From 059fc69b2c7853b937ddb4cfc9d36304dee07893 Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Sat, 1 Apr 2023 10:50:35 +0200 Subject: 2023-04-01 09:31:00 --- tex/context/base/mkiv/lxml-tab.lua | 254 ++++++++++++++----------------------- 1 file changed, 98 insertions(+), 156 deletions(-) (limited to 'tex/context/base/mkiv/lxml-tab.lua') diff --git a/tex/context/base/mkiv/lxml-tab.lua b/tex/context/base/mkiv/lxml-tab.lua index e18362bd8..a06b59065 100644 --- a/tex/context/base/mkiv/lxml-tab.lua +++ b/tex/context/base/mkiv/lxml-tab.lua @@ -18,13 +18,12 @@ local trace_entities = false trackers.register("xml.entities", function(v) trac local report_xml = logs and logs.reporter("xml","core") or function(...) print(string.format(...)) end ---[[ldx-- -

The parser used here is inspired by the variant discussed in the lua book, but -handles comment and processing instructions, has a different structure, provides -parent access; a first version used different trickery but was less optimized to we -went this route. First we had a find based parser, now we have an based one. -The find based parser can be found in l-xml-edu.lua along with other older code.

---ldx]]-- +-- The parser used here is inspired by the variant discussed in the lua book, but +-- handles comment and processing instructions, has a different structure, provides +-- parent access; a first version used different trickery but was less optimized to +-- we went this route. First we had a find based parser, now we have an LPEG based +-- one. The find based parser can be found in l-xml-edu.lua along with other older +-- code. if lpeg.setmaxstack then lpeg.setmaxstack(1000) end -- deeply nested xml files @@ -42,26 +41,19 @@ local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs local formatters = string.formatters ---[[ldx-- -

First a hack to enable namespace resolving. A namespace is characterized by -a . The following function associates a namespace prefix with a -pattern. We use , which in this case is more than twice as fast as a -find based solution where we loop over an array of patterns. Less code and -much cleaner.

---ldx]]-- +-- First a hack to enable namespace resolving. A namespace is characterized by a +-- URL. The following function associates a namespace prefix with a pattern. We use +-- LPEG, which in this case is more than twice as fast as a find based solution +-- where we loop over an array of patterns. Less code and much cleaner. do -- begin of namespace closure (we ran out of locals) xml.xmlns = xml.xmlns or { } ---[[ldx-- -

The next function associates a namespace prefix with an . This -normally happens independent of parsing.

- - -xml.registerns("mml","mathml") - ---ldx]]-- +-- The next function associates a namespace prefix with an URL. This normally +-- happens independent of parsing. +-- +-- xml.registerns("mml","mathml") local check = P(false) local parse = check @@ -71,15 +63,11 @@ function xml.registerns(namespace, pattern) -- pattern can be an lpeg parse = P { P(check) + 1 * V(1) } end ---[[ldx-- -

The next function also registers a namespace, but this time we map a -given namespace prefix onto a registered one, using the given -. This used for attributes like xmlns:m.

- - -xml.checkns("m","http://www.w3.org/mathml") - ---ldx]]-- +-- The next function also registers a namespace, but this time we map a given +-- namespace prefix onto a registered one, using the given URL. This used for +-- attributes like 'xmlns:m'. +-- +-- xml.checkns("m","http://www.w3.org/mathml") function xml.checkns(namespace,url) local ns = lpegmatch(parse,lower(url)) @@ -88,68 +76,54 @@ function xml.checkns(namespace,url) end end ---[[ldx-- -

Next we provide a way to turn an into a registered -namespace. This used for the xmlns attribute.

- - -resolvedns = xml.resolvens("http://www.w3.org/mathml") - - -This returns mml. ---ldx]]-- +-- Next we provide a way to turn an URL into a registered namespace. This used for +-- the 'xmlns' attribute. +-- +-- resolvedns = xml.resolvens("http://www.w3.org/mathml") +-- +-- This returns MATHML. function xml.resolvens(url) return lpegmatch(parse,lower(url)) or "" end ---[[ldx-- -

A namespace in an element can be remapped onto the registered -one efficiently by using the xml.xmlns table.

---ldx]]-- +-- A namespace in an element can be remapped onto the registered one efficiently by +-- using the 'xml.xmlns' table. end -- end of namespace closure ---[[ldx-- -

This version uses . We follow the same approach as before, stack and top and -such. This version is about twice as fast which is mostly due to the fact that -we don't have to prepare the stream for cdata, doctype etc etc. This variant is -is dedicated to Luigi Scarso, who challenged me with 40 megabyte files that -took 12.5 seconds to load (1.5 for file io and the rest for tree building). With -the implementation we got that down to less 7.3 seconds. Loading the 14 - interface definition files (2.6 meg) went down from 1.05 seconds to 0.55.

- -

Next comes the parser. The rather messy doctype definition comes in many -disguises so it is no surprice that later on have to dedicate quite some - code to it.

- - - - - - - - - - -

The code may look a bit complex but this is mostly due to the fact that we -resolve namespaces and attach metatables. There is only one public function:

- - -local x = xml.convert(somestring) - - -

An optional second boolean argument tells this function not to create a root -element.

- -

Valid entities are:

- - - - - - ---ldx]]-- +-- This version uses LPEG. We follow the same approach as before, stack and top and +-- such. This version is about twice as fast which is mostly due to the fact that we +-- don't have to prepare the stream for cdata, doctype etc etc. This variant is is +-- dedicated to Luigi Scarso, who challenged me with 40 megabyte XML files that took +-- 12.5 seconds to load (1.5 for file io and the rest for tree building). With the +-- LPEG implementation we got that down to less 7.3 seconds. Loading the 14 ConTeXt +-- interface definition files (2.6 meg) went down from 1.05 seconds to 0.55. +-- +-- Next comes the parser. The rather messy doctype definition comes in many +-- disguises so it is no surprice that later on have to dedicate quite some LPEG +-- code to it. +-- +-- +-- +-- +-- +-- +-- +-- +-- The code may look a bit complex but this is mostly due to the fact that we +-- resolve namespaces and attach metatables. There is only one public function: +-- +-- local x = xml.convert(somestring) +-- +-- An optional second boolean argument tells this function not to create a root +-- element. +-- +-- Valid entities are: +-- +-- +-- +-- -- not just one big nested table capture (lpeg overflow) @@ -1332,10 +1306,8 @@ function xml.inheritedconvert(data,xmldata,cleanup) -- xmldata is parent return xc end ---[[ldx-- -

Packaging data in an xml like table is done with the following -function. Maybe it will go away (when not used).

---ldx]]-- +-- Packaging data in an xml like table is done with the following function. Maybe it +-- will go away (when not used). function xml.is_valid(root) return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er @@ -1354,11 +1326,8 @@ end xml.errorhandler = report_xml ---[[ldx-- -

We cannot load an from a filehandle so we need to load -the whole file first. The function accepts a string representing -a filename or a file handle.

---ldx]]-- +-- We cannot load an LPEG from a filehandle so we need to load the whole file first. +-- The function accepts a string representing a filename or a file handle. function xml.load(filename,settings) local data = "" @@ -1382,10 +1351,8 @@ function xml.load(filename,settings) end end ---[[ldx-- -

When we inject new elements, we need to convert strings to -valid trees, which is what the next function does.

---ldx]]-- +-- When we inject new elements, we need to convert strings to valid trees, which is +-- what the next function does. local no_root = { no_root = true } @@ -1398,11 +1365,9 @@ function xml.toxml(data) end end ---[[ldx-- -

For copying a tree we use a dedicated function instead of the -generic table copier. Since we know what we're dealing with we -can speed up things a bit. The second argument is not to be used!

---ldx]]-- +-- For copying a tree we use a dedicated function instead of the generic table +-- copier. Since we know what we're dealing with we can speed up things a bit. The +-- second argument is not to be used! -- local function copy(old) -- if old then @@ -1466,13 +1431,10 @@ end xml.copy = copy ---[[ldx-- -

In serializing the tree or parts of the tree is a major -actitivity which is why the following function is pretty optimized resulting -in a few more lines of code than needed. The variant that uses the formatting -function for all components is about 15% slower than the concatinating -alternative.

---ldx]]-- +-- In ConTeXt serializing the tree or parts of the tree is a major actitivity which +-- is why the following function is pretty optimized resulting in a few more lines +-- of code than needed. The variant that uses the formatting function for all +-- components is about 15% slower than the concatinating alternative. -- todo: add when not present @@ -1490,10 +1452,8 @@ function xml.checkbom(root) -- can be made faster end end ---[[ldx-- -

At the cost of some 25% runtime overhead you can first convert the tree to a string -and then handle the lot.

---ldx]]-- +-- At the cost of some 25% runtime overhead you can first convert the tree to a +-- string and then handle the lot. -- new experimental reorganized serialize @@ -1711,21 +1671,18 @@ newhandlers { } } ---[[ldx-- -

How you deal with saving data depends on your preferences. For a 40 MB database -file the timing on a 2.3 Core Duo are as follows (time in seconds):

- - -1.3 : load data from file to string -6.1 : convert string into tree -5.3 : saving in file using xmlsave -6.8 : converting to string using xml.tostring -3.6 : saving converted string in file - -

Beware, these were timing with the old routine but measurements will not be that -much different I guess.

---ldx]]-- +-- How you deal with saving data depends on your preferences. For a 40 MB database +-- file the timing on a 2.3 Core Duo are as follows (time in seconds): +-- +-- 1.3 : load data from file to string +-- 6.1 : convert string into tree +-- 5.3 : saving in file using xmlsave +-- 6.8 : converting to string using xml.tostring +-- 3.6 : saving converted string in file +-- +-- Beware, these were timing with the old routine but measurements will not be that +-- much different I guess. -- maybe this will move to lxml-xml @@ -1827,10 +1784,8 @@ xml.newhandlers = newhandlers xml.serialize = serialize xml.tostring = xmltostring ---[[ldx-- -

The next function operated on the content only and needs a handle function -that accepts a string.

---ldx]]-- +-- The next function operated on the content only and needs a handle function that +-- accepts a string. local function xmlstring(e,handle) if not handle or (e.special and e.tg ~= "@rt@") then @@ -1849,9 +1804,7 @@ end xml.string = xmlstring ---[[ldx-- -

A few helpers:

---ldx]]-- +-- A few helpers: --~ xmlsetproperty(root,"settings",settings) @@ -1899,11 +1852,9 @@ function xml.name(root) end end ---[[ldx-- -

The next helper erases an element but keeps the table as it is, -and since empty strings are not serialized (effectively) it does -not harm. Copying the table would take more time. Usage:

---ldx]]-- +-- The next helper erases an element but keeps the table as it is, and since empty +-- strings are not serialized (effectively) it does not harm. Copying the table +-- would take more time. function xml.erase(dt,k) if dt then @@ -1915,13 +1866,9 @@ function xml.erase(dt,k) end end ---[[ldx-- -

The next helper assigns a tree (or string). Usage:

- - -dt[k] = xml.assign(root) or xml.assign(dt,k,root) - ---ldx]]-- +-- The next helper assigns a tree (or string). Usage: +-- +-- dt[k] = xml.assign(root) or xml.assign(dt,k,root) function xml.assign(dt,k,root) if dt and k then @@ -1932,15 +1879,10 @@ function xml.assign(dt,k,root) end end --- the following helpers may move - ---[[ldx-- -

The next helper assigns a tree (or string). Usage:

- -xml.tocdata(e) -xml.tocdata(e,"error") - ---ldx]]-- +-- The next helper assigns a tree (or string). Usage: +-- +-- xml.tocdata(e) +-- xml.tocdata(e,"error") function xml.tocdata(e,wrapper) -- a few more in the aux module local whatever = type(e) == "table" and xmltostring(e.dt) or e or "" -- cgit v1.2.3