From a57ab2d223a7bcc8e9ae57e148c30ac6e91fafdb Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Fri, 23 Nov 2012 17:35:00 +0100 Subject: beta 2012.11.23 17:35 --- scripts/context/lua/mtxrun.lua | 54 +++++++++++++++++++++++++++++++-- scripts/context/stubs/mswin/mtxrun.lua | 54 +++++++++++++++++++++++++++++++-- scripts/context/stubs/unix/mtxrun | 54 +++++++++++++++++++++++++++++++-- tex/context/base/context-version.pdf | Bin 4148 -> 4152 bytes tex/context/base/context-version.png | Bin 40426 -> 40647 bytes tex/context/base/data-lua.lua | 4 +++ tex/context/base/phys-dim.lua | 15 ++++++--- tex/context/base/status-files.pdf | Bin 24538 -> 24539 bytes tex/context/base/status-lua.pdf | Bin 198684 -> 198711 bytes tex/context/base/util-prs.lua | 50 ++++++++++++++++++++++++++++-- 10 files changed, 214 insertions(+), 17 deletions(-) diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 5e924b07d..403f2ba63 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -5711,7 +5711,7 @@ if not modules then modules = { } end modules ['util-prs'] = { local lpeg, table, string = lpeg, table, string -local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf +local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf, Cp = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.Cp local lpegmatch, patterns = lpeg.match, lpeg.patterns local concat, format, gmatch, find = table.concat, string.format, string.gmatch, string.find local tostring, type, next, rawset = tostring, type, next, rawset @@ -6008,10 +6008,13 @@ end -- inspect(lpeg.match(pattern,[[key="value"]])) -local newline = S('\r\n') +local defaultspecification = { separator = ",", quote = '"' } + +-- this version accepts multiple separators and quotes as used in the +-- database module function parsers.csvsplitter(specification) - specification = specification or { } + specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification local separator = specification.separator local quotechar = specification.quote local separator = S(separator ~= "" and separator or ",") @@ -6035,6 +6038,47 @@ function parsers.csvsplitter(specification) end end +-- and this is a slightly patched version of a version posted by Philipp Gesang + +-- local mycsvsplitter = utilities.parsers.rfc4180splitter() +-- +-- local crap = [[ +-- first,second,third,fourth +-- "1","2","3","4" +-- "a","b","c","d" +-- "foo","bar""baz","boogie","xyzzy" +-- ]] +-- +-- local list, names = mycsvsplitter(crap,true) inspect(list) inspect(names) +-- local list, names = mycsvsplitter(crap) inspect(list) inspect(names) + +function parsers.rfc4180splitter(specification) + specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator = specification.separator --> rfc: COMMA + local quotechar = P(specification.quote) --> DQUOTE + local dquotechar = quotechar * quotechar --> 2DQUOTE + / specification.quote + local separator = S(separator ~= "" and separator or ",") + local escaped = quotechar + * Cs((dquotechar + (1 - quotechar))^0) + * quotechar + local non_escaped = C((1 - quotechar - newline - separator)^1) + local field = escaped + non_escaped + local record = Ct((field * separator^-1)^1) + local headerline = record * Cp() + local wholeblob = Ct((newline^-1 * record)^0) + return function(data,getheader) + if getheader then + local header, position = lpegmatch(headerline,data) + local data = lpegmatch(wholeblob,data,position) + return data, header + else + return lpegmatch(wholeblob,data) + end + end +end + + end -- of closure @@ -16371,6 +16415,10 @@ function package.extraclibpath(...) end end +if not package.loaders then + package.loaders = package.searchers -- 5.2 +end + if not package.loaders[-2] then -- use package-path and package-cpath package.loaders[-2] = package.loaders[2] diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua index 5e924b07d..403f2ba63 100644 --- a/scripts/context/stubs/mswin/mtxrun.lua +++ b/scripts/context/stubs/mswin/mtxrun.lua @@ -5711,7 +5711,7 @@ if not modules then modules = { } end modules ['util-prs'] = { local lpeg, table, string = lpeg, table, string -local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf +local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf, Cp = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.Cp local lpegmatch, patterns = lpeg.match, lpeg.patterns local concat, format, gmatch, find = table.concat, string.format, string.gmatch, string.find local tostring, type, next, rawset = tostring, type, next, rawset @@ -6008,10 +6008,13 @@ end -- inspect(lpeg.match(pattern,[[key="value"]])) -local newline = S('\r\n') +local defaultspecification = { separator = ",", quote = '"' } + +-- this version accepts multiple separators and quotes as used in the +-- database module function parsers.csvsplitter(specification) - specification = specification or { } + specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification local separator = specification.separator local quotechar = specification.quote local separator = S(separator ~= "" and separator or ",") @@ -6035,6 +6038,47 @@ function parsers.csvsplitter(specification) end end +-- and this is a slightly patched version of a version posted by Philipp Gesang + +-- local mycsvsplitter = utilities.parsers.rfc4180splitter() +-- +-- local crap = [[ +-- first,second,third,fourth +-- "1","2","3","4" +-- "a","b","c","d" +-- "foo","bar""baz","boogie","xyzzy" +-- ]] +-- +-- local list, names = mycsvsplitter(crap,true) inspect(list) inspect(names) +-- local list, names = mycsvsplitter(crap) inspect(list) inspect(names) + +function parsers.rfc4180splitter(specification) + specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator = specification.separator --> rfc: COMMA + local quotechar = P(specification.quote) --> DQUOTE + local dquotechar = quotechar * quotechar --> 2DQUOTE + / specification.quote + local separator = S(separator ~= "" and separator or ",") + local escaped = quotechar + * Cs((dquotechar + (1 - quotechar))^0) + * quotechar + local non_escaped = C((1 - quotechar - newline - separator)^1) + local field = escaped + non_escaped + local record = Ct((field * separator^-1)^1) + local headerline = record * Cp() + local wholeblob = Ct((newline^-1 * record)^0) + return function(data,getheader) + if getheader then + local header, position = lpegmatch(headerline,data) + local data = lpegmatch(wholeblob,data,position) + return data, header + else + return lpegmatch(wholeblob,data) + end + end +end + + end -- of closure @@ -16371,6 +16415,10 @@ function package.extraclibpath(...) end end +if not package.loaders then + package.loaders = package.searchers -- 5.2 +end + if not package.loaders[-2] then -- use package-path and package-cpath package.loaders[-2] = package.loaders[2] diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun index 5e924b07d..403f2ba63 100755 --- a/scripts/context/stubs/unix/mtxrun +++ b/scripts/context/stubs/unix/mtxrun @@ -5711,7 +5711,7 @@ if not modules then modules = { } end modules ['util-prs'] = { local lpeg, table, string = lpeg, table, string -local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf +local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf, Cp = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.Cp local lpegmatch, patterns = lpeg.match, lpeg.patterns local concat, format, gmatch, find = table.concat, string.format, string.gmatch, string.find local tostring, type, next, rawset = tostring, type, next, rawset @@ -6008,10 +6008,13 @@ end -- inspect(lpeg.match(pattern,[[key="value"]])) -local newline = S('\r\n') +local defaultspecification = { separator = ",", quote = '"' } + +-- this version accepts multiple separators and quotes as used in the +-- database module function parsers.csvsplitter(specification) - specification = specification or { } + specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification local separator = specification.separator local quotechar = specification.quote local separator = S(separator ~= "" and separator or ",") @@ -6035,6 +6038,47 @@ function parsers.csvsplitter(specification) end end +-- and this is a slightly patched version of a version posted by Philipp Gesang + +-- local mycsvsplitter = utilities.parsers.rfc4180splitter() +-- +-- local crap = [[ +-- first,second,third,fourth +-- "1","2","3","4" +-- "a","b","c","d" +-- "foo","bar""baz","boogie","xyzzy" +-- ]] +-- +-- local list, names = mycsvsplitter(crap,true) inspect(list) inspect(names) +-- local list, names = mycsvsplitter(crap) inspect(list) inspect(names) + +function parsers.rfc4180splitter(specification) + specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator = specification.separator --> rfc: COMMA + local quotechar = P(specification.quote) --> DQUOTE + local dquotechar = quotechar * quotechar --> 2DQUOTE + / specification.quote + local separator = S(separator ~= "" and separator or ",") + local escaped = quotechar + * Cs((dquotechar + (1 - quotechar))^0) + * quotechar + local non_escaped = C((1 - quotechar - newline - separator)^1) + local field = escaped + non_escaped + local record = Ct((field * separator^-1)^1) + local headerline = record * Cp() + local wholeblob = Ct((newline^-1 * record)^0) + return function(data,getheader) + if getheader then + local header, position = lpegmatch(headerline,data) + local data = lpegmatch(wholeblob,data,position) + return data, header + else + return lpegmatch(wholeblob,data) + end + end +end + + end -- of closure @@ -16371,6 +16415,10 @@ function package.extraclibpath(...) end end +if not package.loaders then + package.loaders = package.searchers -- 5.2 +end + if not package.loaders[-2] then -- use package-path and package-cpath package.loaders[-2] = package.loaders[2] diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf index 3349ae881..d17c41110 100644 Binary files a/tex/context/base/context-version.pdf and b/tex/context/base/context-version.pdf differ diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png index d02983feb..43687ee49 100644 Binary files a/tex/context/base/context-version.png and b/tex/context/base/context-version.png differ diff --git a/tex/context/base/data-lua.lua b/tex/context/base/data-lua.lua index 87d346201..c63851525 100644 --- a/tex/context/base/data-lua.lua +++ b/tex/context/base/data-lua.lua @@ -115,6 +115,10 @@ function package.extraclibpath(...) end end +if not package.loaders then + package.loaders = package.searchers -- 5.2 +end + if not package.loaders[-2] then -- use package-path and package-cpath package.loaders[-2] = package.loaders[2] diff --git a/tex/context/base/phys-dim.lua b/tex/context/base/phys-dim.lua index 5b883ac88..88a522195 100644 --- a/tex/context/base/phys-dim.lua +++ b/tex/context/base/phys-dim.lua @@ -82,8 +82,16 @@ local space = P(" ") local lparent = P("(") local rparent = P(")") +local lbrace = P("{") +local rbrace = P("}") + local digits = digit^1 +local powerdigits = plus * C(digits) / context.digitspowerplus + + minus * C(digits) / context.digitspowerminus + + C(digits) / context.digitspower + + local ddigitspace = digitspace / "" / context.digitsspace local ddigit = digits / context.digitsdigit local dsemicomma = semicolon / "" / context.digitsseparatorspace @@ -100,11 +108,8 @@ local dnegative = negative / "" / context.digitsnegative local dhighspace = highspace / "" / context.digitshighspace local dsomesign = plus / "" / context.digitsplus + minus / "" / context.digitsminus -local dpower = power / "" * ( - plus * C(digits) / context.digitspowerplus - + minus * C(digits) / context.digitspowerminus - + C(digits) / context.digitspower - ) +local dpower = power / "" * ( powerdigits + lbrace * powerdigits * rbrace ) + local dpadding = padding / "" / context.digitszeropadding -- todo local dleader = (dpositive + dnegative + dhighspace + dsomesign + dsignspace)^0 diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf index 965f9ab05..83936aba1 100644 Binary files a/tex/context/base/status-files.pdf and b/tex/context/base/status-files.pdf differ diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf index aa44ac99f..736cc36fc 100644 Binary files a/tex/context/base/status-lua.pdf and b/tex/context/base/status-lua.pdf differ diff --git a/tex/context/base/util-prs.lua b/tex/context/base/util-prs.lua index 4166fca25..3cfb3e5f3 100644 --- a/tex/context/base/util-prs.lua +++ b/tex/context/base/util-prs.lua @@ -8,7 +8,7 @@ if not modules then modules = { } end modules ['util-prs'] = { local lpeg, table, string = lpeg, table, string -local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf +local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf, Cp = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.Cp local lpegmatch, patterns = lpeg.match, lpeg.patterns local concat, format, gmatch, find = table.concat, string.format, string.gmatch, string.find local tostring, type, next, rawset = tostring, type, next, rawset @@ -305,10 +305,13 @@ end -- inspect(lpeg.match(pattern,[[key="value"]])) -local newline = S('\r\n') +local defaultspecification = { separator = ",", quote = '"' } + +-- this version accepts multiple separators and quotes as used in the +-- database module function parsers.csvsplitter(specification) - specification = specification or { } + specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification local separator = specification.separator local quotechar = specification.quote local separator = S(separator ~= "" and separator or ",") @@ -331,3 +334,44 @@ function parsers.csvsplitter(specification) return lpegmatch(parser,data) end end + +-- and this is a slightly patched version of a version posted by Philipp Gesang + +-- local mycsvsplitter = utilities.parsers.rfc4180splitter() +-- +-- local crap = [[ +-- first,second,third,fourth +-- "1","2","3","4" +-- "a","b","c","d" +-- "foo","bar""baz","boogie","xyzzy" +-- ]] +-- +-- local list, names = mycsvsplitter(crap,true) inspect(list) inspect(names) +-- local list, names = mycsvsplitter(crap) inspect(list) inspect(names) + +function parsers.rfc4180splitter(specification) + specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator = specification.separator --> rfc: COMMA + local quotechar = P(specification.quote) --> DQUOTE + local dquotechar = quotechar * quotechar --> 2DQUOTE + / specification.quote + local separator = S(separator ~= "" and separator or ",") + local escaped = quotechar + * Cs((dquotechar + (1 - quotechar))^0) + * quotechar + local non_escaped = C((1 - quotechar - newline - separator)^1) + local field = escaped + non_escaped + local record = Ct((field * separator^-1)^1) + local headerline = record * Cp() + local wholeblob = Ct((newline^-1 * record)^0) + return function(data,getheader) + if getheader then + local header, position = lpegmatch(headerline,data) + local data = lpegmatch(wholeblob,data,position) + return data, header + else + return lpegmatch(wholeblob,data) + end + end +end + -- cgit v1.2.3