2018-09-01 23:20:00

1 files changed, 52 insertions, 43 deletions

diff --git a/tex/context/base/mkiv/util-str.lua b/tex/context/base/mkiv/util-str.lua
index 29305f3bb..48c265659 100644
--- a/tex/context/base/mkiv/util-str.lua
+++ b/tex/context/base/mkiv/util-str.lua
@@ -51,9 +51,14 @@ local stripzero   = patterns.stripzero
 local stripzeros  = patterns.stripzeros
 local newline     = patterns.newline
 local endofstring = patterns.endofstring
+local anything    = patterns.anything
 local whitespace  = patterns.whitespace
+local space       = patterns.space
 local spacer      = patterns.spacer
 local spaceortab  = patterns.spaceortab
+local digit       = patterns.digit
+local sign        = patterns.sign
+local period      = patterns.period
 
 -- local function points(n)
 --     n = tonumber(n)
@@ -106,7 +111,6 @@ number.basepoints = basepoints
 
 local rubish     = spaceortab^0 * newline
 local anyrubish  = spaceortab + newline
-local anything   = patterns.anything
 local stripped   = (spaceortab^1 / "") * newline
 local leading    = rubish^0 / ""
 local trailing   = (anyrubish^1 * endofstring) / ""
@@ -174,7 +178,7 @@ local pattern =
     + newline * Cp() / function(position)
           extra, start = 0, position
       end
-    + patterns.anything
+    + anything
   )^1)
 
 function strings.tabtospace(str,tab)
@@ -217,27 +221,31 @@ end
 --     return str
 -- end
 
-local space       = spacer^0
-local nospace     = space/""
-local endofline   = nospace * newline
+local optionalspace = spacer^0
+local nospace       = optionalspace/""
+local endofline     = nospace * newline
 
-local stripend    = (whitespace^1 * endofstring)/""
+local stripend      = (whitespace^1 * endofstring)/""
 
-local normalline  = (nospace * ((1-space*(newline+endofstring))^1) * nospace)
+local normalline    = (nospace * ((1-optionalspace*(newline+endofstring))^1) * nospace)
 
-local stripempty  = endofline^1/""
-local normalempty = endofline^1
-local singleempty = endofline * (endofline^0/"")
-local doubleempty = endofline * endofline^-1 * (endofline^0/"")
+local stripempty    = endofline^1/""
+local normalempty   = endofline^1
+local singleempty   = endofline * (endofline^0/"")
+local doubleempty   = endofline * endofline^-1 * (endofline^0/"")
+local stripstart    = stripempty^0
 
-local stripstart  = stripempty^0
+local intospace     = whitespace^1/" "
+local noleading     = whitespace^1/""
+local notrailing    = noleading * endofstring
 
-local p_prune_normal    = Cs ( stripstart * ( stripend + normalline + normalempty )^0 )
-local p_prune_collapse  = Cs ( stripstart * ( stripend + normalline + doubleempty )^0 )
-local p_prune_noempty   = Cs ( stripstart * ( stripend + normalline + singleempty )^0 )
-local p_retain_normal   = Cs (              (            normalline + normalempty )^0 )
-local p_retain_collapse = Cs (              (            normalline + doubleempty )^0 )
-local p_retain_noempty  = Cs (              (            normalline + singleempty )^0 )
+local p_prune_normal    = Cs ( stripstart * ( stripend   + normalline + normalempty )^0 )
+local p_prune_collapse  = Cs ( stripstart * ( stripend   + normalline + doubleempty )^0 )
+local p_prune_noempty   = Cs ( stripstart * ( stripend   + normalline + singleempty )^0 )
+local p_prune_intospace = Cs ( noleading  * ( notrailing + intospace  + 1           )^0 )
+local p_retain_normal   = Cs (              (              normalline + normalempty )^0 )
+local p_retain_collapse = Cs (              (              normalline + doubleempty )^0 )
+local p_retain_noempty  = Cs (              (              normalline + singleempty )^0 )
 
 -- function striplines(str,prune,collapse,noempty)
 --     if prune then
@@ -263,6 +271,7 @@ local striplinepatterns = {
     ["prune"]               = p_prune_normal,
     ["prune and collapse"]  = p_prune_collapse, -- default
     ["prune and no empty"]  = p_prune_noempty,
+    ["prune and to space"]  = p_prune_intospace,
     ["retain"]              = p_retain_normal,
     ["retain and collapse"] = p_retain_collapse,
     ["retain and no empty"] = p_retain_noempty,
@@ -277,6 +286,10 @@ function strings.striplines(str,how)
     return str and lpegmatch(striplinepatterns[how],str) or str
 end
 
+function strings.collapse(str) -- maybe also in strings
+    return str and lpegmatch(p_prune_intospace,str) or str
+end
+
 -- also see: string.collapsespaces
 
 strings.striplong = strings.striplines -- for old times sake
@@ -292,13 +305,14 @@ strings.striplong = strings.striplines -- for old times sake
 -- "       zus    wim jet",
 -- "    ",
 -- }, "\n")
-
+--
 -- local str = table.concat( {
 -- "  aaaa",
 -- "  bb",
 -- "  cccccc",
+-- " ",
 -- }, "\n")
-
+--
 -- for k, v in table.sortedhash(utilities.strings.striplinepatterns) do
 --     logs.report("stripper","method: %s, result: [[%s]]",k,utilities.strings.striplines(str,k))
 -- end
@@ -445,22 +459,19 @@ end
 
 -- maybe to util-num
 
-local digit  = patterns.digit
-local period = patterns.period
 local two    = digit * digit
 local three  = two * digit
 local prefix = (Carg(1) * three)^1
 
-
 local splitter = Cs (
     (((1 - (three^1 * period))^1 + C(three)) * prefix + C((1-period)^1))
-  * (P(1)/"" * Carg(2)) * C(2)
+  * (anything/"" * Carg(2)) * C(2)
 )
 
 local splitter3 = Cs (
-    three * prefix * P(-1) +
-    two   * prefix * P(-1) +
-    digit * prefix * P(-1) +
+    three * prefix * endofstring +
+    two   * prefix * endofstring +
+    digit * prefix * endofstring +
     three +
     two   +
     digit
@@ -512,9 +523,9 @@ end
 local p = Cs(
         P("-")^0
       * (P("0")^1/"")^0
-      * (1-P("."))^0
-      * (P(".") * P("0")^1 * P(-1)/"" + P(".")^0)
-      * P(1-P("0")^1*P(-1))^0
+      * (1-period)^0
+      * (period * P("0")^1 * endofstring/"" + period^0)
+      * P(1-P("0")^1*endofstring)^0
     )
 
 function number.compactfloat(n,fmt)
@@ -533,12 +544,11 @@ end
 local zero      = P("0")^1 / ""
 local plus      = P("+")   / ""
 local minus     = P("-")
-local separator = S(".")
-local digit     = R("09")
+local separator = period
 local trailing  = zero^1 * #S("eE")
-local exponent  = (S("eE") * (plus + Cs((minus * zero^0 * P(-1))/"") + minus) * zero^0 * (P(-1) * Cc("0") + P(1)^1))
+local exponent  = (S("eE") * (plus + Cs((minus * zero^0 * endofstring)/"") + minus) * zero^0 * (endofstring * Cc("0") + anything^1))
 local pattern_a = Cs(minus^0 * digit^1 * (separator/"" * trailing + separator * (trailing + digit)^0) * exponent)
-local pattern_b = Cs((exponent + P(1))^0)
+local pattern_b = Cs((exponent + anything)^0)
 
 function number.sparseexponent(f,n)
     if not n then
@@ -661,10 +671,10 @@ setmetatable(arguments, { __index =
     end
 })
 
-local prefix_any = C((S("+- .") + R("09"))^0)
-local prefix_sub = (C((S("+-") + R("09"))^0) + Cc(0))
-                 * P(".")
-                 * (C((S("+-") + R("09"))^0) + Cc(0))
+local prefix_any = C((sign + space + period + digit)^0)
+local prefix_sub = (C((sign + digit)^0) + Cc(0))
+                 * period
+                 * (C((sign + digit)^0) + Cc(0))
 local prefix_tab = P("{") * C((1-P("}"))^0) * P("}") + C((1-R("az","AZ","09","%%"))^0)
 
 -- we've split all cases as then we can optimize them (let's omit the fuzzy u)
@@ -1097,7 +1107,7 @@ local builder = Cs { "start",
             )
           + V("*")
         )
-     * (P(-1) + Carg(1))
+     * (endofstring + Carg(1))
     )^0,
     --
     ["s"] = (prefix_any * P("s")) / format_s, -- %s => regular %s (string)
@@ -1169,7 +1179,7 @@ local preset = {
 }
 
 local direct =
-    P("%") * (S("+- .") + R("09"))^0 * S("sqidfgGeExXo") * P(-1)
+    P("%") * (sign + space + period + digit)^0 * S("sqidfgGeExXo") * endofstring
   / [[local format = string.format return function(str) return format("%0",str) end]]
 
 local function make(t,str)
@@ -1298,8 +1308,8 @@ strings.formatters.add = add
 
 -- registered in the default instance (should we fall back on this one?)
 
-patterns.xmlescape = Cs((P("<")/"&lt;" + P(">")/"&gt;" + P("&")/"&amp;" + P('"')/"&quot;" + P(1))^0)
-patterns.texescape = Cs((C(S("#$%\\{}"))/"\\%1" + P(1))^0)
+patterns.xmlescape = Cs((P("<")/"&lt;" + P(">")/"&gt;" + P("&")/"&amp;" + P('"')/"&quot;" + anything)^0)
+patterns.texescape = Cs((C(S("#$%\\{}"))/"\\%1" + anything)^0)
 patterns.luaescape = Cs(((1-S('"\n'))^1 + P('"')/'\\"' + P('\n')/'\\n"')^0) -- maybe also \0
 patterns.luaquoted = Cs(Cc('"') * ((1-S('"\n'))^1 + P('"')/'\\"' + P('\n')/'\\n"')^0 * Cc('"'))
 
@@ -1355,7 +1365,6 @@ end
 
 local dquote = patterns.dquote -- P('"')
 local equote = patterns.escaped + dquote / '\\"' + 1
-local space  = patterns.space
 local cquote = Cc('"')
 
 local pattern =