1 files changed, 63 insertions, 0 deletions
diff --git a/tex/context/base/l-lpeg.lua b/tex/context/base/l-lpeg.lua
index b107a8e64..05bbebab9 100644
--- a/tex/context/base/l-lpeg.lua
+++ b/tex/context/base/l-lpeg.lua
@@ -48,6 +48,11 @@ patterns.whitespace    = patterns.eol + patterns.spacer
 patterns.nonwhitespace = 1 - patterns.whitespace
 patterns.utf8          = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four
 patterns.utfbom        = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191')
+patterns.validutf8     = patterns.utf8^0 * P(-1) * Cc(true) + Cc(false)
+
+patterns.undouble      = P('"')/"" * (1-P('"'))^0 * P('"')/""
+patterns.unsingle      = P("'")/"" * (1-P("'"))^0 * P("'")/""
+patterns.unspacer      = ((patterns.spacer^1)/"")^0
 
 function lpeg.anywhere(pattern) --slightly adapted from website
     return P { P(pattern) + 1 * V(1) } -- why so complex?
@@ -163,3 +168,61 @@ local function f3(s) local c1, c2, c3     = f1(s,1,3) return  (c1 * 64 + c2) * 6
 local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end
 
 patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4
+
+local cache = { }
+
+function lpeg.stripper(str)
+    local s = cache[str]
+    if not s then
+        s = Cs(((S(str)^1)/"" + 1)^0)
+        cache[str] = s
+    end
+    return s
+end
+
+function lpeg.replacer(t)
+    if #t > 0 then
+        local p
+        for i=1,#t do
+            local ti= t[i]
+            local pp = P(ti[1]) / ti[2]
+            p = (p and p + pp ) or pp
+        end
+        return Cs((p + 1)^0)
+    end
+end
+
+--~ print(utf.check(""))
+--~ print(utf.check("abcde"))
+--~ print(utf.check("abcde\255\123"))
+
+local splitters_f, splitters_s = { }, { }
+
+function lpeg.firstofsplit(separator) -- always return value
+    local splitter = splitters_f[separator]
+    if not splitter then
+        separator = P(separator)
+        splitter = C((1 - separator)^0)
+        splitters_f[separator] = splitter
+    end
+    return splitter
+end
+
+function lpeg.secondofsplit(separator) -- nil if not split
+    local splitter = splitters_s[separator]
+    if not splitter then
+        separator = P(separator)
+        splitter = (1 - separator)^0 * separator * C(P(1)^0)
+        splitters_s[separator] = splitter
+    end
+    return splitter
+end
+
+--~ print(1,match(lpeg.firstofsplit(":"),"bc:de"))
+--~ print(2,match(lpeg.firstofsplit(":"),":de")) -- empty
+--~ print(3,match(lpeg.firstofsplit(":"),"bc"))
+--~ print(4,match(lpeg.secondofsplit(":"),"bc:de"))
+--~ print(5,match(lpeg.secondofsplit(":"),"bc:")) -- empty
+--~ print(6,match(lpeg.secondofsplit(":",""),"bc"))
+--~ print(7,match(lpeg.secondofsplit(":"),"bc"))
+--~ print(9,match(lpeg.secondofsplit(":","123"),"bc"))