2021-09-17 10:01:00

5 files changed, 170 insertions, 45 deletions

diff --git a/scripts/context/lua/mtx-unicode.lua b/scripts/context/lua/mtx-unicode.lua
index 060e563c6..0979239db 100644
--- a/scripts/context/lua/mtx-unicode.lua
+++ b/scripts/context/lua/mtx-unicode.lua
@@ -143,6 +143,7 @@ function scripts.unicode.update()
     local eastasianwidth       = texttables.eastasianwidth
     local standardizedvariants = texttables.standardizedvariants
     local arabicshaping        = texttables.arabicshaping
+    local casefolding          = texttables.casefolding
     local index                = texttables.index
     local characterdata        = characters.data
     --
@@ -164,6 +165,8 @@ function scripts.unicode.update()
                 local cjkwd     = ed and lower(ed[2] or "n")
                 local mirror    = bd and tonumber(bd[2],16)
                 local arabic    = nil
+                local lccode    = false
+                local uccode    = false
                 descriptions[description] = unicode
                 if sparse and direction == "l" then
                     direction = nil
@@ -203,6 +206,37 @@ function scripts.unicode.update()
                 if not combining or combining == 0 then
                     combining = nil
                 end
+                --
+                local cf = casefolding[unicode]
+                if cf and  tonumber(cf[1],16) == unicode then
+                    local how = cf[2]
+                    if how == "C" or how == "S" then
+                        local fold = tonumber(cf[3],16)
+                        if fold == unicode then
+                         -- print("SKIPPING",description)
+                        elseif category == "ll" then
+                            uccode = fold
+                        elseif category == "lu" then
+                            lccode = fold
+                        end
+                    elseif how == "F" then
+                        -- we can use the first
+                        local folding = { }
+                        for s in gmatch(cf[3],"%S+") do
+                            folding[#folding+1] = tonumber(s,16)
+                        end
+                        if category == "ll" then
+                            uccode = folding
+                        elseif category == "ul" then
+                            lccode = folding
+                        end
+                    else
+                        -- we skip these
+                     -- print(description)
+                     -- inspect(cf)
+                    end
+                end
+                --
                 if not char then
                     report("%U : adding entry %a",unicode,description)
                     char = {
@@ -218,9 +252,34 @@ function scripts.unicode.update()
                         specials    = specials,
                         arabic      = arabic,
                         combining   = combining,
+                        uccode      = uccode,
+                        lccode      = lccode,
                     }
                     characterdata[unicode] = char
                 else
+                    -- we have more case mapping (e.g. cherokee)
+                    if lccode then
+                        if type(lccode) == "table" then
+                            if type(char.lccode) ~= "table" or not are_equal(lccode,char.lccode) then
+                                report("%U : setting lccode to % t, %a",unicode,lccode,description)
+                                char.lccode = lccode
+                            end
+                        elseif char.lccode ~= lccode then
+                            report("%U : setting lccode to %a, %a, %a",unicode,lccode,description)
+                            char.lccode = lccode
+                        end
+                    end
+                    if uccode then
+                        if type(uccode) == "table" then
+                            if type(char.uccode) ~= "table" or not are_equal(uccode,char.uccode) then
+                                report("%U : setting uccode to % t, %a",unicode,uccode,description)
+                                char.uccode = uccode
+                            end
+                        elseif char.uccode ~= uccode then
+                            report("%U : setting uccode to %a, %a",unicode,uccode,description)
+                            char.uccode = uccode
+                        end
+                    end
                     if direction then
                         if char.direction ~= direction then
                             report("%U : setting direction to %a, %a",unicode,direction,description)
@@ -306,7 +365,7 @@ function scripts.unicode.update()
                                 elseif not find(comment,"check special") then
                                     char.comment = comment .. ", check special"
                                 end
-                                report("%U : check specials % + t, %a",unicode,t,description)
+                             -- report("%U : check specials % + t, %a",unicode,t,description)
                             end
                         end
                     end
@@ -397,7 +456,7 @@ function scripts.unicode.update()
                     mark = descriptions["SOLIDUS"] -- SLASH
                 end
                 if base and mark then
-                    report("adding extra char special for %a",description)
+                 -- report("adding extra char special for %a",description)
                     data.specials = { "with", base, mark }
                     data.comment  = nil
                 end
@@ -429,7 +488,7 @@ function scripts.unicode.update()
     end
     for unicode, ud in table.sortedpairs(characterdata) do
         if not rawget(ud,"category") and rawget(ud,"variants") then
-            report("stripping %U (variant, takes from metacharacter)",unicode)
+         -- report("stripping %U (variant, takes from metacharacter)",unicode)
             characterdata[unicode] = nil
         end
     end
@@ -521,6 +580,7 @@ function scripts.unicode.load()
             eastasianwidth       = resolvers.findfile("eastasianwidth.txt")       or "",
             standardizedvariants = resolvers.findfile("standardizedvariants.txt") or "",
             arabicshaping        = resolvers.findfile("arabicshaping.txt")        or "",
+            casefolding          = resolvers.findfile("casefolding.txt")          or "",
             index                = resolvers.findfile("index.txt")                or "",
         }
         --
@@ -531,6 +591,7 @@ function scripts.unicode.load()
             eastasianwidth       = textfiles.eastasianwidth       ~= "" and io.loaddata(textfiles.eastasianwidth)       or "",
             standardizedvariants = textfiles.standardizedvariants ~= "" and io.loaddata(textfiles.standardizedvariants) or "",
             arabicshaping        = textfiles.arabicshaping        ~= "" and io.loaddata(textfiles.arabicshaping)        or "",
+            casefolding          = textfiles.casefolding          ~= "" and io.loaddata(textfiles.casefolding)          or "",
             index                = textfiles.index                ~= "" and io.loaddata(textfiles.index)                or "",
         }
         texttables = {
@@ -540,6 +601,7 @@ function scripts.unicode.load()
             eastasianwidth       = splitdefinition(textdata.eastasianwidth,true),
             standardizedvariants = splitdefinition(textdata.standardizedvariants,false),
             arabicshaping        = splitdefinition(textdata.arabicshaping,true),
+            casefolding          = splitdefinition(textdata.casefolding,true),
             index                = splitindex(textdata.index),
         }
         --
@@ -563,11 +625,18 @@ end
 --    [0xFE01]="centered form",
 -- }
 
+-- local variants_style={
+--    [0xFE00]="chancery style",
+--    [0xFE01]="roundhand style",
+-- }
+
 function scripts.unicode.save(filename)
     if preamble then
         local data = table.serialize(characters.data,"characters.data", { hexify = true, noquotes = true })
-        data = gsub(data,"%{%s+%[0xFE0E%]=\"text style\",%s+%[0xFE0F%]=\"emoji style\",%s+%}","variants_emoji")
+        data = gsub(data,"%{%s+%[0xFE0E%]=\"text style\",%s+%[0xFE0F%]=\"emoji style\",%s+%}",              "variants_emoji")
         data = gsub(data,"%{%s+%[0xFE00%]=\"corner%-justified form\",%s+%[0xFE01%]=\"centered form\",%s+%}","variants_forms")
+        data = gsub(data,"%{%s+%[0xFE00%]=\"chancery style\",%s+%[0xFE01%]=\"roundhand style\",%s+%}",      "variants_style")
+        data = gsub(data,"%{%s+%[0xFE00%]=\"dotted form\",%s+%}",                                           "variants_dotted")
         io.savedata(filename,preamble .. data)
     end
 end
@@ -764,7 +833,7 @@ else
         scripts.unicode.save("char-def-new.lua")
         scripts.unicode.emoji("char-emj-new.lua")
         report("saved file %a","char-def-new.lua")
-        report("saved file %a (current 12.0, check for updates, see above!)","char-emj-new.lua")
+        report("saved file %a (current 14.0, check for updates, see above!)","char-emj-new.lua")
     else
         report("nothing to do")
     end
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index dc06a5f11..2e8baf15c 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -6525,7 +6525,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["util-str"] = package.loaded["util-str"] or true
 
--- original size: 45916, stripped down to: 23839
+-- original size: 46322, stripped down to: 24128
 
 if not modules then modules={} end modules ['util-str']={
  version=1.001,
@@ -6543,6 +6543,7 @@ local tonumber,type,tostring,next,setmetatable=tonumber,type,tostring,next,setme
 local unpack,concat=table.unpack,table.concat
 local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc
 local patterns,lpegmatch=lpeg.patterns,lpeg.match
+local tsplitat=lpeg.tsplitat
 local utfchar,utfbyte,utflen=utf.char,utf.byte,utf.len
 local loadstripped=function(str,shortcuts)
  if shortcuts then
@@ -7439,7 +7440,6 @@ function number.to16dot16(n)
  return f_16_16(n/65536.0)
 end
 if not string.explode then
- local tsplitat=lpeg.tsplitat
  local p_utf=patterns.utf8character
  local p_check=C(p_utf)*(P("+")*Cc(true))^0
  local p_split=Ct(C(p_utf)^0)
@@ -7459,6 +7459,20 @@ if not string.explode then
   end
  end
 end
+do
+ local p_whitespace=patterns.whitespace^1
+ local cache=setmetatable({},{ __index=function(t,k)
+  local p=tsplitat(p_whitespace*P(k)*p_whitespace)
+  local v=function(s)
+   return lpegmatch(p,s)
+  end
+  t[k]=v
+  return v
+ end })
+ function string.wordsplitter(s)
+  return cache[s]
+ end
+end
 
 
 end -- of closure
@@ -15699,7 +15713,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["util-zip"] = package.loaded["util-zip"] or true
 
--- original size: 23104, stripped down to: 14029
+-- original size: 23490, stripped down to: 14039
 
 if not modules then modules={} end modules ['util-zip']={
  version=1.001,
@@ -16176,11 +16190,11 @@ else
   local timestamp=readcardinal4(s)
   local compression=readbyte(s,1)
   local operating=readbyte(s,1)
-  local isjusttext=(flags & 0x01~=0) and true    or false
-  local extrasize=(flags & 0x04~=0) and readcardinal2(s) or 0
-  local filename=(flags & 0x08~=0) and readcstring(s)   or ""
-  local comment=(flags & 0x10~=0) and readcstring(s)   or ""
-  local checksum=(flags & 0x02~=0) and readcardinal2(s) or 0
+  local isjusttext=band(flags,0x01)~=0 and true    or false
+  local extrasize=band(flags,0x04)~=0 and readcardinal2(s) or 0
+  local filename=band(flags,0x08)~=0 and readcstring(s)   or ""
+  local comment=band(flags,0x10)~=0 and readcstring(s)   or ""
+  local checksum=band(flags,0x02)~=0 and readcardinal2(s) or 0
   local compressed=readstring(s,#str)
   local data=decompress(compressed,gzipwindow) 
   return data
@@ -25932,8 +25946,8 @@ end -- of closure
 
 -- used libraries    : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua libs-ini.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 1027101
--- stripped bytes    : 403437
+-- original bytes    : 1027893
+-- stripped bytes    : 403930
 
 -- end library merge
 
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index dc06a5f11..2e8baf15c 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -6525,7 +6525,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["util-str"] = package.loaded["util-str"] or true
 
--- original size: 45916, stripped down to: 23839
+-- original size: 46322, stripped down to: 24128
 
 if not modules then modules={} end modules ['util-str']={
  version=1.001,
@@ -6543,6 +6543,7 @@ local tonumber,type,tostring,next,setmetatable=tonumber,type,tostring,next,setme
 local unpack,concat=table.unpack,table.concat
 local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc
 local patterns,lpegmatch=lpeg.patterns,lpeg.match
+local tsplitat=lpeg.tsplitat
 local utfchar,utfbyte,utflen=utf.char,utf.byte,utf.len
 local loadstripped=function(str,shortcuts)
  if shortcuts then
@@ -7439,7 +7440,6 @@ function number.to16dot16(n)
  return f_16_16(n/65536.0)
 end
 if not string.explode then
- local tsplitat=lpeg.tsplitat
  local p_utf=patterns.utf8character
  local p_check=C(p_utf)*(P("+")*Cc(true))^0
  local p_split=Ct(C(p_utf)^0)
@@ -7459,6 +7459,20 @@ if not string.explode then
   end
  end
 end
+do
+ local p_whitespace=patterns.whitespace^1
+ local cache=setmetatable({},{ __index=function(t,k)
+  local p=tsplitat(p_whitespace*P(k)*p_whitespace)
+  local v=function(s)
+   return lpegmatch(p,s)
+  end
+  t[k]=v
+  return v
+ end })
+ function string.wordsplitter(s)
+  return cache[s]
+ end
+end
 
 
 end -- of closure
@@ -15699,7 +15713,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["util-zip"] = package.loaded["util-zip"] or true
 
--- original size: 23104, stripped down to: 14029
+-- original size: 23490, stripped down to: 14039
 
 if not modules then modules={} end modules ['util-zip']={
  version=1.001,
@@ -16176,11 +16190,11 @@ else
   local timestamp=readcardinal4(s)
   local compression=readbyte(s,1)
   local operating=readbyte(s,1)
-  local isjusttext=(flags & 0x01~=0) and true    or false
-  local extrasize=(flags & 0x04~=0) and readcardinal2(s) or 0
-  local filename=(flags & 0x08~=0) and readcstring(s)   or ""
-  local comment=(flags & 0x10~=0) and readcstring(s)   or ""
-  local checksum=(flags & 0x02~=0) and readcardinal2(s) or 0
+  local isjusttext=band(flags,0x01)~=0 and true    or false
+  local extrasize=band(flags,0x04)~=0 and readcardinal2(s) or 0
+  local filename=band(flags,0x08)~=0 and readcstring(s)   or ""
+  local comment=band(flags,0x10)~=0 and readcstring(s)   or ""
+  local checksum=band(flags,0x02)~=0 and readcardinal2(s) or 0
   local compressed=readstring(s,#str)
   local data=decompress(compressed,gzipwindow) 
   return data
@@ -25932,8 +25946,8 @@ end -- of closure
 
 -- used libraries    : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua libs-ini.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 1027101
--- stripped bytes    : 403437
+-- original bytes    : 1027893
+-- stripped bytes    : 403930
 
 -- end library merge
 
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index dc06a5f11..2e8baf15c 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -6525,7 +6525,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["util-str"] = package.loaded["util-str"] or true
 
--- original size: 45916, stripped down to: 23839
+-- original size: 46322, stripped down to: 24128
 
 if not modules then modules={} end modules ['util-str']={
  version=1.001,
@@ -6543,6 +6543,7 @@ local tonumber,type,tostring,next,setmetatable=tonumber,type,tostring,next,setme
 local unpack,concat=table.unpack,table.concat
 local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc
 local patterns,lpegmatch=lpeg.patterns,lpeg.match
+local tsplitat=lpeg.tsplitat
 local utfchar,utfbyte,utflen=utf.char,utf.byte,utf.len
 local loadstripped=function(str,shortcuts)
  if shortcuts then
@@ -7439,7 +7440,6 @@ function number.to16dot16(n)
  return f_16_16(n/65536.0)
 end
 if not string.explode then
- local tsplitat=lpeg.tsplitat
  local p_utf=patterns.utf8character
  local p_check=C(p_utf)*(P("+")*Cc(true))^0
  local p_split=Ct(C(p_utf)^0)
@@ -7459,6 +7459,20 @@ if not string.explode then
   end
  end
 end
+do
+ local p_whitespace=patterns.whitespace^1
+ local cache=setmetatable({},{ __index=function(t,k)
+  local p=tsplitat(p_whitespace*P(k)*p_whitespace)
+  local v=function(s)
+   return lpegmatch(p,s)
+  end
+  t[k]=v
+  return v
+ end })
+ function string.wordsplitter(s)
+  return cache[s]
+ end
+end
 
 
 end -- of closure
@@ -15699,7 +15713,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["util-zip"] = package.loaded["util-zip"] or true
 
--- original size: 23104, stripped down to: 14029
+-- original size: 23490, stripped down to: 14039
 
 if not modules then modules={} end modules ['util-zip']={
  version=1.001,
@@ -16176,11 +16190,11 @@ else
   local timestamp=readcardinal4(s)
   local compression=readbyte(s,1)
   local operating=readbyte(s,1)
-  local isjusttext=(flags & 0x01~=0) and true    or false
-  local extrasize=(flags & 0x04~=0) and readcardinal2(s) or 0
-  local filename=(flags & 0x08~=0) and readcstring(s)   or ""
-  local comment=(flags & 0x10~=0) and readcstring(s)   or ""
-  local checksum=(flags & 0x02~=0) and readcardinal2(s) or 0
+  local isjusttext=band(flags,0x01)~=0 and true    or false
+  local extrasize=band(flags,0x04)~=0 and readcardinal2(s) or 0
+  local filename=band(flags,0x08)~=0 and readcstring(s)   or ""
+  local comment=band(flags,0x10)~=0 and readcstring(s)   or ""
+  local checksum=band(flags,0x02)~=0 and readcardinal2(s) or 0
   local compressed=readstring(s,#str)
   local data=decompress(compressed,gzipwindow) 
   return data
@@ -25932,8 +25946,8 @@ end -- of closure
 
 -- used libraries    : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua libs-ini.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 1027101
--- stripped bytes    : 403437
+-- original bytes    : 1027893
+-- stripped bytes    : 403930
 
 -- end library merge
 
diff --git a/scripts/context/stubs/win64/mtxrun.lua b/scripts/context/stubs/win64/mtxrun.lua
index dc06a5f11..2e8baf15c 100644
--- a/scripts/context/stubs/win64/mtxrun.lua
+++ b/scripts/context/stubs/win64/mtxrun.lua
@@ -6525,7 +6525,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["util-str"] = package.loaded["util-str"] or true
 
--- original size: 45916, stripped down to: 23839
+-- original size: 46322, stripped down to: 24128
 
 if not modules then modules={} end modules ['util-str']={
  version=1.001,
@@ -6543,6 +6543,7 @@ local tonumber,type,tostring,next,setmetatable=tonumber,type,tostring,next,setme
 local unpack,concat=table.unpack,table.concat
 local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc
 local patterns,lpegmatch=lpeg.patterns,lpeg.match
+local tsplitat=lpeg.tsplitat
 local utfchar,utfbyte,utflen=utf.char,utf.byte,utf.len
 local loadstripped=function(str,shortcuts)
  if shortcuts then
@@ -7439,7 +7440,6 @@ function number.to16dot16(n)
  return f_16_16(n/65536.0)
 end
 if not string.explode then
- local tsplitat=lpeg.tsplitat
  local p_utf=patterns.utf8character
  local p_check=C(p_utf)*(P("+")*Cc(true))^0
  local p_split=Ct(C(p_utf)^0)
@@ -7459,6 +7459,20 @@ if not string.explode then
   end
  end
 end
+do
+ local p_whitespace=patterns.whitespace^1
+ local cache=setmetatable({},{ __index=function(t,k)
+  local p=tsplitat(p_whitespace*P(k)*p_whitespace)
+  local v=function(s)
+   return lpegmatch(p,s)
+  end
+  t[k]=v
+  return v
+ end })
+ function string.wordsplitter(s)
+  return cache[s]
+ end
+end
 
 
 end -- of closure
@@ -15699,7 +15713,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["util-zip"] = package.loaded["util-zip"] or true
 
--- original size: 23104, stripped down to: 14029
+-- original size: 23490, stripped down to: 14039
 
 if not modules then modules={} end modules ['util-zip']={
  version=1.001,
@@ -16176,11 +16190,11 @@ else
   local timestamp=readcardinal4(s)
   local compression=readbyte(s,1)
   local operating=readbyte(s,1)
-  local isjusttext=(flags & 0x01~=0) and true    or false
-  local extrasize=(flags & 0x04~=0) and readcardinal2(s) or 0
-  local filename=(flags & 0x08~=0) and readcstring(s)   or ""
-  local comment=(flags & 0x10~=0) and readcstring(s)   or ""
-  local checksum=(flags & 0x02~=0) and readcardinal2(s) or 0
+  local isjusttext=band(flags,0x01)~=0 and true    or false
+  local extrasize=band(flags,0x04)~=0 and readcardinal2(s) or 0
+  local filename=band(flags,0x08)~=0 and readcstring(s)   or ""
+  local comment=band(flags,0x10)~=0 and readcstring(s)   or ""
+  local checksum=band(flags,0x02)~=0 and readcardinal2(s) or 0
   local compressed=readstring(s,#str)
   local data=decompress(compressed,gzipwindow) 
   return data
@@ -25932,8 +25946,8 @@ end -- of closure
 
 -- used libraries    : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua libs-ini.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 1027101
--- stripped bytes    : 403437
+-- original bytes    : 1027893
+-- stripped bytes    : 403930
 
 -- end library merge