2017-04-27 01:06:00

6 files changed, 114 insertions, 26 deletions

diff --git a/scripts/context/lua/mtx-fonts.lua b/scripts/context/lua/mtx-fonts.lua
index a3924ef8f..068d38a84 100644
--- a/scripts/context/lua/mtx-fonts.lua
+++ b/scripts/context/lua/mtx-fonts.lua
@@ -17,7 +17,7 @@ local concat = table.concat
 local write_nl = texio.write_nl
 
 local otfversion  = 2.826
-local otlversion  = 3.028
+local otlversion  = 3.029
 
 local helpinfo = [[
 <?xml version="1.0"?>
diff --git a/scripts/context/lua/mtx-unicode.lua b/scripts/context/lua/mtx-unicode.lua
index 418df9261..557e70b79 100644
--- a/scripts/context/lua/mtx-unicode.lua
+++ b/scripts/context/lua/mtx-unicode.lua
@@ -52,7 +52,9 @@ local split, splitlines, strip = string.split, string.splitlines, string.strip
 local are_equal = table.are_equal
 local tonumber, tostring, rawget = tonumber, tostring, rawget
 local lpegmatch = lpeg.match
+local P, C, S, R, Cs, Ct, Cg, Cf, Cc = lpeg.P, lpeg.C, lpeg.S, lpeg.R, lpeg.Cs, lpeg.Ct, lpeg.Cg, lpeg.Cf, lpeg.Cc
 local formatters = string.formatters
+local utfchar = utf.char
 
 local report = application.report
 
@@ -73,7 +75,7 @@ local sparse = false
 local split_space_table = lpeg.tsplitat(" ")
 local split_space_two   = lpeg.splitat (" ")
 local split_range_two   = lpeg.splitat ("..")
-local split_colon_table = lpeg.tsplitat(lpeg.P(" ")^0 * lpeg.P(";") * lpeg.P(" ")^0)
+local split_colon_table = lpeg.tsplitat(P(" ")^0 * P(";") * P(" ")^0)
 
 local skipped = {
     [0x002C6] = true, -- MODIFIER LETTER CIRCUMFLEX ACCENT
@@ -541,10 +543,10 @@ function scripts.unicode.extras() -- old code
             index[k] = nil
         end
     end
--- for k, v in next, data do
---     v.synonym  = nil
---     v.synonyms = nil
--- end
+ -- for k, v in next, data do
+ --     v.synonym  = nil
+ --     v.synonyms = nil
+ -- end
     for k, v in table.sortedhash(index) do
         local d = data[v]
         if d and d.description ~= upper(k) then
@@ -571,6 +573,91 @@ function scripts.unicode.extras() -- old code
     end
 end
 
+do
+
+    local space       = P(" ")
+    local spaces      = space^0
+    local semicolon   = P(";")
+    local hash        = P("#")
+    local newline     = S("\n\r")
+
+    local unicode     = Cs(R("09","AF")^1)/function(n) return tonumber(n,16) end
+                      * spaces
+    local components  = Ct (unicode^1)
+
+ -- local rubish_a    = semicolon
+ --                   * spaces
+ --                   * P("Emoji_ZWJ_Sequence")
+ --                   * spaces
+ --                   * semicolon
+ --                   * spaces
+ -- local description = C((1 - (spaces * (hash+newline)))^1)
+ -- local rubish_b    = (1-newline)^0
+ --                   * newline^1
+ --
+ -- local pattern_1   = Ct ( (
+ --     Cf ( Ct("") *
+ --         Cg (Cc("components") * components)
+ --       * rubish_a
+ --       * Cg (Cc("description") * description )
+ --       * rubish_b
+ --     , rawset)
+ --     + P(1) )^1 )
+
+    local rubish_a    = semicolon
+                      * spaces
+                      * P("non-")^0 * P("fully-qualified")
+                      * spaces
+                      * hash
+                      * spaces
+    local textstring  = C((1 - space)^1)
+                      * spaces
+    local description = ((1 - (spaces * newline))^1) / string.lower
+    local rubish_b    = (1-newline)^0
+                      * newline^1
+
+    local pattern_2   = Ct ( (
+        Cf ( Ct("") *
+            Cg (Cc("components") * components)
+          * rubish_a
+          * Cg (Cc("textstring") * textstring)
+          * Cg (Cc("description") * description )
+          * rubish_b
+        , rawset)
+        + P(1) )^1 )
+
+    function scripts.unicode.emoji(filename)
+
+        local name = resolvers.findfile("emoji-test.txt") or ""
+        if name == "" then
+            return
+        end
+        local l = io.loaddata(name)
+        local t = lpegmatch(pattern_2,l)
+
+        local hash = { }
+
+        local replace = lpeg.replacer {
+            ["#"] = "hash",
+            ["*"] = "asterisk"
+        }
+
+        for i=1,#t do
+            local v = t[i]
+            local d = v.description
+            local k = lpegmatch(replace,d) or d
+            hash[k] = v.components
+        end
+        local new = table.serialize(hash,"return", { hexify = true })
+        local old = io.loaddata(resolvers.findfile("char-emj.lua"))
+        if old and old ~= "" then
+            new = gsub(old,"^(.-)return .*$","%1" .. new)
+        end
+        io.savedata(filename,new)
+    end
+
+end
+
 -- the action
 
 local filename = environment.files[1]
@@ -583,6 +670,7 @@ else
         scripts.unicode.update()
         scripts.unicode.extras()
         scripts.unicode.save("char-def-new.lua")
+        scripts.unicode.emoji("char-emj-new.lua")
     else
         report("nothing to do")
     end
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index 6b998eb74..89c156551 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -1670,7 +1670,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["l-table"] = package.loaded["l-table"] or true
 
--- original size: 39587, stripped down to: 23144
+-- original size: 39608, stripped down to: 23165
 
 if not modules then modules={} end modules ['l-table']={
   version=1.001,
@@ -2003,7 +2003,7 @@ local reserved=table.tohash {
   'in','local','nil','not','or','repeat','return','then','true','until','while',
   'NaN','goto',
 }
-local function is_simple_table(t) 
+local function is_simple_table(t,hexify) 
   local nt=#t
   if nt>0 then
     local n=0
@@ -2121,7 +2121,7 @@ local function do_serialize(root,name,depth,level,indexed)
           if next(v)==nil then
             handle(format("%s {},",depth))
           elseif inline then 
-            local st=is_simple_table(v)
+            local st=is_simple_table(v,hexify)
             if st then
               handle(format("%s { %s },",depth,concat(st,", ")))
             else
@@ -2204,7 +2204,7 @@ local function do_serialize(root,name,depth,level,indexed)
             handle(format("%s [%q]={},",depth,k))
           end
         elseif inline then
-          local st=is_simple_table(v)
+          local st=is_simple_table(v,hexify)
           if st then
             if tk=="number" then
               if hexify then
@@ -20430,7 +20430,7 @@ end -- of closure
 
 -- used libraries    : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 841875
+-- original bytes    : 841896
 -- stripped bytes    : 305446
 
 -- end library merge
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index 6b998eb74..89c156551 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -1670,7 +1670,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["l-table"] = package.loaded["l-table"] or true
 
--- original size: 39587, stripped down to: 23144
+-- original size: 39608, stripped down to: 23165
 
 if not modules then modules={} end modules ['l-table']={
   version=1.001,
@@ -2003,7 +2003,7 @@ local reserved=table.tohash {
   'in','local','nil','not','or','repeat','return','then','true','until','while',
   'NaN','goto',
 }
-local function is_simple_table(t) 
+local function is_simple_table(t,hexify) 
   local nt=#t
   if nt>0 then
     local n=0
@@ -2121,7 +2121,7 @@ local function do_serialize(root,name,depth,level,indexed)
           if next(v)==nil then
             handle(format("%s {},",depth))
           elseif inline then 
-            local st=is_simple_table(v)
+            local st=is_simple_table(v,hexify)
             if st then
               handle(format("%s { %s },",depth,concat(st,", ")))
             else
@@ -2204,7 +2204,7 @@ local function do_serialize(root,name,depth,level,indexed)
             handle(format("%s [%q]={},",depth,k))
           end
         elseif inline then
-          local st=is_simple_table(v)
+          local st=is_simple_table(v,hexify)
           if st then
             if tk=="number" then
               if hexify then
@@ -20430,7 +20430,7 @@ end -- of closure
 
 -- used libraries    : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 841875
+-- original bytes    : 841896
 -- stripped bytes    : 305446
 
 -- end library merge
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index 6b998eb74..89c156551 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -1670,7 +1670,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["l-table"] = package.loaded["l-table"] or true
 
--- original size: 39587, stripped down to: 23144
+-- original size: 39608, stripped down to: 23165
 
 if not modules then modules={} end modules ['l-table']={
   version=1.001,
@@ -2003,7 +2003,7 @@ local reserved=table.tohash {
   'in','local','nil','not','or','repeat','return','then','true','until','while',
   'NaN','goto',
 }
-local function is_simple_table(t) 
+local function is_simple_table(t,hexify) 
   local nt=#t
   if nt>0 then
     local n=0
@@ -2121,7 +2121,7 @@ local function do_serialize(root,name,depth,level,indexed)
           if next(v)==nil then
             handle(format("%s {},",depth))
           elseif inline then 
-            local st=is_simple_table(v)
+            local st=is_simple_table(v,hexify)
             if st then
               handle(format("%s { %s },",depth,concat(st,", ")))
             else
@@ -2204,7 +2204,7 @@ local function do_serialize(root,name,depth,level,indexed)
             handle(format("%s [%q]={},",depth,k))
           end
         elseif inline then
-          local st=is_simple_table(v)
+          local st=is_simple_table(v,hexify)
           if st then
             if tk=="number" then
               if hexify then
@@ -20430,7 +20430,7 @@ end -- of closure
 
 -- used libraries    : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 841875
+-- original bytes    : 841896
 -- stripped bytes    : 305446
 
 -- end library merge
diff --git a/scripts/context/stubs/win64/mtxrun.lua b/scripts/context/stubs/win64/mtxrun.lua
index 6b998eb74..89c156551 100644
--- a/scripts/context/stubs/win64/mtxrun.lua
+++ b/scripts/context/stubs/win64/mtxrun.lua
@@ -1670,7 +1670,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["l-table"] = package.loaded["l-table"] or true
 
--- original size: 39587, stripped down to: 23144
+-- original size: 39608, stripped down to: 23165
 
 if not modules then modules={} end modules ['l-table']={
   version=1.001,
@@ -2003,7 +2003,7 @@ local reserved=table.tohash {
   'in','local','nil','not','or','repeat','return','then','true','until','while',
   'NaN','goto',
 }
-local function is_simple_table(t) 
+local function is_simple_table(t,hexify) 
   local nt=#t
   if nt>0 then
     local n=0
@@ -2121,7 +2121,7 @@ local function do_serialize(root,name,depth,level,indexed)
           if next(v)==nil then
             handle(format("%s {},",depth))
           elseif inline then 
-            local st=is_simple_table(v)
+            local st=is_simple_table(v,hexify)
             if st then
               handle(format("%s { %s },",depth,concat(st,", ")))
             else
@@ -2204,7 +2204,7 @@ local function do_serialize(root,name,depth,level,indexed)
             handle(format("%s [%q]={},",depth,k))
           end
         elseif inline then
-          local st=is_simple_table(v)
+          local st=is_simple_table(v,hexify)
           if st then
             if tk=="number" then
               if hexify then
@@ -20430,7 +20430,7 @@ end -- of closure
 
 -- used libraries    : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 841875
+-- original bytes    : 841896
 -- stripped bytes    : 305446
 
 -- end library merge