2017-04-20 21:37:00

5 files changed, 113 insertions, 24 deletions

diff --git a/scripts/context/lua/mtx-unicode.lua b/scripts/context/lua/mtx-unicode.lua
index 5bf12a871..418df9261 100644
--- a/scripts/context/lua/mtx-unicode.lua
+++ b/scripts/context/lua/mtx-unicode.lua
@@ -46,8 +46,8 @@ local application = logs.application {
     helpinfo = helpinfo,
 }
 
-local gmatch, match, gsub, find, lower, format = string.gmatch, string.match, string.gsub, string.find, string.lower, string.format
-local concat = table.concat
+local gmatch, match, gsub, find, lower, upper, format = string.gmatch, string.match, string.gsub, string.find, string.lower, string.upper, string.format
+local concat, sort = table.concat, table.sort
 local split, splitlines, strip = string.split, string.splitlines, string.strip
 local are_equal = table.are_equal
 local tonumber, tostring, rawget = tonumber, tostring, rawget
@@ -90,6 +90,7 @@ function scripts.unicode.update()
     local eastasianwidth       = texttables.eastasianwidth
     local standardizedvariants = texttables.standardizedvariants
     local arabicshaping        = texttables.arabicshaping
+    local index                = texttables.index
     local characterdata        = characters.data
     --
     for unicode, ud in table.sortedpairs(unicodedata) do
@@ -400,6 +401,22 @@ local function splitdefinition(str,index)
     return t
 end
 
+local function splitindex(str)
+    -- ok, quick and dirty ... could be a nice lpeg instead
+    local l = splitlines(str)
+    local n = { }
+    for i=1,#l do
+        local a, b, c = match(l[i],"([^%,]+)%,?(.-)\t(.*)")
+        if a and b and c then
+            local name = b .. " " .. a
+            name = strip(name)
+            name = gsub(name,"%s+"," ")
+            n[name] = tonumber(c,16)
+        end
+    end
+    return n
+end
+
 function scripts.unicode.load()
     local fullname = resolvers.findfile("char-def.lua")
     report("using: %s",fullname)
@@ -428,6 +445,7 @@ function scripts.unicode.load()
             eastasianwidth       = resolvers.findfile("eastasianwidth.txt")       or "",
             standardizedvariants = resolvers.findfile("standardizedvariants.txt") or "",
             arabicshaping        = resolvers.findfile("arabicshaping.txt")        or "",
+            index                = resolvers.findfile("index.txt")                or "",
         }
         --
         textdata = {
@@ -437,6 +455,7 @@ function scripts.unicode.load()
             eastasianwidth       = textfiles.eastasianwidth       ~= "" and io.loaddata(textfiles.eastasianwidth)       or "",
             standardizedvariants = textfiles.standardizedvariants ~= "" and io.loaddata(textfiles.standardizedvariants) or "",
             arabicshaping        = textfiles.arabicshaping        ~= "" and io.loaddata(textfiles.arabicshaping)        or "",
+            index                = textfiles.index                ~= "" and io.loaddata(textfiles.index)                or "",
         }
         texttables = {
             unicodedata          = splitdefinition(textdata.unicodedata,true),
@@ -445,6 +464,7 @@ function scripts.unicode.load()
             eastasianwidth       = splitdefinition(textdata.eastasianwidth,true),
             standardizedvariants = splitdefinition(textdata.standardizedvariants,false),
             arabicshaping        = splitdefinition(textdata.arabicshaping,true),
+            index                = splitindex(textdata.index),
         }
         return true
     else
@@ -500,11 +520,11 @@ function scripts.unicode.extras() -- old code
                 end
             end
         end
-        table.sort(result)
+        sort(result)
         for i=1,#result do
             report(result[i])
         end
-        table.sort(map)
+        sort(map)
         for i=1,#map do
             local m = map[i]
             if not blocks[m] then
@@ -512,6 +532,43 @@ function scripts.unicode.extras() -- old code
             end
         end
     end
+    --
+    local index  = texttables.index
+    local blocks = characters.blocks
+    local data   = characters.data
+    for k, v in next, index do
+        if k ~= lower(k) then
+            index[k] = nil
+        end
+    end
+-- for k, v in next, data do
+--     v.synonym  = nil
+--     v.synonyms = nil
+-- end
+    for k, v in table.sortedhash(index) do
+        local d = data[v]
+        if d and d.description ~= upper(k) then
+            local synonyms = d.synonyms
+            if synonyms then
+                local n = #synonyms
+                local f = false
+                for i=1,n do
+                    if synonyms[i] == k then
+                        f = true
+                        break
+                    end
+                end
+                if not f then
+                    synonyms[n+1] = k
+                end
+             -- synonyms = table.unique(synonyms)
+             -- d.synonyms = synonyms
+                sort(synonyms)
+            else
+                d.synonyms = { k }
+            end
+        end
+    end
 end
 
 -- the action
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index b2101937a..6b998eb74 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -1670,7 +1670,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["l-table"] = package.loaded["l-table"] or true
 
--- original size: 39197, stripped down to: 22960
+-- original size: 39587, stripped down to: 23144
 
 if not modules then modules={} end modules ['l-table']={
   version=1.001,
@@ -2020,7 +2020,11 @@ local function is_simple_table(t)
         local v=t[i]
         local tv=type(v)
         if tv=="number" then
-          tt[i]=v 
+          if hexify then
+            tt[i]=format("0x%X",v)
+          else
+            tt[i]=v 
+          end
         elseif tv=="string" then
           tt[i]=format("%q",v) 
         elseif tv=="boolean" then
@@ -2036,7 +2040,11 @@ local function is_simple_table(t)
         local v=t[i]
         local tv=type(v)
         if tv=="number" then
-          tt[i+1]=v 
+          if hexify then
+            tt[i+1]=format("0x%X",v)
+          else
+            tt[i+1]=v 
+          end
         elseif tv=="string" then
           tt[i+1]=format("%q",v) 
         elseif tv=="boolean" then
@@ -20422,8 +20430,8 @@ end -- of closure
 
 -- used libraries    : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 841485
--- stripped bytes    : 305240
+-- original bytes    : 841875
+-- stripped bytes    : 305446
 
 -- end library merge
 
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index b2101937a..6b998eb74 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -1670,7 +1670,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["l-table"] = package.loaded["l-table"] or true
 
--- original size: 39197, stripped down to: 22960
+-- original size: 39587, stripped down to: 23144
 
 if not modules then modules={} end modules ['l-table']={
   version=1.001,
@@ -2020,7 +2020,11 @@ local function is_simple_table(t)
         local v=t[i]
         local tv=type(v)
         if tv=="number" then
-          tt[i]=v 
+          if hexify then
+            tt[i]=format("0x%X",v)
+          else
+            tt[i]=v 
+          end
         elseif tv=="string" then
           tt[i]=format("%q",v) 
         elseif tv=="boolean" then
@@ -2036,7 +2040,11 @@ local function is_simple_table(t)
         local v=t[i]
         local tv=type(v)
         if tv=="number" then
-          tt[i+1]=v 
+          if hexify then
+            tt[i+1]=format("0x%X",v)
+          else
+            tt[i+1]=v 
+          end
         elseif tv=="string" then
           tt[i+1]=format("%q",v) 
         elseif tv=="boolean" then
@@ -20422,8 +20430,8 @@ end -- of closure
 
 -- used libraries    : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 841485
--- stripped bytes    : 305240
+-- original bytes    : 841875
+-- stripped bytes    : 305446
 
 -- end library merge
 
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index b2101937a..6b998eb74 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -1670,7 +1670,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["l-table"] = package.loaded["l-table"] or true
 
--- original size: 39197, stripped down to: 22960
+-- original size: 39587, stripped down to: 23144
 
 if not modules then modules={} end modules ['l-table']={
   version=1.001,
@@ -2020,7 +2020,11 @@ local function is_simple_table(t)
         local v=t[i]
         local tv=type(v)
         if tv=="number" then
-          tt[i]=v 
+          if hexify then
+            tt[i]=format("0x%X",v)
+          else
+            tt[i]=v 
+          end
         elseif tv=="string" then
           tt[i]=format("%q",v) 
         elseif tv=="boolean" then
@@ -2036,7 +2040,11 @@ local function is_simple_table(t)
         local v=t[i]
         local tv=type(v)
         if tv=="number" then
-          tt[i+1]=v 
+          if hexify then
+            tt[i+1]=format("0x%X",v)
+          else
+            tt[i+1]=v 
+          end
         elseif tv=="string" then
           tt[i+1]=format("%q",v) 
         elseif tv=="boolean" then
@@ -20422,8 +20430,8 @@ end -- of closure
 
 -- used libraries    : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 841485
--- stripped bytes    : 305240
+-- original bytes    : 841875
+-- stripped bytes    : 305446
 
 -- end library merge
 
diff --git a/scripts/context/stubs/win64/mtxrun.lua b/scripts/context/stubs/win64/mtxrun.lua
index b2101937a..6b998eb74 100644
--- a/scripts/context/stubs/win64/mtxrun.lua
+++ b/scripts/context/stubs/win64/mtxrun.lua
@@ -1670,7 +1670,7 @@ do -- create closure to overcome 200 locals limit
 
 package.loaded["l-table"] = package.loaded["l-table"] or true
 
--- original size: 39197, stripped down to: 22960
+-- original size: 39587, stripped down to: 23144
 
 if not modules then modules={} end modules ['l-table']={
   version=1.001,
@@ -2020,7 +2020,11 @@ local function is_simple_table(t)
         local v=t[i]
         local tv=type(v)
         if tv=="number" then
-          tt[i]=v 
+          if hexify then
+            tt[i]=format("0x%X",v)
+          else
+            tt[i]=v 
+          end
         elseif tv=="string" then
           tt[i]=format("%q",v) 
         elseif tv=="boolean" then
@@ -2036,7 +2040,11 @@ local function is_simple_table(t)
         local v=t[i]
         local tv=type(v)
         if tv=="number" then
-          tt[i+1]=v 
+          if hexify then
+            tt[i+1]=format("0x%X",v)
+          else
+            tt[i+1]=v 
+          end
         elseif tv=="string" then
           tt[i+1]=format("%q",v) 
         elseif tv=="boolean" then
@@ -20422,8 +20430,8 @@ end -- of closure
 
 -- used libraries    : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
 -- skipped libraries : -
--- original bytes    : 841485
--- stripped bytes    : 305240
+-- original bytes    : 841875
+-- stripped bytes    : 305446
 
 -- end library merge