beta 2010.10.21 23:27

18 files changed, 154 insertions, 94 deletions

diff --git a/tex/context/base/char-enc.lua b/tex/context/base/char-enc.lua
index 4d7ceaa57..f4d049738 100644
--- a/tex/context/base/char-enc.lua
+++ b/tex/context/base/char-enc.lua
@@ -8,7 +8,7 @@ if not modules then modules = { } end modules ['char-syn'] = {
 
 -- thanks to tex4ht for these mappings
 
-local allocate = utilities.storage.allocate
+local allocate, setinitializer = utilities.storage.allocate, utilities.storage.setinitializer
 
 characters       = characters or { }
 local characters = characters
@@ -144,23 +144,42 @@ characters.synonyms = allocate { -- afm mess
     Yen                = 0x00A5,
 }
 
-if not characters.enccodes then
+--~ if not characters.enccodes then
+--~
+--~     local enccodes = { } characters.enccodes  = enccodes
+--~
+--~     for unicode, data in next, characters.data do
+--~         local encname = data.adobename or data.contextname
+--~         if encname then
+--~             enccodes[encname] = unicode
+--~         end
+--~     end
+--~
+--~     for name, unicode in next, characters.synonyms do
+--~         if not enccodes[name] then enccodes[name] = unicode end
+--~     end
+--~
+--~
+--~ end
+--~
+--~ storage.register("characters.enccodes", characters.enccodes, "characters.enccodes")
 
-    local enccodes = { }
+-- As this table is seldom used, we can delay its definition. Beware, this means
+-- that table.print would not work on this file unless it is accessed once. This
+-- why the serializer does a dummy access.
 
+local enccodes = allocate()  characters.enccodes = enccodes
+
+local function initialize()
     for unicode, data in next, characters.data do
         local encname = data.adobename or data.contextname
         if encname then
             enccodes[encname] = unicode
         end
     end
-
     for name, unicode in next, characters.synonyms do
         if not enccodes[name] then enccodes[name] = unicode end
     end
-
-    characters.enccodes  = enccodes
-
 end
 
-storage.register("characters.enccodes", characters.enccodes, "characters.enccodes")
+setinitializer(enccodes,initialize)
diff --git a/tex/context/base/char-ini.lua b/tex/context/base/char-ini.lua
index 6d58f6e98..0cce35a1a 100644
--- a/tex/context/base/char-ini.lua
+++ b/tex/context/base/char-ini.lua
@@ -389,6 +389,8 @@ table we derive a few more.</p>
 
 if not characters.fallbacks then
 
+    -- we could the definition by using a metatable
+
     characters.fallbacks   = { }
     characters.directions  = { }
 
@@ -786,7 +788,6 @@ setmetatable(specialchars, { __index = function(t,u)
     end
 end } )
 
-
 function characters.lower(str)
     local new = { }
     for u in utfvalues(str) do
diff --git a/tex/context/base/char-utf.lua b/tex/context/base/char-utf.lua
index 25c072dff..b3ed728f1 100644
--- a/tex/context/base/char-utf.lua
+++ b/tex/context/base/char-utf.lua
@@ -81,10 +81,10 @@ local function initialize()
     initialize = false
 end
 
--- utffilters.add_grapheme(utfchar(318),'l','\string~')
--- utffilters.add_grapheme('c','a','b')
+-- utffilters.addgrapheme(utfchar(318),'l','\string~')
+-- utffilters.addgrapheme('c','a','b')
 
-function utffilters.add_grapheme(result,first,second)
+function utffilters.addgrapheme(result,first,second)
     local r, f, s = tonumber(result), tonumber(first), tonumber(second)
     if r then result = utfchar(r) end
     if f then first  = utfchar(f) end
@@ -96,30 +96,30 @@ function utffilters.add_grapheme(result,first,second)
     end
 end
 
-function utffilters.collapse(str) -- old one
-    if utffilters.collapsing and str and #str > 1 then
-        if initialize then -- saves a call
-            initialize()
-        end
-        local tokens, first, done = { }, false, false
-        for second in utfcharacters(str) do
-            local cgf = graphemes[first]
-            if cgf and cgf[second] then
-                first, done = cgf[second], true
-            elseif first then
-                tokens[#tokens+1] = first
-                first = second
-            else
-                first = second
-            end
-        end
-        if done then
-            tokens[#tokens+1] = first
-            return concat(tokens)
-        end
-    end
-    return str
-end
+--~ function utffilters.collapse(str) -- old one
+--~     if utffilters.collapsing and str and #str > 1 then
+--~         if initialize then -- saves a call
+--~             initialize()
+--~         end
+--~         local tokens, first, done = { }, false, false
+--~         for second in utfcharacters(str) do
+--~             local cgf = graphemes[first]
+--~             if cgf and cgf[second] then
+--~                 first, done = cgf[second], true
+--~             elseif first then
+--~                 tokens[#tokens+1] = first
+--~                 first = second
+--~             else
+--~                 first = second
+--~             end
+--~         end
+--~         if done then
+--~             tokens[#tokens+1] = first
+--~             return concat(tokens)
+--~         end
+--~     end
+--~     return str
+--~ end
 
 --[[ldx--
 <p>In order to deal with 8-bit output, we need to find a way to
diff --git a/tex/context/base/char-utf.mkiv b/tex/context/base/char-utf.mkiv
index b96aec38d..8992b098e 100644
--- a/tex/context/base/char-utf.mkiv
+++ b/tex/context/base/char-utf.mkiv
@@ -42,6 +42,6 @@
 %D \stoptyping
 
 \unexpanded\def\definecomposedutf#1 #2 #3 %
-  {\ctxlua{characters.filters.utf.add_grapheme("#1","#2","#3")}}
+  {\ctxlua{characters.filters.utf.addgrapheme("#1","#2","#3")}}
 
 \protect
diff --git a/tex/context/base/cont-new.tex b/tex/context/base/cont-new.tex
index 90105af26..117d1c356 100644
--- a/tex/context/base/cont-new.tex
+++ b/tex/context/base/cont-new.tex
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2010.10.21 19:55}
+\newcontextversion{2010.10.21 23:27}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/context.tex b/tex/context/base/context.tex
index 876ec1152..0301243fe 100644
--- a/tex/context/base/context.tex
+++ b/tex/context/base/context.tex
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2010.10.21 19:55}
+\edef\contextversion{2010.10.21 23:27}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/enco-ini.mkiv b/tex/context/base/enco-ini.mkiv
index 70cbd2ce0..f9fcf3360 100644
--- a/tex/context/base/enco-ini.mkiv
+++ b/tex/context/base/enco-ini.mkiv
@@ -92,15 +92,15 @@
 
 %D Accent handling (try to avoid this):
 
-\newbox\accenttestbox
+\newbox\accenttestbox % no longer global so we could use \scratchbox
 
 \def\buildmathaccent#1%
   {\mathaccent#1 }
 
-\unexpanded\def\buildtextaccent#1#2%
-  {\begingroup
-   \global\setbox\accenttestbox\hbox{#1}%
-   \scratchcounter\cldcontext{nodes.firstcharinbox(\number\accenttestbox)}%
+\unexpanded\def\buildtextaccent#1#2% we could do all at the lua end
+  {\begingroup                     % but that's no fun (yet)
+   \setbox\accenttestbox\hbox{#1}%
+   \scratchcounter\cldcontext{nodes.firstcharinbox(\number\accenttestbox)}\relax
    \ifcase\scratchcounter\else\accent\scratchcounter\fi
    \relax#2%
    \endgroup}
diff --git a/tex/context/base/font-col.lua b/tex/context/base/font-col.lua
index 59ba0547c..d22a8088e 100644
--- a/tex/context/base/font-col.lua
+++ b/tex/context/base/font-col.lua
@@ -9,7 +9,7 @@ if not modules then modules = { } end modules ['font-col'] = {
 -- possible optimization: delayed initialization of vectors
 
 local gmatch, type = string.gmatch, type
-local traverse_id, first_character = node.traverse_id, node.first_character
+local traverse_id = node.traverse_id
 local lpegmatch = lpeg.match
 local settings_to_hash = utilities.parsers.settings_to_hash
 
diff --git a/tex/context/base/font-enc.lua b/tex/context/base/font-enc.lua
index 000fb21bb..7b4f4a4f8 100644
--- a/tex/context/base/font-enc.lua
+++ b/tex/context/base/font-enc.lua
@@ -53,6 +53,8 @@ Latin Modern or <l n='tex'> Gyre) come in OpenType variants too, so these
 will be used.</p>
 --ldx]]--
 
+local enccodes = characters.enccodes
+
 function enc.load(filename)
     local name = file.removesuffix(filename)
     local data = containers.read(enc.cache,name)
@@ -70,7 +72,6 @@ function enc.load(filename)
     if foundname and foundname ~= "" then
         local ok, encoding, size = resolvers.loadbinfile(foundname)
         if ok and encoding then
-            local enccodes = characters.enccodes
             encoding = gsub(encoding,"%%(.-)\n","")
             local tag, vec = match(encoding,"/(%w+)%s*%[(.*)%]%s*def")
             local i = 0
diff --git a/tex/context/base/font-fbk.lua b/tex/context/base/font-fbk.lua
index 5916319fc..b1556ecbf 100644
--- a/tex/context/base/font-fbk.lua
+++ b/tex/context/base/font-fbk.lua
@@ -76,7 +76,7 @@ function vf.aux.compose_characters(g) -- todo: scaling depends on call location
             report_combining("using compose information from goodies file")
         end
         local done = false
-        for i,c in next, characters.data do
+        for i,c in next, characters.data do -- loop over all characters ... not that efficient but a specials hash takes memory
             if force_composed or not chars[i] then
                 local s = c.specials
                 if s and s[1] == 'char' then
@@ -84,7 +84,7 @@ function vf.aux.compose_characters(g) -- todo: scaling depends on call location
                     local charschr = chars[chr]
                     if charschr then
                         local cc = c.category
-                        if cc == 'll' or cc == 'lu' or cc == 'lt' then
+                        if cc == 'll' or cc == 'lu' or cc == 'lt' then -- characters.is_letter[cc]
                             local acc = s[3]
                             local t = { }
                             for k, v in next, charschr do
@@ -101,7 +101,7 @@ function vf.aux.compose_characters(g) -- todo: scaling depends on call location
                         --~ elseif ca == "me" then
                         --~     -- mark enclosing
                         --~ else
-                            if not charsacc then
+                            if not charsacc then -- fallback accents
                                 acc = fallbacks[acc]
                                 charsacc = acc and chars[acc]
                             end
diff --git a/tex/context/base/font-otn.lua b/tex/context/base/font-otn.lua
index 608e42fb3..2bccc3c1b 100644
--- a/tex/context/base/font-otn.lua
+++ b/tex/context/base/font-otn.lua
@@ -124,6 +124,7 @@ local concat, insert, remove = table.concat, table.insert, table.remove
 local format, gmatch, gsub, find, match, lower, strip = string.format, string.gmatch, string.gsub, string.find, string.match, string.lower, string.strip
 local type, next, tonumber, tostring = type, next, tonumber, tostring
 local lpegmatch = lpeg.match
+local random = math.random
 
 local logs, trackers, fonts, nodes, attributes = logs, trackers, fonts, nodes, attributes
 
@@ -419,7 +420,7 @@ end
 local function alternative_glyph(start,alternatives,kind,chainname,chainlookupname,lookupname) -- chainname and chainlookupname optional
     local value, choice, n = featurevalue or tfmdata.shared.features[kind], nil, #alternatives -- global value, brrr
     if value == "random" then
-        local r = math.random(1,n)
+        local r = random(1,n)
         value, choice = format("random, choice %s",r), alternatives[r]
     elseif value == "first" then
         value, choice = format("first, choice %s",1), alternatives[1]
@@ -1784,22 +1785,22 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
 
                     local i = 1
                     repeat
-if skipped then
-    while true do
-        local char = start.char
-        local ccd = descriptions[char]
-        if ccd then
-            local class = ccd.class
-            if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then
-                start = start.next
-            else
-                break
-            end
-        else
-            break
-        end
-    end
-end
+                        if skipped then
+                            while true do
+                                local char = start.char
+                                local ccd = descriptions[char]
+                                if ccd then
+                                    local class = ccd.class
+                                    if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then
+                                        start = start.next
+                                    else
+                                        break
+                                    end
+                                else
+                                    break
+                                end
+                            end
+                        end
                         local chainlookupname = chainlookups[i]
                         local chainlookup = lookuptable[chainlookupname]
                         local cp = chainmores[chainlookup.type]
diff --git a/tex/context/base/l-table.lua b/tex/context/base/l-table.lua
index e165acba6..70c04e69c 100644
--- a/tex/context/base/l-table.lua
+++ b/tex/context/base/l-table.lua
@@ -597,8 +597,14 @@ local function serialize(root,name,_handle,_reduce,_noquotes,_hexify)
     else
         handle("t={")
     end
-    if root and next(root) then
-        do_serialize(root,name,"",0)
+    if root then
+        -- The dummy access will initialize a table that has a delayed initialization
+        -- using a metatable.
+        local dummy = root.whatever
+        -- Let's forget about empty tables.
+        if next(root) then
+            do_serialize(root,name,"",0)
+        end
     end
     handle("}")
 end
diff --git a/tex/context/base/node-aux.lua b/tex/context/base/node-aux.lua
index 0d4ab665d..275e2ba43 100644
--- a/tex/context/base/node-aux.lua
+++ b/tex/context/base/node-aux.lua
@@ -10,10 +10,12 @@ local nodes, node = nodes, node
 
 local nodecodes       = nodes.nodecodes
 
+local glyph_code      = nodecodes.glyph
 local hlist_code      = nodecodes.hlist
 local vlist_code      = nodecodes.vlist
 
 local traverse_nodes  = node.traverse
+local traverse_id     = node.traverse_id
 local free_node       = node.free
 local hpack_nodes     = node.hpack
 local has_attribute   = node.has_attribute
@@ -157,8 +159,22 @@ nodes.unset_attributes     = unset_attributes
 --     end
 -- end
 
+function nodes.firstcharacter(n,untagged) -- tagged == subtype > 255
+    if untagged then
+        return first_character(n)
+    else
+        for g in traverse_id(glyph_code,n) do
+            return g
+        end
+    end
+end
+
 function nodes.firstcharinbox(n)
     local l = texbox[n].list
-    local f = l and first_character(l)
-    return f and f.char or 0
+    if l then
+        for g in traverse_id(glyph_code,l) do
+            return g.char
+        end
+    end
+    return 0
 end
diff --git a/tex/context/base/node-tra.lua b/tex/context/base/node-tra.lua
index 8094f3736..78042cf44 100644
--- a/tex/context/base/node-tra.lua
+++ b/tex/context/base/node-tra.lua
@@ -49,7 +49,6 @@ local step_tracers    = tracers.steppers
 local copy_node_list  = node.copy_list
 local hpack_node_list = node.hpack
 local free_node_list  = node.flush_list
-local first_character = node.first_character
 local node_type       = node.type
 local traverse_nodes  = node.traverse
 
@@ -237,8 +236,7 @@ function step_tracers.glyphs(n,i)
 end
 
 function step_tracers.features()
---  local f = first_character(collection[1])
---  if f then -- something fishy with first_character
+    -- we cannot use first_character here as it only finds characters with subtype < 256
     local f = collection[1]
     while f do
         if f.id == glyph_code then
diff --git a/tex/context/base/strc-reg.lua b/tex/context/base/strc-reg.lua
index 8ab611677..96b45022f 100644
--- a/tex/context/base/strc-reg.lua
+++ b/tex/context/base/strc-reg.lua
@@ -411,7 +411,7 @@ local function removeemptyentries(result)
         end
     end
     if m > 0 then
-        report_registers("%s empty entries removed in register")
+        report_registers("%s empty entries removed in register",m)
     end
 end
 
diff --git a/tex/context/base/trac-inf.lua b/tex/context/base/trac-inf.lua
index 7a4c336ae..7bd7a7ca4 100644
--- a/tex/context/base/trac-inf.lua
+++ b/tex/context/base/trac-inf.lua
@@ -121,6 +121,7 @@ function statistics.show(reporter)
             local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0
             return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total)
         end)
+        collectgarbage("collect")
         register("current memory usage", statistics.memused)
         register("runtime",statistics.runtime)
         for i=1,#statusinfo do
diff --git a/tex/context/base/util-sto.lua b/tex/context/base/util-sto.lua
index 49abd8c82..34787be9b 100644
--- a/tex/context/base/util-sto.lua
+++ b/tex/context/base/util-sto.lua
@@ -88,3 +88,13 @@ end
 --~ local t = bla.test
 --~ table.print(t)
 --~ print(t.a)
+
+function storage.setinitializer(data,initialize)
+    local m = getmetatable(data) or { }
+    m.__index = function(data,k)
+        m.__index = nil -- so that we can access the entries during initializing
+        initialize()
+        return data[k]
+    end
+    setmetatable(data, m)
+end
diff --git a/tex/generic/context/luatex-fonts-merged.lua b/tex/generic/context/luatex-fonts-merged.lua
index f1b31447b..6bf864ec1 100644
--- a/tex/generic/context/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 10/21/10 19:55:03
+-- merge date  : 10/21/10 23:27:14
 
 do -- begin closure to overcome local limits and interference
 
@@ -1234,8 +1234,14 @@ local function serialize(root,name,_handle,_reduce,_noquotes,_hexify)
     else
         handle("t={")
     end
-    if root and next(root) then
-        do_serialize(root,name,"",0)
+    if root then
+        -- The dummy access will initialize a table that has a delayed initialization
+        -- using a metatable.
+        local dummy = root.whatever
+        -- Let's forget about empty tables.
+        if next(root) then
+            do_serialize(root,name,"",0)
+        end
     end
     handle("}")
 end
@@ -8041,6 +8047,7 @@ local concat, insert, remove = table.concat, table.insert, table.remove
 local format, gmatch, gsub, find, match, lower, strip = string.format, string.gmatch, string.gsub, string.find, string.match, string.lower, string.strip
 local type, next, tonumber, tostring = type, next, tonumber, tostring
 local lpegmatch = lpeg.match
+local random = math.random
 
 local logs, trackers, fonts, nodes, attributes = logs, trackers, fonts, nodes, attributes
 
@@ -8336,7 +8343,7 @@ end
 local function alternative_glyph(start,alternatives,kind,chainname,chainlookupname,lookupname) -- chainname and chainlookupname optional
     local value, choice, n = featurevalue or tfmdata.shared.features[kind], nil, #alternatives -- global value, brrr
     if value == "random" then
-        local r = math.random(1,n)
+        local r = random(1,n)
         value, choice = format("random, choice %s",r), alternatives[r]
     elseif value == "first" then
         value, choice = format("first, choice %s",1), alternatives[1]
@@ -9701,22 +9708,22 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
 
                     local i = 1
                     repeat
-if skipped then
-    while true do
-        local char = start.char
-        local ccd = descriptions[char]
-        if ccd then
-            local class = ccd.class
-            if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then
-                start = start.next
-            else
-                break
-            end
-        else
-            break
-        end
-    end
-end
+                        if skipped then
+                            while true do
+                                local char = start.char
+                                local ccd = descriptions[char]
+                                if ccd then
+                                    local class = ccd.class
+                                    if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then
+                                        start = start.next
+                                    else
+                                        break
+                                    end
+                                else
+                                    break
+                                end
+                            end
+                        end
                         local chainlookupname = chainlookups[i]
                         local chainlookup = lookuptable[chainlookupname]
                         local cp = chainmores[chainlookup.type]