beta 2013.03.11 00:17

14 files changed, 383 insertions, 327 deletions

diff --git a/tex/context/base/buff-ini.lua b/tex/context/base/buff-ini.lua
index 3c26da28f..2519a1551 100644
--- a/tex/context/base/buff-ini.lua
+++ b/tex/context/base/buff-ini.lua
@@ -89,6 +89,44 @@ local function collectcontent(names,separator) -- no print
     end
 end
 
+local function loadcontent(names) -- no print
+    if type(names) == "string" then
+        names = settings_to_array(names)
+    end
+    local nnames = #names
+    local ok = false
+    if nnames == 0 then
+        ok = load(getcontent("")) -- default buffer
+    elseif nnames == 1 then
+        ok = load(getcontent(names[1]))
+    else
+        -- lua 5.2 chunked load
+        local i = 0
+        ok = load(function()
+            while true do
+                i = i + 1
+                if i > nnames then
+                    return nil
+                end
+                local c = getcontent(names[i])
+                if c == "" then
+                    -- would trigger end of load
+                else
+                    return c
+                end
+            end
+        end)
+    end
+    if ok then
+        return ok()
+    elseif nnames == 0 then
+        report_buffers("invalid lua code in default buffer")
+    else
+        report_buffers("invalid lua code in buffer '%s'",concat(names,","))
+    end
+end
+
+
 buffers.raw            = getcontent
 buffers.erase          = erase
 buffers.assign         = assign
@@ -97,6 +135,7 @@ buffers.exists         = exists
 buffers.getcontent     = getcontent
 buffers.getlines       = getlines
 buffers.collectcontent = collectcontent
+buffers.loadcontent    = loadcontent
 
 -- the context interface
 
@@ -305,14 +344,7 @@ function commands.gettexbuffer(name)
     end
 end
 
-function commands.getbufferctxlua(name)
-    local ok = load(getcontent(name))
-    if ok then
-        ok()
-    else
-        report_buffers("invalid lua code in buffer '%s'",name)
-    end
-end
+commands.getbufferctxlua = loadcontent
 
 function commands.doifelsebuffer(name)
     commands.doifelse(exists(name))
diff --git a/tex/context/base/char-ini.lua b/tex/context/base/char-ini.lua
index 277967ef3..9f8a0ce11 100644
--- a/tex/context/base/char-ini.lua
+++ b/tex/context/base/char-ini.lua
@@ -10,8 +10,6 @@ if not modules then modules = { } end modules ['char-ini'] = {
 
 -- we can remove the tag range starting at 0xE0000 (special applications)
 
-local tex = tex
-
 local utfchar, utfbyte, utfvalues, ustring = utf.char, utf.byte, utf.values, utf.ustring
 local concat, unpack, tohash = table.concat, table.unpack, table.tohash
 local next, tonumber, type, rawget, rawset = next, tonumber, type, rawget, rawset
@@ -23,14 +21,6 @@ local utf8char          = patterns.utf8char
 
 local allocate          = utilities.storage.allocate
 local mark              = utilities.storage.mark
-local texsetlccode      = tex.setlccode
-local texsetuccode      = tex.setuccode
-local texsetsfcode      = tex.setsfcode
-local texsetcatcode     = tex.setcatcode
-
-local contextsprint     = context.sprint
-local ctxcatcodes       = catcodes.numbers.ctxcatcodes
-local texcatcodes       = catcodes.numbers.texcatcodes
 
 local setmetatableindex = table.setmetatableindex
 
@@ -318,7 +308,7 @@ end)
 local otfscripts      = utilities.storage.allocate()
 characters.otfscripts = otfscripts
 
-table.setmetatableindex(otfscripts,function(t,unicode)
+setmetatableindex(otfscripts,function(t,unicode)
     for k, v in next, blocks do
         local first, last = v.first, v.last
         if unicode >= first and unicode <= last then
@@ -493,7 +483,9 @@ if not characters.fallbacks then
 
 end
 
-storage.register("characters/fallbacks", characters.fallbacks, "characters.fallbacks") -- accents and such
+if storage then
+    storage.register("characters/fallbacks", characters.fallbacks, "characters.fallbacks") -- accents and such
+end
 
 characters.directions  = { }
 
@@ -511,206 +503,10 @@ setmetatableindex(characters.directions,function(t,k)
 end)
 
 --[[ldx--
-<p>The <type>context</type> namespace is used to store methods and data
-which is rather specific to <l n='context'/>.</p>
---ldx]]--
-
---[[ldx--
-<p>Instead of using a <l n='tex'/> file to define the named glyphs, we
-use the table. After all, we have this information available anyway.</p>
---ldx]]--
-
-function characters.makeactive(n,name) --
-    contextsprint(ctxcatcodes,format("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name))
- -- context("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name)
-end
-
-function tex.uprint(c,n)
-    if n then
-     -- contextsprint(c,charfromnumber(n))
-        contextsprint(c,utfchar(n))
-    else
-     -- contextsprint(charfromnumber(c))
-        contextsprint(utfchar(c))
-    end
-end
-
-local forbidden = tohash { -- at least now
-    0x00A0,
-    0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x200B, 0x200C, 0x200D,
-    0x202F,
-    0x205F,
- -- 0xFEFF,
-}
-
-function characters.define(tobelettered, tobeactivated) -- catcodetables
-
-    if trace_defining then
-        report_defining("defining active character commands")
-    end
-
-    local activated, a = { }, 0
-
-    for u, chr in next, data do -- these will be commands
-        local fallback = chr.fallback
-        if fallback then
-            contextsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\checkedchar{",u,"}{",fallback,"}}}")
-            a = a + 1
-            activated[a] = u
-        else
-            local contextname = chr.contextname
-            if contextname then
-                local category = chr.category
-                if is_character[category] then
-                    if chr.unicodeslot < 128 then
-                        if is_letter[category] then
-                            contextsprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u))) -- has no s
-                        else
-                            contextsprint(ctxcatcodes,format("\\chardef\\%s=%s",contextname,u)) -- has no s
-                        end
-                    else
-                        contextsprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u))) -- has no s
-                    end
-                elseif is_command[category] and not forbidden[u] then
-                    contextsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\"..contextname,"}}")
-                    a = a + 1
-                    activated[a] = u
-                end
-            end
-        end
-    end
-
-    if tobelettered then -- shared
-        local saved = tex.catcodetable
-        for i=1,#tobelettered do
-            tex.catcodetable = tobelettered[i]
-            if trace_defining then
-                report_defining("defining letters (global, shared)")
-            end
-            for u, chr in next, data do
-                if not chr.fallback and is_letter[chr.category] and u >= 128 and u <= 65536 then
-                    texsetcatcode(u,11)
-                end
-                local range = chr.range
-                if range then
-                    for i=1,range.first,range.last do
-                        texsetcatcode(i,11)
-                    end
-                end
-            end
-            texsetcatcode(0x200C,11) -- non-joiner
-            texsetcatcode(0x200D,11) -- joiner
-        end
-        tex.catcodetable = saved
-    end
-
-    local nofactivated = #tobeactivated
-    if tobeactivated and nofactivated > 0 then
-        for i=1,nofactivated do
-            local u = activated[i]
-            if u then
-                report_defining("character 0x%05X is active in sets %s (%s)",u,concat(tobeactivated,","),data[u].description)
-            end
-        end
-        local saved = tex.catcodetable
-        for i=1,#tobeactivated do
-            local vector = tobeactivated[i]
-            if trace_defining then
-                report_defining("defining %s active characters in vector %s",nofactivated,vector)
-            end
-            tex.catcodetable = vector
-            for i=1,nofactivated do
-                local u = activated[i]
-                if u then
-                    texsetcatcode(u,13)
-                end
-            end
-        end
-        tex.catcodetable = saved
-    end
-
-end
-
---[[ldx--
-<p>Setting the lccodes is also done in a loop over the data table.</p>
---ldx]]--
-
-local sfmode = "unset" -- unset, traditional, normal
-
-function characters.setcodes()
-    if trace_defining then
-        report_defining("defining lc and uc codes")
-    end
-    local traditional = sfstate == "traditional" or sfstate == "unset"
-    for code, chr in next, data do
-        local cc = chr.category
-        if is_letter[cc] then
-            local range = chr.range
-            if range then
-                for i=range.first,range.last do
-                    texsetcatcode(i,11) -- letter
-                    texsetlccode(i,i,i) -- self self
-                end
-            else
-                local lc, uc = chr.lccode, chr.uccode
-                if not lc then
-                    chr.lccode, lc = code, code
-                elseif type(lc) == "table" then
-                    lc = code
-                end
-                if not uc then
-                    chr.uccode, uc = code, code
-                elseif type(uc) == "table" then
-                    uc = code
-                end
-                texsetcatcode(code,11)   -- letter
-                texsetlccode(code,lc,uc)
-                if traditional and cc == "lu" then
-                    texsetsfcode(code,999)
-                end
-            end
-        elseif is_mark[cc] then
-            texsetlccode(code,code,code) -- for hyphenation
-        end
-    end
-    if traditional then
-        sfstate = "traditional"
-    end
-end
-
--- If this is something that is not documentwide and used a lot, then we
--- need a more clever approach (trivial but not now).
-
-local function setuppersfcodes(v,n)
-    if sfstate ~= "unset" then
-        report_defining("setting uppercase sf codes to %s",n)
-        for code, chr in next, data do
-            if chr.category == "lu" then
-                texsetsfcode(code,n)
-            end
-        end
-    end
-    sfstate = v
-end
-
-directives.register("characters.spaceafteruppercase",function(v)
-    if v == "traditional" then
-        setuppersfcodes(v,999)
-    elseif v == "normal" then
-        setuppersfcodes(v,1000)
-    end
-end)
-
---[[ldx--
 <p>Next comes a whole series of helper methods. These are (will be) part
 of the official <l n='api'/>.</p>
 --ldx]]--
 
---[[ldx--
-<p>A couple of convenience methods. Beware, these are slower than directly
-accessing the data table.</p>
---ldx]]--
-
 -- we could make them virtual: characters.contextnames[n]
 
 function characters.contextname(n) return data[n].contextname or "" end
@@ -729,32 +525,6 @@ function characters.category(n,verbose)
     end
 end
 
--- xml support (moved)
-
-function characters.remapentity(chr,slot)
-    contextsprint(format("{\\catcode%s=13\\xdef%s{\\string%s}}",slot,utfchar(slot),chr))
-end
-
-characters.activeoffset = 0x10000 -- there will be remapped in that byte range
-
--- xml.entities = xml.entities or { }
---
--- storage.register("xml/entities",xml.entities,"xml.entities") -- this will move to lxml
---
--- function characters.setmkiventities()
---     local entities = xml.entities
---     entities.lt  = "<"
---     entities.amp = "&"
---     entities.gt  = ">"
--- end
---
--- function characters.setmkiientities()
---     local entities = xml.entities
---     entities.lt  = utfchar(characters.activeoffset + utfbyte("<"))
---     entities.amp = utfchar(characters.activeoffset + utfbyte("&"))
---     entities.gt  = utfchar(characters.activeoffset + utfbyte(">"))
--- end
-
 -- -- some day we will make a table .. not that many calls to utfchar
 --
 -- local utfchar = utf.char
@@ -947,6 +717,7 @@ function characters.lettered(str,spacing)
     end
     return concat(new)
 end
+
 --[[ldx--
 <p>Requesting lower and uppercase codes:</p>
 --ldx]]--
@@ -963,15 +734,6 @@ function characters.safechar(n)
     end
 end
 
-function commands.safechar(n)
-    local c = data[n]
-    if c and c.contextname then
-        contextsprint("\\" .. c.contextname) -- context[c.contextname]()
-    else
-        contextsprint(utfchar(n))
-    end
-end
-
 function characters.shape(n)
     local shcode = shcodes[n]
     if not shcode then
@@ -1087,11 +849,248 @@ if not characters.superscripts then
  -- print(table.serialize(superscripts, "superscripts", { hexify = true }))
  -- print(table.serialize(subscripts,   "subscripts",   { hexify = true }))
 
-    storage.register("characters/superscripts", superscripts, "characters.superscripts")
-    storage.register("characters/subscripts",   subscripts,   "characters.subscripts")
+    if storage then
+        storage.register("characters/superscripts", superscripts, "characters.superscripts")
+        storage.register("characters/subscripts",   subscripts,   "characters.subscripts")
+    end
 
 end
 
--- interface
+-- the following code will move to char-tex.lua
+
+-- tex
+
+if not tex or not context or not commands then return characters end
+
+local tex           = tex
+local texsetlccode  = tex.setlccode
+local texsetuccode  = tex.setuccode
+local texsetsfcode  = tex.setsfcode
+local texsetcatcode = tex.setcatcode
+
+local contextsprint = context.sprint
+local ctxcatcodes   = catcodes.numbers.ctxcatcodes
+
+--[[ldx--
+<p>Instead of using a <l n='tex'/> file to define the named glyphs, we
+use the table. After all, we have this information available anyway.</p>
+--ldx]]--
+
+function commands.makeactive(n,name) --
+    contextsprint(ctxcatcodes,format("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name))
+ -- context("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name)
+end
+
+function commands.utfchar(c,n)
+    if n then
+     -- contextsprint(c,charfromnumber(n))
+        contextsprint(c,utfchar(n))
+    else
+     -- contextsprint(charfromnumber(c))
+        contextsprint(utfchar(c))
+    end
+end
+
+function commands.safechar(n)
+    local c = data[n]
+    if c and c.contextname then
+        contextsprint("\\" .. c.contextname) -- context[c.contextname]()
+    else
+        contextsprint(utfchar(n))
+    end
+end
+
+tex.uprint = commands.utfchar
+
+local forbidden = tohash { -- at least now
+    0x00A0,
+    0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x200B, 0x200C, 0x200D,
+    0x202F,
+    0x205F,
+ -- 0xFEFF,
+}
+
+function characters.define(tobelettered, tobeactivated) -- catcodetables
+
+    if trace_defining then
+        report_defining("defining active character commands")
+    end
+
+    local activated, a = { }, 0
+
+    for u, chr in next, data do -- these will be commands
+        local fallback = chr.fallback
+        if fallback then
+            contextsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\checkedchar{",u,"}{",fallback,"}}}")
+            a = a + 1
+            activated[a] = u
+        else
+            local contextname = chr.contextname
+            if contextname then
+                local category = chr.category
+                if is_character[category] then
+                    if chr.unicodeslot < 128 then
+                        if is_letter[category] then
+                            contextsprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u))) -- has no s
+                        else
+                            contextsprint(ctxcatcodes,format("\\chardef\\%s=%s",contextname,u)) -- has no s
+                        end
+                    else
+                        contextsprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u))) -- has no s
+                    end
+                elseif is_command[category] and not forbidden[u] then
+                    contextsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\"..contextname,"}}")
+                    a = a + 1
+                    activated[a] = u
+                end
+            end
+        end
+    end
+
+    if tobelettered then -- shared
+        local saved = tex.catcodetable
+        for i=1,#tobelettered do
+            tex.catcodetable = tobelettered[i]
+            if trace_defining then
+                report_defining("defining letters (global, shared)")
+            end
+            for u, chr in next, data do
+                if not chr.fallback and is_letter[chr.category] and u >= 128 and u <= 65536 then
+                    texsetcatcode(u,11)
+                end
+                local range = chr.range
+                if range then
+                    for i=1,range.first,range.last do
+                        texsetcatcode(i,11)
+                    end
+                end
+            end
+            texsetcatcode(0x200C,11) -- non-joiner
+            texsetcatcode(0x200D,11) -- joiner
+        end
+        tex.catcodetable = saved
+    end
+
+    local nofactivated = #tobeactivated
+    if tobeactivated and nofactivated > 0 then
+        for i=1,nofactivated do
+            local u = activated[i]
+            if u then
+                report_defining("character 0x%05X is active in sets %s (%s)",u,concat(tobeactivated,","),data[u].description)
+            end
+        end
+        local saved = tex.catcodetable
+        for i=1,#tobeactivated do
+            local vector = tobeactivated[i]
+            if trace_defining then
+                report_defining("defining %s active characters in vector %s",nofactivated,vector)
+            end
+            tex.catcodetable = vector
+            for i=1,nofactivated do
+                local u = activated[i]
+                if u then
+                    texsetcatcode(u,13)
+                end
+            end
+        end
+        tex.catcodetable = saved
+    end
+
+end
+
+--[[ldx--
+<p>Setting the lccodes is also done in a loop over the data table.</p>
+--ldx]]--
+
+local sfmode = "unset" -- unset, traditional, normal
+
+function characters.setcodes()
+    if trace_defining then
+        report_defining("defining lc and uc codes")
+    end
+    local traditional = sfstate == "traditional" or sfstate == "unset"
+    for code, chr in next, data do
+        local cc = chr.category
+        if is_letter[cc] then
+            local range = chr.range
+            if range then
+                for i=range.first,range.last do
+                    texsetcatcode(i,11) -- letter
+                    texsetlccode(i,i,i) -- self self
+                end
+            else
+                local lc, uc = chr.lccode, chr.uccode
+                if not lc then
+                    chr.lccode, lc = code, code
+                elseif type(lc) == "table" then
+                    lc = code
+                end
+                if not uc then
+                    chr.uccode, uc = code, code
+                elseif type(uc) == "table" then
+                    uc = code
+                end
+                texsetcatcode(code,11)   -- letter
+                texsetlccode(code,lc,uc)
+                if traditional and cc == "lu" then
+                    texsetsfcode(code,999)
+                end
+            end
+        elseif is_mark[cc] then
+            texsetlccode(code,code,code) -- for hyphenation
+        end
+    end
+    if traditional then
+        sfstate = "traditional"
+    end
+end
+
+-- If this is something that is not documentwide and used a lot, then we
+-- need a more clever approach (trivial but not now).
+
+local function setuppersfcodes(v,n)
+    if sfstate ~= "unset" then
+        report_defining("setting uppercase sf codes to %s",n)
+        for code, chr in next, data do
+            if chr.category == "lu" then
+                texsetsfcode(code,n)
+            end
+        end
+    end
+    sfstate = v
+end
+
+directives.register("characters.spaceafteruppercase",function(v)
+    if v == "traditional" then
+        setuppersfcodes(v,999)
+    elseif v == "normal" then
+        setuppersfcodes(v,1000)
+    end
+end)
+
+-- xml
+
+characters.activeoffset = 0x10000 -- there will be remapped in that byte range
+
+function commands.remapentity(chr,slot)
+    contextsprint(format("{\\catcode%s=13\\xdef%s{\\string%s}}",slot,utfchar(slot),chr))
+end
+
+-- xml.entities = xml.entities or { }
+--
+-- storage.register("xml/entities",xml.entities,"xml.entities") -- this will move to lxml
+--
+-- function characters.setmkiventities()
+--     local entities = xml.entities
+--     entities.lt  = "<"
+--     entities.amp = "&"
+--     entities.gt  = ">"
+-- end
+--
+-- function characters.setmkiientities()
+--     local entities = xml.entities
+--     entities.lt  = utfchar(characters.activeoffset + utfbyte("<"))
+--     entities.amp = utfchar(characters.activeoffset + utfbyte("&"))
+--     entities.gt  = utfchar(characters.activeoffset + utfbyte(">"))
+-- end
 
-commands.utfchar = tex.uprint
diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii
index aa4e3223d..1434b8eee 100644
--- a/tex/context/base/cont-new.mkii
+++ b/tex/context/base/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2013.03.08 01:14}
+\newcontextversion{2013.03.11 00:17}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index debad7bae..8340b5c60 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2013.03.08 01:14}
+\newcontextversion{2013.03.11 00:17}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index bbc4ff28d..a0d46f708 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png
index 43823ceb6..2bdeb57b6 100644
--- a/tex/context/base/context-version.png
+++ b/tex/context/base/context-version.png
diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii
index 20aefca87..8bc5bdd52 100644
--- a/tex/context/base/context.mkii
+++ b/tex/context/base/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2013.03.08 01:14}
+\edef\contextversion{2013.03.11 00:17}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 4fcd7ae07..6fa8db9ff 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2013.03.08 01:14}
+\edef\contextversion{2013.03.11 00:17}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/lxml-ini.mkiv b/tex/context/base/lxml-ini.mkiv
index 042b22e64..8889d906b 100644
--- a/tex/context/base/lxml-ini.mkiv
+++ b/tex/context/base/lxml-ini.mkiv
@@ -418,7 +418,7 @@
 % \stopextendcatcodetable
 %
 % \ctxlua { % entities are remembered in the format
-%     characters.remapentity("<",characters.activeoffset + utf.byte("<"))
-%     characters.remapentity("&",characters.activeoffset + utf.byte("&"))
-%     characters.remapentity(">",characters.activeoffset + utf.byte(">"))
+%     commands.remapentity("<",characters.activeoffset + utf.byte("<"))
+%     commands.remapentity("&",characters.activeoffset + utf.byte("&"))
+%     commands.remapentity(">",characters.activeoffset + utf.byte(">"))
 % }
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 4803e083f..1a6a03c89 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index a0bbca101..27eff5b32 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/util-str.lua b/tex/context/base/util-str.lua
index 959955867..11ca90912 100644
--- a/tex/context/base/util-str.lua
+++ b/tex/context/base/util-str.lua
@@ -12,7 +12,8 @@ local strings     = utilities.strings
 
 local format, gsub, rep, sub = string.format, string.gsub, string.rep, string.sub
 local load, dump = load, string.dump
-local concat = table.concat
+local tonumber, type, tostring = tonumber, type, tostring
+local unpack, concat = table.unpack, table.concat
 local P, V, C, S, R, Ct, Cs, Cp, Carg, Cc = lpeg.P, lpeg.V, lpeg.C, lpeg.S, lpeg.R, lpeg.Ct, lpeg.Cs, lpeg.Cp, lpeg.Carg, lpeg.Cc
 local patterns, lpegmatch = lpeg.patterns, lpeg.match
 local utfchar, utfbyte = utf.char, utf.byte
@@ -168,8 +169,10 @@ local n = 0
 --     print(...,...,...) -- 1,1,1,2,3
 -- end
 
-local template_shortcuts = [[
+local preamble = [[
+local type = type
 local tostring = tostring
+local tonumber = tonumber
 local format = string.format
 local concat = table.concat
 local signed = number.signed
@@ -182,6 +185,23 @@ local xmlescape = lpeg.patterns.xmlescape
 local spaces = string.nspaces
 ]]
 
+local template = [[
+%s
+%s
+return function(%s) return %s end
+]]
+
+
+local arguments = { "a1" } -- faster than previously used (select(n,...))
+
+setmetatable(arguments, { __index =
+    function(t,k)
+        local v = t[k-1] .. ",a" .. k
+        t[k] = v
+        return v
+    end
+})
+
 local prefix_any = C((S("+- .") + R("09"))^0)
 local prefix_tab = C((1-R("az","AZ","09","%%"))^0)
 
@@ -297,9 +317,9 @@ local format_h = function(f)
     n = n + 1
     if f == "-" then
         f = sub(f,2)
-        return format("format('%%%sx',utfbyte(a%s))",f == "" and "05" or f,n)
+        return format("format('%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f == "" and "05" or f,n,n,n)
     else
-        return format("format('0x%%%sx',utfbyte(a%s))",f == "" and "05" or f,n)
+        return format("format('0x%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f == "" and "05" or f,n,n,n)
     end
 end
 
@@ -307,9 +327,9 @@ local format_H = function(f)
     n = n + 1
     if f == "-" then
         f = sub(f,2)
-        return format("format('%%%sX',utfbyte(a%s))",f == "" and "05" or f,n)
+        return format("format('%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f == "" and "05" or f,n,n,n)
     else
-        return format("format('0x%%%sX',utfbyte(a%s))",f == "" and "05" or f,n)
+        return format("format('0x%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f == "" and "05" or f,n,n,n)
     end
 end
 
@@ -317,9 +337,9 @@ local format_u = function(f)
     n = n + 1
     if f == "-" then
         f = sub(f,2)
-        return format("format('%%%sx',utfbyte(a%s))",f == "" and "05" or f,n)
+        return format("format('%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f == "" and "05" or f,n,n,n)
     else
-        return format("format('u+%%%sx',utfbyte(a%s))",f == "" and "05" or f,n)
+        return format("format('u+%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f == "" and "05" or f,n,n,n)
     end
 end
 
@@ -327,9 +347,9 @@ local format_U = function(f)
     n = n + 1
     if f == "-" then
         f = sub(f,2)
-        return format("format('%%%sX',utfbyte(a%s))",f == "" and "05" or f,n)
+        return format("format('%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f == "" and "05" or f,n,n,n)
     else
-        return format("format('U+%%%sX',utfbyte(a%s))",f == "" and "05" or f,n)
+        return format("format('U+%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f == "" and "05" or f,n,n,n)
     end
 end
 
@@ -385,27 +405,26 @@ local format_W = function(f) -- handy when doing depth related indent
     return format("spaces[%s]",tonumber(f) or 0)
 end
 
-local extensions = { }
-
-local format_extension = function(name)
-    n = n + 1
+local format_extension = function(extensions,f,name)
     local extension = extensions[name] or "tostring(%s)"
-    return format(extension,format("a%s",n))
-end
-
-function addextension(name,template,shortcuts)
-    extensions[name] = template
-    if shortcuts then
-        template_shortcuts = shortcuts .. "\n" .. template_shortcuts -- so we can't overload
+    local f = tonumber(f) or 1
+    if f == 0 then
+        return extension
+    elseif f == 1 then
+        n = n + 1
+        return format(extension,"a"..n)
+    elseif f < 0 then
+        return format(extension,"a"..n+f+1)
+    else
+        local t = { }
+        for i=1,f do
+            n = n + 1
+            t[#t+1] = "a"..n
+        end
+        return format(extension,unpack(t))
     end
 end
 
-lpeg.patterns.xmlescape = Cs((P("<")/"&lt;" + P(">")/"&gt;" + P("&")/"&amp;" + P('"')/"&quot;" + P(1))^0)
-lpeg.patterns.texescape = Cs((C(S("#$%\\{}"))/"\\%1" + P(1))^0)
-
-addextension("xml",[[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]])
-addextension("tex",[[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]])
-
 local builder = Cs { "start",
     start = (
         (
@@ -476,7 +495,7 @@ local builder = Cs { "start",
     ["a"] = Cs(((1-P("%"))^1 + P("%%")/"%%%%")^1) / format_a, -- rest (including %%)
     --
  -- ["!"] = P("!xml!") / format_xml, -- %!xml! => hypertext escaped " < > &
-    ["!"] = P("!") * C((1-P("!"))^1) * P("!") / format_extension,
+    ["!"] = Carg(2) * prefix_any * P("!") * C((1-P("!"))^1) * P("!") / format_extension,
 }
 
 -- we can be clever and only alias what is needed
@@ -489,21 +508,6 @@ local direct = Cs (
       * P(-1)
     )
 
-local template  = [[
-%s
-return function(%s) return %s end
-]]
-
-local arguments = { "a1" } -- faster than previously used (select(n,...))
-
-setmetatable(arguments, { __index =
-     function(t,k)
-        local v = t[k-1] .. ",a" .. k
-        t[k] = v
-        return v
-    end
-})
-
 local function make(t,str)
     local f
     local p = lpegmatch(direct,str)
@@ -511,9 +515,9 @@ local function make(t,str)
         f = loadstripped(p)()
     else
         n = 0
-        p = lpegmatch(builder,str,1,"..") -- after this we know n
+        p = lpegmatch(builder,str,1,"..",t._extensions_) -- after this we know n
         if n > 0 then
-            p = format(template,template_shortcuts,arguments[n],p)
+            p = format(template,preamble,t._preamble_,arguments[n],p)
          -- print("builder>",p)
             f = loadstripped(p)()
         else
@@ -528,10 +532,47 @@ local function use(t,fmt,...)
     return t[fmt](...)
 end
 
-local formatters  = string.formatters or { }
-string.formatters = formatters
+strings.formatters = { }
+
+function strings.formatters.new()
+    local t = { _extensions_ = { }, _preamble_ = "", _type_ = "formatter" }
+    setmetatable(t, { __index = make, __call = use })
+    return t
+end
+
+local formatters   = strings.formatters.new() -- the default instance
+
+string.formatters  = formatters -- in the main string namespace
+string.formatter   = function(str,...) return formatters[str](...) end -- sometimes nicer name
+
+local function add(t,name,template,preamble)
+    if type(t) == "table" and t._type_ == "formatter" then
+        t._extensions_[name] = template or "%s"
+        if preamble then
+            t._preamble_ = preamble .. "\n" .. t._preamble_ -- so no overload !
+        end
+    end
+end
+
+strings.formatters.add = add
+
+-- registered in the default instance (should we fall back on this one?)
+
+lpeg.patterns.xmlescape = Cs((P("<")/"&lt;" + P(">")/"&gt;" + P("&")/"&amp;" + P('"')/"&quot;" + P(1))^0)
+lpeg.patterns.texescape = Cs((C(S("#$%\\{}"))/"\\%1" + P(1))^0)
+
+add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]])
+add(formatters,"tex",[[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]])
 
-setmetatable(formatters, { __index = make, __call = use })
+-- add(formatters,"xy",[[ "(" .. %s .. "," .. %s .. ")" ]])
+-- print(formatters["coordinates %2!xy! or (%s,%s)"](1,2,3,4,5,6))
+
+-- local myformatter = utilities.strings.formatters.new()
+--
+-- utilities.strings.formatters.add(myformatter,"upper", [[string.upper(%s)]]) -- maybe handy
+-- utilities.strings.formatters.add(myformatter,"csname",[["\\"..%s]])         -- less usefull
+--
+-- print("\n>>>",myformatter["Is this %!upper! handy %!csname! or %H not %!xml!?"]("really","weird",1234,"a&b"))
 
 -- -- yes or no:
 --
@@ -559,8 +600,6 @@ setmetatable(formatters, { __index = make, __call = use })
 --
 -- setmetatable(formatteds, { __index = make, __call = use })
 
---
-
 -- print(formatters["hans %N and %N done"](123,"0123"))
 -- local test = formatters["1%%23%4w56%s78 %p %!xml! test and %!tex! more %s"]
 -- print(#string.dump(test))
@@ -569,16 +608,6 @@ setmetatable(formatters, { __index = make, __call = use })
 -- print(#string.dump(test))
 -- print(test("okay"))
 
-function string.makeformatter(str) -- redundant
-    return formatters[str]
-end
-
-function string.formatter(str,...) -- redundant
-    return formatters[str](...)
-end
-
-string.addformatter = addextension
-
 -- local p1 = "%s test %f done %p and %c and %V or %+t or %%"
 -- local p2 = "%s test %f done %s and %s and 0x%05X or %s or %%"
 --
diff --git a/tex/context/base/x-mathml.lua b/tex/context/base/x-mathml.lua
index 9565057d0..988758b14 100644
--- a/tex/context/base/x-mathml.lua
+++ b/tex/context/base/x-mathml.lua
@@ -465,10 +465,6 @@ function mathml.stripped(str)
     context(strip(str))
 end
 
-function characters.remapentity(chr,slot) -- Brrrrrr, this will be replaced!
-    context("{\\catcode%s=13\\xdef%s{\\string%s}}",slot,utfchar(slot),chr)
-end
-
 function mathml.mn(id,pattern)
     -- maybe at some point we need to interpret the number, but
     -- currently we assume an upright font
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 1013f5467..b4cc4e1b4 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 03/10/13 14:36:13
+-- merge date  : 03/11/13 00:17:48
 
 do -- begin closure to overcome local limits and interference