beta 2011.05.13 11:37

18 files changed, 264 insertions, 19 deletions

diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index f399b90a4..dfba13b4a 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -3435,8 +3435,10 @@ local unicode = unicode
 
 utf = utf or unicode.utf8
 
-local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
+local concat = table.concat
+local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub
 local char, byte, find, bytepairs, utfvalues, format = string.char, string.byte, string.find, string.bytepairs, string.utfvalues, string.format
+local type = type
 
 local utfsplitlines = string.utfsplitlines
 
@@ -3648,6 +3650,14 @@ function unicode.utfcodes(str)
     return concat(t,separator or " ")
 end
 
+function unicode.ustring(s)
+    return format("U+%05X",type(s) == "number" and s or utfbyte(s))
+end
+
+function unicode.xstring(s)
+    return format("0x%05X",type(s) == "number" and s or utfbyte(s))
+end
+
 
 local lpegmatch = lpeg.match
 local utftype = lpeg.patterns.utftype
@@ -3659,7 +3669,6 @@ end
 
 
 
-
 end -- of closure
 
 do -- create closure to overcome 200 locals limit
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index f399b90a4..dfba13b4a 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -3435,8 +3435,10 @@ local unicode = unicode
 
 utf = utf or unicode.utf8
 
-local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
+local concat = table.concat
+local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub
 local char, byte, find, bytepairs, utfvalues, format = string.char, string.byte, string.find, string.bytepairs, string.utfvalues, string.format
+local type = type
 
 local utfsplitlines = string.utfsplitlines
 
@@ -3648,6 +3650,14 @@ function unicode.utfcodes(str)
     return concat(t,separator or " ")
 end
 
+function unicode.ustring(s)
+    return format("U+%05X",type(s) == "number" and s or utfbyte(s))
+end
+
+function unicode.xstring(s)
+    return format("0x%05X",type(s) == "number" and s or utfbyte(s))
+end
+
 
 local lpegmatch = lpeg.match
 local utftype = lpeg.patterns.utftype
@@ -3659,7 +3669,6 @@ end
 
 
 
-
 end -- of closure
 
 do -- create closure to overcome 200 locals limit
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index f399b90a4..dfba13b4a 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -3435,8 +3435,10 @@ local unicode = unicode
 
 utf = utf or unicode.utf8
 
-local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
+local concat = table.concat
+local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub
 local char, byte, find, bytepairs, utfvalues, format = string.char, string.byte, string.find, string.bytepairs, string.utfvalues, string.format
+local type = type
 
 local utfsplitlines = string.utfsplitlines
 
@@ -3648,6 +3650,14 @@ function unicode.utfcodes(str)
     return concat(t,separator or " ")
 end
 
+function unicode.ustring(s)
+    return format("U+%05X",type(s) == "number" and s or utfbyte(s))
+end
+
+function unicode.xstring(s)
+    return format("0x%05X",type(s) == "number" and s or utfbyte(s))
+end
+
 
 local lpegmatch = lpeg.match
 local utftype = lpeg.patterns.utftype
@@ -3659,7 +3669,6 @@ end
 
 
 
-
 end -- of closure
 
 do -- create closure to overcome 200 locals limit
diff --git a/tex/context/base/char-ini.lua b/tex/context/base/char-ini.lua
index 059332d51..2244048c3 100644
--- a/tex/context/base/char-ini.lua
+++ b/tex/context/base/char-ini.lua
@@ -12,6 +12,7 @@ local tex = tex
 local utf = unicode.utf8
 
 local utfchar, utfbyte, utfvalues = utf.char, utf.byte, string.utfvalues
+local ustring = unicode.ustring
 local concat, unpack = table.concat, table.unpack
 local next, tonumber, type, rawget, rawset = next, tonumber, type, rawget, rawset
 local texsprint, texprint = tex.sprint, tex.print
@@ -939,3 +940,38 @@ end
 --~   direction="l",
 --~   unicodeslot=0x10FFFD,
 --~  },
+
+if not characters.superscripts then
+
+    local superscripts = allocate()   characters.superscripts = superscripts
+    local subscripts   = allocate()   characters.subscripts   = subscripts
+
+    -- skipping U+02120 (service mark) U+02122 (trademark)
+
+    for k, v in next, data do
+        local specials = v.specials
+        if specials then
+            local what = specials[1]
+            if what == "super" then
+                if #specials == 2 then
+                    superscripts[k] = specials[2]
+                else
+                    report_defining("ignoring superscript %s %s: %s",ustring(k),utfchar(k),v.description)
+                end
+            elseif what == "sub" then
+                if #specials == 2 then
+                    subscripts[k] = specials[2]
+                else
+                    report_defining("ignoring subscript %s %s: %s",ustring(k),utfchar(k),v.description)
+                end
+            end
+        end
+    end
+
+ -- print(table.serialize(superscripts, "superscripts", { hexify = true }))
+ -- print(table.serialize(subscripts,   "subscripts",   { hexify = true }))
+
+    storage.register("characters/superscripts", superscripts, "characters.superscripts")
+    storage.register("characters/subscripts",   subscripts,   "characters.subscripts")
+
+end
diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii
index e73e00f0b..1b895fb80 100644
--- a/tex/context/base/cont-new.mkii
+++ b/tex/context/base/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2011.05.12 15:19}
+\newcontextversion{2011.05.13 11:37}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index aefb7d43a..0661ab111 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2011.05.12 15:19}
+\newcontextversion{2011.05.13 11:37}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii
index 4c33eba16..567dc04f8 100644
--- a/tex/context/base/context.mkii
+++ b/tex/context/base/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2011.05.12 15:19}
+\edef\contextversion{2011.05.13 11:37}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 17dde0702..c097015a5 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2011.05.12 15:19}
+\edef\contextversion{2011.05.13 11:37}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/l-unicode.lua b/tex/context/base/l-unicode.lua
index b64969162..2fa50282f 100644
--- a/tex/context/base/l-unicode.lua
+++ b/tex/context/base/l-unicode.lua
@@ -51,8 +51,10 @@ local unicode = unicode
 
 utf = utf or unicode.utf8
 
-local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
+local concat = table.concat
+local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub
 local char, byte, find, bytepairs, utfvalues, format = string.char, string.byte, string.find, string.bytepairs, string.utfvalues, string.format
+local type = type
 
 local utfsplitlines = string.utfsplitlines
 
@@ -367,6 +369,14 @@ function unicode.utfcodes(str)
     return concat(t,separator or " ")
 end
 
+function unicode.ustring(s)
+    return format("U+%05X",type(s) == "number" and s or utfbyte(s))
+end
+
+function unicode.xstring(s)
+    return format("0x%05X",type(s) == "number" and s or utfbyte(s))
+end
+
 --~ print(unicode.utfcodes(str))
 
 local lpegmatch = lpeg.match
@@ -398,4 +408,3 @@ end
 --~     end
 --~ end
 --~ print(os.clock()-t,collectgarbage("count")*1024-u)
-
diff --git a/tex/context/base/math-noa.lua b/tex/context/base/math-noa.lua
index 57361fe04..c84d72906 100644
--- a/tex/context/base/math-noa.lua
+++ b/tex/context/base/math-noa.lua
@@ -18,6 +18,7 @@ if not modules then modules = { } end modules ['math-noa'] = {
 local utf = unicode.utf8
 
 local format, rep  = string.format, string.rep
+local concat = table.concat
 local utfchar, utfbyte = utf.char, utf.byte
 
 local fonts, nodes, node, mathematics = fonts, nodes, node, mathematics
@@ -26,12 +27,14 @@ local otf                 = fonts.handlers.otf
 local otffeatures         = fonts.constructors.newfeatures("otf")
 local registerotffeature  = otffeatures.register
 
-local trace_remapping     = false  trackers.register("math.remapping",  function(v) trace_remapping  = v end)
-local trace_processing    = false  trackers.register("math.processing", function(v) trace_processing = v end)
-local trace_analyzing     = false  trackers.register("math.analyzing",  function(v) trace_analyzing  = v end)
+local trace_remapping     = false  trackers.register("math.remapping",   function(v) trace_remapping   = v end)
+local trace_processing    = false  trackers.register("math.processing",  function(v) trace_processing  = v end)
+local trace_analyzing     = false  trackers.register("math.analyzing",   function(v) trace_analyzing   = v end)
+local trace_normalizing   = false  trackers.register("math.normalizing", function(v) trace_normalizing = v end)
 
 local report_processing   = logs.reporter("mathematics","processing")
 local report_remapping    = logs.reporter("mathematics","remapping")
+local report_normalizing  = logs.reporter("mathematics","normalizing")
 
 local set_attribute       = node.set_attribute
 local has_attribute       = node.has_attribute
@@ -396,6 +399,111 @@ function noads.handlers.collapse(head,style,penalties)
     return true
 end
 
+-- normalize scripts
+
+local unscript = { }  noads.processors.unscript = unscript
+
+local superscripts = characters.superscripts
+local subscripts   = characters.subscripts
+
+local replaced = { }
+
+local function replace(pointer)
+    local next = pointer.next
+    local start_super, stop_super, start_sub, stop_sub
+    local mode = "unset"
+    while next and next.id == math_noad do
+        local nextnucleus = next.nucleus
+        if nextnucleus and not next.sub and not next.sup then
+            local char = nextnucleus.char
+            local s = superscripts[char]
+            if s then
+                if not start_super then
+                    start_super = next
+                    mode = "super"
+                elseif mode == "sub" then
+                    break
+                end
+                stop_super = next
+                next = next.next
+                nextnucleus.char = s
+                replaced[char] = (replaced[char] or 0) + 1
+                if trace_normalizing then
+                    report_normalizing("superscript: U+05X (%s) => U+05X (%s)",char,utfchar(char),s,utfchar(s))
+                end
+            else
+                local s = subscripts[char]
+                if s then
+                    if not start_sub then
+                        start_sub = next
+                        mode = "sub"
+                    elseif mode == "super" then
+                        break
+                    end
+                    stop_sub = next
+                    next = next.next
+                    nextnucleus.char = s
+                    replaced[char] = (replaced[char] or 0) + 1
+                    if trace_normalizing then
+                        report_normalizing("subscript: U+05X (%s) => U+05X (%s)",char,utfchar(char),s,utfchar(s))
+                    end
+                else
+                    break
+                end
+            end
+        else
+            break
+        end
+    end
+    if start_super then
+        if start_super == stop_super then
+            pointer.sup = start_super.nucleus
+        else
+            local list = node.new(math_sub) -- todo attr
+            list.head = start_super
+            pointer.sup = list
+        end
+        if mode == "super" then
+            pointer.next = stop_super.next
+        end
+        stop_super.next = nil
+    end
+    if start_sub then
+        if start_sub == stop_sub then
+            pointer.sub = start_sub.nucleus
+        else
+            local list = node.new(math_sub) -- todo attr
+            list.head = start_sub
+            pointer.sub = list
+        end
+        if mode == "sub" then
+            pointer.next = stop_sub.next
+        end
+        stop_sub.next = nil
+    end
+end
+
+   unscript[math_noad]     = replace
+-- unscript[math_accent]   = replace
+-- unscript[math_radical]  = replace
+-- unscript[math_fraction] = replace
+
+function handlers.unscript(head,style,penalties)
+    process(head,unscript)
+    return true
+end
+
+statistics.register("math script replacements", function()
+    if next(replaced) then
+        local n, t = 0, { }
+        for k, v in table.sortedpairs(replaced) do
+            n = n + v
+            t[#t+1] = format("U+%05X:%s",k,utfchar(k))
+        end
+        return format("%s (n=%s)",concat(t," "),n)
+    end
+end)
+
 -- math alternates: (in xits lgf: $ABC$ $\cal ABC$ $\mathalternate{cal}\cal ABC$)
 
 local function initializemathalternates(tfmdata)
diff --git a/tex/context/base/node-rul.mkiv b/tex/context/base/node-rul.mkiv
index 54d0c22f1..99ff3d36b 100644
--- a/tex/context/base/node-rul.mkiv
+++ b/tex/context/base/node-rul.mkiv
@@ -236,9 +236,9 @@
    \glet\dodoshifted\dodoshiftedindeed
    \dodoshifted}
 
-\def\dostartisolation{\char0 }
-\def\dostopisolation {\char0 }
-\def\doisolator      {\char0 }
+\def\dostartisolation{\zwj} % {\char0 } % some fonts have a char0 rendering
+\def\dostopisolation {\zwj} % {\char0 }
+\def\doisolator      {\zwj} % {\char0 }
 
 \def\doisolatedgroupedalign#1#2%
   {\groupedcommand
diff --git a/tex/context/base/node-ser.lua b/tex/context/base/node-ser.lua
index a58b7d745..aa4615626 100644
--- a/tex/context/base/node-ser.lua
+++ b/tex/context/base/node-ser.lua
@@ -151,6 +151,8 @@ end
 
 -- not ok yet; this will become a module
 
+-- todo: adapt to nodecodes etc
+
 local function serialize(root,name,handle,depth,m)
     handle = handle or print
     if depth then
diff --git a/tex/context/base/s-fnt-32.mkiv b/tex/context/base/s-fnt-32.mkiv
new file mode 100644
index 000000000..ea55e9e74
--- /dev/null
+++ b/tex/context/base/s-fnt-32.mkiv
@@ -0,0 +1,60 @@
+%D \module
+%D   [      file=s-fnt-31,
+%D        version=2011.05.12,
+%D          title=\CONTEXT\ Style File,
+%D       subtitle=Unicode Math Scripts,
+%D         author=Hans Hagen,
+%D           date=\currentdate,
+%D      copyright={PRAGMA / Hans Hagen \& Ton Otten}]
+%C
+%C This module is part of the \CONTEXT\ macro||package and is
+%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
+%C details.
+
+\startluacode
+    function fonts.tracers.mathscripts()
+        context.starttabulate { "|cT|c|cT|c|c|l|" }
+        for k, v in table.sortedpairs(table.merged(characters.superscripts,characters.subscripts)) do
+            local ck = utf.char(k)
+            local cv = utf.char(v)
+            local ss = characters.superscripts[k] and "^" or "_"
+            context.NC() context("U+%05X",k)
+            context.NC() context(ck)
+            context.NC() context("U+%05X",v)
+            context.NC() context(cv)
+            context.NC() context("$x%s = x%s%s$",ck,ss,cv)
+            context.NC() context(string.lower(characters.data[k].description))
+            context.NC() context.NR()
+        end
+        context.stoptabulate()
+    end
+\stopluacode
+
+\doifnotmode{demo}{\endinput}
+
+\setupbodyfont[dejavu,10pt]
+
+% \enabletrackers[math.normalizing]
+% \setupbackend[export=yes]
+
+\setuplayout
+  [backspace=1cm,
+   topspace=1cm,
+   footer=1cm,
+   header=0cm,
+   height=middle,
+   width=middle]
+
+\starttext
+
+\startbuffer
+$e=mc²$ ${}²$ $²$ $x²ᶞ$ $x⁽²⁺²⁼²⁺²⁾$ $x²⁺²⁼²⁺²$ $x₅²$ $x²₅²$
+\stopbuffer
+
+\typebuffer \getbuffer
+
+\startluacode
+    fonts.tracers.mathscripts()
+\stopluacode
+
+\stoptext
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index ad9e92d69..113c2c68f 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 234a294af..1f7e629ca 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/task-ini.lua b/tex/context/base/task-ini.lua
index 4afdaa034..435c85286 100644
--- a/tex/context/base/task-ini.lua
+++ b/tex/context/base/task-ini.lua
@@ -58,6 +58,7 @@ appendaction("shipouts",     "finishers",   "attributes.negatives.handler")
 appendaction("shipouts",     "finishers",   "attributes.effects.handler")                        -- disabled
 appendaction("shipouts",     "finishers",   "attributes.viewerlayers.handler")                   -- disabled
 
+appendaction("math",         "normalizers", "noads.handlers.unscript", nil, "nohead")            -- always on (maybe disabled)
 appendaction("math",         "normalizers", "noads.handlers.relocate", nil, "nohead")            -- always on
 appendaction("math",         "normalizers", "noads.handlers.collapse", nil, "nohead")            -- always on
 appendaction("math",         "normalizers", "noads.handlers.resize",   nil, "nohead")            -- always on
diff --git a/tex/context/base/type-otf.mkiv b/tex/context/base/type-otf.mkiv
index 9bb2d3e47..71f8cd1e3 100644
--- a/tex/context/base/type-otf.mkiv
+++ b/tex/context/base/type-otf.mkiv
@@ -15,6 +15,8 @@
 %D consult its database. However, here we use the names that we used
 %D in good old \TEX, and these may differ a bit. Here we also see
 %D some oldstyle definitions which normally are done with features.
+%D
+%D The ComputerModern* names will become obsolete (all LM now).
 
 % \starttypescriptcollection[myfonts]
 %
diff --git a/tex/generic/context/luatex-fonts-merged.lua b/tex/generic/context/luatex-fonts-merged.lua
index 61f19becf..37de7d03d 100644
--- a/tex/generic/context/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 05/12/11 15:19:18
+-- merge date  : 05/13/11 11:37:06
 
 do -- begin closure to overcome local limits and interference