beta 2012.11.07 23:16

21 files changed, 378 insertions, 220 deletions

diff --git a/tex/context/base/char-ini.lua b/tex/context/base/char-ini.lua
index 778035ff4..6e14982c7 100644
--- a/tex/context/base/char-ini.lua
+++ b/tex/context/base/char-ini.lua
@@ -393,6 +393,8 @@ local is_mark = allocate ( tohash {
     "mn", "ms",
 } )
 
+-- to be redone: store checked characters
+
 characters.is_character = is_character
 characters.is_letter    = is_letter
 characters.is_command   = is_command
@@ -402,7 +404,7 @@ characters.is_mark      = is_mark
 local mt = { -- yes or no ?
     __index = function(t,k)
         if type(k) == "number" then
-            local c = characters.data[k].category
+            local c = data[k].category
             return c and rawget(t,c)
         else
             -- avoid auto conversion in data.characters lookups
diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii
index e2088670d..03cebce07 100644
--- a/tex/context/base/cont-new.mkii
+++ b/tex/context/base/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.11.06 09:56}
+\newcontextversion{2012.11.07 23:16}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 6ada0a7b0..e1f85e6d8 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.11.06 09:56}
+\newcontextversion{2012.11.07 23:16}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 4959623fb..52c1333f5 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png
index ce1935570..32b440ce1 100644
--- a/tex/context/base/context-version.png
+++ b/tex/context/base/context-version.png
diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii
index 0fc178651..784ae89f4 100644
--- a/tex/context/base/context.mkii
+++ b/tex/context/base/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.11.06 09:56}
+\edef\contextversion{2012.11.07 23:16}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 8851f7369..f11caecba 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.11.06 09:56}
+\edef\contextversion{2012.11.07 23:16}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/data-fil.lua b/tex/context/base/data-fil.lua
index 6eb29ac32..2030e5423 100644
--- a/tex/context/base/data-fil.lua
+++ b/tex/context/base/data-fil.lua
@@ -96,7 +96,7 @@ function loaders.file(specification,filetype)
             if trace_locating then
                 report_files("file loader, '%s' loaded",filename)
             end
-            local s = f:read("*a")
+            local s = f:read("*a") -- io.readall(f) is faster but we never have large files here
             if checkgarbage then
                 checkgarbage(#s)
             end
diff --git a/tex/context/base/data-tex.lua b/tex/context/base/data-tex.lua
index 2105f29f4..639bb0ac9 100644
--- a/tex/context/base/data-tex.lua
+++ b/tex/context/base/data-tex.lua
@@ -66,7 +66,7 @@ function helpers.textopener(tag,filename,filehandle,coding)
     elseif t_filehandle == "table" then
         lines = filehandle
     else
-        lines = filehandle:read("*a")
+        lines = filehandle:read("*a") -- io.readall(filehandle) ... but never that large files anyway
         filehandle:close()
     end
     if type(lines) == "string" then
diff --git a/tex/context/base/font-con.lua b/tex/context/base/font-con.lua
index 6074bab18..77d7d3e5c 100644
--- a/tex/context/base/font-con.lua
+++ b/tex/context/base/font-con.lua
@@ -541,13 +541,16 @@ function constructors.scale(tfmdata,specification)
     -- some context specific trickery (this will move to a plugin)
     --
     if hasmath then
-        if properties.mathitalics then
-            italickey = "italic_correction"
-            if trace_defining then
-                report_defining("math italics disabled for: name '%s', fullname: '%s', filename: '%s'",
-                    name or "noname",fullname or "nofullname",filename or "nofilename")
-            end
-        end
+     -- the latest luatex can deal with it itself so we now disable this
+     -- mechanism here
+     --
+     -- if properties.mathitalics then
+     --     italickey = "italic_correction"
+     --     if trace_defining then
+     --         report_defining("math italics disabled for: name '%s', fullname: '%s', filename: '%s'",
+     --             name or "noname",fullname or "nofullname",filename or "nofilename")
+     --     end
+     -- end
         autoitalicamount = false -- new
     else
         if properties.textitalics then
diff --git a/tex/context/base/l-io.lua b/tex/context/base/l-io.lua
index 657b755b8..3456001b7 100644
--- a/tex/context/base/l-io.lua
+++ b/tex/context/base/l-io.lua
@@ -9,6 +9,7 @@ if not modules then modules = { } end modules ['l-io'] = {
 local io = io
 local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format
 local concat = table.concat
+local floor = math.floor
 local type = type
 
 if string.find(os.getenv("PATH"),";") then
@@ -17,10 +18,49 @@ else
     io.fileseparator, io.pathseparator = "/" , ":"
 end
 
+local function readall(f)
+    return f:read("*all")
+end
+
+-- The next one is upto 50% faster on large files and less memory consumption due
+-- to less intermediate large allocations. This phenomena was discussed on the
+-- luatex dev list.
+
+local function readall(f)
+    local size = f:seek("end")
+    if size == 0 then
+        return ""
+    elseif size < 1024*1024 then
+        f:seek("set",0)
+        return f:read('*all')
+    else
+        local done = f:seek("set",0)
+        if size < 1024*1024 then
+            step = 1024 * 1024
+        elseif size > 16*1024*1024 then
+            step = 16*1024*1024
+        else
+            step = floor(size/(1024*1024)) * 1024 * 1024 / 8
+        end
+        local data = { }
+        while true do
+            local r = f:read(step)
+            if not r then
+                return concat(data)
+            else
+                data[#data+1] = r
+            end
+        end
+    end
+end
+
+io.readall = readall
+
 function io.loaddata(filename,textmode) -- return nil if empty
     local f = io.open(filename,(textmode and 'r') or 'rb')
     if f then
-        local data = f:read('*all')
+     -- local data = f:read('*all')
+        local data = readall(f)
         f:close()
         if #data > 0 then
             return data
diff --git a/tex/context/base/lxml-tab.lua b/tex/context/base/lxml-tab.lua
index 6b23953cc..21c7561e8 100644
--- a/tex/context/base/lxml-tab.lua
+++ b/tex/context/base/lxml-tab.lua
@@ -824,13 +824,13 @@ function xml.load(filename,settings)
     local data = ""
     if type(filename) == "string" then
      -- local data = io.loaddata(filename) - -todo: check type in io.loaddata
-        local f = io.open(filename,'r')
+        local f = io.open(filename,'r') -- why not 'rb'
         if f then
-            data = f:read("*all")
+            data = f:read("*all") -- io.readall(f) ... only makes sense for large files
             f:close()
         end
     elseif filename then -- filehandle
-        data = filename:read("*all")
+        data = filename:read("*all") -- io.readall(f) ... only makes sense for large files
     end
     if settings then
         settings.currentresource = filename
diff --git a/tex/context/base/math-fbk.lua b/tex/context/base/math-fbk.lua
index b2b7ed5f0..826f50f7c 100644
--- a/tex/context/base/math-fbk.lua
+++ b/tex/context/base/math-fbk.lua
@@ -181,6 +181,37 @@ virtualcharacters[0x208B] = function(data)
     return raised(data,true)
 end
 
+-- local function repeated(data,char,n,fraction)
+--     local character = data.characters[char]
+--     if character then
+--         local width = character.width
+--         local delta = width - character.italic -- width * fraction
+--         local c = { "char", char }
+--         local r = { "right", right }
+--         local commands = { }
+--         for i=1,n-1 do
+--             width = width + delta
+--             commands[#commands+1] = c
+--             commands[#commands+1] = -delta
+--         end
+--         commands[#commands+1] = c
+--         return {
+--             width    = width,
+--             height   = character.height,
+--             depth    = character.depth,
+--             commands = commands,
+--         }
+--     end
+-- end
+
+-- virtualcharacters[0x222C] = function(data)
+--     return repeated(data,0x222B,2,1/8)
+-- end
+
+-- virtualcharacters[0x222D] = function(data)
+--     return repeated(data,0x222B,3,1/8)
+-- end
+
 local addextra = mathematics.extras.add
 
 addextra(0xFE350, {
diff --git a/tex/context/base/math-ini.mkiv b/tex/context/base/math-ini.mkiv
index 42306c6d5..adf63004c 100644
--- a/tex/context/base/math-ini.mkiv
+++ b/tex/context/base/math-ini.mkiv
@@ -851,7 +851,7 @@
 \to \everymathematics
 
 \setupmathematics
-  [\s!italics=4]
+  [\s!italics=3] % for the moment only this one makes sense .. still experimental
 
 % looks nicer but can generate bogus csnames
 %
diff --git a/tex/context/base/math-int.mkiv b/tex/context/base/math-int.mkiv
index f03df3fdc..6b480961b 100644
--- a/tex/context/base/math-int.mkiv
+++ b/tex/context/base/math-int.mkiv
@@ -19,12 +19,10 @@
 %D   $\int    _a^b f(x)   dx  $ and also
 %D   $\iint   _a^b f(x,y) dxdy$,
 %D   $\iiint  _a^b f(x,y) dxdy$,
-%D   $\iiiint _a^b f(x)   dx  $.
 %D   \startformula
 %D     \int    _a^b f(x) dx \quad
 %D     \iint   _a^b f(x) dx \quad
 %D     \iiint  _a^b f(x) dx \quad
-%D     \iiiint _a^b f(x) dx \quad
 %D   \stopformula
 %D \stopbuffer
 %D
@@ -76,68 +74,68 @@
 %D The following code is used for fallbacks and might become obsolete once
 %D we have enough \OPENTYPE\ math fonts.
 
-\def\math_repeated_integal_i
-  {\int}
-
-\def\math_repeated_integal_ii
-  {\math_repeated_integal_i
-   \math_repeated_integral_kern
-   \math_repeated_integal_i
-   \math_repeat_integral_finish
-   \intlimits}
-
-\def\math_repeated_integal_iii
-  {\math_repeated_integal_i
-   \math_repeated_integral_kern
-   \math_repeated_integal_ii}
-
-\def\math_repeated_integal_iiii
-  {\math_repeated_integal_i
-   \math_repeated_integral_kern
-   \math_repeated_integal_iii}
-
-\unexpanded\def\math_repeat_integral#1%
-  {\let\math_repeat_integral_finish\donothing
-   \iffontchar\textfont\zerocount#1\relax
-     \expandafter\math_repeat_integral_real
-   \else
-     \expandafter\math_repeat_integral_fake
-   \fi}
-
-\def\math_repeat_integral_fake#1#2%
-  {\let\math_repeat_integral_fake_symbol#2%
-   \futurelet\next\math_repeat_integral_fake_indeed}
-
-\def\math_repeat_integral_real#1#2%
-  {#1}
-
-\definemathcommand [iint]   {\math_repeat_integral{"0222C}\normalint  \math_repeated_integal_ii  } % double
-\definemathcommand [iiint]  {\math_repeat_integral{"0222D}\normaliint \math_repeated_integal_iii } % tripple
-\definemathcommand [iiiint] {\math_repeat_integral{"FFFFF}\normaliiint\math_repeated_integal_iiii} % quadruple
-
-\def\math_repeated_integral_kern
-  {\mkern-6mu\mathchoice{\mkern-3mu}{}{}{}}
-
-\def\math_repeat_integral_fake_indeed
-   {\ifx\next\limits
-      \math_repeated_integral_correction
-    \else\ifx\next\displaylimits
-      \math_repeated_integral_correction
-    \else\ifx\next\nolimits
-      % nothing
-    \else\ifcase\mathintlimitmode
-      % nothing
-    \else
-      \math_repeated_integral_correction
-    \fi\fi\fi\fi
-    \math_repeat_integral_fake_symbol}
-
-\def\math_repeated_integral_correction
-  {\mkern-7mu\mathchoice{\mkern-2mu}{}{}{}%
-   \mathop\bgroup\mkern7mu\mathchoice{\mkern2mu}{}{}{}\let\math_repeat_integral_finish\egroup}
-
-%D If the \type{\limits} option is used after \type {\iint}, use \type
-%D {\mathop} and fudge the left hand space a bit to make the subscript
-%D visually centered.
+% \def\math_repeated_integal_i
+%   {\int}
+
+% \def\math_repeated_integal_ii
+%   {\math_repeated_integal_i
+%    \math_repeated_integral_kern
+%    \math_repeated_integal_i
+%    \math_repeat_integral_finish
+%    \intlimits}
+
+% \def\math_repeated_integal_iii
+%   {\math_repeated_integal_i
+%    \math_repeated_integral_kern
+%    \math_repeated_integal_ii}
+%
+% \def\math_repeated_integal_iiii
+%   {\math_repeated_integal_i
+%    \math_repeated_integral_kern
+%    \math_repeated_integal_iii}
+%
+% \unexpanded\def\math_repeat_integral#1%
+%   {\let\math_repeat_integral_finish\donothing
+%    \iffontchar\textfont\zerocount#1\relax
+%      \expandafter\math_repeat_integral_real
+%    \else
+%      \expandafter\math_repeat_integral_fake
+%    \fi}
+%
+% \def\math_repeat_integral_fake#1#2%
+%   {\let\math_repeat_integral_fake_symbol#2%
+%    \futurelet\next\math_repeat_integral_fake_indeed}
+%
+% \def\math_repeat_integral_real#1#2%
+%   {#1}
+%
+% \definemathcommand [iint]   {\math_repeat_integral{"0222C}\normalint  \math_repeated_integal_ii  } % double
+% \definemathcommand [iiint]  {\math_repeat_integral{"0222D}\normaliint \math_repeated_integal_iii } % tripple
+% \definemathcommand [iiiint] {\math_repeat_integral{"FFFFF}\normaliiint\math_repeated_integal_iiii} % quadruple
+%
+% \def\math_repeated_integral_kern
+%   {\mkern-6mu\mathchoice{\mkern-3mu}{}{}{}}
+%
+% \def\math_repeat_integral_fake_indeed
+%    {\ifx\next\limits
+%       \math_repeated_integral_correction
+%     \else\ifx\next\displaylimits
+%       \math_repeated_integral_correction
+%     \else\ifx\next\nolimits
+%       % nothing
+%     \else\ifcase\mathintlimitmode
+%       % nothing
+%     \else
+%       \math_repeated_integral_correction
+%     \fi\fi\fi\fi
+%     \math_repeat_integral_fake_symbol}
+%
+% \def\math_repeated_integral_correction
+%   {\mkern-7mu\mathchoice{\mkern-2mu}{}{}{}%
+%    \mathop\bgroup\mkern7mu\mathchoice{\mkern2mu}{}{}{}\let\math_repeat_integral_finish\egroup}
+%
+% %D If the \type{\limits} option is used after \type {\iint}, use \type
+% %D {\mathop} and fudge the left hand space a bit to make the subscript
+% %D visually centered.
 
 \protect \endinput
diff --git a/tex/context/base/math-noa.lua b/tex/context/base/math-noa.lua
index 5ac6d125d..8716ac726 100644
--- a/tex/context/base/math-noa.lua
+++ b/tex/context/base/math-noa.lua
@@ -61,15 +61,18 @@ local free_node           = node.free
 local new_node            = node.new -- todo: pool: math_noad math_sub
 
 local new_kern            = nodes.pool.kern
+local new_rule            = nodes.pool.rule
+local concat_nodes        = nodes.concat
 
-local topoints            = number.topoints
+local topoints            = number.points
 
 local fonthashes          = fonts.hashes
 local fontdata            = fonthashes.identifiers
 local fontcharacters      = fonthashes.characters
 local fontproperties      = fonthashes.properties
 local fontitalics         = fonthashes.italics
-local fontquads           = fonthashes.quads
+local fontemwidths        = fonthashes.emwidths
+local fontexheights       = fonthashes.exheights
 
 local variables           = interfaces.variables
 local texattribute        = tex.attribute
@@ -437,10 +440,20 @@ local mathpairs = characters.mathpairs
 mathpairs[0x2032] = { [0x2032] = 0x2033, [0x2033] = 0x2034 } -- (prime,prime) (prime,doubleprime)
 mathpairs[0x2033] = { [0x2032] = 0x2034 }                    -- (doubleprime,prime)
 
+mathpairs[0x222B] = { [0x222B] = 0x222C, [0x222C] = 0x222D }
+mathpairs[0x222C] = { [0x222B] = 0x222D }
+
+local validpair = {
+    [noad_rel]             = true,
+    [noad_ord]             = true,
+    [noad_opdisplaylimits] = true,
+    [noad_oplimits]        = true,
+    [noad_opnolimits]      = true,
+}
+
 local function collapsepair(pointer,what,n,parent) -- todo: switch to turn in on and off
     if parent then
-        local subtype = parent.subtype
-        if subtype == noad_rel or subtype == noad_ord then -- ord is new
+        if validpair[parent.subtype] then
             local current_nucleus = parent.nucleus
             if not parent.sub and not parent.sup and current_nucleus.id == math_char then
                 local current_char = current_nucleus.char
@@ -448,8 +461,7 @@ local function collapsepair(pointer,what,n,parent) -- todo: switch to turn in on
                 if mathpair then
                     local next_noad = parent.next
                     if next_noad and next_noad.id == math_noad then
-                        local next_subtype = next_noad.subtype
-                        if next_subtype == noad_rel or next_subtype == noad_ord then -- ord is new
+                        if validpair[next_noad.subtype] then
                             local next_nucleus = next_noad.nucleus
                             if next_nucleus.id == math_char then
                                 local next_char = next_nucleus.char
@@ -767,50 +779,67 @@ local a_mathitalics = attributes.private("mathitalics")
 local italics        = { }
 local default_factor = 1/20
 
-local function getcorrection(method,font,char)
+local function getcorrection(method,font,char) -- -- or character.italic -- (this one is for tex)
 
-    local correction
+    local correction, fromvisual
 
     if method == 1 then
         -- only font data triggered by fontitalics
         local italics = fontitalics[font]
         if italics then
             local character = fontcharacters[font][char]
-            correction = character and character.italic_correction -- or character.italic (this one is for tex)
+            if character then
+                correction = character.italic_correction
+                if correction and correction ~= 0 then
+                    return correction, false
+                end
+            end
         end
     elseif method == 2 then
         -- only font data triggered by fontdata
         local character = fontcharacters[font][char]
-        correction = character and character.italic_correction -- or character.italic (this one is for tex)
+        if character then
+            correction = character.italic_correction
+            if correction and correction ~= 0 then
+                return correction, false
+            end
+        end
     elseif method == 3 then
         -- only quad based by selective
         local visual = chardata[char].visual
         if not visual then
             -- skip
         elseif visual == "it" or visual == "bi" then
-            correction = fontproperties[font].mathitalic_defaultvalue or default_factor*fontquads[font]
+            correction = fontproperties[font].mathitalic_defaultvalue or default_factor*fontemwidths[font]
+            if correction and correction ~= 0 then
+                return correction, true
+            end
         end
     elseif method == 4 then
         -- combination of 1 and 3
         local italics = fontitalics[font]
         if italics then
             local character = fontcharacters[font][char]
-            correction = character and character.italic_correction -- or character.italic (this one is for tex)
+            if character then
+                correction = character.italic_correction
+                if correction and correction ~= 0 then
+                    return correction, false
+                end
+            end
         end
         if not correction then
             local visual = chardata[char].visual
             if not visual then
                 -- skip
             elseif visual == "it" or visual == "bi" then
-                correction = fontproperties[font].mathitalic_defaultvalue or default_factor*fontquads[font]
+                correction = fontproperties[font].mathitalic_defaultvalue or default_factor*fontemwidths[font]
+                if correction and correction ~= 0 then
+                    return correction, true
+                end
             end
         end
     end
 
-    if correction and correction ~= 0 then
-        return correction
-    end
-
 end
 
 local function insert_kern(current,kern)
@@ -822,20 +851,37 @@ local function insert_kern(current,kern)
     return sub
 end
 
--- noad_opdisplaylimits noad_oplimits noad_opnolimits
+local setcolor     = nodes.tracers.colors.set
+local italic_kern  = new_kern
+local c_positive_d = "trace:db"
+local c_negative_d = "trace:dr"
+
+trackers.register("math.italics", function(v)
+    if v then
+        italic_kern = function(k,font)
+            local ex = 1.5 * fontexheights[font]
+            if k > 0 then
+                return setcolor(new_rule(k,ex,ex),c_positive_d)
+            else
+                return concat_nodes {
+                    old_kern(k),
+                    setcolor(new_rule(-k,ex,ex),c_negative_d),
+                    old_kern(k),
+                }
+            end
+        end
+    else
+        italic_kern = new_kern
+    end
+end)
 
 italics[math_char] = function(pointer,what,n,parent)
     local method = has_attribute(pointer,a_mathitalics)
     if method and method > 0 then
         local char = pointer.char
         local font = font_of_family(pointer.fam) -- todo: table
-        local correction = getcorrection(method,font,char)
-
-        -- maybe also correction when next == nil
-        -- when sub/sup -> already done
-
+        local correction, visual = getcorrection(method,font,char)
         if correction then
-            -- maybe only +/- when subtype == opdisplaylimits
             local pid = parent.id
             local sub, sup
             if pid == math_noad then
@@ -843,30 +889,41 @@ italics[math_char] = function(pointer,what,n,parent)
                 sub = parent.sub
             end
             if sup or sub then
-                if sup then
-                    parent.sup = insert_kern(sup,new_kern(correction))
-                    if trace_italics then
-                        report_italics("method %s: adding %s italic correction before superscript after %s (0x%05X)",
-                            method,number.points(correction),utfchar(char),char)
+                local subtype = parent.subtype
+                if subtype == noad_oplimits then
+                    if sup then
+                        parent.sup = insert_kern(sup,italic_kern(correction,font))
+                        if trace_italics then
+                            report_italics("method %s: adding %s italic correction for upper limit of %s (0x%05X)",
+                                method,topoints(correction),utfchar(char),char)
+                        end
                     end
-                end
-                if sub then
-local correction = - correction
-                    parent.sub = insert_kern(sub,new_kern(correction))
-                    if trace_italics then
-                        report_italics("method %s: adding %s italic correction before subscript after %s (0x%05X)",
-                            method,number.points(correction),utfchar(char),char)
+                    if sub then
+                        local correction = - correction
+                        parent.sub = insert_kern(sub,italic_kern(correction,font))
+                        if trace_italics then
+                            report_italics("method %s: adding %s italic correction for lower limit of %s (0x%05X)",
+                                method,topoints(correction),utfchar(char),char)
+                        end
+                    end
+                else
+                    if sup then
+                        parent.sup = insert_kern(sup,italic_kern(correction,font))
+                        if trace_italics then
+                            report_italics("method %s: adding %s italic correction before superscript after %s (0x%05X)",
+                                method,topoints(correction),utfchar(char),char)
+                        end
                     end
                 end
             else
                 local next_noad = parent.next
                 if not next_noad then
-                    if true then -- this might become an option
+                    if n== 1 then -- only at the outer level .. will become an option (always,endonly,none)
                         if trace_italics then
                             report_italics("method %s: adding %s italic correction between %s (0x%05X) and end math",
-                            method,number.points(correction),utfchar(char),char)
+                            method,topoints(correction),utfchar(char),char)
                         end
-                        insert_node_after(parent,parent,new_kern(correction))
+                        insert_node_after(parent,parent,italic_kern(correction,font))
                     end
                 elseif next_noad.id == math_noad then
                     local next_subtype = next_noad.subtype
@@ -874,12 +931,31 @@ local correction = - correction
                         local next_nucleus = next_noad.nucleus
                         if next_nucleus.id == math_char then
                             local next_char = next_nucleus.char
-                            if not chardata[next_char].italic then -- or category
-                                if trace_italics then
-                                    report_italics("method %s: adding %s italic correction between %s (0x%05X) and %s (0x%05X)",
-                                        method,number.points(correction),utfchar(char),char,utfchar(next_char),next_char)
+                            local next_data = chardata[next_char]
+                            local visual = next_data.visual
+                            if visual == "it" or visual == "bi" then
+                             -- if trace_italics then
+                             --     report_italics("method %s: skipping %s italic correction between italic %s (0x%05X) and italic %s (0x%05X)",
+                             --         method,topoints(correction),utfchar(char),char,utfchar(next_char),next_char)
+                             -- end
+                            else
+                                local category = next_data.category
+                                if category == "nd" or category == "ll" or category == "lu" then
+                                    if trace_italics then
+                                        report_italics("method %s: adding %s italic correction between italic %s (0x%05X) and non italic %s (0x%05X)",
+                                            method,topoints(correction),utfchar(char),char,utfchar(next_char),next_char)
+                                    end
+                                    insert_node_after(parent,parent,italic_kern(correction,font))
+                             -- elseif next_data.height > (fontexheights[font]/2) then
+                             --     if trace_italics then
+                             --         report_italics("method %s: adding %s italic correction between %s (0x%05X) and ascending %s (0x%05X)",
+                             --             method,topoints(correction),utfchar(char),char,utfchar(next_char),next_char)
+                             --     end
+                             --     insert_node_after(parent,parent,italic_kern(correction,font))
+                             -- elseif trace_italics then
+                             --  -- report_italics("method %s: skipping %s italic correction between %s (0x%05X) and %s (0x%05X)",
+                             --  --     method,topoints(correction),utfchar(char),char,utfchar(next_char),next_char)
                                 end
-                                insert_node_after(parent,parent,new_kern(correction))
                             end
                         end
                     end
@@ -889,28 +965,7 @@ local correction = - correction
     end
 end
 
--- italics[math_noad] = function(pointer,what,n,parent)
---     local nucleus = pointer.nucleus
---     if nucleus.id == math_char then
---         local method = has_attribute(pointer,a_mathitalics)
---         if method and method > 0 then
---             local char = nucleus.char
---             local font = font_of_family(nucleus.fam) -- todo: table
---             local correction = getcorrection(method,font,char)
---             if correction then
---                 if trace_italics then
---                     report_italics("method %s: adding %s italic correction between %s (0x%05X) and script",
---                         method,number.points(correction),utfchar(char),char)
---                 end
---                 insert_node_after(nucleus,nucleus,new_kern(correction))
---             end
---         end
---     end
--- end
-
 function handlers.italics(head,style,penalties)
--- nodes.showsimplelist(head)
--- inspect(nodes.totable(head))
     processnoads(head,italics,"italics")
     return true
 end
@@ -925,66 +980,6 @@ enable = function()
     enable = false
 end
 
--- -- -- -- -- -- -- --
--- -- -- -- -- -- -- --
-
--- -- nice but not okay with multiple scripts (we need more clever hlist-shift checking then
---
--- local function processitalics(head,previous,previousmethod)
---     local current = head
---     while current do
---         local id = current.id
---         if id == glyph_code then
---             local method = has_attribute(current,a_mathitalics)
---             if method and method > 0 then -- keep method of previous
---                 if previous then
---                     local previousfont = previous.font
---                     local previouschar = previous.char
---                     local currentchar  = current.char
---                     local correction = getcorrection(previousmethod,previousfont,previouschar)
---                     if correction then
---                         if trace_italics then
---                             report_italics("correction %s between U+%05X and U+%05X using method",topoints(correction),previouschar,currentchar,previousmethod)
---                         end
---                         insert_node_after(previous,previous,new_kern(correction))
---                     else
---                         if trace_italics then
---                             report_italics("no correction between U+%05X and U+%05X",previouschar,currentchar)
---                         end
---                     end
---                 end
---                 previous, previousmethod = current, method
---             end
---         elseif id == hlist_code then
---             previous, previousmethod = processitalics(current.list,previous,previousmethod)
---         elseif id == vlist_code then
---             previous, previousmethod = processitalics(current.list,previous,previousmethod)
---         else
---             previous, previousmethod = nil
---         end
---         current = current.next
---     end
---     return previous, previousmethod
--- end
---
--- function handlers.italics(head,style,penalties)
---     processitalics(head)
---     return true
--- end
---
--- local enable
---
--- enable = function()
---     tasks.enableaction("math", "noads.handlers.italics")
---     if trace_italics then
---         report_italics("enabling math italics")
---     end
---     enable = false
--- end
-
--- -- -- -- -- -- -- --
--- -- -- -- -- -- -- --
-
 -- best do this only on math mode (less overhead)
 
 function mathematics.setitalics(n)
diff --git a/tex/context/base/math-vfu.lua b/tex/context/base/math-vfu.lua
index 9b86ddbca..5b41e2fb5 100644
--- a/tex/context/base/math-vfu.lua
+++ b/tex/context/base/math-vfu.lua
@@ -341,6 +341,37 @@ local function stack(main,characters,id,size,unicode,u1,d12,u2)
     end
 end
 
+local function repeated(main,characters,id,size,unicode,u,n,private,fraction) -- math-fbk.lua
+    local c = characters[u]
+    if c then
+        local width  = c.width
+        local italic = fraction*width -- c.italic or 0 -- larger ones have funny italics
+        local tc = { "slot", id, u }
+        local tr = { "right", -italic } -- see hack elsewhere
+        local commands = { }
+        for i=1,n-1 do
+            commands[#commands+1] = tc
+            commands[#commands+1] = tr
+        end
+        commands[#commands+1] = tc
+--         inspect(c)
+--         inspect(commands)
+        local next = c.next
+        if next then
+            repeated(main,characters,id,size,private,next,n,private+1,fraction)
+            next = private
+        end
+        characters[unicode] = {
+            width    = width + (n-1)*(width-italic),
+            height   = c.height,
+            depth    = c.depth,
+            italic   = italic,
+            commands = commands,
+            next     = next,
+        }
+    end
+end
+
 function vfmath.addmissing(main,id,size)
     local characters = main.characters
     local shared = main.shared
@@ -389,6 +420,9 @@ function vfmath.addmissing(main,id,size)
     jointhree(main,characters,id,size,0x27FC,0xFE321,0,0x0002D,joinrelfactor,0x02192) -- \mapstochar\relbar\joinrel\rightarrow
     jointwo  (main,characters,id,size,0x2254,0x03A,0,0x03D)                           -- := (≔)
 
+    repeated(main,characters,id,size,0x222C,0x222B,2,0xFF800,1/3)
+    repeated(main,characters,id,size,0x222D,0x222B,3,0xFF810,1/3)
+
  -- raise    (main,characters,id,size,0x02032,0xFE325,1) -- prime
  -- raise    (main,characters,id,size,0x02033,0xFE325,2) -- double prime
  -- raise    (main,characters,id,size,0x02034,0xFE325,3) -- triple prime
@@ -638,10 +672,9 @@ function vfmath.define(specification,set,goodies)
                             local width = fci.width
                             local italic = fci.italic
                             if italic and italic > 0 then
--- report_virtual("unicode char %s (U+%05X) in font %s has italic correction %s",utfchar(unicode),unicode,fs.properties.name or "unknown",italic)
                                     -- int_a^b
                                 if isextension then
---                                     width = width + italic
+width = width + italic -- for obscure reasons the integral as a width + italic correction
                                 end
                             end
                             if kerns then
@@ -660,6 +693,7 @@ function vfmath.define(specification,set,goodies)
                                     height   = fci.height,
                                     depth    = fci.depth,
                                     italic   = italic,
+-- italic_correction   = italic,
                                     kerns    = krn,
                                     commands = ref,
                                 }
@@ -676,6 +710,7 @@ function vfmath.define(specification,set,goodies)
                                     height   = fci.height,
                                     depth    = fci.depth,
                                     italic   = italic,
+-- italic_correction   = italic,
                                     commands = ref,
                                 }
                             end
@@ -700,6 +735,7 @@ function vfmath.define(specification,set,goodies)
                                     height   = fci.height,
                                     depth    = fci.depth,
                                     italic   = italic,
+-- italic_correction   = italic,
                                     commands = ref,
                                 }
                                 local n = fci.next
@@ -765,6 +801,7 @@ function vfmath.define(specification,set,goodies)
                                             height   = fci.height,
                                             depth    = fci.depth,
                                             italic   = fci.italic,
+-- italic_correction   = italic,
                                             commands = ref,
                                             kerns    = krn,
                                             next     = offset + index,
@@ -775,6 +812,7 @@ function vfmath.define(specification,set,goodies)
                                             height   = fci.height,
                                             depth    = fci.depth,
                                             italic   = fci.italic,
+-- italic_correction   = italic,
                                             commands = ref,
                                             next     = offset + index,
                                         }
diff --git a/tex/context/base/node-tra.lua b/tex/context/base/node-tra.lua
index 6b5f49964..7d5630eaa 100644
--- a/tex/context/base/node-tra.lua
+++ b/tex/context/base/node-tra.lua
@@ -730,9 +730,11 @@ function colors.set(n,c,s)
         end
         set_attribute(n,a_color,mc)
     end
+    return n
 end
 
 function colors.setlist(n,c,s)
+    local f = n
     while n do
         local mc = m_color[c]
         if not mc then
@@ -745,10 +747,12 @@ function colors.setlist(n,c,s)
         end
         n = n.next
     end
+    return f
 end
 
 function colors.reset(n)
     unset_attribute(n,a_color)
+    return n
 end
 
 -- maybe
@@ -766,9 +770,11 @@ function transparencies.set(n,t)
     else
         set_attribute(n,a_transparency,mt)
     end
+    return n
 end
 
 function transparencies.setlist(n,c,s)
+    local f = n
     while n do
         local mt = m_transparency[c]
         if not mt then
@@ -778,10 +784,12 @@ function transparencies.setlist(n,c,s)
         end
         n = n.next
     end
+    return f
 end
 
 function transparencies.reset(n)
     unset_attribute(n,a_transparency)
+    return n
 end
 
 -- for the moment here
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index d24c90a42..83def6697 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index f17102e40..c65b621fc 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 2ddebbed7..0b07ef402 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 11/06/12 09:56:45
+-- merge date  : 11/07/12 23:16:09
 
 do -- begin closure to overcome local limits and interference
 
@@ -2662,6 +2662,7 @@ if not modules then modules = { } end modules ['l-io'] = {
 local io = io
 local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format
 local concat = table.concat
+local floor = math.floor
 local type = type
 
 if string.find(os.getenv("PATH"),";") then
@@ -2670,10 +2671,49 @@ else
     io.fileseparator, io.pathseparator = "/" , ":"
 end
 
+local function readall(f)
+    return f:read("*all")
+end
+
+-- The next one is upto 50% faster on large files and less memory consumption due
+-- to less intermediate large allocations. This phenomena was discussed on the
+-- luatex dev list.
+
+local function readall(f)
+    local size = f:seek("end")
+    if size == 0 then
+        return ""
+    elseif size < 1024*1024 then
+        f:seek("set",0)
+        return f:read('*all')
+    else
+        local done = f:seek("set",0)
+        if size < 1024*1024 then
+            step = 1024 * 1024
+        elseif size > 16*1024*1024 then
+            step = 16*1024*1024
+        else
+            step = floor(size/(1024*1024)) * 1024 * 1024 / 8
+        end
+        local data = { }
+        while true do
+            local r = f:read(step)
+            if not r then
+                return concat(data)
+            else
+                data[#data+1] = r
+            end
+        end
+    end
+end
+
+io.readall = readall
+
 function io.loaddata(filename,textmode) -- return nil if empty
     local f = io.open(filename,(textmode and 'r') or 'rb')
     if f then
-        local data = f:read('*all')
+     -- local data = f:read('*all')
+        local data = readall(f)
         f:close()
         if #data > 0 then
             return data
@@ -4048,13 +4088,16 @@ function constructors.scale(tfmdata,specification)
     -- some context specific trickery (this will move to a plugin)
     --
     if hasmath then
-        if properties.mathitalics then
-            italickey = "italic_correction"
-            if trace_defining then
-                report_defining("math italics disabled for: name '%s', fullname: '%s', filename: '%s'",
-                    name or "noname",fullname or "nofullname",filename or "nofilename")
-            end
-        end
+     -- the latest luatex can deal with it itself so we now disable this
+     -- mechanism here
+     --
+     -- if properties.mathitalics then
+     --     italickey = "italic_correction"
+     --     if trace_defining then
+     --         report_defining("math italics disabled for: name '%s', fullname: '%s', filename: '%s'",
+     --             name or "noname",fullname or "nofullname",filename or "nofilename")
+     --     end
+     -- end
         autoitalicamount = false -- new
     else
         if properties.textitalics then