2015-05-04 19:02:00

14 files changed, 132 insertions, 120 deletions

diff --git a/tex/context/base/buff-ini.lua b/tex/context/base/buff-ini.lua
index 1ad3c905a..dafe2ad0e 100644
--- a/tex/context/base/buff-ini.lua
+++ b/tex/context/base/buff-ini.lua
@@ -9,7 +9,7 @@ if not modules then modules = { } end modules ['buff-ini'] = {
 local concat = table.concat
 local type, next, load = type, next, load
 local sub, format = string.sub, string.format
-local splitlines, validstring = string.splitlines, string.valid
+local splitlines, validstring, replacenewlines = string.splitlines, string.valid, string.replacenewlines
 local P, Cs, patterns, lpegmatch = lpeg.P, lpeg.Cs, lpeg.patterns, lpeg.match
 local utfchar  = utf.char
 local totable  = string.totable
@@ -157,12 +157,14 @@ local function collectcontent(name,separator) -- no print
                 t[n] = c
             end
         end
-        return concat(t,separator or "\n") -- was \r
+        -- the default separator was \r, then \n and is now os.newline because buffers
+        -- can be loaded in other applications
+        return concat(t,separator or os.newline)
     end
 end
 
 local function loadcontent(name) -- no print
-    local content = collectcontent(name,"\n")
+    local content = collectcontent(name,"\n") -- tex likes \n
     local ok, err = load(content)
     if ok then
         return ok()
@@ -454,7 +456,7 @@ local function savebuffer(list,name,prefix) -- name is optional
     if prefix == v_yes then
         name = addsuffix(tex.jobname .. "-" .. name,"tmp")
     end
-    io.savedata(name,content)
+    io.savedata(name,replacenewlines(content))
 end
 
 implement {
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index a99a6cfe7..fc87c4863 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2015.05.03 21:46}
+\newcontextversion{2015.05.04 19:00}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index ff40a9d94..298f490b5 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index a707bbeda..9a2ae1bde 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -39,7 +39,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2015.05.03 21:46}
+\edef\contextversion{2015.05.04 19:00}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/font-inj.lua b/tex/context/base/font-inj.lua
index cb9ed891c..332e92033 100644
--- a/tex/context/base/font-inj.lua
+++ b/tex/context/base/font-inj.lua
@@ -647,10 +647,10 @@ local function inject_cursives(glyphs,nofglyphs)
     end
 end
 
-local function inject_kerns(head,glyphs,nofglyphs)
+local function inject_kerns(head,list,length)
  -- todo: pre/post/replace
-    for i=1,#glyphs do
-        local n = glyphs[i]
+    for i=1,length do
+        local n  = list[i]
         local pn = rawget(properties,n)
         if pn then
             local i = rawget(pn,"injections")
@@ -688,6 +688,9 @@ local function inject_everything(head,where)
         end
         inject_kerns(head,glyphs,nofglyphs)
     end
+    if nofmarks > 0 then
+        inject_kerns(head,marks,nofmarks)
+	end
     if keepregisteredcounts then
         keepregisteredcounts  = false
     else
@@ -998,7 +1001,7 @@ local function inject_pairs_only(head,where)
                     if getsubtype(n) < 256 then
                         local p = rawget(properties,n)
                         if p then
-                            local i = rawget(pn,"replaceinjections")
+                            local i = rawget(p,"replaceinjections")
                             if i then
                                 local yoffset = i.yoffset
                                 if yoffset and yoffset ~= 0 then
diff --git a/tex/context/base/font-odv.lua b/tex/context/base/font-odv.lua
index 40b1efbb4..ca1b9ddf0 100644
--- a/tex/context/base/font-odv.lua
+++ b/tex/context/base/font-odv.lua
@@ -127,6 +127,8 @@ local free_node          = nuts.free
 local remove_node        = nuts.remove
 local flush_list         = nuts.flush_list
 
+local copyinjection      = nodes.injections.copy -- KE: is this necessary? HH: probably not as positioning comes later and we rawget/set
+
 local unsetvalue         = attributes.unsetvalue
 
 local fontdata           = fonts.hashes.identifiers
@@ -157,7 +159,7 @@ local fontprocesses      = fonts.hashes.processes
 local xprocesscharacters = nil
 
 xprocesscharacters = function(head,font)
-    xprocesscharacters = nodes.handlers.characters
+    xprocesscharacters = nodes.handlers.nodepass
     return xprocesscharacters(head,font)
 end
 
@@ -469,6 +471,7 @@ local sequence_reorder_matras = {
     features  = { dv01 = dev2_defaults },
     flags     = false_flags,
     name      = "dv01_reorder_matras",
+    order     = { "dv01" },
     subtables = { "dv01_reorder_matras" },
     type      = "devanagari_reorder_matras",
 }
@@ -478,6 +481,7 @@ local sequence_reorder_reph = {
     features  = { dv02 = dev2_defaults },
     flags     = false_flags,
     name      = "dv02_reorder_reph",
+    order     = { "dv02" },
     subtables = { "dv02_reorder_reph" },
     type      = "devanagari_reorder_reph",
 }
@@ -487,6 +491,7 @@ local sequence_reorder_pre_base_reordering_consonants = {
     features  = { dv03 = dev2_defaults },
     flags     = false_flags,
     name      = "dv03_reorder_pre_base_reordering_consonants",
+    order     = { "dv03" },
     subtables = { "dv03_reorder_pre_base_reordering_consonants" },
     type      = "devanagari_reorder_pre_base_reordering_consonants",
 }
@@ -496,6 +501,7 @@ local sequence_remove_joiners = {
     features  = { dv04 = deva_defaults },
     flags     = false_flags,
     name      = "dv04_remove_joiners",
+    order     = { "dv04" },
     subtables = { "dv04_remove_joiners" },
     type      = "devanagari_remove_joiners",
 }
@@ -698,7 +704,9 @@ local function deva_reorder(head,start,stop,font,attr,nbspaces)
                             local tmp = next and getnext(next) or nil -- needs checking
                             local changestop = next == stop
                             local tempcurrent = copy_node(next)
+							copyinjection(tempcurrent,next)
                             local nextcurrent = copy_node(current)
+							copyinjection(nextcurrent,current) -- KE: necessary? HH: probably not as positioning comes later and we rawget/set
                             setfield(tempcurrent,"next",nextcurrent)
                             setfield(nextcurrent,"prev",tempcurrent)
                             setprop(tempcurrent,a_state,s_blwf)
@@ -707,10 +715,11 @@ local function deva_reorder(head,start,stop,font,attr,nbspaces)
                             if getchar(next) == getchar(tempcurrent) then
                                 flush_list(tempcurrent)
                                 local n = copy_node(current)
+								copyinjection(n,current) -- KE: necessary? HH: probably not as positioning comes later and we rawget/set
                                 setfield(current,"char",dotted_circle)
                                 head = insert_node_after(head, current, n)
                             else
-                                setfield(current,"char",getchar(tempcurrent))    -- (assumes that result of blwf consists of one node)
+                                setfield(current,"char",getchar(tempcurrent)) -- we assumes that the result of blwf consists of one node
                                 local freenode = getnext(current)
                                 setfield(current,"next",tmp)
                                 if tmp then
@@ -1525,16 +1534,15 @@ local function dev2_reorder(head,start,stop,font,attr,nbspaces) -- maybe do a pa
         current = getnext(getnext(start))
     end
 
-    local function stand_alone(is_nbsp)
+    if current ~= getnext(stop) and getchar(current) == c_nbsp then
+        -- Stand Alone cluster
         if current == stop then
             stop = getprev(stop)
             head = remove_node(head,current)
             free_node(current)
             return head, stop, nbspaces
         else
-            if is_nbsp then
-                nbspaces = nbspaces + 1
-            end
+            nbspaces = nbspaces + 1
             base     = current
             current  = getnext(current)
             if current ~= stop then
@@ -1561,6 +1569,7 @@ local function dev2_reorder(head,start,stop,font,attr,nbspaces) -- maybe do a pa
                         if halant[getchar(current)] then
                             setfield(getnext(current),"next",tmp)
                             local nc = copy_node(current)
+							copyinjection(nc,current)
                             setfield(current,"char",dotted_circle)
                             head = insert_node_after(head,current,nc)
                         else
@@ -1573,14 +1582,6 @@ local function dev2_reorder(head,start,stop,font,attr,nbspaces) -- maybe do a pa
                 end
             end
         end
-    end
-
-    if current ~= getnext(stop) then
-        -- Stand Alone cluster
-        stand_alone()
-    elseif getchar(current) == c_nbsp then
-        -- Stand Alone cluster
-        stand_alone(true)
     else -- not Stand Alone cluster
         local last = getnext(stop)
         while current ~= last do    -- find base consonant
@@ -1636,6 +1637,7 @@ local function dev2_reorder(head,start,stop,font,attr,nbspaces) -- maybe do a pa
 local tpm = twopart_mark[char]
 if tpm then
     local extra = copy_node(current)
+	copyinjection(extra,current)
     char = tpm[1]
     setfield(current,"char",char)
     setfield(extra,"char",tpm[2])
@@ -2070,6 +2072,7 @@ end
 
 local function inject_syntax_error(head,current,mark)
     local signal = copy_node(current)
+	copyinjection(signal,current)
     if mark == pre_mark then
         setfield(signal,"char",dotted_circle)
     else
diff --git a/tex/context/base/font-otn.lua b/tex/context/base/font-otn.lua
index 52fcbcbd6..440eb34ec 100644
--- a/tex/context/base/font-otn.lua
+++ b/tex/context/base/font-otn.lua
@@ -2732,7 +2732,7 @@ local function discrun(disc,drun,krun)
         setfield(pre,"prev",nest)
         setfield(prev,"next",disc)
     else
-        run(pre,"preinjections")
+        krun(pre,"preinjections")
     end
     return next
 end
diff --git a/tex/context/base/l-os.lua b/tex/context/base/l-os.lua
index f44b31662..0a86ea6d6 100644
--- a/tex/context/base/l-os.lua
+++ b/tex/context/base/l-os.lua
@@ -349,6 +349,8 @@ else
 
 end
 
+os.newline = name == "windows" and "\013\010" or "\010" -- crlf or lf
+
 function resolvers.bits(t,k)
     local bits = find(os.platform,"64",1,true) and 64 or 32
     os.bits = bits
diff --git a/tex/context/base/math-noa.lua b/tex/context/base/math-noa.lua
index 893ee6f48..cdbbe36c3 100644
--- a/tex/context/base/math-noa.lua
+++ b/tex/context/base/math-noa.lua
@@ -231,6 +231,48 @@ end
 
 noads.process = processnoads
 
+--
+
+local unknowns = { }
+local checked  = { } -- simple case
+local tracked  = false  trackers.register("fonts.missing", function(v) tracked = v end)
+local cached   = table.setmetatableindex("table") -- complex case
+
+local function errorchar(font,char)
+    local done = unknowns[char]
+    if done then
+        unknowns[char] = done  + 1
+    else
+        unknowns[char] = 1
+    end
+    if tracked then
+        -- slower as we check each font too and we always replace as math has
+        -- more demands than text
+        local fake = cached[font][char]
+        if fake then
+            return fake
+        else
+            local kind, fake = fonts.checkers.placeholder(font,char)
+            if not fake or kind ~= "char" then
+                fake = 0x3F
+            end
+            cached[font][char] = fake
+            return fake
+        end
+    else
+        -- only simple checking, report at the end so one should take
+        -- action anyway ... we can miss a few checks but that is ok
+        -- as there is at least one reported
+        if not checked[char] then
+            if trace_normalizing then
+                report_normalizing("character %C is not available",char)
+            end
+            checked[char] = true
+        end
+        return 0x3F
+    end
+end
+
 -- experiment (when not present fall back to fam 0) -- needs documentation
 
 -- 0-2 regular
@@ -377,8 +419,12 @@ local function checked(pointer)
 end
 
 processors.relocate[math_char] = function(pointer)
-    local g = getattr(pointer,a_mathgreek) or 0
-    local a = getattr(pointer,a_mathalphabet) or 0
+    local g          = getattr(pointer,a_mathgreek) or 0
+    local a          = getattr(pointer,a_mathalphabet) or 0
+    local char       = getchar(pointer)
+    local fam        = getfield(pointer,"fam")
+    local font       = font_of_family(fam)
+    local characters = fontcharacters[font]
     if a > 0 or g > 0 then
         if a > 0 then
             setattr(pointer,a_mathgreek,0)
@@ -386,15 +432,11 @@ processors.relocate[math_char] = function(pointer)
         if g > 0 then
             setattr(pointer,a_mathalphabet,0)
         end
-        local char = getchar(pointer)
         local newchar = remapalphabets(char,a,g)
         if newchar then
-            local fam = getfield(pointer,"fam")
-            local id = font_of_family(fam)
-            local characters = fontcharacters[id]
             if characters[newchar] then
                 if trace_remapping then
-                    report_remap("char",id,char,newchar)
+                    report_remap("char",font,char,newchar)
                 end
                 if trace_analyzing then
                     setnodecolor(pointer,"font:isol")
@@ -408,7 +450,7 @@ processors.relocate[math_char] = function(pointer)
                     if newchar then
                         if characters[newchar] then
                             if trace_remapping then
-                                report_remap("char",id,char,newchar," (fallback remapping used)")
+                                report_remap("char",font,char,newchar," (fallback remapping used)")
                             end
                             if trace_analyzing then
                                 setnodecolor(pointer,"font:isol")
@@ -416,17 +458,20 @@ processors.relocate[math_char] = function(pointer)
                             setfield(pointer,"char",newchar)
                             return true
                         elseif trace_remapping then
-                            report_remap("char",id,char,newchar," fails (no fallback character)")
+                            report_remap("char",font,char,newchar," fails (no fallback character)")
                         end
                     elseif trace_remapping then
-                        report_remap("char",id,char,newchar," fails (no fallback remap character)")
+                        report_remap("char",font,char,newchar," fails (no fallback remap character)")
                     end
                 elseif trace_remapping then
-                    report_remap("char",id,char,newchar," fails (no fallback style)")
+                    report_remap("char",font,char,newchar," fails (no fallback style)")
                 end
             end
         end
     end
+    if not characters[char] then
+        setfield(pointer,"char",errorchar(font,char))
+    end
     if trace_analyzing then
         setnodecolor(pointer,"font:medi")
     end
@@ -466,9 +511,9 @@ processors.render[math_char] = function(pointer)
         if renderset then
             local newchar = renderset[char]
             if newchar then
-                local fam = getfield(pointer,"fam")
-                local id = font_of_family(fam)
-                local characters = fontcharacters[id]
+                local fam        = getfield(pointer,"fam")
+                local font       = font_of_family(fam)
+                local characters = fontcharacters[font]
                 if characters and characters[newchar] then
                     setfield(pointer,"char",newchar)
                     setattr(pointer,a_exportstatus,char)
@@ -524,15 +569,11 @@ end
 
 -- normalize scripts
 
-local unscript = { }  noads.processors.unscript = unscript
-local checkers = { }  noads.processors.checkers = checkers
-
+local unscript     = { }  noads.processors.unscript = unscript
 local superscripts = characters.superscripts
 local subscripts   = characters.subscripts
 local fractions    = characters.fractions
-
-local replaced = { }
-local unknowns = { }
+local replaced     = { }
 
 local function replace(pointer,what,n,parent)
     pointer = parent -- we're following the parent list (chars trigger this)
@@ -611,63 +652,11 @@ local function replace(pointer,what,n,parent)
     -- we could return stop
 end
 
-local tracked = false  trackers.register("fonts.missing", function(v) tracked = v end)
-local checked = { } -- simple case
-local cached  = table.setmetatableindex("table") -- complex case
-
-local function check(pointer,what,n,parent)
-    if tracked then
-        -- slower as we check each font too and we always replace as math has
-        -- more demands than text
-        local char = getchar(pointer)
-        local font = font_of_family(getfield(pointer,"fam"))
-        local fake = cached[font][char]
-        if fake then
-            unknowns[char] = unknowns[char]  + 1
-            setfield(pointer,"char",fake)
-        else
-            local chars = fontcharacters[font]
-            if not chars[char] then
-                unknowns[char] = 1
-                local kind, fake = fonts.checkers.placeholder(font,char)
-                if kind == "char" then
-                    cached[font][char] = fake
-                    setfield(pointer,"char",fake)
-                else
-                    cached[font][char] = 0x3F
-                end
-            end
-        end
-    else
-        -- only simple checking, report at the end so one should take
-        -- action anyway ... we can miss a few checks but that is ok
-        -- as there is at least one reported
-        local char = getchar(pointer)
-        local done = unknowns[char]
-        if done then
-            unknowns[char] = done  + 1
-            setfield(pointer,"char",0x3F)
-        elseif not checked[char] then
-            local font  = font_of_family(getfield(pointer,"fam"))
-            local chars = fontcharacters[font]
-            if not chars[char] then
-                unknowns[char] = 1
-                setfield(pointer,"char",0x3F)
-                if trace_normalizing then
-                    report_normalizing("character %C is not available",char)
-                end
-            end
-        end
-        checked[char] = true
-    end
-end
-
 unscript[math_char] = replace -- not noads as we need to recurse
-checkers[math_char] = check  -- not noads as we need to recurse
 
 function handlers.unscript(head,style,penalties)
     processnoads(head,unscript,"unscript")
-    processnoads(head,checkers,"checkers")
+--  processnoads(head,checkers,"checkers")
     return true
 end
 
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index d407b8236..b58500367 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 66d311434..2046da47a 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/strc-bkm.lua b/tex/context/base/strc-bkm.lua
index 1ff75aa5f..96b68b236 100644
--- a/tex/context/base/strc-bkm.lua
+++ b/tex/context/base/strc-bkm.lua
@@ -144,8 +144,10 @@ function bookmarks.place()
                                 else
                                     titledata = { title = first }
                                 end
-                            else
+                            elseif second then
                                 titledata = { title = second }
+                            else
+                                -- ignoring (command and so)
                             end
                         end
                     end
diff --git a/tex/context/base/util-str.lua b/tex/context/base/util-str.lua
index a677a82ed..de4a87e9f 100644
--- a/tex/context/base/util-str.lua
+++ b/tex/context/base/util-str.lua
@@ -44,7 +44,12 @@ end
 
 if not number then number = { } end -- temp hack for luatex-fonts
 
-local stripper = patterns.stripzeros
+local stripper    = patterns.stripzeros
+local newline     = patterns.newline
+local endofstring = patterns.endofstring
+local whitespace  = patterns.whitespace
+local spacer      = patterns.spacer
+local spaceortab  = patterns.spaceortab
 
 local function points(n)
     n = tonumber(n)
@@ -62,12 +67,12 @@ number.basepoints = basepoints
 -- str = " \n \ntest  \n test\ntest "
 -- print("["..string.gsub(string.collapsecrlf(str),"\n","+").."]")
 
-local rubish     = patterns.spaceortab^0 * patterns.newline
-local anyrubish  = patterns.spaceortab + patterns.newline
+local rubish     = spaceortab^0 * newline
+local anyrubish  = spaceortab + newline
 local anything   = patterns.anything
-local stripped   = (patterns.spaceortab^1 / "") * patterns.newline
+local stripped   = (spaceortab^1 / "") * newline
 local leading    = rubish^0 / ""
-local trailing   = (anyrubish^1 * patterns.endofstring) / ""
+local trailing   = (anyrubish^1 * endofstring) / ""
 local redundant  = rubish^3 / "\n"
 
 local pattern = Cs(leading * (trailing + redundant + stripped + anything)^0)
@@ -129,7 +134,7 @@ local pattern =
               return ""
           end
       end
-    + patterns.newline * Cp() / function(position)
+    + newline * Cp() / function(position)
           extra, start = 0, position
       end
     + patterns.anything
@@ -162,11 +167,6 @@ end
 --     return str
 -- end
 
-local newline     = patterns.newline
-local endofstring = patterns.endofstring
-local whitespace  = patterns.whitespace
-local spacer      = patterns.spacer
-
 local space       = spacer^0
 local nospace     = space/""
 local endofline   = nospace * newline
@@ -1117,3 +1117,9 @@ local pattern =
 function string.optionalquoted(str)
     return lpegmatch(pattern,str) or str
 end
+
+local pattern = Cs((newline / os.newline + 1)^0)
+
+function string.replacenewlines(str)
+    return lpegmatch(pattern,str)
+end
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index eabed7cb8..c46c16874 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 05/03/15 21:46:51
+-- merge date  : 05/04/15 19:00:43
 
 do -- begin closure to overcome local limits and interference
 
@@ -2798,6 +2798,11 @@ else
 end
 if not number then number={} end 
 local stripper=patterns.stripzeros
+local newline=patterns.newline
+local endofstring=patterns.endofstring
+local whitespace=patterns.whitespace
+local spacer=patterns.spacer
+local spaceortab=patterns.spaceortab
 local function points(n)
   n=tonumber(n)
   return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536))
@@ -2808,12 +2813,12 @@ local function basepoints(n)
 end
 number.points=points
 number.basepoints=basepoints
-local rubish=patterns.spaceortab^0*patterns.newline
-local anyrubish=patterns.spaceortab+patterns.newline
+local rubish=spaceortab^0*newline
+local anyrubish=spaceortab+newline
 local anything=patterns.anything
-local stripped=(patterns.spaceortab^1/"")*patterns.newline
+local stripped=(spaceortab^1/"")*newline
 local leading=rubish^0/""
-local trailing=(anyrubish^1*patterns.endofstring)/""
+local trailing=(anyrubish^1*endofstring)/""
 local redundant=rubish^3/"\n"
 local pattern=Cs(leading*(trailing+redundant+stripped+anything)^0)
 function strings.collapsecrlf(str)
@@ -2859,17 +2864,13 @@ local pattern=Carg(1)/function(t)
      else
        return ""
      end
-   end+patterns.newline*Cp()/function(position)
+   end+newline*Cp()/function(position)
      extra,start=0,position
    end+patterns.anything
  )^1)
 function strings.tabtospace(str,tab)
   return lpegmatch(pattern,str,1,tab or 7)
 end
-local newline=patterns.newline
-local endofstring=patterns.endofstring
-local whitespace=patterns.whitespace
-local spacer=patterns.spacer
 local space=spacer^0
 local nospace=space/""
 local endofline=nospace*newline
@@ -3438,6 +3439,10 @@ local pattern=Cs(dquote*(equote-P(-2))^0*dquote)
 function string.optionalquoted(str)
   return lpegmatch(pattern,str) or str
 end
+local pattern=Cs((newline/os.newline+1)^0)
+function string.replacenewlines(str)
+  return lpegmatch(pattern,str)
+end
 
 end -- closure