beta 2013.12.14 13:34

17 files changed, 232 insertions, 21 deletions

diff --git a/tex/context/base/char-ini.lua b/tex/context/base/char-ini.lua
index a94aa6b5f..d2236f871 100644
--- a/tex/context/base/char-ini.lua
+++ b/tex/context/base/char-ini.lua
@@ -920,13 +920,13 @@ if not characters.superscripts then
             if what == "super" then
                 if #specials == 2 then
                     superscripts[k] = specials[2]
-                else
+                elseif trace_defining then
                     report_defining("ignoring %s %a, char %c, description %a","superscript",ustring(k),k,v.description)
                 end
             elseif what == "sub" then
                 if #specials == 2 then
                     subscripts[k] = specials[2]
-                else
+                elseif trace_defining then
                     report_defining("ignoring %s %a, char %c, description %a","subscript",ustring(k),k,v.description)
                 end
             end
diff --git a/tex/context/base/char-tex.lua b/tex/context/base/char-tex.lua
index 91aa387b9..4c397ab3b 100644
--- a/tex/context/base/char-tex.lua
+++ b/tex/context/base/char-tex.lua
@@ -164,7 +164,8 @@ local function remap_accents(a,c,braced)
 end
 
 local command_map = allocate {
-    ["i"] = "ı"
+    ["i"] = "ı",
+    ["l"] = "ł",
 }
 
 local function remap_commands(c)
@@ -176,8 +177,9 @@ local function remap_commands(c)
     end
 end
 
-local accents  = (P('\\') * C(S(accents)) * (P("{") * C(U) * P("}" * Cc(true)) + C(U) * Cc(false))) / remap_accents
-local commands = (P('\\') * C(R("az","AZ")^1)) / remap_commands
+local spaces   = P(" ")^0
+local accents  = ( P('\\') * C(S(accents)) * spaces * (P("{") * C(U) * P("}" * Cc(true)) + C(U) * Cc(false)) ) / remap_accents
+local commands = ( P('\\') * C(R("az","AZ")^1) + P("{") * P('\\') * C(R("az","AZ")^1) * spaces * P("}") )/ remap_commands
 
 local convert_accents  = Cs((accents  + P(1))^0)
 local convert_commands = Cs((commands + P(1))^0)
@@ -198,8 +200,12 @@ function characters.tex.toutf(str,strip)
     end
 end
 
---~ print(characters.tex.toutf([[\"{e}]]),true)
---~ print(characters.tex.toutf([[{\"{e}}]],true))
+-- print(characters.tex.toutf([[\"{e}]],true))
+-- print(characters.tex.toutf([[\" {e}]],true))
+-- print(characters.tex.toutf([[{\"{e}}]],true))
+-- print(characters.tex.toutf([[{\" {e}}]],true))
+-- print(characters.tex.toutf([[{\l}]],true))
+-- print(characters.tex.toutf([[{\l }]],true))
 
 function characters.tex.defineaccents()
     for accent, group in next, accentmapping do
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 0cd286558..0d9ac18ef 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2013.12.04 11:34}
+\newcontextversion{2013.12.14 13:34}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 7a8ffb800..aaadd65a6 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 87fe9c8ff..9904e1655 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2013.12.04 11:34}
+\edef\contextversion{2013.12.14 13:34}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/font-otn.lua b/tex/context/base/font-otn.lua
index d9e4b9b64..c57be5f02 100644
--- a/tex/context/base/font-otn.lua
+++ b/tex/context/base/font-otn.lua
@@ -2260,6 +2260,35 @@ local function featuresprocessor(head,font,attr)
                         end
                     end
 
+                    local function kerndisc(disc) -- we can assume that prev and next are glyphs
+                        local prev = disc.prev
+                        local next = disc.next
+                        if prev and next then
+                            prev.next = next
+                         -- next.prev = prev
+                            local a = prev[0]
+                            if a then
+                                a = (a == attr) and (not attribute or prev[a_state] == attribute)
+                            else
+                                a = not attribute or prev[a_state] == attribute
+                            end
+                            if a then
+                                local lookupmatch = lookupcache[prev.char]
+                                if lookupmatch then
+                                    -- sequence kan weg
+                                    local h, d, ok = handler(head,prev,dataset[4],lookupname,lookupmatch,sequence,lookuphash,1)
+                                    if ok then
+                                        done = true
+                                        success = true
+                                    end
+                                end
+                            end
+                            prev.next = disc
+                         -- next.prev = disc
+                        end
+                        return next
+                    end
+
                     while start do
                         local id = start.id
                         if id == glyph_code then
@@ -2305,6 +2334,8 @@ local function featuresprocessor(head,font,attr)
                                     local new = subrun(replace)
                                     if new then start.replace = new end
                                 end
+elseif typ == "gpos_single" or typ == "gpos_pair" then
+    kerndisc(start)
                             end
                             start = start.next
                         elseif id == whatsit_code then -- will be function
@@ -2402,6 +2433,43 @@ local function featuresprocessor(head,font,attr)
                     end
                 end
 
+                local function kerndisc(disc) -- we can assume that prev and next are glyphs
+                    local prev = disc.prev
+                    local next = disc.next
+                    if prev and next then
+                        prev.next = next
+                     -- next.prev = prev
+                        local a = prev[0]
+                        if a then
+                            a = (a == attr) and (not attribute or prev[a_state] == attribute)
+                        else
+                            a = not attribute or prev[a_state] == attribute
+                        end
+                        if a then
+                            for i=1,ns do
+                                local lookupname = subtables[i]
+                                local lookupcache = lookuphash[lookupname]
+                                if lookupcache then
+                                    local lookupmatch = lookupcache[prev.char]
+                                    if lookupmatch then
+                                        -- we could move all code inline but that makes things even more unreadable
+                                        local h, d, ok = handler(head,prev,dataset[4],lookupname,lookupmatch,sequence,lookuphash,i)
+                                        if ok then
+                                            done = true
+                                            break
+                                        end
+                                    end
+                                else
+                                    report_missing_cache(typ,lookupname)
+                                end
+                            end
+                        end
+                        prev.next = disc
+                     -- next.prev = disc
+                    end
+                    return next
+                end
+
                 while start do
                     local id = start.id
                     if id == glyph_code then
@@ -2459,6 +2527,8 @@ local function featuresprocessor(head,font,attr)
                                 local new = subrun(replace)
                                 if new then start.replace = new end
                             end
+elseif typ == "gpos_single" or typ == "gpos_pair" then
+    kerndisc(start)
                         end
                         start = start.next
                     elseif id == whatsit_code then
diff --git a/tex/context/base/l-table.lua b/tex/context/base/l-table.lua
index 11cb66bef..c6bbc62d6 100644
--- a/tex/context/base/l-table.lua
+++ b/tex/context/base/l-table.lua
@@ -1054,3 +1054,24 @@ function table.sorted(t,...)
     sort(t,...)
     return t -- still sorts in-place
 end
+
+--
+
+function table.values(t,s) -- optional sort flag
+    if t then
+        local values, keys, v = { }, { }, 0
+        for key, value in next, t do
+            if not keys[value] then
+                v = v + 1
+                values[v] = value
+                keys[k] = key
+            end
+        end
+        if s then
+            sort(values)
+        end
+        return values
+    else
+        return { }
+    end
+end
diff --git a/tex/context/base/l-unicode.lua b/tex/context/base/l-unicode.lua
index 7ada394d5..902f6a0d0 100644
--- a/tex/context/base/l-unicode.lua
+++ b/tex/context/base/l-unicode.lua
@@ -935,19 +935,27 @@ end
 local _, l_remap = utf.remapper(little)
 local _, b_remap = utf.remapper(big)
 
-function utf.utf8_to_utf16_be(str)
-    return char(254,255) .. lpegmatch(b_remap,str)
+function utf.utf8_to_utf16_be(str,nobom)
+    if nobom then
+        return lpegmatch(b_remap,str)
+    else
+        return char(254,255) .. lpegmatch(b_remap,str)
+    end
 end
 
-function utf.utf8_to_utf16_le(str)
-    return char(255,254) .. lpegmatch(l_remap,str)
+function utf.utf8_to_utf16_le(str,nobom)
+    if nobom then
+        return lpegmatch(l_remap,str)
+    else
+        return char(255,254) .. lpegmatch(l_remap,str)
+    end
 end
 
-function utf.utf8_to_utf16(str,littleendian)
+function utf.utf8_to_utf16(str,littleendian,nobom)
     if littleendian then
-        return utf.utf8_to_utf16_le(str)
+        return utf.utf8_to_utf16_le(str,nobom)
     else
-        return utf.utf8_to_utf16_be(str)
+        return utf.utf8_to_utf16_be(str,nobom)
     end
 end
 
diff --git a/tex/context/base/lang-def.mkiv b/tex/context/base/lang-def.mkiv
index ecd5b3c22..18f572039 100644
--- a/tex/context/base/lang-def.mkiv
+++ b/tex/context/base/lang-def.mkiv
@@ -696,6 +696,26 @@
 
 \installlanguage [\s!thai] [\s!th]
 
+%D Malayalam (needs to be checked)
+
+\installlanguage
+  [\s!ml]
+  [\c!spacing=\v!broad,
+   \c!leftsentence=\emdash,
+   \c!rightsentence=\emdash,
+   \c!leftsubsentence=\emdash,
+   \c!rightsubsentence=\emdash,
+   \c!leftquote=\upperleftsinglesixquote,
+   \c!rightquote=\upperrightsingleninequote,
+   \c!leftquotation=\upperleftdoublesixquote,
+   \c!rightquotation=\upperrightdoubleninequote,
+   \c!date={\v!month,\space,\v!day,{,\space},\v!year},
+   \s!patterns=\s!ml,
+   \s!lefthyphenmin=2,
+   \s!righthyphenmin=3]
+
+\installlanguage [\s!malayalam] [\s!ml]
+
 %D Todo: generate this one from languages.data
 
 \installlanguage[nld][\s!nl]
diff --git a/tex/context/base/mult-def.mkiv b/tex/context/base/mult-def.mkiv
index 15d533009..3d480e187 100644
--- a/tex/context/base/mult-def.mkiv
+++ b/tex/context/base/mult-def.mkiv
@@ -107,6 +107,10 @@
 \ifdefined\v!rule          \else \def\v!rule          {rule}           \fi
 \ifdefined\v!oldstyle      \else \def\v!oldstyle      {oldstyle}       \fi
 
+\def\c!etallimit  {etallimit}
+\def\c!etaldisplay{etaldisplay}
+\def\c!etaltext   {etaltext}
+
 % stop todo
 
 \protect \endinput
diff --git a/tex/context/base/mult-mps.lua b/tex/context/base/mult-mps.lua
index d50beb640..104b9d42e 100644
--- a/tex/context/base/mult-mps.lua
+++ b/tex/context/base/mult-mps.lua
@@ -31,7 +31,7 @@ return {
         "def", "vardef", "enddef", "expr", "suffix", "text", "primary", "secondary",
         "tertiary", "primarydef", "secondarydef", "tertiarydef",
         "randomseed", "also", "contour", "doublepath",
-        "withcolor", "withpen", "dashed", "if", "else", "elseif", "fi", "for", "endfor", "forever", "exitif", "within",
+        "withcolor", "withcmykcolor", "withpen", "dashed", "if", "else", "elseif", "fi", "for", "endfor", "forever", "exitif", "within",
         "forsuffixes", "downto", "upto", "step", "until",
         "charlist", "extensible", "fontdimen", "headerbyte", "kern", "ligtable",
         "boundarychar", "chardp", "charext", "charht", "charic", "charwd", "designsize",
diff --git a/tex/context/base/mult-sys.mkiv b/tex/context/base/mult-sys.mkiv
index 6c6db58e1..8c1bff2bc 100644
--- a/tex/context/base/mult-sys.mkiv
+++ b/tex/context/base/mult-sys.mkiv
@@ -56,6 +56,7 @@
 \definesystemconstant {latin}        \definesystemconstant {la}
 \definesystemconstant {lithuanian}   \definesystemconstant {lt}
 \definesystemconstant {bokmal}       \definesystemconstant {nb}
+\definesystemconstant {malayalam}    \definesystemconstant {ml}
 \definesystemconstant {norwegian}    \definesystemconstant {no}
 \definesystemconstant {nynorsk}      \definesystemconstant {nn}
 \definesystemconstant {polish}       \definesystemconstant {pl}
diff --git a/tex/context/base/sort-lan.lua b/tex/context/base/sort-lan.lua
index 8efc0924a..6d16c0d80 100644
--- a/tex/context/base/sort-lan.lua
+++ b/tex/context/base/sort-lan.lua
@@ -59,7 +59,7 @@ definitions["en"] = { parent = "default" }
 definitions['nl'] = {
     parent = 'default',
     replacements = {
-        { "ij", 'y' }, { "IJ", 'Y' },
+        { "ij", 'y' }, { "IJ", 'Y' }, -- hm
     },
 }
 
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 5bfe22f86..8ace29921 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 4c381f1ce..637a7d51d 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/util-prs.lua b/tex/context/base/util-prs.lua
index 29a57e094..9518b7f38 100644
--- a/tex/context/base/util-prs.lua
+++ b/tex/context/base/util-prs.lua
@@ -483,8 +483,8 @@ function parsers.rfc4180splitter(specification)
                       * Cs((dquotechar + (1 - quotechar))^0)
                       * quotechar
     local non_escaped = C((1 - quotechar - newline - separator)^1)
-    local field       = escaped + non_escaped
-    local record      = Ct((field * separator^-1)^1)
+    local field       = escaped + non_escaped + Cc("")
+    local record      = Ct(field * (separator * field)^1)
     local headerline  = record * Cp()
     local wholeblob   = Ct((newline^-1 * record)^0)
     return function(data,getheader)
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 4366fb070..3a705f242 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 12/04/13 11:34:01
+-- merge date  : 12/14/13 13:34:05
 
 do -- begin closure to overcome local limits and interference
 
@@ -1668,6 +1668,24 @@ function table.sorted(t,...)
   sort(t,...)
   return t 
 end
+function table.values(t,s) 
+  if t then
+    local values,keys,v={},{},0
+    for key,value in next,t do
+      if not keys[value] then
+        v=v+1
+        values[v]=value
+        keys[k]=key
+      end
+    end
+    if s then
+      sort(values)
+    end
+    return values
+  else
+    return {}
+  end
+end
 
 end -- closure
 
@@ -8813,6 +8831,7 @@ nodes.injections=nodes.injections or {}
 local injections=nodes.injections
 local nodecodes=nodes.nodecodes
 local glyph_code=nodecodes.glyph
+local kern_disc=nodecodes.disc
 local kern_code=nodecodes.kern
 local nuts=nodes.nuts
 local nodepool=nuts.pool
@@ -11445,6 +11464,31 @@ local function featuresprocessor(head,font,attr)
               return head
             end
           end
+          local function kerndisc(disc) 
+            local prev=getprev(disc)
+            local next=getnext(disc)
+            if prev and next then
+              setfield(prev,"next",next)
+              local a=getattr(prev,0)
+              if a then
+                a=(a==attr) and (not attribute or getattr(prev,a_state)==attribute)
+              else
+                a=not attribute or getattr(prev,a_state)==attribute
+              end
+              if a then
+                local lookupmatch=lookupcache[getchar(prev)]
+                if lookupmatch then
+                  local h,d,ok=handler(head,prev,dataset[4],lookupname,lookupmatch,sequence,lookuphash,1)
+                  if ok then
+                    done=true
+                    success=true
+                  end
+                end
+              end
+              setfield(prev,"next",disc)
+            end
+            return next
+          end
           while start do
             local id=getid(start)
             if id==glyph_code then
@@ -11488,6 +11532,8 @@ local function featuresprocessor(head,font,attr)
                   local new=subrun(replace)
                   if new then setfield(start,"replace",new) end
                 end
+elseif typ=="gpos_single" or typ=="gpos_pair" then
+  kerndisc(start)
               end
               start=getnext(start)
             elseif id==whatsit_code then 
@@ -11579,6 +11625,39 @@ local function featuresprocessor(head,font,attr)
             return head
           end
         end
+        local function kerndisc(disc) 
+          local prev=getprev(disc)
+          local next=getnext(disc)
+          if prev and next then
+            setfield(prev,"next",next)
+            local a=getattr(prev,0)
+            if a then
+              a=(a==attr) and (not attribute or getattr(prev,a_state)==attribute)
+            else
+              a=not attribute or getattr(prev,a_state)==attribute
+            end
+            if a then
+              for i=1,ns do
+                local lookupname=subtables[i]
+                local lookupcache=lookuphash[lookupname]
+                if lookupcache then
+                  local lookupmatch=lookupcache[getchar(prev)]
+                  if lookupmatch then
+                    local h,d,ok=handler(head,prev,dataset[4],lookupname,lookupmatch,sequence,lookuphash,i)
+                    if ok then
+                      done=true
+                      break
+                    end
+                  end
+                else
+                  report_missing_cache(typ,lookupname)
+                end
+              end
+            end
+            setfield(prev,"next",disc)
+          end
+          return next
+        end
         while start do
           local id=getid(start)
           if id==glyph_code then
@@ -11633,6 +11712,8 @@ local function featuresprocessor(head,font,attr)
                 local new=subrun(replace)
                 if new then setfield(start,"replace",new) end
               end
+elseif typ=="gpos_single" or typ=="gpos_pair" then
+  kerndisc(start)
             end
             start=getnext(start)
           elseif id==whatsit_code then