beta 2010.04.23 18:34

17 files changed, 448 insertions, 168 deletions

diff --git a/tex/context/base/blob-ini.lua b/tex/context/base/blob-ini.lua
new file mode 100644
index 000000000..0f7ccee26
--- /dev/null
+++ b/tex/context/base/blob-ini.lua
@@ -0,0 +1,165 @@
+if not modules then modules = { } end modules ['blob-ini'] = {
+    version   = 1.001,
+    comment   = "companion to blob-ini.mkiv",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- later we will consider an OO variant.
+
+-- This module is just a playground. Occasionally we need to typeset
+-- at the lua and and this is one method. In principle we can construct
+-- pages this way too which sometimes makes sense in dumb cases. Actually,
+-- if one only needs this, one does not really need tex, okay maybe the
+-- parbuilder but that one can be simplified as well then.
+
+-- set fonts, attributes
+-- rest already done in packers etc
+-- add local par whatsit (or wait till cleaned up)
+-- collapse or new pars
+-- interline spacing etc
+
+-- DON'T USE THESE FUNCTIONS AS THEY WILL CHANGE!
+
+local type = type
+
+local utfvalues = string.utfvalues
+local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns
+
+local fontdata = fonts.identifiers
+
+local new_glyph_node    = nodes.glyph
+local new_glue_node     = nodes.glyph
+
+local copy_node         = node.copy
+local copy_node_list    = node.copy_list
+local insert_node_after = node.insert_after
+local flush_node_list   = node.flush_list
+local hpack_node_list   = node.hpack
+local vpack_node_list   = node.vpack
+local write_node        = node.write
+
+local current_font      = font.current
+
+blobs = blobs or  { }
+
+local newline = lpegpatterns.newline
+local space   = lpegpatterns.spacer
+local spacing = newline * space^0
+local content = (space^1)/" " + (1-spacing)
+
+local ctxtextcapture = lpeg.Ct ( (
+    space^0 * (
+        newline^2 * space^0 * lpeg.Cc("")
+      + newline   * space^0 * lpeg.Cc(" ")
+      + lpeg.Cs(content^1)
+    )
+)^0)
+
+local function tonodes(str,fnt,attr) -- (str,template_glyph)
+    if not str or str == "" then
+        return
+    end
+    local head, tail, space, fnt, template = nil, nil, nil, nil, nil
+    if not fnt then
+        fnt = current_font()
+    elseif type(fnt) ~= "number" and fnt.id == "glyph" then
+        fnt, template = nil, fnt
+ -- else
+     -- already a number
+    end
+    for s in utfvalues(str) do
+        local n
+        if s == 32 then
+            if not space then
+                local parameters = fontdata[fnt].parameters
+                space = new_glue_node(parameters.space,parameters.space_stretch,parameters.space_shrink)
+                n = space
+            else
+                n = copy_node(space)
+            end
+        elseif template then
+            n = copy_node(template)
+            n.char = s
+        else
+            n = new_glyph_node(fnt,s)
+        end
+        if attr then -- normall false when template
+            n.attr = copy_node_list(attr)
+        end
+        if head then
+            insert_node_after(head,tail,n)
+        else
+            head = n
+        end
+        tail = n
+    end
+    return head, tail
+end
+
+blobs.tonodes = tonodes
+
+function blobs.new()
+    return {
+        list = { },
+    }
+end
+
+function blobs.append(t,str)
+    local kind = type(str)
+    local dummy = nil
+    if kind == "string" then
+        local pars = lpegmatch(ctxtextcapture,str)
+        local list = t.list
+        for p=1,#pars do
+            local str = pars[p]
+            if #str == 0 then
+                list[#list+1 ] = { head = nil, tail = nil }
+            else
+                local l = list[#list]
+                if not l then
+                    l = { head = nil, tail = nil }
+                    list[#list+1 ] = l
+                end
+                local head, tail = tonodes(str,nil,nil)
+                if head then
+                    if l.head then
+                        l.tail.next = head
+                        head.prev = l.tail
+                        l.tail = tail
+                    else
+                        l.head, l.tail = head, tail
+                    end
+                end
+            end
+        end
+    end
+end
+
+function blobs.pack(t,how)
+    local list = t.list
+    for i=1,#list do
+        local pack = list[i].pack
+        if pack then
+            flush_node_list(node.pack)
+        end
+        if how == "vertical" then
+            -- we need to prepend a local par node
+            -- list[i].pack = node.vpack(list[i].head,"exactly")
+            logs.report("blobs","vpack not yet supported")
+        else
+            list[i].pack = hpack_node_list(list[i].head,"exactly")
+        end
+    end
+end
+
+function blobs.write(t)
+    local list = t.list
+    for i=1,#list do
+        local pack = list[i].pack
+        if pack then
+            write_node(pack)
+        end
+    end
+end
diff --git a/tex/context/base/blob-ini.mkiv b/tex/context/base/blob-ini.mkiv
new file mode 100644
index 000000000..7f63ec73d
--- /dev/null
+++ b/tex/context/base/blob-ini.mkiv
@@ -0,0 +1,34 @@
+%D \module
+%D   [       file=blob-ini,
+%D        version=2010.04.06,
+%D          title=\CONTEXT\ \LUA\ Typesetting,
+%D       subtitle=Initialization,
+%D         author=Hans Hagen,
+%D           date=\currentdate,
+%D      copyright=\PRAGMA]
+%C
+%C This module is part of the \CONTEXT\ macro||package and is
+%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
+%C details.
+
+\writestatus{loading}{ConTeXt Lua Typesetting / Initialization}
+
+%D This is a prelude to typesetting at the \LUA\ end. The code
+%D is already quite old but will only get nice when we are further
+%D down the road (close to version 1.00 of \LUATEX). Typesetting in
+%D pure \LUA\ sometimes makes sense.
+
+\registerctxluafile{blob-ini}{1.001}
+
+\endinput
+
+% \starttext
+%
+% \startluacode
+%     local b = blobs.new()
+%     blobs.append(b,"Hello    world.\n Here we are.\n\n And Again!")
+%     blobs.pack(b)
+%     blobs.write(b)
+% \stopluacode
+%
+% \stoptext
diff --git a/tex/context/base/cont-new.tex b/tex/context/base/cont-new.tex
index 9999d79bd..79fe0c555 100644
--- a/tex/context/base/cont-new.tex
+++ b/tex/context/base/cont-new.tex
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2010.04.16 21:08}
+\newcontextversion{2010.04.23 18:34}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 8b533a34d..13fe9f2cc 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -89,7 +89,7 @@
 \loadmarkfile{trac-lmx}
 \loadmarkfile{trac-deb}
 
-%loadmarkfile{blob-ini} % not yet public (typesetting in pure lua)
+\loadmarkfile{blob-ini} % not to be used, we only use a helper
 
 \loadcorefile{supp-box}
 
diff --git a/tex/context/base/context.tex b/tex/context/base/context.tex
index bba0807fb..188ff748e 100644
--- a/tex/context/base/context.tex
+++ b/tex/context/base/context.tex
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2010.04.16 21:08}
+\edef\contextversion{2010.04.23 18:34}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/core-mis.mkiv b/tex/context/base/core-mis.mkiv
index 00535e586..28ce8406b 100644
--- a/tex/context/base/core-mis.mkiv
+++ b/tex/context/base/core-mis.mkiv
@@ -623,7 +623,7 @@
 
 \def\setuphyphenmark[#1]% sign=normal|wide
   {\dodoubleargument\getparameters[\??kp][#1]%
-   \doifinsetelse\@@kpsign {\v!normal}
+   \doifelse\@@kpsign {\v!normal}% was inset?
      {\let\textmodehyphen\normalhyphen  \let\textmodehyphendiscretionary\normalhyphendiscretionary}
      {\let\textmodehyphen\composedhyphen\let\textmodehyphendiscretionary\composedhyphendiscretionary}}
 
@@ -642,14 +642,14 @@
 
 \def\normalhyphendiscretionary
   {\discretionary
-     {\hbox{\directsymbol\empty\c!lefthyphen}}
      {\hbox{\directsymbol\empty\c!righthyphen}}
+     {\hbox{\directsymbol\empty\c!lefthyphen}}
      {\hbox{\directsymbol\empty\c!hyphen}}}
 
 \def\composedhyphendiscretionary
   {\discretionary
-     {\hbox{\directsymbol\empty\c!leftcompoundhyphen}}
      {\hbox{\directsymbol\empty\c!rightcompoundhyphen}}
+     {\hbox{\directsymbol\empty\c!leftcompoundhyphen}}
      {\hbox{\directsymbol\empty\c!compoundhyphen}}}
 
 \let\textmodehyphen             \composedhyphen
diff --git a/tex/context/base/data-lua.lua b/tex/context/base/data-lua.lua
index af09bbe59..57a434d1d 100644
--- a/tex/context/base/data-lua.lua
+++ b/tex/context/base/data-lua.lua
@@ -12,13 +12,29 @@ if not modules then modules = { } end modules ['data-lua'] = {
 
 local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v end)
 
-local gsub = string.gsub
+local gsub, insert = string.gsub, table.insert
 local unpack = unpack or table.unpack
 
 local  libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs'
 local clibformats = { 'lib' }
-local  libpaths   = file.split_path(package.path)
-local clibpaths   = file.split_path(package.cpath)
+
+local _path_, libpaths, _cpath_, clibpaths
+
+function package.libpaths()
+    if not _path_ or package.path ~= _path_ then
+        _path_ = package.path
+        libpaths = file.split_path(_path_)
+    end
+    return libpaths
+end
+
+function package.clibpaths()
+    if not _cpath_ or package.cpath ~= _cpath_ then
+        _cpath_ = package.cpath
+        clibpaths = file.split_path(_cpath_)
+    end
+    return clibpaths
+end
 
 local function thepath(...)
     local t = { ... } t[#t+1] = "?.lua"
@@ -29,16 +45,35 @@ local function thepath(...)
     return path
 end
 
+local p_libpaths, a_libpaths = { }, { }
+
 function package.append_libpath(...)
-    table.insert(libpaths,thepath(...))
+    insert(a_libpath,thepath(...))
 end
 
 function package.prepend_libpath(...)
-    table.insert(libpaths,1,thepath(...))
+    insert(p_libpaths,1,thepath(...))
 end
 
 -- beware, we need to return a loadfile result !
 
+local function loaded(libpaths,name,simple)
+    for i=1,#libpaths do -- package.path, might become option
+        local libpath = libpaths[i]
+        local resolved = gsub(libpath,"%?",simple)
+        if trace_locating then -- more detail
+            logs.report("fileio","! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved)
+        end
+        if resolvers.isreadable.file(resolved) then
+            if trace_locating then
+                logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved)
+            end
+            return loadfile(resolved)
+        end
+    end
+end
+
+
 package.loaders[2] = function(name) -- was [#package.loaders+1]
     if trace_locating then -- mode detail
         logs.report("fileio","! locating '%s'",name)
@@ -56,21 +91,15 @@ package.loaders[2] = function(name) -- was [#package.loaders+1]
             return loadfile(resolved)
         end
     end
+    -- libpaths
+    local libpaths, clibpaths = package.libpaths(), package.clibpaths()
     local simple = gsub(name,"%.lua$","")
     local simple = gsub(simple,"%.","/")
-    for i=1,#libpaths do -- package.path, might become option
-        local libpath = libpaths[i]
-        local resolved = gsub(libpath,"?",simple)
-        if trace_locating then -- more detail
-            logs.report("fileio","! checking for '%s' on 'package.path': '%s'",simple,libpath)
-        end
-        if resolvers.isreadable.file(resolved) then
-            if trace_locating then
-                logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved)
-            end
-            return loadfile(resolved)
-        end
+    local resolved = loaded(p_libpaths,name,simple) or loaded(libpaths,name,simple) or loaded(a_libpaths,name,simple)
+    if resolved then
+        return resolved
     end
+    --
     local libname = file.addsuffix(simple,os.libsuffix)
     for i=1,#clibformats do
         -- better have a dedicated loop
diff --git a/tex/context/base/lang-ini.mkiv b/tex/context/base/lang-ini.mkiv
index 86870d635..623af4a2a 100644
--- a/tex/context/base/lang-ini.mkiv
+++ b/tex/context/base/lang-ini.mkiv
@@ -241,8 +241,8 @@
 \setuplanguage
   [\s!default]
   [\c!compoundhyphen=\compoundhyphen,
-   \c!leftcompoundhyphen=\compoundhyphen,
-   \c!rightcompoundhyphen=]
+   \c!rightcompoundhyphen=\compoundhyphen,
+   \c!leftcompoundhyphen=]
 
 %D The values \type {leftsentence} and \type
 %D {rightsentence} can be (and are) used to implement
diff --git a/tex/context/base/lang-mis.mkiv b/tex/context/base/lang-mis.mkiv
index eb7bb1a04..0a9d386da 100644
--- a/tex/context/base/lang-mis.mkiv
+++ b/tex/context/base/lang-mis.mkiv
@@ -377,6 +377,12 @@
 % test|^|test
 % \stop
 
+% x\discretionary{1}{2}{3}xxxxxxx
+% xxxxxxx\discretionary{1}{2}{3}x
+%
+% xxx3xxx
+% xxx1<newline>2xxx
+
 \def\hyphenliketextmodediscretionary#1#2%
   {\ifconditional\spaceafterdiscretionary
      \prewordbreak\hbox{#1}\relax
diff --git a/tex/context/base/lxml-ini.mkiv b/tex/context/base/lxml-ini.mkiv
index f5adb0b7a..d2520d974 100644
--- a/tex/context/base/lxml-ini.mkiv
+++ b/tex/context/base/lxml-ini.mkiv
@@ -100,12 +100,12 @@
 
 % todo: \xmldoifelseattribute
 
-\def\xmldoif             #1#2{\ctxlua{lxml.doif("#1","#2")}}
-\def\xmldoifnot          #1#2{\ctxlua{lxml.doifnot("#1","#2")}}
-\def\xmldoifelse         #1#2{\ctxlua{lxml.doifelse("#1","#2")}}
-\def\xmldoiftext         #1#2{\ctxlua{lxml.doiftext("#1","#2")}}
-\def\xmldoifnottext      #1#2{\ctxlua{lxml.doifnottext("#1","#2")}}
-\def\xmldoifelsetext     #1#2{\ctxlua{lxml.doifelsetext("#1","#2")}}
+\def\xmldoif             #1#2{\ctxlua{lxml.doif        (\!!bs#1\!!es,\!!bs#2\!!es)}}
+\def\xmldoifnot          #1#2{\ctxlua{lxml.doifnot     (\!!bs#1\!!es,\!!bs#2\!!es)}}
+\def\xmldoifelse         #1#2{\ctxlua{lxml.doifelse    (\!!bs#1\!!es,\!!bs#2\!!es)}}
+\def\xmldoiftext         #1#2{\ctxlua{lxml.doiftext    (\!!bs#1\!!es,\!!bs#2\!!es)}}
+\def\xmldoifnottext      #1#2{\ctxlua{lxml.doifnottext (\!!bs#1\!!es,\!!bs#2\!!es)}}
+\def\xmldoifelsetext     #1#2{\ctxlua{lxml.doifelsetext(\!!bs#1\!!es,\!!bs#2\!!es)}}
 
 %def\xmldoifelseempty    #1#2{\ctxlua{lxml.doifelseempty("#1","#2")}} % #2, "*" or "" == self not yet implemented
 %def\xmldoifelseselfempty  #1{\ctxlua{lxml.doifelseempty("#1")}}
diff --git a/tex/context/base/lxml-tab.lua b/tex/context/base/lxml-tab.lua
index 08466665e..bc1963eaf 100644
--- a/tex/context/base/lxml-tab.lua
+++ b/tex/context/base/lxml-tab.lua
@@ -145,7 +145,7 @@ element.</p>
 local nsremap, resolvens = xml.xmlns, xml.resolvens
 
 local stack, top, dt, at, xmlns, errorstr, entities = { }, { }, { }, { }, { }, nil, { }
-local strip, cleanup, utfize, resolve, resolve_predefined = false, false, false, false, false
+local strip, cleanup, utfize, resolve, resolve_predefined, unify_predefined = false, false, false, false, false, false
 local dcache, hcache, acache = { }, { }, { }
 
 local mt = { }
@@ -273,22 +273,72 @@ function xml.unknown_dec_entity_format(str) return (str == "" and "&error;") or
 function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end
 function xml.unknown_any_entity_format(str) return format("&#x%s;",str) end
 
+local function fromhex(s)
+    local n = tonumber(s,16)
+    if n then
+        return utfchar(n)
+    else
+        return format("h:%s",s), true
+    end
+end
+
+local function fromdec(s)
+    local n = tonumber(s)
+    if n then
+        return utfchar(n)
+    else
+        return format("d:%s",s), true
+    end
+end
+
+-- one level expansion (simple case), no checking done
+
+local rest = (1-P(";"))^0
+local many = P(1)^0
+
+local parsedentity =
+    P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) +
+             (P("#x")*(many/fromhex) + P("#")*(many/fromdec))
+
+-- parsing in the xml file
+
+local predefined_unified = {
+    [38] = "&amp;",
+    [42] = "&quot;",
+    [47] = "&apos;",
+    [74] = "&lt;",
+    [76] = "&gr;",
+}
+
+local predefined_simplified = {
+    [38] = "&", amp  = "&",
+    [42] = '"', quot = '"',
+    [47] = "'", apos = "'",
+    [74] = "<", lt   = "<",
+    [76] = ">", gt   = ">",
+}
+
 local function handle_hex_entity(str)
     local h = hcache[str]
     if not h then
-        if utfize then
-            local n = tonumber(str,16)
+        local n = tonumber(str,16)
+        h = unify_predefined and predefined_unified[n]
+        if h then
+            if trace_entities then
+                logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h)
+            end
+        elseif utfize then
             h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or ""
             if not n then
                 logs.report("xml","utfize, ignoring hex entity &#x%s;",str)
             elseif trace_entities then
-                logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,c)
+                logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h)
             end
         else
             if trace_entities then
                 logs.report("xml","found entity &#x%s;",str)
             end
-            h = "&#c" .. str .. ";"
+            h = "&#x" .. str .. ";"
         end
         hcache[str] = h
     end
@@ -298,13 +348,18 @@ end
 local function handle_dec_entity(str)
     local d = dcache[str]
     if not d then
-        if utfize then
-            local n = tonumber(str)
+        local n = tonumber(str)
+        d = unify_predefined and predefined_unified[n]
+        if d then
+            if trace_entities then
+                logs.report("xml","utfize, converting dec entity &#%s; into %s",str,d)
+            end
+        elseif utfize then
             d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or ""
             if not n then
                 logs.report("xml","utfize, ignoring dec entity &#%s;",str)
             elseif trace_entities then
-                logs.report("xml","utfize, converting dec entity &#%s; into %s",str,c)
+                logs.report("xml","utfize, converting dec entity &#%s; into %s",str,h)
             end
         else
             if trace_entities then
@@ -317,48 +372,13 @@ local function handle_dec_entity(str)
     return d
 end
 
--- one level expansion (simple case)
-
-local function fromhex(s)
-    local n = tonumber(s,16)
-    if n then
-        return utfchar(n)
-    else
-        return format("h:%s",s), true
-    end
-end
-
-local function fromdec(s)
-    local n = tonumber(s)
-    if n then
-        return utfchar(n)
-    else
-        return format("d:%s",s), true
-    end
-end
-
-local rest = (1-P(";"))^0
-local many = P(1)^0
-
-local parsedentity =
-    P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) +
-             (P("#x")*(many/fromhex) + P("#")*(many/fromdec))
-
 xml.parsedentitylpeg = parsedentity
 
-local predefined = {
-    amp  = "&",
-    lt   = "<",
-    gt   = ">",
-    quot = '"',
-    apos = "'",
-}
-
 local function handle_any_entity(str)
     if resolve then
         local a = acache[str] -- per instance ! todo
         if not a then
-            a = resolve_predefined and predefined[str]
+            a = resolve_predefined and predefined_simplified[str]
             if a then
                 -- one of the predefined
             elseif type(resolve) == "function" then
@@ -404,7 +424,7 @@ local function handle_any_entity(str)
             if trace_entities then
                 logs.report("xml","found entity &%s;",str)
             end
-            a = resolve_predefined and predefined[str]
+            a = resolve_predefined and predefined_simplified[str]
             if a then
                 -- one of the predefined
                 acache[str] = a
@@ -554,6 +574,7 @@ local function xmlconvert(data, settings)
     utfize = settings.utfize_entities
     resolve = settings.resolve_entities
     resolve_predefined = settings.resolve_predefined_entities -- in case we have escaped entities
+    unify_predefined = settings.unify_predefined_entities -- &#038; -> &amp;
     cleanup = settings.text_cleanup
     stack, top, at, xmlns, errorstr, result, entities = { }, { }, { }, { }, nil, nil, settings.entities or { }
     acache, hcache, dcache = { }, { }, { } -- not stored
@@ -660,21 +681,19 @@ the whole file first. The function accepts a string representing
 a filename or a file handle.</p>
 --ldx]]--
 
-function xml.load(filename)
+function xml.load(filename,settings)
+    local data = ""
     if type(filename) == "string" then
+     -- local data = io.loaddata(filename) - -todo: check type in io.loaddata
         local f = io.open(filename,'r')
         if f then
-            local root = xmlconvert(f:read("*all"))
+            data = f:read("*all")
             f:close()
-            return root
-        else
-            return xmlconvert("")
         end
     elseif filename then -- filehandle
-        return xmlconvert(filename:read("*all"))
-    else
-        return xmlconvert("")
+        data = filename:read("*all")
     end
+    return xmlconvert(data,settings)
 end
 
 --[[ldx--
diff --git a/tex/context/base/lxml-tex.lua b/tex/context/base/lxml-tex.lua
index 754c9eb7d..a41502e1b 100644
--- a/tex/context/base/lxml-tex.lua
+++ b/tex/context/base/lxml-tex.lua
@@ -327,11 +327,11 @@ xml.originalload = xml.originalload or xml.load
 
 local noffiles, nofconverted = 0, 0
 
-function xml.load(filename)
+function xml.load(filename,settings)
     noffiles, nofconverted = noffiles + 1, nofconverted + 1
     starttiming(xml)
     local ok, data = resolvers.loadbinfile(filename)
-    local xmltable = xml.convert((ok and data) or "")
+    local xmltable = xml.convert((ok and data) or "",settings)
     stoptiming(xml)
     return xmltable
 end
@@ -343,7 +343,9 @@ local function entityconverter(id,str)
 end
 
 function lxml.convert(id,data,entities,compress)
-    local settings = { }
+    local settings = {
+        unify_predefined_entities = true,
+    }
     if compress and compress == variables.yes then
         settings.strip_cm_and_dt = true
     end
diff --git a/tex/context/base/m-database.tex b/tex/context/base/m-database.tex
index c4fba132a..02b86cf09 100644
--- a/tex/context/base/m-database.tex
+++ b/tex/context/base/m-database.tex
@@ -85,32 +85,14 @@
    \appendseparatedlistcontent{#1}%
    \appendseparatedlistparameter\c!right}
 
+% a version more robust with regard to {a a} b c d situations:
 
-% % % so far
-
-% \def\dodefineprocessseplist#1#2% separator \docommand
-%   {\def\dodoprocessseplist##1##2#1%
-%      {\ifx\relax##1%
-%         \expandafter\nodoprocessseplist
-%       \else\ifx##1#1%
-%         #2{}%
-%         #2{##2}%
-%         \expandafter\expandafter\expandafter\dodoprocessseplist
-%       \else
-%         #2{##1##2}%
-%         \expandafter\expandafter\expandafter\dodoprocessseplist
-%       \fi\fi}%
-%    \def\doprocessseplist##1\relax
-%      {\dodoprocessseplist##1#1\relax#1\relax\relax\end}}
-%
-% a version more robust with regard to {a a} b c d situations
-
-\def\edef@relax{\relax}
+\edef\detokenizedrelax{\detokenize{\relax}}
 
 \def\dodefineprocessseplist#1#2% separator \docommand
   {\def\dodoprocessseplist##1##2#1%
-     {\edef\!!stringa{##1}%
-      \ifx\edef@relax\!!stringa
+     {\edef\!!stringa{\detokenize{##1}}%
+      \ifx\detokenizedrelax\!!stringa
         \expandafter\nodoprocessseplist
       \else\ifx\!!stringa#1%
         #2{}%
@@ -404,4 +386,15 @@ a,b,c
 d,e,f
 \stopseparatedlist
 
+\defineseparatedlist[CSV]
+  [separator=comma,
+   before=\bTABLE, after=\eTABLE,
+   first=\bTR, last=\eTR,
+   left=\bTD, right=\eTD]
+
+\startCSV
+a,b,c,č
+d,e,f,š
+\stopCSV
+
 \stoptext
diff --git a/tex/context/base/node-typ.lua b/tex/context/base/node-typ.lua
index 2562378d1..5ab6b6975 100644
--- a/tex/context/base/node-typ.lua
+++ b/tex/context/base/node-typ.lua
@@ -6,6 +6,8 @@ if not modules then modules = { } end modules ['node-typ'] = {
     license   = "see context related readme files"
 }
 
+-- this will be replaced by blob-ini cum suis so typesetting will go away
+
 local utfvalues = string.utfvalues
 
 local newglyph = nodes.glyph
@@ -15,7 +17,7 @@ local hpack, vpack = node.hpack, node.vpack
 
 typesetting = typesetting or { }
 
-local function tonodes(str,fontid,spacing)
+local function tonodes(str,fontid,spacing) -- don't use this
     local head, prev = nil, nil
     for s in utfvalues(str) do
         local next
diff --git a/tex/context/base/typo-brk.lua b/tex/context/base/typo-brk.lua
index 5cab6a667..8f7a00f9d 100644
--- a/tex/context/base/typo-brk.lua
+++ b/tex/context/base/typo-brk.lua
@@ -9,21 +9,27 @@ if not modules then modules = { } end modules ['typo-brk'] = {
 -- this code dates from the beginning and is kind of experimental; it
 -- will be optimized and improved soon
 
-local next, type = next, type
+local next, type, tonumber = next, type, tonumber
+local utfbyte, utfchar = utf.byte, utf.char
 local format = string.format
 
+local settings_to_array  = aux.settings_to_array
 local has_attribute      = node.has_attribute
 local unset_attribute    = node.unset_attribute
 local set_attribute      = node.set_attribute
 local copy_node          = node.copy
+local free_node          = node.free
 local insert_node_before = node.insert_before
 local insert_node_after  = node.insert_after
 local make_penalty_node  = nodes.penalty
 local make_glue_node     = nodes.glue
 local make_disc_node     = nodes.disc
+local make_glyph_node    = nodes.glyph
+local remove_node        = nodes.remove -- ! nodes
+local tonodes            = blobs.tonodes
 
-local glyph   = node.id("glyph")
-local kern    = node.id("kern")
+local glyph = node.id("glyph")
+local kern  = node.id("kern")
 
 breakpoints           = breakpoints         or { }
 breakpoints.mapping   = breakpoints.mapping or { }
@@ -34,7 +40,8 @@ storage.register("breakpoints/mapping", breakpoints.mapping, "breakpoints.mappin
 
 local mapping = breakpoints.mapping
 
-function breakpoints.setreplacement(id,char,kind,before,after,language)
+function breakpoints.setreplacement(id,char,language,settings)
+    char = utfbyte(char)
     local map = mapping[id]
     if not map then
         map = { }
@@ -45,71 +52,76 @@ function breakpoints.setreplacement(id,char,kind,before,after,language)
         cmap = { }
         map[char] = cmap
     end
-    cmap[language or ""] = { kind or 1, before or 1, after or 1 }
+    local left, right, middle = settings.left, settings.right, settings.middle
+    cmap[language or ""] = {
+        kind   = tonumber(settings.kind)   or 1,
+        nleft  = tonumber(settings.nleft)  or 1,
+        nright = tonumber(settings.nright) or 1,
+        left   = left   ~= "" and left     or nil,
+        right  = right  ~= "" and right    or nil,
+        middle = middle ~= "" and middle   or nil,
+    } -- was { kind or 1, before or 1, after or 1 }
+end
+
+local function insert_break(head,start,before,after)
+    insert_node_before(head,start,make_penalty_node(before))
+    insert_node_before(head,start,make_glue_node(0))
+    insert_node_after(head,start,make_glue_node(0))
+    insert_node_after(head,start,make_penalty_node(after))
 end
 
 breakpoints.methods[1] = function(head,start)
     if start.prev and start.next then
-        insert_node_before(head,start,make_penalty_node(10000))
-        insert_node_before(head,start,make_glue_node(0))
-        insert_node_after(head,start,make_glue_node(0))
-        insert_node_after(head,start,make_penalty_node(0))
+        insert_break(head,start,10000,0)
     end
     return head, start
 end
 breakpoints.methods[2] = function(head,start) -- ( => (-
     if start.prev and start.next then
-        local tmp = start
-        start = make_disc_node()
-        start.prev, start.next = tmp.prev, tmp.next
-        tmp.prev.next, tmp.next.prev = start, start
-        tmp.prev, tmp.next = nil, nil
+        local tmp
+        head, start, tmp = remove_node(head,start)
+        head, start = insert_node_before(head,start,make_disc_node())
         start.replace = tmp
         local tmp, hyphen = copy_node(tmp), copy_node(tmp)
         hyphen.char = languages.prehyphenchar(tmp.lang)
         tmp.next, hyphen.prev = hyphen, tmp
         start.post = tmp
-        insert_node_before(head,start,make_penalty_node(10000))
-        insert_node_before(head,start,make_glue_node(0))
-        insert_node_after(head,start,make_glue_node(0))
-        insert_node_after(head,start,make_penalty_node(10000))
+        insert_break(head,start,10000,10000)
     end
     return head, start
 end
 breakpoints.methods[3] = function(head,start) -- ) => -)
     if start.prev and start.next then
-        local tmp = start
-        start = make_disc_node()
-        start.prev, start.next = tmp.prev, tmp.next
-        tmp.prev.next, tmp.next.prev = start, start
-        tmp.prev, tmp.next = nil, nil
+        local tmp
+        head, start, tmp = remove_node(head,start)
+        head, start = insert_node_before(head,start,make_disc_node())
         start.replace = tmp
         local tmp, hyphen = copy_node(tmp), copy_node(tmp)
         hyphen.char = languages.prehyphenchar(tmp.lang)
         tmp.prev, hyphen.next = hyphen, tmp
         start.pre = hyphen
-        insert_node_before(head,start,make_penalty_node(10000))
-        insert_node_before(head,start,make_glue_node(0))
-        insert_node_after(head,start,make_glue_node(0))
-        insert_node_after(head,start,make_penalty_node(10000))
+        insert_break(head,start,10000,10000)
     end
     return head, start
 end
 breakpoints.methods[4] = function(head,start) -- - => - - -
     if start.prev and start.next then
-        local tmp = start
-        start = make_disc_node()
-        start.prev, start.next = tmp.prev, tmp.next
-        tmp.prev.next, tmp.next.prev = start, start
-        tmp.prev, tmp.next = nil, nil
-        -- maybe prehyphenchar etc
-        start.pre = copy_node(tmp)
-        start.post = copy_node(tmp)
-        start.replace = tmp
-        insert_node_before(head,start,make_penalty_node(10000))
-        insert_node_before(head,start,make_glue_node(0))
-        insert_node_after(head,start,make_glue_node(0))
-        insert_node_after(head,start,make_penalty_node(10000))
+        local tmp
+        head, start, tmp = remove_node(head,start)
+        head, start = insert_node_before(head,start,make_disc_node())
+        start.pre, start.post, start.replace = copy_node(tmp), copy_node(tmp), tmp
+        insert_break(head,start,10000,10000)
+    end
+    return head, start
+end
+breakpoints.methods[5] = function(head,start,settings) -- x => p q r
+    if start.prev and start.next then
+        local tmp
+        head, start, tmp = remove_node(head,start)
+        head, start = insert_node_before(head,start,make_disc_node())
+        start.pre, start.post, start.replace = tonodes(settings.right,tmp), tonodes(settings.left,tmp), tonodes(settings.middle,tmp)
+        free_node(tmp)
+        insert_break(head,start,10000,10000)
     end
     return head, start
 end
@@ -134,18 +146,18 @@ function breakpoints.process(namespace,attribute,head)
                         -- we do a sanity check for language
                         local smap = lang and lang >= 0 and lang < 0x7FFF and (cmap[numbers[lang]] or cmap[""])
                         if smap then
-                            if n >= smap[2] then
-                                local m = smap[3]
+                            if n >= smap.nleft then
+                                local m = smap.nright
                                 local next = start.next
-                                while next do -- gamble on same attribute
+                                while next do -- gamble on same attribute (not that important actually)
                                     local id = next.id
-                                    if id == glyph then -- gamble on same attribute
+                                    if id == glyph then -- gamble on same attribute (not that important actually)
                                         if map[next.char] then
                                             break
                                         elseif m == 1 then
-                                            local method = methods[smap[1]]
+                                            local method = methods[smap.kind]
                                             if method then
-                                                head, start = method(head,start)
+                                                head, start = method(head,start,smap)
                                                 done = true
                                             end
                                             break
@@ -172,6 +184,8 @@ function breakpoints.process(namespace,attribute,head)
                 else
                     n = 0
                 end
+            else
+             -- n = n + 1 -- if we want single char handling (|-|) then we will use grouping and then we need this
             end
         elseif id == kern and start.subtype == 0 then
             -- ignore intercharacter kerning, will go way
diff --git a/tex/context/base/typo-brk.mkiv b/tex/context/base/typo-brk.mkiv
index e63232087..f2a44fee0 100644
--- a/tex/context/base/typo-brk.mkiv
+++ b/tex/context/base/typo-brk.mkiv
@@ -23,9 +23,7 @@
 
 \newbox\breakpointbox
 
-\definesystemvariable {bp}   % BreakPoint
-
-\exhyphenchar=\minusone % we use a different order then base tex, so we really need this
+\exhyphenchar\minusone % we use a different order then base tex, so we really need this
 
 \newcount \maxbreakpointsid
 
@@ -42,12 +40,23 @@
   {\dotripleempty\doinstallbreakpoint}
 
 % hm, we cannot prebuild lists, font dependent
+%
+% -- type nleft nright language left right middle
+%
+% -- we might eventually stick to only method 5
 
 \def\doinstallbreakpoint[#1][#2][#3]%
   {\ifcsname\??bp:#1\endcsname
      \begingroup
-     \getparameters[\??bp][\c!type=1,\c!nleft=3,\c!nright=3,\s!language=,#3]%
-     \ctxlua{breakpoints.setreplacement(\csname\??bp:#1\endcsname,#2,\@@bptype,\@@bpnleft,\@@bpnright,"\@@bplanguage")}%
+     \getparameters[\??bp][\c!type=1,\c!nleft=3,\c!nright=3,\s!language=,\c!left=,\c!right=,\c!middle=,#3]%
+     \ctxlua{breakpoints.setreplacement(\csname\??bp:#1\endcsname, "#2", "\@@bplanguage", {
+            kind =  \@@bptype,
+           nleft = "\@@bpnleft",
+          nright = "\@@bpnright",
+           right = "\@@bpright",
+            left = "\@@bpleft",
+          middle = "\@@bpmiddle"
+     } )}% todo: table
      \endgroup
    \fi}
 
@@ -60,14 +69,21 @@
 
 \definebreakpoints[compound]
 
-\installbreakpoint [compound] [\number`+] [\c!left=3,\c!right=3,\c!type=1]
-\installbreakpoint [compound] [\number`-] [\c!left=3,\c!right=3,\c!type=1]
-\installbreakpoint [compound] [\number`/] [\c!left=3,\c!right=3,\c!type=1]
-\installbreakpoint [compound] [\number`(] [\c!left=3,\c!right=3,\c!type=2]
-\installbreakpoint [compound] [\number`)] [\c!left=3,\c!right=3,\c!type=3]
+\installbreakpoint [compound] [+] [\c!nleft=3,\c!nright=3,\c!type=1] % middle=+,left=,right=
+\installbreakpoint [compound] [-] [\c!nleft=3,\c!nright=3,\c!type=1] % middle=+,left=,right=
+\installbreakpoint [compound] [/] [\c!nleft=3,\c!nright=3,\c!type=1] % middle=+,left=,right=
+\installbreakpoint [compound] [(] [\c!nleft=3,\c!nright=3,\c!type=2] % type=5,middle=(,left=(-,right=
+\installbreakpoint [compound] [)] [\c!nleft=3,\c!nright=3,\c!type=3] % type=5,middle=),left=,right=-)
+
+% \start \hsize 1.5cm \setbreakpoints[compound]
+%     \installbreakpoint [compound] [-] [nleft=3,nright=3,type=5,left=,right={-},middle={-}]
+%     \installbreakpoint [compound] [(] [nleft=3,nright=3,type=5,left=,right={(-},middle={(}]
+%     \installbreakpoint [compound] [)] [nleft=3,nright=3,type=5,left={-)},right=,middle={)}]
+%     composed-word\par composed(word)\par
+% \stop
 
 % \mainlanguage[czech]
-% \installbreakpoint [compound] [\number`-] [language=cs,left=3,right=3,type=4]
+% \installbreakpoint [compound] [\number`-] [language=cs,nleft=3,nright=3,type=4]
 % \setbreakpoints[compound]
 % \start \hsize 1mm test-test \par \stop
 
diff --git a/tex/generic/context/luatex-fonts-merged.lua b/tex/generic/context/luatex-fonts-merged.lua
index 4a1430553..c1f4fc457 100644
--- a/tex/generic/context/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 04/16/10 21:08:09
+-- merge date  : 04/23/10 18:34:16
 
 do -- begin closure to overcome local limits and interference