From de436da91fb6ac8bf2c4f729c86289bc7914676a Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Fri, 23 Apr 2010 18:34:00 +0200 Subject: beta 2010.04.23 18:34 --- tex/context/base/blob-ini.lua | 165 ++++++++++++++++++++++++++++ tex/context/base/blob-ini.mkiv | 34 ++++++ tex/context/base/cont-new.tex | 2 +- tex/context/base/context.mkiv | 2 +- tex/context/base/context.tex | 2 +- tex/context/base/core-mis.mkiv | 6 +- tex/context/base/data-lua.lua | 63 ++++++++--- tex/context/base/lang-ini.mkiv | 4 +- tex/context/base/lang-mis.mkiv | 6 + tex/context/base/lxml-ini.mkiv | 12 +- tex/context/base/lxml-tab.lua | 125 ++++++++++++--------- tex/context/base/lxml-tex.lua | 8 +- tex/context/base/m-database.tex | 37 +++---- tex/context/base/node-typ.lua | 4 +- tex/context/base/typo-brk.lua | 106 ++++++++++-------- tex/context/base/typo-brk.mkiv | 38 +++++-- tex/generic/context/luatex-fonts-merged.lua | 2 +- 17 files changed, 448 insertions(+), 168 deletions(-) create mode 100644 tex/context/base/blob-ini.lua create mode 100644 tex/context/base/blob-ini.mkiv (limited to 'tex') diff --git a/tex/context/base/blob-ini.lua b/tex/context/base/blob-ini.lua new file mode 100644 index 000000000..0f7ccee26 --- /dev/null +++ b/tex/context/base/blob-ini.lua @@ -0,0 +1,165 @@ +if not modules then modules = { } end modules ['blob-ini'] = { + version = 1.001, + comment = "companion to blob-ini.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- later we will consider an OO variant. + +-- This module is just a playground. Occasionally we need to typeset +-- at the lua and and this is one method. In principle we can construct +-- pages this way too which sometimes makes sense in dumb cases. Actually, +-- if one only needs this, one does not really need tex, okay maybe the +-- parbuilder but that one can be simplified as well then. + +-- set fonts, attributes +-- rest already done in packers etc +-- add local par whatsit (or wait till cleaned up) +-- collapse or new pars +-- interline spacing etc + +-- DON'T USE THESE FUNCTIONS AS THEY WILL CHANGE! + +local type = type + +local utfvalues = string.utfvalues +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns + +local fontdata = fonts.identifiers + +local new_glyph_node = nodes.glyph +local new_glue_node = nodes.glyph + +local copy_node = node.copy +local copy_node_list = node.copy_list +local insert_node_after = node.insert_after +local flush_node_list = node.flush_list +local hpack_node_list = node.hpack +local vpack_node_list = node.vpack +local write_node = node.write + +local current_font = font.current + +blobs = blobs or { } + +local newline = lpegpatterns.newline +local space = lpegpatterns.spacer +local spacing = newline * space^0 +local content = (space^1)/" " + (1-spacing) + +local ctxtextcapture = lpeg.Ct ( ( + space^0 * ( + newline^2 * space^0 * lpeg.Cc("") + + newline * space^0 * lpeg.Cc(" ") + + lpeg.Cs(content^1) + ) +)^0) + +local function tonodes(str,fnt,attr) -- (str,template_glyph) + if not str or str == "" then + return + end + local head, tail, space, fnt, template = nil, nil, nil, nil, nil + if not fnt then + fnt = current_font() + elseif type(fnt) ~= "number" and fnt.id == "glyph" then + fnt, template = nil, fnt + -- else + -- already a number + end + for s in utfvalues(str) do + local n + if s == 32 then + if not space then + local parameters = fontdata[fnt].parameters + space = new_glue_node(parameters.space,parameters.space_stretch,parameters.space_shrink) + n = space + else + n = copy_node(space) + end + elseif template then + n = copy_node(template) + n.char = s + else + n = new_glyph_node(fnt,s) + end + if attr then -- normall false when template + n.attr = copy_node_list(attr) + end + if head then + insert_node_after(head,tail,n) + else + head = n + end + tail = n + end + return head, tail +end + +blobs.tonodes = tonodes + +function blobs.new() + return { + list = { }, + } +end + +function blobs.append(t,str) + local kind = type(str) + local dummy = nil + if kind == "string" then + local pars = lpegmatch(ctxtextcapture,str) + local list = t.list + for p=1,#pars do + local str = pars[p] + if #str == 0 then + list[#list+1 ] = { head = nil, tail = nil } + else + local l = list[#list] + if not l then + l = { head = nil, tail = nil } + list[#list+1 ] = l + end + local head, tail = tonodes(str,nil,nil) + if head then + if l.head then + l.tail.next = head + head.prev = l.tail + l.tail = tail + else + l.head, l.tail = head, tail + end + end + end + end + end +end + +function blobs.pack(t,how) + local list = t.list + for i=1,#list do + local pack = list[i].pack + if pack then + flush_node_list(node.pack) + end + if how == "vertical" then + -- we need to prepend a local par node + -- list[i].pack = node.vpack(list[i].head,"exactly") + logs.report("blobs","vpack not yet supported") + else + list[i].pack = hpack_node_list(list[i].head,"exactly") + end + end +end + +function blobs.write(t) + local list = t.list + for i=1,#list do + local pack = list[i].pack + if pack then + write_node(pack) + end + end +end diff --git a/tex/context/base/blob-ini.mkiv b/tex/context/base/blob-ini.mkiv new file mode 100644 index 000000000..7f63ec73d --- /dev/null +++ b/tex/context/base/blob-ini.mkiv @@ -0,0 +1,34 @@ +%D \module +%D [ file=blob-ini, +%D version=2010.04.06, +%D title=\CONTEXT\ \LUA\ Typesetting, +%D subtitle=Initialization, +%D author=Hans Hagen, +%D date=\currentdate, +%D copyright=\PRAGMA] +%C +%C This module is part of the \CONTEXT\ macro||package and is +%C therefore copyrighted by \PRAGMA. See mreadme.pdf for +%C details. + +\writestatus{loading}{ConTeXt Lua Typesetting / Initialization} + +%D This is a prelude to typesetting at the \LUA\ end. The code +%D is already quite old but will only get nice when we are further +%D down the road (close to version 1.00 of \LUATEX). Typesetting in +%D pure \LUA\ sometimes makes sense. + +\registerctxluafile{blob-ini}{1.001} + +\endinput + +% \starttext +% +% \startluacode +% local b = blobs.new() +% blobs.append(b,"Hello world.\n Here we are.\n\n And Again!") +% blobs.pack(b) +% blobs.write(b) +% \stopluacode +% +% \stoptext diff --git a/tex/context/base/cont-new.tex b/tex/context/base/cont-new.tex index 9999d79bd..79fe0c555 100644 --- a/tex/context/base/cont-new.tex +++ b/tex/context/base/cont-new.tex @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2010.04.16 21:08} +\newcontextversion{2010.04.23 18:34} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv index 8b533a34d..13fe9f2cc 100644 --- a/tex/context/base/context.mkiv +++ b/tex/context/base/context.mkiv @@ -89,7 +89,7 @@ \loadmarkfile{trac-lmx} \loadmarkfile{trac-deb} -%loadmarkfile{blob-ini} % not yet public (typesetting in pure lua) +\loadmarkfile{blob-ini} % not to be used, we only use a helper \loadcorefile{supp-box} diff --git a/tex/context/base/context.tex b/tex/context/base/context.tex index bba0807fb..188ff748e 100644 --- a/tex/context/base/context.tex +++ b/tex/context/base/context.tex @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2010.04.16 21:08} +\edef\contextversion{2010.04.23 18:34} %D For those who want to use this: diff --git a/tex/context/base/core-mis.mkiv b/tex/context/base/core-mis.mkiv index 00535e586..28ce8406b 100644 --- a/tex/context/base/core-mis.mkiv +++ b/tex/context/base/core-mis.mkiv @@ -623,7 +623,7 @@ \def\setuphyphenmark[#1]% sign=normal|wide {\dodoubleargument\getparameters[\??kp][#1]% - \doifinsetelse\@@kpsign {\v!normal} + \doifelse\@@kpsign {\v!normal}% was inset? {\let\textmodehyphen\normalhyphen \let\textmodehyphendiscretionary\normalhyphendiscretionary} {\let\textmodehyphen\composedhyphen\let\textmodehyphendiscretionary\composedhyphendiscretionary}} @@ -642,14 +642,14 @@ \def\normalhyphendiscretionary {\discretionary - {\hbox{\directsymbol\empty\c!lefthyphen}} {\hbox{\directsymbol\empty\c!righthyphen}} + {\hbox{\directsymbol\empty\c!lefthyphen}} {\hbox{\directsymbol\empty\c!hyphen}}} \def\composedhyphendiscretionary {\discretionary - {\hbox{\directsymbol\empty\c!leftcompoundhyphen}} {\hbox{\directsymbol\empty\c!rightcompoundhyphen}} + {\hbox{\directsymbol\empty\c!leftcompoundhyphen}} {\hbox{\directsymbol\empty\c!compoundhyphen}}} \let\textmodehyphen \composedhyphen diff --git a/tex/context/base/data-lua.lua b/tex/context/base/data-lua.lua index af09bbe59..57a434d1d 100644 --- a/tex/context/base/data-lua.lua +++ b/tex/context/base/data-lua.lua @@ -12,13 +12,29 @@ if not modules then modules = { } end modules ['data-lua'] = { local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local gsub = string.gsub +local gsub, insert = string.gsub, table.insert local unpack = unpack or table.unpack local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs' local clibformats = { 'lib' } -local libpaths = file.split_path(package.path) -local clibpaths = file.split_path(package.cpath) + +local _path_, libpaths, _cpath_, clibpaths + +function package.libpaths() + if not _path_ or package.path ~= _path_ then + _path_ = package.path + libpaths = file.split_path(_path_) + end + return libpaths +end + +function package.clibpaths() + if not _cpath_ or package.cpath ~= _cpath_ then + _cpath_ = package.cpath + clibpaths = file.split_path(_cpath_) + end + return clibpaths +end local function thepath(...) local t = { ... } t[#t+1] = "?.lua" @@ -29,16 +45,35 @@ local function thepath(...) return path end +local p_libpaths, a_libpaths = { }, { } + function package.append_libpath(...) - table.insert(libpaths,thepath(...)) + insert(a_libpath,thepath(...)) end function package.prepend_libpath(...) - table.insert(libpaths,1,thepath(...)) + insert(p_libpaths,1,thepath(...)) end -- beware, we need to return a loadfile result ! +local function loaded(libpaths,name,simple) + for i=1,#libpaths do -- package.path, might become option + local libpath = libpaths[i] + local resolved = gsub(libpath,"%?",simple) + if trace_locating then -- more detail + logs.report("fileio","! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved) + end + return loadfile(resolved) + end + end +end + + package.loaders[2] = function(name) -- was [#package.loaders+1] if trace_locating then -- mode detail logs.report("fileio","! locating '%s'",name) @@ -56,21 +91,15 @@ package.loaders[2] = function(name) -- was [#package.loaders+1] return loadfile(resolved) end end + -- libpaths + local libpaths, clibpaths = package.libpaths(), package.clibpaths() local simple = gsub(name,"%.lua$","") local simple = gsub(simple,"%.","/") - for i=1,#libpaths do -- package.path, might become option - local libpath = libpaths[i] - local resolved = gsub(libpath,"?",simple) - if trace_locating then -- more detail - logs.report("fileio","! checking for '%s' on 'package.path': '%s'",simple,libpath) - end - if resolvers.isreadable.file(resolved) then - if trace_locating then - logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved) - end - return loadfile(resolved) - end + local resolved = loaded(p_libpaths,name,simple) or loaded(libpaths,name,simple) or loaded(a_libpaths,name,simple) + if resolved then + return resolved end + -- local libname = file.addsuffix(simple,os.libsuffix) for i=1,#clibformats do -- better have a dedicated loop diff --git a/tex/context/base/lang-ini.mkiv b/tex/context/base/lang-ini.mkiv index 86870d635..623af4a2a 100644 --- a/tex/context/base/lang-ini.mkiv +++ b/tex/context/base/lang-ini.mkiv @@ -241,8 +241,8 @@ \setuplanguage [\s!default] [\c!compoundhyphen=\compoundhyphen, - \c!leftcompoundhyphen=\compoundhyphen, - \c!rightcompoundhyphen=] + \c!rightcompoundhyphen=\compoundhyphen, + \c!leftcompoundhyphen=] %D The values \type {leftsentence} and \type %D {rightsentence} can be (and are) used to implement diff --git a/tex/context/base/lang-mis.mkiv b/tex/context/base/lang-mis.mkiv index eb7bb1a04..0a9d386da 100644 --- a/tex/context/base/lang-mis.mkiv +++ b/tex/context/base/lang-mis.mkiv @@ -377,6 +377,12 @@ % test|^|test % \stop +% x\discretionary{1}{2}{3}xxxxxxx +% xxxxxxx\discretionary{1}{2}{3}x +% +% xxx3xxx +% xxx12xxx + \def\hyphenliketextmodediscretionary#1#2% {\ifconditional\spaceafterdiscretionary \prewordbreak\hbox{#1}\relax diff --git a/tex/context/base/lxml-ini.mkiv b/tex/context/base/lxml-ini.mkiv index f5adb0b7a..d2520d974 100644 --- a/tex/context/base/lxml-ini.mkiv +++ b/tex/context/base/lxml-ini.mkiv @@ -100,12 +100,12 @@ % todo: \xmldoifelseattribute -\def\xmldoif #1#2{\ctxlua{lxml.doif("#1","#2")}} -\def\xmldoifnot #1#2{\ctxlua{lxml.doifnot("#1","#2")}} -\def\xmldoifelse #1#2{\ctxlua{lxml.doifelse("#1","#2")}} -\def\xmldoiftext #1#2{\ctxlua{lxml.doiftext("#1","#2")}} -\def\xmldoifnottext #1#2{\ctxlua{lxml.doifnottext("#1","#2")}} -\def\xmldoifelsetext #1#2{\ctxlua{lxml.doifelsetext("#1","#2")}} +\def\xmldoif #1#2{\ctxlua{lxml.doif (\!!bs#1\!!es,\!!bs#2\!!es)}} +\def\xmldoifnot #1#2{\ctxlua{lxml.doifnot (\!!bs#1\!!es,\!!bs#2\!!es)}} +\def\xmldoifelse #1#2{\ctxlua{lxml.doifelse (\!!bs#1\!!es,\!!bs#2\!!es)}} +\def\xmldoiftext #1#2{\ctxlua{lxml.doiftext (\!!bs#1\!!es,\!!bs#2\!!es)}} +\def\xmldoifnottext #1#2{\ctxlua{lxml.doifnottext (\!!bs#1\!!es,\!!bs#2\!!es)}} +\def\xmldoifelsetext #1#2{\ctxlua{lxml.doifelsetext(\!!bs#1\!!es,\!!bs#2\!!es)}} %def\xmldoifelseempty #1#2{\ctxlua{lxml.doifelseempty("#1","#2")}} % #2, "*" or "" == self not yet implemented %def\xmldoifelseselfempty #1{\ctxlua{lxml.doifelseempty("#1")}} diff --git a/tex/context/base/lxml-tab.lua b/tex/context/base/lxml-tab.lua index 08466665e..bc1963eaf 100644 --- a/tex/context/base/lxml-tab.lua +++ b/tex/context/base/lxml-tab.lua @@ -145,7 +145,7 @@ element.

local nsremap, resolvens = xml.xmlns, xml.resolvens local stack, top, dt, at, xmlns, errorstr, entities = { }, { }, { }, { }, { }, nil, { } -local strip, cleanup, utfize, resolve, resolve_predefined = false, false, false, false, false +local strip, cleanup, utfize, resolve, resolve_predefined, unify_predefined = false, false, false, false, false, false local dcache, hcache, acache = { }, { }, { } local mt = { } @@ -273,22 +273,72 @@ function xml.unknown_dec_entity_format(str) return (str == "" and "&error;") or function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end function xml.unknown_any_entity_format(str) return format("&#x%s;",str) end +local function fromhex(s) + local n = tonumber(s,16) + if n then + return utfchar(n) + else + return format("h:%s",s), true + end +end + +local function fromdec(s) + local n = tonumber(s) + if n then + return utfchar(n) + else + return format("d:%s",s), true + end +end + +-- one level expansion (simple case), no checking done + +local rest = (1-P(";"))^0 +local many = P(1)^0 + +local parsedentity = + P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + + (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) + +-- parsing in the xml file + +local predefined_unified = { + [38] = "&", + [42] = """, + [47] = "'", + [74] = "<", + [76] = "&gr;", +} + +local predefined_simplified = { + [38] = "&", amp = "&", + [42] = '"', quot = '"', + [47] = "'", apos = "'", + [74] = "<", lt = "<", + [76] = ">", gt = ">", +} + local function handle_hex_entity(str) local h = hcache[str] if not h then - if utfize then - local n = tonumber(str,16) + local n = tonumber(str,16) + h = unify_predefined and predefined_unified[n] + if h then + if trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + end + elseif utfize then h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or "" if not n then logs.report("xml","utfize, ignoring hex entity &#x%s;",str) elseif trace_entities then - logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,c) + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) end else if trace_entities then logs.report("xml","found entity &#x%s;",str) end - h = "&#c" .. str .. ";" + h = "&#x" .. str .. ";" end hcache[str] = h end @@ -298,13 +348,18 @@ end local function handle_dec_entity(str) local d = dcache[str] if not d then - if utfize then - local n = tonumber(str) + local n = tonumber(str) + d = unify_predefined and predefined_unified[n] + if d then + if trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,d) + end + elseif utfize then d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or "" if not n then logs.report("xml","utfize, ignoring dec entity &#%s;",str) elseif trace_entities then - logs.report("xml","utfize, converting dec entity &#%s; into %s",str,c) + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,h) end else if trace_entities then @@ -317,48 +372,13 @@ local function handle_dec_entity(str) return d end --- one level expansion (simple case) - -local function fromhex(s) - local n = tonumber(s,16) - if n then - return utfchar(n) - else - return format("h:%s",s), true - end -end - -local function fromdec(s) - local n = tonumber(s) - if n then - return utfchar(n) - else - return format("d:%s",s), true - end -end - -local rest = (1-P(";"))^0 -local many = P(1)^0 - -local parsedentity = - P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + - (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) - xml.parsedentitylpeg = parsedentity -local predefined = { - amp = "&", - lt = "<", - gt = ">", - quot = '"', - apos = "'", -} - local function handle_any_entity(str) if resolve then local a = acache[str] -- per instance ! todo if not a then - a = resolve_predefined and predefined[str] + a = resolve_predefined and predefined_simplified[str] if a then -- one of the predefined elseif type(resolve) == "function" then @@ -404,7 +424,7 @@ local function handle_any_entity(str) if trace_entities then logs.report("xml","found entity &%s;",str) end - a = resolve_predefined and predefined[str] + a = resolve_predefined and predefined_simplified[str] if a then -- one of the predefined acache[str] = a @@ -554,6 +574,7 @@ local function xmlconvert(data, settings) utfize = settings.utfize_entities resolve = settings.resolve_entities resolve_predefined = settings.resolve_predefined_entities -- in case we have escaped entities + unify_predefined = settings.unify_predefined_entities -- & -> & cleanup = settings.text_cleanup stack, top, at, xmlns, errorstr, result, entities = { }, { }, { }, { }, nil, nil, settings.entities or { } acache, hcache, dcache = { }, { }, { } -- not stored @@ -660,21 +681,19 @@ the whole file first. The function accepts a string representing a filename or a file handle.

--ldx]]-- -function xml.load(filename) +function xml.load(filename,settings) + local data = "" if type(filename) == "string" then + -- local data = io.loaddata(filename) - -todo: check type in io.loaddata local f = io.open(filename,'r') if f then - local root = xmlconvert(f:read("*all")) + data = f:read("*all") f:close() - return root - else - return xmlconvert("") end elseif filename then -- filehandle - return xmlconvert(filename:read("*all")) - else - return xmlconvert("") + data = filename:read("*all") end + return xmlconvert(data,settings) end --[[ldx-- diff --git a/tex/context/base/lxml-tex.lua b/tex/context/base/lxml-tex.lua index 754c9eb7d..a41502e1b 100644 --- a/tex/context/base/lxml-tex.lua +++ b/tex/context/base/lxml-tex.lua @@ -327,11 +327,11 @@ xml.originalload = xml.originalload or xml.load local noffiles, nofconverted = 0, 0 -function xml.load(filename) +function xml.load(filename,settings) noffiles, nofconverted = noffiles + 1, nofconverted + 1 starttiming(xml) local ok, data = resolvers.loadbinfile(filename) - local xmltable = xml.convert((ok and data) or "") + local xmltable = xml.convert((ok and data) or "",settings) stoptiming(xml) return xmltable end @@ -343,7 +343,9 @@ local function entityconverter(id,str) end function lxml.convert(id,data,entities,compress) - local settings = { } + local settings = { + unify_predefined_entities = true, + } if compress and compress == variables.yes then settings.strip_cm_and_dt = true end diff --git a/tex/context/base/m-database.tex b/tex/context/base/m-database.tex index c4fba132a..02b86cf09 100644 --- a/tex/context/base/m-database.tex +++ b/tex/context/base/m-database.tex @@ -85,32 +85,14 @@ \appendseparatedlistcontent{#1}% \appendseparatedlistparameter\c!right} +% a version more robust with regard to {a a} b c d situations: -% % % so far - -% \def\dodefineprocessseplist#1#2% separator \docommand -% {\def\dodoprocessseplist##1##2#1% -% {\ifx\relax##1% -% \expandafter\nodoprocessseplist -% \else\ifx##1#1% -% #2{}% -% #2{##2}% -% \expandafter\expandafter\expandafter\dodoprocessseplist -% \else -% #2{##1##2}% -% \expandafter\expandafter\expandafter\dodoprocessseplist -% \fi\fi}% -% \def\doprocessseplist##1\relax -% {\dodoprocessseplist##1#1\relax#1\relax\relax\end}} -% -% a version more robust with regard to {a a} b c d situations - -\def\edef@relax{\relax} +\edef\detokenizedrelax{\detokenize{\relax}} \def\dodefineprocessseplist#1#2% separator \docommand {\def\dodoprocessseplist##1##2#1% - {\edef\!!stringa{##1}% - \ifx\edef@relax\!!stringa + {\edef\!!stringa{\detokenize{##1}}% + \ifx\detokenizedrelax\!!stringa \expandafter\nodoprocessseplist \else\ifx\!!stringa#1% #2{}% @@ -404,4 +386,15 @@ a,b,c d,e,f \stopseparatedlist +\defineseparatedlist[CSV] + [separator=comma, + before=\bTABLE, after=\eTABLE, + first=\bTR, last=\eTR, + left=\bTD, right=\eTD] + +\startCSV +a,b,c,č +d,e,f,š +\stopCSV + \stoptext diff --git a/tex/context/base/node-typ.lua b/tex/context/base/node-typ.lua index 2562378d1..5ab6b6975 100644 --- a/tex/context/base/node-typ.lua +++ b/tex/context/base/node-typ.lua @@ -6,6 +6,8 @@ if not modules then modules = { } end modules ['node-typ'] = { license = "see context related readme files" } +-- this will be replaced by blob-ini cum suis so typesetting will go away + local utfvalues = string.utfvalues local newglyph = nodes.glyph @@ -15,7 +17,7 @@ local hpack, vpack = node.hpack, node.vpack typesetting = typesetting or { } -local function tonodes(str,fontid,spacing) +local function tonodes(str,fontid,spacing) -- don't use this local head, prev = nil, nil for s in utfvalues(str) do local next diff --git a/tex/context/base/typo-brk.lua b/tex/context/base/typo-brk.lua index 5cab6a667..8f7a00f9d 100644 --- a/tex/context/base/typo-brk.lua +++ b/tex/context/base/typo-brk.lua @@ -9,21 +9,27 @@ if not modules then modules = { } end modules ['typo-brk'] = { -- this code dates from the beginning and is kind of experimental; it -- will be optimized and improved soon -local next, type = next, type +local next, type, tonumber = next, type, tonumber +local utfbyte, utfchar = utf.byte, utf.char local format = string.format +local settings_to_array = aux.settings_to_array local has_attribute = node.has_attribute local unset_attribute = node.unset_attribute local set_attribute = node.set_attribute local copy_node = node.copy +local free_node = node.free local insert_node_before = node.insert_before local insert_node_after = node.insert_after local make_penalty_node = nodes.penalty local make_glue_node = nodes.glue local make_disc_node = nodes.disc +local make_glyph_node = nodes.glyph +local remove_node = nodes.remove -- ! nodes +local tonodes = blobs.tonodes -local glyph = node.id("glyph") -local kern = node.id("kern") +local glyph = node.id("glyph") +local kern = node.id("kern") breakpoints = breakpoints or { } breakpoints.mapping = breakpoints.mapping or { } @@ -34,7 +40,8 @@ storage.register("breakpoints/mapping", breakpoints.mapping, "breakpoints.mappin local mapping = breakpoints.mapping -function breakpoints.setreplacement(id,char,kind,before,after,language) +function breakpoints.setreplacement(id,char,language,settings) + char = utfbyte(char) local map = mapping[id] if not map then map = { } @@ -45,71 +52,76 @@ function breakpoints.setreplacement(id,char,kind,before,after,language) cmap = { } map[char] = cmap end - cmap[language or ""] = { kind or 1, before or 1, after or 1 } + local left, right, middle = settings.left, settings.right, settings.middle + cmap[language or ""] = { + kind = tonumber(settings.kind) or 1, + nleft = tonumber(settings.nleft) or 1, + nright = tonumber(settings.nright) or 1, + left = left ~= "" and left or nil, + right = right ~= "" and right or nil, + middle = middle ~= "" and middle or nil, + } -- was { kind or 1, before or 1, after or 1 } +end + +local function insert_break(head,start,before,after) + insert_node_before(head,start,make_penalty_node(before)) + insert_node_before(head,start,make_glue_node(0)) + insert_node_after(head,start,make_glue_node(0)) + insert_node_after(head,start,make_penalty_node(after)) end breakpoints.methods[1] = function(head,start) if start.prev and start.next then - insert_node_before(head,start,make_penalty_node(10000)) - insert_node_before(head,start,make_glue_node(0)) - insert_node_after(head,start,make_glue_node(0)) - insert_node_after(head,start,make_penalty_node(0)) + insert_break(head,start,10000,0) end return head, start end breakpoints.methods[2] = function(head,start) -- ( => (- if start.prev and start.next then - local tmp = start - start = make_disc_node() - start.prev, start.next = tmp.prev, tmp.next - tmp.prev.next, tmp.next.prev = start, start - tmp.prev, tmp.next = nil, nil + local tmp + head, start, tmp = remove_node(head,start) + head, start = insert_node_before(head,start,make_disc_node()) start.replace = tmp local tmp, hyphen = copy_node(tmp), copy_node(tmp) hyphen.char = languages.prehyphenchar(tmp.lang) tmp.next, hyphen.prev = hyphen, tmp start.post = tmp - insert_node_before(head,start,make_penalty_node(10000)) - insert_node_before(head,start,make_glue_node(0)) - insert_node_after(head,start,make_glue_node(0)) - insert_node_after(head,start,make_penalty_node(10000)) + insert_break(head,start,10000,10000) end return head, start end breakpoints.methods[3] = function(head,start) -- ) => -) if start.prev and start.next then - local tmp = start - start = make_disc_node() - start.prev, start.next = tmp.prev, tmp.next - tmp.prev.next, tmp.next.prev = start, start - tmp.prev, tmp.next = nil, nil + local tmp + head, start, tmp = remove_node(head,start) + head, start = insert_node_before(head,start,make_disc_node()) start.replace = tmp local tmp, hyphen = copy_node(tmp), copy_node(tmp) hyphen.char = languages.prehyphenchar(tmp.lang) tmp.prev, hyphen.next = hyphen, tmp start.pre = hyphen - insert_node_before(head,start,make_penalty_node(10000)) - insert_node_before(head,start,make_glue_node(0)) - insert_node_after(head,start,make_glue_node(0)) - insert_node_after(head,start,make_penalty_node(10000)) + insert_break(head,start,10000,10000) end return head, start end breakpoints.methods[4] = function(head,start) -- - => - - - if start.prev and start.next then - local tmp = start - start = make_disc_node() - start.prev, start.next = tmp.prev, tmp.next - tmp.prev.next, tmp.next.prev = start, start - tmp.prev, tmp.next = nil, nil - -- maybe prehyphenchar etc - start.pre = copy_node(tmp) - start.post = copy_node(tmp) - start.replace = tmp - insert_node_before(head,start,make_penalty_node(10000)) - insert_node_before(head,start,make_glue_node(0)) - insert_node_after(head,start,make_glue_node(0)) - insert_node_after(head,start,make_penalty_node(10000)) + local tmp + head, start, tmp = remove_node(head,start) + head, start = insert_node_before(head,start,make_disc_node()) + start.pre, start.post, start.replace = copy_node(tmp), copy_node(tmp), tmp + insert_break(head,start,10000,10000) + end + return head, start +end +breakpoints.methods[5] = function(head,start,settings) -- x => p q r + if start.prev and start.next then + local tmp + head, start, tmp = remove_node(head,start) + head, start = insert_node_before(head,start,make_disc_node()) + start.pre, start.post, start.replace = tonodes(settings.right,tmp), tonodes(settings.left,tmp), tonodes(settings.middle,tmp) + free_node(tmp) + insert_break(head,start,10000,10000) end return head, start end @@ -134,18 +146,18 @@ function breakpoints.process(namespace,attribute,head) -- we do a sanity check for language local smap = lang and lang >= 0 and lang < 0x7FFF and (cmap[numbers[lang]] or cmap[""]) if smap then - if n >= smap[2] then - local m = smap[3] + if n >= smap.nleft then + local m = smap.nright local next = start.next - while next do -- gamble on same attribute + while next do -- gamble on same attribute (not that important actually) local id = next.id - if id == glyph then -- gamble on same attribute + if id == glyph then -- gamble on same attribute (not that important actually) if map[next.char] then break elseif m == 1 then - local method = methods[smap[1]] + local method = methods[smap.kind] if method then - head, start = method(head,start) + head, start = method(head,start,smap) done = true end break @@ -172,6 +184,8 @@ function breakpoints.process(namespace,attribute,head) else n = 0 end + else + -- n = n + 1 -- if we want single char handling (|-|) then we will use grouping and then we need this end elseif id == kern and start.subtype == 0 then -- ignore intercharacter kerning, will go way diff --git a/tex/context/base/typo-brk.mkiv b/tex/context/base/typo-brk.mkiv index e63232087..f2a44fee0 100644 --- a/tex/context/base/typo-brk.mkiv +++ b/tex/context/base/typo-brk.mkiv @@ -23,9 +23,7 @@ \newbox\breakpointbox -\definesystemvariable {bp} % BreakPoint - -\exhyphenchar=\minusone % we use a different order then base tex, so we really need this +\exhyphenchar\minusone % we use a different order then base tex, so we really need this \newcount \maxbreakpointsid @@ -42,12 +40,23 @@ {\dotripleempty\doinstallbreakpoint} % hm, we cannot prebuild lists, font dependent +% +% -- type nleft nright language left right middle +% +% -- we might eventually stick to only method 5 \def\doinstallbreakpoint[#1][#2][#3]% {\ifcsname\??bp:#1\endcsname \begingroup - \getparameters[\??bp][\c!type=1,\c!nleft=3,\c!nright=3,\s!language=,#3]% - \ctxlua{breakpoints.setreplacement(\csname\??bp:#1\endcsname,#2,\@@bptype,\@@bpnleft,\@@bpnright,"\@@bplanguage")}% + \getparameters[\??bp][\c!type=1,\c!nleft=3,\c!nright=3,\s!language=,\c!left=,\c!right=,\c!middle=,#3]% + \ctxlua{breakpoints.setreplacement(\csname\??bp:#1\endcsname, "#2", "\@@bplanguage", { + kind = \@@bptype, + nleft = "\@@bpnleft", + nright = "\@@bpnright", + right = "\@@bpright", + left = "\@@bpleft", + middle = "\@@bpmiddle" + } )}% todo: table \endgroup \fi} @@ -60,14 +69,21 @@ \definebreakpoints[compound] -\installbreakpoint [compound] [\number`+] [\c!left=3,\c!right=3,\c!type=1] -\installbreakpoint [compound] [\number`-] [\c!left=3,\c!right=3,\c!type=1] -\installbreakpoint [compound] [\number`/] [\c!left=3,\c!right=3,\c!type=1] -\installbreakpoint [compound] [\number`(] [\c!left=3,\c!right=3,\c!type=2] -\installbreakpoint [compound] [\number`)] [\c!left=3,\c!right=3,\c!type=3] +\installbreakpoint [compound] [+] [\c!nleft=3,\c!nright=3,\c!type=1] % middle=+,left=,right= +\installbreakpoint [compound] [-] [\c!nleft=3,\c!nright=3,\c!type=1] % middle=+,left=,right= +\installbreakpoint [compound] [/] [\c!nleft=3,\c!nright=3,\c!type=1] % middle=+,left=,right= +\installbreakpoint [compound] [(] [\c!nleft=3,\c!nright=3,\c!type=2] % type=5,middle=(,left=(-,right= +\installbreakpoint [compound] [)] [\c!nleft=3,\c!nright=3,\c!type=3] % type=5,middle=),left=,right=-) + +% \start \hsize 1.5cm \setbreakpoints[compound] +% \installbreakpoint [compound] [-] [nleft=3,nright=3,type=5,left=,right={-},middle={-}] +% \installbreakpoint [compound] [(] [nleft=3,nright=3,type=5,left=,right={(-},middle={(}] +% \installbreakpoint [compound] [)] [nleft=3,nright=3,type=5,left={-)},right=,middle={)}] +% composed-word\par composed(word)\par +% \stop % \mainlanguage[czech] -% \installbreakpoint [compound] [\number`-] [language=cs,left=3,right=3,type=4] +% \installbreakpoint [compound] [\number`-] [language=cs,nleft=3,nright=3,type=4] % \setbreakpoints[compound] % \start \hsize 1mm test-test \par \stop diff --git a/tex/generic/context/luatex-fonts-merged.lua b/tex/generic/context/luatex-fonts-merged.lua index 4a1430553..c1f4fc457 100644 --- a/tex/generic/context/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua --- merge date : 04/16/10 21:08:09 +-- merge date : 04/23/10 18:34:16 do -- begin closure to overcome local limits and interference -- cgit v1.2.3