diff options
author | Marius <mariausol@gmail.com> | 2013-05-20 03:20:28 +0300 |
---|---|---|
committer | Marius <mariausol@gmail.com> | 2013-05-20 03:20:28 +0300 |
commit | 5fc5cfb5014ddcc2942e13a559f4082fb66aa6e7 (patch) | |
tree | 53f81e99fac8c80ddd2fb70e233a7e5d5735722f /tex/context/base/font-sol.lua | |
parent | 13ec4b540e0d46c97fd7b089e0b7413da81e0a9f (diff) | |
download | context-5fc5cfb5014ddcc2942e13a559f4082fb66aa6e7.tar.gz |
beta 2013.05.20 02:00
Diffstat (limited to 'tex/context/base/font-sol.lua')
-rw-r--r-- | tex/context/base/font-sol.lua | 1768 |
1 files changed, 884 insertions, 884 deletions
diff --git a/tex/context/base/font-sol.lua b/tex/context/base/font-sol.lua index b37ab8869..db2dd24c2 100644 --- a/tex/context/base/font-sol.lua +++ b/tex/context/base/font-sol.lua @@ -1,884 +1,884 @@ -if not modules then modules = { } end modules ['font-sol'] = { -- this was: node-spl
- version = 1.001,
- comment = "companion to font-sol.mkiv",
- author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
- copyright = "PRAGMA ADE / ConTeXt Development Team",
- license = "see context related readme files"
-}
-
--- This module is dedicated to the oriental tex project and for
--- the moment is too experimental to be publicly supported.
---
--- We could cache solutions: say that we store the featureset and
--- all 'words' -> replacement ... so we create a large solution
--- database (per font)
---
--- This module can be optimized by using a dedicated dynamics handler
--- but I'll only do that when the rest of the code is stable.
---
--- Todo: bind setups to paragraph.
-
-local gmatch, concat, format, remove = string.gmatch, table.concat, string.format, table.remove
-local next, tostring, tonumber = next, tostring, tonumber
-local insert, remove = table.insert, table.remove
-local utfchar = utf.char
-local random = math.random
-
-local utilities, logs, statistics, fonts, trackers = utilities, logs, statistics, fonts, trackers
-local interfaces, commands, attributes = interfaces, commands, attributes
-local nodes, node, tex = nodes, node, tex
-
-local trace_split = false trackers.register("builders.paragraphs.solutions.splitters.splitter", function(v) trace_split = v end)
-local trace_optimize = false trackers.register("builders.paragraphs.solutions.splitters.optimizer", function(v) trace_optimize = v end)
-local trace_colors = false trackers.register("builders.paragraphs.solutions.splitters.colors", function(v) trace_colors = v end)
-local trace_goodies = false trackers.register("fonts.goodies", function(v) trace_goodies = v end)
-
-local report_solutions = logs.reporter("fonts","solutions")
-local report_splitters = logs.reporter("fonts","splitters")
-local report_optimizers = logs.reporter("fonts","optimizers")
-
-local variables = interfaces.variables
-
-local v_normal = variables.normal
-local v_reverse = variables.reverse
-local v_preroll = variables.preroll
-local v_random = variables.random
-local v_split = variables.split
-
-local settings_to_array = utilities.parsers.settings_to_array
-local settings_to_hash = utilities.parsers.settings_to_hash
-
-local find_node_tail = node.tail or node.slide
-local free_node = node.free
-local free_nodelist = node.flush_list
-local copy_nodelist = node.copy_list
-local traverse_nodes = node.traverse
-local traverse_ids = node.traverse_id
-local protect_glyphs = nodes.handlers.protectglyphs or node.protect_glyphs
-local hpack_nodes = node.hpack
-local insert_node_before = node.insert_before
-local insert_node_after = node.insert_after
-local repack_hlist = nodes.repackhlist
-local nodes_to_utf = nodes.listtoutf
-
-local setnodecolor = nodes.tracers.colors.set
-
-local nodecodes = nodes.nodecodes
-local whatsitcodes = nodes.whatsitcodes
-local kerncodes = nodes.kerncodes
-
-local glyph_code = nodecodes.glyph
-local disc_code = nodecodes.disc
-local kern_code = nodecodes.kern
-local hlist_code = nodecodes.hlist
-local whatsit_code = nodecodes.whatsit
-
-local fontkern_code = kerncodes.fontkern
-
-local localpar_code = whatsitcodes.localpar
-local dir_code = whatsitcodes.dir
-local userdefined_code = whatsitcodes.userdefined
-
-local nodepool = nodes.pool
-local tasks = nodes.tasks
-local usernodeids = nodepool.userids
-
-local new_textdir = nodepool.textdir
-local new_usernumber = nodepool.usernumber
-local new_glue = nodepool.glue
-local new_leftskip = nodepool.leftskip
-
-local starttiming = statistics.starttiming
-local stoptiming = statistics.stoptiming
-local process_characters = nodes.handlers.characters
-local inject_kerns = nodes.injections.handler
-
-local fonthashes = fonts.hashes
-local fontdata = fonthashes.identifiers
-local setfontdynamics = fonthashes.setdynamics
-local fontprocesses = fonthashes.processes
-
-local texsetattribute = tex.setattribute
-local unsetvalue = attributes.unsetvalue
-
-local parbuilders = builders.paragraphs
-parbuilders.solutions = parbuilders.solutions or { }
-local parsolutions = parbuilders.solutions
-parsolutions.splitters = parsolutions.splitters or { }
-local splitters = parsolutions.splitters
-
-local solutions = { } -- attribute sets
-local registered = { } -- backmapping
-splitters.registered = registered
-
-local a_split = attributes.private('splitter')
-
-local preroll = true
-local criterium = 0
-local randomseed = nil
-local optimize = nil -- set later
-local variant = v_normal
-local splitwords = true
-
-local cache = { }
-local variants = { }
-local max_less = 0
-local max_more = 0
-
-local stack = { }
-
-local dummy = {
- attribute = unsetvalue,
- randomseed = 0,
- criterium = 0,
- preroll = false,
- optimize = nil,
- splitwords = false,
- variant = v_normal,
-}
-
-local function checksettings(r,settings)
- local s = r.settings
- local method = settings_to_hash(settings.method or "")
- local optimize, preroll, splitwords
- for k, v in next, method do
- if k == v_preroll then
- preroll = true
- elseif k == v_split then
- splitwords = true
- elseif variants[k] then
- variant = k
- optimize = variants[k] -- last one wins
- end
- end
- r.randomseed = tonumber(settings.randomseed) or s.randomseed or r.randomseed or 0
- r.criterium = tonumber(settings.criterium ) or s.criterium or r.criterium or 0
- r.preroll = preroll or false
- r.splitwords = splitwords or false
- r.optimize = optimize or s.optimize or r.optimize or variants[v_normal]
-end
-
-local function pushsplitter(name,settings)
- local r = name and registered[name]
- if r then
- if settings then
- checksettings(r,settings)
- end
- else
- r = dummy
- end
- insert(stack,r)
- -- brr
- randomseed = r.randomseed or 0
- criterium = r.criterium or 0
- preroll = r.preroll or false
- optimize = r.optimize or nil
- splitwords = r.splitwords or nil
- --
- texsetattribute(a_split,r.attribute)
- return #stack
-end
-
-local function popsplitter()
- remove(stack)
- local n = #stack
- local r = stack[n] or dummy
- --
- randomseed = r.randomseed or 0
- criterium = r.criterium or 0
- preroll = r.preroll or false
- optimize = r.optimize or nil
- --
- texsetattribute(a_split,r.attribute)
- return n
-end
-
-local contextsetups = fonts.specifiers.contextsetups
-
-local function convert(featuresets,name,list)
- if list then
- local numbers = { }
- local nofnumbers = 0
- for i=1,#list do
- local feature = list[i]
- local fs = featuresets[feature]
- local fn = fs and fs.number
- if not fn then
- -- fall back on global features
- fs = contextsetups[feature]
- fn = fs and fs.number
- end
- if fn then
- nofnumbers = nofnumbers + 1
- numbers[nofnumbers] = fn
- if trace_goodies or trace_optimize then
- report_solutions("solution %a of %a uses feature %a with number %s",i,name,feature,fn)
- end
- else
- report_solutions("solution %a of %a has an invalid feature reference %a",i,name,feature)
- end
- end
- return nofnumbers > 0 and numbers
- end
-end
-
-local function initialize(goodies)
- local solutions = goodies.solutions
- if solutions then
- local featuresets = goodies.featuresets
- local goodiesname = goodies.name
- if trace_goodies or trace_optimize then
- report_solutions("checking solutions in %a",goodiesname)
- end
- for name, set in next, solutions do
- set.less = convert(featuresets,name,set.less)
- set.more = convert(featuresets,name,set.more)
- end
- end
-end
-
-fonts.goodies.register("solutions",initialize)
-
-function splitters.define(name,settings)
- local goodies = settings.goodies
- local solution = settings.solution
- local less = settings.less
- local more = settings.more
- local less_set, more_set
- local l = less and settings_to_array(less)
- local m = more and settings_to_array(more)
- if goodies then
- goodies = fonts.goodies.load(goodies) -- also in tfmdata
- if goodies then
- local featuresets = goodies.featuresets
- local solution = solution and goodies.solutions[solution]
- if l and #l > 0 then
- less_set = convert(featuresets,name,less) -- take from settings
- else
- less_set = solution and solution.less -- take from goodies
- end
- if m and #m > 0 then
- more_set = convert(featuresets,name,more) -- take from settings
- else
- more_set = solution and solution.more -- take from goodies
- end
- end
- else
- if l then
- local n = #less_set
- for i=1,#l do
- local ss = contextsetups[l[i]]
- if ss then
- n = n + 1
- less_set[n] = ss.number
- end
- end
- end
- if m then
- local n = #more_set
- for i=1,#m do
- local ss = contextsetups[m[i]]
- if ss then
- n = n + 1
- more_set[n] = ss.number
- end
- end
- end
- end
- if trace_optimize then
- report_solutions("defining solutions %a, less %a, more %a",name,concat(less_set or {}," "),concat(more_set or {}," "))
- end
- local nofsolutions = #solutions + 1
- local t = {
- solution = solution,
- less = less_set or { },
- more = more_set or { },
- settings = settings, -- for tracing
- attribute = nofsolutions,
- }
- solutions[nofsolutions] = t
- registered[name] = t
- return nofsolutions
-end
-
-local nofwords, noftries, nofadapted, nofkept, nofparagraphs = 0, 0, 0, 0, 0
-
-local splitter_one = usernodeids["splitters.one"]
-local splitter_two = usernodeids["splitters.two"]
-
-local a_word = attributes.private('word')
-local a_fontkern = attributes.private('fontkern')
-
-local encapsulate = false
-
-directives.register("builders.paragraphs.solutions.splitters.encapsulate", function(v)
- encapsulate = v
-end)
-
-function splitters.split(head)
- -- quite fast
- local current, done, rlmode, start, stop, attribute = head, false, false, nil, nil, 0
- cache, max_less, max_more = { }, 0, 0
- local function flush() -- we can move this
- local font = start.font
- local last = stop.next
- local list = last and copy_nodelist(start,last) or copy_nodelist(start)
- local n = #cache + 1
- if encapsulate then
- local user_one = new_usernumber(splitter_one,n)
- local user_two = new_usernumber(splitter_two,n)
- head, start = insert_node_before(head,start,user_one)
- insert_node_after(head,stop,user_two)
- else
- local current = start
- while true do
- current[a_word] = n
- if current == stop then
- break
- else
- current = current.next
- end
- end
- end
- if rlmode == "TRT" or rlmode == "+TRT" then
- local dirnode = new_textdir("+TRT")
- list.prev = dirnode
- dirnode.next = list
- list = dirnode
- end
- local c = {
- original = list,
- attribute = attribute,
- direction = rlmode,
- font = font
- }
- if trace_split then
- report_splitters("cached %4i: font %a, attribute %a, direction %a, word %a",
- n, font, attribute, nodes_to_utf(list,true), rlmode and "r2l" or "l2r")
- end
- cache[n] = c
- local solution = solutions[attribute]
- local l, m = #solution.less, #solution.more
- if l > max_less then max_less = l end
- if m > max_more then max_more = m end
- start, stop, done = nil, nil, true
- end
- while current do -- also nextid
- local next = current.next
- local id = current.id
- if id == glyph_code then
- if current.subtype < 256 then
- local a = current[a_split]
- if not a then
- start, stop = nil, nil
- elseif not start then
- start, stop, attribute = current, current, a
- elseif a ~= attribute then
- start, stop = nil, nil
- else
- stop = current
- end
- end
- elseif id == disc_code then
- if splitwords then
- if start then
- flush()
- end
- elseif start and next and next.id == glyph_code and next.subtype < 256 then
- -- beware: we can cross future lines
- stop = next
- else
- start, stop = nil, nil
- end
- elseif id == whatsit_code then
- if start then
- flush()
- end
- local subtype = current.subtype
- if subtype == dir_code or subtype == localpar_code then
- rlmode = current.dir
- end
- else
- if start then
- flush()
- end
- end
- current = next
- end
- if start then
- flush()
- end
- nofparagraphs = nofparagraphs + 1
- nofwords = nofwords + #cache
- return head, done
-end
-
-local function collect_words(list) -- can be made faster for attributes
- local words, w, word = { }, 0, nil
- if encapsulate then
- for current in traverse_ids(whatsit_code,list) do
- if current.subtype == userdefined_code then -- hm
- local user_id = current.user_id
- if user_id == splitter_one then
- word = { current.value, current, current }
- w = w + 1
- words[w] = word
- elseif user_id == splitter_two then
- if word then
- word[3] = current
- else
- -- something is wrong
- end
- end
- end
- end
- else
- local current, first, last, index = list, nil, nil, nil
- while current do
- -- todo: disc and kern
- local id = current.id
- if id == glyph_code or id == disc_code then
- local a = current[a_word]
- if a then
- if a == index then
- -- same word
- last = current
- elseif index then
- w = w + 1
- words[w] = { index, first, last }
- first = current
- last = current
- index = a
- elseif first then
- last = current
- index = a
- else
- first = current
- last = current
- index = a
- end
- elseif index then
- if first then
- w = w + 1
- words[w] = { index, first, last }
- end
- index = nil
- first = nil
- elseif trace_split then
- if id == disc_code then
- report_splitters("skipped: disc node")
- else
- report_splitters("skipped: %C",current.char)
- end
- end
- elseif id == kern_code and (current.subtype == fontkern_code or current[a_fontkern]) then
- if first then
- last = current
- else
- first = current
- last = current
- end
- elseif index then
- w = w + 1
- words[w] = { index, first, last }
- index = nil
- first = nil
- if id == disc_node then
- if trace_split then
- report_splitters("skipped: disc node")
- end
- end
- end
- current = current.next
- end
- if index then
- w = w + 1
- words[w] = { index, first, last }
- end
- if trace_split then
- for i=1,#words do
- local w = words[i]
- local n, f, l = w[1], w[2], w[3]
- local c = cache[n]
- if c then
- report_splitters("found %4i: word %a, cached %a",n,nodes_to_utf(f,true,true,l),nodes_to_utf(c.original,true))
- else
- report_splitters("found %4i: word %a, not in cache",n,nodes_to_utf(f,true,true,l))
- end
- end
- end
- end
- return words, list -- check for empty (elsewhere)
-end
-
--- we could avoid a hpack but hpack is not that slow
-
-local function doit(word,list,best,width,badness,line,set,listdir)
- local changed = 0
- local n = word[1]
- local found = cache[n]
- if found then
- local h, t
- if encapsulate then
- h = word[2].next -- head of current word
- t = word[3].prev -- tail of current word
- else
- h = word[2]
- t = word[3]
- end
- if splitwords then
- -- there are no lines crossed in a word
- else
- local ok = false
- local c = h
- while c do
- if c == t then
- ok = true
- break
- else
- c = c.next
- end
- end
- if not ok then
- report_solutions("skipping hyphenated word (for now)")
- -- todo: mark in words as skipped, saves a bit runtime
- return false, changed
- end
- end
- local original, attribute, direction = found.original, found.attribute, found.direction
- local solution = solutions[attribute]
- local features = solution and solution[set]
- if features then
- local featurenumber = features[best] -- not ok probably
- if featurenumber then
- noftries = noftries + 1
- local first = copy_nodelist(original)
- if not trace_colors then
- for n in traverse_nodes(first) do -- maybe fast force so no attr needed
- n[0] = featurenumber -- this forces dynamics
- end
- elseif set == "less" then
- for n in traverse_nodes(first) do
- setnodecolor(n,"font:isol") -- yellow
- n[0] = featurenumber
- end
- else
- for n in traverse_nodes(first) do
- setnodecolor(n,"font:medi") -- green
- n[0] = featurenumber
- end
- end
- local font = found.font
- local setdynamics = setfontdynamics[font]
- if setdynamics then
- local processes = setdynamics(font,featurenumber)
- for i=1,#processes do -- often more than 1
- first = processes[i](first,font,featurenumber)
- end
- else
- report_solutions("fatal error, no dynamics for font %a",font)
- end
- first = inject_kerns(first)
- if first.id == whatsit_code then
- local temp = first
- first = first.next
- free_node(temp)
- end
- local last = find_node_tail(first)
- -- replace [u]h->t by [u]first->last
- local prev = h.prev
- local next = t.next
- prev.next = first
- first.prev = prev
- if next then
- last.next = next
- next.prev = last
- end
- -- check new pack
- local temp, b = repack_hlist(list,width,'exactly',listdir)
- if b > badness then
- if trace_optimize then
- report_optimizers("line %a, badness before %a, after %a, criterium %a, verdict %a",line,badness,b,criterium,"quit")
- end
- -- remove last insert
- prev.next = h
- h.prev = prev
- if next then
- t.next = next
- next.prev = t
- else
- t.next = nil
- end
- last.next = nil
- free_nodelist(first)
- else
- if trace_optimize then
- report_optimizers("line %a, badness before: %a, after %a, criterium %a, verdict %a",line,badness,b,criterium,"continue")
- end
- -- free old h->t
- t.next = nil
- free_nodelist(h) -- somhow fails
- if not encapsulate then
- word[2] = first
- word[3] = last
- end
- changed, badness = changed + 1, b
- end
- if b <= criterium then
- return true, changed
- end
- end
- end
- end
- return false, changed
-end
-
--- We repeat some code but adding yet another layer of indirectness is not
--- making things better.
-
-variants[v_normal] = function(words,list,best,width,badness,line,set,listdir)
- local changed = 0
- for i=1,#words do
- local done, c = doit(words[i],list,best,width,badness,line,set,listdir)
- changed = changed + c
- if done then
- break
- end
- end
- if changed > 0 then
- nofadapted = nofadapted + 1
- -- todo: get rid of pack when ok because we already have packed and we only need the last b
- local list, b = repack_hlist(list,width,'exactly',listdir)
- return list, true, changed, b -- badness
- else
- nofkept = nofkept + 1
- return list, false, 0, badness
- end
-end
-
-variants[v_reverse] = function(words,list,best,width,badness,line,set,listdir)
- local changed = 0
- for i=#words,1,-1 do
- local done, c = doit(words[i],list,best,width,badness,line,set,listdir)
- changed = changed + c
- if done then
- break
- end
- end
- if changed > 0 then
- nofadapted = nofadapted + 1
- -- todo: get rid of pack when ok because we already have packed and we only need the last b
- local list, b = repack_hlist(list,width,'exactly',listdir)
- return list, true, changed, b -- badness
- else
- nofkept = nofkept + 1
- return list, false, 0, badness
- end
-end
-
-variants[v_random] = function(words,list,best,width,badness,line,set,listdir)
- local changed = 0
- while #words > 0 do
- local done, c = doit(remove(words,random(1,#words)),list,best,width,badness,line,set,listdir)
- changed = changed + c
- if done then
- break
- end
- end
- if changed > 0 then
- nofadapted = nofadapted + 1
- -- todo: get rid of pack when ok because we already have packed and we only need the last b
- local list, b = repack_hlist(list,width,'exactly',listdir)
- return list, true, changed, b -- badness
- else
- nofkept = nofkept + 1
- return list, false, 0, badness
- end
-end
-
-local function show_quality(current,what,line)
- local set = current.glue_set
- local sign = current.glue_sign
- local order = current.glue_order
- local amount = set * ((sign == 2 and -1) or 1)
- report_optimizers("line %a, category %a, amount %a, set %a, sign %a, how %a, order %a",line,what,amount,set,sign,how,order)
-end
-
-function splitters.optimize(head)
- if not optimize then
- report_optimizers("no optimizer set")
- return
- end
- local nc = #cache
- if nc == 0 then
- return
- end
- starttiming(splitters)
- local listdir = nil -- todo ! ! !
- if randomseed then
- math.setrandomseedi(randomseed)
- randomseed = nil
- end
- local line = 0
- local tex_hbadness, tex_hfuzz = tex.hbadness, tex.hfuzz
- tex.hbadness, tex.hfuzz = 10000, number.maxdimen
- if trace_optimize then
- report_optimizers("preroll %a, variant %a, criterium %a, cache size %a",preroll,variant,criterium,nc)
- end
- for current in traverse_ids(hlist_code,head) do
- -- report_splitters("before: [%s] => %s",current.dir,nodes.tosequence(current.list,nil))
- line = line + 1
- local sign, dir, list, width = current.glue_sign, current.dir, current.list, current.width
- if not encapsulate and list.id == glyph_code then
- -- nasty .. we always assume a prev being there .. future luatex will always have a leftskip set
- -- current.list, list = insert_node_before(list,list,new_glue(0))
- current.list, list = insert_node_before(list,list,new_leftskip(0))
- end
- local temp, badness = repack_hlist(list,width,'exactly',dir) -- it would be nice if the badness was stored in the node
- if badness > 0 then
- if sign == 0 then
- if trace_optimize then
- report_optimizers("line %a, badness %a, outcome %a, verdict %a",line,badness,"okay","okay")
- end
- else
- local set, max
- if sign == 1 then
- if trace_optimize then
- report_optimizers("line %a, badness %a, outcome %a, verdict %a",line,badness,"underfull","trying more")
- end
- set, max = "more", max_more
- else
- if trace_optimize then
- report_optimizers("line %a, badness %a, outcome %a, verdict %a",line,badness,"overfull","trying less")
- end
- set, max = "less", max_less
- end
- -- we can keep the best variants
- local lastbest, lastbadness = nil, badness
- if preroll then
- local bb, base
- for i=1,max do
- if base then
- free_nodelist(base)
- end
- base = copy_nodelist(list)
- local words = collect_words(base) -- beware: words is adapted
- for j=i,max do
- local temp, done, changes, b = optimize(words,base,j,width,badness,line,set,dir)
- base = temp
- if trace_optimize then
- report_optimizers("line %a, alternative %a.%a, changes %a, badness %a",line,i,j,changes,b)
- end
- bb = b
- if b <= criterium then
- break
- end
- -- if done then
- -- break
- -- end
- end
- if bb and bb > criterium then -- needs checking
- if not lastbest then
- lastbest, lastbadness = i, bb
- elseif bb > lastbadness then
- lastbest, lastbadness = i, bb
- end
- else
- break
- end
- end
- free_nodelist(base)
- end
- local words = collect_words(list)
- for best=lastbest or 1,max do
- local temp, done, changes, b = optimize(words,list,best,width,badness,line,set,dir)
- current.list = temp
- if trace_optimize then
- report_optimizers("line %a, alternative %a, changes %a, badness %a",line,best,changes,b)
- end
- if done then
- if b <= criterium then -- was == 0
- protect_glyphs(list)
- break
- end
- end
- end
- end
- else
- if trace_optimize then
- report_optimizers("line %a, verdict %a",line,"not bad enough")
- end
- end
- -- we pack inside the outer hpack and that way keep the original wd/ht/dp as bonus
- current.list = hpack_nodes(current.list,width,'exactly',listdir)
- -- report_splitters("after: [%s] => %s",temp.dir,nodes.tosequence(temp.list,nil))
- end
- for i=1,nc do
- local ci = cache[i]
- free_nodelist(ci.original)
- end
- cache = { }
- tex.hbadness, tex.hfuzz = tex_hbadness, tex_hfuzz
- stoptiming(splitters)
-end
-
-statistics.register("optimizer statistics", function()
- if nofwords > 0 then
- local elapsed = statistics.elapsedtime(splitters)
- local average = noftries/elapsed
- return format("%s words identified in %s paragraphs, %s words retried, %s lines tried, %0.3f seconds used, %s adapted, %0.1f lines per second",
- nofwords,nofparagraphs,noftries,nofadapted+nofkept,elapsed,nofadapted,average)
- end
-end)
-
--- we could use a stack
-
-local enableaction = tasks.enableaction
-local disableaction = tasks.disableaction
-
-local function enable()
- enableaction("processors", "builders.paragraphs.solutions.splitters.split")
- enableaction("finalizers", "builders.paragraphs.solutions.splitters.optimize")
-end
-
-local function disable()
- disableaction("processors", "builders.paragraphs.solutions.splitters.split")
- disableaction("finalizers", "builders.paragraphs.solutions.splitters.optimize")
-end
-
-function splitters.start(name,settings)
- if pushsplitter(name,settings) == 1 then
- enable()
- end
-end
-
-function splitters.stop()
- if popsplitter() == 0 then
- disable()
- end
-end
-
-function splitters.set(name,settings)
- if #stack > 0 then
- stack = { }
- else
- enable()
- end
- pushsplitter(name,settings) -- sets attribute etc
-end
-
-function splitters.reset()
- if #stack > 0 then
- stack = { }
- popsplitter() -- resets attribute etc
- disable()
- end
-end
-
--- interface
-
-commands.definefontsolution = splitters.define
-commands.startfontsolution = splitters.start
-commands.stopfontsolution = splitters.stop
-commands.setfontsolution = splitters.set
-commands.resetfontsolution = splitters.reset
+if not modules then modules = { } end modules ['font-sol'] = { -- this was: node-spl + version = 1.001, + comment = "companion to font-sol.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This module is dedicated to the oriental tex project and for +-- the moment is too experimental to be publicly supported. +-- +-- We could cache solutions: say that we store the featureset and +-- all 'words' -> replacement ... so we create a large solution +-- database (per font) +-- +-- This module can be optimized by using a dedicated dynamics handler +-- but I'll only do that when the rest of the code is stable. +-- +-- Todo: bind setups to paragraph. + +local gmatch, concat, format, remove = string.gmatch, table.concat, string.format, table.remove +local next, tostring, tonumber = next, tostring, tonumber +local insert, remove = table.insert, table.remove +local utfchar = utf.char +local random = math.random + +local utilities, logs, statistics, fonts, trackers = utilities, logs, statistics, fonts, trackers +local interfaces, commands, attributes = interfaces, commands, attributes +local nodes, node, tex = nodes, node, tex + +local trace_split = false trackers.register("builders.paragraphs.solutions.splitters.splitter", function(v) trace_split = v end) +local trace_optimize = false trackers.register("builders.paragraphs.solutions.splitters.optimizer", function(v) trace_optimize = v end) +local trace_colors = false trackers.register("builders.paragraphs.solutions.splitters.colors", function(v) trace_colors = v end) +local trace_goodies = false trackers.register("fonts.goodies", function(v) trace_goodies = v end) + +local report_solutions = logs.reporter("fonts","solutions") +local report_splitters = logs.reporter("fonts","splitters") +local report_optimizers = logs.reporter("fonts","optimizers") + +local variables = interfaces.variables + +local v_normal = variables.normal +local v_reverse = variables.reverse +local v_preroll = variables.preroll +local v_random = variables.random +local v_split = variables.split + +local settings_to_array = utilities.parsers.settings_to_array +local settings_to_hash = utilities.parsers.settings_to_hash + +local find_node_tail = node.tail or node.slide +local free_node = node.free +local free_nodelist = node.flush_list +local copy_nodelist = node.copy_list +local traverse_nodes = node.traverse +local traverse_ids = node.traverse_id +local protect_glyphs = nodes.handlers.protectglyphs or node.protect_glyphs +local hpack_nodes = node.hpack +local insert_node_before = node.insert_before +local insert_node_after = node.insert_after +local repack_hlist = nodes.repackhlist +local nodes_to_utf = nodes.listtoutf + +local setnodecolor = nodes.tracers.colors.set + +local nodecodes = nodes.nodecodes +local whatsitcodes = nodes.whatsitcodes +local kerncodes = nodes.kerncodes + +local glyph_code = nodecodes.glyph +local disc_code = nodecodes.disc +local kern_code = nodecodes.kern +local hlist_code = nodecodes.hlist +local whatsit_code = nodecodes.whatsit + +local fontkern_code = kerncodes.fontkern + +local localpar_code = whatsitcodes.localpar +local dir_code = whatsitcodes.dir +local userdefined_code = whatsitcodes.userdefined + +local nodepool = nodes.pool +local tasks = nodes.tasks +local usernodeids = nodepool.userids + +local new_textdir = nodepool.textdir +local new_usernumber = nodepool.usernumber +local new_glue = nodepool.glue +local new_leftskip = nodepool.leftskip + +local starttiming = statistics.starttiming +local stoptiming = statistics.stoptiming +local process_characters = nodes.handlers.characters +local inject_kerns = nodes.injections.handler + +local fonthashes = fonts.hashes +local fontdata = fonthashes.identifiers +local setfontdynamics = fonthashes.setdynamics +local fontprocesses = fonthashes.processes + +local texsetattribute = tex.setattribute +local unsetvalue = attributes.unsetvalue + +local parbuilders = builders.paragraphs +parbuilders.solutions = parbuilders.solutions or { } +local parsolutions = parbuilders.solutions +parsolutions.splitters = parsolutions.splitters or { } +local splitters = parsolutions.splitters + +local solutions = { } -- attribute sets +local registered = { } -- backmapping +splitters.registered = registered + +local a_split = attributes.private('splitter') + +local preroll = true +local criterium = 0 +local randomseed = nil +local optimize = nil -- set later +local variant = v_normal +local splitwords = true + +local cache = { } +local variants = { } +local max_less = 0 +local max_more = 0 + +local stack = { } + +local dummy = { + attribute = unsetvalue, + randomseed = 0, + criterium = 0, + preroll = false, + optimize = nil, + splitwords = false, + variant = v_normal, +} + +local function checksettings(r,settings) + local s = r.settings + local method = settings_to_hash(settings.method or "") + local optimize, preroll, splitwords + for k, v in next, method do + if k == v_preroll then + preroll = true + elseif k == v_split then + splitwords = true + elseif variants[k] then + variant = k + optimize = variants[k] -- last one wins + end + end + r.randomseed = tonumber(settings.randomseed) or s.randomseed or r.randomseed or 0 + r.criterium = tonumber(settings.criterium ) or s.criterium or r.criterium or 0 + r.preroll = preroll or false + r.splitwords = splitwords or false + r.optimize = optimize or s.optimize or r.optimize or variants[v_normal] +end + +local function pushsplitter(name,settings) + local r = name and registered[name] + if r then + if settings then + checksettings(r,settings) + end + else + r = dummy + end + insert(stack,r) + -- brr + randomseed = r.randomseed or 0 + criterium = r.criterium or 0 + preroll = r.preroll or false + optimize = r.optimize or nil + splitwords = r.splitwords or nil + -- + texsetattribute(a_split,r.attribute) + return #stack +end + +local function popsplitter() + remove(stack) + local n = #stack + local r = stack[n] or dummy + -- + randomseed = r.randomseed or 0 + criterium = r.criterium or 0 + preroll = r.preroll or false + optimize = r.optimize or nil + -- + texsetattribute(a_split,r.attribute) + return n +end + +local contextsetups = fonts.specifiers.contextsetups + +local function convert(featuresets,name,list) + if list then + local numbers = { } + local nofnumbers = 0 + for i=1,#list do + local feature = list[i] + local fs = featuresets[feature] + local fn = fs and fs.number + if not fn then + -- fall back on global features + fs = contextsetups[feature] + fn = fs and fs.number + end + if fn then + nofnumbers = nofnumbers + 1 + numbers[nofnumbers] = fn + if trace_goodies or trace_optimize then + report_solutions("solution %a of %a uses feature %a with number %s",i,name,feature,fn) + end + else + report_solutions("solution %a of %a has an invalid feature reference %a",i,name,feature) + end + end + return nofnumbers > 0 and numbers + end +end + +local function initialize(goodies) + local solutions = goodies.solutions + if solutions then + local featuresets = goodies.featuresets + local goodiesname = goodies.name + if trace_goodies or trace_optimize then + report_solutions("checking solutions in %a",goodiesname) + end + for name, set in next, solutions do + set.less = convert(featuresets,name,set.less) + set.more = convert(featuresets,name,set.more) + end + end +end + +fonts.goodies.register("solutions",initialize) + +function splitters.define(name,settings) + local goodies = settings.goodies + local solution = settings.solution + local less = settings.less + local more = settings.more + local less_set, more_set + local l = less and settings_to_array(less) + local m = more and settings_to_array(more) + if goodies then + goodies = fonts.goodies.load(goodies) -- also in tfmdata + if goodies then + local featuresets = goodies.featuresets + local solution = solution and goodies.solutions[solution] + if l and #l > 0 then + less_set = convert(featuresets,name,less) -- take from settings + else + less_set = solution and solution.less -- take from goodies + end + if m and #m > 0 then + more_set = convert(featuresets,name,more) -- take from settings + else + more_set = solution and solution.more -- take from goodies + end + end + else + if l then + local n = #less_set + for i=1,#l do + local ss = contextsetups[l[i]] + if ss then + n = n + 1 + less_set[n] = ss.number + end + end + end + if m then + local n = #more_set + for i=1,#m do + local ss = contextsetups[m[i]] + if ss then + n = n + 1 + more_set[n] = ss.number + end + end + end + end + if trace_optimize then + report_solutions("defining solutions %a, less %a, more %a",name,concat(less_set or {}," "),concat(more_set or {}," ")) + end + local nofsolutions = #solutions + 1 + local t = { + solution = solution, + less = less_set or { }, + more = more_set or { }, + settings = settings, -- for tracing + attribute = nofsolutions, + } + solutions[nofsolutions] = t + registered[name] = t + return nofsolutions +end + +local nofwords, noftries, nofadapted, nofkept, nofparagraphs = 0, 0, 0, 0, 0 + +local splitter_one = usernodeids["splitters.one"] +local splitter_two = usernodeids["splitters.two"] + +local a_word = attributes.private('word') +local a_fontkern = attributes.private('fontkern') + +local encapsulate = false + +directives.register("builders.paragraphs.solutions.splitters.encapsulate", function(v) + encapsulate = v +end) + +function splitters.split(head) + -- quite fast + local current, done, rlmode, start, stop, attribute = head, false, false, nil, nil, 0 + cache, max_less, max_more = { }, 0, 0 + local function flush() -- we can move this + local font = start.font + local last = stop.next + local list = last and copy_nodelist(start,last) or copy_nodelist(start) + local n = #cache + 1 + if encapsulate then + local user_one = new_usernumber(splitter_one,n) + local user_two = new_usernumber(splitter_two,n) + head, start = insert_node_before(head,start,user_one) + insert_node_after(head,stop,user_two) + else + local current = start + while true do + current[a_word] = n + if current == stop then + break + else + current = current.next + end + end + end + if rlmode == "TRT" or rlmode == "+TRT" then + local dirnode = new_textdir("+TRT") + list.prev = dirnode + dirnode.next = list + list = dirnode + end + local c = { + original = list, + attribute = attribute, + direction = rlmode, + font = font + } + if trace_split then + report_splitters("cached %4i: font %a, attribute %a, direction %a, word %a", + n, font, attribute, nodes_to_utf(list,true), rlmode and "r2l" or "l2r") + end + cache[n] = c + local solution = solutions[attribute] + local l, m = #solution.less, #solution.more + if l > max_less then max_less = l end + if m > max_more then max_more = m end + start, stop, done = nil, nil, true + end + while current do -- also nextid + local next = current.next + local id = current.id + if id == glyph_code then + if current.subtype < 256 then + local a = current[a_split] + if not a then + start, stop = nil, nil + elseif not start then + start, stop, attribute = current, current, a + elseif a ~= attribute then + start, stop = nil, nil + else + stop = current + end + end + elseif id == disc_code then + if splitwords then + if start then + flush() + end + elseif start and next and next.id == glyph_code and next.subtype < 256 then + -- beware: we can cross future lines + stop = next + else + start, stop = nil, nil + end + elseif id == whatsit_code then + if start then + flush() + end + local subtype = current.subtype + if subtype == dir_code or subtype == localpar_code then + rlmode = current.dir + end + else + if start then + flush() + end + end + current = next + end + if start then + flush() + end + nofparagraphs = nofparagraphs + 1 + nofwords = nofwords + #cache + return head, done +end + +local function collect_words(list) -- can be made faster for attributes + local words, w, word = { }, 0, nil + if encapsulate then + for current in traverse_ids(whatsit_code,list) do + if current.subtype == userdefined_code then -- hm + local user_id = current.user_id + if user_id == splitter_one then + word = { current.value, current, current } + w = w + 1 + words[w] = word + elseif user_id == splitter_two then + if word then + word[3] = current + else + -- something is wrong + end + end + end + end + else + local current, first, last, index = list, nil, nil, nil + while current do + -- todo: disc and kern + local id = current.id + if id == glyph_code or id == disc_code then + local a = current[a_word] + if a then + if a == index then + -- same word + last = current + elseif index then + w = w + 1 + words[w] = { index, first, last } + first = current + last = current + index = a + elseif first then + last = current + index = a + else + first = current + last = current + index = a + end + elseif index then + if first then + w = w + 1 + words[w] = { index, first, last } + end + index = nil + first = nil + elseif trace_split then + if id == disc_code then + report_splitters("skipped: disc node") + else + report_splitters("skipped: %C",current.char) + end + end + elseif id == kern_code and (current.subtype == fontkern_code or current[a_fontkern]) then + if first then + last = current + else + first = current + last = current + end + elseif index then + w = w + 1 + words[w] = { index, first, last } + index = nil + first = nil + if id == disc_node then + if trace_split then + report_splitters("skipped: disc node") + end + end + end + current = current.next + end + if index then + w = w + 1 + words[w] = { index, first, last } + end + if trace_split then + for i=1,#words do + local w = words[i] + local n, f, l = w[1], w[2], w[3] + local c = cache[n] + if c then + report_splitters("found %4i: word %a, cached %a",n,nodes_to_utf(f,true,true,l),nodes_to_utf(c.original,true)) + else + report_splitters("found %4i: word %a, not in cache",n,nodes_to_utf(f,true,true,l)) + end + end + end + end + return words, list -- check for empty (elsewhere) +end + +-- we could avoid a hpack but hpack is not that slow + +local function doit(word,list,best,width,badness,line,set,listdir) + local changed = 0 + local n = word[1] + local found = cache[n] + if found then + local h, t + if encapsulate then + h = word[2].next -- head of current word + t = word[3].prev -- tail of current word + else + h = word[2] + t = word[3] + end + if splitwords then + -- there are no lines crossed in a word + else + local ok = false + local c = h + while c do + if c == t then + ok = true + break + else + c = c.next + end + end + if not ok then + report_solutions("skipping hyphenated word (for now)") + -- todo: mark in words as skipped, saves a bit runtime + return false, changed + end + end + local original, attribute, direction = found.original, found.attribute, found.direction + local solution = solutions[attribute] + local features = solution and solution[set] + if features then + local featurenumber = features[best] -- not ok probably + if featurenumber then + noftries = noftries + 1 + local first = copy_nodelist(original) + if not trace_colors then + for n in traverse_nodes(first) do -- maybe fast force so no attr needed + n[0] = featurenumber -- this forces dynamics + end + elseif set == "less" then + for n in traverse_nodes(first) do + setnodecolor(n,"font:isol") -- yellow + n[0] = featurenumber + end + else + for n in traverse_nodes(first) do + setnodecolor(n,"font:medi") -- green + n[0] = featurenumber + end + end + local font = found.font + local setdynamics = setfontdynamics[font] + if setdynamics then + local processes = setdynamics(font,featurenumber) + for i=1,#processes do -- often more than 1 + first = processes[i](first,font,featurenumber) + end + else + report_solutions("fatal error, no dynamics for font %a",font) + end + first = inject_kerns(first) + if first.id == whatsit_code then + local temp = first + first = first.next + free_node(temp) + end + local last = find_node_tail(first) + -- replace [u]h->t by [u]first->last + local prev = h.prev + local next = t.next + prev.next = first + first.prev = prev + if next then + last.next = next + next.prev = last + end + -- check new pack + local temp, b = repack_hlist(list,width,'exactly',listdir) + if b > badness then + if trace_optimize then + report_optimizers("line %a, badness before %a, after %a, criterium %a, verdict %a",line,badness,b,criterium,"quit") + end + -- remove last insert + prev.next = h + h.prev = prev + if next then + t.next = next + next.prev = t + else + t.next = nil + end + last.next = nil + free_nodelist(first) + else + if trace_optimize then + report_optimizers("line %a, badness before: %a, after %a, criterium %a, verdict %a",line,badness,b,criterium,"continue") + end + -- free old h->t + t.next = nil + free_nodelist(h) -- somhow fails + if not encapsulate then + word[2] = first + word[3] = last + end + changed, badness = changed + 1, b + end + if b <= criterium then + return true, changed + end + end + end + end + return false, changed +end + +-- We repeat some code but adding yet another layer of indirectness is not +-- making things better. + +variants[v_normal] = function(words,list,best,width,badness,line,set,listdir) + local changed = 0 + for i=1,#words do + local done, c = doit(words[i],list,best,width,badness,line,set,listdir) + changed = changed + c + if done then + break + end + end + if changed > 0 then + nofadapted = nofadapted + 1 + -- todo: get rid of pack when ok because we already have packed and we only need the last b + local list, b = repack_hlist(list,width,'exactly',listdir) + return list, true, changed, b -- badness + else + nofkept = nofkept + 1 + return list, false, 0, badness + end +end + +variants[v_reverse] = function(words,list,best,width,badness,line,set,listdir) + local changed = 0 + for i=#words,1,-1 do + local done, c = doit(words[i],list,best,width,badness,line,set,listdir) + changed = changed + c + if done then + break + end + end + if changed > 0 then + nofadapted = nofadapted + 1 + -- todo: get rid of pack when ok because we already have packed and we only need the last b + local list, b = repack_hlist(list,width,'exactly',listdir) + return list, true, changed, b -- badness + else + nofkept = nofkept + 1 + return list, false, 0, badness + end +end + +variants[v_random] = function(words,list,best,width,badness,line,set,listdir) + local changed = 0 + while #words > 0 do + local done, c = doit(remove(words,random(1,#words)),list,best,width,badness,line,set,listdir) + changed = changed + c + if done then + break + end + end + if changed > 0 then + nofadapted = nofadapted + 1 + -- todo: get rid of pack when ok because we already have packed and we only need the last b + local list, b = repack_hlist(list,width,'exactly',listdir) + return list, true, changed, b -- badness + else + nofkept = nofkept + 1 + return list, false, 0, badness + end +end + +local function show_quality(current,what,line) + local set = current.glue_set + local sign = current.glue_sign + local order = current.glue_order + local amount = set * ((sign == 2 and -1) or 1) + report_optimizers("line %a, category %a, amount %a, set %a, sign %a, how %a, order %a",line,what,amount,set,sign,how,order) +end + +function splitters.optimize(head) + if not optimize then + report_optimizers("no optimizer set") + return + end + local nc = #cache + if nc == 0 then + return + end + starttiming(splitters) + local listdir = nil -- todo ! ! ! + if randomseed then + math.setrandomseedi(randomseed) + randomseed = nil + end + local line = 0 + local tex_hbadness, tex_hfuzz = tex.hbadness, tex.hfuzz + tex.hbadness, tex.hfuzz = 10000, number.maxdimen + if trace_optimize then + report_optimizers("preroll %a, variant %a, criterium %a, cache size %a",preroll,variant,criterium,nc) + end + for current in traverse_ids(hlist_code,head) do + -- report_splitters("before: [%s] => %s",current.dir,nodes.tosequence(current.list,nil)) + line = line + 1 + local sign, dir, list, width = current.glue_sign, current.dir, current.list, current.width + if not encapsulate and list.id == glyph_code then + -- nasty .. we always assume a prev being there .. future luatex will always have a leftskip set + -- current.list, list = insert_node_before(list,list,new_glue(0)) + current.list, list = insert_node_before(list,list,new_leftskip(0)) + end + local temp, badness = repack_hlist(list,width,'exactly',dir) -- it would be nice if the badness was stored in the node + if badness > 0 then + if sign == 0 then + if trace_optimize then + report_optimizers("line %a, badness %a, outcome %a, verdict %a",line,badness,"okay","okay") + end + else + local set, max + if sign == 1 then + if trace_optimize then + report_optimizers("line %a, badness %a, outcome %a, verdict %a",line,badness,"underfull","trying more") + end + set, max = "more", max_more + else + if trace_optimize then + report_optimizers("line %a, badness %a, outcome %a, verdict %a",line,badness,"overfull","trying less") + end + set, max = "less", max_less + end + -- we can keep the best variants + local lastbest, lastbadness = nil, badness + if preroll then + local bb, base + for i=1,max do + if base then + free_nodelist(base) + end + base = copy_nodelist(list) + local words = collect_words(base) -- beware: words is adapted + for j=i,max do + local temp, done, changes, b = optimize(words,base,j,width,badness,line,set,dir) + base = temp + if trace_optimize then + report_optimizers("line %a, alternative %a.%a, changes %a, badness %a",line,i,j,changes,b) + end + bb = b + if b <= criterium then + break + end + -- if done then + -- break + -- end + end + if bb and bb > criterium then -- needs checking + if not lastbest then + lastbest, lastbadness = i, bb + elseif bb > lastbadness then + lastbest, lastbadness = i, bb + end + else + break + end + end + free_nodelist(base) + end + local words = collect_words(list) + for best=lastbest or 1,max do + local temp, done, changes, b = optimize(words,list,best,width,badness,line,set,dir) + current.list = temp + if trace_optimize then + report_optimizers("line %a, alternative %a, changes %a, badness %a",line,best,changes,b) + end + if done then + if b <= criterium then -- was == 0 + protect_glyphs(list) + break + end + end + end + end + else + if trace_optimize then + report_optimizers("line %a, verdict %a",line,"not bad enough") + end + end + -- we pack inside the outer hpack and that way keep the original wd/ht/dp as bonus + current.list = hpack_nodes(current.list,width,'exactly',listdir) + -- report_splitters("after: [%s] => %s",temp.dir,nodes.tosequence(temp.list,nil)) + end + for i=1,nc do + local ci = cache[i] + free_nodelist(ci.original) + end + cache = { } + tex.hbadness, tex.hfuzz = tex_hbadness, tex_hfuzz + stoptiming(splitters) +end + +statistics.register("optimizer statistics", function() + if nofwords > 0 then + local elapsed = statistics.elapsedtime(splitters) + local average = noftries/elapsed + return format("%s words identified in %s paragraphs, %s words retried, %s lines tried, %0.3f seconds used, %s adapted, %0.1f lines per second", + nofwords,nofparagraphs,noftries,nofadapted+nofkept,elapsed,nofadapted,average) + end +end) + +-- we could use a stack + +local enableaction = tasks.enableaction +local disableaction = tasks.disableaction + +local function enable() + enableaction("processors", "builders.paragraphs.solutions.splitters.split") + enableaction("finalizers", "builders.paragraphs.solutions.splitters.optimize") +end + +local function disable() + disableaction("processors", "builders.paragraphs.solutions.splitters.split") + disableaction("finalizers", "builders.paragraphs.solutions.splitters.optimize") +end + +function splitters.start(name,settings) + if pushsplitter(name,settings) == 1 then + enable() + end +end + +function splitters.stop() + if popsplitter() == 0 then + disable() + end +end + +function splitters.set(name,settings) + if #stack > 0 then + stack = { } + else + enable() + end + pushsplitter(name,settings) -- sets attribute etc +end + +function splitters.reset() + if #stack > 0 then + stack = { } + popsplitter() -- resets attribute etc + disable() + end +end + +-- interface + +commands.definefontsolution = splitters.define +commands.startfontsolution = splitters.start +commands.stopfontsolution = splitters.stop +commands.setfontsolution = splitters.set +commands.resetfontsolution = splitters.reset |