path: root/tex/context/base/mkxl/lang-hyp.lmt

if not modules then modules = { } end modules ['lang-hyp'] = {
    version   = 1.001,
    comment   = "companion to lang-ini.mkiv",
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

-- In an automated workflow hypenation of long titles can be somewhat problematic
-- especially when demands conflict. For that reason I played a bit with a Lua based
-- variant of the traditional hyphenation machinery. This mechanism has been extended
-- several times in projects, of which a good description can be found in TUGboat,
-- Volume 27 (2006), No. 2 — Proceedings of EuroTEX2006: Automatic non-standard
-- hyphenation in OpenOffice.org by László Németh.
--
-- Being the result of two days experimenting the following implementation is probably
-- not completely okay yet. If there is demand I might add some more features and plugs.
-- The performance is quite okay but can probably improved a bit, although this is not
-- the most critital code. For instance, on a metafun manual run the overhead is about
-- 0.3 seconds on 19 seconds which is not that bad.
--
-- In the procecess of wrapping up (for the ctx conference proceedings) I cleaned up
-- and extended the code a bit. It can be used in production.
--
-- . a l g o r i t h m .
--    4l1g4
--     l g o3
--      1g o
--            2i t h
--                4h1m
-- ---------------------
--    4 1 4 3 2 0 4 1
--   a l-g o-r i t h-m

-- . a s s z o n n y a l .
--     s1s z/sz=sz,1,3
--             n1n y/ny=ny,1,3
-- -----------------------
--    0 1 0 0 0 1 0 0 0/sz=sz,2,3,ny=ny,6,3
--   a s-s z o n-n y a l/sz=sz,2,3,ny=ny,6,3
--
-- ab1cd/ef=gh,2,2 : acd - efd (pattern/replacement,start,length
--
-- todo  : support hjcodes (<32 == length) like luatex does now (no need/demand so far)
-- maybe : support hyphenation over range (can alsready be done using attributes/language)
-- maybe : reset dictionary.hyphenated when a pattern is added and/or forced reset option
-- todo  : check subtypes (because they have subtle meanings in the line breaking)
--
-- word start (in tex engine):
--
-- boundary  : yes when wordboundary
-- hlist     : when hyphenationbounds 1 or 3
-- vlist     : when hyphenationbounds 1 or 3
-- rule      : when hyphenationbounds 1 or 3
-- dir       : when hyphenationbounds 1 or 3
-- whatsit   : when hyphenationbounds 1 or 3
-- glue      : yes
-- math      : skipped
-- glyph     : exhyphenchar (one only) : yes (so no -- ---)
-- otherwise : yes
--
-- word end (in tex engine):
--
-- boundary  : yes
-- glyph     : yes when different language
-- glue      : yes
-- penalty   : yes
-- kern      : yes when not italic (for some historic reason)
-- hlist     : when hyphenationbounds 2 or 3
-- vlist     : when hyphenationbounds 2 or 3
-- rule      : when hyphenationbounds 2 or 3
-- dir       : when hyphenationbounds 2 or 3
-- whatsit   : when hyphenationbounds 2 or 3
-- ins       : when hyphenationbounds 2 or 3
-- adjust    : when hyphenationbounds 2 or 3

local type, rawget, rawset, tonumber, next = type, rawget, rawset, tonumber, next

local P, R, S, Cg, Cf, Ct, Cc, C, Carg, Cs = lpeg.P, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cf, lpeg.Ct, lpeg.Cc, lpeg.C, lpeg.Carg, lpeg.Cs
local lpegmatch = lpeg.match

local context    = context

local concat     = table.concat
local insert     = table.insert
local remove     = table.remove
local formatters = string.formatters
local utfchar    = utf.char
local utfbyte    = utf.byte

if not characters then
    require("char-ini")
end

local setmetatableindex = table.setmetatableindex

-- \enabletrackers[hyphenator.steps=silent] will not write to the terminal

local trace_steps       = false  trackers.register("hyphenator.steps",    function(v) trace_steps     = v end)
local trace_visualize   = false  trackers.register("hyphenator.visualize",function(v) trace_visualize = v end)

local report            = logs.reporter("hyphenator")

local implement         = interfaces and interfaces.implement or function() end

languages               = languages or { }
local hyphenators       = languages.hyphenators or { }
languages.hyphenators   = hyphenators
local traditional       = hyphenators.traditional or { }
hyphenators.traditional = traditional

local dictionaries = setmetatableindex(function(t,k)
    local v = {
        patterns   = { },
        hyphenated = { },
        specials   = { },
        exceptions = { },
        loaded     = false,
    }
    t[k] = v
    return v
end)

hyphenators.dictionaries = dictionaries

local character      = lpeg.patterns.utf8character
local digit          = R("09")
local weight         = digit/tonumber + Cc(0)
local fence          = P(".")
local hyphen         = P("-")
local space          = P(" ")
local char           = character - space
local validcharacter = (character - S("./"))
local keycharacter   =  character - S("/")
----- basepart       = Ct( (Cc(0) * fence)^-1 * (weight * validcharacter)^1 * weight * (fence * Cc(0))^-1)
local specpart       = (P("/") * Cf ( Ct("") *
        Cg ( Cc("before") * C((1-P("="))^1) * P("=") ) *
        Cg ( Cc("after")  * C((1-P(","))^1)  ) *
        (   P(",") *
            Cg ( Cc("start")  * ((1-P(","))^1/tonumber) * P(",") ) *
            Cg ( Cc("length") * ((1-P(-1) )^1/tonumber)          )
        )^-1
    , rawset))^-1

local make_hashkey_p = Cs((digit/"" + keycharacter)^1)
----- make_pattern_p = basepart * specpart
local make_hashkey_e = Cs((hyphen/"" + keycharacter)^1)
local make_pattern_e = Ct(P(char) * (hyphen * Cc(true) * P(char) + P(char) * Cc(false))^1) -- catch . and char after -

-- local make_hashkey_c = Cs((digit + keycharacter/"")^1)
-- local make_pattern_c = Ct((P(1)/tonumber)^1)

-- local cache = setmetatableindex(function(t,k)
--     local n = lpegmatch(make_hashkey_c,k)
--     local v = lpegmatch(make_pattern_c,n)
--     t[k] = v
--     return v
-- end)
--
-- local weight_n       = digit + Cc("0")
-- local basepart_n     = Cs( (Cc("0") * fence)^-1 * (weight * validcharacter)^1 * weight * (fence * Cc("0"))^-1) / cache
-- local make_pattern_n = basepart_n * specpart

local make_pattern_c = Ct((P(1)/tonumber)^1)

-- us + nl: 17664 entries -> 827 unique (saves some 3M)

local cache = setmetatableindex(function(t,k)
    local v = lpegmatch(make_pattern_c,k)
    t[k] = v
    return v
end)

local weight_n       = digit + Cc("0")
local fence_n        = fence / "0"
local char_n         = validcharacter / ""
local basepart_n     = Cs(fence_n^-1 * (weight_n * char_n)^1 * weight_n * fence_n^-1) / cache
local make_pattern_n = basepart_n * specpart

local function register_pattern(patterns,specials,str,specification)
    local k = lpegmatch(make_hashkey_p,str)
 -- local v1, v2 = lpegmatch(make_pattern_p,str)
    local v1, v2 = lpegmatch(make_pattern_n,str)
    patterns[k] = v1 -- is this key still ok for complex patterns
    if specification then
        specials[k] = specification
    elseif v2 then
        specials[k] = v2
    end
end

local function unregister_pattern(patterns,specials,str)
    local k = lpegmatch(make_hashkey_p,str)
    patterns[k] = nil
    specials[k] = nil
end

local p_lower = lpeg.patterns.utf8lower

local function register_exception(exceptions,str,specification)
    local l = lpegmatch(p_lower,str)
    local k = lpegmatch(make_hashkey_e,l)
    local v = lpegmatch(make_pattern_e,l)
    exceptions[k] = v
end

local p_pattern   = ((Carg(1) * Carg(2) * C(char^1)) / register_pattern   + 1)^1
local p_exception = ((Carg(1)           * C(char^1)) / register_exception + 1)^1
local p_split     = Ct(C(character)^1)

function traditional.loadpatterns(language,filename)
    local dictionary    = dictionaries[language]
    if not dictionary.loaded then
        if not filename or filename == "" then
            filename = "lang-" .. language
        end
        filename = file.addsuffix(filename,"lua")
        local fullname = resolvers.findfile(filename)
        if fullname and fullname ~= "" then
            local specification = dofile(fullname)
            if specification then
                local patterns = specification.patterns
                if patterns then
                    local data = patterns.data
                    if data and data ~= "" then
                        lpegmatch(p_pattern,data,1,dictionary.patterns,dictionary.specials)
                    end
                end
                local exceptions = specification.exceptions
                if exceptions then
                    local data = exceptions.data
                    if data and data ~= "" then
                        lpegmatch(p_exception,data,1,dictionary.exceptions)
                    end
                end
            end
        end
        dictionary.loaded = true
    end
    return dictionary
end

local lcchars    = characters.lcchars
local uccodes    = characters.uccodes
local categories = characters.categories
local nofwords   = 0
local nofhashed  = 0

local steps     = nil
local f_show    = formatters["%w%s"]

local function show_log()
    if trace_steps == true then
        report()
        local w = #steps[1][1]
        for i=1,#steps do
            local s = steps[i]
            report("%s%w%S  %S",s[1],w - #s[1] + 3,s[2],s[3] or "")
        end
        report()
    end
end

local function show_1(wsplit)
    local u = concat(wsplit," ")
    steps = { { f_show(0,u), f_show(0,u) } }
end

local function show_2(c,m,wsplit,done,i,spec)
    local s = lpegmatch(p_split,c)
    local t = { }
    local n = #m
    local w = #wsplit
    for j=1,n do
        t[#t+1] = m[j]
        t[#t+1] = s[j]
    end
    local m = 2*i-2
    local l = #t
    local s = spec and table.sequenced(spec) or ""
    if m == 0 then
        steps[#steps+1] = { f_show(m,  concat(t,"",2)),      f_show(1,concat(done," ",2,#done),s) }
    elseif i+1 == w then
        steps[#steps+1] = { f_show(m-1,concat(t,"",1,#t-1)), f_show(1,concat(done," ",2,#done),s) }
    else
        steps[#steps+1] = { f_show(m-1,concat(t)),           f_show(1,concat(done," ",2,#done),s) }
    end
end

local function show_3(wsplit,done)
    local t = { }
    local h = { }
    local n = #wsplit
    for i=1,n do
        local w = wsplit[i]
        if i > 1 then
            local d = done[i]
            t[#t+1] = i > 2 and d % 2 == 1 and "-" or " "
            h[#h+1] = d
        end
        t[#t+1] = w
        h[#h+1] = w
    end
    steps[#steps+1] = { f_show(0,concat(h)), f_show(0,concat(t)) }
    show_log()
end

local function show_4(wsplit,done)
    steps = { { concat(wsplit," ") } }
    show_log()
end

function traditional.lasttrace()
    return steps
end

-- We could reuse the w table but as we cache the resolved words there is not much gain in
-- that complication.
--
-- Beware: word can be a table and when n is passed to we can assume reuse so we need to
-- honor that n then.
--
-- todo: a fast variant for tex ... less lookups (we could check is dictionary has changed)
-- ... although due to caching the already done words, we don't do much here

local function hyphenate(dictionary,word,n) -- odd is okay
    nofwords = nofwords + 1
    local hyphenated = dictionary.hyphenated
    local isstring = type(word) == "string"
    if isstring then
        local done = hyphenated[word]
        if done ~= nil then
            return done
        end
    elseif n then
        local done = hyphenated[concat(word,"",1,n)]
        if done ~= nil then
            return done
        end
    else
        local done = hyphenated[concat(word)]
        if done ~= nil then
            return done
        end
    end
    local key
    if isstring then
        key = word
        word = lpegmatch(p_split,word)
        if not n then
            n = #word
        end
    else
        if not n then
            n = #word
        end
        key = concat(word,"",1,n)
    end
    local l = 1
    local w = { "." }
 -- local d = dictionary.codehash
    for i=1,n do
        local c = word[i]
     -- l = l + (d[c] or 1)
        l = l + 1
        w[l] = lcchars[c] or c
    end
    l = l + 1
    w[l] = "."
    local c = concat(w,"",2,l-1)
    --
    local done = hyphenated[c]
    if done ~= nil then
        hyphenated[key] = done
        nofhashed = nofhashed + 1
        return done
    end
    --
    local exceptions = dictionary.exceptions
    local exception  = exceptions[c]
    if exception then
        if trace_steps then
            show_4(w,exception)
        end
        hyphenated[key] = exception
        nofhashed = nofhashed + 1
        return exception
    end
    --
    if trace_steps then
        show_1(w)
    end
    --
    local specials = dictionary.specials
    local patterns = dictionary.patterns
    --
    local spec
    for i=1,l do
        for j=i,l do
            local c = concat(w,"",i,j)
            local m = patterns[c]
            if m then
                local s = specials[c]
                if not done then
                    done = { }
                    spec = nil
                    -- the string that we resolve has explicit fences (.) so done starts at
                    -- the first fence and runs upto the last one so we need one slot less
                    for i=1,l do
                        done[i] = 0
                    end
                end
                -- we run over the pattern that always has a (zero) value for each character
                -- plus one more as we look at both sides
                for k=1,#m do
                    local new = m[k]
                    if not new then
                        break
                    elseif new == true then
                        report("fatal error")
                        break
                    elseif new > 0 then
                        local pos = i + k - 1
                        local old = done[pos]
                        if not old then
                            -- break ?
                        elseif new > old then
                            done[pos] = new
                            if s then
                                local b = i + (s.start or 1) - 1
                                if b > 0 then
                                    local e = b + (s.length or 2) - 1
                                    if e > 0 then
                                        if pos >= b and pos <= e then
                                            if spec then
                                                spec[pos] = { s, k - 1 }
                                            else
                                                spec = { [pos] = { s, k - 1 } }
                                            end
                                        end
                                    end
                                end
                            end
                        end
                    end
                end
                if trace_steps and done then
                    show_2(c,m,w,done,i,s)
                end
            end
        end
    end
    if trace_steps and done then
        show_3(w,done)
    end
    if done then
        local okay = false
        for i=3,#done do
            if done[i] % 2 == 1 then
                done[i-2] = spec and spec[i] or true
                okay = true
            else
                done[i-2] = false
            end
        end
        if okay then
            done[#done] = nil
            done[#done] = nil
        else
            done = false
        end
    else
        done = false
    end
    hyphenated[key] = done
    nofhashed = nofhashed + 1
    return done
end

function traditional.gettrace(language,word)
    if not word or word == "" then
        return
    end
    local dictionary = dictionaries[language]
    if dictionary then
        local hyphenated = dictionary.hyphenated
        hyphenated[word] = nil
        hyphenate(dictionary,word)
        return steps
    end
end

local methods = setmetatableindex(function(t,k) local v = hyphenate t[k] = v return v end)

function traditional.installmethod(name,f)
    if rawget(methods,name) then
        report("overloading %a is not permitted",name)
    else
        methods[name] = f
    end
end

local s_detail_1 = "-"
local f_detail_2 = formatters["%s-%s"]
local f_detail_3 = formatters["{%s}{%s}{}"]
local f_detail_4 = formatters["{%s%s}{%s%s}{%s}"]

function traditional.injecthyphens(dictionary,word,specification)
    if not word then
        return false
    end
    if not specification then
        return word
    end
    local hyphens = hyphenate(dictionary,word)
    if not hyphens then
        return word
    end

    -- the following code is similar to code later on but here we have strings while there
    -- we have hyphen specs

    local word      = lpegmatch(p_split,word)
    local size      = #word

    local leftmin   = specification.leftcharmin or 2
    local rightmin  = size - (specification.rightcharmin or leftmin)
    local leftchar  = specification.leftchar
    local rightchar = specification.rightchar

    local result    = { }
    local rsize     = 0
    local position  = 1

    while position <= size do
        if position >= leftmin and position <= rightmin then
            local hyphen = hyphens[position]
            if not hyphen then
                rsize = rsize + 1
                result[rsize] = word[position]
                position = position + 1
            elseif hyphen == true then
                rsize = rsize + 1
                result[rsize] = word[position]
                rsize = rsize + 1
                if leftchar and rightchar then
                    result[rsize] = f_detail_3(rightchar,leftchar)
                else
                    result[rsize] = s_detail_1
                end
                position = position + 1
            else
                local o, h = hyphen[2]
                if o then
                    h = hyphen[1]
                else
                    h = hyphen
                    o = 1
                end
                local b = position - o + (h.start  or 1)
                local e = b + (h.length or 2) - 1
                if b > 0 and e >= b then
                    for i=1,b-position do
                        rsize = rsize + 1
                        result[rsize] = word[position]
                        position = position + 1
                    end
                    rsize = rsize + 1
                    if leftchar and rightchar then
                        result[rsize] = f_detail_4(h.before,rightchar,leftchar,h.after,concat(word,"",b,e))
                    else
                        result[rsize] = f_detail_2(h.before,h.after)
                    end
                    position = e + 1
                else
                    -- error
                    rsize = rsize + 1
                    result[rsize] = word[position]
                    position = position + 1
                end
            end
        else
            rsize = rsize + 1
            result[rsize] = word[position]
            position = position + 1
        end
    end
    return concat(result)
end

do

    local word      = C((1-space)^1)
    local spaces    = space^1

    local u_pattern = (Carg(1) * Carg(2) * word           / unregister_pattern + spaces)^1
    local r_pattern = (Carg(1) * Carg(2) * word * Carg(3) /   register_pattern + spaces)^1
    local e_pattern = (Carg(1)           * word           / register_exception + spaces)^1

    function traditional.registerpattern(language,str,specification)
        local dictionary = dictionaries[language]
        if specification == false then
            lpegmatch(u_pattern,str,1,dictionary.patterns,dictionary.specials)
         -- unregister_pattern(dictionary.patterns,dictionary.specials,str)
        else
            lpegmatch(r_pattern,str,1,dictionary.patterns,dictionary.specials,type(specification) == "table" and specification or false)
         -- register_pattern(dictionary.patterns,dictionary.specials,str,specification)
        end
    end

    function traditional.registerexception(language,str)
        lpegmatch(e_pattern,str,1,dictionaries[language].exceptions)
    end

end

-- todo: unicodes or utfhash ?

if context then

    local nodecodes          = nodes.nodecodes
    local disccodes          = nodes.disccodes

    local glyph_code         = nodecodes.glyph
    local disc_code          = nodecodes.disc
    local math_code          = nodecodes.math
    local hlist_code         = nodecodes.hlist

    local automaticdisc_code = disccodes.automatic
    local regulardisc_code   = disccodes.regular

    local nuts               = nodes.nuts
    local tonode             = nodes.tonode
    local nodepool           = nuts.pool

    local new_disc           = nodepool.disc
    local new_penalty        = nodepool.penalty

    local getfield           = nuts.getfield
    local getfont            = nuts.getfont
    local getid              = nuts.getid
    local getattr            = nuts.getattr
    local getnext            = nuts.getnext
    local getprev            = nuts.getprev
    local getsubtype         = nuts.getsubtype
    local getlist            = nuts.getlist
    local getlanguage        = nuts.getlanguage
    local getattrlist        = nuts.getattrlist
    local setattrlist        = nuts.setattrlist
    local isglyph            = nuts.isglyph
    local ischar             = nuts.ischar

    local setchar            = nuts.setchar
    local setdisc            = nuts.setdisc
    local setlink            = nuts.setlink
    local setprev            = nuts.setprev
    local setnext            = nuts.setnext

    local insertbefore       = nuts.insertbefore
    local insertafter        = nuts.insertafter
    local copy_node          = nuts.copy
    local copylist           = nuts.copylist
    local remove_node        = nuts.remove
    local endofmath          = nuts.endofmath
    local node_tail          = nuts.tail

    local nexthlist          = nuts.traversers.hlist
    local nextdisc           = nuts.traversers.disc

    local setcolor           = nodes.tracers.colors.set

    local variables          = interfaces.variables
    local v_reset            = variables.reset
    local v_yes              = variables.yes
    local v_word             = variables.word
    local v_all              = variables.all

    local settings_to_array  = utilities.parsers.settings_to_array

    local unsetvalue         = attributes.unsetvalue
    local texsetattribute    = tex.setattribute

    local prehyphenchar      = language.prehyphenchar
    local posthyphenchar     = language.posthyphenchar
    local preexhyphenchar    = language.preexhyphenchar
    local postexhyphenchar   = language.postexhyphenchar

    local a_hyphenation      = attributes.private("hyphenation")

    local interwordpenalty   = 5000

    function traditional.loadpatterns(language)
        return dictionaries[language]
    end

    -- for the moment we use an independent data structure

    setmetatableindex(dictionaries,function(t,k)
        if type(k) == "string" then
            -- this will force a load if not yet loaded (we need a nicer way) for the moment
            -- that will do (nneeded for examples that register a pattern specification
            languages.getnumber(k)
        end
        local specification = languages.getdata(k)
        local dictionary = {
            patterns   = { },
            exceptions = { },
            hyphenated = { },
            specials   = { },
            instance   = false,
            characters = { },
            unicodes   = { },
        }
        if specification then
            local resources = specification.resources
            if resources then
                local characters = dictionary.characters or { }
                local unicodes   = dictionary.unicodes   or { }
                for i=1,#resources do
                    local r = resources[i]
                    if not r.in_dictionary then
                        r.in_dictionary = true
                        local patterns = r.patterns
                        if patterns then
                            local data = patterns.data
                            if data then
                                -- regular patterns
                                lpegmatch(p_pattern,data,1,dictionary.patterns,dictionary.specials)
                            end
                            local extra = patterns.extra
                            if extra then
                                -- special patterns
                                lpegmatch(p_pattern,extra,1,dictionary.patterns,dictionary.specials)
                            end
                        end
                        local exceptions = r.exceptions
                        if exceptions then
                            local data = exceptions.data
                            if data and data ~= "" then
                                lpegmatch(p_exception,data,1,dictionary.exceptions)
                            end
                        end
                        local usedchars  = lpegmatch(p_split,patterns.characters)
                        for i=1,#usedchars do
                            local char  = usedchars[i]
                            local code  = utfbyte(char)
                            local upper = uccodes[code]
                            characters[char]  = code
                            unicodes  [code]  = char
                            if type(upper) == "table" then
                                for i=1,#upper do
                                    local u = upper[i]
                                    unicodes[u] = utfchar(u)
                                end
                            else
                                unicodes[upper] = utfchar(upper)
                            end
                        end
                    end
                end
                dictionary.characters = characters
                dictionary.unicodes   = unicodes
                setmetatableindex(characters,function(t,k) local v = k and utfbyte(k) t[k] = v return v end)
            end
            t[specification.number] = dictionary
            dictionary.instance = specification.instance -- needed for hyphenchars
        end
        t[k] = dictionary
        return dictionary
    end)

    -- Beware: left and right min doesn't mean that in a 1 mmm hsize there can be snippets
    -- with less characters than either of them! This could be an option but such a narrow
    -- hsize doesn't make sense anyway.

    -- We assume that featuresets are defined global ... local definitions (also mid paragraph)
    -- make not much sense anyway. For the moment we assume no predefined sets so we don't need
    -- to store them. Nor do we need to hash them in order to save space ... no sane user will
    -- define many of them.

    local featuresets       = hyphenators.featuresets or { }
    hyphenators.featuresets = featuresets

    storage.shared.noflanguagesfeaturesets = storage.shared.noflanguagesfeaturesets or 0

    local noffeaturesets = storage.shared.noflanguagesfeaturesets

    storage.register("languages/hyphenators/featuresets",featuresets,"languages.hyphenators.featuresets")

    ----- hash = table.sequenced(featureset,",") -- no need now

    local function register(name,featureset)
        noffeaturesets = noffeaturesets + 1
        featureset.attribute = noffeaturesets
        featuresets[noffeaturesets] = featureset  -- access by attribute
        featuresets[name] = featureset            -- access by name
        storage.shared.noflanguagesfeaturesets = noffeaturesets
        return noffeaturesets
    end

    local function makeset(...)
        -- a bit overkill, supporting variants but who cares
        local set = { }
        for i=1,select("#",...) do
            local list = select(i,...)
            local kind = type(list)
            local used = nil
            if kind == "string" then
                if list == v_all then
                    -- not ok ... now all get ignored
                    return setmetatableindex(function(t,k) local v = utfchar(k) t[k] = v return v end)
                elseif list ~= "" then
                    used = lpegmatch(p_split,list)
                    set  = set or { }
                    for i=1,#used do
                        local char = used[i]
                        set[utfbyte(char)] = char
                    end
                end
            elseif kind == "table" then
                if next(list) then
                    set = set or { }
                    for byte, char in next, list do
                        set[byte] = char == true and utfchar(byte) or char
                    end
                elseif #list > 0 then
                    set = set or { }
                    for i=1,#list do
                        local l = list[i]
                        if type(l) == "number" then
                            set[l] = utfchar(l)
                        else
                            set[utfbyte(l)] = l
                        end
                    end
                end
            end
        end
        return set
    end

    -- category pd (tex also sees --- and -- as hyphens but do we really want that

    local defaulthyphens = {
        [0x002D] = true,   -- HYPHEN-MINUS
        [0x00AD] = 0x002D, -- SOFT HYPHEN (active in ConTeXt)
     -- [0x058A] = true,   -- ARMENIAN HYPHEN
     -- [0x1400] = true,   -- CANADIAN SYLLABICS HYPHEN
     -- [0x1806] = true,   -- MONGOLIAN TODO SOFT HYPHEN
        [0x2010] = true,   -- HYPHEN
     -- [0x2011] = true,   -- NON-BREAKING HYPHEN
     -- [0x2012] = true,   -- FIGURE DASH
        [0x2013] = true,   -- EN DASH
        [0x2014] = true,   -- EM DASH
     -- [0x2015] = true,   -- HORIZONTAL BAR
     -- [0x2027] = true,   -- HYPHENATION POINT
     -- [0x2E17] = true,   -- DOUBLE OBLIQUE HYPHEN
     -- [0x2E1A] = true,   -- HYPHEN WITH DIAERESIS
     -- [0x2E3A] = true,   -- TWO-EM DASH
     -- [0x2E3B] = true,   -- THREE-EM DASH
     -- [0x2E40] = true,   -- DOUBLE HYPHEN
     -- [0x301C] = true,   -- WAVE DASH
     -- [0x3030] = true,   -- WAVY DASH
     -- [0x30A0] = true,   -- KATAKANA-HIRAGANA DOUBLE HYPHEN
     -- [0xFE31] = true,   -- PRESENTATION FORM FOR VERTICAL EM DASH
     -- [0xFE32] = true,   -- PRESENTATION FORM FOR VERTICAL EN DASH
     -- [0xFE58] = true,   -- SMALL EM DASH
     -- [0xFE63] = true,   -- SMALL HYPHEN-MINUS
     -- [0xFF0D] = true,   -- FULLWIDTH HYPHEN-MINUS
    }

    local defaultjoiners = {
        [0x200C] = true, -- nzwj
        [0x200D] = true, -- zwj
    }

    local function somehyphenchar(c)
        c = tonumber(c)
        return c ~= 0 and c or nil
    end

    local function definefeatures(name,featureset)
        local extrachars   = featureset.characters -- "[]()"
        local hyphenchars  = featureset.hyphens
        local joinerchars  = featureset.joiners
        local alternative  = featureset.alternative
        local rightwordmin = tonumber(featureset.rightwordmin)
        local charmin      = tonumber(featureset.charmin) -- luatex now also has hyphenationmin
        local leftcharmin  = tonumber(featureset.leftcharmin)
        local rightcharmin = tonumber(featureset.rightcharmin)
        local leftchar     = somehyphenchar(featureset.leftchar)
        local rightchar    = somehyphenchar(featureset.rightchar)
        local rightchars   = featureset.rightchars
local rightedge    = featureset.rightedge
local autohyphen   = v_yes -- featureset.autohyphen -- insert disc
local hyphenonly   = v_yes -- featureset.hyphenonly -- don't hyphenate around
        rightchars  = rightchars  == v_word and true           or tonumber(rightchars)
        joinerchars = joinerchars == v_yes  and defaultjoiners or joinerchars -- table
        hyphenchars = hyphenchars == v_yes  and defaulthyphens or hyphenchars -- table
        -- not yet ok: extrachars have to be ignored  so it cannot be all)
        featureset.extrachars   = makeset(joinerchars or "",extrachars or "")
        featureset.hyphenchars  = makeset(hyphenchars or "")
        featureset.alternative  = alternative or "hyphenate"
        featureset.rightwordmin = rightwordmin and rightwordmin > 0 and rightwordmin or nil
        featureset.charmin      = charmin      and charmin      > 0 and charmin      or nil
        featureset.leftcharmin  = leftcharmin  and leftcharmin  > 0 and leftcharmin  or nil
        featureset.rightcharmin = rightcharmin and rightcharmin > 0 and rightcharmin or nil
        featureset.rightchars   = rightchars
        featureset.leftchar     = leftchar
        featureset.rightchar    = rightchar
     -- featureset.strict       = rightedge  == "tex"
featureset.autohyphen   = autohyphen == v_yes
featureset.hyphenonly   = hyphenonly == v_yes
        return register(name,featureset)
    end

    local function setfeatures(n)
        if not n or n == v_reset then
            n = false
        else
            local f = featuresets[n]
            if not f and type(n) == "string" then
                local t = settings_to_array(n)
                local s = { }
                for i=1,#t do
                    local ti = t[i]
                    local fs = featuresets[ti]
                    if fs then
                        for k, v in next, fs do
                            s[k] = v
                        end
                    end
                end
                n = register(n,s)
            else
                n = f and f.attribute
            end
        end
        texsetattribute(a_hyphenation,n or unsetvalue)
    end

    traditional.definefeatures = definefeatures
    traditional.setfeatures    = setfeatures

    implement {
        name      = "definehyphenationfeatures",
        actions   = definefeatures,
        arguments = {
            "string",
            {
                { "characters" },
                { "hyphens" },
                { "joiners" },
                { "rightchars" },
                { "rightwordmin", "integer" },
                { "charmin", "integer" },
                { "leftcharmin", "integer" },
                { "rightcharmin", "integer" },
                { "leftchar", "integer" },
                { "rightchar", "integer" },
                { "alternative" },
                { "rightedge" },
            }
        }
    }

    implement {
        name      = "sethyphenationfeatures",
        actions   = setfeatures,
        arguments = "string"
    }

    implement {
        name      = "registerhyphenationpattern",
        actions   = traditional.registerpattern,
        arguments = { "string",  "string",  "boolean" }
    }

    implement {
        name      = "registerhyphenationexception",
        actions   = traditional.registerexception,
        arguments = "2 strings",
    }

    -- This is a relative large function with local variables and local functions. A previous
    -- implementation had the functions outside but this is cleaner and as efficient. The test
    -- runs 100 times over tufte.tex, knuth.tex, zapf.tex, ward.tex and darwin.tex in lower
    -- and uppercase with a 1mm hsize.
    --
    --         language=0     language>0     4 | 3 * slower
    --
    -- tex     2.34 | 1.30    2.55 | 1.45    0.21 | 0.15
    -- lua     2.42 | 1.38    3.30 | 1.84    0.88 | 0.46
    --
    -- Of course we have extra overhead (virtual Lua machine) but also we check attributes and
    -- support specific local options). The test puts the typeset text in boxes and discards
    -- it. If we also flush the runtime is 4.31|2.56 and 4.99|2.94 seconds so the relative
    -- difference is (somehow) smaller. The test has 536 pages. There is a little bit of extra
    -- overhead because we store the patterns in a different way.
    --
    -- As usual I will look for speedups. Some 0.01 seconds could be gained by sharing patterns
    -- which is not impressive but it does save some 3M memory on this test. (Some optimizations
    -- already brought the 3.30 seconds down to 3.14 but it all depends on aggressive caching.)

    -- As we kick in the hyphenator before fonts get handled, we don't look at implicit (font)
    -- kerns or ligatures.

    local starttiming = statistics.starttiming
    local stoptiming  = statistics.stoptiming

 -- local strictids = {
 --     [nodecodes.hlist]   = true,
 --     [nodecodes.vlist]   = true,
 --     [nodecodes.rule]    = true,
 --     [nodecodes.dir]     = true,
 --     [nodecodes.whatsit] = true,
 --     [nodecodes.insert]  = true,
 --     [nodecodes.adjust]  = true,
 --
 --     [nodecodes.math]    = true,
 --     [nodecodes.disc]    = true,
 --
 --     [nodecodes.accent]  = true, -- never used in context
 -- }

    -- a lot of overhead when only one char

    function traditional.hyphenate(head)

        local first           = head
        local tail            = nil
        local last            = nil
        local current         = first
        local dictionary      = nil
        local instance        = nil
        local characters      = nil
        local unicodes        = nil
        local exhyphenchar    = tex.exhyphenchar
        local extrachars      = nil
        local hyphenchars     = nil
        local language        = nil
        local lastfont        = nil
        local start           = nil
        local stop            = nil
        local word            = { } -- we reuse this table
        local size            = 0
     -- local leftchar        = false
     -- local rightchar       = false -- utfbyte("-")
        local leftexchar      = false
        local rightexchar     = false -- utfbyte("-")
        local leftmin         = 0
        local rightmin        = 0
        local charmin         = 1
        local leftcharmin     = nil
        local rightcharmin    = nil
        ----- leftwordmin     = nil
        local rightwordmin    = nil
        local rightchars      = nil
        local leftchar        = nil
        local rightchar       = nil
        local attr            = nil
        local lastwordlast    = nil
        local hyphenated      = hyphenate
        ----- strict          = nil
        local exhyphenpenalty = tex.exhyphenpenalty
        local hyphenpenalty   = tex.hyphenpenalty
        local autohyphen      = false
        local hyphenonly      = false

        -- We cannot use an 'enabled' boolean (false when no characters or extras) because we
        -- can have plugins that set a characters metatable and so) ... it doesn't save much
        -- anyway. Using (unicodes and unicodes[code]) and a nil table when no characters also
        -- doesn't save much. So there not that much to gain for languages that don't hyphenate.
        --
        -- enabled = (unicodes and (next(unicodes) or getmetatable(unicodes)))
        --        or (extrachars and next(extrachars))
        --
        -- This can be used to not add characters i.e. keep size 0 but then we need to check for
        -- attributes that change it, which costs time too. Not much to gain there.

        starttiming(traditional)

        local function insertpenalty()
            local p = new_penalty(interwordpenalty)
            setattrlist(p,last)
            if trace_visualize then
                nuts.setvisual(p,"penalty")
            end
            last = getprev(last)
            first, last = insertafter(first,last,p)
        end

        local function synchronizefeatureset(a)
            local f = a and featuresets[a]
            if f then
                hyphenated   = methods[f.alternative or "hyphenate"]
                extrachars   = f.extrachars
                hyphenchars  = f.hyphenchars
                rightwordmin = f.rightwordmin
                charmin      = f.charmin
                leftcharmin  = f.leftcharmin
                rightcharmin = f.rightcharmin
                leftchar     = f.leftchar
                rightchar    = f.rightchar
             -- strict       = f.strict and strictids
                rightchars   = f.rightchars
                autohyphen   = f.autohyphen
                hyphenonly   = f.hyphenonly
                if rightwordmin and rightwordmin > 0 and lastwordlast ~= rightwordmin then
                    -- so we can change mid paragraph but it's kind of unpredictable then
                    if not tail then
                        tail = node_tail(first)
                    end
                    last = tail
                    local inword = false
                    local count  = 0
                    while last and rightwordmin > 0 do
                        local id = getid(last)
                        if id == glyph_code then
                            count = count + 1
                            inword = true
                            if trace_visualize then
                                setcolor(last,"darkgreen")
                            end
                        elseif inword then
                            inword = false
                            rightwordmin = rightwordmin - 1
                            if rightchars == true then
                                if rightwordmin > 0 then
                                    insertpenalty()
                                end
                            elseif rightchars and count <= rightchars then
                                insertpenalty()
                            end
                        end
                        last = getprev(last)
                    end
                    lastwordlast = rightwordmin
                end
                if not charmin or charmin == 0 then
                    charmin = 1
                end
            else
                hyphenated   = methods.hyphenate
                extrachars   = false
                hyphenchars  = false
                rightwordmin = false
                charmin      = 1
                leftcharmin  = false
                rightcharmin = false
                leftchar     = false
                rightchar    = false
             -- strict       = false
                autohyphen   = false
                hyphenonly   = false
            end

            return a
        end

        local function flush(hyphens) -- todo: no need for result

            local rightmin = size - rightmin
            local result   = { }
            local rsize    = 0
            local position = 1

            -- todo: remember last dics and don't go back to before that (plus message) ...
            -- for simplicity we also assume that we don't start with a dics node
            --
            -- there can be a conflict: if we backtrack then we can end up in another disc
            -- and get out of sync (dup chars and so)

            while position <= size do
                if position >= leftmin and position <= rightmin then
                    local hyphen = hyphens[position]
                    if not hyphen then
                        rsize = rsize + 1
                        result[rsize] = word[position]
                        position = position + 1
                    elseif hyphen == true then
                        rsize = rsize + 1
                        result[rsize] = word[position]
                        rsize = rsize + 1
                        result[rsize] = true
                        position = position + 1
                    else
                        local o, h = hyphen[2]
                        if o then
                            -- { hyphen, offset)
                            h = hyphen[1]
                        else
                            -- hyphen
                            h = hyphen
                            o = 1
                        end
                        local b = position - o + (h.start  or 1)
                        local e = b + (h.length or 2) - 1
                        if b > 0 and e >= b then
                            for i=1,b-position do
                                rsize = rsize + 1
                                result[rsize] = word[position]
                                position = position + 1
                            end
                            rsize = rsize + 1
                            result[rsize] = {
                                h.before or "",      -- pre
                                h.after or "",       -- post
                                concat(word,"",b,e), -- replace
                                h.right,             -- optional after pre
                                h.left,              -- optional before post
                            }
                            position = e + 1
                        else
                            -- error
                            rsize = rsize + 1
                            result[rsize] = word[position]
                            position = position + 1
                        end
                    end
                else
                    rsize = rsize + 1
                    result[rsize] = word[position]
                    position = position + 1
                end
            end

            local function serialize(replacement,leftchar,rightchar)
                if not replacement then
                    return
                elseif replacement == true then
                    local glyph = copy_node(stop)
                    setchar(glyph,leftchar or rightchar)
                    return glyph
                end
                local head    = nil
                local current = nil
                if leftchar then
                    head    = copy_node(stop)
                    current = head
                    setchar(head,leftchar)
                end
                local rsize = #replacement
                if rsize == 1 then
                    local glyph = copy_node(stop)
                    setchar(glyph,characters[replacement])
                    if head then
                        insertafter(current,current,glyph)
                    else
                        head = glyph
                    end
                    current = glyph
                elseif rsize > 0 then
                    local list = lpegmatch(p_split,replacement) -- this is an utf split (could be cached)
                    for i=1,#list do
                        local glyph = copy_node(stop)
                        setchar(glyph,characters[list[i]])
                        if head then
                            insertafter(current,current,glyph)
                        else
                            head = glyph
                        end
                        current = glyph
                    end
                end
                if rightchar then
                    local glyph = copy_node(stop)
                    insertafter(current,current,glyph)
                    setchar(glyph,rightchar)
                end
                return head
            end

            local current  = start
            local attrnode = start -- will be different, just the first char

            for i=1,rsize do
                local r = result[i]
                if r == true then
                    local disc = new_disc()
                    local pre  = nil
                    local post = nil
                    if rightchar then
                        pre = serialize(true,rightchar)
                    end
                    if leftchar then
                        post = serialize(true,leftchar)
                    end
                    setdisc(disc,pre,post,nil,regulardisc_code,hyphenpenalty)
                    if attrnode then
                        setattrlist(disc,attrnode)
                    end
                    -- could be a replace as well
                    insertbefore(first,current,disc)
                elseif type(r) == "table" then
                    local disc    = new_disc()
                    local pre     = r[1]
                    local post    = r[2]
                    local replace = r[3]
                    local right   = r[4] ~= false and rightchar
                    local left    = r[5] ~= false and leftchar
                    if pre then
                        if pre ~= "" then
                            pre = serialize(pre,false,right)
                        else
                            pre = nil
                        end
                    end
                    if post then
                        if post ~= "" then
                            post = serialize(post,left,false)
                        else
                            post = nil
                        end
                    end
                    if replace then
                        if replace ~= "" then
                            replace = serialize(replace)
                        else
                            replace = nil
                        end
                    end
                    -- maybe regular code
                    setdisc(disc,pre,post,replace,regulardisc_code,hyphenpenalty)
                    if attrnode then
                        setattrlist(disc,attrnode)
                    end
                    insertbefore(first,current,disc)
                else
                    setchar(current,characters[r])
                    if i < rsize then
                        current = getnext(current)
                    end
                end
            end
            if current and current ~= stop then
                local current = getnext(current)
                local last    = getnext(stop)
                while current ~= last do
                    first, current = remove_node(first,current,true)
                end
            end

        end

        local function inject(leftchar,rightchar,code,attrnode)
            if first ~= current then
                local disc = new_disc()
                first, current, glyph = remove_node(first,current)
                first, current = insertbefore(first,current,disc)
                if trace_visualize then
                    setcolor(glyph,"darkred")  -- these get checked
                    setcolor(disc,"darkgreen") -- in the colorizer
                end
                local pre     = nil
                local post    = nil
                local replace = glyph
                if leftchar and leftchar > 0 then
                    post = copy_node(glyph)
                    setchar(post,leftchar)
                end
                pre = copy_node(glyph)
                setchar(pre,rightchar and rightchar > 0 and rightchar or code)
                setdisc(disc,pre,post,replace,automaticdisc_code,hyphenpenalty) -- ex ?
                if attrnode then
                    setattrlist(disc,attrnode)
                end
            end
            return current
        end

        local function injectseries(current,last,next,attrnode)
            local disc  = new_disc()
            local start = current
            first, current = insertbefore(first,current,disc)
            setprev(start)
            setnext(last)
            if next then
                setlink(current,next)
            else
                setnext(current)
            end
            local pre     = copylist(start)
            local post    = nil
            local replace = start
            setdisc(disc,pre,post,replace,automaticdisc_code,hyphenpenalty) -- ex ?
            if attrnode then
                setattrlist(disc,attrnode)
            end
            return current
        end

        local a = getattr(first,a_hyphenation)
        if a ~= attr then
            attr = synchronizefeatureset(a)
        end

        -- The first attribute in a word determines the way a word gets hyphenated and if
        -- relevant, other properties are also set then. We could optimize for silly one-char
        -- cases but it has no priority as the code is still not that much slower than the
        -- native hyphenator and this variant also provides room for extensions.

        local skipping = false

        -- In "word word word." the sequences "word" and "." can be a different font!

        while current and current ~= last do -- and current
            local code, id = isglyph(current)
            if code then
                if skipping then
                    current = getnext(current)
                else
                    local lang = getlanguage(current)
                    local font = getfont(current)
                    if lang ~= language or font ~= lastfont then
                        if dictionary and size > charmin and leftmin + rightmin <= size then
                            -- only german has many words starting with an uppercase character
                            if categories[word[1]] == "lu" and getfield(start,"uchyph") < 0 then
                                -- skip
                            else
                                local hyphens = hyphenated(dictionary,word,size)
                                if hyphens then
                                    flush(hyphens)
                                end
                            end
                        end
                        lastfont = font
                        if language ~= lang and lang > 0 then
                            --
                            dictionary = dictionaries[lang]
                            instance   = dictionary.instance
                            characters = dictionary.characters
                            unicodes   = dictionary.unicodes
                            --
                            local a = getattr(current,a_hyphenation)
                            attr        = synchronizefeatureset(a)
                            leftchar    = leftchar     or (instance and posthyphenchar  (instance)) -- we can make this more
                            rightchar   = rightchar    or (instance and prehyphenchar   (instance)) -- efficient if needed
                            leftexchar  =                 (instance and preexhyphenchar (instance))
                            rightexchar =                 (instance and postexhyphenchar(instance))
                            leftmin     = leftcharmin  or getfield(current,"lhmin")
                            rightmin    = rightcharmin or getfield(current,"rhmin")
                            if not leftchar or leftchar < 0 then
                                leftchar = false
                            end
                            if not rightchar or rightchar < 0 then
                                rightchar = false
                            end
                            --
                            local char = unicodes[code] or (extrachars and extrachars[code])
                            if char then
                                word[1] = char
                                size    = 1
                                start   = current
                            else
                                size = 0
                            end
                        else
                            size = 0
                        end
                        language = lang
                    elseif language <= 0 then
                        --
                    elseif size > 0 then
                        local char = unicodes[code] or (extrachars and extrachars[code])
                        if char then
                            size = size + 1
                            word[size] = char
                        elseif dictionary then
                            if not hyphenonly or code ~= exhyphenchar then
                                if size > charmin and leftmin + rightmin <= size then
                                    if categories[word[1]] == "lu" and getfield(start,"uchyph") < 0 then
                                        -- skip
                                    else
                                        local hyphens = hyphenated(dictionary,word,size)
                                        if hyphens then
                                            flush(hyphens)
                                        end
                                    end
                                end
                            end
                            size = 0
                            if code == exhyphenchar then -- normally the -
                                local next = getnext(current)
                                local last = current
                                local font = getfont(current)
                                while next and ischar(next,font) == code do
                                    last = next
                                    next = getnext(next)
                                end
                                if not autohyphen then
                                    current = last
                                elseif current == last then
                                    current = inject(leftexchar,rightexchar,code,current)
                                else
                                    current = injectseries(current,last,next,current)
                                end
                                if hyphenonly then
                                    skipping = true
                                end
                            elseif hyphenchars then
                                local char = hyphenchars[code]
                                if char == true then
                                    char = code
                                end
                                if char then
                                    current = inject(leftchar and char or nil,rightchar and char or nil,char,current)
                                end
                            end
                        end
                    else
                        local a = getattr(current,a_hyphenation)
                        if a ~= attr then
                            attr        = synchronizefeatureset(a) -- influences extrachars
                            leftchar    = leftchar     or (instance and posthyphenchar  (instance)) -- we can make this more
                            rightchar   = rightchar    or (instance and prehyphenchar   (instance)) -- efficient if needed
                            leftexchar  =                 (instance and preexhyphenchar (instance))
                            rightexchar =                 (instance and postexhyphenchar(instance))
                            leftmin     = leftcharmin  or getfield(current,"lhmin")
                            rightmin    = rightcharmin or getfield(current,"rhmin")
                            if not leftchar or leftchar < 0 then
                                leftchar = false
                            end
                            if not rightchar or rightchar < 0 then
                                rightchar = false
                            end
                        end
                        --
                        local char = unicodes[code] or (extrachars and extrachars[code])
                        if char then
                            word[1] = char
                            size    = 1
                            start   = current
                        end
                    end
                    stop    = current
                    current = getnext(current)
                end
            else
                if skipping then
                    skipping = false
                end
                if id == disc_code then
                    size = 0
                    current = getnext(current)
                    if hyphenonly then
                        skipping = true
                    end
             -- elseif strict and strict[id] then
             --     current = id == math_code and getnext(endofmath(current)) or getnext(current)
             --     size = 0
                else
                    current = id == math_code and getnext(endofmath(current)) or getnext(current)
                end
                if size > 0 then
                    if dictionary and size > charmin and leftmin + rightmin <= size then
                        if categories[word[1]] == "lu" and getfield(start,"uchyph") < 0 then
                            -- skip
                        else
                            local hyphens = hyphenated(dictionary,word,size)
                            if hyphens then
                                flush(hyphens)
                            end
                        end
                    end
                    size = 0
                end
            end
        end
        -- we can have quit due to last so we need to flush the last seen word, we could move
        -- this in the loop and test for current but ... messy
        if dictionary and size > charmin and leftmin + rightmin <= size then
            if categories[word[1]] == "lu" and getfield(start,"uchyph") < 0 then
                -- skip
            else
                local hyphens = hyphenated(dictionary,word,size)
                if hyphens then
                    flush(hyphens)
                end
            end
        end

        stoptiming(traditional)

        return head
    end

    statistics.register("hyphenation",function()
        if nofwords > 0 or statistics.elapsed(traditional) > 0 then
            return string.format("%s words hyphenated, %s unique, used time %s",
                nofwords,nofhashed,statistics.elapsedseconds(traditional) or 0)
        end
    end)

    local texmethod = "builders.kernel.hyphenation"
    local oldmethod = texmethod
    local newmethod = texmethod

 -- local newmethod = "languages.hyphenators.traditional.hyphenate"
 --
 -- nodes.tasks.prependaction("processors","words",newmethod)
 -- nodes.tasks.disableaction("processors",oldmethod)
 --
 -- nodes.tasks.replaceaction("processors","words",oldmethod,newmethod)

 -- \enabledirectives[hyphenators.method=traditional]
 -- \enabledirectives[hyphenators.method=builtin]

    -- push / pop ? check first attribute

    -- local replaceaction = nodes.tasks.replaceaction -- no longer overload this way (too many local switches)

    local hyphenate    = language.hyphenate
    local hyphenating  = nuts.hyphenating
    local methods      = { }
    local usedmethod   = false
    local stack        = { }

    local original = hyphenating and
        function(head)
            return (hyphenating(head))
        end
    or
        function(head)
            hyphenate(tonode(head))
            return head -- a nut
        end

 -- local has_language = language.has_language
 --
 -- local function original(head) -- kernel.hyphenation(head)
 --     local h = tonode(head)
 --     if has_language(h) then
 --         hyphenate(h)
 --     end
 --     return head
 -- end

    local getcount = tex.getcount

    hyphenators.methods  = methods
    local optimize       = false

    directives.register("hyphenator.optimize", function(v) optimize = v end)

    function hyphenators.handler(head,groupcode)
        if usedmethod then
            if optimize and (groupcode == "hbox" or groupcode == "adjustedhbox") then
                if getcount("hyphenstate") > 0 then
                    forced = false
                    return usedmethod(head)
                else
                    return head
                end
            else
                return usedmethod(head)
            end
        else
            return head
        end
    end

    methods.tex         = original
    methods.original    = original
    methods.expanded    = original -- was expanded before 1.005
    methods.traditional = languages.hyphenators.traditional.hyphenate
    methods.none        = false -- function(head) return head, false end

    usedmethod          = original

    local function setmethod(method)
        usedmethod = type(method) == "string" and methods[method]
        if usedmethod == nil then
            usedmethod = methods.tex
        end
    end
    local function pushmethod(method)
        insert(stack,usedmethod)
        usedmethod = type(method) == "string" and methods[method]
        if usedmethod == nil then
            usedmethod = methods.tex
        end
    end
    local function popmethod()
        usedmethod = remove(stack) or methods.tex
    end

    hyphenators.setmethod  = setmethod
    hyphenators.pushmethod = pushmethod
    hyphenators.popmethod  = popmethod

    directives.register("hyphenators.method",setmethod)

    function hyphenators.setup(specification)
        local method = specification.method
        if method then
            setmethod(method)
        end
    end

    implement { name = "sethyphenationmethod", actions = setmethod,  arguments = "string" }
    implement { name = "pushhyphenation",      actions = pushmethod, arguments = "string" }
    implement { name = "pophyphenation",       actions = popmethod }

    -- can become a runtime loaded one:

    local context      = context
    local ctx_NC       = context.NC
    local ctx_NR       = context.NR
    local ctx_verbatim = context.verbatim

    function hyphenators.showhyphenationtrace(language,word)
        if not word or word == "" then
            return
        end
        local saved = trace_steps
        trace_steps = "silent"
        local steps = traditional.gettrace(language,word)
        trace_steps = saved
        if steps then
            local n = #steps
            if n > 0 then
                context.starttabulate { "|r|l|l|l|" }
                for i=1,n do
                    local s = steps[i]
                    ctx_NC() if i > 1 and i < n then context(i-1) end
                    ctx_NC() ctx_verbatim(s[1])
                    ctx_NC() ctx_verbatim(s[2])
                    ctx_NC() ctx_verbatim(s[3])
                    ctx_NC()
                    ctx_NR()
                end
                context.stoptabulate()
            end
        end
    end

    implement {
        name      = "showhyphenationtrace",
        actions   = hyphenators.showhyphenationtrace,
        arguments = "2 strings",
    }

    function nodes.stripdiscretionaries(head)
        for l in nexthlist, head do
            for d in nextdisc, getlist(l) do
                remove_node(h,false,true)
            end
        end
        return head
    end


else

    -- traditional.loadpatterns("nl","lang-nl")
    -- traditional.loadpatterns("de","lang-de")
    -- traditional.loadpatterns("us","lang-us")

    -- traditional.registerpattern("nl","e1ë",      { start = 1, length = 2, before = "e",  after = "e"  } )
    -- traditional.registerpattern("nl","oo7ë",     { start = 2, length = 3, before = "o",  after = "e"  } )
    -- traditional.registerpattern("de","qqxc9xkqq",{ start = 3, length = 4, before = "ab", after = "cd" } )

    -- local specification = {
    --     leftcharmin     = 2,
    --     rightcharmin    = 2,
    --     leftchar        = "<",
    --     rightchar       = ">",
    -- }

    -- print("reëel",       traditional.injecthyphens(dictionaries.nl,"reëel",       specification),"r{e>}{<e}{eë}el")
    -- print("reeëel",      traditional.injecthyphens(dictionaries.nl,"reeëel",      specification),"re{e>}{<e}{eë}el")
    -- print("rooëel",      traditional.injecthyphens(dictionaries.nl,"rooëel",      specification),"r{o>}{<e}{ooë}el")

    -- print(   "qxcxkq",   traditional.injecthyphens(dictionaries.de,   "qxcxkq",   specification),"")
    -- print(  "qqxcxkqq",  traditional.injecthyphens(dictionaries.de,  "qqxcxkqq",  specification),"")
    -- print( "qqqxcxkqqq", traditional.injecthyphens(dictionaries.de, "qqqxcxkqqq", specification),"")
    -- print("qqqqxcxkqqqq",traditional.injecthyphens(dictionaries.de,"qqqqxcxkqqqq",specification),"")

    -- print("kunstmatig",       traditional.injecthyphens(dictionaries.nl,"kunstmatig",       specification),"")
    -- print("kunststofmatig",   traditional.injecthyphens(dictionaries.nl,"kunststofmatig",   specification),"")
    -- print("kunst[stof]matig", traditional.injecthyphens(dictionaries.nl,"kunst[stof]matig", specification),"")

    -- traditional.loadpatterns("us","lang-us")

    -- local specification = {
    --     leftcharmin     = 2,
    --     rightcharmin    = 2,
    --     leftchar        = false,
    --     rightchar       = false,
    -- }

    -- trace_steps = true

    -- print("components",    traditional.injecthyphens(dictionaries.us,"components", specification),"")
    -- print("single",        traditional.injecthyphens(dictionaries.us,"single",     specification),"sin-gle")
    -- print("everyday",      traditional.injecthyphens(dictionaries.us,"everyday",   specification),"every-day")
    -- print("associate",     traditional.injecthyphens(dictionaries.us,"associate",     specification),"as-so-ciate")
    -- print("philanthropic", traditional.injecthyphens(dictionaries.us,"philanthropic", specification),"phil-an-thropic")
    -- print("projects",      traditional.injecthyphens(dictionaries.us,"projects",      specification),"projects")
    -- print("Associate",     traditional.injecthyphens(dictionaries.us,"Associate",     specification),"As-so-ciate")
    -- print("Philanthropic", traditional.injecthyphens(dictionaries.us,"Philanthropic", specification),"Phil-an-thropic")
    -- print("Projects",      traditional.injecthyphens(dictionaries.us,"Projects",      specification),"Projects")

end