path: root/tex/context/base/mkiv/char-utf.lua

if not modules then modules = { } end modules ['char-utf'] = {
    version   = 1.001,
    comment   = "companion to char-utf.mkiv",
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

--[[ldx--
<p>When a sequence of <l n='utf'/> characters enters the application, it may be
neccessary to collapse subsequences into their composed variant.</p>

<p>This module implements methods for collapsing and expanding <l n='utf'/>
sequences. We also provide means to deal with characters that are special to
<l n='tex'/> as well as 8-bit characters that need to end up in special kinds
of output (for instance <l n='pdf'/>).</p>

<p>We implement these manipulations as filters. One can run multiple filters
over a string.</p>
--ldx]]--

local gsub, find = string.gsub, string.find
local concat, sortedhash, keys, sort = table.concat, table.sortedhash, table.keys, table.sort
local utfchar, utfbyte, utfcharacters, utfvalues = utf.char, utf.byte, utf.characters, utf.values
local P, Cs, Cmt, Ct = lpeg.P, lpeg.Cs, lpeg.Cmt, lpeg.Ct

if not characters        then require("char-def") end
if not characters.blocks then require("char-ini") end

local lpegmatch             = lpeg.match
local lpegpatterns          = lpeg.patterns
local p_utf8character       = lpegpatterns.utf8character
local p_utf8byte            = lpegpatterns.utf8byte
local utfchartabletopattern = lpeg.utfchartabletopattern

local formatters            = string.formatters

local allocate              = utilities.storage.allocate or function() return { } end

local charfromnumber        = characters.fromnumber

characters                  = characters or { }
local characters            = characters

local graphemes             = allocate()
characters.graphemes        = graphemes

local collapsed             = allocate()
characters.collapsed        = collapsed

local combined              = allocate()
characters.combined         = combined

local decomposed            = allocate()
characters.decomposed       = decomposed

local mathpairs             = allocate()
characters.mathpairs        = mathpairs

local filters               = allocate()
characters.filters          = filters

local utffilters            = { }
characters.filters.utf      = utffilters

local data                  = characters.data

-- is characters.combined cached?

--[[ldx--
<p>It only makes sense to collapse at runtime, since we don't expect source code
to depend on collapsing.</p>
--ldx]]--

-- for the moment, will be entries in char-def.lua .. this is just a subset that for
-- typographic (font) reasons we want to have split ... if we decompose all, we get
-- problems with fonts

local decomposed = allocate {
    ["IJ"] = "IJ",
    ["ij"] = "ij",
    ["և"] = "եւ",
    ["ff"] = "ff",
    ["fi"] = "fi",
    ["fl"] = "fl",
    ["ffi"] = "ffi",
    ["ffl"] = "ffl",
    ["ſt"] = "ſt",
    ["st"] = "st",
    ["ﬓ"] = "մն",
    ["ﬔ"] = "մե",
    ["ﬕ"] = "մի",
    ["ﬖ"] = "վն",
    ["ﬗ"] = "մխ",
}

characters.decomposed = decomposed

-- local function initialize() -- maybe only 'mn'
--     local data = characters.data
--     for unicode, v in next, data do
--         -- using vs and first testing for length is faster (.02->.01 s)
--         local vs = v.specials
--         if vs and #vs == 3 then
--             local vc = vs[1]
--             if vc == "char" then
--                 local one, two = vs[2], vs[3]
--                 if data[two].category == "mn" then
--                     local cgf = combined[one]
--                     if not cgf then
--                         cgf = { [two] = unicode }
--                         combined[one]  = cgf
--                     else
--                         cgf[two] = unicode
--                     end
--                 end
--                 local first, second, combination = utfchar(one), utfchar(two), utfchar(unicode)
--                 local cgf = graphemes[first]
--                 if not cgf then
--                     cgf = { [second] = combination }
--                     graphemes[first] = cgf
--                 else
--                     cgf[second] = combination
--                 end
--                 if v.mathclass or v.mathspec then
--                     local mps = mathpairs[two]
--                     if not mps then
--                         mps = { [one] = unicode }
--                         mathpairs[two] = mps
--                     else
--                         mps[one] = unicode -- here unicode
--                     end
--                     local mps = mathpairs[second]
--                     if not mps then
--                         mps = { [first] = combination }
--                         mathpairs[second] = mps
--                     else
--                         mps[first] = combination
--                     end
--                 end
--          -- elseif vc == "compat" then
--          -- else
--          --     local description = v.description
--          --     if find(description,"LIGATURE") then
--          --         if vs then
--          --             local t = { }
--          --             for i=2,#vs do
--          --                 t[#t+1] = utfchar(vs[i])
--          --             end
--          --             decomposed[utfchar(unicode)] = concat(t)
--          --         else
--          --             local vs = v.shcode
--          --             if vs then
--          --                 local t = { }
--          --                 for i=1,#vs do
--          --                     t[i] = utfchar(vs[i])
--          --                 end
--          --                 decomposed[utfchar(unicode)] = concat(t)
--          --             end
--          --         end
--          --     end
--             end
--         end
--     end
--     initialize = false
--     characters.initialize = function() end -- when used outside tex
-- end

local function initialize() -- maybe in tex mode store in format !
    local data = characters.data
    local function backtrack(v,last,target)
        local vs = v.specials
        if vs and #vs == 3 and vs[1] == "char" then
            local one, two = vs[2], vs[3]
            local first, second = utfchar(one), utfchar(two) .. last
            collapsed[first..second] = target
            backtrack(data[one],second,target)
        end
    end
    local function setpair(one,two,unicode,first,second,combination)
        local mps = mathpairs[one]
        if not mps then
            mps = { [two] = unicode }
            mathpairs[one] = mps
        else
            mps[two] = unicode
        end
        local mps = mathpairs[first]
        if not mps then
            mps = { [second] = combination }
            mathpairs[first] = mps
        else
            mps[second] = combination
        end
    end
    for unicode, v in next, data do
        local vs = v.specials
        if vs and #vs == 3 and vs[1] == "char" then
            --
            local one, two = vs[2], vs[3]
            local first, second, combination = utfchar(one), utfchar(two), utfchar(unicode)
            --
            collapsed[first..second] = combination
            backtrack(data[one],second,combination)
            -- sort of obsolete:
            local cgf = graphemes[first]
            if not cgf then
                cgf = { [second] = combination }
                graphemes[first] = cgf
            else
                cgf[second] = combination
            end
            --
            if v.mathclass or v.mathspec then
                setpair(two,one,unicode,second,first,combination) -- watch order
            end
        end
        local mp = v.mathpair
        if mp then
            local one, two = mp[1], mp[2]
            local first, second, combination = utfchar(one), utfchar(two), utfchar(unicode)
            setpair(one,two,unicode,first,second,combination)
        end
    end
    initialize = false
    characters.initialize = function() end
end

characters.initialize = initialize

--[[ldx--
<p>The next variant has lazy token collecting, on a 140 page mk.tex this saves
about .25 seconds, which is understandable because we have no graphemes and
not collecting tokens is not only faster but also saves garbage collecting.
</p>
--ldx]]--

local skippable  = { }
local filesuffix = file.suffix

function utffilters.setskippable(suffix,value)
    if value == nil then
        value = true
    end
    if type(suffix) == "table" then
        for i=1,#suffix do
            skippable[suffix[i]] = value
        end
    else
        skippable[suffix] = value
    end
end

-- function utffilters.collapse(str,filename)   -- we can make high a seperate pass (never needed with collapse)
--     if skippable[filesuffix(filename)] then
--         return str
--  -- elseif find(filename,"^virtual://") then
--  --     return str
--  -- else
--  --  -- print("\n"..filename)
--     end
--     if str and str ~= "" then
--         local nstr = #str
--         if nstr > 1 then
--             if initialize then -- saves a call
--                 initialize()
--             end
--             local tokens, t, first, done, n = { }, 0, false, false, 0
--             for second in utfcharacters(str) do
--                 if done then
--                     if first then
--                         if second == " " then
--                             t = t + 1
--                             tokens[t] = first
--                             first = second
--                         else
--                          -- local crs = high[second]
--                          -- if crs then
--                          --     t = t + 1
--                          --     tokens[t] = first
--                          --     first = crs
--                          -- else
--                                 local cgf = graphemes[first]
--                                 if cgf and cgf[second] then
--                                     first = cgf[second]
--                                 else
--                                     t = t + 1
--                                     tokens[t] = first
--                                     first = second
--                                 end
--                          -- end
--                         end
--                     elseif second == " " then
--                         first = second
--                     else
--                      -- local crs = high[second]
--                      -- if crs then
--                      --     first = crs
--                      -- else
--                             first = second
--                      -- end
--                     end
--                 elseif second == " " then
--                     first = nil
--                     n = n + 1
--                 else
--                  -- local crs = high[second]
--                  -- if crs then
--                  --     for s in utfcharacters(str) do
--                  --         if n == 1 then
--                  --             break
--                  --         else
--                  --             t = t + 1
--                  --             tokens[t] = s
--                  --             n = n - 1
--                  --         end
--                  --     end
--                  --     if first then
--                  --         t = t + 1
--                  --         tokens[t] = first
--                  --     end
--                  --     first = crs
--                  --     done = true
--                  -- else
--                         local cgf = graphemes[first]
--                         if cgf and cgf[second] then
--                             for s in utfcharacters(str) do
--                                 if n == 1 then
--                                     break
--                                 else
--                                     t = t + 1
--                                     tokens[t] = s
--                                     n = n - 1
--                                 end
--                             end
--                             first = cgf[second]
--                             done = true
--                         else
--                             first = second
--                             n = n + 1
--                         end
--                  -- end
--                 end
--             end
--             if done then
--                 if first then
--                     t = t + 1
--                     tokens[t] = first
--                 end
--                 return concat(tokens) -- seldom called
--             end
--         elseif nstr > 0 then
--             return high[str] or str -- this will go from here
--         end
--     end
--     return str
-- end

-- this is about twice as fast

local p_collapse = nil -- so we can reset if needed

local function prepare()
    if initialize then
        initialize()
    end
    local tree = utfchartabletopattern(collapsed)
    p_collapse = Cs((tree/collapsed + p_utf8character)^0 * P(-1)) -- the P(1) is needed in order to accept non utf
end

function utffilters.collapse(str,filename)
    if not p_collapse then
        prepare()
    end
    if not str or #str == "" or #str == 1 then
        return str
    elseif filename and skippable[filesuffix(filename)] then -- we could hash the collapsables or do a quicker test
        return str
    else
        return lpegmatch(p_collapse,str) or str
    end
end

-- function utffilters.decompose(str)
--     if str and str ~= "" then
--         local nstr = #str
--         if nstr > 1 then
--          -- if initialize then -- saves a call
--          --     initialize()
--          -- end
--             local tokens, t, done, n = { }, 0, false, 0
--             for s in utfcharacters(str) do
--                 local dec = decomposed[s]
--                 if dec then
--                     if not done then
--                         if n > 0 then
--                             for s in utfcharacters(str) do
--                                 if n == 0 then
--                                     break
--                                 else
--                                     t = t + 1
--                                     tokens[t] = s
--                                     n = n - 1
--                                 end
--                             end
--                         end
--                         done = true
--                     end
--                     t = t + 1
--                     tokens[t] = dec
--                 elseif done then
--                     t = t + 1
--                     tokens[t] = s
--                 else
--                     n = n + 1
--                 end
--             end
--             if done then
--                 return concat(tokens) -- seldom called
--             end
--         end
--     end
--     return str
-- end

-- local replacer = nil
-- local finder   = nil
--
-- function utffilters.decompose(str) -- 3 to 4 times faster than the above
--     if not replacer then
--         if initialize then
--             initialize()
--         end
--         local tree = utfchartabletopattern(decomposed)
--         finder   = lpeg.finder(tree,false,true)
--         replacer = lpeg.replacer(tree,decomposed,false,true)
--     end
--     if str and str ~= "" and #str > 1 and lpegmatch(finder,str) then
--         return lpegmatch(replacer,str)
--     end
--     return str
-- end

local p_decompose = nil

local function prepare()
    if initialize then
        initialize()
    end
    local tree = utfchartabletopattern(decomposed)
    p_decompose = Cs((tree/decomposed + p_utf8character)^0 * P(-1))
end

function utffilters.decompose(str,filename) -- 3 to 4 times faster than the above
    if not p_decompose then
        prepare()
    end
    if str and str ~= "" and #str > 1 then
        return lpegmatch(p_decompose,str)
    end
    if not str or #str == "" or #str < 2 then
        return str
    elseif filename and skippable[filesuffix(filename)] then
        return str
    else
        return lpegmatch(p_decompose,str) or str
    end
    return str
end

-- utffilters.addgrapheme(utfchar(318),'l','\string~')
-- utffilters.addgrapheme('c','a','b')

function utffilters.addgrapheme(result,first,second) -- can be U+ 0x string or utf or number
    local result = charfromnumber(result)
    local first  = charfromnumber(first)
    local second = charfromnumber(second)
    if not graphemes[first] then
        graphemes[first] = { [second] = result }
    else
        graphemes[first][second] = result
    end
    local pair = first .. second
    if not composed[pair] then
        composed[pair] = result
        p_composed = nil
    end
end

if interfaces then -- eventually this goes to char-ctx.lua

    interfaces.implement {
        name      = "addgrapheme",
        actions   = utffilters.addgrapheme,
        arguments = { "string", "string", "string" }
    }

end

-- --

local p_reorder = nil

-- local sorter = function(a,b) return b[2] < a[2] end
--
-- local function swapper(s,p,t)
--     local old = { }
--     for i=1,#t do
--         old[i] = t[i][1]
--     end
--     old = concat(old)
--     sort(t,sorter)
--     for i=1,#t do
--         t[i] = t[i][1]
--     end
--     local new = concat(t)
--     if old ~= new then
--         print("reordered",old,"->",new)
--     end
--     return p, new
-- end

-- -- the next one isnto stable for similar weights

local sorter = function(a,b)
    return b[2] < a[2]
end

local function swapper(s,p,t)
    sort(t,sorter)
    for i=1,#t do
        t[i] = t[i][1]
    end
    return p, concat(t)
end

-- -- the next one keeps similar weights in the original order
--
-- local sorter = function(a,b)
--     local b2, a2 = b[2], a[2]
--     if a2 == b2 then
--         return b[3] > a[3]
--     else
--         return b2 < a2
--     end
-- end
--
-- local function swapper(s,p,t)
--     for i=1,#t do
--         t[i][3] = i
--     end
--     sort(t,sorter)
--     for i=1,#t do
--         t[i] = t[i][1]
--     end
--     return p, concat(t)
-- end

-- at some point exceptions will become an option, for now it's an experiment
-- to overcome bugs (that have become features) in unicode .. or we might decide
-- for an extra ordering key in char-def that takes precedence over combining

local exceptions = {
    -- frozen unicode bug
    ["َّ"] = "َّ", -- U+64E .. U+651 => U+651 .. U+64E
}

local function prepare()
    local hash = { }
    for k, v in sortedhash(characters.data) do
        local combining = v.combining -- v.ordering or v.combining
        if combining then
            hash[utfchar(k)] = { utfchar(k), combining, 0 } -- slot 3 can be used in sort
        end
    end
    local e = utfchartabletopattern(exceptions)
    local p = utfchartabletopattern(hash)
    p_reorder = Cs((e/exceptions + Cmt(Ct((p/hash)^2),swapper) + p_utf8character)^0) * P(-1)
end

function utffilters.reorder(str,filename)
    if not p_reorder then
        prepare()
    end
    if not str or #str == "" or #str < 2 then
        return str
    elseif filename and skippable[filesuffix(filename)] then
        return str
    else
        return lpegmatch(p_reorder,str) or str
    end
    return str
end

-- local collapse   = utffilters.collapse
-- local decompose  = utffilters.decompose
-- local preprocess = utffilters.preprocess
--
-- local c1, c2, c3 = "a", "̂", "̃"
-- local r2, r3 = "â", "ẫ"
-- local l1 = "ffl"
--
-- local str  = c1..c2..c3 .. " " .. c1..c2 .. " " .. l1
-- local res  = r3 .. " " .. r2 .. " " .. "ffl"
--
-- local text  = io.loaddata("t:/sources/tufte.tex")
--
-- local function test(n)
--     local data = text .. string.rep(str,100) .. text
--     local okay = text .. string.rep(res,100) .. text
--     local t = os.clock()
--     for i=1,10000 do
--         collapse(data)
--         decompose(data)
--      -- preprocess(data)
--     end
--     print(os.clock()-t,decompose(collapse(data))==okay,decompose(collapse(str)))
-- end
--
-- test(050)
-- test(150)
--
-- local old = "foo" .. string.char(0xE1) .. "bar"
-- local new = collapse(old)
-- print(old,new)

-- local one_old = "فَأَصَّدَّقَ دَّ" local one_new = utffilters.reorder(one_old)
-- local two_old = "فَأَصَّدَّقَ دَّ" local two_new = utffilters.reorder(two_old)
--
-- print(one_old,two_old,one_old==two_old,false)
-- print(one_new,two_new,one_new==two_new,true)
--
-- local test = "foo" .. utf.reverse("ؚ" .. "ً" .. "ٌ" .. "ٍ" .. "َ" .. "ُ" .. "ِ" .. "ّ" .. "ْ" ) .. "bar"
-- local done = utffilters.reorder(test)
--
-- print(test,done,test==done,false)

local f_default     = formatters["[%U] "]
local f_description = formatters["[%s] "]

local function convert(n)
    local d = data[n]
    d = d and d.description
    if d then
        return f_description(d)
    else
        return f_default(n)
    end
end

local pattern = Cs((p_utf8byte / convert)^1)

function utffilters.verbose(data)
    return data and lpegmatch(pattern,data) or ""
end

return characters