2016-01-12 16:26:00

1 files changed, 478 insertions, 0 deletions

diff --git a/tex/context/base/mkiv/lang-ini.lua b/tex/context/base/mkiv/lang-ini.lua
new file mode 100644
index 000000000..46ccdec35
--- /dev/null
+++ b/tex/context/base/mkiv/lang-ini.lua
@@ -0,0 +1,478 @@
+if not modules then modules = { } end modules ['lang-ini'] = {
+    version   = 1.001,
+    comment   = "companion to lang-ini.mkiv",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- needs a cleanup (share locals)
+-- discard language when redefined
+
+-- 002D : hyphen-minus (ascii)
+-- 2010 : hyphen
+-- 2011 : nonbreakable hyphen
+-- 2013 : endash (compound hyphen)
+
+--~ lang:hyphenation(string) string = lang:hyphenation() lang:clear_hyphenation()
+
+-- todo: no foo:bar but foo(bar,...)
+
+local type, tonumber = type, tonumber
+local utfbyte = utf.byte
+local format, gsub = string.format, string.gsub
+local concat, sortedkeys, sortedpairs = table.concat, table.sortedkeys, table.sortedpairs
+local utfbytes = string.utfvalues
+
+local context   = context
+local commands  = commands
+local implement = interfaces.implement
+
+local settings_to_array = utilities.parsers.settings_to_array
+
+local trace_patterns = false  trackers.register("languages.patterns", function(v) trace_patterns = v end)
+
+local report_initialization = logs.reporter("languages","initialization")
+
+local lang             = lang
+
+local prehyphenchar    = lang.prehyphenchar    -- global per language
+local posthyphenchar   = lang.posthyphenchar   -- global per language
+local preexhyphenchar  = lang.preexhyphenchar  -- global per language
+local postexhyphenchar = lang.postexhyphenchar -- global per language
+local lefthyphenmin    = lang.lefthyphenmin
+local righthyphenmin   = lang.righthyphenmin
+local sethjcode        = lang.sethjcode
+
+local uccodes          = characters.uccodes
+
+lang.exceptions        = lang.hyphenation
+local new_langage      = lang.new
+
+languages              = languages or {}
+local languages        = languages
+
+languages.version      = 1.010
+
+languages.registered   = languages.registered or { }
+local registered       = languages.registered
+
+languages.associated   = languages.associated or { }
+local associated       = languages.associated
+
+languages.numbers      = languages.numbers    or { }
+local numbers          = languages.numbers
+
+languages.data         = languages.data       or { }
+local data             = languages.data
+
+storage.register("languages/registered",registered,"languages.registered")
+storage.register("languages/associated",associated,"languages.associated")
+storage.register("languages/numbers",   numbers,   "languages.numbers")
+storage.register("languages/data",      data,      "languages.data")
+
+local nofloaded  = 0
+
+local function resolve(tag)
+    local data, instance = registered[tag], nil
+    if data then
+        instance = data.instance
+        if not instance then
+            instance = new_langage(data.number)
+            data.instance = instance
+        end
+    end
+    return data, instance
+end
+
+local function tolang(what) -- returns lang object
+    if not what then
+        what = tex.language
+    end
+    local tag = numbers[what]
+    local data = tag and registered[tag] or registered[what]
+    if data then
+        local instance = data.lang
+        if not instance then
+            instance = new_langage(data.number)
+            data.instance = instance
+        end
+        return instance
+    end
+end
+
+function languages.getdata(tag) -- or number
+    if tag then
+        return registered[tag] or registered[numbers[tag]]
+    else
+        return registered[numbers[tex.language]]
+    end
+end
+
+-- languages.tolang = tolang
+
+-- patterns=en
+-- patterns=en,de
+
+local function validdata(loaded,what,tag)
+    local dataset = loaded[what]
+    if dataset then
+        local data = dataset.data
+        if not data or data == "" then
+            return nil
+        elseif dataset.compression == "zlib" then
+            data = zlib.decompress(data)
+            if dataset.length and dataset.length ~= #data then
+                report_initialization("compression error in %a for language %a","patterns",what,tag)
+            end
+            return data
+        else
+            return data
+        end
+    end
+end
+
+local function sethjcodes(instance,loaded,what)
+    local l = loaded[what]
+    local c = l and l.characters
+    if c then
+        local h = l.codehash
+        if not h then
+            h = { }
+            l.codehash = h
+        end
+        local s = tex.savinghyphcodes
+        tex.savinghyphcodes = 0
+        for l in utfbytes(c) do
+            local u = uccodes[l]
+            sethjcode(instance,l,l)
+            h[l] = l
+            if type(u) == "number" then
+                -- we don't want ß -> SS
+                sethjcode(instance,u,l)
+                h[u] = l
+            end
+        end
+        tex.savinghyphcodes = s
+    end
+end
+
+local function loaddefinitions(tag,specification)
+    statistics.starttiming(languages)
+    local data, instance = resolve(tag)
+    local definitions = settings_to_array(specification.patterns or "")
+    if #definitions > 0 then
+        if trace_patterns then
+            report_initialization("pattern specification for language %a: %s",tag,specification.patterns)
+        end
+        local dataused  = data.used
+        local ok        = false
+        local resources = data.resources or { }
+        data.resources  = resources
+        for i=1,#definitions do
+            local definition = definitions[i]
+            if definition == "" then
+                -- error
+            elseif definition == "reset" then -- interfaces.variables.reset
+                if trace_patterns then
+                    report_initialization("clearing patterns for language %a",tag)
+                end
+                instance:clear_patterns()
+            elseif not dataused[definition] then
+                dataused[definition] = definition
+                local filename = "lang-" .. definition .. ".lua"
+                local fullname = resolvers.findfile(filename) or ""
+                if fullname == "" then
+                    fullname = resolvers.findfile(filename .. ".gz") or ""
+                end
+                if fullname ~= "" then
+                    if trace_patterns then
+                        report_initialization("loading definition %a for language %a from %a",definition,tag,fullname)
+                    end
+                    local suffix, gzipped = gzip.suffix(fullname)
+                    local loaded = table.load(fullname,gzipped and gzip.load)
+                    if loaded then -- todo: version test
+                        ok, nofloaded = true, nofloaded + 1
+if sethjcodes then -- for now
+                        sethjcodes(instance,loaded,"patterns")
+                        sethjcodes(instance,loaded,"exceptions")
+end
+                        instance:patterns   (validdata(loaded,"patterns",  tag) or "")
+                        instance:hyphenation(validdata(loaded,"exceptions",tag) or "")
+                        resources[#resources+1] = loaded -- so we can use them otherwise
+                    else
+                        report_initialization("invalid definition %a for language %a in %a",definition,tag,filename)
+                    end
+                elseif trace_patterns then
+                    report_initialization("invalid definition %a for language %a in %a",definition,tag,filename)
+                end
+            elseif trace_patterns then
+                report_initialization("definition %a for language %a already loaded",definition,tag)
+            end
+        end
+        return ok
+    elseif trace_patterns then
+        report_initialization("no definitions for language %a",tag)
+    end
+    statistics.stoptiming(languages)
+end
+
+storage.shared.noflanguages = storage.shared.noflanguages or 0
+
+local noflanguages = storage.shared.noflanguages
+
+function languages.define(tag,parent)
+    noflanguages = noflanguages + 1
+    if trace_patterns then
+        report_initialization("assigning number %a to %a",noflanguages,tag)
+    end
+    numbers[noflanguages] = tag
+    registered[tag] = {
+        tag      = tag,
+        parent   = parent or "",
+        patterns = "",
+        loaded   = false,
+        used     = { },
+        dirty    = true,
+        number   = noflanguages,
+        instance = nil, -- luatex data structure
+        synonyms = { },
+    }
+    storage.shared.noflanguages = noflanguages
+end
+
+function languages.setsynonym(synonym,tag) -- convenience function
+    local l = registered[tag]
+    if l then
+        l.synonyms[synonym] = true -- maybe some day more info
+    end
+end
+
+function languages.installed(separator)
+    return concat(sortedkeys(registered),separator or ",")
+end
+
+function languages.current(n)
+    return numbers[n and tonumber(n) or tex.language]
+end
+
+function languages.associate(tag,script,language) -- not yet used
+    associated[tag] = { script, language }
+end
+
+function languages.association(tag) -- not yet used
+    if type(tag) == "number" then
+        tag = numbers[tag]
+    end
+    local lat = tag and associated[tag]
+    if lat then
+        return lat[1], lat[2]
+    end
+end
+
+function languages.loadable(tag,defaultlanguage) -- hack
+    local l = registered[tag] -- no synonyms
+    if l and resolvers.findfile("lang-"..l.patterns..".lua") then
+        return true
+    else
+        return false
+    end
+end
+
+-- a bit messy, we will do all language setting in lua as we can now assign
+-- and 'patterns' will go away here.
+
+function languages.unload(tag)
+    local l = registered[tag]
+    if l then
+        l.dirty = true
+    end
+end
+
+if environment.initex then
+
+    function languages.getnumber()
+        return 0
+    end
+
+else
+
+    function languages.getnumber(tag,default,patterns)
+        local l = registered[tag]
+        if l then
+            if l.dirty then
+                if trace_patterns then
+                    report_initialization("checking patterns for %a with default %a",tag,default)
+                end
+                -- patterns is already resolved to parent patterns if applicable
+                if patterns and patterns ~= "" then
+                    if l.patterns ~= patterns then
+                        l.patterns = patterns
+                        if trace_patterns then
+                            report_initialization("loading patterns for %a using specification %a",tag,patterns)
+                        end
+                        loaddefinitions(tag,l)
+                    else
+                        -- unchanged
+                    end
+                elseif l.patterns == "" then
+                    l.patterns = tag
+                    if trace_patterns then
+                        report_initialization("loading patterns for %a using tag",tag)
+                    end
+                    local ok = loaddefinitions(tag,l)
+                    if not ok and tag ~= default then
+                        l.patterns = default
+                        if trace_patterns then
+                            report_initialization("loading patterns for %a using default",tag)
+                        end
+                        loaddefinitions(tag,l)
+                    end
+                end
+                l.loaded = true
+                l.dirty = false
+            end
+            return l.number
+        else
+            return 0
+        end
+    end
+
+end
+
+-- not that usefull, global values
+
+function languages.prehyphenchar   (what) return prehyphenchar   (tolang(what)) end
+function languages.posthyphenchar  (what) return posthyphenchar  (tolang(what)) end
+function languages.preexhyphenchar (what) return preexhyphenchar (tolang(what)) end
+function languages.postexhyphenchar(what) return postexhyphenchar(tolang(what)) end
+function languages.lefthyphenmin   (what) return lefthyphenmin   (tolang(what)) end
+function languages.righthyphenmin  (what) return righthyphenmin  (tolang(what)) end
+
+-- e['implementer']= 'imple{m}{-}{-}menter'
+-- e['manual'] = 'man{}{}{}'
+-- e['as'] = 'a-s'
+-- e['user-friendly'] = 'user=friend-ly'
+-- e['exceptionally-friendly'] = 'excep-tionally=friend-ly'
+
+function languages.loadwords(tag,filename)
+    local data, instance = resolve(tag)
+    if data then
+        statistics.starttiming(languages)
+        instance:hyphenation(io.loaddata(filename) or "")
+        statistics.stoptiming(languages)
+    end
+end
+
+function languages.setexceptions(tag,str)
+    local data, instance = resolve(tag)
+    if data then
+        instance:hyphenation(string.strip(str)) -- we need to strip leading spaces
+    end
+end
+
+function languages.hyphenate(tag,str)
+    -- todo: does this still work?
+    local data, instance = resolve(tag)
+    if data then
+        return instance:hyphenate(str)
+    else
+        return str
+    end
+end
+
+-- hyphenation.define        ("zerolanguage")
+-- hyphenation.loadpatterns  ("zerolanguage") -- else bug
+-- hyphenation.loadexceptions("zerolanguage") -- else bug
+
+languages.logger = languages.logger or { }
+
+function languages.logger.report()
+    local result, r = { }, 0
+    for tag, l in sortedpairs(registered) do
+        if l.loaded then
+            r = r + 1
+            result[r] = format("%s:%s:%s",tag,l.parent,l.number)
+        end
+    end
+    return r > 0 and concat(result," ") or "none"
+end
+
+-- must happen at the tex end .. will use lang-def.lua
+
+languages.associate('en','latn','eng')
+languages.associate('uk','latn','eng')
+languages.associate('nl','latn','nld')
+languages.associate('de','latn','deu')
+languages.associate('fr','latn','fra')
+
+statistics.register("loaded patterns", function()
+    local result = languages.logger.report()
+    if result ~= "none" then
+     -- return result
+        return format("%s, load time: %s",result,statistics.elapsedtime(languages))
+    end
+end)
+
+-- statistics.register("language load time", function()
+--     -- often zero so we can merge that in the above
+--     return statistics.elapsedseconds(languages, format(", nofpatterns: %s",nofloaded))
+-- end)
+
+-- interface
+
+implement {
+    name      = "languagenumber",
+    actions   = { languages.getnumber, context },
+    arguments = { "string", "string", "string" }
+}
+
+implement {
+    name      = "installedlanguages",
+    actions   = { languages.installed, context },
+}
+
+implement {
+    name      = "definelanguage",
+    actions   = languages.define,
+    arguments = { "string", "string" }
+}
+
+implement {
+    name      = "setlanguagesynonym",
+    actions   = languages.setsynonym,
+    arguments = { "string", "string" }
+}
+
+implement {
+    name      = "unloadlanguage",
+    actions   = languages.unload,
+    arguments = { "string" }
+}
+
+implement {
+    name      = "setlanguageexceptions",
+    actions   = languages.setexceptions,
+    arguments = { "string", "string" }
+}
+
+
+implement {
+    name      = "currentprehyphenchar",
+    actions   = function()
+        local c = prehyphenchar(tolang())
+        if c and c > 0 then
+            context.char(c)
+        end
+    end
+}
+
+implement {
+    name      = "currentposthyphenchar",
+    actions   = function()
+        local c = posthyphenchar(tolang())
+        if c and c > 0 then
+            context.char(c)
+        end
+    end
+}