Merge pull request #260 from phi-gamma/master

add import helper, new directory layout, revised letterspacing, fontloader gone nuts

1 files changed, 452 insertions, 0 deletions

diff --git a/src/fontloader/misc/fontloader-font-def.lua b/src/fontloader/misc/fontloader-font-def.lua
new file mode 100644
index 0000000..fdded3c
--- /dev/null
+++ b/src/fontloader/misc/fontloader-font-def.lua
@@ -0,0 +1,452 @@
+if not modules then modules = { } end modules ['font-def'] = {
+    version   = 1.001,
+    comment   = "companion to font-ini.mkiv",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- We can overload some of the definers.functions so we don't local them.
+
+local format, gmatch, match, find, lower, gsub = string.format, string.gmatch, string.match, string.find, string.lower, string.gsub
+local tostring, next = tostring, next
+local lpegmatch = lpeg.match
+local suffixonly, removesuffix = file.suffix, file.removesuffix
+
+local allocate = utilities.storage.allocate
+
+local trace_defining     = false  trackers  .register("fonts.defining", function(v) trace_defining     = v end)
+local directive_embedall = false  directives.register("fonts.embedall", function(v) directive_embedall = v end)
+
+trackers.register("fonts.loading", "fonts.defining", "otf.loading", "afm.loading", "tfm.loading")
+trackers.register("fonts.all", "fonts.*", "otf.*", "afm.*", "tfm.*")
+
+local report_defining = logs.reporter("fonts","defining")
+
+--[[ldx--
+<p>Here we deal with defining fonts. We do so by intercepting the
+default loader that only handles <l n='tfm'/>.</p>
+--ldx]]--
+
+local fonts         = fonts
+local fontdata      = fonts.hashes.identifiers
+local readers       = fonts.readers
+local definers      = fonts.definers
+local specifiers    = fonts.specifiers
+local constructors  = fonts.constructors
+local fontgoodies   = fonts.goodies
+
+readers.sequence    = allocate { 'otf', 'ttf', 'afm', 'tfm', 'lua' } -- dfont ttc
+
+local variants      = allocate()
+specifiers.variants = variants
+
+definers.methods    = definers.methods or { }
+
+local internalized  = allocate() -- internal tex numbers (private)
+local lastdefined   = nil -- we don't want this one to end up in s-tra-02
+
+local loadedfonts   = constructors.loadedfonts
+local designsizes   = constructors.designsizes
+
+-- not in generic (some day I'll make two defs, one for context, one for generic)
+
+local resolvefile   = fontgoodies and fontgoodies.filenames and fontgoodies.filenames.resolve or function(s) return s end
+
+--[[ldx--
+<p>We hardly gain anything when we cache the final (pre scaled)
+<l n='tfm'/> table. But it can be handy for debugging, so we no
+longer carry this code along. Also, we now have quite some reference
+to other tables so we would end up with lots of catches.</p>
+--ldx]]--
+
+--[[ldx--
+<p>We can prefix a font specification by <type>name:</type> or
+<type>file:</type>. The first case will result in a lookup in the
+synonym table.</p>
+
+<typing>
+[ name: | file: ] identifier [ separator [ specification ] ]
+</typing>
+
+<p>The following function split the font specification into components
+and prepares a table that will move along as we proceed.</p>
+--ldx]]--
+
+-- beware, we discard additional specs
+--
+-- method:name method:name(sub) method:name(sub)*spec method:name*spec
+-- name name(sub) name(sub)*spec name*spec
+-- name@spec*oeps
+
+local splitter, splitspecifiers = nil, "" -- not so nice
+
+local P, C, S, Cc = lpeg.P, lpeg.C, lpeg.S, lpeg.Cc
+
+local left  = P("(")
+local right = P(")")
+local colon = P(":")
+local space = P(" ")
+
+definers.defaultlookup = "file"
+
+local prefixpattern = P(false)
+
+local function addspecifier(symbol)
+    splitspecifiers     = splitspecifiers .. symbol
+    local method        = S(splitspecifiers)
+    local lookup        = C(prefixpattern) * colon
+    local sub           = left * C(P(1-left-right-method)^1) * right
+    local specification = C(method) * C(P(1)^1)
+    local name          = C((1-sub-specification)^1)
+    splitter = P((lookup + Cc("")) * name * (sub + Cc("")) * (specification + Cc("")))
+end
+
+local function addlookup(str,default)
+    prefixpattern = prefixpattern + P(str)
+end
+
+definers.addlookup = addlookup
+
+addlookup("file")
+addlookup("name")
+addlookup("spec")
+
+local function getspecification(str)
+    return lpegmatch(splitter,str or "") -- weird catch
+end
+
+definers.getspecification = getspecification
+
+function definers.registersplit(symbol,action,verbosename)
+    addspecifier(symbol)
+    variants[symbol] = action
+    if verbosename then
+        variants[verbosename] = action
+    end
+end
+
+local function makespecification(specification,lookup,name,sub,method,detail,size)
+    size = size or 655360
+    if not lookup or lookup == "" then
+        lookup = definers.defaultlookup
+    end
+    if trace_defining then
+        report_defining("specification %a, lookup %a, name %a, sub %a, method %a, detail %a",
+            specification, lookup, name, sub, method, detail)
+    end
+    local t = {
+        lookup        = lookup,        -- forced type
+        specification = specification, -- full specification
+        size          = size,          -- size in scaled points or -1000*n
+        name          = name,          -- font or filename
+        sub           = sub,           -- subfont (eg in ttc)
+        method        = method,        -- specification method
+        detail        = detail,        -- specification
+        resolved      = "",            -- resolved font name
+        forced        = "",            -- forced loader
+        features      = { },           -- preprocessed features
+    }
+    return t
+end
+
+
+definers.makespecification = makespecification
+
+function definers.analyze(specification, size)
+    -- can be optimized with locals
+    local lookup, name, sub, method, detail = getspecification(specification or "")
+    return makespecification(specification, lookup, name, sub, method, detail, size)
+end
+
+--[[ldx--
+<p>We can resolve the filename using the next function:</p>
+--ldx]]--
+
+definers.resolvers = definers.resolvers or { }
+local resolvers    = definers.resolvers
+
+-- todo: reporter
+
+function resolvers.file(specification)
+    local name = resolvefile(specification.name) -- catch for renames
+    local suffix = lower(suffixonly(name))
+    if fonts.formats[suffix] then
+        specification.forced     = suffix
+        specification.forcedname = name
+        specification.name       = removesuffix(name)
+    else
+        specification.name       = name -- can be resolved
+    end
+end
+
+function resolvers.name(specification)
+    local resolve = fonts.names.resolve
+    if resolve then
+        local resolved, sub = resolve(specification.name,specification.sub,specification) -- we pass specification for overloaded versions
+        if resolved then
+            specification.resolved = resolved
+            specification.sub      = sub
+            local suffix = lower(suffixonly(resolved))
+            if fonts.formats[suffix] then
+                specification.forced     = suffix
+                specification.forcedname = resolved
+                specification.name       = removesuffix(resolved)
+            else
+                specification.name       = resolved
+            end
+        end
+    else
+        resolvers.file(specification)
+    end
+end
+
+function resolvers.spec(specification)
+    local resolvespec = fonts.names.resolvespec
+    if resolvespec then
+        local resolved, sub = resolvespec(specification.name,specification.sub,specification) -- we pass specification for overloaded versions
+        if resolved then
+            specification.resolved   = resolved
+            specification.sub        = sub
+            specification.forced     = lower(suffixonly(resolved))
+            specification.forcedname = resolved
+            specification.name       = removesuffix(resolved)
+        end
+    else
+        resolvers.name(specification)
+    end
+end
+
+function definers.resolve(specification)
+    if not specification.resolved or specification.resolved == "" then -- resolved itself not per se in mapping hash
+        local r = resolvers[specification.lookup]
+        if r then
+            r(specification)
+        end
+    end
+    if specification.forced == "" then
+        specification.forced     = nil
+        specification.forcedname = nil
+    end
+    specification.hash = lower(specification.name .. ' @ ' .. constructors.hashfeatures(specification))
+    if specification.sub and specification.sub ~= "" then
+        specification.hash = specification.sub .. ' @ ' .. specification.hash
+    end
+    return specification
+end
+
+--[[ldx--
+<p>The main read function either uses a forced reader (as determined by
+a lookup) or tries to resolve the name using the list of readers.</p>
+
+<p>We need to cache when possible. We do cache raw tfm data (from <l
+n='tfm'/>, <l n='afm'/> or <l n='otf'/>). After that we can cache based
+on specificstion (name) and size, that is, <l n='tex'/> only needs a number
+for an already loaded fonts. However, it may make sense to cache fonts
+before they're scaled as well (store <l n='tfm'/>'s with applied methods
+and features). However, there may be a relation between the size and
+features (esp in virtual fonts) so let's not do that now.</p>
+
+<p>Watch out, here we do load a font, but we don't prepare the
+specification yet.</p>
+--ldx]]--
+
+-- very experimental:
+
+function definers.applypostprocessors(tfmdata)
+    local postprocessors = tfmdata.postprocessors
+    if postprocessors then
+        local properties = tfmdata.properties
+        for i=1,#postprocessors do
+            local extrahash = postprocessors[i](tfmdata) -- after scaling etc
+            if type(extrahash) == "string" and extrahash ~= "" then
+                -- e.g. a reencoding needs this
+                extrahash = gsub(lower(extrahash),"[^a-z]","-")
+                properties.fullname = format("%s-%s",properties.fullname,extrahash)
+            end
+        end
+    end
+    return tfmdata
+end
+
+-- function definers.applypostprocessors(tfmdata)
+--     return tfmdata
+-- end
+
+local function checkembedding(tfmdata)
+    local properties = tfmdata.properties
+    local embedding
+    if directive_embedall then
+        embedding = "full"
+    elseif properties and properties.filename and constructors.dontembed[properties.filename] then
+        embedding = "no"
+    else
+        embedding = "subset"
+    end
+    if properties then
+        properties.embedding = embedding
+    else
+        tfmdata.properties = { embedding = embedding }
+    end
+    tfmdata.embedding = embedding
+end
+
+function definers.loadfont(specification)
+    local hash = constructors.hashinstance(specification)
+    local tfmdata = loadedfonts[hash] -- hashes by size !
+    if not tfmdata then
+        local forced = specification.forced or ""
+        if forced ~= "" then
+            local reader = readers[lower(forced)] -- normally forced is already lowered
+            tfmdata = reader and reader(specification)
+            if not tfmdata then
+                report_defining("forced type %a of %a not found",forced,specification.name)
+            end
+        else
+            local sequence = readers.sequence -- can be overloaded so only a shortcut here
+            for s=1,#sequence do
+                local reader = sequence[s]
+                if readers[reader] then -- we skip not loaded readers
+                    if trace_defining then
+                        report_defining("trying (reader sequence driven) type %a for %a with file %a",reader,specification.name,specification.filename)
+                    end
+                    tfmdata = readers[reader](specification)
+                    if tfmdata then
+                        break
+                    else
+                        specification.filename = nil
+                    end
+                end
+            end
+        end
+        if tfmdata then
+            tfmdata = definers.applypostprocessors(tfmdata)
+            checkembedding(tfmdata) -- todo: general postprocessor
+            loadedfonts[hash] = tfmdata
+            designsizes[specification.hash] = tfmdata.parameters.designsize
+        end
+    end
+    if not tfmdata then
+        report_defining("font with asked name %a is not found using lookup %a",specification.name,specification.lookup)
+    end
+    return tfmdata
+end
+
+function constructors.checkvirtualids()
+    -- dummy in plain version
+end
+
+function constructors.readanddefine(name,size) -- no id -- maybe a dummy first
+    local specification = definers.analyze(name,size)
+    local method = specification.method
+    if method and variants[method] then
+        specification = variants[method](specification)
+    end
+    specification = definers.resolve(specification)
+    local hash = constructors.hashinstance(specification)
+    local id = definers.registered(hash)
+    if not id then
+        local tfmdata = definers.loadfont(specification)
+        if tfmdata then
+            tfmdata.properties.hash = hash
+            constructors.checkvirtualids(tfmdata) -- experiment, will become obsolete when slots can selfreference
+            id = font.define(tfmdata)
+            definers.register(tfmdata,id)
+        else
+            id = 0  -- signal
+        end
+    end
+    return fontdata[id], id
+end
+
+--[[ldx--
+<p>So far the specifiers. Now comes the real definer. Here we cache
+based on id's. Here we also intercept the virtual font handler. Since
+it evolved stepwise I may rewrite this bit (combine code).</p>
+
+In the previously defined reader (the one resulting in a <l n='tfm'/>
+table) we cached the (scaled) instances. Here we cache them again, but
+this time based on id. We could combine this in one cache but this does
+not gain much. By the way, passing id's back to in the callback was
+introduced later in the development.</p>
+--ldx]]--
+
+function definers.current() -- or maybe current
+    return lastdefined
+end
+
+function definers.registered(hash)
+    local id = internalized[hash]
+    return id, id and fontdata[id]
+end
+
+function definers.register(tfmdata,id)
+    if tfmdata and id then
+        local hash = tfmdata.properties.hash
+        if not hash then
+            report_defining("registering font, id %a, name %a, invalid hash",id,tfmdata.properties.filename or "?")
+        elseif not internalized[hash] then
+            internalized[hash] = id
+            if trace_defining then
+                report_defining("registering font, id %s, hash %a",id,hash)
+            end
+            fontdata[id] = tfmdata
+        end
+    end
+end
+
+function definers.read(specification,size,id) -- id can be optional, name can already be table
+    statistics.starttiming(fonts)
+    if type(specification) == "string" then
+        specification = definers.analyze(specification,size)
+    end
+    local method = specification.method
+    if method and variants[method] then
+        specification = variants[method](specification)
+    end
+    specification = definers.resolve(specification)
+    local hash = constructors.hashinstance(specification)
+    local tfmdata = definers.registered(hash) -- id
+    if tfmdata then
+        if trace_defining then
+            report_defining("already hashed: %s",hash)
+        end
+    else
+        tfmdata = definers.loadfont(specification) -- can be overloaded
+        if tfmdata then
+            if trace_defining then
+                report_defining("loaded and hashed: %s",hash)
+            end
+            tfmdata.properties.hash = hash
+            if id then
+                definers.register(tfmdata,id)
+            end
+        else
+            if trace_defining then
+                report_defining("not loaded and hashed: %s",hash)
+            end
+        end
+    end
+    lastdefined = tfmdata or id -- todo ! ! ! ! !
+    if not tfmdata then -- or id?
+        report_defining( "unknown font %a, loading aborted",specification.name)
+    elseif trace_defining and type(tfmdata) == "table" then
+        local properties = tfmdata.properties or { }
+        local parameters = tfmdata.parameters or { }
+        report_defining("using %a font with id %a, name %a, size %a, bytes %a, encoding %a, fullname %a, filename %a",
+            properties.format or "unknown", id, properties.name, parameters.size, properties.encodingbytes,
+            properties.encodingname, properties.fullname, file.basename(properties.filename))
+    end
+    statistics.stoptiming(fonts)
+    return tfmdata
+end
+
+function font.getfont(id)
+    return fontdata[id] -- otherwise issues
+end
+
+--[[ldx--
+<p>We overload the <l n='tfm'/> reader.</p>
+--ldx]]--
+
+callbacks.register('define_font', definers.read, "definition of fonts (tfmdata preparation)")