From 2b89b4ede02fd02fe66597e9c314dbfcfa4c0ecb Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Sat, 27 Sep 2014 14:46:00 +0200 Subject: beta 2014.09.27 14:46 --- tex/context/base/cont-new.mkiv | 2 +- tex/context/base/context-version.pdf | Bin 4386 -> 4391 bytes tex/context/base/context.mkiv | 2 +- tex/context/base/font-agl.lua | 10 +- tex/context/base/font-cid.lua | 25 +++- tex/context/base/font-map.lua | 135 ++++++++++++++++++ tex/context/base/font-mis.lua | 2 +- tex/context/base/font-otf.lua | 17 +-- tex/context/base/lang-hyp.lua | 38 +++--- tex/context/base/s-fnt-21.mkiv | 2 +- tex/context/base/status-files.pdf | Bin 24748 -> 24767 bytes tex/context/base/status-lua.pdf | Bin 325287 -> 325265 bytes tex/generic/context/luatex/luatex-fonts-merged.lua | 152 +++++++++++++++++++-- 13 files changed, 335 insertions(+), 50 deletions(-) diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv index 58f53214e..0083e7aee 100644 --- a/tex/context/base/cont-new.mkiv +++ b/tex/context/base/cont-new.mkiv @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2014.09.26 11:42} +\newcontextversion{2014.09.27 14:46} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf index 71b88d27b..ae0441b81 100644 Binary files a/tex/context/base/context-version.pdf and b/tex/context/base/context-version.pdf differ diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv index 509472134..6e183c88b 100644 --- a/tex/context/base/context.mkiv +++ b/tex/context/base/context.mkiv @@ -28,7 +28,7 @@ %D up and the dependencies are more consistent. \edef\contextformat {\jobname} -\edef\contextversion{2014.09.26 11:42} +\edef\contextversion{2014.09.27 14:46} \edef\contextkind {beta} %D For those who want to use this: diff --git a/tex/context/base/font-agl.lua b/tex/context/base/font-agl.lua index 5ee34b028..42a41a15d 100644 --- a/tex/context/base/font-agl.lua +++ b/tex/context/base/font-agl.lua @@ -657,11 +657,15 @@ end -- We load this table only when needed. We could use a loading mechanism -- return the table but there are no more vectors like this so why bother. -fonts.encodings = fonts.encodings or { } - -fonts.encodings.agl = { +local agl = { names = names, -- unicode -> name unicodes = unicodes, -- name -> unicode synonyms = synonyms, -- merged into the other two extras = extras, -- merged into the other two } + +fonts = fonts or { } +fonts.encodings = fonts.encodings or { } +fonts.encodings.agl = agl + +return agl diff --git a/tex/context/base/font-cid.lua b/tex/context/base/font-cid.lua index e4b565313..0eaacdfbd 100644 --- a/tex/context/base/font-cid.lua +++ b/tex/context/base/font-cid.lua @@ -74,7 +74,7 @@ local function loadcidfile(filename) ordering = ordering, filename = filename, unicodes = unicodes, - names = names + names = names, } end end @@ -112,15 +112,28 @@ function cid.getmap(specification) report_otf("invalid cidinfo specification, table expected") return end - local registry = specification.registry - local ordering = specification.ordering + local registry = specification.registry + local ordering = specification.ordering local supplement = specification.supplement - -- check for already loaded file - local filename = format(registry,ordering,supplement) - local found = cidmap[lower(filename)] + local filename = format(registry,ordering,supplement) + local lowername = lower(filename) + local found = cidmap[lowername] if found then return found end + if ordering == "Identity" then + local found = { + supplement = supplement, + registry = registry, + ordering = ordering, + filename = filename, + unicodes = { }, + names = { }, + } + cidmap[lowername] = found + return found + end + -- check for already loaded file if trace_loading then report_otf("cidmap needed, registry %a, ordering %a, supplement %a",registry,ordering,supplement) end diff --git a/tex/context/base/font-map.lua b/tex/context/base/font-map.lua index e5f587105..429c73597 100644 --- a/tex/context/base/font-map.lua +++ b/tex/context/base/font-map.lua @@ -163,6 +163,7 @@ function mappings.addtounicode(data,filename) local properties = data.properties local descriptions = data.descriptions local unicodes = resources.unicodes + local lookuptypes = resources.lookuptypes if not unicodes then return end @@ -175,8 +176,10 @@ function mappings.addtounicode(data,filename) local private = fonts.constructors.privateoffset local unknown = format("%04X",utfbyte("?")) local unicodevector = fonts.encodings.agl.unicodes -- loaded runtime in context + ----- namevector = fonts.encodings.agl.names -- loaded runtime in context local tounicode = { } local originals = { } + local missing = { } resources.tounicode = tounicode resources.originals = originals local lumunic, uparser, oparser @@ -311,12 +314,144 @@ function mappings.addtounicode(data,filename) end end end + -- check using substitutes and alternates + -- + if not unicode then + missing[name] = true + end -- if not unicode then -- originals[index] = 0xFFFD -- tounicode[index] = "FFFD" -- end end end + if next(missing) then +-- inspect(missing) + local guess = { } + -- helper + local function check(gname,code,unicode) + local description = descriptions[code] + -- no need to add a self reference + local variant = description.name + if variant == gname then + return + end + -- the variant already has a unicode (normally that resultrs in a default tounicode to self) + local unic = unicodes[variant] + if unic == -1 or unic >= private or (unic >= 0xE000 and unic <= 0xF8FF) or unic == 0xFFFE or unic == 0xFFFF then + -- no default mapping and therefore maybe no tounicode yet + else + return + end + -- the variant already has a tounicode + local index = descriptions[code].index + if tounicode[index] then + return + end + -- add to the list + local g = guess[variant] + if g then + g[gname] = unicode + else + guess[variant] = { [gname] = unicode } + end + end + -- + for unicode, description in next, descriptions do + local slookups = description.slookups + if slookups then + local gname = description.name + for tag, data in next, slookups do + local lookuptype = lookuptypes[tag] + if lookuptype == "alternate" then + for i=1,#data do + check(gname,data[i],unicode) + end + elseif lookuptype == "substitution" then + check(gname,data,unicode) + end + end + end + local mlookups = description.mlookups + if mlookups then + local gname = description.name + for tag, list in next, mlookups do + local lookuptype = lookuptypes[tag] + if lookuptype == "alternate" then + for i=1,#list do + local data = list[i] + for i=1,#data do + check(gname,data[i],unicode) + end + end + elseif lookuptype == "substitution" then + for i=1,#list do + check(gname,list[i],unicode) + end + end + end + end + end + -- resolve references + local done = true + while done do + done = false + for k, v in next, guess do + if type(v) ~= "number" then + for kk, vv in next, v do + if vv == -1 or vv >= private or (vv >= 0xE000 and vv <= 0xF8FF) or vv == 0xFFFE or vv == 0xFFFF then + local uu = guess[kk] + if type(uu) == "number" then + guess[k] = uu + done = true + end + else + guess[k] = vv + done = true + end + end + end + end + end + -- generate tounicodes + for k, v in next, guess do + if type(v) == "number" then + guess[k] = tounicode16(v) + else + local t = nil + local l = lower(k) + local u = unicodes[l] + if not u then + -- forget about it + elseif u == -1 or u >= private or (u >= 0xE000 and u <= 0xF8FF) or u == 0xFFFE or u == 0xFFFF then + t = tounicode[descriptions[u].index] + else + -- t = u + end + if t then + guess[k] = t + else + guess[k] = "FFFD" + end + end + end + local orphans = 0 + local guessed = 0 + for k, v in next, guess do + tounicode[descriptions[unicodes[k]].index] = v + if v == "FFFD" then + orphans = orphans + 1 + guess[k] = false + else + guessed = guessed + 1 + guess[k] = true + end + end + -- resources.nounicode = guess -- only when we test things + if trace_loading and orphans > 0 or guessed > 0 then + report_fonts("%s glyphs with no related unicode, %s guessed, %s orphans",guessed+orphans,guessed,orphans) + end + end if trace_mapping then for unic, glyph in table.sortedhash(descriptions) do local name = glyph.name diff --git a/tex/context/base/font-mis.lua b/tex/context/base/font-mis.lua index 1b50977ea..b934837f4 100644 --- a/tex/context/base/font-mis.lua +++ b/tex/context/base/font-mis.lua @@ -22,7 +22,7 @@ local handlers = fonts.handlers handlers.otf = handlers.otf or { } local otf = handlers.otf -otf.version = otf.version or 2.760 +otf.version = otf.version or 2.762 otf.cache = otf.cache or containers.define("fonts", "otf", otf.version, true) function otf.loadcached(filename,format,sub) diff --git a/tex/context/base/font-otf.lua b/tex/context/base/font-otf.lua index c1b23983d..c1bb4419c 100644 --- a/tex/context/base/font-otf.lua +++ b/tex/context/base/font-otf.lua @@ -48,7 +48,7 @@ local otf = fonts.handlers.otf otf.glists = { "gsub", "gpos" } -otf.version = 2.760 -- beware: also sync font-mis.lua +otf.version = 2.762 -- beware: also sync font-mis.lua otf.cache = containers.define("fonts", "otf", otf.version, true) local fontdata = fonts.hashes.identifiers @@ -255,7 +255,7 @@ local ordered_enhancers = { "analyze glyphs", "analyze math", - "prepare tounicode", -- maybe merge with prepare + -- "prepare tounicode", "reorganize lookups", "reorganize mark classes", @@ -274,6 +274,8 @@ local ordered_enhancers = { "check metadata", "check extra features", -- after metadata + "prepare tounicode", + "check encoding", -- moved "add duplicates", @@ -684,17 +686,18 @@ actions["prepare glyphs"] = function(data,filename,raw) if includesubfonts then metadata.subfonts[cidindex] = somecopy(subfont) end + -- we have delayed loading so we cannot use next for index=0,subfont.glyphcnt-1 do -- we could take the previous glyphcnt instead of 0 local glyph = cidglyphs[index] if glyph then local unicode = glyph.unicode -if unicode >= 0x00E000 and unicode <= 0x00F8FF then + if unicode >= 0x00E000 and unicode <= 0x00F8FF then unicode = -1 -elseif unicode >= 0x0F0000 and unicode <= 0x0FFFFD then + elseif unicode >= 0x0F0000 and unicode <= 0x0FFFFD then unicode = -1 -elseif unicode >= 0x100000 and unicode <= 0x10FFFD then + elseif unicode >= 0x100000 and unicode <= 0x10FFFD then unicode = -1 -end + end local name = glyph.name or cidnames[index] if not unicode or unicode == -1 then -- or unicode >= criterium then unicode = cidunicodes[index] @@ -736,7 +739,6 @@ end nofunicodes = nofunicodes + 1 end indices[index] = unicode -- each index is unique (at least now) - local description = { -- width = glyph.width, boundingbox = glyph.boundingbox, @@ -745,7 +747,6 @@ end index = index, glyph = glyph, } - descriptions[unicode] = description else -- report_otf("potential problem: glyph %U is used but empty",index) diff --git a/tex/context/base/lang-hyp.lua b/tex/context/base/lang-hyp.lua index 3b5eac9ba..87216cc8b 100644 --- a/tex/context/base/lang-hyp.lua +++ b/tex/context/base/lang-hyp.lua @@ -639,25 +639,25 @@ else -- traditional.loadpatterns("nl","lang-nl") -- traditional.loadpatterns("de","lang-de") - traditional.registerpattern("nl","e1ë", { start = 1, length = 2, before = "e", after = "e" } ) - traditional.registerpattern("nl","oo1ë", { start = 2, length = 3, before = "o", after = "e" } ) - traditional.registerpattern("de","qqxc9xkqq",{ start = 3, length = 4, before = "ab", after = "cd" } ) - - local specification = { - lefthyphenmin = 2, - righthyphenmin = 2, - lefthyphenchar = "<", - righthyphenchar = ">", - } - - print("reëel", traditional.injecthyphens(dictionaries.nl,"reëel", specification),"r{e>}{}{}{}{}{}{=private or (unic>=0xE000 and unic<=0xF8FF) or unic==0xFFFE or unic==0xFFFF then + else + return + end + local index=descriptions[code].index + if tounicode[index] then + return + end + local g=guess[variant] + if g then + g[gname]=unicode + else + guess[variant]={ [gname]=unicode } + end + end + for unicode,description in next,descriptions do + local slookups=description.slookups + if slookups then + local gname=description.name + for tag,data in next,slookups do + local lookuptype=lookuptypes[tag] + if lookuptype=="alternate" then + for i=1,#data do + check(gname,data[i],unicode) + end + elseif lookuptype=="substitution" then + check(gname,data,unicode) + end + end + end + local mlookups=description.mlookups + if mlookups then + local gname=description.name + for tag,list in next,mlookups do + local lookuptype=lookuptypes[tag] + if lookuptype=="alternate" then + for i=1,#list do + local data=list[i] + for i=1,#data do + check(gname,data[i],unicode) + end + end + elseif lookuptype=="substitution" then + for i=1,#list do + check(gname,list[i],unicode) + end + end + end + end + end + local done=true + while done do + done=false + for k,v in next,guess do + if type(v)~="number" then + for kk,vv in next,v do + if vv==-1 or vv>=private or (vv>=0xE000 and vv<=0xF8FF) or vv==0xFFFE or vv==0xFFFF then + local uu=guess[kk] + if type(uu)=="number" then + guess[k]=uu + done=true + end + else + guess[k]=vv + done=true + end + end + end + end + end + for k,v in next,guess do + if type(v)=="number" then + guess[k]=tounicode16(v) + else + local t=nil + local l=lower(k) + local u=unicodes[l] + if not u then + elseif u==-1 or u>=private or (u>=0xE000 and u<=0xF8FF) or u==0xFFFE or u==0xFFFF then + t=tounicode[descriptions[u].index] + else + end + if t then + guess[k]=t + else + guess[k]="FFFD" + end + end + end + local orphans=0 + local guessed=0 + for k,v in next,guess do + tounicode[descriptions[unicodes[k]].index]=v + if v=="FFFD" then + orphans=orphans+1 + guess[k]=false + else + guessed=guessed+1 + guess[k]=true + end + end + if trace_loading and orphans>0 or guessed>0 then + report_fonts("%s glyphs with no related unicode, %s guessed, %s orphans",guessed+orphans,guessed,orphans) end end if trace_mapping then @@ -6706,7 +6838,7 @@ local report_otf=logs.reporter("fonts","otf loading") local fonts=fonts local otf=fonts.handlers.otf otf.glists={ "gsub","gpos" } -otf.version=2.760 +otf.version=2.762 otf.cache=containers.define("fonts","otf",otf.version,true) local fontdata=fonts.hashes.identifiers local chardata=characters and characters.data @@ -6869,7 +7001,6 @@ local ordered_enhancers={ "prepare lookups", "analyze glyphs", "analyze math", - "prepare tounicode", "reorganize lookups", "reorganize mark classes", "reorganize anchor classes", @@ -6882,6 +7013,7 @@ local ordered_enhancers={ "check glyphs", "check metadata", "check extra features", + "prepare tounicode", "check encoding", "add duplicates", "cleanup tables", @@ -7249,13 +7381,13 @@ actions["prepare glyphs"]=function(data,filename,raw) local glyph=cidglyphs[index] if glyph then local unicode=glyph.unicode -if unicode>=0x00E000 and unicode<=0x00F8FF then + if unicode>=0x00E000 and unicode<=0x00F8FF then unicode=-1 -elseif unicode>=0x0F0000 and unicode<=0x0FFFFD then + elseif unicode>=0x0F0000 and unicode<=0x0FFFFD then unicode=-1 -elseif unicode>=0x100000 and unicode<=0x10FFFD then + elseif unicode>=0x100000 and unicode<=0x10FFFD then unicode=-1 -end + end local name=glyph.name or cidnames[index] if not unicode or unicode==-1 then unicode=cidunicodes[index] -- cgit v1.2.3