diff options
| author | Hans Hagen <pragma@wxs.nl> | 2022-12-01 13:43:10 +0100 |
|---|---|---|
| committer | Context Git Mirror Bot <phg@phi-gamma.net> | 2022-12-01 13:43:10 +0100 |
| commit | 08fa92c1c94d9faddee48590a1a20506e89c191c (patch) | |
| tree | 12bbf6fa56b69c47340d4c15f8f601be2d9487b5 | |
| parent | 2593c827482f6c5a315d504cd5316879d6172656 (diff) | |
| download | context-08fa92c1c94d9faddee48590a1a20506e89c191c.tar.gz | |
2022-12-01 12:41:00
142 files changed, 8639 insertions, 1985 deletions
diff --git a/context/data/scite/context/lexers/data/scite-context-data-tex.lua b/context/data/scite/context/lexers/data/scite-context-data-tex.lua index e9d97ef2c..6d5a2656a 100644 --- a/context/data/scite/context/lexers/data/scite-context-data-tex.lua +++ b/context/data/scite/context/lexers/data/scite-context-data-tex.lua @@ -1,7 +1,7 @@ return { ["aleph"]={ "Alephminorversion", "Alephrevision", "Alephversion" }, ["etex"]={ "botmarks", "clubpenalties", "currentgrouplevel", "currentgrouptype", "currentifbranch", "currentiflevel", "currentiftype", "detokenize", "dimexpr", "displaywidowpenalties", "everyeof", "firstmarks", "fontchardp", "fontcharht", "fontcharic", "fontcharwd", "glueexpr", "glueshrink", "glueshrinkorder", "gluestretch", "gluestretchorder", "gluetomu", "ifcsname", "ifdefined", "iffontchar", "interactionmode", "interlinepenalties", "lastlinefit", "lastnodetype", "marks", "muexpr", "mutoglue", "numexpr", "pagediscards", "parshapedimen", "parshapeindent", "parshapelength", "predisplaydirection", "protected", "savinghyphcodes", "savingvdiscards", "scantokens", "showgroups", "showifs", "showtokens", "splitbotmarks", "splitdiscards", "splitfirstmarks", "topmarks", "tracingassigns", "tracinggroups", "tracingifs", "tracingnesting", "unexpanded", "unless", "widowpenalties" }, - ["luatex"]={ "Uabove", "Uabovewithdelims", "Uatop", "Uatopwithdelims", "Uchar", "Udelcode", "Udelimited", "Udelimiter", "Udelimiterover", "Udelimiterunder", "Uhextensible", "Uleft", "Umathaccent", "Umathaccentbasedepth", "Umathaccentbaseheight", "Umathaccentbottomovershoot", "Umathaccentbottomshiftdown", "Umathaccentextendmargin", "Umathaccentsuperscriptdrop", "Umathaccentsuperscriptpercent", "Umathaccenttopovershoot", "Umathaccenttopshiftup", "Umathaccentvariant", "Umathadapttoleft", "Umathadapttoright", "Umathaxis", "Umathbottomaccentvariant", "Umathchar", "Umathcharclass", "Umathchardef", "Umathcharfam", "Umathcharslot", "Umathclass", "Umathcode", "Umathconnectoroverlapmin", "Umathdegreevariant", "Umathdelimiterovervariant", "Umathdelimiterpercent", "Umathdelimitershortfall", "Umathdelimiterundervariant", "Umathdenominatorvariant", "Umathdict", "Umathdictdef", "Umathdiscretionary", "Umathextrasubpreshift", "Umathextrasubprespace", "Umathextrasubshift", "Umathextrasubspace", "Umathextrasuppreshift", "Umathextrasupprespace", "Umathextrasupshift", "Umathextrasupspace", "Umathflattenedaccentbasedepth", "Umathflattenedaccentbaseheight", "Umathflattenedaccentbottomshiftdown", "Umathflattenedaccenttopshiftup", "Umathfractiondelsize", "Umathfractiondenomdown", "Umathfractiondenomvgap", "Umathfractionnumup", "Umathfractionnumvgap", "Umathfractionrule", "Umathfractionvariant", "Umathhextensiblevariant", "Umathlimitabovebgap", "Umathlimitabovekern", "Umathlimitabovevgap", "Umathlimitbelowbgap", "Umathlimitbelowkern", "Umathlimitbelowvgap", "Umathlimits", "Umathnoaxis", "Umathnolimits", "Umathnolimitsubfactor", "Umathnolimitsupfactor", "Umathnumeratorvariant", "Umathopenupdepth", "Umathopenupheight", "Umathoperatorsize", "Umathoverbarkern", "Umathoverbarrule", "Umathoverbarvgap", "Umathoverdelimiterbgap", "Umathoverdelimitervariant", "Umathoverdelimitervgap", "Umathoverlayaccentvariant", "Umathoverlinevariant", "Umathphantom", "Umathpresubshiftdistance", "Umathpresupshiftdistance", "Umathprimeraise", "Umathprimeraisecomposed", "Umathprimeshiftdrop", "Umathprimeshiftup", "Umathprimespaceafter", "Umathprimevariant", "Umathprimewidth", "Umathquad", "Umathradicaldegreeafter", "Umathradicaldegreebefore", "Umathradicaldegreeraise", "Umathradicalextensibleafter", "Umathradicalextensiblebefore", "Umathradicalkern", "Umathradicalrule", "Umathradicalvariant", "Umathradicalvgap", "Umathruledepth", "Umathruleheight", "Umathskeweddelimitertolerance", "Umathskewedfractionhgap", "Umathskewedfractionvgap", "Umathsource", "Umathspaceafterscript", "Umathspacebeforescript", "Umathstackdenomdown", "Umathstacknumup", "Umathstackvariant", "Umathstackvgap", "Umathsubscriptvariant", "Umathsubshiftdistance", "Umathsubshiftdown", "Umathsubshiftdrop", "Umathsubsupshiftdown", "Umathsubsupvgap", "Umathsubtopmax", "Umathsupbottommin", "Umathsuperscriptvariant", "Umathsupshiftdistance", "Umathsupshiftdrop", "Umathsupshiftup", "Umathsupsubbottommax", "Umathtopaccentvariant", "Umathunderbarkern", "Umathunderbarrule", "Umathunderbarvgap", "Umathunderdelimiterbgap", "Umathunderdelimitervariant", "Umathunderdelimitervgap", "Umathunderlinevariant", "Umathuseaxis", "Umathvextensiblevariant", "Umathvoid", "Umathxscale", "Umathyscale", "Umiddle", "Unosubprescript", "Unosubscript", "Unosuperprescript", "Unosuperscript", "Uoperator", "Uover", "Uoverdelimiter", "Uoverwithdelims", "Uprimescript", "Uradical", "Uright", "Uroot", "Urooted", "Ushiftedsubprescript", "Ushiftedsubscript", "Ushiftedsuperprescript", "Ushiftedsuperscript", "Uskewed", "Uskewedwithdelims", "Ustack", "Ustartdisplaymath", "Ustartmath", "Ustartmathmode", "Ustopdisplaymath", "Ustopmath", "Ustopmathmode", "Ustretched", "Ustretchedwithdelims", "Ustyle", "Usubprescript", "Usubscript", "Usuperprescript", "Usuperscript", "Uunderdelimiter", "Uvextensible", "adjustspacing", "adjustspacingshrink", "adjustspacingstep", "adjustspacingstretch", "afterassigned", "aftergrouped", "aliased", "aligncontent", "alignmark", "alignmentcellsource", "alignmentwrapsource", "aligntab", "allcrampedstyles", "alldisplaystyles", "allmainstyles", "allmathstyles", "allscriptscriptstyles", "allscriptstyles", "allsplitstyles", "alltextstyles", "alluncrampedstyles", "allunsplitstyles", "atendofgroup", "atendofgrouped", "attribute", "attributedef", "automaticdiscretionary", "automatichyphenpenalty", "automigrationmode", "autoparagraphmode", "begincsname", "beginlocalcontrol", "beginmathgroup", "beginsimplegroup", "boundary", "boxadapt", "boxanchor", "boxanchors", "boxattribute", "boxdirection", "boxfreeze", "boxgeometry", "boxorientation", "boxrepack", "boxshift", "boxsource", "boxtarget", "boxtotal", "boxxmove", "boxxoffset", "boxymove", "boxyoffset", "catcodetable", "clearmarks", "copymathatomrule", "copymathparent", "copymathspacing", "crampeddisplaystyle", "crampedscriptscriptstyle", "crampedscriptstyle", "crampedtextstyle", "csstring", "currentloopiterator", "currentloopnesting", "currentmarks", "defcsname", "detokenized", "dimensiondef", "dimexpression", "directlua", "edefcsname", "efcode", "endlocalcontrol", "endmathgroup", "endsimplegroup", "enforced", "etoks", "etoksapp", "etokspre", "everybeforepar", "everymathatom", "everytab", "exceptionpenalty", "expand", "expandafterpars", "expandafterspaces", "expandcstoken", "expanded", "expandedafter", "expandedloop", "expandtoken", "explicitdiscretionary", "explicithyphenpenalty", "firstvalidlanguage", "flushmarks", "fontcharta", "fontid", "fontmathcontrol", "fontspecdef", "fontspecid", "fontspecifiedname", "fontspecifiedsize", "fontspecscale", "fontspecxscale", "fontspecyscale", "fonttextcontrol", "formatname", "frozen", "futurecsname", "futuredef", "futureexpand", "futureexpandis", "futureexpandisap", "gdefcsname", "gleaders", "glet", "gletcsname", "glettonothing", "gluespecdef", "glyph", "glyphdatafield", "glyphoptions", "glyphscale", "glyphscriptfield", "glyphscriptscale", "glyphscriptscriptscale", "glyphstatefield", "glyphtextscale", "glyphxoffset", "glyphxscale", "glyphxscaled", "glyphyoffset", "glyphyscale", "glyphyscaled", "gtoksapp", "gtokspre", "hccode", "hjcode", "hmcode", "holdingmigrations", "hpack", "hyphenationmin", "hyphenationmode", "ifabsdim", "ifabsnum", "ifarguments", "ifboolean", "ifchkdim", "ifchknum", "ifcmpdim", "ifcmpnum", "ifcondition", "ifcstok", "ifdimexpression", "ifdimval", "ifempty", "ifflags", "ifhaschar", "ifhastok", "ifhastoks", "ifhasxtoks", "ifincsname", "ifinsert", "ifmathparameter", "ifmathstyle", "ifnumexpression", "ifnumval", "ifparameter", "ifparameters", "ifrelax", "iftok", "ifzerodim", "ifzeronum", "ignorearguments", "ignorepars", "immediate", "immutable", "indexofcharacter", "indexofregister", "inherited", "initcatcodetable", "insertbox", "insertcopy", "insertdepth", "insertdistance", "insertheight", "insertheights", "insertlimit", "insertmaxdepth", "insertmode", "insertmultiplier", "insertpenalty", "insertprogress", "insertstorage", "insertstoring", "insertunbox", "insertuncopy", "insertwidth", "instance", "integerdef", "lastarguments", "lastatomclass", "lastboundary", "lastchkdim", "lastchknum", "lastleftclass", "lastloopiterator", "lastnamedcs", "lastnodesubtype", "lastpageextra", "lastparcontext", "lastrightclass", "leftmarginkern", "letcharcode", "letcsname", "letfrozen", "letmathatomrule", "letmathparent", "letmathspacing", "letprotected", "lettonothing", "linebreakcriterium", "linedirection", "localbrokenpenalty", "localcontrol", "localcontrolled", "localcontrolledloop", "localinterlinepenalty", "localleftbox", "localleftboxbox", "localmiddlebox", "localmiddleboxbox", "localrightbox", "localrightboxbox", "lpcode", "luabytecode", "luabytecodecall", "luacopyinputnodes", "luadef", "luaescapestring", "luafunction", "luafunctioncall", "luatexbanner", "luatexrevision", "luatexversion", "mathaccent", "mathatom", "mathatomglue", "mathatomskip", "mathbackwardpenalties", "mathbeginclass", "mathcheckfencesmode", "mathdictgroup", "mathdictproperties", "mathdirection", "mathdisplaymode", "mathdisplayskipmode", "mathdoublescriptmode", "mathendclass", "matheqnogapstep", "mathfenced", "mathfontcontrol", "mathforwardpenalties", "mathfrac", "mathghost", "mathgluemode", "mathgroupingmode", "mathleftclass", "mathlimitsmode", "mathmainstyle", "mathmiddle", "mathnolimitsmode", "mathpenaltiesmode", "mathrad", "mathrightclass", "mathrulesfam", "mathrulesmode", "mathscale", "mathscriptsmode", "mathslackmode", "mathspacingmode", "mathstackstyle", "mathstyle", "mathstylefontid", "mathsurroundmode", "mathsurroundskip", "maththreshold", "meaningasis", "meaningfull", "meaningless", "mugluespecdef", "mutable", "noaligned", "noatomruling", "noboundary", "nohrule", "norelax", "normalizelinemode", "normalizeparmode", "nospaces", "novrule", "numericscale", "numexpression", "orelse", "orphanpenalties", "orphanpenalty", "orunless", "outputbox", "overloaded", "overloadmode", "overshoot", "pageboundary", "pageextragoal", "pagevsize", "parametercount", "parametermark", "parattribute", "pardirection", "permanent", "pettymuskip", "postexhyphenchar", "posthyphenchar", "postinlinepenalty", "prebinoppenalty", "predisplaygapfactor", "preexhyphenchar", "prehyphenchar", "preinlinepenalty", "prerelpenalty", "protrudechars", "protrusionboundary", "pxdimen", "quitloop", "quitvmode", "resetmathspacing", "retokenized", "rightmarginkern", "rpcode", "savecatcodetable", "scaledemwidth", "scaledexheight", "scaledextraspace", "scaledfontdimen", "scaledinterwordshrink", "scaledinterwordspace", "scaledinterwordstretch", "scaledmathstyle", "scaledslantperpoint", "scantextokens", "semiexpand", "semiexpanded", "semiprotected", "setdefaultmathcodes", "setfontid", "setmathatomrule", "setmathdisplaypostpenalty", "setmathdisplayprepenalty", "setmathignore", "setmathoptions", "setmathpostpenalty", "setmathprepenalty", "setmathspacing", "shapingpenaltiesmode", "shapingpenalty", "snapshotpar", "srule", "supmarkmode", "swapcsvalues", "tabsize", "textdirection", "thewithoutunit", "tinymuskip", "todimension", "tohexadecimal", "tointeger", "tokenized", "toksapp", "tokspre", "tolerant", "tomathstyle", "toscaled", "tosparsedimension", "tosparsescaled", "tpack", "tracingadjusts", "tracingalignments", "tracingexpressions", "tracingfonts", "tracingfullboxes", "tracinghyphenation", "tracinginserts", "tracinglevels", "tracingmarks", "tracingmath", "tracingnodes", "tracingpenalties", "uleaders", "undent", "unexpandedloop", "unhpack", "unletfrozen", "unletprotected", "untraced", "unvpack", "variablefam", "vpack", "wordboundary", "wrapuppar", "xdefcsname", "xtoks", "xtoksapp", "xtokspre" }, + ["luatex"]={ "Uabove", "Uabovewithdelims", "Uatop", "Uatopwithdelims", "Uchar", "Udelcode", "Udelimited", "Udelimiter", "Udelimiterover", "Udelimiterunder", "Uhextensible", "Uleft", "Umathaccent", "Umathaccentbasedepth", "Umathaccentbaseheight", "Umathaccentbottomovershoot", "Umathaccentbottomshiftdown", "Umathaccentextendmargin", "Umathaccentsuperscriptdrop", "Umathaccentsuperscriptpercent", "Umathaccenttopovershoot", "Umathaccenttopshiftup", "Umathaccentvariant", "Umathadapttoleft", "Umathadapttoright", "Umathaxis", "Umathbottomaccentvariant", "Umathchar", "Umathcharclass", "Umathchardef", "Umathcharfam", "Umathcharslot", "Umathclass", "Umathcode", "Umathconnectoroverlapmin", "Umathdegreevariant", "Umathdelimiterovervariant", "Umathdelimiterpercent", "Umathdelimitershortfall", "Umathdelimiterundervariant", "Umathdenominatorvariant", "Umathdict", "Umathdictdef", "Umathdiscretionary", "Umathextrasubpreshift", "Umathextrasubprespace", "Umathextrasubshift", "Umathextrasubspace", "Umathextrasuppreshift", "Umathextrasupprespace", "Umathextrasupshift", "Umathextrasupspace", "Umathflattenedaccentbasedepth", "Umathflattenedaccentbaseheight", "Umathflattenedaccentbottomshiftdown", "Umathflattenedaccenttopshiftup", "Umathfractiondelsize", "Umathfractiondenomdown", "Umathfractiondenomvgap", "Umathfractionnumup", "Umathfractionnumvgap", "Umathfractionrule", "Umathfractionvariant", "Umathhextensiblevariant", "Umathlimitabovebgap", "Umathlimitabovekern", "Umathlimitabovevgap", "Umathlimitbelowbgap", "Umathlimitbelowkern", "Umathlimitbelowvgap", "Umathlimits", "Umathnoaxis", "Umathnolimits", "Umathnolimitsubfactor", "Umathnolimitsupfactor", "Umathnumeratorvariant", "Umathopenupdepth", "Umathopenupheight", "Umathoperatorsize", "Umathoverbarkern", "Umathoverbarrule", "Umathoverbarvgap", "Umathoverdelimiterbgap", "Umathoverdelimitervariant", "Umathoverdelimitervgap", "Umathoverlayaccentvariant", "Umathoverlinevariant", "Umathphantom", "Umathpresubshiftdistance", "Umathpresupshiftdistance", "Umathprimeraise", "Umathprimeraisecomposed", "Umathprimeshiftdrop", "Umathprimeshiftup", "Umathprimespaceafter", "Umathprimevariant", "Umathprimewidth", "Umathquad", "Umathradicaldegreeafter", "Umathradicaldegreebefore", "Umathradicaldegreeraise", "Umathradicalextensibleafter", "Umathradicalextensiblebefore", "Umathradicalkern", "Umathradicalrule", "Umathradicalvariant", "Umathradicalvgap", "Umathruledepth", "Umathruleheight", "Umathskeweddelimitertolerance", "Umathskewedfractionhgap", "Umathskewedfractionvgap", "Umathsource", "Umathspaceafterscript", "Umathspacebeforescript", "Umathstackdenomdown", "Umathstacknumup", "Umathstackvariant", "Umathstackvgap", "Umathsubscriptvariant", "Umathsubshiftdistance", "Umathsubshiftdown", "Umathsubshiftdrop", "Umathsubsupshiftdown", "Umathsubsupvgap", "Umathsubtopmax", "Umathsupbottommin", "Umathsuperscriptvariant", "Umathsupshiftdistance", "Umathsupshiftdrop", "Umathsupshiftup", "Umathsupsubbottommax", "Umathtopaccentvariant", "Umathunderbarkern", "Umathunderbarrule", "Umathunderbarvgap", "Umathunderdelimiterbgap", "Umathunderdelimitervariant", "Umathunderdelimitervgap", "Umathunderlinevariant", "Umathuseaxis", "Umathvextensiblevariant", "Umathvoid", "Umathxscale", "Umathyscale", "Umiddle", "Unosubprescript", "Unosubscript", "Unosuperprescript", "Unosuperscript", "Uoperator", "Uover", "Uoverdelimiter", "Uoverwithdelims", "Uprimescript", "Uradical", "Uright", "Uroot", "Urooted", "Ushiftedsubprescript", "Ushiftedsubscript", "Ushiftedsuperprescript", "Ushiftedsuperscript", "Uskewed", "Uskewedwithdelims", "Ustack", "Ustartdisplaymath", "Ustartmath", "Ustartmathmode", "Ustopdisplaymath", "Ustopmath", "Ustopmathmode", "Ustretched", "Ustretchedwithdelims", "Ustyle", "Usubprescript", "Usubscript", "Usuperprescript", "Usuperscript", "Uunderdelimiter", "Uvextensible", "adjustspacing", "adjustspacingshrink", "adjustspacingstep", "adjustspacingstretch", "afterassigned", "aftergrouped", "aliased", "aligncontent", "alignmark", "alignmentcellsource", "alignmentwrapsource", "aligntab", "allcrampedstyles", "alldisplaystyles", "allmainstyles", "allmathstyles", "allscriptscriptstyles", "allscriptstyles", "allsplitstyles", "alltextstyles", "alluncrampedstyles", "allunsplitstyles", "amcode", "atendofgroup", "atendofgrouped", "attribute", "attributedef", "automaticdiscretionary", "automatichyphenpenalty", "automigrationmode", "autoparagraphmode", "begincsname", "beginlocalcontrol", "beginmathgroup", "beginsimplegroup", "boundary", "boxadapt", "boxanchor", "boxanchors", "boxattribute", "boxdirection", "boxfreeze", "boxgeometry", "boxorientation", "boxrepack", "boxshift", "boxsource", "boxtarget", "boxtotal", "boxxmove", "boxxoffset", "boxymove", "boxyoffset", "catcodetable", "cfcode", "clearmarks", "copymathatomrule", "copymathparent", "copymathspacing", "crampeddisplaystyle", "crampedscriptscriptstyle", "crampedscriptstyle", "crampedtextstyle", "csactive", "csstring", "currentloopiterator", "currentloopnesting", "currentmarks", "defcsname", "detokenized", "dimensiondef", "dimexpression", "directlua", "edefcsname", "efcode", "endlocalcontrol", "endmathgroup", "endsimplegroup", "enforced", "etoks", "etoksapp", "etokspre", "everybeforepar", "everymathatom", "everytab", "exceptionpenalty", "expand", "expandactive", "expandafterpars", "expandafterspaces", "expandcstoken", "expanded", "expandedafter", "expandedloop", "expandtoken", "explicitdiscretionary", "explicithyphenpenalty", "firstvalidlanguage", "flushmarks", "fontcharta", "fontid", "fontmathcontrol", "fontspecdef", "fontspecid", "fontspecifiedname", "fontspecifiedsize", "fontspecscale", "fontspecxscale", "fontspecyscale", "fonttextcontrol", "formatname", "frozen", "futurecsname", "futuredef", "futureexpand", "futureexpandis", "futureexpandisap", "gdefcsname", "gleaders", "glet", "gletcsname", "glettonothing", "gluespecdef", "glyph", "glyphdatafield", "glyphoptions", "glyphscale", "glyphscriptfield", "glyphscriptscale", "glyphscriptscriptscale", "glyphstatefield", "glyphtextscale", "glyphxoffset", "glyphxscale", "glyphxscaled", "glyphyoffset", "glyphyscale", "glyphyscaled", "gtoksapp", "gtokspre", "hccode", "hjcode", "hmcode", "holdingmigrations", "hpack", "hyphenationmin", "hyphenationmode", "ifabsdim", "ifabsnum", "ifarguments", "ifboolean", "ifchkdim", "ifchknum", "ifcmpdim", "ifcmpnum", "ifcondition", "ifcstok", "ifdimexpression", "ifdimval", "ifempty", "ifflags", "ifhaschar", "ifhastok", "ifhastoks", "ifhasxtoks", "ifincsname", "ifinsert", "ifmathparameter", "ifmathstyle", "ifnumexpression", "ifnumval", "ifparameter", "ifparameters", "ifrelax", "iftok", "ifzerodim", "ifzeronum", "ignorearguments", "ignorepars", "immediate", "immutable", "indexofcharacter", "indexofregister", "inherited", "initcatcodetable", "insertbox", "insertcopy", "insertdepth", "insertdistance", "insertheight", "insertheights", "insertlimit", "insertmaxdepth", "insertmode", "insertmultiplier", "insertpenalty", "insertprogress", "insertstorage", "insertstoring", "insertunbox", "insertuncopy", "insertwidth", "instance", "integerdef", "lastarguments", "lastatomclass", "lastboundary", "lastchkdim", "lastchknum", "lastleftclass", "lastloopiterator", "lastnamedcs", "lastnodesubtype", "lastpageextra", "lastparcontext", "lastrightclass", "leftmarginkern", "letcharcode", "letcsname", "letfrozen", "letmathatomrule", "letmathparent", "letmathspacing", "letprotected", "lettonothing", "linebreakcriterium", "linedirection", "localbrokenpenalty", "localcontrol", "localcontrolled", "localcontrolledloop", "localinterlinepenalty", "localleftbox", "localleftboxbox", "localmiddlebox", "localmiddleboxbox", "localrightbox", "localrightboxbox", "lpcode", "luabytecode", "luabytecodecall", "luacopyinputnodes", "luadef", "luaescapestring", "luafunction", "luafunctioncall", "luatexbanner", "luatexrevision", "luatexversion", "mathaccent", "mathatom", "mathatomglue", "mathatomskip", "mathbackwardpenalties", "mathbeginclass", "mathcheckfencesmode", "mathdictgroup", "mathdictproperties", "mathdirection", "mathdisplaymode", "mathdisplayskipmode", "mathdoublescriptmode", "mathendclass", "matheqnogapstep", "mathfenced", "mathfontcontrol", "mathforwardpenalties", "mathfrac", "mathghost", "mathgluemode", "mathgroupingmode", "mathleftclass", "mathlimitsmode", "mathmainstyle", "mathmiddle", "mathnolimitsmode", "mathpenaltiesmode", "mathrad", "mathrightclass", "mathrulesfam", "mathrulesmode", "mathscale", "mathscriptsmode", "mathslackmode", "mathspacingmode", "mathstackstyle", "mathstyle", "mathstylefontid", "mathsurroundmode", "mathsurroundskip", "maththreshold", "meaningasis", "meaningfull", "meaningless", "mugluespecdef", "mutable", "noaligned", "noatomruling", "noboundary", "nohrule", "norelax", "normalizelinemode", "normalizeparmode", "nospaces", "novrule", "numericscale", "numexpression", "orelse", "orphanpenalties", "orphanpenalty", "orunless", "outputbox", "overloaded", "overloadmode", "overshoot", "pageboundary", "pageextragoal", "pagevsize", "parametercount", "parametermark", "parattribute", "pardirection", "permanent", "pettymuskip", "postexhyphenchar", "posthyphenchar", "postinlinepenalty", "prebinoppenalty", "predisplaygapfactor", "preexhyphenchar", "prehyphenchar", "preinlinepenalty", "prerelpenalty", "protrudechars", "protrusionboundary", "pxdimen", "quitloop", "quitvmode", "resetmathspacing", "retokenized", "rightmarginkern", "rpcode", "savecatcodetable", "scaledemwidth", "scaledexheight", "scaledextraspace", "scaledfontdimen", "scaledinterwordshrink", "scaledinterwordspace", "scaledinterwordstretch", "scaledmathstyle", "scaledslantperpoint", "scantextokens", "semiexpand", "semiexpanded", "semiprotected", "setdefaultmathcodes", "setfontid", "setmathatomrule", "setmathdisplaypostpenalty", "setmathdisplayprepenalty", "setmathignore", "setmathoptions", "setmathpostpenalty", "setmathprepenalty", "setmathspacing", "shapingpenaltiesmode", "shapingpenalty", "snapshotpar", "srule", "supmarkmode", "swapcsvalues", "tabsize", "textdirection", "thewithoutunit", "tinymuskip", "todimension", "tohexadecimal", "tointeger", "tokenized", "toksapp", "tokspre", "tolerant", "tomathstyle", "toscaled", "tosparsedimension", "tosparsescaled", "tpack", "tracingadjusts", "tracingalignments", "tracingexpressions", "tracingfonts", "tracingfullboxes", "tracinghyphenation", "tracinginserts", "tracinglevels", "tracingmarks", "tracingmath", "tracingnodes", "tracingpenalties", "uleaders", "undent", "unexpandedloop", "unhpack", "unletfrozen", "unletprotected", "untraced", "unvpack", "variablefam", "vpack", "wordboundary", "wrapuppar", "xdefcsname", "xtoks", "xtoksapp", "xtokspre" }, ["omega"]={ "Omegaminorversion", "Omegarevision", "Omegaversion" }, ["pdftex"]={ "ifpdfabsdim", "ifpdfabsnum", "ifpdfprimitive", "pdfadjustspacing", "pdfannot", "pdfcatalog", "pdfcolorstack", "pdfcolorstackinit", "pdfcompresslevel", "pdfcopyfont", "pdfcreationdate", "pdfdecimaldigits", "pdfdest", "pdfdestmargin", "pdfdraftmode", "pdfeachlinedepth", "pdfeachlineheight", "pdfendlink", "pdfendthread", "pdffirstlineheight", "pdffontattr", "pdffontexpand", "pdffontname", "pdffontobjnum", "pdffontsize", "pdfgamma", "pdfgentounicode", "pdfglyphtounicode", "pdfhorigin", "pdfignoreddimen", "pdfignoreunknownimages", "pdfimageaddfilename", "pdfimageapplygamma", "pdfimagegamma", "pdfimagehicolor", "pdfimageresolution", "pdfincludechars", "pdfinclusioncopyfonts", "pdfinclusionerrorlevel", "pdfinfo", "pdfinfoomitdate", "pdfinsertht", "pdflastannot", "pdflastlinedepth", "pdflastlink", "pdflastobj", "pdflastxform", "pdflastximage", "pdflastximagepages", "pdflastxpos", "pdflastypos", "pdflinkmargin", "pdfliteral", "pdfmajorversion", "pdfmapfile", "pdfmapline", "pdfminorversion", "pdfnames", "pdfnoligatures", "pdfnormaldeviate", "pdfobj", "pdfobjcompresslevel", "pdfomitcharset", "pdfomitcidset", "pdfomitinfodict", "pdfoutline", "pdfoutput", "pdfpageattr", "pdfpagebox", "pdfpageheight", "pdfpageref", "pdfpageresources", "pdfpagesattr", "pdfpagewidth", "pdfpkfixeddpi", "pdfpkmode", "pdfpkresolution", "pdfprimitive", "pdfprotrudechars", "pdfpxdimen", "pdfrandomseed", "pdfrecompress", "pdfrefobj", "pdfrefxform", "pdfrefximage", "pdfreplacefont", "pdfrestore", "pdfretval", "pdfsave", "pdfsavepos", "pdfsetmatrix", "pdfsetrandomseed", "pdfstartlink", "pdfstartthread", "pdfsuppressoptionalinfo", "pdfsuppressptexinfo", "pdftexbanner", "pdftexrevision", "pdftexversion", "pdfthread", "pdfthreadmargin", "pdftracingfonts", "pdftrailer", "pdftrailerid", "pdfuniformdeviate", "pdfuniqueresname", "pdfvorigin", "pdfxform", "pdfxformattr", "pdfxformmargin", "pdfxformname", "pdfxformresources", "pdfximage" }, ["tex"]={ " ", "-", "/", "above", "abovedisplayshortskip", "abovedisplayskip", "abovewithdelims", "accent", "adjdemerits", "advance", "afterassignment", "aftergroup", "atop", "atopwithdelims", "badness", "baselineskip", "batchmode", "begingroup", "belowdisplayshortskip", "belowdisplayskip", "binoppenalty", "botmark", "box", "boxmaxdepth", "brokenpenalty", "catcode", "char", "chardef", "cleaders", "clubpenalty", "copy", "count", "countdef", "cr", "crcr", "csname", "day", "deadcycles", "def", "defaulthyphenchar", "defaultskewchar", "delcode", "delimiter", "delimiterfactor", "delimitershortfall", "dimen", "dimendef", "discretionary", "displayindent", "displaylimits", "displaystyle", "displaywidowpenalty", "displaywidth", "divide", "doublehyphendemerits", "dp", "dump", "edef", "else", "emergencystretch", "end", "endcsname", "endgroup", "endinput", "endlinechar", "eqno", "errhelp", "errmessage", "errorcontextlines", "errorstopmode", "escapechar", "everycr", "everydisplay", "everyhbox", "everyjob", "everymath", "everypar", "everyvbox", "exhyphenchar", "exhyphenpenalty", "expandafter", "fam", "fi", "finalhyphendemerits", "firstmark", "floatingpenalty", "font", "fontdimen", "fontname", "futurelet", "gdef", "global", "globaldefs", "halign", "hangafter", "hangindent", "hbadness", "hbox", "hfil", "hfill", "hfilneg", "hfuzz", "holdinginserts", "hrule", "hsize", "hskip", "hss", "ht", "hyphenation", "hyphenchar", "hyphenpenalty", "if", "ifcase", "ifcat", "ifdim", "iffalse", "ifhbox", "ifhmode", "ifinner", "ifmmode", "ifnum", "ifodd", "iftrue", "ifvbox", "ifvmode", "ifvoid", "ifx", "ignorespaces", "indent", "input", "inputlineno", "insert", "insertpenalties", "interlinepenalty", "jobname", "kern", "language", "lastbox", "lastkern", "lastpenalty", "lastskip", "lccode", "leaders", "left", "lefthyphenmin", "leftskip", "leqno", "let", "limits", "linepenalty", "lineskip", "lineskiplimit", "long", "looseness", "lower", "lowercase", "mark", "mathbin", "mathchar", "mathchardef", "mathchoice", "mathclose", "mathcode", "mathinner", "mathop", "mathopen", "mathord", "mathpunct", "mathrel", "mathsurround", "maxdeadcycles", "maxdepth", "meaning", "medmuskip", "message", "middle", "mkern", "month", "moveleft", "moveright", "mskip", "multiply", "muskip", "muskipdef", "newlinechar", "noalign", "noexpand", "noindent", "nolimits", "nonscript", "nonstopmode", "nulldelimiterspace", "nullfont", "number", "omit", "or", "outer", "output", "outputpenalty", "over", "overfullrule", "overline", "overwithdelims", "pagedepth", "pagefilllstretch", "pagefillstretch", "pagefilstretch", "pagegoal", "pageshrink", "pagestretch", "pagetotal", "par", "parfillleftskip", "parfillskip", "parindent", "parinitleftskip", "parinitrightskip", "parshape", "parskip", "patterns", "pausing", "penalty", "postdisplaypenalty", "predisplaypenalty", "predisplaysize", "pretolerance", "prevdepth", "prevgraf", "radical", "raise", "relax", "relpenalty", "right", "righthyphenmin", "rightskip", "romannumeral", "scriptfont", "scriptscriptfont", "scriptscriptstyle", "scriptspace", "scriptstyle", "scrollmode", "setbox", "setlanguage", "sfcode", "shipout", "show", "showbox", "showboxbreadth", "showboxdepth", "showlists", "shownodedetails", "showthe", "skewchar", "skip", "skipdef", "spacefactor", "spaceskip", "span", "splitbotmark", "splitfirstmark", "splitmaxdepth", "splittopskip", "string", "tabskip", "textfont", "textstyle", "the", "thickmuskip", "thinmuskip", "time", "toks", "toksdef", "tolerance", "topmark", "topskip", "tracingcommands", "tracinglostchars", "tracingmacros", "tracingonline", "tracingoutput", "tracingpages", "tracingparagraphs", "tracingrestores", "tracingstats", "uccode", "uchyph", "unboundary", "underline", "unhbox", "unhcopy", "unkern", "unpenalty", "unskip", "unvbox", "unvcopy", "uppercase", "vadjust", "valign", "vbadness", "vbox", "vcenter", "vfil", "vfill", "vfilneg", "vfuzz", "vrule", "vsize", "vskip", "vsplit", "vss", "vtop", "wd", "widowpenalty", "xdef", "xleaders", "xspaceskip", "year" }, diff --git a/context/data/scite/context/scite-context-data-tex.properties b/context/data/scite/context/scite-context-data-tex.properties index 8675453ea..f7b6a3991 100644 --- a/context/data/scite/context/scite-context-data-tex.properties +++ b/context/data/scite/context/scite-context-data-tex.properties @@ -56,85 +56,86 @@ Uunderdelimiter Uvextensible adjustspacing adjustspacingshrink adjustspacingstep adjustspacingstretch afterassigned aftergrouped aliased aligncontent \ alignmark alignmentcellsource alignmentwrapsource aligntab allcrampedstyles \ alldisplaystyles allmainstyles allmathstyles allscriptscriptstyles allscriptstyles \ -allsplitstyles alltextstyles alluncrampedstyles allunsplitstyles atendofgroup \ -atendofgrouped attribute attributedef automaticdiscretionary automatichyphenpenalty \ -automigrationmode autoparagraphmode begincsname beginlocalcontrol beginmathgroup \ -beginsimplegroup boundary boxadapt boxanchor boxanchors \ -boxattribute boxdirection boxfreeze boxgeometry boxorientation \ -boxrepack boxshift boxsource boxtarget boxtotal \ -boxxmove boxxoffset boxymove boxyoffset catcodetable \ -clearmarks copymathatomrule copymathparent copymathspacing crampeddisplaystyle \ -crampedscriptscriptstyle crampedscriptstyle crampedtextstyle csstring currentloopiterator \ -currentloopnesting currentmarks defcsname detokenized dimensiondef \ -dimexpression directlua edefcsname efcode endlocalcontrol \ -endmathgroup endsimplegroup enforced etoks etoksapp \ -etokspre everybeforepar everymathatom everytab exceptionpenalty \ -expand expandafterpars expandafterspaces expandcstoken expanded \ -expandedafter expandedloop expandtoken explicitdiscretionary explicithyphenpenalty \ -firstvalidlanguage flushmarks fontcharta fontid fontmathcontrol \ -fontspecdef fontspecid fontspecifiedname fontspecifiedsize fontspecscale \ -fontspecxscale fontspecyscale fonttextcontrol formatname frozen \ -futurecsname futuredef futureexpand futureexpandis futureexpandisap \ -gdefcsname gleaders glet gletcsname glettonothing \ -gluespecdef glyph glyphdatafield glyphoptions glyphscale \ -glyphscriptfield glyphscriptscale glyphscriptscriptscale glyphstatefield glyphtextscale \ -glyphxoffset glyphxscale glyphxscaled glyphyoffset glyphyscale \ -glyphyscaled gtoksapp gtokspre hccode hjcode \ -hmcode holdingmigrations hpack hyphenationmin hyphenationmode \ -ifabsdim ifabsnum ifarguments ifboolean ifchkdim \ -ifchknum ifcmpdim ifcmpnum ifcondition ifcstok \ -ifdimexpression ifdimval ifempty ifflags ifhaschar \ -ifhastok ifhastoks ifhasxtoks ifincsname ifinsert \ -ifmathparameter ifmathstyle ifnumexpression ifnumval ifparameter \ -ifparameters ifrelax iftok ifzerodim ifzeronum \ -ignorearguments ignorepars immediate immutable indexofcharacter \ -indexofregister inherited initcatcodetable insertbox insertcopy \ -insertdepth insertdistance insertheight insertheights insertlimit \ -insertmaxdepth insertmode insertmultiplier insertpenalty insertprogress \ -insertstorage insertstoring insertunbox insertuncopy insertwidth \ -instance integerdef lastarguments lastatomclass lastboundary \ -lastchkdim lastchknum lastleftclass lastloopiterator lastnamedcs \ -lastnodesubtype lastpageextra lastparcontext lastrightclass leftmarginkern \ -letcharcode letcsname letfrozen letmathatomrule letmathparent \ -letmathspacing letprotected lettonothing linebreakcriterium linedirection \ -localbrokenpenalty localcontrol localcontrolled localcontrolledloop localinterlinepenalty \ -localleftbox localleftboxbox localmiddlebox localmiddleboxbox localrightbox \ -localrightboxbox lpcode luabytecode luabytecodecall luacopyinputnodes \ -luadef luaescapestring luafunction luafunctioncall luatexbanner \ -luatexrevision luatexversion mathaccent mathatom mathatomglue \ -mathatomskip mathbackwardpenalties mathbeginclass mathcheckfencesmode mathdictgroup \ -mathdictproperties mathdirection mathdisplaymode mathdisplayskipmode mathdoublescriptmode \ -mathendclass matheqnogapstep mathfenced mathfontcontrol mathforwardpenalties \ -mathfrac mathghost mathgluemode mathgroupingmode mathleftclass \ -mathlimitsmode mathmainstyle mathmiddle mathnolimitsmode mathpenaltiesmode \ -mathrad mathrightclass mathrulesfam mathrulesmode mathscale \ -mathscriptsmode mathslackmode mathspacingmode mathstackstyle mathstyle \ -mathstylefontid mathsurroundmode mathsurroundskip maththreshold meaningasis \ -meaningfull meaningless mugluespecdef mutable noaligned \ -noatomruling noboundary nohrule norelax normalizelinemode \ -normalizeparmode nospaces novrule numericscale numexpression \ -orelse orphanpenalties orphanpenalty orunless outputbox \ -overloaded overloadmode overshoot pageboundary pageextragoal \ -pagevsize parametercount parametermark parattribute pardirection \ -permanent pettymuskip postexhyphenchar posthyphenchar postinlinepenalty \ -prebinoppenalty predisplaygapfactor preexhyphenchar prehyphenchar preinlinepenalty \ -prerelpenalty protrudechars protrusionboundary pxdimen quitloop \ -quitvmode resetmathspacing retokenized rightmarginkern rpcode \ -savecatcodetable scaledemwidth scaledexheight scaledextraspace scaledfontdimen \ -scaledinterwordshrink scaledinterwordspace scaledinterwordstretch scaledmathstyle scaledslantperpoint \ -scantextokens semiexpand semiexpanded semiprotected setdefaultmathcodes \ -setfontid setmathatomrule setmathdisplaypostpenalty setmathdisplayprepenalty setmathignore \ -setmathoptions setmathpostpenalty setmathprepenalty setmathspacing shapingpenaltiesmode \ -shapingpenalty snapshotpar srule supmarkmode swapcsvalues \ -tabsize textdirection thewithoutunit tinymuskip todimension \ -tohexadecimal tointeger tokenized toksapp tokspre \ -tolerant tomathstyle toscaled tosparsedimension tosparsescaled \ -tpack tracingadjusts tracingalignments tracingexpressions tracingfonts \ -tracingfullboxes tracinghyphenation tracinginserts tracinglevels tracingmarks \ -tracingmath tracingnodes tracingpenalties uleaders undent \ -unexpandedloop unhpack unletfrozen unletprotected untraced \ -unvpack variablefam vpack wordboundary wrapuppar \ -xdefcsname xtoks xtoksapp xtokspre +allsplitstyles alltextstyles alluncrampedstyles allunsplitstyles amcode \ +atendofgroup atendofgrouped attribute attributedef automaticdiscretionary \ +automatichyphenpenalty automigrationmode autoparagraphmode begincsname beginlocalcontrol \ +beginmathgroup beginsimplegroup boundary boxadapt boxanchor \ +boxanchors boxattribute boxdirection boxfreeze boxgeometry \ +boxorientation boxrepack boxshift boxsource boxtarget \ +boxtotal boxxmove boxxoffset boxymove boxyoffset \ +catcodetable cfcode clearmarks copymathatomrule copymathparent \ +copymathspacing crampeddisplaystyle crampedscriptscriptstyle crampedscriptstyle crampedtextstyle \ +csactive csstring currentloopiterator currentloopnesting currentmarks \ +defcsname detokenized dimensiondef dimexpression directlua \ +edefcsname efcode endlocalcontrol endmathgroup endsimplegroup \ +enforced etoks etoksapp etokspre everybeforepar \ +everymathatom everytab exceptionpenalty expand expandactive \ +expandafterpars expandafterspaces expandcstoken expanded expandedafter \ +expandedloop expandtoken explicitdiscretionary explicithyphenpenalty firstvalidlanguage \ +flushmarks fontcharta fontid fontmathcontrol fontspecdef \ +fontspecid fontspecifiedname fontspecifiedsize fontspecscale fontspecxscale \ +fontspecyscale fonttextcontrol formatname frozen futurecsname \ +futuredef futureexpand futureexpandis futureexpandisap gdefcsname \ +gleaders glet gletcsname glettonothing gluespecdef \ +glyph glyphdatafield glyphoptions glyphscale glyphscriptfield \ +glyphscriptscale glyphscriptscriptscale glyphstatefield glyphtextscale glyphxoffset \ +glyphxscale glyphxscaled glyphyoffset glyphyscale glyphyscaled \ +gtoksapp gtokspre hccode hjcode hmcode \ +holdingmigrations hpack hyphenationmin hyphenationmode ifabsdim \ +ifabsnum ifarguments ifboolean ifchkdim ifchknum \ +ifcmpdim ifcmpnum ifcondition ifcstok ifdimexpression \ +ifdimval ifempty ifflags ifhaschar ifhastok \ +ifhastoks ifhasxtoks ifincsname ifinsert ifmathparameter \ +ifmathstyle ifnumexpression ifnumval ifparameter ifparameters \ +ifrelax iftok ifzerodim ifzeronum ignorearguments \ +ignorepars immediate immutable indexofcharacter indexofregister \ +inherited initcatcodetable insertbox insertcopy insertdepth \ +insertdistance insertheight insertheights insertlimit insertmaxdepth \ +insertmode insertmultiplier insertpenalty insertprogress insertstorage \ +insertstoring insertunbox insertuncopy insertwidth instance \ +integerdef lastarguments lastatomclass lastboundary lastchkdim \ +lastchknum lastleftclass lastloopiterator lastnamedcs lastnodesubtype \ +lastpageextra lastparcontext lastrightclass leftmarginkern letcharcode \ +letcsname letfrozen letmathatomrule letmathparent letmathspacing \ +letprotected lettonothing linebreakcriterium linedirection localbrokenpenalty \ +localcontrol localcontrolled localcontrolledloop localinterlinepenalty localleftbox \ +localleftboxbox localmiddlebox localmiddleboxbox localrightbox localrightboxbox \ +lpcode luabytecode luabytecodecall luacopyinputnodes luadef \ +luaescapestring luafunction luafunctioncall luatexbanner luatexrevision \ +luatexversion mathaccent mathatom mathatomglue mathatomskip \ +mathbackwardpenalties mathbeginclass mathcheckfencesmode mathdictgroup mathdictproperties \ +mathdirection mathdisplaymode mathdisplayskipmode mathdoublescriptmode mathendclass \ +matheqnogapstep mathfenced mathfontcontrol mathforwardpenalties mathfrac \ +mathghost mathgluemode mathgroupingmode mathleftclass mathlimitsmode \ +mathmainstyle mathmiddle mathnolimitsmode mathpenaltiesmode mathrad \ +mathrightclass mathrulesfam mathrulesmode mathscale mathscriptsmode \ +mathslackmode mathspacingmode mathstackstyle mathstyle mathstylefontid \ +mathsurroundmode mathsurroundskip maththreshold meaningasis meaningfull \ +meaningless mugluespecdef mutable noaligned noatomruling \ +noboundary nohrule norelax normalizelinemode normalizeparmode \ +nospaces novrule numericscale numexpression orelse \ +orphanpenalties orphanpenalty orunless outputbox overloaded \ +overloadmode overshoot pageboundary pageextragoal pagevsize \ +parametercount parametermark parattribute pardirection permanent \ +pettymuskip postexhyphenchar posthyphenchar postinlinepenalty prebinoppenalty \ +predisplaygapfactor preexhyphenchar prehyphenchar preinlinepenalty prerelpenalty \ +protrudechars protrusionboundary pxdimen quitloop quitvmode \ +resetmathspacing retokenized rightmarginkern rpcode savecatcodetable \ +scaledemwidth scaledexheight scaledextraspace scaledfontdimen scaledinterwordshrink \ +scaledinterwordspace scaledinterwordstretch scaledmathstyle scaledslantperpoint scantextokens \ +semiexpand semiexpanded semiprotected setdefaultmathcodes setfontid \ +setmathatomrule setmathdisplaypostpenalty setmathdisplayprepenalty setmathignore setmathoptions \ +setmathpostpenalty setmathprepenalty setmathspacing shapingpenaltiesmode shapingpenalty \ +snapshotpar srule supmarkmode swapcsvalues tabsize \ +textdirection thewithoutunit tinymuskip todimension tohexadecimal \ +tointeger tokenized toksapp tokspre tolerant \ +tomathstyle toscaled tosparsedimension tosparsescaled tpack \ +tracingadjusts tracingalignments tracingexpressions tracingfonts tracingfullboxes \ +tracinghyphenation tracinginserts tracinglevels tracingmarks tracingmath \ +tracingnodes tracingpenalties uleaders undent unexpandedloop \ +unhpack unletfrozen unletprotected untraced unvpack \ +variablefam vpack wordboundary wrapuppar xdefcsname \ +xtoks xtoksapp xtokspre keywordclass.tex.omega=\ Omegaminorversion Omegarevision Omegaversion diff --git a/doc/context/sources/general/manuals/math/math-tweaks.tex b/doc/context/sources/general/manuals/math/math-tweaks.tex index cf3ffaa0e..6a775688d 100644 --- a/doc/context/sources/general/manuals/math/math-tweaks.tex +++ b/doc/context/sources/general/manuals/math/math-tweaks.tex @@ -634,6 +634,104 @@ the loading. \stopchapter +\startchapter[title=Script parameters] + +Here we show the parameters that deal with script placement. This chapter was +added after we found that some values in Latin Modern were off. The tables show a +factor that relates the value to the x-height which is for some part of the +recommendation. + +\starttabulate[|c|l|c|l|] +\NC \NC SubscriptBaselineDropMin \NC .10 \NC harmless, seldom triggered \NC \NR +\NC \star \NC SubscriptShiftDown \NC .40 \NC by inspection in several files \NC \NR +\NC \star \NC SubscriptShiftDownWithSuperscript \NC .40 \NC as above \NC \NR +\NC \NC SubscriptTopMax \NC .80 \NC Microsoft recommendation \NC \NR +\NC \NC SuperscriptBaselineDropMax \NC .10 \NC see Subscript..Min variant \NC \NR +\NC \NC SuperscriptBottomMaxWithSubscript \NC .80 \NC Microsoft recommendation \NC \NR +\NC \NC SuperscriptBottomMin \NC .25 \NC Microsoft recommendation \NC \NR +\NC \star \NC SuperscriptShiftUp \NC .65 \NC by inspection, a gamble \NC \NR +\NC \star \NC SuperscriptShiftUpCramped \NC .65 \NC see above, non-\TEX \NC \NR +\stoptabulate + +In the end only the ones marked with a star are really important. The other ones +are often small and the drop ones between .5 and 1.5 points with (to us) no clear +logic. We decided to not enable a tweak but explicitly check and set the (four) +values in the goodie file. + +\starttexdefinition ShowScriptsValue #1#2#3 + \NC \csstring#2 + \NC \bgroup \switchtobodyfont[#1-nt]\normalexpanded{\egroup\fam0 \cldcontext{"\letterpercent0.3f",\number#2\textstyle/\number\exheight}} + \NC \bgroup \switchtobodyfont[#1-nt]\normalexpanded{\egroup\fam0 \the#2\textstyle} + \NC \bgroup \switchtobodyfont [#1]\normalexpanded{\egroup\fam0 \the#2\textstyle} + \NC \NR +\stoptexdefinition + +\starttexdefinition ShowScripts #1 + \testpage[4] + \starttabulate[|l|r|r|r|] + \FL + \NC \bf #1 + \NC factor + \NC original + \NC tweaked + \NC \NR + \ML + % \ShowScriptsValue{#1}{\Umathaxis} {MathAxis} + \ShowScriptsValue{#1}{\Umathsubshiftdown} {SubscriptShiftDown} + \ShowScriptsValue{#1}{\Umathsubshiftdrop} {SubscriptBaselineDropMin} + \ShowScriptsValue{#1}{\Umathsubsupshiftdown}{SubscriptShiftDownWithSuperscript} + % \ShowScriptsValue{#1}{\Umathsubsupvgap} {} + \ShowScriptsValue{#1}{\Umathsubtopmax} {SubscriptTopMax} + \ShowScriptsValue{#1}{\Umathsupbottommin} {SuperscriptBottomMin} + \ShowScriptsValue{#1}{\Umathsupshiftdrop} {SuperscriptBaselineDropMax} + \ShowScriptsValue{#1}{\Umathsupshiftup} {SuperscriptShiftUp} + \ShowScriptsValue{#1}{\Umathsupsubbottommax}{SuperscriptBaselineDropMax} + \LL + \stoptabulate +\stoptexdefinition + +\ShowSomething{ShowScripts} + +One of the pitfalls is that in \CONTEXT\ we actually set \type +{\Umathsubsupshiftdown} which the results in a bad parameter being used. This +also makes that we do need to fix the font. footnote {Other macro packages are +less likely to suffer from this as long as they don't set that parameter.} These +are issues that we easily spent days on checking, double checking and exploring +variants before settling on some approach. In this case we compared traditional +\PDFTEX\ output using Computer Modern with \OPENTYPE\ rendering using Latin +Modern which exposed the issue. + +\stopchapter + +\startchapter[title=primes] + +A few words on primes, a pain in the butt symbol. In traditional \TEX\ (fonts) a +prime is a special symbol. Macro packages are set up in such a way that single +quotes become primes. In practice that then boils down to putting the symbol in +the superscript which means that at the \TEX\ level the smaller size is used. + +This smaller size is somehow reflected in the \OPENTYPE\ math fonts: the script +and scriptscript sized (ssty variant 1 and 2) are basically meant to be +superscripted. The text size however often looks differently and is positioned +like an accent. Supposedly it is meant to be used without placement, as minute or +second indicator. This is one of the cases where a text, script and scriptscript +variant of the same glyphs looks pretty incompatible. + +Combined with up to four primes being combined as well as up to three reverse +primes being provided this a rather messy implementation. In \CONTEXT\ we always +tweaked the dimensions to consistently suit our purpose. + +When we experimented with primes as operator, implemented like we do with fourier +using right delimited radicals, we realized that hooking this into the text level +symbol gave inconsistent results so in the end we simplified the \type +{fixprimes} tweak in a way that suits both usage patterns: we just use the \type +{ssty}~1 variant (of course scaled to the right relative size) for all three +sizes: there is no need to be more granular and it safes us some trouble. We +thereby also sacrifice some detailed tweaking but also got rid of potential +visual incompatibilities as for instance in \STIX. + +\stopchapter + \startchapter[title=Tweak: dimensions] \ShowTweakUsage{dimensions} Usage of this tweak can best be observed (and experimented) in the goodie files @@ -1414,11 +1512,12 @@ You can substitute a character in a font by one from a substitution feature: \NC #1 \NC \showglyphs \switchtobodyfont[#1] $f'(x)+e^{g'(x)} - a''+ b''' + c''''$ \NC \showglyphs \switchtobodyfont[#1-nt] $f'(x)+e^{g'(x)} - a''+ b''' + c''''$ + \NC \showglyphs \switchtobodyfont[#1] $\primed{f}$ \NC \NR \stoptexdefinition -\starttabulate[|l|c|c|c|c|] - \NC \NC tweaked \NC original \NC \NR +\starttabulate[|l|c|c|c|] + \NC \NC tweaked \NC original \NC primed \NC \NR \ctxlua { for i=1,#document.usedfonts do if i > 1 then @@ -1617,16 +1716,13 @@ hand symbol. } \stoptabulate - - - \stopchapter \startchapter[title=Tweak: addmirrors] \ShowTweakUsage{addmirrors} This adds symbols for right|-|to|-|left math by simply mirroring existing ones. Deep down the engine doesn't really care about in what direction typesetting -happens (wrt spacing and building structures) , and it is the backend that does +happens (wrt spacing and building structures), and it is the backend that does the real work. However, symbols need to be flipped. \stopchapter diff --git a/scripts/context/lua/mtx-fonts.lua b/scripts/context/lua/mtx-fonts.lua index 25be6890a..a7cf817d5 100644 --- a/scripts/context/lua/mtx-fonts.lua +++ b/scripts/context/lua/mtx-fonts.lua @@ -127,7 +127,7 @@ loadmodule("font-otr.lua") loadmodule("font-cff.lua") loadmodule("font-ttf.lua") loadmodule("font-tmp.lua") -loadmodule("font-dsp.lua") +loadmodule("font-dsp.lua") -- autosuffix loadmodule("font-oup.lua") loadmodule("font-otl.lua") diff --git a/source/luametatex/source/libraries/mimalloc/CMakeLists.txt b/source/luametatex/source/libraries/mimalloc/CMakeLists.txt index 8127e0965..6cd826650 100644 --- a/source/luametatex/source/libraries/mimalloc/CMakeLists.txt +++ b/source/luametatex/source/libraries/mimalloc/CMakeLists.txt @@ -6,14 +6,16 @@ set(CMAKE_CXX_STANDARD 17) option(MI_SECURE "Use full security mitigations (like guard pages, allocation randomization, double-free mitigation, and free-list corruption detection)" OFF) option(MI_DEBUG_FULL "Use full internal heap invariant checking in DEBUG mode (expensive)" OFF) -option(MI_PADDING "Enable padding to detect heap block overflow (used only in DEBUG mode)" ON) +option(MI_PADDING "Enable padding to detect heap block overflow (used only in DEBUG mode or with Valgrind)" ON) option(MI_OVERRIDE "Override the standard malloc interface (e.g. define entry points for malloc() etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) +option(MI_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) +option(MI_WIN_REDIRECT "Use redirection module ('mimalloc-redirect') on Windows if compiling mimalloc as a DLL" ON) option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_SHARED "Build shared library" ON) option(MI_BUILD_STATIC "Build static library" ON) @@ -28,6 +30,7 @@ option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version (deprecated)" OFF) option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) +include(CheckIncludeFiles) include(GNUInstallDirs) include("cmake/mimalloc-config-version.cmake") @@ -103,9 +106,37 @@ if(MI_OVERRIDE) endif() endif() +if(WIN32) + if (MI_WIN_REDIRECT) + if (MSVC_C_ARCHITECTURE_ID MATCHES "ARM") + message(STATUS "Cannot use redirection on Windows ARM (MI_WIN_REDIRECT=OFF)") + set(MI_WIN_REDIRECT OFF) + endif() + endif() + if (NOT MI_WIN_REDIRECT) + # use a negative define for backward compatibility + list(APPEND mi_defines MI_WIN_NOREDIRECT=1) + endif() +endif() + if(MI_SECURE) message(STATUS "Set full secure build (MI_SECURE=ON)") list(APPEND mi_defines MI_SECURE=4) + #if (MI_VALGRIND) + # message(WARNING "Secure mode is a bit weakened when compiling with Valgrind support as buffer overflow detection is no longer byte-precise (if running without valgrind)") + #endif() +endif() + +if(MI_VALGRIND) + CHECK_INCLUDE_FILES("valgrind/valgrind.h;valgrind/memcheck.h" MI_HAS_VALGRINDH) + if (NOT MI_HAS_VALGRINDH) + set(MI_VALGRIND OFF) + message(WARNING "Cannot find the 'valgrind/valgrind.h' and 'valgrind/memcheck.h' -- install valgrind first") + message(STATUS "Compile **without** Valgrind support (MI_VALGRIND=OFF)") + else() + message(STATUS "Compile with Valgrind support (MI_VALGRIND=ON)") + list(APPEND mi_defines MI_VALGRIND=1) + endif() endif() if(MI_SEE_ASM) @@ -179,7 +210,7 @@ if(MI_USE_CXX) if(CMAKE_CXX_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wno-deprecated) endif() - if(CMAKE_CXX_COMPILER_ID MATCHES "Intel") + if(CMAKE_CXX_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM") list(APPEND mi_cflags -Kc++) endif() endif() @@ -217,18 +248,26 @@ endif() # extra needed libraries if(WIN32) list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) + set(pc_libraries "-lpsapi -lshell32 -luser32 -ladvapi32 -lbcrypt") else() + set(pc_libraries "") find_library(MI_LIBPTHREAD pthread) if (MI_LIBPTHREAD) list(APPEND mi_libraries ${MI_LIBPTHREAD}) + set(pc_libraries "${pc_libraries} -pthread") endif() find_library(MI_LIBRT rt) if(MI_LIBRT) list(APPEND mi_libraries ${MI_LIBRT}) + set(pc_libraries "${pc_libraries} -lrt") endif() find_library(MI_LIBATOMIC atomic) - if (MI_LIBATOMIC OR MI_USE_LIBATOMIC) - list(APPEND mi_libraries atomic) + if (NOT MI_LIBATOMIC AND MI_USE_LIBATOMIC) + set(MI_LIBATOMIC atomic) + endif() + if (MI_LIBATOMIC) + list(APPEND mi_libraries ${MI_LIBATOMIC}) + set(pc_libraries "${pc_libraries} -latomic") endif() endif() @@ -251,16 +290,18 @@ else() set(mi_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/mimalloc-${mi_version}") # for cmake package info endif() -if(MI_SECURE) - set(mi_basename "mimalloc-secure") -else() set(mi_basename "mimalloc") +if(MI_SECURE) + set(mi_basename "${mi_basename}-secure") +endif() +if(MI_VALGRIND) + set(mi_basename "${mi_basename}-valgrind") endif() - string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$")) set(mi_basename "${mi_basename}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version endif() + if(MI_BUILD_SHARED) list(APPEND mi_build_targets "shared") endif() @@ -304,8 +345,8 @@ if(MI_BUILD_SHARED) $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> $<INSTALL_INTERFACE:${mi_install_incdir}> ) - if(WIN32) - # On windows copy the mimalloc redirection dll too. + if(WIN32 AND MI_WIN_REDIRECT) + # On windows, link and copy the mimalloc redirection dll too. if(CMAKE_SIZEOF_VOID_P EQUAL 4) set(MIMALLOC_REDIRECT_SUFFIX "32") else() @@ -376,6 +417,15 @@ if (MI_BUILD_OBJECT) RENAME ${mi_basename}${CMAKE_C_OUTPUT_EXTENSION} ) endif() +# pkg-config file support +include("cmake/JoinPaths.cmake") +join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") +join_paths(libdir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_LIBDIR}") + +configure_file(mimalloc.pc.in mimalloc.pc @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/mimalloc.pc" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/") + # ----------------------------------------------------------------------------- # API surface testing # ----------------------------------------------------------------------------- diff --git a/source/luametatex/source/libraries/mimalloc/cmake/JoinPaths.cmake b/source/luametatex/source/libraries/mimalloc/cmake/JoinPaths.cmake new file mode 100644 index 000000000..c68d91b84 --- /dev/null +++ b/source/luametatex/source/libraries/mimalloc/cmake/JoinPaths.cmake @@ -0,0 +1,23 @@ +# This module provides function for joining paths +# known from most languages +# +# SPDX-License-Identifier: (MIT OR CC0-1.0) +# Copyright 2020 Jan Tojnar +# https://github.com/jtojnar/cmake-snips +# +# Modelled after Python’s os.path.join +# https://docs.python.org/3.7/library/os.path.html#os.path.join +# Windows not supported +function(join_paths joined_path first_path_segment) + set(temp_path "${first_path_segment}") + foreach(current_segment IN LISTS ARGN) + if(NOT ("${current_segment}" STREQUAL "")) + if(IS_ABSOLUTE "${current_segment}") + set(temp_path "${current_segment}") + else() + set(temp_path "${temp_path}/${current_segment}") + endif() + endif() + endforeach() + set(${joined_path} "${temp_path}" PARENT_SCOPE) +endfunction() diff --git a/source/luametatex/source/libraries/mimalloc/cmake/mimalloc-config-version.cmake b/source/luametatex/source/libraries/mimalloc/cmake/mimalloc-config-version.cmake index 8063afe6b..f0669c84d 100644 --- a/source/luametatex/source/libraries/mimalloc/cmake/mimalloc-config-version.cmake +++ b/source/luametatex/source/libraries/mimalloc/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 2) set(mi_version_minor 0) -set(mi_version_patch 6) +set(mi_version_patch 7) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/source/luametatex/source/libraries/mimalloc/include/mimalloc-internal.h b/source/luametatex/source/libraries/mimalloc/include/mimalloc-internal.h index d691eca58..550b65433 100644 --- a/source/luametatex/source/libraries/mimalloc/include/mimalloc-internal.h +++ b/source/luametatex/source/libraries/mimalloc/include/mimalloc-internal.h @@ -9,6 +9,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MIMALLOC_INTERNAL_H #include "mimalloc-types.h" +#include "mimalloc-track.h" #if (MI_DEBUG>0) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) @@ -88,12 +89,14 @@ size_t _mi_os_good_alloc_size(size_t size); bool _mi_os_has_overcommit(void); // arena.c -void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); -void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); +void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); +void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); void _mi_arena_free(void* p, size_t size, size_t memid, bool is_committed, mi_os_tld_t* tld); +mi_arena_id_t _mi_arena_id_none(void); +bool _mi_arena_memid_is_suitable(size_t memid, mi_arena_id_t req_arena_id); // "segment-cache.c" -void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); +void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld); void _mi_segment_cache_collect(bool force, mi_os_tld_t* tld); void _mi_segment_map_allocated_at(const mi_segment_t* segment); @@ -115,16 +118,18 @@ void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* // "page.c" -void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... -void _mi_heap_delayed_free(mi_heap_t* heap); +void _mi_heap_delayed_free_all(mi_heap_t* heap); +bool _mi_heap_delayed_free_partial(mi_heap_t* heap); void _mi_heap_collect_retired(mi_heap_t* heap, bool force); void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); +bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append); void _mi_deferred_free(mi_heap_t* heap, bool force); @@ -138,6 +143,7 @@ uint8_t _mi_bin(size_t size); // for stats void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); void _mi_heap_set_default_direct(mi_heap_t* heap); +bool _mi_heap_memid_is_suitable(mi_heap_t* heap, size_t memid); // "stats.c" void _mi_stats_done(mi_stats_t* stats); @@ -147,12 +153,11 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start); mi_msecs_t _mi_clock_start(void); // "alloc.c" -void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_malloc_generic` +void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_malloc_generic` void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); bool _mi_free_delayed_block(mi_block_t* block); -void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size); #if MI_DEBUG>1 bool _mi_page_is_valid(mi_page_t* page); @@ -164,8 +169,11 @@ bool _mi_page_is_valid(mi_page_t* page); // ------------------------------------------------------ #if defined(__GNUC__) || defined(__clang__) -#define mi_unlikely(x) __builtin_expect(!!(x),false) -#define mi_likely(x) __builtin_expect(!!(x),true) +#define mi_unlikely(x) (__builtin_expect(!!(x),false)) +#define mi_likely(x) (__builtin_expect(!!(x),true)) +#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) +#define mi_unlikely(x) (x) [[unlikely]] +#define mi_likely(x) (x) [[likely]] #else #define mi_unlikely(x) (x) #define mi_likely(x) (x) @@ -224,6 +232,12 @@ static inline bool _mi_is_power_of_two(uintptr_t x) { return ((x & (x - 1)) == 0); } +// Is a pointer aligned? +static inline bool _mi_is_aligned(void* p, size_t alignment) { + mi_assert_internal(alignment != 0); + return (((uintptr_t)p % alignment) == 0); +} + // Align upwards static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { mi_assert_internal(alignment != 0); @@ -289,8 +303,8 @@ static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) *total = count * size; - return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) - && size > 0 && (SIZE_MAX / size) < count); + // note: gcc/clang optimize this to directly check the overflow flag + return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); } #endif @@ -300,8 +314,10 @@ static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* tot *total = size; return false; } - else if (mi_unlikely(mi_mul_overflow(count, size, total))) { + else if mi_unlikely(mi_mul_overflow(count, size, total)) { + #if MI_DEBUG > 0 _mi_error_message(EOVERFLOW, "allocation request is too large (%zu * %zu bytes)\n", count, size); + #endif *total = SIZE_MAX; return true; } @@ -372,7 +388,7 @@ extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate static inline mi_heap_t* mi_get_default_heap(void) { #if defined(MI_TLS_SLOT) mi_heap_t* heap = (mi_heap_t*)mi_tls_slot(MI_TLS_SLOT); - if (mi_unlikely(heap == NULL)) { + if mi_unlikely(heap == NULL) { #ifdef __GNUC__ __asm(""); // prevent conditional load of the address of _mi_heap_empty #endif @@ -486,7 +502,7 @@ static inline mi_page_t* _mi_ptr_page(void* p) { static inline size_t mi_page_block_size(const mi_page_t* page) { const size_t bsize = page->xblock_size; mi_assert_internal(bsize > 0); - if (mi_likely(bsize < MI_HUGE_BLOCK_SIZE)) { + if mi_likely(bsize < MI_HUGE_BLOCK_SIZE) { return bsize; } else { @@ -649,30 +665,36 @@ static inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) { static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]); - return (mi_unlikely(p==null) ? NULL : p); + return (p==null ? NULL : p); } static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { - uintptr_t x = (uintptr_t)(mi_unlikely(p==NULL) ? null : p); + uintptr_t x = (uintptr_t)(p==NULL ? null : p); return mi_rotl(x ^ keys[1], keys[0]) + keys[0]; } static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { + mi_track_mem_defined(block,sizeof(mi_block_t)); + mi_block_t* next; #ifdef MI_ENCODE_FREELIST - return (mi_block_t*)mi_ptr_decode(null, block->next, keys); + next = (mi_block_t*)mi_ptr_decode(null, block->next, keys); #else MI_UNUSED(keys); MI_UNUSED(null); - return (mi_block_t*)block->next; + next = (mi_block_t*)block->next; #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); + return next; } static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { + mi_track_mem_undefined(block,sizeof(mi_block_t)); #ifdef MI_ENCODE_FREELIST block->next = mi_ptr_encode(null, next, keys); #else MI_UNUSED(keys); MI_UNUSED(null); block->next = (mi_encoded_t)next; #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); } static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { @@ -680,7 +702,7 @@ static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* mi_block_t* next = mi_block_nextx(page,block,page->keys); // check for free list corruption: is `next` at least in the same page? // TODO: check if `next` is `page->block_size` aligned? - if (mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next))) { + if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) { _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); next = NULL; } @@ -779,12 +801,12 @@ size_t _mi_os_numa_node_count_get(void); extern _Atomic(size_t) _mi_numa_node_count; static inline int _mi_os_numa_node(mi_os_tld_t* tld) { - if (mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1)) return 0; + if mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1) { return 0; } else return _mi_os_numa_node_get(tld); } static inline size_t _mi_os_numa_node_count(void) { const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count); - if (mi_likely(count>0)) return count; + if mi_likely(count > 0) { return count; } else return _mi_os_numa_node_count_get(); } @@ -1003,7 +1025,7 @@ static inline size_t mi_bsr(uintptr_t x) { // (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253. // --------------------------------------------------------------------------------- -#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) +#if !MI_TRACK_ENABLED && defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) #include <intrin.h> #include <string.h> extern bool _mi_cpu_has_fsrm; @@ -1012,7 +1034,15 @@ static inline void _mi_memcpy(void* dst, const void* src, size_t n) { __movsb((unsigned char*)dst, (const unsigned char*)src, n); } else { - memcpy(dst, src, n); // todo: use noinline? + memcpy(dst, src, n); + } +} +static inline void _mi_memzero(void* dst, size_t n) { + if (_mi_cpu_has_fsrm) { + __stosb((unsigned char*)dst, 0, n); + } + else { + memset(dst, 0, n); } } #else @@ -1020,6 +1050,9 @@ static inline void _mi_memcpy(void* dst, const void* src, size_t n) { static inline void _mi_memcpy(void* dst, const void* src, size_t n) { memcpy(dst, src, n); } +static inline void _mi_memzero(void* dst, size_t n) { + memset(dst, 0, n); +} #endif @@ -1037,12 +1070,23 @@ static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); _mi_memcpy(adst, asrc, n); } + +static inline void _mi_memzero_aligned(void* dst, size_t n) { + mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); + void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); + _mi_memzero(adst, n); +} #else // Default fallback on `_mi_memcpy` static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); _mi_memcpy(dst, src, n); } + +static inline void _mi_memzero_aligned(void* dst, size_t n) { + mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); + _mi_memzero(dst, n); +} #endif diff --git a/source/luametatex/source/libraries/mimalloc/include/mimalloc-new-delete.h b/source/luametatex/source/libraries/mimalloc/include/mimalloc-new-delete.h index 2749a0be9..1c12fad2f 100644 --- a/source/luametatex/source/libraries/mimalloc/include/mimalloc-new-delete.h +++ b/source/luametatex/source/libraries/mimalloc/include/mimalloc-new-delete.h @@ -44,8 +44,8 @@ terms of the MIT license. A copy of the license can be found in the file void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); } void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); }; void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); }; - void operator delete (void* p, std::align_val_t al, const std::nothrow_t& tag) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); } - void operator delete[](void* p, std::align_val_t al, const std::nothrow_t& tag) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); } + void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); } + void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); } void* operator new (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); } void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); } diff --git a/source/luametatex/source/libraries/mimalloc/include/mimalloc-track.h b/source/luametatex/source/libraries/mimalloc/include/mimalloc-track.h new file mode 100644 index 000000000..bb9df4fa3 --- /dev/null +++ b/source/luametatex/source/libraries/mimalloc/include/mimalloc-track.h @@ -0,0 +1,43 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_TRACK_H +#define MIMALLOC_TRACK_H + +// ------------------------------------------------------ +// Track memory ranges with macros for tools like Valgrind +// or other memory checkers. +// ------------------------------------------------------ + +#if MI_VALGRIND + +#define MI_TRACK_ENABLED 1 + +#include <valgrind/valgrind.h> +#include <valgrind/memcheck.h> + +#define mi_track_malloc(p,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero) +#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/) +#define mi_track_free(p) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/) +#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size) +#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) +#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) + +#else + +#define MI_TRACK_ENABLED 0 + +#define mi_track_malloc(p,size,zero) +#define mi_track_resize(p,oldsize,newsize) +#define mi_track_free(p) +#define mi_track_mem_defined(p,size) +#define mi_track_mem_undefined(p,size) +#define mi_track_mem_noaccess(p,size) + +#endif + +#endif diff --git a/source/luametatex/source/libraries/mimalloc/include/mimalloc-types.h b/source/luametatex/source/libraries/mimalloc/include/mimalloc-types.h index fb75ea464..1387a7200 100644 --- a/source/luametatex/source/libraries/mimalloc/include/mimalloc-types.h +++ b/source/luametatex/source/libraries/mimalloc/include/mimalloc-types.h @@ -29,6 +29,9 @@ terms of the MIT license. A copy of the license can be found in the file // Define NDEBUG in the release version to disable assertions. // #define NDEBUG +// Define MI_VALGRIND to enable valgrind support +// #define MI_VALGRIND 1 + // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). // #define MI_STAT 1 @@ -56,14 +59,14 @@ terms of the MIT license. A copy of the license can be found in the file // Reserve extra padding at the end of each block to be more resilient against heap block overflows. // The padding can detect byte-precise buffer overflow on free. -#if !defined(MI_PADDING) && (MI_DEBUG>=1) +#if !defined(MI_PADDING) && (MI_DEBUG>=1 || MI_VALGRIND) #define MI_PADDING 1 #endif // Encoded free lists allow detection of corrupted free lists // and can detect buffer overflows, modify after free, and double `free`s. -#if (MI_SECURE>=3 || MI_DEBUG>=1 || MI_PADDING > 0) +#if (MI_SECURE>=3 || MI_DEBUG>=1) #define MI_ENCODE_FREELIST 1 #endif @@ -435,6 +438,7 @@ struct mi_heap_s { mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") _Atomic(mi_block_t*) thread_delayed_free; mi_threadid_t thread_id; // thread this heap belongs too + mi_arena_id_t arena_id; // arena id if the heap belongs to a specific arena (or 0) uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list mi_random_ctx_t random; // random number context used for secure allocation diff --git a/source/luametatex/source/libraries/mimalloc/include/mimalloc.h b/source/luametatex/source/libraries/mimalloc/include/mimalloc.h index c752ac247..32eab19ea 100644 --- a/source/luametatex/source/libraries/mimalloc/include/mimalloc.h +++ b/source/luametatex/source/libraries/mimalloc/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 206 // major + 2 digits minor +#define MI_MALLOC_VERSION 207 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes @@ -95,6 +95,7 @@ terms of the MIT license. A copy of the license can be found in the file #include <stddef.h> // size_t #include <stdbool.h> // bool +#include <stdint.h> // INTPTR_MAX #ifdef __cplusplus extern "C" { @@ -166,7 +167,11 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s // Note that `alignment` always follows `size` for consistency with unaligned // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. // ------------------------------------------------------------------------------------- -#define MI_ALIGNMENT_MAX (1024*1024UL) // maximum supported alignment is 1MiB +#if (INTPTR_MAX > INT32_MAX) +#define MI_ALIGNMENT_MAX (16*1024*1024UL) // maximum supported alignment is 16MiB +#else +#define MI_ALIGNMENT_MAX (1024*1024UL) // maximum supported alignment for 32-bit systems is 1MiB +#endif mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); @@ -275,6 +280,17 @@ mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_commit mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept; +// Experimental: heaps associated with specific memory arena's +typedef int mi_arena_id_t; +mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); +mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; +mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; +mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; + +#if MI_MALLOC_VERSION >= 200 +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id); +#endif + // deprecated mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; diff --git a/source/luametatex/source/libraries/mimalloc/readme.md b/source/luametatex/source/libraries/mimalloc/readme.md index 6142dbc5e..588630992 100644 --- a/source/luametatex/source/libraries/mimalloc/readme.md +++ b/source/luametatex/source/libraries/mimalloc/readme.md @@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the run-time systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release tag: `v2.0.6` (2022-04-14). -Latest stable tag: `v1.7.6` (2022-02-14). +Latest release tag: `v2.0.7` (2022-11-03). +Latest stable tag: `v1.7.7` (2022-11-03). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -73,10 +73,13 @@ Enjoy! ### Releases -Note: the `v2.x` version has a new algorithm for managing internal mimalloc pages that tends to use reduce memory usage +Note: the `v2.x` version has a new algorithm for managing internal mimalloc pages that tends to reduce memory usage and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance (see [below](#performance)); please report if you observe any significant performance regression. +* 2022-11-03, `v1.7.7`, `v2.0.7`: Initial support for [Valgrind](#valgrind) for leak testing and heap block overflow detection. Initial + support for attaching heaps to a speficic memory area (only in v2). Fix `realloc` behavior for zero size blocks, remove restriction to integral multiple of the alignment in `alloc_align`, improved aligned allocation performance, reduced contention with many threads on few processors (thank you @dposluns!), vs2022 support, support `pkg-config`, . + * 2022-04-14, `v1.7.6`, `v2.0.6`: fix fallback path for aligned OS allocation on Windows, improve Windows aligned allocation even when compiling with older SDK's, fix dynamic overriding on macOS Monterey, fix MSVC C++ dynamic overriding, fix warnings under Clang 14, improve performance if many OS threads are created and destroyed, fix statistics for large object @@ -337,6 +340,44 @@ When _mimalloc_ is built using debug mode, various checks are done at runtime to - Double free's, and freeing invalid heap pointers are detected. - Corrupted free-lists and some forms of use-after-free are detected. +## Valgrind + +Generally, we recommend using the standard allocator with the amazing [Valgrind] tool (and +also for other address sanitizers). +However, it is possible to build mimalloc with Valgrind support. This has a small performance +overhead but does allow detecting memory leaks and byte-precise buffer overflows directly on final +executables. To build with valgrind support, use the `MI_VALGRIND=ON` cmake option: + +``` +> cmake ../.. -DMI_VALGRIND=ON +``` + +This can also be combined with secure mode or debug mode. +You can then run your programs directly under valgrind: + +``` +> valgrind <myprogram> +``` + +If you rely on overriding `malloc`/`free` by mimalloc (instead of using the `mi_malloc`/`mi_free` API directly), +you also need to tell `valgrind` to not intercept those calls itself, and use: + +``` +> MIMALLOC_SHOW_STATS=1 valgrind --soname-synonyms=somalloc=*mimalloc* -- <myprogram> +``` + +By setting the `MIMALLOC_SHOW_STATS` environment variable you can check that mimalloc is indeed +used and not the standard allocator. Even though the [Valgrind option][valgrind-soname] +is called `--soname-synonyms`, this also +works when overriding with a static library or object file. Unfortunately, it is not possible to +dynamically override mimalloc using `LD_PRELOAD` together with `valgrind`. +See also the `test/test-wrong.c` file to test with `valgrind`. + +Valgrind support is in its initial development -- please report any issues. + +[Valgrind]: https://valgrind.org/ +[valgrind-soname]: https://valgrind.org/docs/manual/manual-core.html#opt.soname-synonyms + # Overriding Standard Malloc diff --git a/source/luametatex/source/libraries/mimalloc/src/alloc-aligned.c b/source/luametatex/source/libraries/mimalloc/src/alloc-aligned.c index fce0fd749..9614aa092 100644 --- a/source/luametatex/source/libraries/mimalloc/src/alloc-aligned.c +++ b/source/luametatex/source/libraries/mimalloc/src/alloc-aligned.c @@ -31,7 +31,8 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* } // otherwise over-allocate - void* p = _mi_heap_malloc_zero(heap, size + alignment - 1, zero); + const size_t oversize = size + alignment - 1; + void* p = _mi_heap_malloc_zero(heap, oversize, zero); if (p == NULL) return NULL; // .. and align within the allocation @@ -41,6 +42,16 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true); mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p), _mi_ptr_page(aligned_p), aligned_p)); + + #if MI_TRACK_ENABLED + if (p != aligned_p) { + mi_track_free(p); + mi_track_malloc(aligned_p,size,zero); + } + else { + mi_track_resize(aligned_p,oversize,size); + } + #endif return aligned_p; } @@ -49,19 +60,19 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t { // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. mi_assert(alignment > 0); - if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>) + if mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>) #if MI_DEBUG > 0 _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment); #endif return NULL; } - if (mi_unlikely(alignment > MI_ALIGNMENT_MAX)) { // we cannot align at a boundary larger than this (or otherwise we cannot find segment headers) + if mi_unlikely(alignment > MI_ALIGNMENT_MAX) { // we cannot align at a boundary larger than this (or otherwise we cannot find segment headers) #if MI_DEBUG > 0 _mi_error_message(EOVERFLOW, "aligned allocation has a maximum alignment of %zu (size %zu, alignment %zu)\n", MI_ALIGNMENT_MAX, size, alignment); #endif return NULL; } - if (mi_unlikely(size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>) + if mi_unlikely(size > PTRDIFF_MAX) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>) #if MI_DEBUG > 0 _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); #endif @@ -71,18 +82,18 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t const size_t padsize = size + MI_PADDING_SIZE; // note: cannot overflow due to earlier size > PTRDIFF_MAX check // try first if there happens to be a small block available with just the right alignment - if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) { + if mi_likely(padsize <= MI_SMALL_SIZE_MAX) { mi_page_t* page = _mi_heap_get_free_small_page(heap, padsize); const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0; - if (mi_likely(page->free != NULL && is_aligned)) + if mi_likely(page->free != NULL && is_aligned) { #if MI_STAT>1 mi_heap_stat_increase(heap, malloc, size); #endif - void* p = _mi_page_malloc(heap, page, padsize); // TODO: inline _mi_page_malloc + void* p = _mi_page_malloc(heap, page, padsize, zero); // TODO: inline _mi_page_malloc mi_assert_internal(p != NULL); mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); - if (zero) { _mi_block_zero_init(page, p, size); } + mi_track_malloc(p,size,zero); return p; } } @@ -95,19 +106,19 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t // Optimized mi_heap_malloc_aligned / mi_malloc_aligned // ------------------------------------------------------ -mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false); } -mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { #if !MI_PADDING // without padding, any small sized allocation is naturally aligned (see also `_mi_segment_page_start`) if (!_mi_is_power_of_two(alignment)) return NULL; - if (mi_likely(_mi_is_power_of_two(size) && size >= alignment && size <= MI_SMALL_SIZE_MAX)) + if mi_likely(_mi_is_power_of_two(size) && size >= alignment && size <= MI_SMALL_SIZE_MAX) #else // with padding, we can only guarantee this for fixed alignments - if (mi_likely((alignment == sizeof(void*) || (alignment == MI_MAX_ALIGN_SIZE && size > (MI_MAX_ALIGN_SIZE/2))) - && size <= MI_SMALL_SIZE_MAX)) + if mi_likely((alignment == sizeof(void*) || (alignment == MI_MAX_ALIGN_SIZE && size > (MI_MAX_ALIGN_SIZE/2))) + && size <= MI_SMALL_SIZE_MAX) #endif { // fast path for common alignment and size @@ -122,45 +133,45 @@ mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size // Aligned Allocation // ------------------------------------------------------ -mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true); } -mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_zalloc_aligned_at(heap, size, alignment, 0); } -mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; return mi_heap_zalloc_aligned_at(heap, total, alignment, offset); } -mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_calloc_aligned_at(heap,count,size,alignment,0); } -mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_malloc_aligned_at(mi_get_default_heap(), size, alignment, offset); } -mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_malloc_aligned(mi_get_default_heap(), size, alignment); } -mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_zalloc_aligned_at(mi_get_default_heap(), size, alignment, offset); } -mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_zalloc_aligned(mi_get_default_heap(), size, alignment); } -mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_calloc_aligned_at(mi_get_default_heap(), count, size, alignment, offset); } -mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_calloc_aligned(mi_get_default_heap(), count, size, alignment); } @@ -207,55 +218,55 @@ static void* mi_heap_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsi return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero); } -void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false); } -void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { return mi_heap_realloc_zero_aligned(heap,p,newsize,alignment,false); } -void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true); } -void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { return mi_heap_realloc_zero_aligned(heap, p, newsize, alignment, true); } -void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(newcount, size, &total)) return NULL; return mi_heap_rezalloc_aligned_at(heap, p, total, alignment, offset); } -void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(newcount, size, &total)) return NULL; return mi_heap_rezalloc_aligned(heap, p, total, alignment); } -void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_realloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset); } -void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { return mi_heap_realloc_aligned(mi_get_default_heap(), p, newsize, alignment); } -void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_rezalloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset); } -void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { return mi_heap_rezalloc_aligned(mi_get_default_heap(), p, newsize, alignment); } -void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_recalloc_aligned_at(mi_get_default_heap(), p, newcount, size, alignment, offset); } -void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_recalloc_aligned(mi_get_default_heap(), p, newcount, size, alignment); } diff --git a/source/luametatex/source/libraries/mimalloc/src/alloc-override-osx.c b/source/luametatex/source/libraries/mimalloc/src/alloc-override-osx.c index 41d0a386e..ba2313a2a 100644 --- a/source/luametatex/source/libraries/mimalloc/src/alloc-override-osx.c +++ b/source/luametatex/source/libraries/mimalloc/src/alloc-override-osx.c @@ -254,7 +254,7 @@ static malloc_zone_t mi_malloc_zone = { static inline malloc_zone_t* mi_get_default_zone(void) { static bool init; - if (mi_unlikely(!init)) { + if mi_unlikely(!init) { init = true; malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see <http://eatmyrandom.blogspot.com/2010/03/mallocfree-interception-on-mac-os-x.html>) } diff --git a/source/luametatex/source/libraries/mimalloc/src/alloc-override.c b/source/luametatex/source/libraries/mimalloc/src/alloc-override.c index e29cb4b23..6b9845d39 100644 --- a/source/luametatex/source/libraries/mimalloc/src/alloc-override.c +++ b/source/luametatex/source/libraries/mimalloc/src/alloc-override.c @@ -29,7 +29,7 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; // Override system malloc // ------------------------------------------------------ -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !defined(MI_VALGRIND) +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !MI_TRACK_ENABLED // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward @@ -123,10 +123,10 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; // we just override new/delete which does work in a static library. #else // On all other systems forward to our API - void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) - void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n) - void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize) - void free(void* p) MI_FORWARD0(mi_free, p) + mi_decl_export void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) + mi_decl_export void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n) + mi_decl_export void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize) + mi_decl_export void free(void* p) MI_FORWARD0(mi_free, p) #endif #if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) diff --git a/source/luametatex/source/libraries/mimalloc/src/alloc-posix.c b/source/luametatex/source/libraries/mimalloc/src/alloc-posix.c index 176e7ec30..214a97345 100644 --- a/source/luametatex/source/libraries/mimalloc/src/alloc-posix.c +++ b/source/luametatex/source/libraries/mimalloc/src/alloc-posix.c @@ -83,13 +83,16 @@ mi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcep } mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept { - if (mi_unlikely((size&(alignment-1)) != 0)) { // C11 requires alignment>0 && integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc> + // C11 requires the size to be an integral multiple of the alignment, see <https://en.cppreference.com/w/c/memory/aligned_alloc>. + // unfortunately, it turns out quite some programs pass a size that is not an integral multiple so skip this check.. + /* if mi_unlikely((size & (alignment - 1)) != 0) { // C11 requires alignment>0 && integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc> #if MI_DEBUG > 0 _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment); #endif return NULL; } - // C11 also requires alignment to be a power-of-two which is checked in mi_malloc_aligned + */ + // C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned void* p = mi_malloc_aligned(size, alignment); mi_assert_internal(((uintptr_t)p % alignment) == 0); return p; @@ -109,7 +112,7 @@ mi_decl_nodiscard int mi_reallocarr( void* p, size_t count, size_t size ) mi_att } void** op = (void**)p; void* newp = mi_reallocarray(*op, count, size); - if (mi_unlikely(newp == NULL)) return errno; + if mi_unlikely(newp == NULL) { return errno; } *op = newp; return 0; } diff --git a/source/luametatex/source/libraries/mimalloc/src/alloc.c b/source/luametatex/source/libraries/mimalloc/src/alloc.c index 1a36b5da8..348218246 100644 --- a/source/luametatex/source/libraries/mimalloc/src/alloc.c +++ b/source/luametatex/source/libraries/mimalloc/src/alloc.c @@ -12,6 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-internal.h" #include "mimalloc-atomic.h" + #include <string.h> // memset, strlen #include <stdlib.h> // malloc, exit @@ -25,11 +26,11 @@ terms of the MIT license. A copy of the license can be found in the file // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. -extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept { +extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size); mi_block_t* const block = page->free; - if (mi_unlikely(block == NULL)) { - return _mi_malloc_generic(heap, size); + if mi_unlikely(block == NULL) { + return _mi_malloc_generic(heap, size, zero); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); // pop from the free list @@ -37,10 +38,22 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz page->free = mi_block_next(page, block); mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); -#if (MI_DEBUG>0) - if (!page->is_zero) { memset(block, MI_DEBUG_UNINIT, size); } + // allow use of the block internally + // note: when tracking we need to avoid ever touching the MI_PADDING since + // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc-track.h`) + mi_track_mem_undefined(block, mi_page_usable_block_size(page)); + + // zero the block? note: we need to zero the full block size (issue #63) + if mi_unlikely(zero) { + mi_assert_internal(page->xblock_size != 0); // do not call with zero'ing for huge blocks (see _mi_malloc_generic) + const size_t zsize = (page->is_zero ? sizeof(block->next) + MI_PADDING_SIZE : page->xblock_size); + _mi_memzero_aligned(block, zsize - MI_PADDING_SIZE); + } + +#if (MI_DEBUG>0) && !MI_TRACK_ENABLED + if (!page->is_zero && !zero) { memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); } #elif (MI_SECURE!=0) - block->next = 0; // don't leak internal data + if (!zero) { block->next = 0; } // don't leak internal data #endif #if (MI_STAT>0) @@ -55,10 +68,13 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz } #endif -#if (MI_PADDING > 0) && defined(MI_ENCODE_FREELIST) +#if (MI_PADDING > 0) && defined(MI_ENCODE_FREELIST) && !MI_TRACK_ENABLED mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page)); ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); + #if (MI_DEBUG>1) mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta)); + mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess + #endif padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys)); padding->delta = (uint32_t)(delta); uint8_t* fill = (uint8_t*)padding - delta; @@ -69,8 +85,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz return block; } -// allocate a small block -extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { +static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local mi_assert(size <= MI_SMALL_SIZE_MAX); @@ -80,7 +95,7 @@ extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_ } #endif mi_page_t* page = _mi_heap_get_free_small_page(heap,size + MI_PADDING_SIZE); - void* p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE); + void* p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE, zero); mi_assert_internal(p==NULL || mi_usable_size(p) >= size); #if MI_STAT>1 if (p != NULL) { @@ -88,22 +103,28 @@ extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_ mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); } #endif + mi_track_malloc(p,size,zero); return p; } -extern inline mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { +// allocate a small block +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_heap_malloc_small_zero(heap, size, false); +} + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { return mi_heap_malloc_small(mi_get_default_heap(), size); } // The main allocation function -extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - if (mi_likely(size <= MI_SMALL_SIZE_MAX)) { - return mi_heap_malloc_small(heap, size); +extern inline void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { + if mi_likely(size <= MI_SMALL_SIZE_MAX) { + return mi_heap_malloc_small_zero(heap, size, zero); } else { mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local - void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE); // note: size can overflow but it is detected in malloc_generic + void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero); // note: size can overflow but it is detected in malloc_generic mi_assert_internal(p == NULL || mi_usable_size(p) >= size); #if MI_STAT>1 if (p != NULL) { @@ -111,55 +132,29 @@ extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); } #endif + mi_track_malloc(p,size,zero); return p; } } -extern inline mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { - return mi_heap_malloc(mi_get_default_heap(), size); +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return _mi_heap_malloc_zero(heap, size, false); } - -void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size) { - // note: we need to initialize the whole usable block size to zero, not just the requested size, - // or the recalloc/rezalloc functions cannot safely expand in place (see issue #63) - MI_UNUSED(size); - mi_assert_internal(p != NULL); - mi_assert_internal(mi_usable_size(p) >= size); // size can be zero - mi_assert_internal(_mi_ptr_page(p)==page); - if (page->is_zero && size > sizeof(mi_block_t)) { - // already zero initialized memory - ((mi_block_t*)p)->next = 0; // clear the free list pointer - mi_assert_expensive(mi_mem_is_zero(p, mi_usable_size(p))); - } - else { - // otherwise memset - memset(p, 0, mi_usable_size(p)); - } +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { + return mi_heap_malloc(mi_get_default_heap(), size); } // zero initialized small block -mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { - void* p = mi_malloc_small(size); - if (p != NULL) { - _mi_block_zero_init(_mi_ptr_page(p), p, size); // todo: can we avoid getting the page again? - } - return p; -} - -void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { - void* p = mi_heap_malloc(heap,size); - if (zero && p != NULL) { - _mi_block_zero_init(_mi_ptr_page(p),p,size); // todo: can we avoid getting the page again? - } - return p; +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { + return mi_heap_malloc_small_zero(mi_get_default_heap(), size, true); } -extern inline mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { return _mi_heap_malloc_zero(heap, size, true); } -mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { return mi_heap_zalloc(mi_get_default_heap(),size); } @@ -192,16 +187,19 @@ static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, con return false; } +#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); } + static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { + bool is_double_free = false; mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL? { // Suspicous: decoded value a in block is in the same page (or NULL) -- maybe a double free? // (continue in separate function to improve code generation) - return mi_check_is_double_freex(page, block); + is_double_free = mi_check_is_double_freex(page, block); } - return false; + return is_double_free; } #else static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { @@ -215,12 +213,19 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block // Check for heap block overflow by setting up padding at the end of the block // --------------------------------------------------------------------------- -#if (MI_PADDING>0) && defined(MI_ENCODE_FREELIST) +#if (MI_PADDING>0) && defined(MI_ENCODE_FREELIST) && !MI_TRACK_ENABLED static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) { *bsize = mi_page_usable_block_size(page); const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); + mi_track_mem_defined(padding,sizeof(mi_padding_t)); *delta = padding->delta; - return ((uint32_t)mi_ptr_encode(page,block,page->keys) == padding->canary && *delta <= *bsize); + uint32_t canary = padding->canary; + uintptr_t keys[2]; + keys[0] = page->keys[0]; + keys[1] = page->keys[1]; + bool ok = ((uint32_t)mi_ptr_encode(page,block,keys) == canary && *delta <= *bsize); + mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); + return ok; } // Return the exact usable size of a block. @@ -242,13 +247,16 @@ static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, si *size = bsize - delta; uint8_t* fill = (uint8_t*)block + bsize - delta; const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes + mi_track_mem_defined(fill,maxpad); for (size_t i = 0; i < maxpad; i++) { if (fill[i] != MI_DEBUG_PADDING) { *wrong = bsize - delta + i; - return false; + ok = false; + break; } } - return true; + mi_track_mem_noaccess(fill,maxpad); + return ok; } static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { @@ -347,14 +355,14 @@ static void mi_stat_huge_free(const mi_page_t* page) { // Free // ------------------------------------------------------ -// multi-threaded free +// multi-threaded free (or free in huge block) static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block) { // The padding check may access the non-thread-owned page for the key values. // that is safe as these are constant and the page won't be freed (as the block is not freed yet). mi_check_padding(page, block); mi_padding_shrink(page, block, sizeof(mi_block_t)); // for small size, ensure we can fit the delayed thread pointers without triggering overflow detection - #if (MI_DEBUG!=0) + #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED // note: when tracking, cannot use mi_usable_size with multi-threading memset(block, MI_DEBUG_FREED, mi_usable_size(block)); #endif @@ -372,7 +380,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free); do { use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE); - if (mi_unlikely(use_delayed)) { + if mi_unlikely(use_delayed) { // unlikely: this only happens on the first concurrent free in a page that is in the full list tfreex = mi_tf_set_delayed(tfree,MI_DELAYED_FREEING); } @@ -383,7 +391,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc } } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex)); - if (mi_unlikely(use_delayed)) { + if mi_unlikely(use_delayed) { // racy read on `heap`, but ok because MI_DELAYED_FREEING is set (see `mi_heap_delete` and `mi_heap_collect_abandon`) mi_heap_t* const heap = (mi_heap_t*)(mi_atomic_load_acquire(&page->xheap)); //mi_page_heap(page); mi_assert_internal(heap != NULL); @@ -409,20 +417,21 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block) { // and push it on the free list - if (mi_likely(local)) { + //const size_t bsize = mi_page_block_size(page); + if mi_likely(local) { // owning thread can free a block directly - if (mi_unlikely(mi_check_is_double_free(page, block))) return; + if mi_unlikely(mi_check_is_double_free(page, block)) return; mi_check_padding(page, block); - #if (MI_DEBUG!=0) + #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif mi_block_set_next(page, block, page->local_free); page->local_free = block; page->used--; - if (mi_unlikely(mi_page_all_free(page))) { + if mi_unlikely(mi_page_all_free(page)) { _mi_page_retire(page); } - else if (mi_unlikely(mi_page_is_in_full(page))) { + else if mi_unlikely(mi_page_is_in_full(page)) { _mi_page_unfull(page); } } @@ -444,7 +453,8 @@ mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* p static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, bool local, void* p) mi_attr_noexcept { mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p); - mi_stat_free(page, block); + mi_stat_free(page, block); // stat_free may access the padding + mi_track_free(p); _mi_free_block(page, local, block); } @@ -455,26 +465,26 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms { MI_UNUSED(msg); #if (MI_DEBUG>0) - if (mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0)) { + if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); return NULL; } #endif mi_segment_t* const segment = _mi_ptr_segment(p); - if (mi_unlikely(segment == NULL)) return NULL; // checks also for (p==NULL) + if mi_unlikely(segment == NULL) return NULL; // checks also for (p==NULL) #if (MI_DEBUG>0) - if (mi_unlikely(!mi_is_in_heap_region(p))) { + if mi_unlikely(!mi_is_in_heap_region(p)) { _mi_warning_message("%s: pointer might not point to a valid heap region: %p\n" "(this may still be a valid very large allocation (over 64MiB))\n", msg, p); - if (mi_likely(_mi_ptr_cookie(segment) == segment->cookie)) { + if mi_likely(_mi_ptr_cookie(segment) == segment->cookie) { _mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p); } } #endif #if (MI_DEBUG>0 || MI_SECURE>=4) - if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { + if mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie) { _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p); return NULL; } @@ -486,23 +496,24 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms void mi_free(void* p) mi_attr_noexcept { mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); - if (mi_unlikely(segment == NULL)) return; + if mi_unlikely(segment == NULL) return; mi_threadid_t tid = _mi_thread_id(); mi_page_t* const page = _mi_segment_page_of(segment, p); - if (mi_likely(tid == mi_atomic_load_relaxed(&segment->thread_id) && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks + if mi_likely(tid == mi_atomic_load_relaxed(&segment->thread_id) && page->flags.full_aligned == 0) { // the thread id matches and it is not a full page, nor has aligned blocks // local, and not full or aligned mi_block_t* block = (mi_block_t*)(p); - if (mi_unlikely(mi_check_is_double_free(page,block))) return; + if mi_unlikely(mi_check_is_double_free(page,block)) return; mi_check_padding(page, block); mi_stat_free(page, block); - #if (MI_DEBUG!=0) + #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif + mi_track_free(p); mi_block_set_next(page, block, page->local_free); page->local_free = block; - if (mi_unlikely(--page->used == 0)) { // using this expression generates better code than: page->used--; if (mi_page_all_free(page)) + if mi_unlikely(--page->used == 0) { // using this expression generates better code than: page->used--; if (mi_page_all_free(page)) _mi_page_retire(page); } } @@ -513,6 +524,7 @@ void mi_free(void* p) mi_attr_noexcept } } +// return true if successful bool _mi_free_delayed_block(mi_block_t* block) { // get segment and page const mi_segment_t* const segment = _mi_ptr_segment(block); @@ -525,7 +537,9 @@ bool _mi_free_delayed_block(mi_block_t* block) { // some blocks may end up in the page `thread_free` list with no blocks in the // heap `thread_delayed_free` list which may cause the page to be never freed! // (it would only be freed if we happen to scan it in `mi_page_queue_find_free_ex`) - _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, false /* dont overwrite never delayed */); + if (!_mi_page_try_use_delayed_free(page, MI_USE_DELAYED_FREE, false /* dont overwrite never delayed */)) { + return false; + } // collect all other non-local frees to ensure up-to-date `used` count _mi_page_free_collect(page, false); @@ -548,7 +562,7 @@ static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noe const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg); if (segment==NULL) return 0; // also returns 0 if `p == NULL` const mi_page_t* const page = _mi_segment_page_of(segment, p); - if (mi_likely(!mi_page_has_aligned(page))) { + if mi_likely(!mi_page_has_aligned(page)) { const mi_block_t* block = (const mi_block_t*)p; return mi_page_usable_size_of(page, block); } @@ -558,7 +572,7 @@ static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noe } } -size_t mi_usable_size(const void* p) mi_attr_noexcept { +mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { return _mi_usable_size(p, "mi_usable_size"); } @@ -570,6 +584,7 @@ size_t mi_usable_size(const void* p) mi_attr_noexcept { #ifdef __cplusplus void* _mi_externs[] = { (void*)&_mi_page_malloc, + (void*)&_mi_heap_malloc_zero, (void*)&mi_malloc, (void*)&mi_malloc_small, (void*)&mi_zalloc_small, @@ -602,24 +617,24 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { mi_free(p); } -extern inline mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count,size,&total)) return NULL; return mi_heap_zalloc(heap,total); } -mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { return mi_heap_calloc(mi_get_default_heap(),count,size); } // Uninitialized `calloc` -extern mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { +mi_decl_nodiscard extern mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; return mi_heap_malloc(heap, total); } -mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { return mi_heap_mallocn(mi_get_default_heap(),count,size); } @@ -638,31 +653,40 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { } void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept { - const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL - if (mi_unlikely(newsize <= size && newsize >= (size / 2))) { + // if p == NULL then behave as malloc. + // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)). + // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) + const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL (with size 0) + if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) // todo: adjust potential padding to reflect the new size? + mi_track_free(p); + mi_track_malloc(p,newsize,true); return p; // reallocation still fits and not more than 50% waste } void* newp = mi_heap_malloc(heap,newsize); - if (mi_likely(newp != NULL)) { + if mi_likely(newp != NULL) { if (zero && newsize > size) { // also set last word in the previous allocation to zero to ensure any padding is zero-initialized const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); memset((uint8_t*)newp + start, 0, newsize - start); } - if (mi_likely(p != NULL)) { - _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize)); + if mi_likely(p != NULL) { + if mi_likely(_mi_is_aligned(p, sizeof(uintptr_t))) { // a client may pass in an arbitrary pointer `p`.. + const size_t copysize = (newsize > size ? size : newsize); + mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking.. + _mi_memcpy_aligned(newp, p, copysize); + } mi_free(p); // only free the original pointer if successful } } return newp; } -void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { return _mi_heap_realloc_zero(heap, p, newsize, false); } -void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; return mi_heap_realloc(heap, p, total); @@ -670,41 +694,41 @@ void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_a // Reallocate but free `p` on errors -void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { void* newp = mi_heap_realloc(heap, p, newsize); if (newp==NULL && p!=NULL) mi_free(p); return newp; } -void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { return _mi_heap_realloc_zero(heap, p, newsize, true); } -void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { +mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; return mi_heap_rezalloc(heap, p, total); } -void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { +mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { return mi_heap_realloc(mi_get_default_heap(),p,newsize); } -void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { +mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { return mi_heap_reallocn(mi_get_default_heap(),p,count,size); } // Reallocate but free `p` on errors -void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { +mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { return mi_heap_reallocf(mi_get_default_heap(),p,newsize); } -void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept { +mi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept { return mi_heap_rezalloc(mi_get_default_heap(), p, newsize); } -void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { +mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { return mi_heap_recalloc(mi_get_default_heap(), p, count, size); } @@ -715,7 +739,7 @@ void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { // ------------------------------------------------------ // `strdup` using mi_malloc -mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept { if (s == NULL) return NULL; size_t n = strlen(s); char* t = (char*)mi_heap_malloc(heap,n+1); @@ -723,12 +747,12 @@ mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_no return t; } -mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept { return mi_heap_strdup(mi_get_default_heap(), s); } // `strndup` using mi_malloc -mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept { if (s == NULL) return NULL; const char* end = (const char*)memchr(s, 0, n); // find end of string in the first `n` characters (returns NULL if not found) const size_t m = (end != NULL ? (size_t)(end - s) : n); // `m` is the minimum of `n` or the end-of-string @@ -740,7 +764,7 @@ mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) return t; } -mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { return mi_heap_strndup(mi_get_default_heap(),s,n); } @@ -751,7 +775,7 @@ mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { #define PATH_MAX MAX_PATH #endif #include <windows.h> -mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { // todo: use GetFullPathNameW to allow longer file names char buf[PATH_MAX]; DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL); @@ -797,7 +821,7 @@ char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) } #endif -mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { return mi_heap_realpath(mi_get_default_heap(),fname,resolved_name); } #endif @@ -835,8 +859,8 @@ static bool mi_try_new_handler(bool nothrow) { #else typedef void (*std_new_handler_t)(void); -#if (defined(__GNUC__) || defined(__clang__)) -std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv(void) { +#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) // exclude clang-cl, see issue #631 +std_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) { return NULL; } static std_new_handler_t mi_get_new_handler(void) { @@ -873,19 +897,19 @@ static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow ) { return p; } -mi_decl_restrict void* mi_new(size_t size) { +mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { void* p = mi_malloc(size); - if (mi_unlikely(p == NULL)) return mi_try_new(size,false); + if mi_unlikely(p == NULL) return mi_try_new(size,false); return p; } -mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { void* p = mi_malloc(size); - if (mi_unlikely(p == NULL)) return mi_try_new(size, true); + if mi_unlikely(p == NULL) return mi_try_new(size, true); return p; } -mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { +mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { void* p; do { p = mi_malloc_aligned(size, alignment); @@ -894,7 +918,7 @@ mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { return p; } -mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept { void* p; do { p = mi_malloc_aligned(size, alignment); @@ -903,9 +927,9 @@ mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_ return p; } -mi_decl_restrict void* mi_new_n(size_t count, size_t size) { +mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { size_t total; - if (mi_unlikely(mi_count_size_overflow(count, size, &total))) { + if mi_unlikely(mi_count_size_overflow(count, size, &total)) { mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc return NULL; } @@ -914,7 +938,7 @@ mi_decl_restrict void* mi_new_n(size_t count, size_t size) { } } -void* mi_new_realloc(void* p, size_t newsize) { +mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) { void* q; do { q = mi_realloc(p, newsize); @@ -922,9 +946,9 @@ void* mi_new_realloc(void* p, size_t newsize) { return q; } -void* mi_new_reallocn(void* p, size_t newcount, size_t size) { +mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { size_t total; - if (mi_unlikely(mi_count_size_overflow(newcount, size, &total))) { + if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) { mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc return NULL; } diff --git a/source/luametatex/source/libraries/mimalloc/src/arena.c b/source/luametatex/source/libraries/mimalloc/src/arena.c index 6b1e951f3..56b87d083 100644 --- a/source/luametatex/source/libraries/mimalloc/src/arena.c +++ b/source/luametatex/source/libraries/mimalloc/src/arena.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019-2021, Microsoft Research, Daan Leijen +Copyright (c) 2019-2022, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -45,16 +45,17 @@ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); Arena allocation ----------------------------------------------------------- */ - // Block info: bit 0 contains the `in_use` bit, the upper bits the // size in count of arena blocks. typedef uintptr_t mi_block_info_t; #define MI_ARENA_BLOCK_SIZE (MI_SEGMENT_SIZE) // 8MiB (must be at least MI_SEGMENT_ALIGN) #define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_BLOCK_SIZE/2) // 4MiB -#define MI_MAX_ARENAS (64) // not more than 256 (since we use 8 bits in the memid) +#define MI_MAX_ARENAS (64) // not more than 126 (since we use 7 bits in the memid and an arena index + 1) // A memory arena descriptor typedef struct mi_arena_s { + mi_arena_id_t id; // arena id; 0 for non-specific + bool exclusive; // only allow allocations if specifically for this arena _Atomic(uint8_t*) start; // the start of the memory area size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`) size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) @@ -75,23 +76,58 @@ static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0 /* ----------------------------------------------------------- + Arena id's + 0 is used for non-arena's (like OS memory) + id = arena_index + 1 +----------------------------------------------------------- */ + +static size_t mi_arena_id_index(mi_arena_id_t id) { + return (size_t)(id <= 0 ? MI_MAX_ARENAS : id - 1); +} + +static mi_arena_id_t mi_arena_id_create(size_t arena_index) { + mi_assert_internal(arena_index < MI_MAX_ARENAS); + mi_assert_internal(MI_MAX_ARENAS <= 126); + int id = (int)arena_index + 1; + mi_assert_internal(id >= 1 && id <= 127); + return id; +} + +mi_arena_id_t _mi_arena_id_none(void) { + return 0; +} + +static bool mi_arena_id_is_suitable(mi_arena_id_t arena_id, bool arena_is_exclusive, mi_arena_id_t req_arena_id) { + return ((!arena_is_exclusive && req_arena_id == _mi_arena_id_none()) || + (arena_id == req_arena_id)); +} + + +/* ----------------------------------------------------------- Arena allocations get a memory id where the lower 8 bits are - the arena index +1, and the upper bits the block index. + the arena id, and the upper bits the block index. ----------------------------------------------------------- */ // Use `0` as a special id for direct OS allocated memory. #define MI_MEMID_OS 0 -static size_t mi_arena_id_create(size_t arena_index, mi_bitmap_index_t bitmap_index) { - mi_assert_internal(arena_index < 0xFE); +static size_t mi_arena_memid_create(mi_arena_id_t id, bool exclusive, mi_bitmap_index_t bitmap_index) { mi_assert_internal(((bitmap_index << 8) >> 8) == bitmap_index); // no overflow? - return ((bitmap_index << 8) | ((arena_index+1) & 0xFF)); + mi_assert_internal(id >= 0 && id <= 0x7F); + return ((bitmap_index << 8) | ((uint8_t)id & 0x7F) | (exclusive ? 0x80 : 0)); +} + +static bool mi_arena_memid_indices(size_t arena_memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index) { + *bitmap_index = (arena_memid >> 8); + mi_arena_id_t id = (int)(arena_memid & 0x7F); + *arena_index = mi_arena_id_index(id); + return ((arena_memid & 0x80) != 0); } -static void mi_arena_id_indices(size_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index) { - mi_assert_internal(memid != MI_MEMID_OS); - *arena_index = (memid & 0xFF) - 1; - *bitmap_index = (memid >> 8); +bool _mi_arena_memid_is_suitable(size_t arena_memid, mi_arena_id_t request_arena_id) { + mi_arena_id_t id = (int)(arena_memid & 0x7F); + bool exclusive = ((arena_memid & 0x80) != 0); + return mi_arena_id_is_suitable(id, exclusive, request_arena_id); } static size_t mi_block_count_of_size(size_t size) { @@ -117,14 +153,19 @@ static bool mi_arena_alloc(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* ----------------------------------------------------------- */ static mi_decl_noinline void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t needed_bcount, - bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) + bool* commit, bool* large, bool* is_pinned, bool* is_zero, + mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld) { + MI_UNUSED(arena_index); + mi_assert_internal(mi_arena_id_index(arena->id) == arena_index); + if (!mi_arena_id_is_suitable(arena->id, arena->exclusive, req_arena_id)) return NULL; + mi_bitmap_index_t bitmap_index; if (!mi_arena_alloc(arena, needed_bcount, &bitmap_index)) return NULL; // claimed it! set the dirty bits (todo: no need for an atomic op here?) void* p = arena->start + (mi_bitmap_index_bit(bitmap_index)*MI_ARENA_BLOCK_SIZE); - *memid = mi_arena_id_create(arena_index, bitmap_index); + *memid = mi_arena_memid_create(arena->id, arena->exclusive, bitmap_index); *is_zero = _mi_bitmap_claim_across(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL); *large = arena->is_large; *is_pinned = (arena->is_large || !arena->allow_decommit); @@ -149,15 +190,31 @@ static mi_decl_noinline void* mi_arena_alloc_from(mi_arena_t* arena, size_t aren return p; } -static mi_decl_noinline void* mi_arena_allocate(int numa_node, size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) +static mi_decl_noinline void* mi_arena_allocate(int numa_node, size_t size, size_t alignment, bool* commit, bool* large, + bool* is_pinned, bool* is_zero, + mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld) { MI_UNUSED_RELEASE(alignment); mi_assert_internal(alignment <= MI_SEGMENT_ALIGN); const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); const size_t bcount = mi_block_count_of_size(size); - if (mi_likely(max_arena == 0)) return NULL; + if mi_likely(max_arena == 0) return NULL; mi_assert_internal(size <= bcount*MI_ARENA_BLOCK_SIZE); + size_t arena_index = mi_arena_id_index(req_arena_id); + if (arena_index < MI_MAX_ARENAS) { + // try a specific arena if requested + mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[arena_index]); + if (arena != NULL && + (arena->numa_node < 0 || arena->numa_node == numa_node) && // numa local? + (*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages + { + void* p = mi_arena_alloc_from(arena, arena_index, bcount, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); + mi_assert_internal((uintptr_t)p % alignment == 0); + if (p != NULL) return p; + } + } + else { // try numa affine allocation for (size_t i = 0; i < max_arena; i++) { mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]); @@ -165,11 +222,9 @@ static mi_decl_noinline void* mi_arena_allocate(int numa_node, size_t size, size if ((arena->numa_node<0 || arena->numa_node==numa_node) && // numa local? (*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages { - void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_pinned, is_zero, memid, tld); + void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); mi_assert_internal((uintptr_t)p % alignment == 0); - if (p != NULL) { - return p; - } + if (p != NULL) return p; } } @@ -180,10 +235,9 @@ static mi_decl_noinline void* mi_arena_allocate(int numa_node, size_t size, size if ((arena->numa_node>=0 && arena->numa_node!=numa_node) && // not numa local! (*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages { - void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_pinned, is_zero, memid, tld); + void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); mi_assert_internal((uintptr_t)p % alignment == 0); - if (p != NULL) { - return p; + if (p != NULL) return p; } } } @@ -192,7 +246,7 @@ static mi_decl_noinline void* mi_arena_allocate(int numa_node, size_t size, size void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, - size_t* memid, mi_os_tld_t* tld) + mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld) { mi_assert_internal(commit != NULL && is_pinned != NULL && is_zero != NULL && memid != NULL && tld != NULL); mi_assert_internal(size > 0); @@ -206,12 +260,12 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* // try to allocate in an arena if the alignment is small enough and the object is not too small (as for heap meta data) if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN) { - void* p = mi_arena_allocate(numa_node, size, alignment, commit, large, is_pinned, is_zero, memid, tld); + void* p = mi_arena_allocate(numa_node, size, alignment, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); if (p != NULL) return p; } // finally, fall back to the OS - if (mi_option_is_enabled(mi_option_limit_os_alloc)) { + if (mi_option_is_enabled(mi_option_limit_os_alloc) || req_arena_id != _mi_arena_id_none()) { errno = ENOMEM; return NULL; } @@ -222,9 +276,19 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* return p; } -void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) +void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld) { - return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, commit, large, is_pinned, is_zero, memid, tld); + return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); +} + +void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) { + if (size != NULL) *size = 0; + size_t arena_index = mi_arena_id_index(arena_id); + if (arena_index >= MI_MAX_ARENAS) return NULL; + mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[arena_index]); + if (arena == NULL) return NULL; + if (size != NULL) *size = arena->block_count * MI_ARENA_BLOCK_SIZE; + return arena->start; } /* ----------------------------------------------------------- @@ -244,7 +308,7 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_o // allocated in an arena size_t arena_idx; size_t bitmap_idx; - mi_arena_id_indices(memid, &arena_idx, &bitmap_idx); + mi_arena_memid_indices(memid, &arena_idx, &bitmap_idx); mi_assert_internal(arena_idx < MI_MAX_ARENAS); mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t,&mi_arenas[arena_idx]); mi_assert_internal(arena != NULL); @@ -281,10 +345,11 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_o Add an arena. ----------------------------------------------------------- */ -static bool mi_arena_add(mi_arena_t* arena) { +static bool mi_arena_add(mi_arena_t* arena, mi_arena_id_t* arena_id) { mi_assert_internal(arena != NULL); mi_assert_internal((uintptr_t)mi_atomic_load_ptr_relaxed(uint8_t,&arena->start) % MI_SEGMENT_ALIGN == 0); mi_assert_internal(arena->block_count > 0); + if (arena_id != NULL) *arena_id = -1; size_t i = mi_atomic_increment_acq_rel(&mi_arena_count); if (i >= MI_MAX_ARENAS) { @@ -292,11 +357,14 @@ static bool mi_arena_add(mi_arena_t* arena) { return false; } mi_atomic_store_ptr_release(mi_arena_t,&mi_arenas[i], arena); + arena->id = mi_arena_id_create(i); + if (arena_id != NULL) *arena_id = arena->id; return true; } -bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept +bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + if (arena_id != NULL) *arena_id = _mi_arena_id_none(); if (size < MI_ARENA_BLOCK_SIZE) return false; if (is_large) { @@ -311,6 +379,8 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la mi_arena_t* arena = (mi_arena_t*)_mi_os_alloc(asize, &_mi_stats_main); // TODO: can we avoid allocating from the OS? if (arena == NULL) return false; + arena->id = _mi_arena_id_none(); + arena->exclusive = exclusive; arena->block_count = bcount; arena->field_count = fields; arena->start = (uint8_t*)start; @@ -335,18 +405,19 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la _mi_bitmap_claim(arena->blocks_inuse, fields, post, postidx, NULL); } - mi_arena_add(arena); - return true; + return mi_arena_add(arena, arena_id); + } // Reserve a range of regular OS memory -int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept +int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + if (arena_id != NULL) *arena_id = _mi_arena_id_none(); size = _mi_align_up(size, MI_ARENA_BLOCK_SIZE); // at least one block bool large = allow_large; void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, &large, &_mi_stats_main); if (start==NULL) return ENOMEM; - if (!mi_manage_os_memory(start, size, (large || commit), large, true, -1)) { + if (!mi_manage_os_memory_ex(start, size, (large || commit), large, true, -1, exclusive, arena_id)) { _mi_os_free_ex(start, size, commit, &_mi_stats_main); _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size,1024)); return ENOMEM; @@ -355,6 +426,19 @@ int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noe return 0; } +bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept { + return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false, NULL); +} + +int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept { + return mi_reserve_os_memory_ex(size, commit, allow_large, false, NULL); +} + + +/* ----------------------------------------------------------- + Debugging +----------------------------------------------------------- */ + static size_t mi_debug_show_bitmap(const char* prefix, mi_bitmap_field_t* fields, size_t field_count ) { size_t inuse_count = 0; for (size_t i = 0; i < field_count; i++) { @@ -383,11 +467,13 @@ void mi_debug_show_arenas(void) mi_attr_noexcept { } } + /* ----------------------------------------------------------- Reserve a huge page arena. ----------------------------------------------------------- */ // reserve at a specific numa node -int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept { +int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + if (arena_id != NULL) *arena_id = -1; if (pages==0) return 0; if (numa_node < -1) numa_node = -1; if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count(); @@ -400,13 +486,16 @@ int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msec } _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); - if (!mi_manage_os_memory(p, hsize, true, true, true, numa_node)) { + if (!mi_manage_os_memory_ex(p, hsize, true, true, true, numa_node, exclusive, arena_id)) { _mi_os_free_huge_pages(p, hsize, &_mi_stats_main); return ENOMEM; } return 0; } +int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept { + return mi_reserve_huge_os_pages_at_ex(pages, numa_node, timeout_msecs, false, NULL); +} // reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected) int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept { diff --git a/source/luametatex/source/libraries/mimalloc/src/bitmap.c b/source/luametatex/source/libraries/mimalloc/src/bitmap.c index af6de0a12..4fc7a1f3d 100644 --- a/source/luametatex/source/libraries/mimalloc/src/bitmap.c +++ b/source/luametatex/source/libraries/mimalloc/src/bitmap.c @@ -108,6 +108,25 @@ bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fiel return false; } +// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled +bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, + const size_t start_field_idx, const size_t count, + mi_bitmap_pred_fun_t pred_fun, void* pred_arg, + mi_bitmap_index_t* bitmap_idx) { + size_t idx = start_field_idx; + for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { + if (idx >= bitmap_fields) idx = 0; // wrap + if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { + if (pred_fun == NULL || pred_fun(*bitmap_idx, pred_arg)) { + return true; + } + // predicate returned false, unclaim and look further + _mi_bitmap_unclaim(bitmap, bitmap_fields, count, *bitmap_idx); + } + } + return false; +} + /* // Find `count` bits of 0 and set them to 1 atomically; returns `true` on success. // For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never span fields. @@ -283,7 +302,7 @@ bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitm static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, size_t* pre_mask, size_t* mid_mask, size_t* post_mask) { MI_UNUSED_RELEASE(bitmap_fields); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); - if (mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS)) { + if mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS) { *pre_mask = mi_bitmap_mask_(count, bitidx); *mid_mask = 0; *post_mask = 0; diff --git a/source/luametatex/source/libraries/mimalloc/src/bitmap.h b/source/luametatex/source/libraries/mimalloc/src/bitmap.h index 7bd3106c9..0c501ec1f 100644 --- a/source/luametatex/source/libraries/mimalloc/src/bitmap.h +++ b/source/luametatex/source/libraries/mimalloc/src/bitmap.h @@ -72,6 +72,10 @@ bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_ // For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); +// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled +typedef bool (mi_cdecl *mi_bitmap_pred_fun_t)(mi_bitmap_index_t bitmap_idx, void* pred_arg); +bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_pred_fun_t pred_fun, void* pred_arg, mi_bitmap_index_t* bitmap_idx); + // Set `count` bits at `bitmap_idx` to 0 atomically // Returns `true` if all `count` bits were 1 previously. bool _mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); diff --git a/source/luametatex/source/libraries/mimalloc/src/heap.c b/source/luametatex/source/libraries/mimalloc/src/heap.c index 816d961ae..15ca36031 100644 --- a/source/luametatex/source/libraries/mimalloc/src/heap.c +++ b/source/luametatex/source/libraries/mimalloc/src/heap.c @@ -139,9 +139,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL); } - // free thread delayed blocks. + // free all current thread delayed blocks. // (if abandoning, after this there are no more thread-delayed references into the pages.) - _mi_heap_delayed_free(heap); + _mi_heap_delayed_free_all(heap); // collect retired pages _mi_heap_collect_retired(heap, force); @@ -200,13 +200,14 @@ mi_heap_t* mi_heap_get_backing(void) { return bheap; } -mi_heap_t* mi_heap_new(void) { +mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena( mi_arena_id_t arena_id ) { mi_heap_t* bheap = mi_heap_get_backing(); mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? if (heap==NULL) return NULL; _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); heap->tld = bheap->tld; heap->thread_id = _mi_thread_id(); + heap->arena_id = arena_id; _mi_random_split(&bheap->random, &heap->random); heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); @@ -218,6 +219,14 @@ mi_heap_t* mi_heap_new(void) { return heap; } +mi_decl_nodiscard mi_heap_t* mi_heap_new(void) { + return mi_heap_new_in_arena(_mi_arena_id_none()); +} + +bool _mi_heap_memid_is_suitable(mi_heap_t* heap, size_t memid) { + return _mi_arena_memid_is_suitable(memid, heap->arena_id); +} + uintptr_t _mi_heap_random_next(mi_heap_t* heap) { return _mi_random_next(&heap->random); } @@ -350,7 +359,7 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { if (from==NULL || from->page_count == 0) return; // reduce the size of the delayed frees - _mi_heap_delayed_free(from); + _mi_heap_delayed_free_partial(from); // transfer all pages by appending the queues; this will set a new heap field // so threads may do delayed frees in either heap for a while. @@ -369,7 +378,7 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { // note: be careful here as the `heap` field in all those pages no longer point to `from`, // turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a // the regular `_mi_free_delayed_block` which is safe. - _mi_heap_delayed_free(from); + _mi_heap_delayed_free_all(from); #if !defined(_MSC_VER) || (_MSC_VER > 1900) // somehow the following line gives an error in VS2015, issue #353 mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_block_t,&from->thread_delayed_free) == NULL); #endif @@ -421,7 +430,7 @@ static mi_heap_t* mi_heap_of_block(const void* p) { mi_segment_t* segment = _mi_ptr_segment(p); bool valid = (_mi_ptr_cookie(segment) == segment->cookie); mi_assert_internal(valid); - if (mi_unlikely(!valid)) return NULL; + if mi_unlikely(!valid) return NULL; return mi_page_heap(_mi_segment_page_of(segment,p)); } @@ -543,7 +552,7 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa xarea.area.reserved = page->reserved * bsize; xarea.area.committed = page->capacity * bsize; xarea.area.blocks = _mi_page_start(_mi_page_segment(page), page, NULL); - xarea.area.used = page->used * bsize; + xarea.area.used = page->used; // number of blocks in use (#553) xarea.area.block_size = ubsize; xarea.area.full_block_size = bsize; return fun(heap, &xarea, arg); diff --git a/source/luametatex/source/libraries/mimalloc/src/init.c b/source/luametatex/source/libraries/mimalloc/src/init.c index 19124afef..4f37b7176 100644 --- a/source/luametatex/source/libraries/mimalloc/src/init.c +++ b/source/luametatex/source/libraries/mimalloc/src/init.c @@ -109,6 +109,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { MI_ATOMIC_VAR_INIT(NULL), 0, // tid 0, // cookie + 0, // arena id { 0, 0 }, // keys { {0}, {0}, 0 }, 0, // page count @@ -149,6 +150,7 @@ mi_heap_t _mi_heap_main = { MI_ATOMIC_VAR_INIT(NULL), 0, // thread id 0, // initial cookie + 0, // arena id { 0, 0 }, // the key of the main heap can be fixed (unlike page keys that need to be secure!) { {0x846ca68b}, {0}, 0 }, // random 0, // page count @@ -475,7 +477,7 @@ void _mi_heap_set_default_direct(mi_heap_t* heap) { // -------------------------------------------------------- // Run functions on process init/done, and thread init/done // -------------------------------------------------------- -static void mi_process_done(void); +static void mi_cdecl mi_process_done(void); static bool os_preloading = true; // true until this module is initialized static bool mi_redirected = false; // true if malloc redirects to mi_malloc @@ -490,7 +492,7 @@ mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept { } // Communicate with the redirection module on Windows -#if defined(_WIN32) && defined(MI_SHARED_LIB) +#if defined(_WIN32) && defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) #ifdef __cplusplus extern "C" { #endif @@ -506,8 +508,8 @@ mi_decl_export void _mi_redirect_entry(DWORD reason) { mi_thread_done(); } } -__declspec(dllimport) bool mi_allocator_init(const char** message); -__declspec(dllimport) void mi_allocator_done(void); +__declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message); +__declspec(dllimport) void mi_cdecl mi_allocator_done(void); #ifdef __cplusplus } #endif @@ -606,7 +608,7 @@ void mi_process_init(void) mi_attr_noexcept { } // Called when the process is done (through `at_exit`) -static void mi_process_done(void) { +static void mi_cdecl mi_process_done(void) { // only shutdown if we were initialized if (!_mi_process_is_initialized) return; // ensure we are called once diff --git a/source/luametatex/source/libraries/mimalloc/src/options.c b/source/luametatex/source/libraries/mimalloc/src/options.c index 6b2379322..0182671ce 100644 --- a/source/luametatex/source/libraries/mimalloc/src/options.c +++ b/source/luametatex/source/libraries/mimalloc/src/options.c @@ -120,7 +120,7 @@ mi_decl_nodiscard long mi_option_get(mi_option_t option) { if (option < 0 || option >= _mi_option_last) return 0; mi_option_desc_t* desc = &options[option]; mi_assert(desc->option == option); // index should match the option - if (mi_unlikely(desc->init == UNINIT)) { + if mi_unlikely(desc->init == UNINIT) { mi_option_init(desc); } return desc->value; @@ -170,7 +170,7 @@ void mi_option_disable(mi_option_t option) { } -static void mi_out_stderr(const char* msg, void* arg) { +static void mi_cdecl mi_out_stderr(const char* msg, void* arg) { MI_UNUSED(arg); if (msg == NULL) return; #ifdef _WIN32 @@ -203,7 +203,7 @@ static void mi_out_stderr(const char* msg, void* arg) { static char out_buf[MI_MAX_DELAY_OUTPUT+1]; static _Atomic(size_t) out_len; -static void mi_out_buf(const char* msg, void* arg) { +static void mi_cdecl mi_out_buf(const char* msg, void* arg) { MI_UNUSED(arg); if (msg==NULL) return; if (mi_atomic_load_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return; @@ -235,7 +235,7 @@ static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) { // Once this module is loaded, switch to this routine // which outputs to stderr and the delayed output buffer. -static void mi_out_buf_stderr(const char* msg, void* arg) { +static void mi_cdecl mi_out_buf_stderr(const char* msg, void* arg) { mi_out_stderr(msg,arg); mi_out_buf(msg,arg); } @@ -346,7 +346,7 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) { static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) { if (prefix != NULL && strlen(prefix) <= 32 && !_mi_is_main_thread()) { char tprefix[64]; - snprintf(tprefix, sizeof(tprefix), "%sthread 0x%x: ", prefix, (unsigned) _mi_thread_id()); /* HH: %z is unknown */ +/* HH */ snprintf(tprefix, sizeof(tprefix), "%sthread 0x%x: ", prefix, (unsigned) _mi_thread_id()); /* HH: %z is unknown */ mi_vfprintf(out, arg, tprefix, fmt, args); } else { diff --git a/source/luametatex/source/libraries/mimalloc/src/os.c b/source/luametatex/source/libraries/mimalloc/src/os.c index 72959d818..6d7249873 100644 --- a/source/luametatex/source/libraries/mimalloc/src/os.c +++ b/source/luametatex/source/libraries/mimalloc/src/os.c @@ -122,7 +122,7 @@ size_t _mi_os_good_alloc_size(size_t size) { else if (size < 8*MI_MiB) align_size = 256*MI_KiB; else if (size < 32*MI_MiB) align_size = 1*MI_MiB; else align_size = 4*MI_MiB; - if (mi_unlikely(size >= (SIZE_MAX - align_size))) return size; // possible overflow? + if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow? return _mi_align_up(size, align_size); } @@ -362,9 +362,9 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside // the memory region returned by VirtualAlloc; in that case we need to free using // the start of the region. - MEMORY_BASIC_INFORMATION info = { 0, 0 }; + MEMORY_BASIC_INFORMATION info = { 0 }; VirtualQuery(addr, &info, sizeof(info)); - if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < MI_SEGMENT_SIZE) { + if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)MI_SEGMENT_SIZE) { errcode = 0; err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0); if (err) { errcode = GetLastError(); } @@ -986,7 +986,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) else _mi_stat_decrease(&stats->reset, csize); if (!reset) return true; // nothing to do on unreset! - #if (MI_DEBUG>1) + #if (MI_DEBUG>1) && !MI_TRACK_ENABLED if (MI_SECURE==0) { memset(start, 0, csize); // pretend it is eagerly reset } diff --git a/source/luametatex/source/libraries/mimalloc/src/page.c b/source/luametatex/source/libraries/mimalloc/src/page.c index fd6c5397d..4b321156c 100644 --- a/source/luametatex/source/libraries/mimalloc/src/page.c +++ b/source/luametatex/source/libraries/mimalloc/src/page.c @@ -124,14 +124,23 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) { + while (!_mi_page_try_use_delayed_free(page, delay, override_never)) { + mi_atomic_yield(); + } +} + +bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) { mi_thread_free_t tfreex; mi_delayed_t old_delay; mi_thread_free_t tfree; + size_t yield_count = 0; do { tfree = mi_atomic_load_acquire(&page->xthread_free); // note: must acquire as we can break/repeat this loop and not do a CAS; tfreex = mi_tf_set_delayed(tfree, delay); old_delay = mi_tf_delayed(tfree); - if (mi_unlikely(old_delay == MI_DELAYED_FREEING)) { + if mi_unlikely(old_delay == MI_DELAYED_FREEING) { + if (yield_count >= 4) return false; // give up after 4 tries + yield_count++; mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done. // tfree = mi_tf_set_delayed(tfree, MI_NO_DELAYED_FREE); // will cause CAS to busy fail } @@ -143,6 +152,8 @@ void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool overrid } } while ((old_delay == MI_DELAYED_FREEING) || !mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex)); + + return true; // success } /* ----------------------------------------------------------- @@ -199,7 +210,7 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // and the local free list if (page->local_free != NULL) { - if (mi_likely(page->free == NULL)) { + if mi_likely(page->free == NULL) { // usual case page->free = page->local_free; page->local_free = NULL; @@ -272,10 +283,18 @@ static mi_page_t* mi_page_fresh(mi_heap_t* heap, mi_page_queue_t* pq) { Do any delayed frees (put there by other threads if they deallocated in a full page) ----------------------------------------------------------- */ -void _mi_heap_delayed_free(mi_heap_t* heap) { +void _mi_heap_delayed_free_all(mi_heap_t* heap) { + while (!_mi_heap_delayed_free_partial(heap)) { + mi_atomic_yield(); + } +} + +// returns true if all delayed frees were processed +bool _mi_heap_delayed_free_partial(mi_heap_t* heap) { // take over the list (note: no atomic exchange since it is often NULL) mi_block_t* block = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free); while (block != NULL && !mi_atomic_cas_ptr_weak_acq_rel(mi_block_t, &heap->thread_delayed_free, &block, NULL)) { /* nothing */ }; + bool all_freed = true; // and free them all while(block != NULL) { @@ -283,7 +302,9 @@ void _mi_heap_delayed_free(mi_heap_t* heap) { // use internal free instead of regular one to keep stats etc correct if (!_mi_free_delayed_block(block)) { // we might already start delayed freeing while another thread has not yet - // reset the delayed_freeing flag; in that case delay it further by reinserting. + // reset the delayed_freeing flag; in that case delay it further by reinserting the current block + // into the delayed free list + all_freed = false; mi_block_t* dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free); do { mi_block_set_nextx(heap, block, dfree, heap->keys); @@ -291,6 +312,7 @@ void _mi_heap_delayed_free(mi_heap_t* heap) { } block = next; } + return all_freed; } /* ----------------------------------------------------------- @@ -403,7 +425,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { // how to check this efficiently though... // for now, we don't retire if it is the only page left of this size class. mi_page_queue_t* pq = mi_page_queue_of(page); - if (mi_likely(page->xblock_size <= MI_MAX_RETIRE_SIZE && !mi_page_is_in_full(page))) { + if mi_likely(page->xblock_size <= MI_MAX_RETIRE_SIZE && !mi_page_is_in_full(page)) { if (pq->last==page && pq->first==page) { // the only page in the queue? mi_stat_counter_increase(_mi_stats_main.page_no_retire,1); page->retire_expire = 1 + (page->xblock_size <= MI_SMALL_OBJ_SIZE_MAX ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); @@ -619,7 +641,9 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_page_set_heap(page, heap); page->xblock_size = (block_size < MI_HUGE_BLOCK_SIZE ? (uint32_t)block_size : MI_HUGE_BLOCK_SIZE); // initialize before _mi_segment_page_start size_t page_size; - _mi_segment_page_start(segment, page, &page_size); + const void* page_start = _mi_segment_page_start(segment, page, &page_size); + MI_UNUSED(page_start); + mi_track_mem_noaccess(page_start,page_size); mi_assert_internal(mi_page_block_size(page) <= page_size); mi_assert_internal(page_size <= page->slice_count*MI_SEGMENT_SLICE_SIZE); mi_assert_internal(page_size / block_size < (1L<<16)); @@ -812,8 +836,8 @@ static mi_page_t* mi_large_huge_page_alloc(mi_heap_t* heap, size_t size) { static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size) mi_attr_noexcept { // huge allocation? const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if (mi_unlikely(req_size > (MI_MEDIUM_OBJ_SIZE_MAX - MI_PADDING_SIZE) )) { - if (mi_unlikely(req_size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>) + if mi_unlikely(req_size > (MI_MEDIUM_OBJ_SIZE_MAX - MI_PADDING_SIZE)) { + if mi_unlikely(req_size > PTRDIFF_MAX) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>) _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); return NULL; } @@ -830,32 +854,32 @@ static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size) mi_attr_noexcept { // Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. -void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept +void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { mi_assert_internal(heap != NULL); // initialize if necessary - if (mi_unlikely(!mi_heap_is_initialized(heap))) { + if mi_unlikely(!mi_heap_is_initialized(heap)) { mi_thread_init(); // calls `_mi_heap_init` in turn heap = mi_get_default_heap(); - if (mi_unlikely(!mi_heap_is_initialized(heap))) { return NULL; } + if mi_unlikely(!mi_heap_is_initialized(heap)) { return NULL; } } mi_assert_internal(mi_heap_is_initialized(heap)); // call potential deferred free routines _mi_deferred_free(heap, false); - // free delayed frees from other threads - _mi_heap_delayed_free(heap); + // free delayed frees from other threads (but skip contended ones) + _mi_heap_delayed_free_partial(heap); // find (or allocate) a page of the right size mi_page_t* page = mi_find_page(heap, size); - if (mi_unlikely(page == NULL)) { // first time out of memory, try to collect and retry the allocation once more + if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more mi_heap_collect(heap, true /* force */); page = mi_find_page(heap, size); } - if (mi_unlikely(page == NULL)) { // out of memory + if mi_unlikely(page == NULL) { // out of memory const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size); return NULL; @@ -864,6 +888,15 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(mi_page_block_size(page) >= size); - // and try again, this time succeeding! (i.e. this should never recurse) - return _mi_page_malloc(heap, page, size); + // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc) + if mi_unlikely(zero && page->xblock_size == 0) { + // note: we cannot call _mi_page_malloc with zeroing for huge blocks; we zero it afterwards in that case. + void* p = _mi_page_malloc(heap, page, size, false); + mi_assert_internal(p != NULL); + _mi_memzero_aligned(p, mi_page_usable_block_size(page)); + return p; + } + else { + return _mi_page_malloc(heap, page, size, zero); + } } diff --git a/source/luametatex/source/libraries/mimalloc/src/random.c b/source/luametatex/source/libraries/mimalloc/src/random.c index d474a53a0..a5f5e6b82 100644 --- a/source/luametatex/source/libraries/mimalloc/src/random.c +++ b/source/luametatex/source/libraries/mimalloc/src/random.c @@ -172,6 +172,8 @@ If we cannot get good randomness, we fall back to weak randomness based on a tim // We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using // dynamic overriding, we observed it can raise an exception when compiled with C++, and // sometimes deadlocks when also running under the VS debugger. +// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom. +// To be continued.. #pragma comment (lib,"advapi32.lib") #define RtlGenRandom SystemFunction036 #ifdef __cplusplus diff --git a/source/luametatex/source/libraries/mimalloc/src/region.c b/source/luametatex/source/libraries/mimalloc/src/region.c index 72ce84947..57d11fe8d 100644 --- a/source/luametatex/source/libraries/mimalloc/src/region.c +++ b/source/luametatex/source/libraries/mimalloc/src/region.c @@ -49,9 +49,10 @@ bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats); bool _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats); // arena.c +mi_arena_id_t _mi_arena_id_none(void); void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_stats_t* stats); -void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); -void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); +void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); +void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); @@ -180,7 +181,7 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, bool is_zero = false; bool is_pinned = false; size_t arena_memid = 0; - void* const start = _mi_arena_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, ®ion_commit, ®ion_large, &is_pinned, &is_zero, &arena_memid, tld); + void* const start = _mi_arena_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, ®ion_commit, ®ion_large, &is_pinned, &is_zero, _mi_arena_id_none(), & arena_memid, tld); if (start == NULL) return false; mi_assert_internal(!(region_large && !allow_large)); mi_assert_internal(!region_large || region_commit); @@ -330,7 +331,7 @@ static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* large, bool* } mi_assert_internal(!_mi_bitmap_is_any_claimed(®ion->reset, 1, blocks, bit_idx)); - #if (MI_DEBUG>=2) + #if (MI_DEBUG>=2) && !MI_TRACK_ENABLED if (*commit) { ((uint8_t*)p)[0] = 0; } #endif @@ -370,13 +371,13 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* l } if (p == NULL) { // and otherwise fall back to the OS - p = _mi_arena_alloc_aligned(size, alignment, commit, large, is_pinned, is_zero, &arena_memid, tld); + p = _mi_arena_alloc_aligned(size, alignment, commit, large, is_pinned, is_zero, _mi_arena_id_none(), & arena_memid, tld); *memid = mi_memid_create_from_arena(arena_memid); } if (p != NULL) { mi_assert_internal((uintptr_t)p % alignment == 0); -#if (MI_DEBUG>=2) + #if (MI_DEBUG>=2) && !MI_TRACK_ENABLED if (*commit) { ((uint8_t*)p)[0] = 0; } // ensure the memory is committed #endif } diff --git a/source/luametatex/source/libraries/mimalloc/src/segment-cache.c b/source/luametatex/source/libraries/mimalloc/src/segment-cache.c index aacdbc11d..da726716a 100644 --- a/source/luametatex/source/libraries/mimalloc/src/segment-cache.c +++ b/source/luametatex/source/libraries/mimalloc/src/segment-cache.c @@ -39,8 +39,13 @@ static mi_decl_cache_align mi_bitmap_field_t cache_available[MI_CACHE_FIELDS] = static mi_decl_cache_align mi_bitmap_field_t cache_available_large[MI_CACHE_FIELDS] = { MI_CACHE_BITS_SET }; static mi_decl_cache_align mi_bitmap_field_t cache_inuse[MI_CACHE_FIELDS]; // zero bit = free +static bool mi_cdecl mi_segment_cache_is_suitable(mi_bitmap_index_t bitidx, void* arg) { + mi_arena_id_t req_arena_id = *((mi_arena_id_t*)arg); + mi_cache_slot_t* slot = &cache[mi_bitmap_index_bit(bitidx)]; + return _mi_arena_memid_is_suitable(slot->memid, req_arena_id); +} -mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) +mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t _req_arena_id, size_t* memid, mi_os_tld_t* tld) { #ifdef MI_CACHE_DISABLE return NULL; @@ -60,12 +65,15 @@ mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* comm // find an available slot mi_bitmap_index_t bitidx = 0; bool claimed = false; + mi_arena_id_t req_arena_id = _req_arena_id; + mi_bitmap_pred_fun_t pred_fun = &mi_segment_cache_is_suitable; // cannot pass NULL as the arena may be exclusive itself; todo: do not put exclusive arenas in the cache? + if (*large) { // large allowed? - claimed = _mi_bitmap_try_find_from_claim(cache_available_large, MI_CACHE_FIELDS, start_field, 1, &bitidx); + claimed = _mi_bitmap_try_find_from_claim_pred(cache_available_large, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx); if (claimed) *large = true; } if (!claimed) { - claimed = _mi_bitmap_try_find_from_claim(cache_available, MI_CACHE_FIELDS, start_field, 1, &bitidx); + claimed = _mi_bitmap_try_find_from_claim_pred (cache_available, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx); if (claimed) *large = false; } @@ -283,7 +291,7 @@ static mi_segment_t* _mi_segment_of(const void* p) { size_t index = mi_segment_map_index_of(segment, &bitidx); // fast path: for any pointer to valid small/medium/large object or first MI_SEGMENT_SIZE in huge const uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); - if (mi_likely((mask & ((uintptr_t)1 << bitidx)) != 0)) { + if mi_likely((mask & ((uintptr_t)1 << bitidx)) != 0) { return segment; // yes, allocated by us } if (index==MI_SEGMENT_MAP_WSIZE) return NULL; @@ -324,7 +332,7 @@ static mi_segment_t* _mi_segment_of(const void* p) { mi_assert_internal((void*)segment < p); bool cookie_ok = (_mi_ptr_cookie(segment) == segment->cookie); mi_assert_internal(cookie_ok); - if (mi_unlikely(!cookie_ok)) return NULL; + if mi_unlikely(!cookie_ok) return NULL; if (((uint8_t*)segment + mi_segment_size(segment)) <= (uint8_t*)p) return NULL; // outside the range mi_assert_internal(p >= (void*)segment && (uint8_t*)p < (uint8_t*)segment + mi_segment_size(segment)); return segment; diff --git a/source/luametatex/source/libraries/mimalloc/src/segment.c b/source/luametatex/source/libraries/mimalloc/src/segment.c index 800d4fc31..c76c2259e 100644 --- a/source/luametatex/source/libraries/mimalloc/src/segment.c +++ b/source/luametatex/source/libraries/mimalloc/src/segment.c @@ -721,7 +721,7 @@ static mi_page_t* mi_segment_span_allocate(mi_segment_t* segment, size_t slice_i return page; } -static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_segments_tld_t* tld) { +static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) { mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_LARGE_OBJ_SIZE_MAX); // search from best fit up mi_span_queue_t* sq = mi_span_queue_for(slice_count, tld); @@ -730,8 +730,11 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_segm for (mi_slice_t* slice = sq->first; slice != NULL; slice = slice->next) { if (slice->slice_count >= slice_count) { // found one - mi_span_queue_delete(sq, slice); mi_segment_t* segment = _mi_ptr_segment(slice); + if (_mi_arena_memid_is_suitable(segment->memid, req_arena_id)) { + // found a suitable page span + mi_span_queue_delete(sq, slice); + if (slice->slice_count > slice_count) { mi_segment_slice_split(segment, slice, slice_count, tld); } @@ -745,6 +748,7 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_segm return page; } } + } sq++; } // could not find a page.. @@ -757,7 +761,7 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_segm ----------------------------------------------------------- */ // Allocate a segment from the OS aligned to `MI_SEGMENT_SIZE` . -static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_segments_tld_t* tld, mi_os_tld_t* os_tld, mi_page_t** huge_page) +static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld, mi_page_t** huge_page) { mi_assert_internal((required==0 && huge_page==NULL) || (required>0 && huge_page != NULL)); mi_assert_internal((segment==NULL) || (segment!=NULL && required==0)); @@ -793,9 +797,9 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ bool mem_large = (!eager_delay && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy bool is_pinned = false; size_t memid = 0; - segment = (mi_segment_t*)_mi_segment_cache_pop(segment_size, &commit_mask, &decommit_mask, &mem_large, &is_pinned, &is_zero, &memid, os_tld); + segment = (mi_segment_t*)_mi_segment_cache_pop(segment_size, &commit_mask, &decommit_mask, &mem_large, &is_pinned, &is_zero, req_arena_id, &memid, os_tld); if (segment==NULL) { - segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_pinned, &is_zero, &memid, os_tld); + segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_pinned, &is_zero, req_arena_id, &memid, os_tld); if (segment == NULL) return NULL; // failed to allocate if (commit) { mi_commit_mask_create_full(&commit_mask); @@ -817,6 +821,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ if (!ok) return NULL; // failed to commit mi_commit_mask_set(&commit_mask, &commit_needed_mask); } + mi_track_mem_undefined(segment,commit_needed); segment->memid = memid; segment->mem_is_pinned = is_pinned; segment->mem_is_large = mem_large; @@ -907,8 +912,8 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ // Allocate a segment from the OS aligned to `MI_SEGMENT_SIZE` . -static mi_segment_t* mi_segment_alloc(size_t required, mi_segments_tld_t* tld, mi_os_tld_t* os_tld, mi_page_t** huge_page) { - return mi_segment_init(NULL, required, tld, os_tld, huge_page); +static mi_segment_t* mi_segment_alloc(size_t required, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld, mi_page_t** huge_page) { + return mi_segment_init(NULL, required, req_arena_id, tld, os_tld, huge_page); } @@ -1149,8 +1154,8 @@ static mi_segment_t* mi_abandoned_pop(void) { // Check efficiently if it is empty (or if the visited list needs to be moved) mi_tagged_segment_t ts = mi_atomic_load_relaxed(&abandoned); segment = mi_tagged_segment_ptr(ts); - if (mi_likely(segment == NULL)) { - if (mi_likely(!mi_abandoned_visited_revisit())) { // try to swap in the visited list on NULL + if mi_likely(segment == NULL) { + if mi_likely(!mi_abandoned_visited_revisit()) { // try to swap in the visited list on NULL return NULL; } } @@ -1367,6 +1372,9 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice long max_tries = mi_option_get_clamp(mi_option_max_segment_reclaim, 8, 1024); // limit the work to bound allocation times while ((max_tries-- > 0) && ((segment = mi_abandoned_pop()) != NULL)) { segment->abandoned_visits++; + // todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments + // and push them into the visited list and use many tries. Perhaps we can skip non-suitable ones in a better way? + bool is_suitable = _mi_heap_memid_is_suitable(heap, segment->memid); bool has_page = mi_segment_check_free(segment,needed_slices,block_size,tld); // try to free up pages (due to concurrent frees) if (segment->used == 0) { // free the segment (by forced reclaim) to make it available to other threads. @@ -1376,13 +1384,13 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice // freeing but that would violate some invariants temporarily) mi_segment_reclaim(segment, heap, 0, NULL, tld); } - else if (has_page) { + else if (has_page && is_suitable) { // found a large enough free span, or a page of the right block_size with free space // we return the result of reclaim (which is usually `segment`) as it might free // the segment due to concurrent frees (in which case `NULL` is returned). return mi_segment_reclaim(segment, heap, block_size, reclaimed, tld); } - else if (segment->abandoned_visits > 3) { + else if (segment->abandoned_visits > 3 && is_suitable) { // always reclaim on 3rd visit to limit the abandoned queue length. mi_segment_reclaim(segment, heap, 0, NULL, tld); } @@ -1442,7 +1450,7 @@ static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_ return segment; } // 2. otherwise allocate a fresh segment - return mi_segment_alloc(0, tld, os_tld, NULL); + return mi_segment_alloc(0, heap->arena_id, tld, os_tld, NULL); } @@ -1458,7 +1466,7 @@ static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_ki size_t page_size = _mi_align_up(required, (required > MI_MEDIUM_PAGE_SIZE ? MI_MEDIUM_PAGE_SIZE : MI_SEGMENT_SLICE_SIZE)); size_t slices_needed = page_size / MI_SEGMENT_SLICE_SIZE; mi_assert_internal(slices_needed * MI_SEGMENT_SLICE_SIZE == page_size); - mi_page_t* page = mi_segments_page_find_and_allocate(slices_needed, tld); //(required <= MI_SMALL_SIZE_MAX ? 0 : slices_needed), tld); + mi_page_t* page = mi_segments_page_find_and_allocate(slices_needed, heap->arena_id, tld); //(required <= MI_SMALL_SIZE_MAX ? 0 : slices_needed), tld); if (page==NULL) { // no free page, allocate a new segment and try again if (mi_segment_reclaim_or_alloc(heap, slices_needed, block_size, tld, os_tld) == NULL) { @@ -1482,10 +1490,10 @@ static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_ki Huge page allocation ----------------------------------------------------------- */ -static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { mi_page_t* page = NULL; - mi_segment_t* segment = mi_segment_alloc(size,tld,os_tld,&page); + mi_segment_t* segment = mi_segment_alloc(size,req_arena_id,tld,os_tld,&page); if (segment == NULL || page==NULL) return NULL; mi_assert_internal(segment->used==1); mi_assert_internal(mi_page_block_size(page) >= size); @@ -1535,8 +1543,9 @@ mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, mi_segment page = mi_segments_page_alloc(heap,MI_PAGE_LARGE,block_size,block_size,tld, os_tld); } else { - page = mi_segment_huge_page_alloc(block_size,tld,os_tld); + page = mi_segment_huge_page_alloc(block_size,heap->arena_id,tld,os_tld); } + mi_assert_internal(page == NULL || _mi_heap_memid_is_suitable(heap, _mi_page_segment(page)->memid)); mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld)); return page; } diff --git a/source/luametatex/source/libraries/mimalloc/src/stats.c b/source/luametatex/source/libraries/mimalloc/src/stats.c index 134a7bcb6..f82c7c67f 100644 --- a/source/luametatex/source/libraries/mimalloc/src/stats.c +++ b/source/luametatex/source/libraries/mimalloc/src/stats.c @@ -267,7 +267,7 @@ static void mi_buffered_flush(buffered_t* buf) { buf->used = 0; } -static void mi_buffered_out(const char* msg, void* arg) { +static void mi_cdecl mi_buffered_out(const char* msg, void* arg) { buffered_t* buf = (buffered_t*)arg; if (msg==NULL || buf==NULL) return; for (const char* src = msg; *src != 0; src++) { diff --git a/source/luametatex/source/libraries/miniz/ChangeLog.md b/source/luametatex/source/libraries/miniz/ChangeLog.md index 4ae15a8cd..f88df3313 100644 --- a/source/luametatex/source/libraries/miniz/ChangeLog.md +++ b/source/luametatex/source/libraries/miniz/ChangeLog.md @@ -1,5 +1,44 @@ ## Changelog +### 3.0.1 + + - Fix compilation error with MINIZ_USE_UNALIGNED_LOADS_AND_STORES=1 + +### 3.0.0 + + - Reduce memory usage for inflate. This changes `struct tinfl_decompressor_tag` and therefore requires a major version bump (breaks ABI compatibility) + - Add padding to structures so it continues to work if features differ. This also changes some structures + - Use _ftelli64, _fseeki64 and stat with MinGW32 and OpenWatcom + - Fix varios warnings with OpenWatcom compiler + - Avoid using unaligned memory access in UBSan builds + - Set MINIZ_LITTLE_ENDIAN only if not set + - Add MINIZ_NO_DEFLATE_APIS and MINIZ_NO_INFLATE_APIS + - Fix use of uninitialized memory in tinfl_decompress_mem_to_callback() + - Use wfopen on windows + - Use _wstat64 instead _stat64 on windows + - Use level_and_flags after MZ_DEFAULT_COMPRESSION has been handled + - Improve endianess detection + - Don't use unaligned stores and loads per default + - Fix function declaration if MINIZ_NO_STDIO is used + - Fix MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_UTF8 not being set + - Remove total files check (its 32-bit uint) + - tinfl_decompress: avoid NULL ptr arithmetic UB + - miniz_zip: fix mz_zip_reader_extract_to_heap to read correct sizes + - Eliminate 64-bit operations on 32-bit machines + - Disable treating warnings as error with MSVC + - Disable building shared lib via CMake by default + - Fixed alignment problems on MacOS + - Fixed get error string for MZ_ZIP_TOTAL_ERRORS + - Write correct FLEVEL 2-bit value in zlib header + - miniz.pc.in: fix include path not containing the "miniz" suffix + - Fix compatibility with FreeBSD + - pkg-config tweaks + - Fix integer overflow in header corruption check + - Fix some warnings + - tdefl_compress_normal: Avoid NULL ptr arithmetic UB + - replace use of stdint.h types with mz_ variants + + ### 2.2.0 - Fix examples with amalgamation diff --git a/source/luametatex/source/libraries/miniz/miniz.c b/source/luametatex/source/libraries/miniz/miniz.c index 87bdedb18..c197c181c 100644 --- a/source/luametatex/source/libraries/miniz/miniz.c +++ b/source/luametatex/source/libraries/miniz/miniz.c @@ -187,6 +187,8 @@ const char *mz_version(void) #ifndef MINIZ_NO_ZLIB_APIS +#ifndef MINIZ_NO_DEFLATE_APIS + int mz_deflateInit(mz_streamp pStream, int level) { return mz_deflateInit2(pStream, level, MZ_DEFLATED, MZ_DEFAULT_WINDOW_BITS, 9, MZ_DEFAULT_STRATEGY); @@ -321,7 +323,7 @@ int mz_compress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char memset(&stream, 0, sizeof(stream)); /* In case mz_ulong is 64-bits (argh I hate longs). */ - if ((source_len | *pDest_len) > 0xFFFFFFFFU) + if ((mz_uint64)(source_len | *pDest_len) > 0xFFFFFFFFU) return MZ_PARAM_ERROR; stream.next_in = pSource; @@ -354,6 +356,10 @@ mz_ulong mz_compressBound(mz_ulong source_len) return mz_deflateBound(NULL, source_len); } +#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/ + +#ifndef MINIZ_NO_INFLATE_APIS + typedef struct { tinfl_decompressor m_decomp; @@ -560,7 +566,7 @@ int mz_uncompress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned cha memset(&stream, 0, sizeof(stream)); /* In case mz_ulong is 64-bits (argh I hate longs). */ - if ((*pSource_len | *pDest_len) > 0xFFFFFFFFU) + if ((mz_uint64)(*pSource_len | *pDest_len) > 0xFFFFFFFFU) return MZ_PARAM_ERROR; stream.next_in = pSource; @@ -589,6 +595,8 @@ int mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char return mz_uncompress2(pDest, pDest_len, pSource, &source_len); } +#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ + const char *mz_error(int err) { static struct @@ -666,6 +674,8 @@ const char *mz_error(int err) +#ifndef MINIZ_NO_DEFLATE_APIS + #ifdef __cplusplus extern "C" { #endif @@ -744,7 +754,7 @@ static tdefl_sym_freq *tdefl_radix_sort_syms(mz_uint num_syms, tdefl_sym_freq *p { mz_uint32 total_passes = 2, pass_shift, pass, i, hist[256 * 2]; tdefl_sym_freq *pCur_syms = pSyms0, *pNew_syms = pSyms1; - MZ_CLEAR_OBJ(hist); + MZ_CLEAR_ARR(hist); for (i = 0; i < num_syms; i++) { mz_uint freq = pSyms0[i].m_key; @@ -862,7 +872,7 @@ static void tdefl_optimize_huffman_table(tdefl_compressor *d, int table_num, int { int i, j, l, num_codes[1 + TDEFL_MAX_SUPPORTED_HUFF_CODESIZE]; mz_uint next_code[TDEFL_MAX_SUPPORTED_HUFF_CODESIZE + 1]; - MZ_CLEAR_OBJ(num_codes); + MZ_CLEAR_ARR(num_codes); if (static_table) { for (i = 0; i < table_len; i++) @@ -888,8 +898,8 @@ static void tdefl_optimize_huffman_table(tdefl_compressor *d, int table_num, int tdefl_huffman_enforce_max_code_size(num_codes, num_used_syms, code_size_limit); - MZ_CLEAR_OBJ(d->m_huff_code_sizes[table_num]); - MZ_CLEAR_OBJ(d->m_huff_codes[table_num]); + MZ_CLEAR_ARR(d->m_huff_code_sizes[table_num]); + MZ_CLEAR_ARR(d->m_huff_codes[table_num]); for (i = 1, j = num_used_syms; i <= code_size_limit; i++) for (l = num_codes[i]; l > 0; l--) d->m_huff_code_sizes[table_num][pSyms[--j].m_sym_index] = (mz_uint8)(i); @@ -975,7 +985,7 @@ static void tdefl_optimize_huffman_table(tdefl_compressor *d, int table_num, int } \ } -static mz_uint8 s_tdefl_packed_code_size_syms_swizzle[] = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; +static const mz_uint8 s_tdefl_packed_code_size_syms_swizzle[] = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; static void tdefl_start_dynamic_block(tdefl_compressor *d) { @@ -1113,7 +1123,8 @@ static mz_bool tdefl_compress_lz_codes(tdefl_compressor *d) if (flags & 1) { mz_uint s0, s1, n0, n1, sym, num_extra_bits; - mz_uint match_len = pLZ_codes[0], match_dist = *(const mz_uint16 *)(pLZ_codes + 1); + mz_uint match_len = pLZ_codes[0]; + mz_uint match_dist = (pLZ_codes[1] | (pLZ_codes[2] << 8)); pLZ_codes += 3; MZ_ASSERT(d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]); @@ -1158,7 +1169,7 @@ static mz_bool tdefl_compress_lz_codes(tdefl_compressor *d) if (pOutput_buf >= d->m_pOutput_buf_end) return MZ_FALSE; - *(mz_uint64 *)pOutput_buf = bit_buffer; + memcpy(pOutput_buf, &bit_buffer, sizeof(mz_uint64)); pOutput_buf += (bits_in >> 3); bit_buffer >>= (bits_in & ~7); bits_in &= 7; @@ -1240,6 +1251,8 @@ static mz_bool tdefl_compress_block(tdefl_compressor *d, mz_bool static_block) return tdefl_compress_lz_codes(d); } +static const mz_uint s_tdefl_num_probes[11]; + static int tdefl_flush_block(tdefl_compressor *d, int flush) { mz_uint saved_bit_buf, saved_bits_in; @@ -1260,8 +1273,27 @@ static int tdefl_flush_block(tdefl_compressor *d, int flush) if ((d->m_flags & TDEFL_WRITE_ZLIB_HEADER) && (!d->m_block_index)) { - TDEFL_PUT_BITS(0x78, 8); - TDEFL_PUT_BITS(0x01, 8); + const mz_uint8 cmf = 0x78; + mz_uint8 flg, flevel = 3; + mz_uint header, i, mz_un = sizeof(s_tdefl_num_probes) / sizeof(mz_uint); + + /* Determine compression level by reversing the process in tdefl_create_comp_flags_from_zip_params() */ + for (i = 0; i < mz_un; i++) + if (s_tdefl_num_probes[i] == (d->m_flags & 0xFFF)) break; + + if (i < 2) + flevel = 0; + else if (i < 6) + flevel = 1; + else if (i == 6) + flevel = 2; + + header = cmf << 8 | (flevel << 6); + header += 31 - (header % 31); + flg = header & 0xFF; + + TDEFL_PUT_BITS(cmf, 8); + TDEFL_PUT_BITS(flg, 8); } TDEFL_PUT_BITS(flush == TDEFL_FINISH, 1); @@ -1732,7 +1764,7 @@ static mz_bool tdefl_compress_normal(tdefl_compressor *d) mz_uint dst_pos = (d->m_lookahead_pos + d->m_lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK, ins_pos = d->m_lookahead_pos + d->m_lookahead_size - 2; mz_uint hash = (d->m_dict[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] << TDEFL_LZ_HASH_SHIFT) ^ d->m_dict[(ins_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK]; mz_uint num_bytes_to_process = (mz_uint)MZ_MIN(src_buf_left, TDEFL_MAX_MATCH_LEN - d->m_lookahead_size); - const mz_uint8 *pSrc_end = pSrc + num_bytes_to_process; + const mz_uint8 *pSrc_end = pSrc ? pSrc + num_bytes_to_process : NULL; src_buf_left -= num_bytes_to_process; d->m_lookahead_size += num_bytes_to_process; while (pSrc != pSrc_end) @@ -1942,8 +1974,8 @@ tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pI d->m_finished = (flush == TDEFL_FINISH); if (flush == TDEFL_FULL_FLUSH) { - MZ_CLEAR_OBJ(d->m_hash); - MZ_CLEAR_OBJ(d->m_next); + MZ_CLEAR_ARR(d->m_hash); + MZ_CLEAR_ARR(d->m_next); d->m_dict_size = 0; } } @@ -1966,7 +1998,7 @@ tdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_fun d->m_greedy_parsing = (flags & TDEFL_GREEDY_PARSING_FLAG) != 0; d->m_max_probes[1] = 1 + (((flags & 0xFFF) >> 2) + 2) / 3; if (!(flags & TDEFL_NONDETERMINISTIC_PARSING_FLAG)) - MZ_CLEAR_OBJ(d->m_hash); + MZ_CLEAR_ARR(d->m_hash); d->m_lookahead_pos = d->m_lookahead_size = d->m_dict_size = d->m_total_lz_bytes = d->m_lz_code_buf_dict_pos = d->m_bits_in = 0; d->m_output_flush_ofs = d->m_output_flush_remaining = d->m_finished = d->m_block_index = d->m_bit_buffer = d->m_wants_to_finish = 0; d->m_pLZ_code_buf = d->m_lz_code_buf + 1; @@ -1987,7 +2019,7 @@ tdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_fun d->m_src_buf_left = 0; d->m_out_buf_ofs = 0; if (!(flags & TDEFL_NONDETERMINISTIC_PARSING_FLAG)) - MZ_CLEAR_OBJ(d->m_dict); + MZ_CLEAR_ARR(d->m_dict); memset(&d->m_huff_count[0][0], 0, sizeof(d->m_huff_count[0][0]) * TDEFL_MAX_HUFF_SYMBOLS_0); memset(&d->m_huff_count[1][0], 0, sizeof(d->m_huff_count[1][0]) * TDEFL_MAX_HUFF_SYMBOLS_1); return TDEFL_STATUS_OKAY; @@ -2197,7 +2229,7 @@ void *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, /* Allocate the tdefl_compressor and tinfl_decompressor structures in C so that */ /* non-C language bindings to tdefL_ and tinfl_ API don't need to worry about */ /* structure size and allocation mechanism. */ -tdefl_compressor *tdefl_compressor_alloc() +tdefl_compressor *tdefl_compressor_alloc(void) { return (tdefl_compressor *)MZ_MALLOC(sizeof(tdefl_compressor)); } @@ -2215,6 +2247,8 @@ void tdefl_compressor_free(tdefl_compressor *pComp) #ifdef __cplusplus } #endif + +#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/ /************************************************************************** * * Copyright 2013-2014 RAD Game Tools and Valve Software @@ -2243,6 +2277,8 @@ void tdefl_compressor_free(tdefl_compressor *pComp) +#ifndef MINIZ_NO_INFLATE_APIS + #ifdef __cplusplus extern "C" { #endif @@ -2323,10 +2359,10 @@ extern "C" { /* It reads just enough bytes from the input stream that are needed to decode the next Huffman code (and absolutely no more). It works by trying to fully decode a */ /* Huffman code by using whatever bits are currently present in the bit buffer. If this fails, it reads another byte, and tries again until it succeeds or until the */ /* bit buffer contains >=15 bits (deflate's max. Huffman code size). */ -#define TINFL_HUFF_BITBUF_FILL(state_index, pHuff) \ +#define TINFL_HUFF_BITBUF_FILL(state_index, pLookUp, pTree) \ do \ { \ - temp = (pHuff)->m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]; \ + temp = pLookUp[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]; \ if (temp >= 0) \ { \ code_len = temp >> 9; \ @@ -2338,7 +2374,7 @@ extern "C" { code_len = TINFL_FAST_LOOKUP_BITS; \ do \ { \ - temp = (pHuff)->m_tree[~temp + ((bit_buf >> code_len++) & 1)]; \ + temp = pTree[~temp + ((bit_buf >> code_len++) & 1)]; \ } while ((temp < 0) && (num_bits >= (code_len + 1))); \ if (temp >= 0) \ break; \ @@ -2354,7 +2390,7 @@ extern "C" { /* The slow path is only executed at the very end of the input buffer. */ /* v1.16: The original macro handled the case at the very end of the passed-in input buffer, but we also need to handle the case where the user passes in 1+zillion bytes */ /* following the deflate data and our non-conservative read-ahead path won't kick in here on this code. This is much trickier. */ -#define TINFL_HUFF_DECODE(state_index, sym, pHuff) \ +#define TINFL_HUFF_DECODE(state_index, sym, pLookUp, pTree) \ do \ { \ int temp; \ @@ -2363,7 +2399,7 @@ extern "C" { { \ if ((pIn_buf_end - pIn_buf_cur) < 2) \ { \ - TINFL_HUFF_BITBUF_FILL(state_index, pHuff); \ + TINFL_HUFF_BITBUF_FILL(state_index, pLookUp, pTree); \ } \ else \ { \ @@ -2372,14 +2408,14 @@ extern "C" { num_bits += 16; \ } \ } \ - if ((temp = (pHuff)->m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) \ + if ((temp = pLookUp[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) \ code_len = temp >> 9, temp &= 511; \ else \ { \ code_len = TINFL_FAST_LOOKUP_BITS; \ do \ { \ - temp = (pHuff)->m_tree[~temp + ((bit_buf >> code_len++) & 1)]; \ + temp = pTree[~temp + ((bit_buf >> code_len++) & 1)]; \ } while (temp < 0); \ } \ sym = temp; \ @@ -2388,20 +2424,33 @@ extern "C" { } \ MZ_MACRO_END +static void tinfl_clear_tree(tinfl_decompressor *r) +{ + if (r->m_type == 0) + MZ_CLEAR_ARR(r->m_tree_0); + else if (r->m_type == 1) + MZ_CLEAR_ARR(r->m_tree_1); + else + MZ_CLEAR_ARR(r->m_tree_2); +} + tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags) { - static const int s_length_base[31] = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0 }; - static const int s_length_extra[31] = { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 }; - static const int s_dist_base[32] = { 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, 0, 0 }; - static const int s_dist_extra[32] = { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 }; + static const mz_uint16 s_length_base[31] = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0 }; + static const mz_uint8 s_length_extra[31] = { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 }; + static const mz_uint16 s_dist_base[32] = { 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, 0, 0 }; + static const mz_uint8 s_dist_extra[32] = { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 }; static const mz_uint8 s_length_dezigzag[19] = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; - static const int s_min_table_sizes[3] = { 257, 1, 4 }; + static const mz_uint16 s_min_table_sizes[3] = { 257, 1, 4 }; + + mz_int16 *pTrees[3]; + mz_uint8 *pCode_sizes[3]; tinfl_status status = TINFL_STATUS_FAILED; mz_uint32 num_bits, dist, counter, num_extra; tinfl_bit_buf_t bit_buf; const mz_uint8 *pIn_buf_cur = pIn_buf_next, *const pIn_buf_end = pIn_buf_next + *pIn_buf_size; - mz_uint8 *pOut_buf_cur = pOut_buf_next, *const pOut_buf_end = pOut_buf_next + *pOut_buf_size; + mz_uint8 *pOut_buf_cur = pOut_buf_next, *const pOut_buf_end = pOut_buf_next ? pOut_buf_next + *pOut_buf_size : NULL; size_t out_buf_size_mask = (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF) ? (size_t)-1 : ((pOut_buf_next - pOut_buf_start) + *pOut_buf_size) - 1, dist_from_out_buf_start; /* Ensure the output buffer's size is a power of 2, unless the output buffer is large enough to hold the entire output file (in which case it doesn't matter). */ @@ -2411,6 +2460,13 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex return TINFL_STATUS_BAD_PARAM; } + pTrees[0] = r->m_tree_0; + pTrees[1] = r->m_tree_1; + pTrees[2] = r->m_tree_2; + pCode_sizes[0] = r->m_code_size_0; + pCode_sizes[1] = r->m_code_size_1; + pCode_sizes[2] = r->m_code_size_2; + num_bits = r->m_num_bits; bit_buf = r->m_bit_buf; dist = r->m_dist; @@ -2427,7 +2483,7 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex TINFL_GET_BYTE(2, r->m_zhdr1); counter = (((r->m_zhdr0 * 256 + r->m_zhdr1) % 31 != 0) || (r->m_zhdr1 & 32) || ((r->m_zhdr0 & 15) != 8)); if (!(decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF)) - counter |= (((1U << (8U + (r->m_zhdr0 >> 4))) > 32768U) || ((out_buf_size_mask + 1) < (size_t)(1U << (8U + (r->m_zhdr0 >> 4))))); + counter |= (((1U << (8U + (r->m_zhdr0 >> 4))) > 32768U) || ((out_buf_size_mask + 1) < (size_t)((size_t)1 << (8U + (r->m_zhdr0 >> 4))))); if (counter) { TINFL_CR_RETURN_FOREVER(36, TINFL_STATUS_FAILED); @@ -2488,11 +2544,11 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex { if (r->m_type == 1) { - mz_uint8 *p = r->m_tables[0].m_code_size; + mz_uint8 *p = r->m_code_size_0; mz_uint i; r->m_table_sizes[0] = 288; r->m_table_sizes[1] = 32; - TINFL_MEMSET(r->m_tables[1].m_code_size, 5, 32); + TINFL_MEMSET(r->m_code_size_1, 5, 32); for (i = 0; i <= 143; ++i) *p++ = 8; for (; i <= 255; ++i) @@ -2509,26 +2565,30 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex TINFL_GET_BITS(11, r->m_table_sizes[counter], "\05\05\04"[counter]); r->m_table_sizes[counter] += s_min_table_sizes[counter]; } - MZ_CLEAR_OBJ(r->m_tables[2].m_code_size); + MZ_CLEAR_ARR(r->m_code_size_2); for (counter = 0; counter < r->m_table_sizes[2]; counter++) { mz_uint s; TINFL_GET_BITS(14, s, 3); - r->m_tables[2].m_code_size[s_length_dezigzag[counter]] = (mz_uint8)s; + r->m_code_size_2[s_length_dezigzag[counter]] = (mz_uint8)s; } r->m_table_sizes[2] = 19; } for (; (int)r->m_type >= 0; r->m_type--) { int tree_next, tree_cur; - tinfl_huff_table *pTable; + mz_int16 *pLookUp; + mz_int16 *pTree; + mz_uint8 *pCode_size; mz_uint i, j, used_syms, total, sym_index, next_code[17], total_syms[16]; - pTable = &r->m_tables[r->m_type]; - MZ_CLEAR_OBJ(total_syms); - MZ_CLEAR_OBJ(pTable->m_look_up); - MZ_CLEAR_OBJ(pTable->m_tree); + pLookUp = r->m_look_up[r->m_type]; + pTree = pTrees[r->m_type]; + pCode_size = pCode_sizes[r->m_type]; + MZ_CLEAR_ARR(total_syms); + TINFL_MEMSET(pLookUp, 0, sizeof(r->m_look_up[0])); + tinfl_clear_tree(r); for (i = 0; i < r->m_table_sizes[r->m_type]; ++i) - total_syms[pTable->m_code_size[i]]++; + total_syms[pCode_size[i]]++; used_syms = 0, total = 0; next_code[0] = next_code[1] = 0; for (i = 1; i <= 15; ++i) @@ -2542,7 +2602,7 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex } for (tree_next = -1, sym_index = 0; sym_index < r->m_table_sizes[r->m_type]; ++sym_index) { - mz_uint rev_code = 0, l, cur_code, code_size = pTable->m_code_size[sym_index]; + mz_uint rev_code = 0, l, cur_code, code_size = pCode_size[sym_index]; if (!code_size) continue; cur_code = next_code[code_size]++; @@ -2553,14 +2613,14 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex mz_int16 k = (mz_int16)((code_size << 9) | sym_index); while (rev_code < TINFL_FAST_LOOKUP_SIZE) { - pTable->m_look_up[rev_code] = k; + pLookUp[rev_code] = k; rev_code += (1 << code_size); } continue; } - if (0 == (tree_cur = pTable->m_look_up[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)])) + if (0 == (tree_cur = pLookUp[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)])) { - pTable->m_look_up[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)] = (mz_int16)tree_next; + pLookUp[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)] = (mz_int16)tree_next; tree_cur = tree_next; tree_next -= 2; } @@ -2568,24 +2628,24 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex for (j = code_size; j > (TINFL_FAST_LOOKUP_BITS + 1); j--) { tree_cur -= ((rev_code >>= 1) & 1); - if (!pTable->m_tree[-tree_cur - 1]) + if (!pTree[-tree_cur - 1]) { - pTable->m_tree[-tree_cur - 1] = (mz_int16)tree_next; + pTree[-tree_cur - 1] = (mz_int16)tree_next; tree_cur = tree_next; tree_next -= 2; } else - tree_cur = pTable->m_tree[-tree_cur - 1]; + tree_cur = pTree[-tree_cur - 1]; } tree_cur -= ((rev_code >>= 1) & 1); - pTable->m_tree[-tree_cur - 1] = (mz_int16)sym_index; + pTree[-tree_cur - 1] = (mz_int16)sym_index; } if (r->m_type == 2) { for (counter = 0; counter < (r->m_table_sizes[0] + r->m_table_sizes[1]);) { mz_uint s; - TINFL_HUFF_DECODE(16, dist, &r->m_tables[2]); + TINFL_HUFF_DECODE(16, dist, r->m_look_up[2], r->m_tree_2); if (dist < 16) { r->m_len_codes[counter++] = (mz_uint8)dist; @@ -2605,8 +2665,8 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex { TINFL_CR_RETURN_FOREVER(21, TINFL_STATUS_FAILED); } - TINFL_MEMCPY(r->m_tables[0].m_code_size, r->m_len_codes, r->m_table_sizes[0]); - TINFL_MEMCPY(r->m_tables[1].m_code_size, r->m_len_codes + r->m_table_sizes[0], r->m_table_sizes[1]); + TINFL_MEMCPY(r->m_code_size_0, r->m_len_codes, r->m_table_sizes[0]); + TINFL_MEMCPY(r->m_code_size_1, r->m_len_codes + r->m_table_sizes[0], r->m_table_sizes[1]); } } for (;;) @@ -2616,7 +2676,7 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex { if (((pIn_buf_end - pIn_buf_cur) < 4) || ((pOut_buf_end - pOut_buf_cur) < 2)) { - TINFL_HUFF_DECODE(23, counter, &r->m_tables[0]); + TINFL_HUFF_DECODE(23, counter, r->m_look_up[0], r->m_tree_0); if (counter >= 256) break; while (pOut_buf_cur >= pOut_buf_end) @@ -2644,14 +2704,14 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex num_bits += 16; } #endif - if ((sym2 = r->m_tables[0].m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) + if ((sym2 = r->m_look_up[0][bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) code_len = sym2 >> 9; else { code_len = TINFL_FAST_LOOKUP_BITS; do { - sym2 = r->m_tables[0].m_tree[~sym2 + ((bit_buf >> code_len++) & 1)]; + sym2 = r->m_tree_0[~sym2 + ((bit_buf >> code_len++) & 1)]; } while (sym2 < 0); } counter = sym2; @@ -2668,14 +2728,14 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex num_bits += 16; } #endif - if ((sym2 = r->m_tables[0].m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) + if ((sym2 = r->m_look_up[0][bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) code_len = sym2 >> 9; else { code_len = TINFL_FAST_LOOKUP_BITS; do { - sym2 = r->m_tables[0].m_tree[~sym2 + ((bit_buf >> code_len++) & 1)]; + sym2 = r->m_tree_0[~sym2 + ((bit_buf >> code_len++) & 1)]; } while (sym2 < 0); } bit_buf >>= code_len; @@ -2704,7 +2764,7 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex counter += extra_bits; } - TINFL_HUFF_DECODE(26, dist, &r->m_tables[1]); + TINFL_HUFF_DECODE(26, dist, r->m_look_up[1], r->m_tree_1); num_extra = s_dist_extra[dist]; dist = s_dist_base[dist]; if (num_extra) @@ -2789,7 +2849,7 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_nex --pIn_buf_cur; num_bits -= 8; } - bit_buf &= (tinfl_bit_buf_t)((((mz_uint64)1) << num_bits) - (mz_uint64)1); + bit_buf &= ~(~(tinfl_bit_buf_t)0 << num_bits); MZ_ASSERT(!num_bits); /* if this assert fires then we've read beyond the end of non-deflate/zlib streams with following data (such as gzip streams). */ if (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER) @@ -2821,7 +2881,7 @@ common_exit: } } r->m_num_bits = num_bits; - r->m_bit_buf = bit_buf & (tinfl_bit_buf_t)((((mz_uint64)1) << num_bits) - (mz_uint64)1); + r->m_bit_buf = bit_buf & ~(~(tinfl_bit_buf_t)0 << num_bits); r->m_dist = dist; r->m_counter = counter; r->m_num_extra = num_extra; @@ -2916,6 +2976,7 @@ int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, size_t in_buf_ofs = 0, dict_ofs = 0; if (!pDict) return TINFL_STATUS_FAILED; + memset(pDict,0,TINFL_LZ_DICT_SIZE); tinfl_init(&decomp); for (;;) { @@ -2938,7 +2999,7 @@ int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, } #ifndef MINIZ_NO_MALLOC -tinfl_decompressor *tinfl_decompressor_alloc() +tinfl_decompressor *tinfl_decompressor_alloc(void) { tinfl_decompressor *pDecomp = (tinfl_decompressor *)MZ_MALLOC(sizeof(tinfl_decompressor)); if (pDecomp) @@ -2955,6 +3016,8 @@ void tinfl_decompressor_free(tinfl_decompressor *pDecomp) #ifdef __cplusplus } #endif + +#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ /************************************************************************** * * Copyright 2013-2014 RAD Game Tools and Valve Software @@ -2997,19 +3060,48 @@ extern "C" { #include <sys/stat.h> #if defined(_MSC_VER) || defined(__MINGW64__) + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> + +static WCHAR* mz_utf8z_to_widechar(const char* str) +{ + int reqChars = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0); + WCHAR* wStr = (WCHAR*)malloc(reqChars * sizeof(WCHAR)); + MultiByteToWideChar(CP_UTF8, 0, str, -1, wStr, sizeof(WCHAR) * reqChars); + return wStr; +} + static FILE *mz_fopen(const char *pFilename, const char *pMode) { - FILE *pFile = NULL; - fopen_s(&pFile, pFilename, pMode); - return pFile; + WCHAR* wFilename = mz_utf8z_to_widechar(pFilename); + WCHAR* wMode = mz_utf8z_to_widechar(pMode); + FILE* pFile = NULL; + errno_t err = _wfopen_s(&pFile, wFilename, wMode); + free(wFilename); + free(wMode); + return err ? NULL : pFile; } + static FILE *mz_freopen(const char *pPath, const char *pMode, FILE *pStream) { - FILE *pFile = NULL; - if (freopen_s(&pFile, pPath, pMode, pStream)) - return NULL; - return pFile; + WCHAR* wPath = mz_utf8z_to_widechar(pPath); + WCHAR* wMode = mz_utf8z_to_widechar(pMode); + FILE* pFile = NULL; + errno_t err = _wfreopen_s(&pFile, wPath, wMode, pStream); + free(wPath); + free(wMode); + return err ? NULL : pFile; +} + +static int mz_stat64(const char *path, struct __stat64 *buffer) +{ + WCHAR* wPath = mz_utf8z_to_widechar(path); + int res = _wstat64(wPath, buffer); + free(wPath); + return res; } + #ifndef MINIZ_NO_TIME #include <sys/utime.h> #endif @@ -3020,11 +3112,12 @@ static FILE *mz_freopen(const char *pPath, const char *pMode, FILE *pStream) #define MZ_FTELL64 _ftelli64 #define MZ_FSEEK64 _fseeki64 #define MZ_FILE_STAT_STRUCT _stat64 -#define MZ_FILE_STAT _stat64 +#define MZ_FILE_STAT mz_stat64 #define MZ_FFLUSH fflush #define MZ_FREOPEN mz_freopen #define MZ_DELETE_FILE remove -#elif defined(__MINGW32__) + +#elif defined(__MINGW32__) || defined(__WATCOMC__) #ifndef MINIZ_NO_TIME #include <sys/utime.h> #endif @@ -3032,13 +3125,14 @@ static FILE *mz_freopen(const char *pPath, const char *pMode, FILE *pStream) #define MZ_FCLOSE fclose #define MZ_FREAD fread #define MZ_FWRITE fwrite -#define MZ_FTELL64 ftello64 -#define MZ_FSEEK64 fseeko64 -#define MZ_FILE_STAT_STRUCT _stat -#define MZ_FILE_STAT _stat +#define MZ_FTELL64 _ftelli64 +#define MZ_FSEEK64 _fseeki64 +#define MZ_FILE_STAT_STRUCT stat +#define MZ_FILE_STAT stat #define MZ_FFLUSH fflush #define MZ_FREOPEN(f, m, s) freopen(f, m, s) #define MZ_DELETE_FILE remove + #elif defined(__TINYC__) #ifndef MINIZ_NO_TIME #include <sys/utime.h> @@ -3054,6 +3148,7 @@ static FILE *mz_freopen(const char *pPath, const char *pMode, FILE *pStream) #define MZ_FFLUSH fflush #define MZ_FREOPEN(f, m, s) freopen(f, m, s) #define MZ_DELETE_FILE remove + #elif defined(__USE_LARGEFILE64) /* gcc, clang */ #ifndef MINIZ_NO_TIME #include <utime.h> @@ -3069,7 +3164,8 @@ static FILE *mz_freopen(const char *pPath, const char *pMode, FILE *pStream) #define MZ_FFLUSH fflush #define MZ_FREOPEN(p, m, s) freopen64(p, m, s) #define MZ_DELETE_FILE remove -#elif defined(__APPLE__) + +#elif defined(__APPLE__) || defined(__FreeBSD__) #ifndef MINIZ_NO_TIME #include <utime.h> #endif @@ -3215,7 +3311,7 @@ struct mz_zip_internal_state_tag mz_zip_array m_sorted_central_dir_offsets; /* The flags passed in when the archive is initially opened. */ - uint32_t m_init_flags; + mz_uint32 m_init_flags; /* MZ_TRUE if the archive has a zip64 end of central directory headers, etc. */ mz_bool m_zip64; @@ -3651,7 +3747,7 @@ static mz_bool mz_zip_reader_read_central_dir(mz_zip_archive *pZip, mz_uint flag if (((num_this_disk | cdir_disk_index) != 0) && ((num_this_disk != 1) || (cdir_disk_index != 1))) return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_MULTIDISK); - if (cdir_size < pZip->m_total_files * MZ_ZIP_CENTRAL_DIR_HEADER_SIZE) + if (cdir_size < (mz_uint64)pZip->m_total_files * MZ_ZIP_CENTRAL_DIR_HEADER_SIZE) return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED); if ((cdir_ofs + (mz_uint64)cdir_size) > pZip->m_archive_size) @@ -3802,7 +3898,7 @@ static mz_bool mz_zip_reader_read_central_dir(mz_zip_archive *pZip, mz_uint flag void mz_zip_zero_struct(mz_zip_archive *pZip) { if (pZip) - MZ_CLEAR_OBJ(*pZip); + MZ_CLEAR_PTR(pZip); } static mz_bool mz_zip_reader_end_internal(mz_zip_archive *pZip, mz_bool set_last_error) @@ -4276,7 +4372,7 @@ static mz_bool mz_zip_locate_file_binary_search(mz_zip_archive *pZip, const char const mz_zip_array *pCentral_dir_offsets = &pState->m_central_dir_offsets; const mz_zip_array *pCentral_dir = &pState->m_central_dir; mz_uint32 *pIndices = &MZ_ZIP_ARRAY_ELEMENT(&pState->m_sorted_central_dir_offsets, mz_uint32, 0); - const uint32_t size = pZip->m_total_files; + const mz_uint32 size = pZip->m_total_files; const mz_uint filename_len = (mz_uint)strlen(pFilename); if (pIndex) @@ -4291,7 +4387,7 @@ static mz_bool mz_zip_locate_file_binary_search(mz_zip_archive *pZip, const char while (l <= h) { mz_int64 m = l + ((h - l) >> 1); - uint32_t file_index = pIndices[(uint32_t)m]; + mz_uint32 file_index = pIndices[(mz_uint32)m]; int comp = mz_zip_filename_compare(pCentral_dir, pCentral_dir_offsets, file_index, pFilename, filename_len); if (!comp) @@ -4384,7 +4480,8 @@ mz_bool mz_zip_reader_locate_file_v2(mz_zip_archive *pZip, const char *pName, co return mz_zip_set_error(pZip, MZ_ZIP_FILE_NOT_FOUND); } -mz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size) +static +mz_bool mz_zip_reader_extract_to_mem_no_alloc1(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size, const mz_zip_archive_file_stat *st) { int status = TINFL_STATUS_DONE; mz_uint64 needed_size, cur_file_ofs, comp_remaining, out_buf_ofs = 0, read_buf_size, read_buf_ofs = 0, read_buf_avail; @@ -4397,6 +4494,9 @@ mz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file if ((!pZip) || (!pZip->m_pState) || ((buf_size) && (!pBuf)) || ((user_read_buf_size) && (!pUser_read_buf)) || (!pZip->m_pRead)) return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER); + if (st) { + file_stat = *st; + } else if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat)) return MZ_FALSE; @@ -4527,17 +4627,22 @@ mz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file return status == TINFL_STATUS_DONE; } +mz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size) +{ + return mz_zip_reader_extract_to_mem_no_alloc1(pZip, file_index, pBuf, buf_size, flags, pUser_read_buf, user_read_buf_size, NULL); +} + mz_bool mz_zip_reader_extract_file_to_mem_no_alloc(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size) { mz_uint32 file_index; if (!mz_zip_reader_locate_file_v2(pZip, pFilename, NULL, flags, &file_index)) return MZ_FALSE; - return mz_zip_reader_extract_to_mem_no_alloc(pZip, file_index, pBuf, buf_size, flags, pUser_read_buf, user_read_buf_size); + return mz_zip_reader_extract_to_mem_no_alloc1(pZip, file_index, pBuf, buf_size, flags, pUser_read_buf, user_read_buf_size, NULL); } mz_bool mz_zip_reader_extract_to_mem(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags) { - return mz_zip_reader_extract_to_mem_no_alloc(pZip, file_index, pBuf, buf_size, flags, NULL, 0); + return mz_zip_reader_extract_to_mem_no_alloc1(pZip, file_index, pBuf, buf_size, flags, NULL, 0, NULL); } mz_bool mz_zip_reader_extract_file_to_mem(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags) @@ -4547,23 +4652,17 @@ mz_bool mz_zip_reader_extract_file_to_mem(mz_zip_archive *pZip, const char *pFil void *mz_zip_reader_extract_to_heap(mz_zip_archive *pZip, mz_uint file_index, size_t *pSize, mz_uint flags) { - mz_uint64 comp_size, uncomp_size, alloc_size; - const mz_uint8 *p = mz_zip_get_cdh(pZip, file_index); + mz_zip_archive_file_stat file_stat; + mz_uint64 alloc_size; void *pBuf; if (pSize) *pSize = 0; - if (!p) - { - mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER); + if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat)) return NULL; - } - - comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS); - uncomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS); - alloc_size = (flags & MZ_ZIP_FLAG_COMPRESSED_DATA) ? comp_size : uncomp_size; + alloc_size = (flags & MZ_ZIP_FLAG_COMPRESSED_DATA) ? file_stat.m_comp_size : file_stat.m_uncomp_size; if (((sizeof(size_t) == sizeof(mz_uint32))) && (alloc_size > 0x7FFFFFFF)) { mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR); @@ -4576,7 +4675,7 @@ void *mz_zip_reader_extract_to_heap(mz_zip_archive *pZip, mz_uint file_index, si return NULL; } - if (!mz_zip_reader_extract_to_mem(pZip, file_index, pBuf, (size_t)alloc_size, flags)) + if (!mz_zip_reader_extract_to_mem_no_alloc1(pZip, file_index, pBuf, (size_t)alloc_size, flags, NULL, 0, &file_stat)) { pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf); return NULL; @@ -5037,7 +5136,7 @@ size_t mz_zip_reader_extract_iter_read(mz_zip_reader_extract_iter_state* pState, size_t to_copy = MZ_MIN( (buf_size - copied_to_caller), pState->out_blk_remain ); /* Copy data to caller's buffer */ - memcpy( (uint8_t*)pvBuf + copied_to_caller, pWrite_buf_cur, to_copy ); + memcpy( (mz_uint8*)pvBuf + copied_to_caller, pWrite_buf_cur, to_copy ); #ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS /* Perform CRC */ @@ -5406,7 +5505,7 @@ handle_failure: mz_bool mz_zip_validate_archive(mz_zip_archive *pZip, mz_uint flags) { mz_zip_internal_state *pState; - uint32_t i; + mz_uint32 i; if ((!pZip) || (!pZip->m_pState) || (!pZip->m_pAlloc) || (!pZip->m_pFree) || (!pZip->m_pRead)) return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER); @@ -5424,9 +5523,6 @@ mz_bool mz_zip_validate_archive(mz_zip_archive *pZip, mz_uint flags) } else { - if (pZip->m_total_files >= MZ_UINT32_MAX) - return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE); - if (pState->m_central_dir.m_size >= MZ_UINT32_MAX) return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE); } @@ -5788,7 +5884,7 @@ mz_bool mz_zip_writer_init_file_v2(mz_zip_archive *pZip, const char *pFilename, mz_uint64 cur_ofs = 0; char buf[4096]; - MZ_CLEAR_OBJ(buf); + MZ_CLEAR_ARR(buf); do { @@ -6151,7 +6247,7 @@ mz_bool mz_zip_writer_add_mem_ex_v2(mz_zip_archive *pZip, const char *pArchive_n pState->m_zip64 = MZ_TRUE; /*return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES); */ } - if ((buf_size > 0xFFFFFFFF) || (uncomp_size > 0xFFFFFFFF)) + if (((mz_uint64)buf_size > 0xFFFFFFFF) || (uncomp_size > 0xFFFFFFFF)) { pState->m_zip64 = MZ_TRUE; /*return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE); */ @@ -6244,7 +6340,7 @@ mz_bool mz_zip_writer_add_mem_ex_v2(mz_zip_archive *pZip, const char *pArchive_n } cur_archive_file_ofs += num_alignment_padding_bytes; - MZ_CLEAR_OBJ(local_dir_header); + MZ_CLEAR_ARR(local_dir_header); if (!store_data_uncompressed || (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) { @@ -6394,7 +6490,7 @@ mz_bool mz_zip_writer_add_mem_ex_v2(mz_zip_archive *pZip, const char *pArchive_n mz_bool mz_zip_writer_add_read_buf_callback(mz_zip_archive *pZip, const char *pArchive_name, mz_file_read_func read_callback, void* callback_opaque, mz_uint64 max_size, const MZ_TIME_T *pFile_time, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, const char *user_extra_data, mz_uint user_extra_data_len, const char *user_extra_data_central, mz_uint user_extra_data_central_len) { - mz_uint16 gen_flags = (level_and_flags & MZ_ZIP_FLAG_WRITE_HEADER_SET_SIZE) ? 0 : MZ_ZIP_LDH_BIT_FLAG_HAS_LOCATOR; + mz_uint16 gen_flags; mz_uint uncomp_crc32 = MZ_CRC32_INIT, level, num_alignment_padding_bytes; mz_uint16 method = 0, dos_time = 0, dos_date = 0, ext_attributes = 0; mz_uint64 local_dir_header_ofs, cur_archive_file_ofs = pZip->m_archive_size, uncomp_size = 0, comp_size = 0; @@ -6406,13 +6502,15 @@ mz_bool mz_zip_writer_add_read_buf_callback(mz_zip_archive *pZip, const char *pA mz_zip_internal_state *pState; mz_uint64 file_ofs = 0, cur_archive_header_file_ofs; - if (!(level_and_flags & MZ_ZIP_FLAG_ASCII_FILENAME)) - gen_flags |= MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_UTF8; - if ((int)level_and_flags < 0) level_and_flags = MZ_DEFAULT_LEVEL; level = level_and_flags & 0xF; + gen_flags = (level_and_flags & MZ_ZIP_FLAG_WRITE_HEADER_SET_SIZE) ? 0 : MZ_ZIP_LDH_BIT_FLAG_HAS_LOCATOR; + + if (!(level_and_flags & MZ_ZIP_FLAG_ASCII_FILENAME)) + gen_flags |= MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_UTF8; + /* Sanity checks */ if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) || (!pArchive_name) || ((comment_size) && (!pComment)) || (level > MZ_UBER_COMPRESSION)) return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER); @@ -6497,7 +6595,7 @@ mz_bool mz_zip_writer_add_read_buf_callback(mz_zip_archive *pZip, const char *pA method = MZ_DEFLATED; } - MZ_CLEAR_OBJ(local_dir_header); + MZ_CLEAR_ARR(local_dir_header); if (pState->m_zip64) { if (max_size >= MZ_UINT32_MAX || local_dir_header_ofs >= MZ_UINT32_MAX) @@ -6801,7 +6899,7 @@ mz_bool mz_zip_writer_add_file(mz_zip_archive *pZip, const char *pArchive_name, } #endif /* #ifndef MINIZ_NO_STDIO */ -static mz_bool mz_zip_writer_update_zip64_extension_block(mz_zip_array *pNew_ext, mz_zip_archive *pZip, const mz_uint8 *pExt, uint32_t ext_len, mz_uint64 *pComp_size, mz_uint64 *pUncomp_size, mz_uint64 *pLocal_header_ofs, mz_uint32 *pDisk_start) +static mz_bool mz_zip_writer_update_zip64_extension_block(mz_zip_array *pNew_ext, mz_zip_archive *pZip, const mz_uint8 *pExt, mz_uint32 ext_len, mz_uint64 *pComp_size, mz_uint64 *pUncomp_size, mz_uint64 *pLocal_header_ofs, mz_uint32 *pDisk_start) { /* + 64 should be enough for any new zip64 data */ if (!mz_zip_array_reserve(pZip, pNew_ext, ext_len + 64, MZ_FALSE)) @@ -7117,10 +7215,10 @@ mz_bool mz_zip_writer_add_from_zip_reader(mz_zip_archive *pZip, mz_zip_archive * if (pZip->m_pState->m_zip64) { /* dest is zip64, so upgrade the data descriptor */ - const mz_uint32 *pSrc_descriptor = (const mz_uint32 *)((const mz_uint8 *)pBuf + (has_id ? sizeof(mz_uint32) : 0)); - const mz_uint32 src_crc32 = pSrc_descriptor[0]; - const mz_uint64 src_comp_size = pSrc_descriptor[1]; - const mz_uint64 src_uncomp_size = pSrc_descriptor[2]; + const mz_uint8 *pSrc_descriptor = (const mz_uint8 *)pBuf + (has_id ? sizeof(mz_uint32) : 0); + const mz_uint32 src_crc32 = MZ_READ_LE32(pSrc_descriptor); + const mz_uint64 src_comp_size = MZ_READ_LE32(pSrc_descriptor + sizeof(mz_uint32)); + const mz_uint64 src_uncomp_size = MZ_READ_LE32(pSrc_descriptor + 2*sizeof(mz_uint32)); mz_write_le32((mz_uint8 *)pBuf, MZ_ZIP_DATA_DESCRIPTOR_ID); mz_write_le32((mz_uint8 *)pBuf + sizeof(mz_uint32) * 1, src_crc32); @@ -7256,7 +7354,7 @@ mz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip) if (pState->m_zip64) { - if ((pZip->m_total_files > MZ_UINT32_MAX) || (pState->m_central_dir.m_size >= MZ_UINT32_MAX)) + if ((mz_uint64)pState->m_central_dir.m_size >= MZ_UINT32_MAX) return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES); } else @@ -7284,7 +7382,7 @@ mz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip) /* Write zip64 end of central directory header */ mz_uint64 rel_ofs_to_zip64_ecdr = pZip->m_archive_size; - MZ_CLEAR_OBJ(hdr); + MZ_CLEAR_ARR(hdr); MZ_WRITE_LE32(hdr + MZ_ZIP64_ECDH_SIG_OFS, MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIG); MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_SIZE_OF_RECORD_OFS, MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE - sizeof(mz_uint32) - sizeof(mz_uint64)); MZ_WRITE_LE16(hdr + MZ_ZIP64_ECDH_VERSION_MADE_BY_OFS, 0x031E); /* TODO: always Unix */ @@ -7299,7 +7397,7 @@ mz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip) pZip->m_archive_size += MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE; /* Write zip64 end of central directory locator */ - MZ_CLEAR_OBJ(hdr); + MZ_CLEAR_ARR(hdr); MZ_WRITE_LE32(hdr + MZ_ZIP64_ECDL_SIG_OFS, MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIG); MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDL_REL_OFS_TO_ZIP64_ECDR_OFS, rel_ofs_to_zip64_ecdr); MZ_WRITE_LE32(hdr + MZ_ZIP64_ECDL_TOTAL_NUMBER_OF_DISKS_OFS, 1); @@ -7310,7 +7408,7 @@ mz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip) } /* Write end of central directory record */ - MZ_CLEAR_OBJ(hdr); + MZ_CLEAR_ARR(hdr); MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_SIG_OFS, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG); MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS, MZ_MIN(MZ_UINT16_MAX, pZip->m_total_files)); MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS, MZ_MIN(MZ_UINT16_MAX, pZip->m_total_files)); @@ -7626,7 +7724,9 @@ const char *mz_zip_get_error_string(mz_zip_error mz_err) case MZ_ZIP_VALIDATION_FAILED: return "validation failed"; case MZ_ZIP_WRITE_CALLBACK_FAILED: - return "write calledback failed"; + return "write callback failed"; + case MZ_ZIP_TOTAL_ERRORS: + return "total errors"; default: break; } diff --git a/source/luametatex/source/libraries/miniz/miniz.h b/source/luametatex/source/libraries/miniz/miniz.h index 0e65e38b1..35c740c72 100644 --- a/source/luametatex/source/libraries/miniz/miniz.h +++ b/source/luametatex/source/libraries/miniz/miniz.h @@ -1,5 +1,7 @@ +#ifndef MINIZ_EXPORT #define MINIZ_EXPORT -/* miniz.c 2.2.0 - public domain deflate/inflate, zlib-subset, ZIP reading/writing/appending, PNG writing +#endif +/* miniz.c 3.0.0 - public domain deflate/inflate, zlib-subset, ZIP reading/writing/appending, PNG writing See "unlicense" statement at the end of this file. Rich Geldreich <richgel99@gmail.com>, last updated Oct. 13, 2013 Implements RFC 1950: http://www.ietf.org/rfc/rfc1950.txt and RFC 1951: http://www.ietf.org/rfc/rfc1951.txt @@ -115,8 +117,8 @@ -/* Defines to completely disable specific portions of miniz.c: - If all macros here are defined the only functionality remaining will be CRC-32, adler-32, tinfl, and tdefl. */ +/* Defines to completely disable specific portions of miniz.c: + If all macros here are defined the only functionality remaining will be CRC-32 and adler-32. */ /* Define MINIZ_NO_STDIO to disable all usage and any functions which rely on stdio for file I/O. */ /*#define MINIZ_NO_STDIO */ @@ -126,6 +128,12 @@ /* The current downside is the times written to your archives will be from 1979. */ /*#define MINIZ_NO_TIME */ +/* Define MINIZ_NO_DEFLATE_APIS to disable all compression API's. */ +/*#define MINIZ_NO_DEFLATE_APIS */ + +/* Define MINIZ_NO_INFLATE_APIS to disable all decompression API's. */ +/*#define MINIZ_NO_INFLATE_APIS */ + /* Define MINIZ_NO_ARCHIVE_APIS to disable all ZIP archive API's. */ /*#define MINIZ_NO_ARCHIVE_APIS */ @@ -138,12 +146,20 @@ /* Define MINIZ_NO_ZLIB_COMPATIBLE_NAME to disable zlib names, to prevent conflicts against stock zlib. */ /*#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES */ -/* Define MINIZ_NO_MALLOC to disable all calls to malloc, free, and realloc. +/* Define MINIZ_NO_MALLOC to disable all calls to malloc, free, and realloc. Note if MINIZ_NO_MALLOC is defined then the user must always provide custom user alloc/free/realloc callbacks to the zlib and archive API's, and a few stand-alone helper API's which don't provide custom user functions (such as tdefl_compress_mem_to_heap() and tinfl_decompress_mem_to_heap()) won't work. */ /*#define MINIZ_NO_MALLOC */ +#ifdef MINIZ_NO_INFLATE_APIS +#define MINIZ_NO_ARCHIVE_APIS +#endif + +#ifdef MINIZ_NO_DEFLATE_APIS +#define MINIZ_NO_ARCHIVE_WRITING_APIS +#endif + #if defined(__TINYC__) && (defined(__linux) || defined(__linux__)) /* TODO: Work around "error: include file 'sys\utime.h' when compiling with tcc on Linux */ #define MINIZ_NO_TIME @@ -162,18 +178,40 @@ #define MINIZ_X86_OR_X64_CPU 0 #endif -#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || MINIZ_X86_OR_X64_CPU +/* Set MINIZ_LITTLE_ENDIAN only if not set */ +#if !defined(MINIZ_LITTLE_ENDIAN) +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) + +#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) /* Set MINIZ_LITTLE_ENDIAN to 1 if the processor is little endian. */ #define MINIZ_LITTLE_ENDIAN 1 #else #define MINIZ_LITTLE_ENDIAN 0 #endif +#else + +#if MINIZ_X86_OR_X64_CPU +#define MINIZ_LITTLE_ENDIAN 1 +#else +#define MINIZ_LITTLE_ENDIAN 0 +#endif + +#endif +#endif + +/* Using unaligned loads and stores causes errors when using UBSan */ +#if defined(__has_feature) +#if __has_feature(undefined_behavior_sanitizer) +#define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 0 +#endif +#endif + /* Set MINIZ_USE_UNALIGNED_LOADS_AND_STORES only if not set */ #if !defined(MINIZ_USE_UNALIGNED_LOADS_AND_STORES) #if MINIZ_X86_OR_X64_CPU /* Set MINIZ_USE_UNALIGNED_LOADS_AND_STORES to 1 on CPU's that permit efficient integer loads and stores from unaligned addresses. */ -#define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 1 +#define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 0 #define MINIZ_UNALIGNED_USE_MEMCPY #else #define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 0 @@ -237,10 +275,10 @@ enum MZ_DEFAULT_COMPRESSION = -1 }; -#define MZ_VERSION "10.2.0" -#define MZ_VERNUM 0xA100 -#define MZ_VER_MAJOR 10 -#define MZ_VER_MINOR 2 +#define MZ_VERSION "11.0.1" +#define MZ_VERNUM 0xB001 +#define MZ_VER_MAJOR 11 +#define MZ_VER_MINOR 1 #define MZ_VER_REVISION 0 #define MZ_VER_SUBREVISION 0 @@ -305,6 +343,8 @@ typedef mz_stream *mz_streamp; /* Returns the version string of miniz.c. */ MINIZ_EXPORT const char *mz_version(void); +#ifndef MINIZ_NO_DEFLATE_APIS + /* mz_deflateInit() initializes a compressor with default options: */ /* Parameters: */ /* pStream must point to an initialized mz_stream struct. */ @@ -357,6 +397,10 @@ MINIZ_EXPORT int mz_compress2(unsigned char *pDest, mz_ulong *pDest_len, const u /* mz_compressBound() returns a (very) conservative upper bound on the amount of data that could be generated by calling mz_compress(). */ MINIZ_EXPORT mz_ulong mz_compressBound(mz_ulong source_len); +#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/ + +#ifndef MINIZ_NO_INFLATE_APIS + /* Initializes a decompressor. */ MINIZ_EXPORT int mz_inflateInit(mz_streamp pStream); @@ -390,6 +434,7 @@ MINIZ_EXPORT int mz_inflateEnd(mz_streamp pStream); /* Returns MZ_OK on success, or one of the error codes from mz_inflate() on failure. */ MINIZ_EXPORT int mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len); MINIZ_EXPORT int mz_uncompress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong *pSource_len); +#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ /* Returns a string description of the specified error code, or NULL if the error code is invalid. */ MINIZ_EXPORT const char *mz_error(int err); @@ -440,6 +485,8 @@ typedef void *const voidpc; #define free_func mz_free_func #define internal_state mz_internal_state #define z_stream mz_stream + +#ifndef MINIZ_NO_DEFLATE_APIS #define deflateInit mz_deflateInit #define deflateInit2 mz_deflateInit2 #define deflateReset mz_deflateReset @@ -449,6 +496,9 @@ typedef void *const voidpc; #define compress mz_compress #define compress2 mz_compress2 #define compressBound mz_compressBound +#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/ + +#ifndef MINIZ_NO_INFLATE_APIS #define inflateInit mz_inflateInit #define inflateInit2 mz_inflateInit2 #define inflateReset mz_inflateReset @@ -456,6 +506,8 @@ typedef void *const voidpc; #define inflateEnd mz_inflateEnd #define uncompress mz_uncompress #define uncompress2 mz_uncompress2 +#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ + #define crc32 mz_crc32 #define adler32 mz_adler32 #define MAX_WBITS 15 @@ -519,7 +571,8 @@ typedef int mz_bool; #ifdef MINIZ_NO_TIME typedef struct mz_dummy_time_t_tag { - int m_dummy; + mz_uint32 m_dummy1; + mz_uint32 m_dummy2; } mz_dummy_time_t; #define MZ_TIME_T mz_dummy_time_t #else @@ -541,6 +594,8 @@ typedef struct mz_dummy_time_t_tag #define MZ_MAX(a, b) (((a) > (b)) ? (a) : (b)) #define MZ_MIN(a, b) (((a) < (b)) ? (a) : (b)) #define MZ_CLEAR_OBJ(obj) memset(&(obj), 0, sizeof(obj)) +#define MZ_CLEAR_ARR(obj) memset((obj), 0, sizeof(obj)) +#define MZ_CLEAR_PTR(obj) memset((obj), 0, sizeof(*obj)) #if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN #define MZ_READ_LE16(p) *((const mz_uint16 *)(p)) @@ -577,6 +632,8 @@ extern MINIZ_EXPORT void *miniz_def_realloc_func(void *opaque, void *address, si #pragma once +#ifndef MINIZ_NO_DEFLATE_APIS + #ifdef __cplusplus extern "C" { #endif @@ -764,10 +821,14 @@ MINIZ_EXPORT void tdefl_compressor_free(tdefl_compressor *pComp); #ifdef __cplusplus } #endif + +#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/ #pragma once /* ------------------- Low-level Decompression API Definitions */ +#ifndef MINIZ_NO_INFLATE_APIS + #ifdef __cplusplus extern "C" { #endif @@ -876,12 +937,6 @@ enum TINFL_FAST_LOOKUP_SIZE = 1 << TINFL_FAST_LOOKUP_BITS }; -typedef struct -{ - mz_uint8 m_code_size[TINFL_MAX_HUFF_SYMBOLS_0]; - mz_int16 m_look_up[TINFL_FAST_LOOKUP_SIZE], m_tree[TINFL_MAX_HUFF_SYMBOLS_0 * 2]; -} tinfl_huff_table; - #if MINIZ_HAS_64BIT_REGISTERS #define TINFL_USE_64BIT_BITBUF 1 #else @@ -901,7 +956,13 @@ struct tinfl_decompressor_tag mz_uint32 m_state, m_num_bits, m_zhdr0, m_zhdr1, m_z_adler32, m_final, m_type, m_check_adler32, m_dist, m_counter, m_num_extra, m_table_sizes[TINFL_MAX_HUFF_TABLES]; tinfl_bit_buf_t m_bit_buf; size_t m_dist_from_out_buf_start; - tinfl_huff_table m_tables[TINFL_MAX_HUFF_TABLES]; + mz_int16 m_look_up[TINFL_MAX_HUFF_TABLES][TINFL_FAST_LOOKUP_SIZE]; + mz_int16 m_tree_0[TINFL_MAX_HUFF_SYMBOLS_0 * 2]; + mz_int16 m_tree_1[TINFL_MAX_HUFF_SYMBOLS_1 * 2]; + mz_int16 m_tree_2[TINFL_MAX_HUFF_SYMBOLS_2 * 2]; + mz_uint8 m_code_size_0[TINFL_MAX_HUFF_SYMBOLS_0]; + mz_uint8 m_code_size_1[TINFL_MAX_HUFF_SYMBOLS_1]; + mz_uint8 m_code_size_2[TINFL_MAX_HUFF_SYMBOLS_2]; mz_uint8 m_raw_header[4], m_len_codes[TINFL_MAX_HUFF_SYMBOLS_0 + TINFL_MAX_HUFF_SYMBOLS_1 + 137]; }; @@ -909,6 +970,8 @@ struct tinfl_decompressor_tag } #endif +#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ + #pragma once @@ -942,10 +1005,6 @@ typedef struct mz_uint16 m_bit_flag; mz_uint16 m_method; -#ifndef MINIZ_NO_TIME - MZ_TIME_T m_time; -#endif - /* CRC-32 of uncompressed data. */ mz_uint32 m_crc32; @@ -982,6 +1041,11 @@ typedef struct /* Guaranteed to be zero terminated, may be truncated to fit. */ char m_comment[MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE]; +#ifdef MINIZ_NO_TIME + MZ_TIME_T m_padding; +#else + MZ_TIME_T m_time; +#endif } mz_zip_archive_file_stat; typedef size_t (*mz_file_read_func)(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n); @@ -1093,9 +1157,7 @@ typedef struct mz_uint flags; int status; -#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS - mz_uint file_crc32; -#endif + mz_uint64 read_buf_size, read_buf_ofs, read_buf_avail, comp_remaining, out_buf_ofs, cur_file_ofs; mz_zip_archive_file_stat file_stat; void *pRead_buf; @@ -1105,6 +1167,12 @@ typedef struct tinfl_decompressor inflator; +#ifdef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS + mz_uint padding; +#else + mz_uint file_crc32; +#endif + } mz_zip_reader_extract_iter_state; /* -------- ZIP reading */ @@ -1228,9 +1296,9 @@ MINIZ_EXPORT mz_bool mz_zip_reader_extract_file_to_cfile(mz_zip_archive *pZip, c /* TODO */ typedef void *mz_zip_streaming_extract_state_ptr; mz_zip_streaming_extract_state_ptr mz_zip_streaming_extract_begin(mz_zip_archive *pZip, mz_uint file_index, mz_uint flags); - uint64_t mz_zip_streaming_extract_get_size(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState); - uint64_t mz_zip_streaming_extract_get_cur_ofs(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState); - mz_bool mz_zip_streaming_extract_seek(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState, uint64_t new_ofs); + mz_uint64 mz_zip_streaming_extract_get_size(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState); + mz_uint64 mz_zip_streaming_extract_get_cur_ofs(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState); + mz_bool mz_zip_streaming_extract_seek(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState, mz_uint64 new_ofs); size_t mz_zip_streaming_extract_read(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState, void *pBuf, size_t buf_size); mz_bool mz_zip_streaming_extract_end(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState); #endif @@ -1244,7 +1312,9 @@ MINIZ_EXPORT mz_bool mz_zip_validate_archive(mz_zip_archive *pZip, mz_uint flags /* Misc utils/helpers, valid for ZIP reading or writing */ MINIZ_EXPORT mz_bool mz_zip_validate_mem_archive(const void *pMem, size_t size, mz_uint flags, mz_zip_error *pErr); +#ifndef MINIZ_NO_STDIO MINIZ_EXPORT mz_bool mz_zip_validate_file_archive(const char *pFilename, mz_uint flags, mz_zip_error *pErr); +#endif /* Universal end function - calls either mz_zip_reader_end() or mz_zip_writer_end(). */ MINIZ_EXPORT mz_bool mz_zip_end(mz_zip_archive *pZip); @@ -1318,7 +1388,7 @@ MINIZ_EXPORT mz_bool mz_zip_writer_add_from_zip_reader(mz_zip_archive *pZip, mz_ /* An archive must be manually finalized by calling this function for it to be valid. */ MINIZ_EXPORT mz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip); -/* Finalizes a heap archive, returning a poiner to the heap block and its size. */ +/* Finalizes a heap archive, returning a pointer to the heap block and its size. */ /* The heap block will be allocated using the mz_zip_archive's alloc/realloc callbacks. */ MINIZ_EXPORT mz_bool mz_zip_writer_finalize_heap_archive(mz_zip_archive *pZip, void **ppBuf, size_t *pSize); @@ -1335,11 +1405,13 @@ MINIZ_EXPORT mz_bool mz_zip_writer_end(mz_zip_archive *pZip); MINIZ_EXPORT mz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags); MINIZ_EXPORT mz_bool mz_zip_add_mem_to_archive_file_in_place_v2(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, mz_zip_error *pErr); +#ifndef MINIZ_NO_STDIO /* Reads a single file from an archive into a heap block. */ /* If pComment is not NULL, only the file with the specified comment will be extracted. */ /* Returns NULL on failure. */ MINIZ_EXPORT void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name, size_t *pSize, mz_uint flags); MINIZ_EXPORT void *mz_zip_extract_archive_file_to_heap_v2(const char *pZip_filename, const char *pArchive_name, const char *pComment, size_t *pSize, mz_uint flags, mz_zip_error *pErr); +#endif #endif /* #ifndef MINIZ_NO_ARCHIVE_WRITING_APIS */ diff --git a/source/luametatex/source/libraries/miniz/readme.md b/source/luametatex/source/libraries/miniz/readme.md index 3f8fd7324..9734435fa 100644 --- a/source/luametatex/source/libraries/miniz/readme.md +++ b/source/luametatex/source/libraries/miniz/readme.md @@ -4,7 +4,7 @@ Miniz is a lossless, high performance data compression library in a single sourc ## Usage -Please use the files from the [releases page](https://github.com/richgel999/miniz/releases) in your projects. Do not use the git checkout directly! The different source and header files are [amalgamated](https://www.sqlite.org/amalgamation.html) into one `miniz.c`/`miniz.h` pair in a build step (`amalgamate.sh`). Include `miniz.c` and `miniz.h` in your project to use Miniz. +Releases are available at the [releases page](https://github.com/richgel999/miniz/releases) as a pair of `miniz.c`/`miniz.h` files which can be simply added to a project. To create this file pair the different source and header files are [amalgamated](https://www.sqlite.org/amalgamation.html) during build. Alternatively use as cmake or meson module (or build system of your choice). ## Features @@ -18,6 +18,18 @@ Please use the files from the [releases page](https://github.com/richgel999/mini * Entire inflater (including optional zlib header parsing and Adler-32 checking) is implemented in a single function as a coroutine, which is separately available in a small (~550 line) source file: miniz_tinfl.c * A fairly complete (but totally optional) set of .ZIP archive manipulation and extraction API's. The archive functionality is intended to solve common problems encountered in embedded, mobile, or game development situations. (The archive API's are purposely just powerful enough to write an entire archiver given a bit of additional higher-level logic.) +## Building miniz - Using vcpkg + +You can download and install miniz using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager: + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + ./vcpkg install miniz + +The miniz port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + ## Known Problems * No support for encrypted archives. Not sure how useful this stuff is in practice. diff --git a/source/luametatex/source/lua/lmtcallbacklib.h b/source/luametatex/source/lua/lmtcallbacklib.h index 6faa4ddac..7801e1a70 100644 --- a/source/luametatex/source/lua/lmtcallbacklib.h +++ b/source/luametatex/source/lua/lmtcallbacklib.h @@ -83,23 +83,23 @@ typedef enum callback_keys { callback_result_key = 'R', /*tex a string (return value) but nil is also okay */ } callback_keys; -inline static int lmt_callback_defined (int a) { return lmt_callback_state.values[a]; } -inline static int lmt_callback_call (lua_State *L, int i, int o, int top) { return lua_pcallk(L, i, o, top + 2, 0, NULL); } +inline static int lmt_callback_defined (int a) { return lmt_callback_state.values[a]; } +inline static int lmt_callback_call (lua_State *L, int i, int o, int top) { return lua_pcallk(L, i, o, top + 2, 0, NULL); } -extern int lmt_callback_okay (lua_State *L, int i, int *top); -extern void lmt_callback_error (lua_State *L, int top, int i); -inline void lmt_callback_wrapup (lua_State *L, int top) { lua_settop(L, top); } +extern int lmt_callback_okay (lua_State *L, int i, int *top); +extern void lmt_callback_error (lua_State *L, int top, int i); +inline static void lmt_callback_wrapup (lua_State *L, int top) { lua_settop(L, top); } + +extern int lmt_run_callback (lua_State *L, int i, const char *values, ...); +extern int lmt_run_and_save_callback (lua_State *L, int i, const char *values, ...); +extern int lmt_run_saved_callback_line (lua_State *L, int i, int firstpos); +extern int lmt_run_saved_callback_close (lua_State *L, int i); -extern int lmt_run_callback (lua_State *L, int i, const char *values, ...); -extern int lmt_run_and_save_callback (lua_State *L, int i, const char *values, ...); -extern int lmt_run_saved_callback_line (lua_State *L, int i, int firstpos); -extern int lmt_run_saved_callback_close (lua_State *L, int i); +extern void lmt_destroy_saved_callback (lua_State *L, int i); -extern void lmt_destroy_saved_callback (lua_State *L, int i); +extern void lmt_run_memory_callback (const char *what, int success); -extern void lmt_run_memory_callback (const char *what, int success); - -extern void lmt_push_callback_usage (lua_State *L); +extern void lmt_push_callback_usage (lua_State *L); # endif diff --git a/source/luametatex/source/lua/lmtnodelib.c b/source/luametatex/source/lua/lmtnodelib.c index 4f0c7d73a..2a02fbd11 100644 --- a/source/luametatex/source/lua/lmtnodelib.c +++ b/source/luametatex/source/lua/lmtnodelib.c @@ -1363,12 +1363,12 @@ static int nodelib_direct_getanchors(lua_State *L) lua_pushnil(L); } /* bonus detail: source, target */ - if (box_source_anchor(n)) { + if (box_anchor(n)) { lua_pushinteger(L, box_anchor(n) & 0x0FFF); } else { lua_pushnil(L); } - if (box_target_anchor(n)) { + if (box_anchor(n)) { lua_pushinteger(L, (box_anchor(n) >> 16) & 0x0FFF); } else { lua_pushnil(L); @@ -7861,6 +7861,57 @@ static int nodelib_direct_unprotectglyphs(lua_State *L) return 0; } +/*tex This is an experiment. */ + +inline static void nodelib_aux_protect_all_none(halfword h) +{ + while (h) { + if (node_type(h) == glyph_node) { + halfword f = glyph_font(h); + if (f >= 0 && f <= lmt_font_state.font_data.ptr && lmt_font_state.fonts[f] && has_font_text_control(f, text_control_none_protected)) { + glyph_protected(h) = glyph_protected_text_code; + } + } + h = node_next(h); + } +} + +inline static void nodelib_aux_protect_node_none(halfword n) +{ + switch (node_type(n)) { + case glyph_node: + { + halfword f = glyph_font(n); + if (f >= 0 && f <= lmt_font_state.font_data.ptr && lmt_font_state.fonts[f] && has_font_text_control(f, text_control_none_protected)) { + glyph_protected(n) = glyph_protected_text_code; + } + } + break; + case disc_node: + nodelib_aux_protect_all_none(disc_no_break_head(n)); + nodelib_aux_protect_all_none(disc_pre_break_head(n)); + nodelib_aux_protect_all_none(disc_post_break_head(n)); + break; + } +} + +static int nodelib_direct_protectglyphs_none(lua_State *L) +{ + halfword head = nodelib_valid_direct_from_index(L, 1); + halfword tail = nodelib_valid_direct_from_index(L, 2); + if (head) { + while (head) { + nodelib_aux_protect_node_none(head); + if (head == tail) { + break; + } else { + head = node_next(head); + } + } + } + return 0; +} + /* node.direct.first_glyph */ static int nodelib_direct_firstglyph(lua_State *L) @@ -7946,7 +7997,7 @@ static int nodelib_direct_hasglyph(lua_State *L) /* node.getword */ -static inline int nodelib_aux_in_word(halfword n) +inline static int nodelib_aux_in_word(halfword n) { switch (node_type(n)) { case glyph_node: @@ -9644,6 +9695,7 @@ static const struct luaL_Reg nodelib_direct_function_list[] = { { "newmathglyph", nodelib_direct_newmathglyph }, { "protectglyph", nodelib_direct_protectglyph }, { "protectglyphs", nodelib_direct_protectglyphs }, + { "protectglyphsnone", nodelib_direct_protectglyphs_none }, { "protrusionskippable", nodelib_direct_protrusionskipable }, { "rangedimensions", nodelib_direct_rangedimensions }, /* maybe get... */ { "getglyphdimensions", nodelib_direct_getglyphdimensions }, diff --git a/source/luametatex/source/lua/lmttexlib.c b/source/luametatex/source/lua/lmttexlib.c index c88d13490..7d1f3c32f 100644 --- a/source/luametatex/source/lua/lmttexlib.c +++ b/source/luametatex/source/lua/lmttexlib.c @@ -2173,7 +2173,7 @@ return 0; static int texlib_getmathcode(lua_State* L) { - mathcodeval mval = { 0, 0, 0 }; + mathcodeval mval = tex_no_math_code(); int ch = lmt_checkinteger(L, -1); if (character_in_range(ch)) { mval = tex_get_math_code(ch); @@ -2192,7 +2192,7 @@ static int texlib_getmathcode(lua_State* L) static int texlib_getmathcodes(lua_State* L) { - mathcodeval mval = { 0, 0, 0 }; + mathcodeval mval = tex_no_math_code(); int ch = lmt_checkinteger(L, -1); if (character_in_range(ch)) { mval = tex_get_math_code(ch); @@ -2603,6 +2603,7 @@ static int texlib_aux_convert(lua_State *L, int cur_code) /* case lua_token_string_code: */ /* arg token list */ case string_code: /* arg token */ case cs_string_code: /* arg token */ + case cs_active_code: /* arg token */ case detokenized_code: /* arg token */ case meaning_code: /* arg token */ case to_mathstyle_code: @@ -5191,8 +5192,11 @@ static int texlib_getmathcontrolvalues(lua_State *L) static int texlib_gettextcontrolvalues(lua_State *L) { - lua_createtable(L, 1, 0); + lua_createtable(L, 2, 2); lua_set_string_by_index(L, text_control_collapse_hyphens, "collapsehyphens"); + lua_set_string_by_index(L, text_control_base_ligaturing, "baseligaturing"); + lua_set_string_by_index(L, text_control_base_kerning, "basekerning"); + lua_set_string_by_index(L, text_control_none_protected, "noneprotected"); return 1; } diff --git a/source/luametatex/source/lua/lmttokenlib.c b/source/luametatex/source/lua/lmttokenlib.c index e15b2de5a..97ca1b144 100644 --- a/source/luametatex/source/lua/lmttokenlib.c +++ b/source/luametatex/source/lua/lmttokenlib.c @@ -1874,6 +1874,7 @@ static int tokenlib_scan_next_char(lua_State *L) break; case letter_cmd: case other_char_cmd: + case active_char_cmd: /* needs testing */ { char buffer[6]; char *uindex = aux_uni2string((char *) buffer, (unsigned int) cur_chr); @@ -3069,6 +3070,8 @@ static int tokenlib_push_macro(lua_State *L) // todo: just store cmd and flag to /*tex We need to check for a valid hit, but what is best here, for instance using |(cmd >= call_cmd)| is not okay as we miss a lot then. + + Active characters: maybe when we pass a number ... */ if (lua_type(L, 1) == LUA_TSTRING) { size_t lname = 0; @@ -3088,6 +3091,15 @@ static int tokenlib_push_macro(lua_State *L) // todo: just store cmd and flag to return 0; } +static int tokenlib_pop_macro(lua_State *L) +{ + lua_token_package *p = tokenlib_aux_check_ispackage(L, 1); + if (p) { + tex_forced_define(p->how, p->cs, p->flag, p->cmd, p->chr); + } + return 0; +} + char *lmt_get_expansion(halfword head, int *len) { char *str = NULL; @@ -3127,15 +3139,6 @@ static int tokenlib_get_expansion(lua_State* L) return 1; } -static int tokenlib_pop_macro(lua_State *L) -{ - lua_token_package *p = tokenlib_aux_check_ispackage(L, 1); - if (p) { - tex_forced_define(p->how, p->cs, p->flag, p->cmd, p->chr); - } - return 0; -} - static int tokenlib_save_lua(lua_State *L) { halfword f = lmt_tohalfword(L, 1); diff --git a/source/luametatex/source/luametatex.h b/source/luametatex/source/luametatex.h index 514eb191d..6bb728234 100644 --- a/source/luametatex/source/luametatex.h +++ b/source/luametatex/source/luametatex.h @@ -89,7 +89,7 @@ # define luametatex_version 210 # define luametatex_revision 02 # define luametatex_version_string "2.10.02" -# define luametatex_development_id 20221105 +# define luametatex_development_id 20221118 # define luametatex_name_camelcase "LuaMetaTeX" # define luametatex_name_lowercase "luametatex" diff --git a/source/luametatex/source/luarest/lmtstrlibext.c b/source/luametatex/source/luarest/lmtstrlibext.c index 78d7f760c..5478c4b67 100644 --- a/source/luametatex/source/luarest/lmtstrlibext.c +++ b/source/luametatex/source/luarest/lmtstrlibext.c @@ -15,8 +15,6 @@ /*tex Helpers */ -# define utf_fffd "\xEF\xBF\xBD" - inline static int strlib_aux_tounicode(const char *s, size_t l, size_t *p) { unsigned char i = s[*p]; @@ -183,7 +181,7 @@ static int strlib_aux_utf_failed(lua_State *L, int new_ind) { lua_pushinteger(L, new_ind); lua_replace(L, lua_upvalueindex(2)); - lua_pushliteral(L, utf_fffd); + lua_pushliteral(L, utf_fffd_string); return 1; } @@ -356,7 +354,7 @@ static int strlib_utfcharactertable(lua_State *L) lua_pushlstring(L, s + p, b); p += b; } else { - lua_pushliteral(L, utf_fffd); + lua_pushliteral(L, utf_fffd_string); p += 1; } lua_rawseti(L, -2, n++); diff --git a/source/luametatex/source/mp/mpc/mp.c b/source/luametatex/source/mp/mpc/mp.c index beb836de6..a79f7db3f 100644 --- a/source/luametatex/source/mp/mpc/mp.c +++ b/source/luametatex/source/mp/mpc/mp.c @@ -54,9 +54,9 @@ # define max_num_knot_nodes 1000 # define max_num_value_nodes 1000 # define max_num_symbolic_nodes 1000 -//define mp_link(A) (A)->link -//define mp_type(A) (A)->type -//define mp_name_type(A) (A)->name_type +# define mp_link(A) (A)->link +# define mp_type(A) (A)->type +# define mp_name_type(A) (A)->name_type # define mp_set_link(A,B) (A)->link = (mp_node) (B) # define mp_max_command_code mp_stop # define mp_max_pre_command mp_etex_command diff --git a/source/luametatex/source/mp/mpc/mpmathdouble.c b/source/luametatex/source/mp/mpc/mpmathdouble.c index 8349a3b66..ab661d96b 100644 --- a/source/luametatex/source/mp/mpc/mpmathdouble.c +++ b/source/luametatex/source/mp/mpc/mpmathdouble.c @@ -117,9 +117,9 @@ static void mp_set_double_half_from_addition (mp_number *A, mp_number *B, mp static void mp_set_double_half_from_subtraction(mp_number *A, mp_number *B, mp_number *C); static void mp_wrapup_numeric_token (MP mp, unsigned char *start, unsigned char *stop); static char *mp_double_number_tostring (MP mp, mp_number *n); -inline double mp_double_make_fraction (double p, double q) { return (p / q) * fraction_multiplier; } -inline double mp_double_take_fraction (double p, double q) { return (p * q) / fraction_multiplier; } -inline double mp_double_make_scaled (double p, double q) { return p / q; } +inline static double mp_double_make_fraction (double p, double q) { return (p / q) * fraction_multiplier; } +inline static double mp_double_take_fraction (double p, double q) { return (p * q) / fraction_multiplier; } +inline static double mp_double_make_scaled (double p, double q) { return p / q; } math_data *mp_initialize_double_math(MP mp) { diff --git a/source/luametatex/source/mp/mpw/mpmathdouble.w b/source/luametatex/source/mp/mpw/mpmathdouble.w index ab9caf460..6f0f8df99 100644 --- a/source/luametatex/source/mp/mpw/mpmathdouble.w +++ b/source/luametatex/source/mp/mpw/mpmathdouble.w @@ -147,9 +147,9 @@ static void mp_set_double_half_from_subtraction(mp_number *A, mp_number *B, mp static void mp_wrapup_numeric_token (MP mp, unsigned char *start, unsigned char *stop); static char *mp_double_number_tostring (MP mp, mp_number *n); -inline double mp_double_make_fraction (double p, double q) { return (p / q) * fraction_multiplier; } -inline double mp_double_take_fraction (double p, double q) { return (p * q) / fraction_multiplier; } -inline double mp_double_make_scaled (double p, double q) { return p / q; } +inline static double mp_double_make_fraction (double p, double q) { return (p / q) * fraction_multiplier; } +inline static double mp_double_take_fraction (double p, double q) { return (p * q) / fraction_multiplier; } +inline static double mp_double_make_scaled (double p, double q) { return p / q; } @c math_data *mp_initialize_double_math(MP mp) diff --git a/source/luametatex/source/tex/texcommands.c b/source/luametatex/source/tex/texcommands.c index 5a4cc48b3..0e9bb7ac9 100644 --- a/source/luametatex/source/tex/texcommands.c +++ b/source/luametatex/source/tex/texcommands.c @@ -138,7 +138,6 @@ void tex_initialize_commands(void) tex_primitive(tex_command, "exhyphenchar", internal_int_cmd, ex_hyphen_char_code, internal_int_base); tex_primitive(tex_command, "exhyphenpenalty", internal_int_cmd, ex_hyphen_penalty_code, internal_int_base); tex_primitive(tex_command, "fam", internal_int_cmd, family_code, internal_int_base); - tex_primitive(luatex_command, "variablefam", internal_int_cmd, variable_family_code, internal_int_base); tex_primitive(tex_command, "finalhyphendemerits", internal_int_cmd, final_hyphen_demerits_code, internal_int_base); tex_primitive(tex_command, "floatingpenalty", internal_int_cmd, floating_penalty_code, internal_int_base); tex_primitive(tex_command, "globaldefs", internal_int_cmd, global_defs_code, internal_int_base); @@ -198,7 +197,8 @@ void tex_initialize_commands(void) tex_primitive(luatex_command, "tracingnodes", internal_int_cmd, tracing_nodes_code, internal_int_base); tex_primitive(luatex_command, "tracingfullboxes", internal_int_cmd, tracing_full_boxes_code, internal_int_base); tex_primitive(luatex_command, "tracingpenalties", internal_int_cmd, tracing_penalties_code, internal_int_base); - tex_primitive(tex_command, "uchyph", internal_int_cmd, uc_hyph_code, internal_int_base); /* obsolete */ + tex_primitive(tex_command, "uchyph", internal_int_cmd, uc_hyph_code, internal_int_base); /* obsolete, not needed */ + tex_primitive(luatex_command, "variablefam", internal_int_cmd, variable_family_code, internal_int_base); /* obsolete, not used */ tex_primitive(tex_command, "vbadness", internal_int_cmd, vbadness_code, internal_int_base); tex_primitive(tex_command, "widowpenalty", internal_int_cmd, widow_penalty_code, internal_int_base); tex_primitive(tex_command, "year", internal_int_cmd, year_code, internal_int_base); @@ -386,6 +386,7 @@ void tex_initialize_commands(void) tex_primitive(luatex_command, "expandtoken", expand_after_cmd, expand_token_code, 0); tex_primitive(luatex_command, "expandcstoken", expand_after_cmd, expand_cs_token_code, 0); tex_primitive(luatex_command, "expand", expand_after_cmd, expand_code, 0); + tex_primitive(luatex_command, "expandactive", expand_after_cmd, expand_active_code, 0); tex_primitive(luatex_command, "semiexpand", expand_after_cmd, semi_expand_code, 0); tex_primitive(luatex_command, "expandedafter", expand_after_cmd, expand_after_toks_code, 0); /* tex_primitive(luatex_command, "expandafterfi", expand_after_cmd, expand_after_fi, 0); */ @@ -655,6 +656,7 @@ void tex_initialize_commands(void) tex_primitive(tex_command, "string", convert_cmd, string_code, 0); tex_primitive(luatex_command, "directlua", convert_cmd, lua_code, 0); tex_primitive(luatex_command, "csstring", convert_cmd, cs_string_code, 0); + tex_primitive(luatex_command, "csactive", convert_cmd, cs_active_code, 0); tex_primitive(luatex_command, "detokenized", convert_cmd, detokenized_code, 0); tex_primitive(luatex_command, "expanded", convert_cmd, expanded_code, 0); tex_primitive(luatex_command, "semiexpanded", convert_cmd, semi_expanded_code, 0); @@ -759,6 +761,7 @@ void tex_initialize_commands(void) tex_primitive(tex_command, "sfcode", define_char_code_cmd, sfcode_charcode, 0); tex_primitive(luatex_command, "hccode", define_char_code_cmd, hccode_charcode, 0); tex_primitive(luatex_command, "hmcode", define_char_code_cmd, hmcode_charcode, 0); + tex_primitive(luatex_command, "amcode", define_char_code_cmd, amcode_charcode, 0); tex_primitive(tex_command, "mathcode", define_char_code_cmd, mathcode_charcode, 0); tex_primitive(tex_command, "delcode", define_char_code_cmd, delcode_charcode, 0); diff --git a/source/luametatex/source/tex/texcommands.h b/source/luametatex/source/tex/texcommands.h index 6d85f1ed7..a5c157a44 100644 --- a/source/luametatex/source/tex/texcommands.h +++ b/source/luametatex/source/tex/texcommands.h @@ -442,6 +442,7 @@ typedef enum convert_codes { semi_expanded_code, /*tex command code for |\constantexpanded| */ string_code, /*tex command code for |\string| */ cs_string_code, /*tex command code for |\csstring| */ + cs_active_code, /*tex command code for |\csactive| */ detokenized_code, /*tex command code for |\detokenized| */ roman_numeral_code, /*tex command code for |\romannumeral| */ meaning_code, /*tex command code for |\meaning| */ @@ -743,6 +744,7 @@ typedef enum expand_after_codes { expand_token_code, expand_cs_token_code, expand_code, + expand_active_code, semi_expand_code, expand_after_toks_code, /* expand_after_fi, */ @@ -1034,6 +1036,7 @@ typedef enum charcode_codes { sfcode_charcode, hccode_charcode, hmcode_charcode, + amcode_charcode, mathcode_charcode, extmathcode_charcode, delcode_charcode, diff --git a/source/luametatex/source/tex/texdumpdata.h b/source/luametatex/source/tex/texdumpdata.h index e6f51d323..d9b4e5cdd 100644 --- a/source/luametatex/source/tex/texdumpdata.h +++ b/source/luametatex/source/tex/texdumpdata.h @@ -55,7 +55,7 @@ */ -# define luametatex_format_fingerprint 675 +# define luametatex_format_fingerprint 676 /* These end up in the string pool. */ diff --git a/source/luametatex/source/tex/texequivalents.c b/source/luametatex/source/tex/texequivalents.c index 4de7617c9..bdf21446e 100644 --- a/source/luametatex/source/tex/texequivalents.c +++ b/source/luametatex/source/tex/texequivalents.c @@ -1125,7 +1125,7 @@ void tex_geq_word_define(halfword p, int w) a side effect of looking at the code through a visual studio lense.) */ -static inline void tex_aux_set_eq_data(halfword p, singleword t, halfword e, singleword f, quarterword l) +inline static void tex_aux_set_eq_data(halfword p, singleword t, halfword e, singleword f, quarterword l) { singleword flag = eq_flag(p); set_eq_level(p, l); diff --git a/source/luametatex/source/tex/texequivalents.h b/source/luametatex/source/tex/texequivalents.h index 2feab6858..1a6c41b2a 100644 --- a/source/luametatex/source/tex/texequivalents.h +++ b/source/luametatex/source/tex/texequivalents.h @@ -571,7 +571,12 @@ typedef enum int_codes { alignment_wrap_source_code, /* page_boundary_penalty_code, */ line_break_criterium_code, - variable_family_code, + /* + This one was added as experiment to \LUATEX\ (answer to a forwarded question) but as it + didn't get tested it will go away. \CONTEXT\ doesn't need it and we don't need to be + compatible anyway. Lesson learned. + */ + variable_family_code, /* those below these are not interfaced via primitives */ internal_par_state_code, internal_dir_state_code, @@ -601,7 +606,7 @@ typedef enum int_codes { } int_codes; # define first_int_code pre_tolerance_code -# define last_int_code line_break_criterium_code +# define last_int_code variable_family_code typedef enum dimen_codes { par_indent_code, /*tex indentation of paragraphs */ @@ -836,7 +841,7 @@ extern save_state_info lmt_save_state; # define saved_value(A) lmt_save_state.save_stack[lmt_save_state.save_stack_data.ptr + (A)].saved_value # define saved_word(A) lmt_save_state.save_stack[lmt_save_state.save_stack_data.ptr + (A)].saved_word -inline void tex_set_saved_record(halfword ptr, quarterword type, quarterword level, halfword value) +inline static void tex_set_saved_record(halfword ptr, quarterword type, quarterword level, halfword value) { saved_type(ptr) = type; saved_level(ptr) = level; diff --git a/source/luametatex/source/tex/texexpand.c b/source/luametatex/source/tex/texexpand.c index e74a9c08d..cec254d2b 100644 --- a/source/luametatex/source/tex/texexpand.c +++ b/source/luametatex/source/tex/texexpand.c @@ -251,7 +251,7 @@ void tex_expand_current_token(void) case expand_token_code: { /* we can share code with lmtokenlib .. todo */ - halfword cat = tex_scan_category_code(); + halfword cat = tex_scan_category_code(0); halfword chr = tex_scan_char_number(0); /* too fragile: halfword tok = null; @@ -304,6 +304,7 @@ void tex_expand_current_token(void) case spacer_cmd: case letter_cmd: case other_char_cmd: + case active_char_cmd: /* new */ cur_tok = token_val(cmd, eq_value(cur_cs)); break; } @@ -322,6 +323,20 @@ void tex_expand_current_token(void) } break; } + case expand_active_code: + { + tex_get_token(); + if (cur_cmd == active_char_cmd) { + cur_cs = tex_active_to_cs(cur_chr, ! lmt_hash_state.no_new_cs); + if (cur_cs) { + cur_tok = cs_token_flag + cur_cs; + } else { + cur_tok = token_val(cur_cmd, cur_chr); + } + } + tex_back_input(cur_tok); + break; + } case semi_expand_code: { tex_get_token(); @@ -618,12 +633,12 @@ static int tex_aux_collect_cs_tokens(halfword *p, int *n) case spacer_cmd: case letter_cmd: case other_char_cmd: + case active_char_cmd: /* new */ // cur_tok = token_val(cur_cmd, cur_chr); // *p = tex_store_new_token(*p, cur_tok); *p = tex_store_new_token(*p, token_val(cur_cmd, cur_chr)); *n += 1; break; - /* case active_char_cmd: */ /* case comment_cmd: */ /* case invalid_char_cmd: */ /* diff --git a/source/luametatex/source/tex/texfont.c b/source/luametatex/source/tex/texfont.c index f3b26d99a..ac0ea1290 100644 --- a/source/luametatex/source/tex/texfont.c +++ b/source/luametatex/source/tex/texfont.c @@ -26,6 +26,65 @@ # include "luametatex.h" +/*tex + Finally the base mode ligaturing and kerning code has also been made more consistent with the + rest: abstraction, more tight equality testing, helpers, merged some experiments, etc. It will + probably evolve a bit more; not that we use basemode frequently in \CONTEXT. Keep in mind that + it is not that hard to mess up the list when using \LUA\ but we do little checking here. + + From now on base mode ligaturing and kerning will only be applied when |text_font_control| + have the |text_control_base_ligaturing| and |text_control_base_kerning| bits set. +*/ + +inline static halfword tex_aux_discretionary_node(halfword target, int location) +{ + switch (location) { + case pre_break_code : return disc_pre_break_node(target); + case post_break_code: return disc_post_break_node(target); + case no_break_code : return disc_no_break_node(target); + default : return null; + } +} + +inline static int tex_aux_same_font_properties(halfword a, halfword b) // also in kern +{ + return node_type(a) == glyph_node && node_type(b) == glyph_node + && glyph_font(a) == glyph_font(b) + && glyph_x_scale(a) == glyph_x_scale(b) + && glyph_y_scale(a) == glyph_y_scale(b) + && glyph_scale(a) == glyph_scale(b); +} + +inline static int tex_aux_apply_base_kerning(halfword n) +{ + if (glyph_protected(n)) { + return 0; + } else { + halfword f = glyph_font(n); + if (f >= 0 && f <= lmt_font_state.font_data.ptr && lmt_font_state.fonts[f]) { + return has_font_text_control(f, text_control_base_kerning); + } else { + return 0; + } + } +} + +inline static int tex_aux_apply_base_ligaturing(halfword n) +{ + if (glyph_protected(n)) { + return 0; + } else { + halfword f = glyph_font(n); + if (f >= 0 && f <= lmt_font_state.font_data.ptr && lmt_font_state.fonts[f]) { + return has_font_text_control(f, text_control_base_ligaturing); + } else { + return 0; + } + } +} + +/* */ + inline static scaled tex_aux_font_x_scaled(scaled v) { return v ? scaledround(0.000001 * (glyph_scale_par ? glyph_scale_par : 1000) * (glyph_x_scale_par ? glyph_x_scale_par : 1000) * v) : 0; @@ -135,34 +194,34 @@ void tex_undump_font_data(dumpstream f) { void tex_set_charinfo_extensible_recipe(charinfo *ci, extinfo *ext) { if (ci->math) { - extinfo *lst = ci->math->extensible_recipe; - if (lst) { - while (lst) { - extinfo *c = lst->next; - lmt_memory_free(lst); - lst = c; + extinfo *list = ci->math->extensible_recipe; + if (list) { + while (list) { + extinfo *c = list->next; + lmt_memory_free(list); + list = c; } } ci->math->extensible_recipe = ext; } } -void tex_set_font_parameters(halfword f, int b) +void tex_set_font_parameters(halfword f, int index) { int i = font_parameter_count(f); - if (b > i) { + if (index > i) { /*tex If really needed this can be a calloc. */ - int s = (b + 2) * (int) sizeof(int); - int *a = lmt_memory_realloc(font_parameter_base(f), (size_t) s); - if (a) { - lmt_font_state.font_data.allocated += (b - i + 1) * (int) sizeof(scaled); - font_parameter_base(f) = a; - font_parameter_count(f) = b; - while (i < b) { + int size = (index + 2) * (int) sizeof(int); + int *list = lmt_memory_realloc(font_parameter_base(f), (size_t) size); + if (list) { + lmt_font_state.font_data.allocated += (index - i + 1) * (int) sizeof(scaled); + font_parameter_base(f) = list; + font_parameter_count(f) = index; + while (i < index) { font_parameter(f, ++i) = 0; } } else { - tex_overflow_error("font", s); + tex_overflow_error("font", size); } } } @@ -174,14 +233,14 @@ int tex_new_font(void) int size = sizeof(charinfo); charinfo *ci = lmt_memory_calloc(1, (size_t) size); if (ci) { - texfont *t = NULL; + texfont *tf = NULL; size = sizeof(texfont); - t = lmt_memory_calloc(1, (size_t) size); - if (t) { + tf = lmt_memory_calloc(1, (size_t) size); + if (tf) { sa_tree_item sa_value = { 0 }; int id = tex_new_font_id(); lmt_font_state.font_data.allocated += size; - lmt_font_state.fonts[id] = t; + lmt_font_state.fonts[id] = tf; set_font_name(id, NULL); set_font_original(id, NULL); set_font_left_boundary(id, NULL); @@ -194,13 +253,13 @@ int tex_new_font(void) set_font_skew_char(id, -1); /*tex allocate eight values including 0 */ tex_set_font_parameters(id, 7); - for (int k = 0; k <= 7; k++) { - tex_set_font_parameter(id, k, 0); + for (int i = 0; i <= 7; i++) { + tex_set_font_parameter(id, i, 0); } /*tex character info zero is reserved for |notdef|. The stack size 1, default item value 0. */ - t->characters = sa_new_tree(1, 4, sa_value); - t->chardata = ci; - t->chardata_size = 1; + tf->characters = sa_new_tree(1, 4, sa_value); + tf->chardata = ci; + tf->chardata_size = 1; return id; } } @@ -208,16 +267,16 @@ int tex_new_font(void) return 0; } -void tex_font_malloc_charinfo(halfword f, int num) +void tex_font_malloc_charinfo(halfword f, int index) { int glyph = lmt_font_state.fonts[f]->chardata_size; - int size = (glyph + num) * sizeof(charinfo); + int size = (glyph + index) * sizeof(charinfo); charinfo *data = lmt_memory_realloc(lmt_font_state.fonts[f]->chardata , (size_t) size); if (data) { - lmt_font_state.font_data.allocated += num * sizeof(charinfo); + lmt_font_state.font_data.allocated += index * sizeof(charinfo); lmt_font_state.fonts[f]->chardata = data; - memset(&data[glyph], 0, (size_t) num * sizeof(charinfo)); - lmt_font_state.fonts[f]->chardata_size += num; + memset(&data[glyph], 0, (size_t) index * sizeof(charinfo)); + lmt_font_state.fonts[f]->chardata_size += index; } else { tex_overflow_error("font", size); } @@ -690,10 +749,8 @@ halfword tex_get_font_identifier(halfword fontspec) } /*tex - Here come some subroutines to deal with expanded fonts. Returning 1 means that they are identical. - */ ligatureinfo tex_get_ligature(halfword f, int lc, int rc) @@ -793,11 +850,7 @@ halfword tex_checked_font_adjust(halfword adjust_spacing, halfword adjust_spacin return adjust_spacing; } -/*tex - - This returns the multiple of |font_step(f)| that is nearest to |e|. - -*/ +/*tex This returns the multiple of |font_step(f)| that is nearest to |e|. */ int tex_fix_expand_value(halfword f, int e) { @@ -936,34 +989,39 @@ halfword tex_get_parameter_glue(quarterword p, quarterword s) /*tex Ligaturing starts here */ -static void tex_aux_nesting_append(halfword nest1, halfword newn) +static void tex_aux_discretionary_append(halfword target, int location, halfword n) { - halfword tail = node_tail(nest1); - tex_couple_nodes(tail ? tail : nest1, newn); - node_tail(nest1) = newn; + halfword node = tex_aux_discretionary_node(target, location); + if (node_tail(node)) { + tex_couple_nodes(node_tail(node), n); + } else { + node_head(node) = n; + } + node_tail(node) = n; } -static void tex_aux_nesting_prepend(halfword nest1, halfword newn) +static void tex_aux_discretionary_prepend(halfword target, int location, halfword n) { - halfword head = node_next(nest1); - tex_couple_nodes(nest1, newn); - if (head) { - tex_couple_nodes(newn, head); + halfword node = tex_aux_discretionary_node(target, location); + if (node_head(node)) { + tex_couple_nodes(n, node_head(node)); } else { - node_tail(nest1) = newn; - } + node_tail(node) = n; + } + node_head(node) = n; } -static void tex_aux_nesting_prepend_list(halfword nest1, halfword newn) +static void tex_aux_nesting_prepend_list(halfword target, int location, halfword n) /* n is prepended to target */ { - halfword head = node_next(nest1); - halfword tail = tex_tail_of_node_list(newn); - tex_couple_nodes(nest1, newn); - if (head) { - tex_couple_nodes(tail, head); - } else { - node_tail(nest1) = tail; + halfword node = tex_aux_discretionary_node(target, location); + halfword copy = tex_copy_node_list(n, null); + halfword tail = tex_tail_of_node_list(copy); + if (node_head(node)) { + tex_couple_nodes(tail, node_head(node)); + } else { + node_tail(node) = tail; } + node_head(node) = copy; } int tex_valid_ligature(halfword left, halfword right, int *slot) @@ -987,7 +1045,9 @@ int tex_valid_ligature(halfword left, halfword right, int *slot) static int tex_aux_found_ligature(halfword left, halfword right) { - if (node_type(left) != glyph_node) { + if (! left || ! right) { + return 0; + } else if (node_type(left) != glyph_node || node_type(right) != glyph_node) { return 0; } else if (glyph_font(left) != glyph_font(right)) { return 0; @@ -999,77 +1059,101 @@ static int tex_aux_found_ligature(halfword left, halfword right) } /*tex - We could be more efficient and reuse the possibly later removed node but it takes more code and - we don't have that many ligatures anyway. + In principle we only support simple ligatures, i.e.\ |move_after|, |keep_right| and |keep_left| + are zero. At some point we might even drop special ones, including those with boundaries because + the likelyhood of encountering these in the \OPENTYPE\ arena is close to zero. */ -static int tex_aux_try_ligature(halfword *first, halfword forward) -{ - halfword cur = *first; - if (glyph_scale(cur) == glyph_scale(forward) && glyph_x_scale(cur) == glyph_x_scale(forward) && glyph_y_scale(cur) == glyph_y_scale(forward)) { - halfword slot; - halfword type = tex_valid_ligature(cur, forward, &slot); - if (type >= 0) { - int move_after = (type & 0x0C) >> 2; - int keep_right = (type & 0x01) != 0; - int keep_left = (type & 0x02) != 0; - halfword parent = (glyph_character(cur) >= 0) ? cur : ((glyph_character(forward) >= 0) ? forward : null); - halfword ligature = tex_new_glyph_node(glyph_ligature_subtype, glyph_font(cur), slot, parent); - if (keep_left) { - tex_couple_nodes(cur, ligature); - if (move_after) { - move_after--; - cur = ligature; - } - } else { - halfword prev = node_prev(cur); - tex_uncouple_node(cur); - tex_flush_node(cur); - tex_couple_nodes(prev, ligature); - cur = ligature; +static int tex_aux_try_ligature(halfword *first, halfword second, halfword *nextone) +{ + halfword current = *first; + halfword slot; + halfword type = tex_valid_ligature(current, second, &slot); + if (type >= 0) { + int move_after = (type & 0x0C) >> 2; + int keep_right = (type & 0x01) != 0; + int keep_left = (type & 0x02) != 0; + halfword next = node_next(second); + if (keep_left && keep_right) { + halfword ligature = tex_copy_node(current); + glyph_character(ligature) = slot; + tex_couple_nodes(*first, ligature); + tex_couple_nodes(ligature, second); + if (nextone) { + *nextone = second; } - if (keep_right) { - tex_couple_nodes(ligature, forward); - if (move_after) { - move_after--; - cur = forward; - } - } else { - halfword next = node_next(forward); - tex_uncouple_node(forward); - tex_flush_node(forward); - if (next) { - tex_couple_nodes(ligature, next); - } + } else if (keep_right) { + glyph_character(*first) = slot; + if (nextone) { + *nextone = second; + } + } else if (keep_left) { + glyph_character(second) = slot; + if (nextone) { + *nextone = second; + } + } else { + glyph_character(*first) = slot; + tex_uncouple_node(second); + tex_flush_node(second); + tex_try_couple_nodes(*first, next); + if (nextone) { + *nextone = *first; + } + } + /* untested */ + if (nextone) { + while (move_after-- > 0 && *nextone) { + *nextone = node_next(*nextone); } - *first = cur; - return 1; } + return 1; + } else { + return 0; } - return 0; } -/*tex - - There shouldn't be any ligatures here - we only add them at the end of |xxx_break| in a |DISC-1 - - DISC-2| situation and we stop processing |DISC-1| (we continue with |DISC-1|'s |post_| and - |no_break|. - -*/ +static void tex_aux_handle_ligature_list(halfword target, int location) +{ + halfword node = tex_aux_discretionary_node(target, location); + halfword head = node_head(node); + halfword tail = node_tail(node); + if (head && head != tail) { + halfword current = head; + while (node_next(current)) { + halfword next = node_next(current); + int ishead = current == head; + halfword nextone = next; + if (tex_aux_same_font_properties(current, next) && tex_aux_try_ligature(¤t, next, &nextone)) { + if (ishead) { + head = current; + node_head(node) = current; + } + current = nextone; + } else { + current = next; + } + } + node_tail(node) = current; + } +} -static halfword tex_aux_handle_ligature_nesting(halfword root, halfword cur) +static void tex_aux_handle_ligature_pair(halfword target, int location) { - if (cur) { - while (node_next(cur)) { - halfword fwd = node_next(cur); - if (node_type(cur) == glyph_node && node_type(fwd) == glyph_node && glyph_font(cur) == glyph_font(fwd) && tex_aux_try_ligature(&cur, fwd)) { - continue; + halfword node = tex_aux_discretionary_node(target, location); + halfword head = node_head(node); + halfword tail = node_tail(node); + if (head && head != tail) { + halfword previous = node_prev(tail); + int ishead = previous == head; + if (tex_aux_same_font_properties(previous, tail) && tex_aux_try_ligature(&previous, tail, NULL)) { + if (ishead) { + head = previous; + node_head(node) = previous; } - cur = node_next(cur); + node_tail(node) = previous; } - node_tail(root) = cur; } - return root; } /*tex @@ -1083,207 +1167,204 @@ static halfword tex_aux_handle_ligature_nesting(halfword root, halfword cur) have (any kind of) discretionaries. It is still on my agenda to look into nested discretionaries i.e. discs nodes in disc fields but it might never result in useable code. + Todo: check this boundary mess (check for subtype too). + Todo: maybe get rid of weird ligatures, turn boundary into space and such. + */ -static halfword tex_aux_handle_ligature_word(halfword cur) +static halfword tex_aux_handle_ligature_word(halfword current) { halfword right = null; - if (node_type(cur) == boundary_node) { - halfword prev = node_prev(cur); - halfword fwd = node_next(cur); + halfword last = null; /* cf LuaTeX patch, an ancient border case buglet. */ + if (node_type(current) == boundary_node) { + halfword previous = node_prev(current); + halfword next = node_next(current); /*tex There is no need to uncouple |cur|, it is freed. */ - tex_flush_node(cur); - if (fwd) { - tex_couple_nodes(prev, fwd); - if (node_type(fwd) != glyph_node) { - return prev; + tex_flush_node(current); + if (next) { + tex_couple_nodes(previous, next); + if (node_type(next) != glyph_node) { + return previous; } else { - cur = fwd; + current = next; } } else { - node_next(prev) = fwd; - return prev; + node_next(previous) = next; + return previous; } - } else if (font_has_left_boundary(glyph_font(cur))) { - halfword prev = node_prev(cur); - halfword p = tex_new_glyph_node(glyph_unset_subtype, glyph_font(cur), left_boundary_char, cur); - tex_couple_nodes(prev, p); - tex_couple_nodes(p, cur); - cur = p; + } else if (node_type(current) == glyph_node && font_has_left_boundary(glyph_font(current))) { + halfword previous = node_prev(current); + halfword glyph = tex_new_glyph_node(glyph_unset_subtype, glyph_font(current), left_boundary_char, current); + tex_couple_nodes(previous, glyph); + tex_couple_nodes(glyph, current); + current = glyph; } - if (font_has_right_boundary(glyph_font(cur))) { - right = tex_new_glyph_node(glyph_unset_subtype, glyph_font(cur), right_boundary_char, cur); + if (node_type(current) == glyph_node && font_has_right_boundary(glyph_font(current))) { + right = tex_new_glyph_node(glyph_unset_subtype, glyph_font(current), right_boundary_char, current); } - /* todo: switch */ + // tex_print_node_list(current, "GOING",max_integer, max_integer); while (1) { - halfword t = node_type(cur); + halfword currenttype = node_type(current); /*tex A glyph followed by \unknown */ - if (t == glyph_node) { - halfword fwd = node_next(cur); - if (fwd) { - t = node_type(fwd); - if (t == glyph_node) { - /*tex a glyph followed by a glyph */ - if (glyph_font(cur) != glyph_font(fwd)) { + if (currenttype == glyph_node) { + if (tex_aux_apply_base_ligaturing(current)) { + halfword forward = node_next(current); + if (forward) { + halfword forwardtype = node_type(forward); + if (forwardtype == glyph_node) { + if (! tex_aux_apply_base_ligaturing(forward)) { + // break; + } else if (! tex_aux_same_font_properties(current, forward)) { + // break; + } else { + halfword nextone = current; + if (tex_aux_try_ligature(¤t, forward, &nextone)) { + current = nextone; + continue; + } + } + } else if (forwardtype == disc_node) { + /*tex a glyph followed by a disc */ + halfword pre = disc_pre_break_head(forward); + halfword replace = disc_no_break_head(forward); + halfword next; + /*tex Check on: |a{b?}{?}{?}| and |a+b=>B| : |{B?}{?}{a?}| */ + /*tex Check on: |a{?}{?}{b?}| and |a+b=>B| : |{a?}{?}{B?}| */ + if (tex_aux_found_ligature(current, pre) || tex_aux_found_ligature(current, replace)) { + /*tex Move |cur| from before disc to skipped part */ + halfword previous = node_prev(current); + tex_uncouple_node(current); + tex_couple_nodes(previous, forward); + tex_aux_discretionary_prepend(forward, no_break_code, current); + tex_aux_discretionary_prepend(forward, pre_break_code, tex_copy_node(current)); + /*tex As we have removed cur, we need to start again. */ + current = previous; + } + /*tex Check on: |a{?}{?}{}b| and |a+b=>B| : |{a?}{?b}{B}|. */ + next = node_next(forward); + if (! replace && tex_aux_found_ligature(current, next)) { + /*tex Move |cur| from before |disc| to |no_break| part. */ + halfword previous = node_prev(current); + halfword tail = node_next(next); + tex_uncouple_node(current); + tex_couple_nodes(previous, forward); + tex_aux_discretionary_prepend(forward, pre_break_code, tex_copy_node(current)); + /*tex Move next from after disc to |no_break| part. */ + tex_uncouple_node(next); + tex_try_couple_nodes(forward, tail); + /*tex We {\em know} this works. */ + tex_couple_nodes(current, next); + /*tex Make sure the list is correct. */ + tex_aux_discretionary_append(forward, post_break_code, tex_copy_node(next)); + /*tex As we have removed cur, we need to start again. */ + current = previous; + } + /*tex We are finished with the |pre_break|. */ + tex_aux_handle_ligature_list(forward, pre_break_code); + } else if (forwardtype == boundary_node) { + halfword next = node_next(forward); + tex_try_couple_nodes(current, next); + tex_flush_node(forward); + if (right) { + /*tex Shame, didn't need it. */ + tex_flush_node(right); + /*tex No need to reset |right|, we're going to leave the loop anyway. */ + } break; - } else if (tex_aux_try_ligature(&cur, fwd)) { + } else if (right) { + tex_couple_nodes(current, right); + tex_couple_nodes(right, forward); + right = null; continue; + } else { + break; } - } else if (t == disc_node) { - /*tex a glyph followed by a disc */ - halfword pre = disc_pre_break_head(fwd); - halfword nob = disc_no_break_head(fwd); - halfword next, tail; - /*tex Check on: |a{b?}{?}{?}| and |a+b=>B| : |{B?}{?}{a?}| */ - /*tex Check on: |a{?}{?}{b?}| and |a+b=>B| : |{a?}{?}{B?}| */ - if ((pre && node_type(pre) == glyph_node && tex_aux_found_ligature(cur, pre)) - || (nob && node_type(nob) == glyph_node && tex_aux_found_ligature(cur, nob))) { - /*tex Move |cur| from before disc to skipped part */ - halfword prev = node_prev(cur); - tex_uncouple_node(cur); - tex_couple_nodes(prev, fwd); - tex_aux_nesting_prepend(disc_no_break(fwd), cur); - /*tex Now ligature the |pre_break|. */ - tex_aux_nesting_prepend(disc_pre_break(fwd), tex_copy_node(cur)); - /*tex As we have removed cur, we need to start again. */ - cur = prev; - } - /*tex Check on: |a{?}{?}{}b| and |a+b=>B| : |{a?}{?b}{B}|. */ - next = node_next(fwd); - if ((! nob) && next && node_type(next) == glyph_node && tex_aux_found_ligature(cur, next)) { - /*tex Move |cur| from before |disc| to |no_break| part. */ - halfword prev = node_prev(cur); - tex_uncouple_node(cur); - tex_couple_nodes(prev, fwd); - /*tex We {\em know} it's empty. */ - tex_couple_nodes(disc_no_break(fwd), cur); - /*tex Now copy |cur| the |pre_break|. */ - tex_aux_nesting_prepend(disc_pre_break(fwd), tex_copy_node(cur)); - /*tex Move next from after disc to |no_break| part. */ - tail = node_next(next); - tex_uncouple_node(next); - tex_try_couple_nodes(fwd, tail); - /*tex We {\em know} this works. */ - tex_couple_nodes(cur, next); - /*tex Make sure the list is correct. */ - disc_no_break_tail(fwd) = next; - /*tex Now copy next to the |post_break|. */ - tex_aux_nesting_append(disc_post_break(fwd), tex_copy_node(next)); - /*tex As we have removed cur, we need to start again. */ - cur = prev; - } - /*tex We are finished with the |pre_break|. */ - tex_aux_handle_ligature_nesting(disc_pre_break(fwd), disc_pre_break_head(fwd)); - } else if (t == boundary_node) { - halfword next = node_next(fwd); - tex_try_couple_nodes(cur, next); - tex_flush_node(fwd); + } else { + /*tex The last character of a paragraph. */ if (right) { - /*tex Shame, didn't need it. */ - tex_flush_node(right); - /*tex No need to reset |right|, we're going to leave the loop anyway. */ + /*tex |par| prohibits the use of |couple_nodes| here. */ + tex_try_couple_nodes(current, right); + right = null; + continue; + } else { + break; } - break; - } else if (right) { - tex_couple_nodes(cur, right); - tex_couple_nodes(right, fwd); - right = null; - continue; - } else { - break; - } - } else { - /*tex The last character of a paragraph. */ - if (right) { - /*tex |par| prohibits the use of |couple_nodes| here. */ - tex_try_couple_nodes(cur, right); - right = null; - continue; - } else { - break; } + /*tex A discretionary followed by \unknown */ } - /*tex A discretionary followed by \unknown */ - } else if (t == disc_node) { + } else if (currenttype == disc_node) { /*tex If |{?}{x}{?}| or |{?}{?}{y}| then: */ - if (disc_no_break_head(cur) || disc_post_break_head(cur)) { - halfword fwd; - if (disc_post_break_head(cur)) { - tex_aux_handle_ligature_nesting(disc_post_break(cur), disc_post_break_head(cur)); + if (disc_no_break_head(current) || disc_post_break_head(current)) { + /*tex Is this nesting okay (and needed)? */ + halfword forward; + if (disc_post_break_head(current)) { + tex_aux_handle_ligature_list(current, post_break_code); } - if (disc_no_break_head(cur)) { - tex_aux_handle_ligature_nesting(disc_no_break(cur), disc_no_break_head(cur)); + if (disc_no_break_head(current)) { + tex_aux_handle_ligature_list(current, no_break_code); } - fwd = node_next(cur); - while (fwd) { - if (node_type(fwd) == glyph_node) { - halfword nob = disc_no_break_tail(cur); - halfword pst = disc_post_break_tail(cur); - if ((! nob || ! tex_aux_found_ligature(nob, fwd)) && (! pst || ! tex_aux_found_ligature(pst, fwd))) { - break; - } else { - halfword next = node_next(fwd); - tex_aux_nesting_append(disc_no_break(cur), tex_copy_node(fwd)); - tex_aux_handle_ligature_nesting(disc_no_break(cur), nob); - tex_uncouple_node(fwd); - tex_try_couple_nodes(cur, next); - tex_aux_nesting_append(disc_post_break(cur), fwd); - tex_aux_handle_ligature_nesting(disc_post_break(cur), pst); - fwd = node_next(cur); - } + forward = node_next(current); + while (forward && node_type(forward) == glyph_node && tex_aux_apply_base_ligaturing(forward)) { + halfword replace = disc_no_break_tail(current); + halfword post = disc_post_break_tail(current); + if (tex_aux_found_ligature(replace, forward) || tex_aux_found_ligature(post, forward)) { + tex_try_couple_nodes(current, node_next(forward)); + tex_uncouple_node(forward); + tex_aux_discretionary_append(current, no_break_code, tex_copy_node(forward)); + tex_aux_handle_ligature_pair(current, no_break_code); + tex_aux_handle_ligature_pair(current, post_break_code); + forward = node_next(current); } else { break; } } - if (fwd && node_type(fwd) == disc_node) { + if (forward && node_type(forward) == disc_node) { /*tex This only deals with simple pre-only discretionaries and a following glyph. */ - halfword next = node_next(fwd); + halfword next = node_next(forward); if (next - && ! disc_no_break_head(fwd) - && ! disc_post_break_head(fwd) + && ! disc_no_break_head(forward) + && ! disc_post_break_head(forward) && node_type(next) == glyph_node - && ((disc_post_break_tail(cur) && tex_aux_found_ligature(disc_post_break_tail(cur), next)) || - (disc_no_break_tail (cur) && tex_aux_found_ligature(disc_no_break_tail (cur), next)))) { + && tex_aux_apply_base_ligaturing(next) + && ((disc_post_break_tail(current) && tex_aux_found_ligature(disc_post_break_tail(current), next)) || + (disc_no_break_tail (current) && tex_aux_found_ligature(disc_no_break_tail (current), next)))) { halfword last = node_next(next); tex_uncouple_node(next); - tex_try_couple_nodes(fwd, last); + tex_try_couple_nodes(forward, last); /*tex Just a hidden flag, used for (base mode) experiments. */ if (hyphenation_permitted(hyphenation_mode_par, lazy_ligatures_hyphenation_mode)) { /*tex f-f-i -> f-fi */ - halfword tail = disc_no_break_tail(cur); - tex_aux_nesting_append(disc_no_break(cur), tex_copy_node(next)); - tex_aux_handle_ligature_nesting(disc_no_break(cur), tail); - tail = disc_post_break_tail(cur); - tex_aux_nesting_append(disc_post_break(cur), next); - tex_aux_handle_ligature_nesting(disc_post_break(cur), tail); - tex_try_couple_nodes(node_prev(fwd), node_next(fwd)); - tex_flush_node(fwd); + tex_aux_discretionary_append(current, no_break_code, tex_copy_node(next)); + tex_aux_handle_ligature_pair(current,no_break_code); + tex_aux_discretionary_append(current, post_break_code, next); + tex_aux_handle_ligature_pair(current,post_break_code); + tex_try_couple_nodes(node_prev(forward), node_next(forward)); + tex_flush_node(forward); } else { /*tex f-f-i -> ff-i : |{a-}{b}{AB} {-}{c}{}| => |{AB-}{c}{ABc}| */ - tex_aux_nesting_append(disc_post_break(fwd), tex_copy_node(next)); - if (disc_no_break_head(cur)) { - halfword tail; - tex_aux_nesting_prepend_list(disc_no_break(fwd), tex_copy_node_list(disc_no_break_head(cur), null)); - tail = disc_no_break_tail(fwd); - tex_aux_nesting_append(disc_no_break(fwd), next); - tex_aux_handle_ligature_nesting(disc_no_break(fwd), tail); - tex_aux_nesting_prepend_list(disc_pre_break(fwd), tex_copy_node_list(disc_no_break_head(cur), null)); + tex_aux_discretionary_append(forward, post_break_code, tex_copy_node(next)); + if (disc_no_break_head(current)) { + tex_aux_nesting_prepend_list(forward, no_break_code, disc_no_break_head(current)); + tex_aux_discretionary_append(forward, no_break_code, next); + tex_aux_handle_ligature_pair(forward, no_break_code); + tex_aux_nesting_prepend_list(forward, pre_break_code, disc_no_break_head(current)); } - tex_try_couple_nodes(node_prev(cur), node_next(cur)); - tex_flush_node(cur); - cur = fwd; + tex_try_couple_nodes(node_prev(current), node_next(current)); + tex_flush_node(current); + current = forward; } } } } } else { /*tex We have glyph nor disc. */ - return cur; + return last; } /*tex Goto the next node, where |\par| allows |node_next(cur)| to be NULL. */ - cur = node_next(cur); + last = current; + current = node_next(current); } - return cur; + return current; } /*tex The return value is the new tail, head should be a dummy: */ @@ -1293,27 +1374,35 @@ halfword tex_handle_ligaturing(halfword head, halfword tail) if (node_next(head)) { /*tex A trick to allow explicit |node == null| tests. */ halfword save_tail = null; - halfword cur, prev; + halfword current, previous; if (tail) { save_tail = node_next(tail); node_next(tail) = null; } - prev = head; - cur = node_next(prev); - while (cur) { - if (node_type(cur) == glyph_node || node_type(cur) == boundary_node) { - cur = tex_aux_handle_ligature_word(cur); + previous = head; + current = node_next(previous); + while (current) { + switch(node_type(current)) { + case glyph_node: + if (tex_aux_apply_base_ligaturing(current)) { + current = tex_aux_handle_ligature_word(current); + } + break; + case disc_node: + case boundary_node: + current = tex_aux_handle_ligature_word(current); + break; + } + previous = current; + if (current) { + current = node_next(current); } - prev = cur; - cur = node_next(cur); } - if (! prev) { - prev = tail; + if (! previous) { + previous = tex_tail_of_node_list(head); } - tex_try_couple_nodes(prev, save_tail); - // if (tail) { - // } - return prev; + tex_try_couple_nodes(previous, save_tail); + return previous; } else { return tail; } @@ -1321,13 +1410,9 @@ halfword tex_handle_ligaturing(halfword head, halfword tail) /*tex Kerning starts here: */ -static void tex_aux_add_kern_before(halfword left, halfword right) +static halfword tex_aux_add_kern_before(halfword left, halfword right) { - if ( - glyph_font(left) == glyph_font(right) && - glyph_scale(left) == glyph_scale(right) && - glyph_x_scale(left) == glyph_x_scale(right) && - glyph_y_scale(left) == glyph_y_scale(right) && + if (tex_aux_same_font_properties(left, right) && ! tex_has_glyph_option(left, glyph_option_no_right_kern) && ! tex_has_glyph_option(right, glyph_option_no_left_kern) && tex_has_kern(glyph_font(left), glyph_character(left)) @@ -1335,21 +1420,19 @@ static void tex_aux_add_kern_before(halfword left, halfword right) scaled k = tex_raw_get_kern(glyph_font(left), glyph_character(left), glyph_character(right)); if (k) { scaled kern = tex_new_kern_node(k, font_kern_subtype); - halfword prev = node_prev(right); - tex_couple_nodes(prev, kern); + halfword previous = node_prev(right); + tex_couple_nodes(previous, kern); tex_couple_nodes(kern, right); tex_attach_attribute_list_copy(kern, left); + return kern; } } + return null; } -static void tex_aux_add_kern_after(halfword left, halfword right, halfword aft) +static halfword tex_aux_add_kern_after(halfword left, halfword right, halfword after) { - if ( - glyph_font(left) == glyph_font(right) && - glyph_scale(left) == glyph_scale(right) && - glyph_x_scale(left) == glyph_x_scale(right) && - glyph_y_scale(left) == glyph_y_scale(right) && + if (tex_aux_same_font_properties(left, right) && ! tex_has_glyph_option(left, glyph_option_no_right_kern) && ! tex_has_glyph_option(right, glyph_option_no_left_kern) && tex_has_kern(glyph_font(left), glyph_character(left)) @@ -1357,78 +1440,96 @@ static void tex_aux_add_kern_after(halfword left, halfword right, halfword aft) scaled k = tex_raw_get_kern(glyph_font(left), glyph_character(left), glyph_character(right)); if (k) { scaled kern = tex_new_kern_node(k, font_kern_subtype); - halfword next = node_next(aft); - tex_couple_nodes(aft, kern); + halfword next = node_next(after); + tex_couple_nodes(after, kern); tex_try_couple_nodes(kern, next); - tex_attach_attribute_list_copy(kern, aft); + tex_attach_attribute_list_copy(kern, after); + return kern; } } + return null; } -static void tex_aux_do_handle_kerning(halfword root, halfword init_left, halfword init_right) +static halfword tex_aux_do_handle_kerning(halfword root, halfword init_left, halfword init_right); + +static void tex_aux_handle_discretionary_kerning(halfword target, int location, halfword left, halfword right) { - halfword cur = node_next(root); - if (cur) { + halfword node = tex_aux_discretionary_node(target, location); + if (node_head(node)) { + halfword kern = tex_aux_do_handle_kerning(node_head(node), left, right); + if (kern) { + node_head(node) = kern; + node_tail(node) = tex_tail_of_node_list(node_head(node)); + } + } +} + +static halfword tex_aux_do_handle_kerning(halfword root, halfword init_left, halfword init_right) +{ + // halfword head = node_next(root); // todo: get rid of this one + halfword head = root; // todo: get rid of this one + halfword current = head; + halfword initial = null; + if (current) { halfword left = null; - if (node_type(cur) == glyph_node) { + if (node_type(current) == glyph_node && tex_aux_apply_base_kerning(current)) { if (init_left) { - tex_aux_add_kern_before(init_left, cur); + halfword kern = tex_aux_add_kern_before(init_left, current); + if (current == head) { + initial = kern; + } } - left = cur; + left = current; } - cur = node_next(cur); - while (cur) { - halfword t = node_type(cur); - if (t == glyph_node) { - if (left) { - tex_aux_add_kern_before(left, cur); - if (glyph_character(left) < 0) { - halfword prev = node_prev(left); - tex_couple_nodes(prev, cur); - tex_flush_node(left); + current = node_next(current); + while (current) { + halfword currenttype = node_type(current); + if (currenttype == glyph_node) { + if (tex_aux_apply_base_kerning(current)) { + if (left) { + tex_aux_add_kern_before(left, current); + if (glyph_character(left) < 0) { + halfword previous = node_prev(left); + tex_couple_nodes(previous, current); + tex_flush_node(left); + } } + left = current; + } else { + left = null; } - left = cur; } else { - if (t == disc_node) { - halfword right = node_type(node_next(cur)) == glyph_node ? node_next(cur) : null; - tex_aux_do_handle_kerning(disc_pre_break(cur), left, null); - if (disc_pre_break_head(cur)) { - disc_pre_break_tail(cur) = tex_tail_of_node_list(disc_pre_break_head(cur)); - } - tex_aux_do_handle_kerning(disc_post_break(cur), null, right); - if (disc_post_break_head(cur)) { - disc_post_break_tail(cur) = tex_tail_of_node_list(disc_post_break_head(cur)); - } - tex_aux_do_handle_kerning(disc_no_break(cur), left, right); - if (disc_no_break_head(cur)) { - disc_no_break_tail(cur) = tex_tail_of_node_list(disc_no_break_head(cur)); - } + if (currenttype == disc_node) { + halfword next = node_next(current); + halfword right = node_type(next) == glyph_node && tex_aux_apply_base_kerning(next) ? next : null; + tex_aux_handle_discretionary_kerning(current, pre_break_code, left, null); + tex_aux_handle_discretionary_kerning(current, post_break_code, null, right); + tex_aux_handle_discretionary_kerning(current, no_break_code, left, right); } if (left) { - if (glyph_character(left) < 0) { - halfword prev = node_prev(left); - tex_couple_nodes(prev, cur); + if (glyph_character(left) < 0) { /* boundary ? */ + halfword previous = node_prev(left); + tex_couple_nodes(previous, current); tex_flush_node(left); } left = null; } } - cur = node_next(cur); + current = node_next(current); } if (left) { if (init_right) { tex_aux_add_kern_after(left, init_right, left); } if (glyph_character(left) < 0) { - halfword prev = node_prev(left); + halfword previous = node_prev(left); halfword next = node_next(left); if (next) { - tex_couple_nodes(prev, next); + tex_couple_nodes(previous, next); node_tail(root) = next; - } else if (prev != root) { - node_next(prev) = null; - node_tail(root) = prev; + } else if (previous != root) { + node_next(previous) = null; + node_tail(root) = previous; } else { node_next(root) = null; node_tail(root) = null; @@ -1440,6 +1541,7 @@ static void tex_aux_do_handle_kerning(halfword root, halfword init_left, halfwor tex_aux_add_kern_after(init_left, init_right, root); node_tail(root) = node_next(root); } + return initial; } halfword tex_handle_kerning(halfword head, halfword tail) @@ -1449,7 +1551,7 @@ halfword tex_handle_kerning(halfword head, halfword tail) save_link = node_next(tail); node_next(tail) = null; node_tail(head) = tail; - tex_aux_do_handle_kerning(head, null, null); + tex_aux_do_handle_kerning(node_next(head), null, null); /*tex There is no need to check initial here. */ tail = node_tail(head); if (tex_valid_node(save_link)) { /* no need for check */ @@ -1457,7 +1559,7 @@ halfword tex_handle_kerning(halfword head, halfword tail) } } else { node_tail(head) = null; - tex_aux_do_handle_kerning(head, null, null); + tex_aux_do_handle_kerning(node_next(head), null, null); /*tex There is no need to check initial here. */ } return tail; } @@ -1494,19 +1596,17 @@ halfword tex_handle_glyphrun(halfword head, halfword group, halfword direction) if (callback_id) { head = tex_aux_run_lua_ligkern_callback(lmt_lua_state.lua_instance, head, group, direction, callback_id); } else { + // what if disc at start tex_handle_ligaturing(head, null); } callback_id = lmt_callback_defined(kerning_callback); if (callback_id) { head = tex_aux_run_lua_ligkern_callback(lmt_lua_state.lua_instance, head, group, direction, callback_id); } else { - halfword nest = tex_new_node(nesting_node, unset_nesting_code); - tex_couple_nodes(nest, head); - tex_aux_do_handle_kerning(nest, null, null); - head = node_next(nest); - node_prev(head) = null; - node_next(nest) = null; - tex_flush_node(nest); + halfword kern = tex_aux_do_handle_kerning(head, null, null); + if (kern) { + head = kern; + } } } } @@ -1534,10 +1634,8 @@ void tex_set_cur_font(halfword g, halfword f) } /*tex - Because we do fonts in \LUA\ we can decide to drop this one and assume a definition using the token scanner. It also avoids the filename (split) mess. - */ int tex_tex_def_font(int a) @@ -1555,7 +1653,7 @@ int tex_tex_def_font(int a) /*tex Stated 'at' size, or negative of scaled magnification. */ scaled s = -1000; char *fn; - /*tex Here |a| detemines if we define global or not. */ + /*tex Here |a| determines if we define global or not. */ if (is_global(a)) { update_tex_font_global(u, null_font); } else { @@ -1603,11 +1701,9 @@ int tex_tex_def_font(int a) } /*tex - When \TEX\ wants to typeset a character that doesn't exist, the character node is not created; thus the output routine can assume that characters exist when it sees them. The following procedure prints a warning message unless the user has suppressed it. - */ void tex_char_warning(halfword f, int c) diff --git a/source/luametatex/source/tex/texfont.h b/source/luametatex/source/tex/texfont.h index 994cdd894..947cdf446 100644 --- a/source/luametatex/source/tex/texfont.h +++ b/source/luametatex/source/tex/texfont.h @@ -324,6 +324,9 @@ extern font_state_info lmt_font_state; typedef enum text_control_codes { text_control_collapse_hyphens = 0x00001, + text_control_base_ligaturing = 0x00002, + text_control_base_kerning = 0x00004, + text_control_none_protected = 0x00008, } text_control_codes; # define has_font_text_control(f,c) ((font_textcontrol(f) & c) == c) diff --git a/source/luametatex/source/tex/texinputstack.h b/source/luametatex/source/tex/texinputstack.h index 7ae677d56..51ff2ef56 100644 --- a/source/luametatex/source/tex/texinputstack.h +++ b/source/luametatex/source/tex/texinputstack.h @@ -68,12 +68,12 @@ typedef struct input_file_state_info { extern input_file_state_info input_file_state; -static inline int input_file_value(void) +inline static int input_file_value(void) { return input_file_state.forced_file ? input_file_state.forced_file : lmt_input_state.cur_input.state_file; } -static inline int input_line_value(void) +inline static int input_line_value(void) { return input_file_state.forced_line ? input_file_state.forced_line : (input_file_state.line ? input_file_state.line : lmt_input_state.input_line); } diff --git a/source/luametatex/source/tex/texinserts.c b/source/luametatex/source/tex/texinserts.c index dbd164926..3d634dd39 100644 --- a/source/luametatex/source/tex/texinserts.c +++ b/source/luametatex/source/tex/texinserts.c @@ -155,7 +155,7 @@ halfword tex_get_insert_distance(halfword i) } } -static inline halfword tex_aux_insert_box(halfword i) +inline static halfword tex_aux_insert_box(halfword i) { if (tex_valid_insert_id(i)) { return lmt_insert_state.mode == index_insert_mode ? insert_content(i) : lmt_insert_state.inserts[i].content; diff --git a/source/luametatex/source/tex/texlinebreak.c b/source/luametatex/source/tex/texlinebreak.c index 2e0c945a7..7c9e6b768 100644 --- a/source/luametatex/source/tex/texlinebreak.c +++ b/source/luametatex/source/tex/texlinebreak.c @@ -1372,7 +1372,7 @@ halfword tex_badness(scaled t, scaled s) } } -static inline void tex_split_line_break_criterium(halfword criterium, halfword *semi_tight, halfword *decent, halfword *semi_loose, halfword *loose) { +inline static void tex_split_line_break_criterium(halfword criterium, halfword *semi_tight, halfword *decent, halfword *semi_loose, halfword *loose) { *semi_tight = (criterium >> 24) & 0x7F; *decent = (criterium >> 16) & 0x7F; *semi_loose = (criterium >> 8) & 0x7F; @@ -1391,7 +1391,7 @@ static inline void tex_split_line_break_criterium(halfword criterium, halfword * } } -static inline halfword tex_normalized_loose_badness(halfword b, halfword loose, halfword semi_loose, halfword decent) +inline static halfword tex_normalized_loose_badness(halfword b, halfword loose, halfword semi_loose, halfword decent) { // if (b > loose_criterium) { // return very_loose_fit; @@ -1411,7 +1411,7 @@ static inline halfword tex_normalized_loose_badness(halfword b, halfword loose, } } -static inline halfword tex_normalized_tight_badness(halfword b, halfword decent, halfword semi_tight) +inline static halfword tex_normalized_tight_badness(halfword b, halfword decent, halfword semi_tight) { // if (b > decent_criterium) { // return tight_fit; diff --git a/source/luametatex/source/tex/texmaincontrol.c b/source/luametatex/source/tex/texmaincontrol.c index b0a9d2a4b..6de36d7ee 100644 --- a/source/luametatex/source/tex/texmaincontrol.c +++ b/source/luametatex/source/tex/texmaincontrol.c @@ -68,7 +68,7 @@ main_control_state_info lmt_main_control_state = { A few helpers: */ -inline scaled tex_aux_checked_dimen1(scaled v) +inline static scaled tex_aux_checked_dimen1(scaled v) { if (v > max_dimen) { return max_dimen; @@ -79,7 +79,7 @@ inline scaled tex_aux_checked_dimen1(scaled v) } } -inline scaled tex_aux_checked_dimen2(scaled v) +inline static scaled tex_aux_checked_dimen2(scaled v) { if (v > max_dimen) { return max_dimen; @@ -475,6 +475,25 @@ static void tex_aux_run_relax(void) { return; } +static void tex_aux_run_active(void) { +// if (lmt_input_state.scanner_status == scanner_is_tolerant || lmt_input_state.scanner_status == scanner_is_matching) { +// cur_cs = tex_active_to_cs(cur_chr, ! lmt_hash_state.no_new_cs); +// cur_cmd = eq_type(cur_cs); +// cur_chr = eq_value(cur_cs); +// tex_x_token(); +// } else + if ((cur_mode == mmode || lmt_nest_state.math_mode) && tex_check_active_math_char(cur_chr)) { + /*tex We have an intercept. */ + tex_back_input(cur_tok); + } else { + cur_cs = tex_active_to_cs(cur_chr, ! lmt_hash_state.no_new_cs); + cur_cmd = eq_type(cur_cs); + cur_chr = eq_value(cur_cs); + tex_x_token(); + tex_back_input(cur_tok); + } +} + /*tex |ignore_spaces| is a special case: after it has acted, |get_x_token| has already fetched the @@ -1664,7 +1683,7 @@ void tex_local_control(int obeymode) tex_unsave_full_scanner_status(saved_full_status); } -inline int tex_aux_is_iterator_value(halfword tokeninfo) +inline static int tex_aux_is_iterator_value(halfword tokeninfo) { if (tokeninfo >= cs_token_flag) { halfword cs = tokeninfo - cs_token_flag; @@ -3543,7 +3562,7 @@ inline static void tex_aux_update_register(int a, int level, halfword index, hal if ((register_attribute_number(index)) > lmt_node_memory_state.max_used_attribute) { lmt_node_memory_state.max_used_attribute = register_attribute_number(index); } - change_attribute_register(a, index, value); + tex_change_attribute_register(a, index, value); tex_word_define(a, index, value); break; case dimen_val_level: @@ -4497,9 +4516,9 @@ static void tex_aux_set_let(int a, int force) case let_charcode_code: /*tex |\letcharcode| (todo: protection) */ { - halfword v = tex_scan_int(0, NULL); - if (v > 0) { - p = tex_active_to_cs(v, 1); + halfword character = tex_scan_int(0, NULL); + if (character > 0) { + p = tex_active_to_cs(character, 1); do { tex_get_token(); } while (cur_cmd == spacer_cmd); @@ -4773,6 +4792,13 @@ static void tex_aux_set_define_char_code(int a) /* maybe make |a| already a bool tex_set_hm_code(chr, val, global_or_local(a)); } break; + case amcode_charcode: + { + halfword chr = tex_scan_char_number(0); + halfword val = tex_scan_category_code(1); + tex_set_am_code(chr, val, global_or_local(a)); + } + break; case mathcode_charcode: tex_scan_extdef_math_code((is_global(a)) ? level_one: cur_level, tex_mathcode); break; @@ -5181,7 +5207,7 @@ static void tex_aux_set_internal_attr(int a) if (internal_attribute_number(p) > lmt_node_memory_state.max_used_attribute) { lmt_node_memory_state.max_used_attribute = internal_attribute_number(p); } - change_attribute_register(a, p, v); + tex_change_attribute_register(a, p, v); tex_word_define(a, p, v); } @@ -5192,7 +5218,7 @@ static void tex_aux_set_register_attr(int a) if (register_attribute_number(p) > lmt_node_memory_state.max_used_attribute) { lmt_node_memory_state.max_used_attribute = register_attribute_number(p); } - change_attribute_register(a, p, v); + tex_change_attribute_register(a, p, v); tex_word_define(a, p, v); } @@ -5488,6 +5514,15 @@ void tex_get_r_token(void) tex_get_token(); } while (cur_tok == space_token); if (eqtb_invalid_cs(cur_cs)) { + if (cur_cmd == active_char_cmd) { + cur_cs = tex_active_to_cs(cur_chr, 1); + cur_cmd = eq_type(cur_cs); + cur_chr = eq_value(cur_cs); + // tex_x_token(); + // if (! eqtb_invalid_cs(cur_cs)) { + return; + // } + } if (cur_cs == 0) { tex_back_input(cur_tok); } @@ -5777,7 +5812,7 @@ void tex_assign_internal_attribute_value(int a, halfword p, int val) if (register_attribute_number(p) > lmt_node_memory_state.max_used_attribute) { lmt_node_memory_state.max_used_attribute = register_attribute_number(p); } - change_attribute_register(a, p, val); + tex_change_attribute_register(a, p, val); tex_word_define(a, p, val); } @@ -6349,6 +6384,8 @@ inline static void tex_aux_big_switch(int mode, int cmd) register_runner(ignore_cmd, tex_aux_run_relax, tex_aux_run_relax, tex_aux_run_relax); + register_runner(active_char_cmd, tex_aux_run_active, tex_aux_run_active, tex_aux_run_active); + /*tex The next is unlikely to happen but compilers like the check. */ # if (main_control_mode == 0) diff --git a/source/luametatex/source/tex/texmath.c b/source/luametatex/source/tex/texmath.c index f650704cd..1424e5e03 100644 --- a/source/luametatex/source/tex/texmath.c +++ b/source/luametatex/source/tex/texmath.c @@ -138,7 +138,7 @@ static void tex_aux_math_math_component (halfword n, int append); inline static mathdictval tex_fake_math_dict(halfword chr) { - mathdictval d = { 0, 0, 0 }; + mathdictval d = tex_no_dict_code(); if (math_dict_properties_par || math_dict_group_par) { d.properties = (unsigned short) math_dict_properties_par; d.group = (unsigned short) math_dict_group_par; @@ -957,7 +957,7 @@ int tex_show_math_node(halfword n, int threshold, int max) return 1; } -inline halfword tex_aux_valid_delimiter(halfword d) +inline static halfword tex_aux_valid_delimiter(halfword d) { return (d && (delimiter_small_family(d) || delimiter_small_character(d) || delimiter_large_family(d) || delimiter_large_character(d))) ? d : null; } @@ -1040,6 +1040,9 @@ static void tex_aux_display_radical_noad(halfword n, int threshold, int max) if (radical_depth(n)) { tex_print_format(", depth %D", radical_depth(n), pt_unit); } + if (radical_size(n)) { + tex_print_format(", size %i", radical_size(n)); + } if (noad_source(n) != 0) { tex_print_format(", source %i", noad_source(n)); } @@ -1246,7 +1249,9 @@ void tex_run_math_initialize(void) switch(cur_cmd) { case math_shift_cmd: /*tex |get_x_token| would fail on |\ifmmode|! */ + lmt_nest_state.math_mode = 1; tex_get_token(); + lmt_nest_state.math_mode = 0; if (cur_cmd == math_shift_cmd && cur_list.mode > nomode) { tex_aux_enter_display_math(math_shift_cmd); } else { @@ -1513,7 +1518,7 @@ void tex_scan_extdef_del_code(int level, int extcode) mathdictval tex_scan_mathdict(void) { - mathdictval d = { 0, 0, 0 }; /* use this one directly */ + mathdictval d = tex_no_dict_code(); /* use this one directly */ d.properties = (unsigned short) tex_scan_math_properties_number(); d.group = (unsigned short) tex_scan_math_group_number(); d.index = (unsigned int) tex_scan_math_index_number(); @@ -1522,7 +1527,7 @@ mathdictval tex_scan_mathdict(void) mathcodeval tex_scan_mathchar(int extcode) { - mathcodeval d = { 0, 0, 0 }; /* use this one directly */ + mathcodeval d = tex_no_math_code(); /* use this one directly */ switch (extcode) { case tex_mathcode: /*tex |"<4bits><4bits><8bits>| */ @@ -1600,7 +1605,7 @@ halfword tex_new_math_dict_spec(mathdictval d, mathcodeval m, quarterword code) mathcodeval tex_get_math_spec(halfword s) { - mathcodeval m = { 0, 0, 0 }; + mathcodeval m = tex_no_math_code(); if (s) { m.class_value = math_spec_class(s); m.family_value = math_spec_family(s); @@ -1611,7 +1616,7 @@ mathcodeval tex_get_math_spec(halfword s) mathdictval tex_get_math_dict(halfword s) { - mathdictval d = { 0, 0, 0 }; + mathdictval d = tex_no_dict_code(); if (s) { d.properties = math_spec_properties(s); d.group = math_spec_group(s); @@ -1666,19 +1671,114 @@ mathcodeval tex_scan_delimiter_as_mathchar(int extcode) For some reason |$\char44$| gives an undefined |$| when we made that character active in math. */ -static void tex_aux_scan_active_math_char(void) +static void tex_aux_report_active(int where, const char *what, int code, int character) +{ + tex_begin_diagnostic(); + tex_print_format("[active: location %i, %s, code %i, char %i]",where, what, code, character); + tex_end_diagnostic(); +} + +static void tex_aux_append_math_char(mathcodeval mval, mathdictval dval, int automatic); + +int tex_check_active_math_char(int character) +{ + halfword code = tex_get_am_code(character); + if (code) { + switch (code) { + case alignment_tab_cmd: + case superscript_cmd: + case subscript_cmd: + cur_cmd = code; + cur_chr = character; + cur_tok = token_val(cur_cmd, cur_chr); + if (tracing_commands_par >= 4) { + tex_aux_report_active(4, "control", code, character); + } + return 1; + case letter_cmd: + case other_char_cmd: + cur_cmd = code; + cur_chr = character; + cur_tok = token_val(cur_cmd, cur_chr); + if (tracing_commands_par >= 4) { + tex_aux_report_active(4, "inject", code, character); + } + return 1; + default: + if (tracing_commands_par >= 4) { + tex_aux_report_active(4, "ignore", code, character); + } + return 1; + } + } else { + return 0; + } +} + +int tex_pass_active_math_char(int character) { - cur_cs = tex_active_to_cs(cur_chr, 1); - cur_cmd = eq_type(cur_cs); - cur_chr = eq_value(cur_cs); - tex_x_token(); - tex_back_input(cur_tok); + halfword code = tex_get_am_code(character); + if (code) { + return 1; + } else { + return 0; + } +} + +static int tex_aux_scan_active_math_char(mathcodeval *mval, int where) +{ + halfword character = mval->character_value; + halfword code = tex_get_am_code(character); + if (code) { + switch (code) { + case alignment_tab_cmd: + case superscript_cmd: + case subscript_cmd: + cur_cmd = code; + cur_chr = character; + cur_tok = token_val(cur_cmd, cur_chr); + tex_back_input(cur_tok); + if (tracing_commands_par >= 4) { + tex_aux_report_active(where, "control", code, character); + } + return 1; + case letter_cmd: + case other_char_cmd: + cur_cmd = code; + cur_chr = character; + cur_tok = token_val(cur_cmd, cur_chr); + if (tracing_commands_par >= 4) { + tex_aux_report_active(where, "inject", code, character); + } + return 0; + default: + if (tracing_commands_par >= 4) { + tex_aux_report_active(where, "ignore", code, character); + } + return 1; + } + } else if (mval->class_value == active_math_class_value) { + cur_cs = tex_active_to_cs(cur_chr, 1); + cur_cmd = eq_type(cur_cs); + cur_chr = eq_value(cur_cs); + tex_x_token(); + tex_back_input(cur_tok); + if (tracing_commands_par >= 4) { + tex_aux_report_active(where, "active", code, character); + } + return 1; + } else { + // if (tracing_commands_par >= 4) { + // tex_aux_report_active(where, "keep", code, mval->character_value); + // } + return 0; + } } static int tex_aux_scan_math(halfword target, halfword style, int usetextfont, halfword toks, halfword toks_text, int nocomponent, halfword cls, halfword all) { - mathcodeval mval = { 0, 0, 0 }; - mathdictval dval = { 0, 0, 0 }; + mathcodeval mval = tex_no_math_code(); + mathdictval dval = tex_no_dict_code(); lmt_math_state.last_atom = cls; RESTART: do { @@ -1694,13 +1794,12 @@ static int tex_aux_scan_math(halfword target, halfword style, int usetextfont, h case other_char_cmd: case char_given_cmd: mval = tex_get_math_code(cur_chr); - if (mval.class_value == active_math_class_value) { - /*tex An active character is allowed here. */ - tex_aux_scan_active_math_char(); - goto RESTART; + if (tex_aux_scan_active_math_char(&mval, 1)) { + goto RESTART; /* rescan pushed back token */ + } else { + dval = tex_fake_math_dict(mval.character_value); + break; } - dval = tex_fake_math_dict(mval.character_value); - break; // case char_number_cmd: // /* The |\glyph| variant is accepted but no keywords here. */ // cur_chr = tex_scan_char_number(); @@ -1875,18 +1974,16 @@ static void tex_aux_append_math_fence_val(mathcodeval mval, mathdictval dval, qu /* todo : share the next three with the regular fences */ noad_options(fence) |= noad_option_no_check; if (class == middle_noad_subtype && cur_group != math_fence_group) { - tex_aux_append_math_fence_val((mathcodeval) { 0, 0, 0 }, (mathdictval) { 0, 0, 0 }, open_noad_subtype); + tex_aux_append_math_fence_val(tex_no_math_code(), tex_no_dict_code(), open_noad_subtype); } tex_aux_append_math_fence(fence, class); } static void tex_aux_append_math_char(mathcodeval mval, mathdictval dval, int automatic) { - if (mval.class_value == active_math_class_value) { - /*tex An active character is allowed here */ - tex_aux_scan_active_math_char(); - return; - } else { + if (tex_aux_scan_active_math_char(&mval, 2)) { + return; /* rescan pushed back token */ + } else { if (automatic && tex_math_has_class_option(mval.class_value, auto_inject_class_option)) { switch (mval.class_value) { case accent_noad_subtype: @@ -1923,10 +2020,9 @@ static void tex_aux_append_math_char(mathcodeval mval, mathdictval dval, int aut static void tex_aux_append_math_char_in_text(mathcodeval mval, mathdictval dval) { (void) dval; - if (mval.class_value == active_math_class_value) { - /*tex An active character is allowed here. But why in text mode too. */ - tex_aux_scan_active_math_char(); - } else { + if (tex_aux_scan_active_math_char(&mval, 3)) { + return; /* rescan pushed back token */ + } else { halfword p = tex_new_char_node(glyph_character_subtype, tex_fam_fnt(mval.family_value, text_size), mval.character_value, 1); /* todo: data */ tex_tail_append(p); } @@ -1942,8 +2038,8 @@ void tex_run_math_char_number(void) { Both |\char| and |\glyph| get the same treatment. Scanning can change |cur_chr| so we do that first. We no longer check for active here! */ - mathcodeval mval = { 0, 0, 0 }; - mathdictval dval = { 0, 0, 0 }; + mathcodeval mval = tex_no_math_code(); + mathdictval dval = tex_no_dict_code(); cur_chr = tex_scan_char_number(0); mval.character_value = cur_chr; mval.family_value = (short) cur_fam_par; @@ -2057,16 +2153,16 @@ int tex_scan_math_code_val(halfword code, mathcodeval *mval, mathdictval *dval) } void tex_run_text_math_char_number(void) { - mathcodeval mval = { 0, 0, 0 }; - mathdictval dval = { 0, 0, 0 }; + mathcodeval mval = tex_no_math_code(); + mathdictval dval = tex_no_dict_code(); if (tex_scan_math_code_val(cur_chr, &mval, &dval)) { tex_aux_append_math_char_in_text(mval, dval); } } void tex_run_math_math_char_number(void) { - mathcodeval mval = { 0, 0, 0 }; - mathdictval dval = { 0, 0, 0 }; + mathcodeval mval = tex_no_math_code(); + mathdictval dval = tex_no_dict_code(); if (tex_scan_math_code_val(cur_chr, &mval, &dval)) { tex_aux_append_math_char(mval, dval, 1); } @@ -2075,10 +2171,10 @@ void tex_run_math_math_char_number(void) { void tex_run_math_delimiter_number(void) { switch (cur_chr) { case math_delimiter_code: - tex_aux_append_math_char(tex_scan_delimiter_as_mathchar(tex_mathcode), (mathdictval) { 0, 0, 0 }, 0); + tex_aux_append_math_char(tex_scan_delimiter_as_mathchar(tex_mathcode), tex_no_dict_code(), 0); break; case math_udelimiter_code: - tex_aux_append_math_char(tex_scan_delimiter_as_mathchar(umath_mathcode), (mathdictval) { 0, 0, 0 }, 0); + tex_aux_append_math_char(tex_scan_delimiter_as_mathchar(umath_mathcode), tex_no_dict_code(), 0); break; } } @@ -2532,7 +2628,7 @@ void tex_run_math_radical(void) } break; case 's': case 'S': - switch (tex_scan_character("toTO", 0, 0, 0)) { + switch (tex_scan_character("itoITO", 0, 0, 0)) { case 't': case 'T': if (tex_scan_mandate_keyword("style", 2)) { switch (code) { @@ -2554,6 +2650,11 @@ void tex_run_math_radical(void) noad_source(radical) = tex_scan_int(0, NULL); } break; + case 'i': case 'I': + if (tex_scan_mandate_keyword("size", 2)) { + radical_size(radical) = tex_scan_int(0, NULL); + } + break; default: tex_aux_show_keyword_error("style|source"); goto DONE; @@ -3931,7 +4032,7 @@ void tex_run_math_fence(void) break; default: if (cur_group != math_fence_group) { - tex_aux_append_math_fence_val((mathcodeval) { 0, 0, 0 }, (mathdictval) { 0, 0, 0 }, open_noad_subtype); + tex_aux_append_math_fence_val(tex_no_math_code(), tex_no_dict_code(), open_noad_subtype); } switch (cur_group) { case math_fence_group: @@ -5578,7 +5679,7 @@ halfword tex_to_math_rules_parameter(halfword left, halfword right) void tex_set_default_math_codes(void) { - mathcodeval mval = { 0, 0, 0 }; + mathcodeval mval = tex_no_math_code(); /*tex This will remap old font families at runtime. */ mval.class_value = math_use_current_family_code; /*tex Upright math digts come from family 0. */ diff --git a/source/luametatex/source/tex/texmath.h b/source/luametatex/source/tex/texmath.h index a522008f3..79f40c13b 100644 --- a/source/luametatex/source/tex/texmath.h +++ b/source/luametatex/source/tex/texmath.h @@ -282,7 +282,7 @@ typedef enum math_atom_font_options { math_atom_math_font_option = 2, } math_atom_font_options; -inline int math_parameter_value_type(int n) +inline static int math_parameter_value_type(int n) { if (n < last_math_parameter) { return lmt_interface.math_parameter_values[n].type; @@ -563,7 +563,7 @@ extern scaled tex_get_math_x_parameter_default (int style, int param, scaled d extern scaled tex_get_math_y_parameter (int style, int param); extern scaled tex_get_math_y_parameter_checked (int style, int param); -extern scaled tex_get_math_y_parameter_default (int style, int paramm, scaled dflt); +extern scaled tex_get_math_y_parameter_default (int style, int param, scaled dflt); extern scaled tex_get_font_math_parameter (int font, int size, int param); extern scaled tex_get_font_math_x_parameter (int font, int size, int param); @@ -624,6 +624,9 @@ extern void tex_run_text_math_spec (void); extern void tex_set_default_math_codes (void); +extern int tex_check_active_math_char (int character); +extern int tex_pass_active_math_char (int character); + /*tex The runners in maincontrol: */ extern void tex_run_math_left_brace (void); diff --git a/source/luametatex/source/tex/texmathcodes.c b/source/luametatex/source/tex/texmathcodes.c index f19cde91c..33aac71ee 100644 --- a/source/luametatex/source/tex/texmathcodes.c +++ b/source/luametatex/source/tex/texmathcodes.c @@ -82,7 +82,7 @@ mathcodeval tex_mathchar_from_integer(int value, int extcode) mathcodeval tex_mathchar_from_spec(int value) { - mathcodeval mval = { 0, 0, 0 }; + mathcodeval mval = tex_no_math_code(); if (value) { mval.class_value = math_spec_class(value); mval.family_value = math_spec_family(value); @@ -165,7 +165,7 @@ void tex_set_math_code(int n, mathcodeval v, int level) mathcodeval tex_get_math_code(int n) { sa_tree_item item; - mathcodeval m = { 0, 0, 0 }; + mathcodeval m = tex_no_math_code(); sa_get_item_4(lmt_mathcode_state.mathcode_head, n, &item); if (item.uint_value == MATHCODEDEFAULT) { m.character_value = n; @@ -282,6 +282,14 @@ delcodeval tex_get_del_code(int n) return d; } +/*tex */ + +mathdictval tex_no_dict_code(void) +{ + return (mathdictval) { 0, 0, 0 }; +} + + /*tex This really only works for old-style delcodes! */ int tex_get_del_code_number(int n) diff --git a/source/luametatex/source/tex/texmathcodes.h b/source/luametatex/source/tex/texmathcodes.h index a45132171..8017c2c65 100644 --- a/source/luametatex/source/tex/texmathcodes.h +++ b/source/luametatex/source/tex/texmathcodes.h @@ -74,4 +74,6 @@ extern void tex_undump_math_codes (dumpstream f); extern void tex_free_math_codes (void); +extern mathdictval tex_no_dict_code (void); + # endif diff --git a/source/luametatex/source/tex/texmlist.c b/source/luametatex/source/tex/texmlist.c index 0ea1f82e8..b5412872a 100644 --- a/source/luametatex/source/tex/texmlist.c +++ b/source/luametatex/source/tex/texmlist.c @@ -222,7 +222,7 @@ typedef enum limits_modes { limits_horizontal_mode, // no limits } limits_modes; -inline void tex_math_wipe_kerns(kernset *kerns) { +inline static void tex_math_wipe_kerns(kernset *kerns) { if (kerns) { kerns->topright = 0; kerns->topleft = 0; @@ -239,7 +239,7 @@ inline void tex_math_wipe_kerns(kernset *kerns) { } } -inline void tex_math_copy_kerns(kernset *kerns, kernset *parent) { +inline static void tex_math_copy_kerns(kernset *kerns, kernset *parent) { if (kerns && parent) { kerns->topright = parent->topright; kerns->topleft = parent->topleft; @@ -511,7 +511,7 @@ static int tex_aux_math_followed_by_italic_kern(halfword current, const char *tr return 0; } -static inline int tex_aux_checked_left_kern_fnt_chr(halfword fnt, halfword chr, halfword state, halfword subtype) +inline static int tex_aux_checked_left_kern_fnt_chr(halfword fnt, halfword chr, halfword state, halfword subtype) { halfword top = 0; halfword bot = 0; @@ -532,7 +532,7 @@ static inline int tex_aux_checked_left_kern_fnt_chr(halfword fnt, halfword chr, } } -static inline int tex_aux_checked_left_kern(halfword list, halfword state, halfword subtype) +inline static int tex_aux_checked_left_kern(halfword list, halfword state, halfword subtype) { if (list && node_type(list) == glyph_node) { return tex_aux_checked_left_kern_fnt_chr(glyph_font(list), glyph_character(list), state, subtype); @@ -541,7 +541,7 @@ static inline int tex_aux_checked_left_kern(halfword list, halfword state, halfw } } -static inline int tex_aux_checked_right_kern_fnt_chr(halfword fnt, halfword chr, halfword state, halfword subtype) +inline static int tex_aux_checked_right_kern_fnt_chr(halfword fnt, halfword chr, halfword state, halfword subtype) { halfword top = 0; halfword bot = 0; @@ -562,7 +562,7 @@ static inline int tex_aux_checked_right_kern_fnt_chr(halfword fnt, halfword chr, } } -static inline int tex_aux_checked_right_kern(halfword list, halfword state, halfword subtype) +inline static int tex_aux_checked_right_kern(halfword list, halfword state, halfword subtype) { if (list && node_type(list) == glyph_node) { return tex_aux_checked_right_kern_fnt_chr(glyph_font(list), glyph_character(list), state, subtype); @@ -2472,6 +2472,12 @@ static void tex_aux_make_delimited_radical(halfword target, int style, int size, halfword total = height + depth; delimiterextremes extremes = { .tfont = null_font, .tchar = 0, .bfont = null_font, .bchar = 0, .height = 0, .depth = 0 }; noad_new_hlist(target) = null; + size += radical_size(target); + if (size < text_size) { + size = text_size; + } else if (size > script_script_size) { + size = script_script_size; + } delimiter = tex_aux_make_delimiter(target, delimiter, size, total, 0, style, 2, NULL, NULL, 0, has_noad_option_nooverflow(target), &extremes, depth); if (companion) { /*tex For now we assume symmetry and same height and depth! */ @@ -4771,7 +4777,7 @@ inline static scaled tex_aux_insert_italic_now(halfword target, halfword kernel, return italic; } -static inline int tex_aux_raise_prime_composed(halfword target) +inline static int tex_aux_raise_prime_composed(halfword target) { int mainclass = -1 ; /* maybe also mainclass */ @@ -4891,12 +4897,14 @@ static void tex_aux_make_scripts(halfword target, halfword kernel, scaled italic primedata.node = tex_aux_analyze_script(noad_prime(target), &primedata); maxleftkern = tex_aux_math_left_kern(glyph_font(kernel), glyph_character(kernel)); // maxrightkern = tex_aux_math_right_kern(glyph_font(kernel), glyph_character(kernel)); - prime_up = tex_get_math_y_parameter_default(style, math_parameter_prime_shift_drop, 0); - shift_up = tex_get_math_y_parameter_checked(style, math_parameter_superscript_shift_drop); - shift_down = tex_get_math_y_parameter_checked(style, math_parameter_subscript_shift_drop); - break; // fallthrough + prime_up = 0; + shift_up = 0; + shift_down = 0; + break; default: - kernelsize.ht -= supdrop; /* new */ + /*tex Used for optimizing accents. */ + kernelsize.ht -= supdrop; + /*tex These parameters are only applied in an assembly (and often some 0.5 .. 1.5 pt on 12pt). */ prime_up = kernelsize.ht - tex_get_math_y_parameter_default(style, math_parameter_prime_shift_drop, 0); shift_up = kernelsize.ht - tex_get_math_y_parameter_checked(style, math_parameter_superscript_shift_drop); shift_down = kernelsize.dp + tex_get_math_y_parameter_checked(style, math_parameter_subscript_shift_drop); @@ -5372,7 +5380,7 @@ static void tex_aux_make_scripts(halfword target, halfword kernel, scaled italic */ -// static inline int tex_aux_is_extensible(halfword result) +// inline static int tex_aux_is_extensible(halfword result) // { // if (result) { // switch (node_type(result)) { @@ -6119,7 +6127,7 @@ static halfword tex_aux_unroll_noad(halfword tail, halfword l, quarterword s) while (l) { halfword n = node_next(l); node_next(l) = null; - if (node_type(l) == hlist_node && (node_subtype(l) == s) && ! box_source_anchor(l)) { + if (node_type(l) == hlist_node && node_subtype(l) == s && ! box_source_anchor(l)) { if (box_list(l)) { tex_couple_nodes(tail, box_list(l)); tail = tex_tail_of_node_list(tail); @@ -7052,6 +7060,9 @@ static void tex_mlist_to_hlist_finalize_list(mliststate *state) } else { tex_couple_nodes(p, l); } + } else if ((current_subtype == open_noad_subtype || current_subtype == fenced_noad_subtype) && tex_math_has_class_option(fenced_noad_subtype, unpack_class_option)) { + /*tex tricky as we have an open subtype for spacing now. */ + p = tex_aux_unroll_noad(p, l, math_fence_list); } else if (has_noad_option_unpacklist(current) || tex_math_has_class_option(current_subtype, unpack_class_option)) { /*tex So here we only unpack a math list. */ p = tex_aux_unroll_noad(p, l, math_list_list); diff --git a/source/luametatex/source/tex/texnesting.c b/source/luametatex/source/tex/texnesting.c index d699d58fc..f281eee2e 100644 --- a/source/luametatex/source/tex/texnesting.c +++ b/source/luametatex/source/tex/texnesting.c @@ -182,7 +182,7 @@ nest_state_info lmt_nest_state = { .offset = 0, }, .shown_mode = 0, - .padding = 0, + .math_mode = 0, }; /*tex @@ -239,7 +239,8 @@ void tex_initialize_nesting(void) { lmt_nest_state.nest_data.ptr = 0; lmt_nest_state.nest_data.top = 0; - lmt_nest_state.shown_mode = 0; + // lmt_nest_state.shown_mode = 0; + // lmt_nest_state.math_mode = 0; cur_list.mode = vmode; cur_list.head = contribute_head; cur_list.tail = contribute_head; @@ -290,6 +291,8 @@ void tex_push_nest(void) { list_state_record *top = &lmt_nest_state.nest[lmt_nest_state.nest_data.ptr]; lmt_nest_state.nest_data.ptr += 1; + // lmt_nest_state.shown_mode = 0; // needs checking + lmt_nest_state.math_mode = 0; if (tex_aux_room_on_nest_stack()) { cur_list.mode = top->mode; cur_list.head = tex_new_temp_node(); diff --git a/source/luametatex/source/tex/texnesting.h b/source/luametatex/source/tex/texnesting.h index f940094a0..91251eee0 100644 --- a/source/luametatex/source/tex/texnesting.h +++ b/source/luametatex/source/tex/texnesting.h @@ -30,7 +30,7 @@ typedef struct nest_state_info { list_state_record *nest; memory_data nest_data; int shown_mode; - int padding; + int math_mode; } nest_state_info; extern nest_state_info lmt_nest_state; diff --git a/source/luametatex/source/tex/texnodes.c b/source/luametatex/source/tex/texnodes.c index 0134eb419..ab2601a43 100644 --- a/source/luametatex/source/tex/texnodes.c +++ b/source/luametatex/source/tex/texnodes.c @@ -1800,6 +1800,24 @@ halfword tex_list_node_mem_usage(void) (actually for each node tyep I guess). */ +extern void tex_change_attribute_register(halfword a, halfword id, halfword value) +{ + if (eq_value(id) != value) { + if (is_global(a)) { + int i; + for (i = (lmt_save_state.save_stack_data.ptr - 1); i >= 0; i--) { + if (save_type(i) == attribute_list_save_type) { + delete_attribute_reference(save_value(i)); + save_value(i) = attribute_cache_disabled; + } + } + } else { + delete_attribute_reference(current_attribute_state); + } + set_current_attribute_state(attribute_cache_disabled); + } +} + inline static halfword tex_aux_new_attribute_list_node(halfword count) { halfword r = tex_get_node(attribute_node_size); diff --git a/source/luametatex/source/tex/texnodes.h b/source/luametatex/source/tex/texnodes.h index 9e24ada27..50dbeaa97 100644 --- a/source/luametatex/source/tex/texnodes.h +++ b/source/luametatex/source/tex/texnodes.h @@ -130,8 +130,9 @@ typedef enum node_types { align_stack_node, noad_state_node, if_node, - unhyphenated_node, /*tex These are both active nodes. */ - hyphenated_node, /*tex These are both active nodes. */ + /*tex These two are active nodes. */ + unhyphenated_node, + hyphenated_node, delta_node, passive_node, } node_types; @@ -988,17 +989,17 @@ typedef enum rule_codes { # define first_rule_code normal_rule_code # define last_rule_code strut_rule_code -# define rule_node_size 7 -# define rule_width(a) vlink(a,2) -# define rule_x_offset(a) vinfo(a,2) -# define rule_depth(a) vlink(a,3) -# define rule_y_offset(a) vinfo(a,3) -# define rule_height(a) vlink(a,4) -# define rule_data(a) vinfo(a,4) -# define rule_left(a) vinfo(a,5) -# define rule_right(a) vlink(a,5) -# define rule_font(a) vinfo(a,6) -# define rule_character(a) vlink(a,6) +# define rule_node_size 7 +# define rule_width(a) vlink(a,2) +# define rule_x_offset(a) vinfo(a,2) +# define rule_depth(a) vlink(a,3) +# define rule_y_offset(a) vinfo(a,3) +# define rule_height(a) vlink(a,4) +# define rule_data(a) vinfo(a,4) +# define rule_left(a) vinfo(a,5) +# define rule_right(a) vlink(a,5) +# define rule_font(a) vinfo(a,6) +# define rule_character(a) vlink(a,6) # define rule_total(a) (rule_height(a) + rule_depth(a)) @@ -1057,36 +1058,36 @@ typedef enum rule_codes { */ -//define glyph_node_size 12 -# define glyph_node_size 13 -# define glyph_character(a) vinfo(a,2) -# define glyph_font(a) vlink(a,2) -# define glyph_data(a) vinfo(a,3) /*tex We had that unused, so now it's like an attribute. */ -# define glyph_state(a) vlink(a,3) /*tex A user field (can be handy in \LUA). */ -# define glyph_language(a) vinfo(a,4) -# define glyph_script(a) vlink(a,4) -# define glyph_options(a) vinfo(a,5) -# define glyph_hyphenate(a) vlink(a,5) -# define glyph_protected(a) vinfo00(a,6) -# define glyph_lhmin(a) vinfo01(a,6) -# define glyph_rhmin(a) vinfo02(a,6) -# define glyph_discpart(a) vinfo03(a,6) -# define glyph_expansion(a) vlink(a,6) -# define glyph_x_scale(a) vinfo(a,7) -# define glyph_y_scale(a) vlink(a,7) -# define glyph_scale(a) vinfo(a,8) -# define glyph_raise(a) vlink(a,8) -# define glyph_left(a) vinfo(a,9) -# define glyph_right(a) vlink(a,9) -# define glyph_x_offset(a) vinfo(a,10) -# define glyph_y_offset(a) vlink(a,10) -//define glyph_input_file(a) vinfo(a,11) /* aka glyph_synctex_tag */ -//define glyph_input_line(a) vlink(a,11) /* aka glyph_synctex_line */ -# define glyph_properties(a) vinfo0(a,11) -# define glyph_group(a) vinfo1(a,11) -# define glyph_index(a) vlink(a,11) -# define glyph_input_file(a) vinfo(a,12) -# define glyph_input_line(a) vlink(a,12) +//define glyph_node_size 12 +# define glyph_node_size 13 +# define glyph_character(a) vinfo(a,2) +# define glyph_font(a) vlink(a,2) +# define glyph_data(a) vinfo(a,3) /*tex We had that unused, so now it's like an attribute. */ +# define glyph_state(a) vlink(a,3) /*tex A user field (can be handy in \LUA). */ +# define glyph_language(a) vinfo(a,4) +# define glyph_script(a) vlink(a,4) +# define glyph_options(a) vinfo(a,5) +# define glyph_hyphenate(a) vlink(a,5) +# define glyph_protected(a) vinfo00(a,6) +# define glyph_lhmin(a) vinfo01(a,6) +# define glyph_rhmin(a) vinfo02(a,6) +# define glyph_discpart(a) vinfo03(a,6) +# define glyph_expansion(a) vlink(a,6) +# define glyph_x_scale(a) vinfo(a,7) +# define glyph_y_scale(a) vlink(a,7) +# define glyph_scale(a) vinfo(a,8) +# define glyph_raise(a) vlink(a,8) +# define glyph_left(a) vinfo(a,9) +# define glyph_right(a) vlink(a,9) +# define glyph_x_offset(a) vinfo(a,10) +# define glyph_y_offset(a) vlink(a,10) +//define glyph_input_file(a) vinfo(a,11) /* aka glyph_synctex_tag */ +//define glyph_input_line(a) vlink(a,11) /* aka glyph_synctex_line */ +# define glyph_properties(a) vinfo0(a,11) +# define glyph_group(a) vinfo1(a,11) +# define glyph_index(a) vlink(a,11) +# define glyph_input_file(a) vinfo(a,12) +# define glyph_input_line(a) vlink(a,12) # define get_glyph_data(a) ((halfword) glyph_data(a)) # define get_glyph_state(a) ((halfword) glyph_state(a)) @@ -1477,6 +1478,10 @@ typedef enum specification_options { # define specification_n(a,n) (specification_repeat(a) ? ((n - 1) % specification_count(a) + 1) : (n > specification_count(a) ? specification_count(a) : n)) +/* interesting: 1Kb smaller bin: */ + +// inline static halfword specification_n(halfword a, halfword n) { return specification_repeat(a) ? ((n - 1) % specification_count(a) + 1) : (n > specification_count(a) ? specification_count(a) : n); } + extern void tex_null_specification_list (halfword a); extern void tex_new_specification_list (halfword a, halfword n, halfword o); extern void tex_dispose_specification_list (halfword a); @@ -1596,15 +1601,15 @@ typedef enum simple_choice_subtypes { */ -# define noad_state_node_size 6 -# define noad_state_topright(a) vlink(a,2) -# define noad_state_bottomright(a) vinfo(a,2) -# define noad_state_topleft(a) vlink(a,3) -# define noad_state_bottomleft(a) vinfo(a,3) -# define noad_state_height(a) vlink(a,4) -# define noad_state_depth(a) vinfo(a,4) -# define noad_state_toptotal(a) vlink(a,5) -# define noad_state_bottomtotal(a) vinfo(a,5) +# define noad_state_node_size 6 +# define noad_state_topright(a) vlink(a,2) +# define noad_state_bottomright(a) vinfo(a,2) +# define noad_state_topleft(a) vlink(a,3) +# define noad_state_bottomleft(a) vinfo(a,3) +# define noad_state_height(a) vlink(a,4) +# define noad_state_depth(a) vinfo(a,4) +# define noad_state_toptotal(a) vlink(a,5) +# define noad_state_bottomtotal(a) vinfo(a,5) # define noad_size 14 # define noad_new_hlist(a) vlink(a,2) /*tex the translation of an mlist; a bit confusing name */ @@ -1754,7 +1759,7 @@ inline static void tex_add_noad_option (halfword a, halfword r) { noad_option inline static void tex_remove_noad_option (halfword a, halfword r) { noad_options(a) &= ~(r | noad_options(a)); } inline static int tex_has_noad_option (halfword a, halfword r) { return (noad_options(a) & r) == r; } -inline int has_noad_no_script_option(halfword n, halfword option) +inline static int has_noad_no_script_option(halfword n, halfword option) { switch (node_type(n)) { case simple_noad: @@ -1880,13 +1885,13 @@ typedef enum math_modifier_types { /*tex accent noads: todo, left and right offsets and options */ -# define accent_noad_size noad_size -# define accent_top_character noad_extra_1 /*tex the |top_accent_chr| field of an accent noad */ -# define accent_bottom_character noad_extra_2 /*tex the |bot_accent_chr| field of an accent noad */ -# define accent_middle_character noad_extra_3 /*tex the |overlay_accent_chr| field of an accent noad */ -# define accent_fraction noad_extra_4 -# define accent_top_overshoot noad_extra_5 -# define accent_bot_overshoot noad_extra_6 +# define accent_noad_size noad_size +# define accent_top_character noad_extra_1 /*tex the |top_accent_chr| field of an accent noad */ +# define accent_bottom_character noad_extra_2 /*tex the |bot_accent_chr| field of an accent noad */ +# define accent_middle_character noad_extra_3 /*tex the |overlay_accent_chr| field of an accent noad */ +# define accent_fraction noad_extra_4 +# define accent_top_overshoot noad_extra_5 +# define accent_bot_overshoot noad_extra_6 typedef enum math_accent_subtypes { bothflexible_accent_subtype, @@ -1965,6 +1970,7 @@ typedef enum fraction_subtypes { # define radical_degree noad_extra_1 # define radical_left_delimiter noad_extra_2 # define radical_right_delimiter noad_extra_3 +# define radical_size noad_extra_4 # define radical_height noad_extra_5 # define radical_depth noad_extra_6 @@ -2513,9 +2519,10 @@ inline static void tex_attach_attribute_list_attribute(halfword target, halfword # define attach_current_attribute_list tex_build_attribute_list /* (target) */ # define set_current_attribute_state(v) do { \ - current_attribute_state = v; \ + current_attribute_state = v; \ } while (0) +/* # define change_attribute_register(a,id,value) do { \ if (eq_value(id) != value) { \ if (is_global(a)) { \ @@ -2532,6 +2539,9 @@ inline static void tex_attach_attribute_list_attribute(halfword target, halfword set_current_attribute_state(attribute_cache_disabled); \ } \ } while (0) +*/ + +extern void tex_change_attribute_register(halfword a, halfword id, halfword value); # define save_attribute_state_before() do { \ halfword c = current_attribute_state; \ @@ -2621,8 +2631,8 @@ typedef enum glue_signs { # define normal_glue_multiplier 0.0 -inline halfword tex_checked_glue_sign (halfword sign) { return ((sign < min_glue_sign ) || (sign > max_glue_sign )) ? normal_glue_sign : sign ; } -inline halfword tex_checked_glue_order (halfword order) { return ((order < min_glue_order) || (order > max_glue_order)) ? normal_glue_order : order; } +inline static halfword tex_checked_glue_sign (halfword sign) { return ((sign < min_glue_sign ) || (sign > max_glue_sign )) ? normal_glue_sign : sign ; } +inline static halfword tex_checked_glue_order (halfword order) { return ((order < min_glue_order) || (order > max_glue_order)) ? normal_glue_order : order; } /*tex These are reserved nodes that sit at the start of main memory. We could actually just allocate diff --git a/source/luametatex/source/tex/texprimitive.c b/source/luametatex/source/tex/texprimitive.c index d5f2b0927..9875e06dd 100644 --- a/source/luametatex/source/tex/texprimitive.c +++ b/source/luametatex/source/tex/texprimitive.c @@ -743,8 +743,10 @@ void tex_print_cmd_chr(singleword cmd, halfword chr) case other_char_cmd: tex_aux_print_chr_cmd("the", cmd, chr); break; - /* case active_char_cmd: + tex_aux_print_chr_cmd("active", cmd, chr); + break; + /* case comment_cmd: case invalid_char_cmd: break; diff --git a/source/luametatex/source/tex/texprinting.h b/source/luametatex/source/tex/texprinting.h index b323b1ae5..5e311a0bb 100644 --- a/source/luametatex/source/tex/texprinting.h +++ b/source/luametatex/source/tex/texprinting.h @@ -105,12 +105,11 @@ inline static int tex_single_letter(strnumber s) inline static int tex_is_active_cs(strnumber s) { if (s && str_length(s) > 3) { - const unsigned char *ss = str_string(s); - return (ss[0] == 0xEF) && (ss[1] == 0xBF) && (ss[2] == 0xBF); + const unsigned char *ss = str_string(s); // or signed and active_character ... + return (ss[0] == active_first) && (ss[1] == active_second) && (ss[2] == active_third); } else { return 0; } } -# define active_cs_value(A) aux_str2uni((str_string((A))+3)) # endif diff --git a/source/luametatex/source/tex/texscanning.c b/source/luametatex/source/tex/texscanning.c index 5480910dc..3c9859de9 100644 --- a/source/luametatex/source/tex/texscanning.c +++ b/source/luametatex/source/tex/texscanning.c @@ -11,7 +11,7 @@ static void tex_aux_scan_expression (int level); A helper. */ -inline void tex_push_back(halfword tok, halfword cmd, halfword chr) +inline static void tex_push_back(halfword tok, halfword cmd, halfword chr) { if (cmd != spacer_cmd && tok != deep_frozen_relax_token && ! (cmd == relax_cmd && chr == no_relax_code)) { tex_back_input(tok); @@ -670,7 +670,7 @@ static int tex_aux_set_cur_val_by_some_cmd(int code) case math_char_slot_code: /* we actually need two commands or we need to look ahead */ { - mathcodeval mval = { 0, 0, 0 }; + mathcodeval mval = tex_no_math_code(); mathdictval dval = { 0, 0, 0 }; if (tex_scan_math_cmd_val(&mval, &dval)) { switch (code) { @@ -1059,6 +1059,9 @@ static void tex_aux_set_cur_val_by_define_char_cmd(int chr) case hmcode_charcode: chr = tex_get_hm_code(index); break; + case amcode_charcode: + chr = tex_get_am_code(index); + break; case mathcode_charcode: case extmathcode_charcode: chr = tex_get_math_code_number(index); @@ -1820,7 +1823,7 @@ halfword tex_scan_math_index_number (void) { return tex_ halfword tex_scan_math_discretionary_number (int optional_equal) { return tex_aux_scan_limited_int(optional_equal, 0, max_math_discretionary, "Math discretionary"); } singleword tex_scan_box_index (void) { return (singleword) tex_aux_scan_limited_int(0, 0, max_box_index, "Box index"); } singleword tex_scan_box_axis (void) { return (singleword) tex_aux_scan_limited_int(0, 0, max_box_axis, "Box axis"); } -halfword tex_scan_category_code (void) { return tex_aux_scan_limited_int(0, 0, max_category_code,"Category code"); } +halfword tex_scan_category_code (int optional_equal) { return tex_aux_scan_limited_int(optional_equal, 0, max_category_code,"Category code"); } halfword tex_scan_function_reference (int optional_equal) { return tex_aux_scan_limited_int(optional_equal, 0, max_function_reference, "Function reference"); } halfword tex_scan_bytecode_reference (int optional_equal) { return tex_aux_scan_limited_int(optional_equal, 0, max_bytecode_index, "Bytecode reference"); } halfword tex_scan_limited_scale (int optional_equal) { return tex_aux_scan_limited_int(optional_equal, -max_limited_scale, max_limited_scale, "Limited scale"); } diff --git a/source/luametatex/source/tex/texscanning.h b/source/luametatex/source/tex/texscanning.h index 90897bf54..303143587 100644 --- a/source/luametatex/source/tex/texscanning.h +++ b/source/luametatex/source/tex/texscanning.h @@ -120,7 +120,7 @@ extern halfword tex_scan_math_properties_number (void); extern halfword tex_scan_math_group_number (void); extern halfword tex_scan_math_index_number (void); extern halfword tex_scan_math_discretionary_number (int optional_equal); -extern halfword tex_scan_category_code (void); +extern halfword tex_scan_category_code (int optional_equal); extern singleword tex_scan_box_index (void); /*tex For local boxes: small for now! */ extern singleword tex_scan_box_axis (void); extern halfword tex_scan_function_reference (int optional_equal); diff --git a/source/luametatex/source/tex/texstringpool.c b/source/luametatex/source/tex/texstringpool.c index fd93758b5..ffe4177bf 100644 --- a/source/luametatex/source/tex/texstringpool.c +++ b/source/luametatex/source/tex/texstringpool.c @@ -82,7 +82,7 @@ string_pool_info lmt_string_pool_state = { # define initial_temp_string_slots 256 # define reserved_temp_string_slots 2 -static inline void tex_aux_increment_pool_string(int n) +inline static void tex_aux_increment_pool_string(int n) { lmt_string_pool_state.string_body_data.allocated += n; if (lmt_string_pool_state.string_body_data.allocated > lmt_string_pool_state.string_body_data.size) { @@ -90,7 +90,7 @@ static inline void tex_aux_increment_pool_string(int n) } } -static inline void tex_aux_decrement_pool_string(int n) +inline static void tex_aux_decrement_pool_string(int n) { lmt_string_pool_state.string_body_data.allocated -= n; } diff --git a/source/luametatex/source/tex/texstringpool.h b/source/luametatex/source/tex/texstringpool.h index 7302e7fb4..a15b9fad5 100644 --- a/source/luametatex/source/tex/texstringpool.h +++ b/source/luametatex/source/tex/texstringpool.h @@ -76,37 +76,37 @@ extern string_pool_info lmt_string_pool_state; /*tex Forget the last character in the pool. */ -inline void tex_flush_char(void) { --lmt_string_pool_state.string_temp_top; } - -extern strnumber tex_make_string (void); -extern strnumber tex_push_string (const unsigned char *s, int l); -extern char *tex_take_string (int *len); -extern int tex_str_eq_buf (strnumber s, int k, int n); -extern int tex_str_eq_str (strnumber s, strnumber t); -extern int tex_str_eq_cstr (strnumber s, const char *, size_t); -extern int tex_get_strings_started (void); -extern void tex_reset_cur_string (void); -/* strnumber tex_search_string (strnumber search); */ -/* int tex_used_strings (void); */ -extern strnumber tex_maketexstring (const char *s); -extern strnumber tex_maketexlstring (const char *s, size_t); -extern void tex_append_char (unsigned char c); -extern void tex_append_string (const unsigned char *s, unsigned l); -extern char *tex_makecstring (int s, int *allocated); -extern char *tex_makeclstring (int s, size_t *len); -extern void tex_dump_string_pool (dumpstream f); -extern void tex_undump_string_pool (dumpstream f); -extern void tex_initialize_string_pool (void); -extern void tex_initialize_string_mem (void); -extern void tex_flush_str (strnumber s); -extern strnumber tex_save_cur_string (void); -extern void tex_restore_cur_string (strnumber u); - -/* void tex_increment_pool_string (int n); */ -/* void tex_decrement_pool_string (int n); */ - -extern void tex_compact_string_pool (void); - -inline char *tex_to_cstring (int s) { return str_length(s) > 0 ? (char *) str_string(s) : ""; } +inline static void tex_flush_char(void) { --lmt_string_pool_state.string_temp_top; } + +extern strnumber tex_make_string (void); +extern strnumber tex_push_string (const unsigned char *s, int l); +extern char *tex_take_string (int *len); +extern int tex_str_eq_buf (strnumber s, int k, int n); +extern int tex_str_eq_str (strnumber s, strnumber t); +extern int tex_str_eq_cstr (strnumber s, const char *, size_t); +extern int tex_get_strings_started (void); +extern void tex_reset_cur_string (void); +/* strnumber tex_search_string (strnumber search); */ +/* int tex_used_strings (void); */ +extern strnumber tex_maketexstring (const char *s); +extern strnumber tex_maketexlstring (const char *s, size_t); +extern void tex_append_char (unsigned char c); +extern void tex_append_string (const unsigned char *s, unsigned l); +extern char *tex_makecstring (int s, int *allocated); +extern char *tex_makeclstring (int s, size_t *len); +extern void tex_dump_string_pool (dumpstream f); +extern void tex_undump_string_pool (dumpstream f); +extern void tex_initialize_string_pool (void); +extern void tex_initialize_string_mem (void); +extern void tex_flush_str (strnumber s); +extern strnumber tex_save_cur_string (void); +extern void tex_restore_cur_string (strnumber u); + +/* void tex_increment_pool_string (int n); */ +/* void tex_decrement_pool_string (int n); */ + +extern void tex_compact_string_pool (void); + +inline static char *tex_to_cstring (int s) { return str_length(s) > 0 ? (char *) str_string(s) : ""; } # endif diff --git a/source/luametatex/source/tex/textextcodes.c b/source/luametatex/source/tex/textextcodes.c index 39fc258c7..2fef9857f 100644 --- a/source/luametatex/source/tex/textextcodes.c +++ b/source/luametatex/source/tex/textextcodes.c @@ -218,12 +218,16 @@ static void tex_aux_free_catcodes(void) # define HMCODESTACK 8 # define HMCODEDEFAULT 0 +# define AMCODESTACK 8 +# define AMCODEDEFAULT 0 + typedef struct luscode_state_info { sa_tree uccode_head; sa_tree lccode_head; sa_tree sfcode_head; sa_tree hccode_head; sa_tree hmcode_head; + sa_tree amcode_head; } luscode_state_info; static luscode_state_info lmt_luscode_state = { @@ -231,7 +235,8 @@ static luscode_state_info lmt_luscode_state = { .lccode_head = NULL, .sfcode_head = NULL, .hccode_head = NULL, - .hmcode_head = NULL + .hmcode_head = NULL, + .amcode_head = NULL }; void tex_set_lc_code(int n, halfword v, int gl) @@ -445,6 +450,45 @@ static void tex_aux_free_hmcodes(void) sa_destroy_tree(lmt_luscode_state.hmcode_head); } +/*tex Experiment. */ + + +void tex_set_am_code(int n, halfword v, int gl) +{ + sa_set_item_1(lmt_luscode_state.amcode_head, n, v, gl); +} + +halfword tex_get_am_code(int n) +{ + return sa_return_item_1(lmt_luscode_state.amcode_head, n); +} + +static void tex_aux_unsave_amcodes(int gl) +{ + sa_restore_stack(lmt_luscode_state.amcode_head, gl); +} + +static void tex_aux_initialize_amcodes(void) +{ + sa_tree_item item = { .int_value = AMCODEDEFAULT }; + lmt_luscode_state.amcode_head = sa_new_tree(AMCODESTACK, 1, item); +} + +static void tex_aux_dump_amcodes(dumpstream f) +{ + sa_dump_tree(f, lmt_luscode_state.amcode_head); +} + +static void tex_aux_undump_amcodes(dumpstream f) +{ + lmt_luscode_state.amcode_head = sa_undump_tree(f); +} + +static void tex_aux_free_amcodes(void) +{ + sa_destroy_tree(lmt_luscode_state.amcode_head); +} + /*tex The hyphenation codes are indeed stored in a tree and are used instead of lowercase codes when @@ -544,6 +588,7 @@ void tex_unsave_text_codes(int grouplevel) tex_aux_unsave_sfcodes(grouplevel); tex_aux_unsave_hccodes(grouplevel); tex_aux_unsave_hmcodes(grouplevel); + tex_aux_unsave_amcodes(grouplevel); } void tex_initialize_text_codes(void) @@ -554,6 +599,7 @@ void tex_initialize_text_codes(void) tex_aux_initialize_sfcodes(); tex_aux_initialize_hccodes(); tex_aux_initialize_hmcodes(); + tex_aux_initialize_amcodes(); /* initializehjcodes(); */ } @@ -565,6 +611,7 @@ void tex_free_text_codes(void) tex_aux_free_sfcodes(); tex_aux_free_hccodes(); tex_aux_free_hmcodes(); + tex_aux_free_amcodes(); /* freehjcodes(); */ } @@ -576,6 +623,7 @@ void tex_dump_text_codes(dumpstream f) tex_aux_dump_sfcodes(f); tex_aux_dump_hccodes(f); tex_aux_dump_hmcodes(f); + tex_aux_dump_amcodes(f); /* dumphjcodes(f); */ } @@ -587,6 +635,7 @@ void tex_undump_text_codes(dumpstream f) tex_aux_undump_sfcodes(f); tex_aux_undump_hccodes(f); tex_aux_undump_hmcodes(f); + tex_aux_undump_amcodes(f); /* undumphjcodes(f); */ } diff --git a/source/luametatex/source/tex/textextcodes.h b/source/luametatex/source/tex/textextcodes.h index 476f0f03e..d9fd7fbe4 100644 --- a/source/luametatex/source/tex/textextcodes.h +++ b/source/luametatex/source/tex/textextcodes.h @@ -29,6 +29,8 @@ extern void tex_set_hc_code (int n, halfword v, int gl); extern halfword tex_get_hc_code (int n); extern void tex_set_hm_code (int n, halfword v, int gl); extern halfword tex_get_hm_code (int n); +extern void tex_set_am_code (int n, halfword v, int gl); +extern halfword tex_get_am_code (int n); extern void tex_set_hj_code (int l, int n, halfword v, int gl); extern halfword tex_get_hj_code (int l, int n); extern void tex_initialize_xx_codes (void); diff --git a/source/luametatex/source/tex/textoken.c b/source/luametatex/source/tex/textoken.c index 7dd9c888b..ba457491a 100644 --- a/source/luametatex/source/tex/textoken.c +++ b/source/luametatex/source/tex/textoken.c @@ -573,7 +573,7 @@ static const char *tex_aux_special_cmd_string(halfword cmd, halfword chr, const case begin_local_cmd : return "[[special cmd: begin local call]]"; case end_local_cmd : return "[[special cmd: end local call]]"; // case prefix_cmd : return "[[special cmd: enforced]]"; - case prefix_cmd : return "\\always"; + case prefix_cmd : return "\\always "; default : printf("[[unknown cmd: (%i,%i)]\n", cmd, chr); return unknown; } } @@ -625,6 +625,7 @@ halfword tex_show_token_list(halfword p, halfword q, int l, int asis) case spacer_cmd: case letter_cmd: case other_char_cmd: + case active_char_cmd: /* new */ case ignore_cmd: /* new */ tex_print_tex_str(chr); break; @@ -691,7 +692,7 @@ halfword tex_show_token_list(halfword p, halfword q, int l, int asis) } while (0) */ -inline halfword get_unichar_from_buffer(int *b) +inline static halfword get_unichar_from_buffer(int *b) { halfword a = (halfword) ((const unsigned char) *(lmt_fileio_state.io_buffer + *b)); if (a <= 0x80) { @@ -1032,14 +1033,13 @@ int tex_scan_keyword_case_sensitive(const char *s) halfword tex_active_to_cs(int c, int force) { halfword cs = -1; - if (c > 0) { - /*tex This is not that efficient: we can make a helper that doesn't use an alloc. */ - char utfbytes[8] = { '\xEF', '\xBF', '\xBF', 0 }; + if (c >= 0 && c <= max_character_code) { + char utfbytes[8] = { active_character_first, active_character_second, active_character_third, 0 }; aux_uni2string((char *) &utfbytes[3], c); cs = tex_string_locate(utfbytes, (size_t) utf8_size(c) + 3, force); } if (cs < 0) { - cs = tex_string_locate("\xEF\xBF\xBF", 4, force); /*tex Including the zero sentinel. */ + cs = tex_string_locate(active_character_unknown, 4, force); /*tex Including the zero sentinel. */ } return cs; } @@ -1195,10 +1195,18 @@ static int tex_aux_get_next_file(void) case mid_line_state + active_char_cmd: case new_line_state + active_char_cmd: case skip_blanks_state + active_char_cmd: - /*tex Process an active-character. */ - cur_cs = tex_active_to_cs(cur_chr, ! lmt_hash_state.no_new_cs); - cur_cmd = eq_type(cur_cs); - cur_chr = eq_value(cur_cs); + /*tex Process an active-character. */ + if ((lmt_input_state.scanner_status == scanner_is_tolerant || lmt_input_state.scanner_status == scanner_is_matching) && tex_pass_active_math_char(cur_chr)) { + /*tex We need to intercept a delimiter in arguments. */ + } else if ((lmt_input_state.scanner_status == scanner_is_defining || lmt_input_state.scanner_status == scanner_is_absorbing) && tex_pass_active_math_char(cur_chr)) { + /*tex We are storing stuff in a token list or macro body. */ + } else if ((cur_mode == mmode || lmt_nest_state.math_mode) && tex_check_active_math_char(cur_chr)) { + /*tex We have an intercept. */ + } else { + cur_cs = tex_active_to_cs(cur_chr, ! lmt_hash_state.no_new_cs); + cur_cmd = eq_type(cur_cs); + cur_chr = eq_value(cur_cs); + } lmt_input_state.cur_input.state = mid_line_state; break; case mid_line_state + superscript_cmd: @@ -2730,6 +2738,7 @@ void tex_run_convert_tokens(halfword code) break; } case cs_string_code: + case cs_active_code: { int saved_selector; int saved_scanner_status = lmt_input_state.scanner_status; @@ -2737,7 +2746,18 @@ void tex_run_convert_tokens(halfword code) tex_get_token(); lmt_input_state.scanner_status = saved_scanner_status; push_selector; - if (cur_cs) { + if (code == cs_active_code) { + // tex_print_char(active_first); + // tex_print_char(active_second); + // tex_print_char(active_third); + tex_print_str(active_character_namespace); + if (cur_cmd == active_char_cmd) { + tex_print_char(cur_chr); + } else { + /*tex So anything else will just inject the hash (abstraction, saves a command). */ + tex_back_input(cur_tok); + } + } else if (cur_cs) { tex_print_cs_name(cur_cs); } else { tex_print_tex_str(cur_chr); @@ -3122,6 +3142,7 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb case spacer_cmd: case letter_cmd: case other_char_cmd: + case active_char_cmd: if (! skip) { tex_aux_append_uchar_to_buffer(chr); } @@ -3335,7 +3356,7 @@ void tex_set_tex_attribute_register(int j, halfword v, int flags, int internal) if (j > lmt_node_memory_state.max_used_attribute) { lmt_node_memory_state.max_used_attribute = j; } - change_attribute_register(flags, register_attribute_location(j), v); + tex_change_attribute_register(flags, register_attribute_location(j), v); tex_word_define(flags, internal ? internal_attribute_location(j) : register_attribute_location(j), v); } diff --git a/source/luametatex/source/tex/textoken.h b/source/luametatex/source/tex/textoken.h index ad67dfcb5..843304405 100644 --- a/source/luametatex/source/tex/textoken.h +++ b/source/luametatex/source/tex/textoken.h @@ -390,6 +390,43 @@ extern halfword tex_copy_token_list (halfword h, halfword *t); extern halfword tex_parse_str_to_tok (halfword head, halfword *tail, halfword ct, const char *str, size_t lstr, int option); -inline int tex_valid_token (int t) { return ((t >= 0) && (t <= (int) lmt_token_memory_state.tokens_data.top)); } +inline static int tex_valid_token (int t) { return ((t >= 0) && (t <= (int) lmt_token_memory_state.tokens_data.top)); } + +/*tex + + This is also a sort of documentation. Active characters are stored in the hash using a prefix + which assumes that users don't use that one. So far we've seen no clashes which is due to the + fact that the namespace prefix U+FFFF is an invalid \UNICODE\ character and it's kind of hard + to get that one into the input anyway. + + The replacement character U+FFFD is a kind of fallback when we run into some troubles or when + a control sequence is expected (and undefined is unacceptable). + + U+FFFD REPLACEMENT CHARACTER + U+FFFE NOT A CHARACTER + U+FFFF NOT A CHARACTER + + I experimented with a namespace character (catcodtable id) as fourth character but there are + some unwanted side effects, for instance in testing an active character as separator (in + arguments) so that code waa eventually removed. I might come back to this one day (active + characters in the catcode regime namespace). + +*/ + +# define utf_fffd_string "\xEF\xBF\xBD" /* U+FFFD : 65533 */ + +# define active_character_namespace "\xEF\xBF\xBF" /* U+FFFF : 65535 */ + +# define active_character_first '\xEF' +# define active_character_second '\xBF' +# define active_character_third '\xBF' + +# define active_first 0xEF +# define active_second 0xBF +# define active_third 0xBF + +# define active_character_unknown "\xEF\xBF\xBD" /* utf_fffd_string */ + +# define active_cs_value(A) aux_str2uni(str_string(A)+3) # endif diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii index ad689d99c..163951adb 100644 --- a/tex/context/base/mkii/cont-new.mkii +++ b/tex/context/base/mkii/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2022.11.18 13:15} +\newcontextversion{2022.12.01 12:38} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii index 09f548302..d07c96f1a 100644 --- a/tex/context/base/mkii/context.mkii +++ b/tex/context/base/mkii/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2022.11.18 13:15} +\edef\contextversion{2022.12.01 12:38} %D For those who want to use this: diff --git a/tex/context/base/mkiv/char-def.lua b/tex/context/base/mkiv/char-def.lua index a5a21687b..276d0f700 100644 --- a/tex/context/base/mkiv/char-def.lua +++ b/tex/context/base/mkiv/char-def.lua @@ -1467,6 +1467,7 @@ characters.data={ description="TILDE", direction="on", linebreak="al", + mathclass="relation", synonyms={ "spacing tilde" }, unicodeslot=0x7E, }, diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv index db7e8f9a2..820a372bb 100644 --- a/tex/context/base/mkiv/cont-new.mkiv +++ b/tex/context/base/mkiv/cont-new.mkiv @@ -13,7 +13,7 @@ % \normalend % uncomment this to get the real base runtime -\newcontextversion{2022.11.18 13:15} +\newcontextversion{2022.12.01 12:38} %D This file is loaded at runtime, thereby providing an excellent place for hacks, %D patches, extensions and new features. There can be local overloads in cont-loc diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv index 107874ffd..d7a655b4f 100644 --- a/tex/context/base/mkiv/context.mkiv +++ b/tex/context/base/mkiv/context.mkiv @@ -49,7 +49,7 @@ %D {YYYY.MM.DD HH:MM} format. \edef\contextformat {\jobname} -\edef\contextversion{2022.11.18 13:15} +\edef\contextversion{2022.12.01 12:38} %D Kind of special: diff --git a/tex/context/base/mkiv/font-cff.lua b/tex/context/base/mkiv/font-cff.lua index 4b60cd6c0..f1c377637 100644 --- a/tex/context/base/mkiv/font-cff.lua +++ b/tex/context/base/mkiv/font-cff.lua @@ -2061,24 +2061,26 @@ end if t == 8 and top > 48 then -- let's assume this only happens for rrcurveto .. the other ones would need some more -- complex handling (cff2 stuff) + -- + -- dx1 dy1 (dx1+dx2) (dy1+dy2) (dx1+dx2+dx3) (dy1+dy2+dy3) rcurveto. local n = 0 for i=1,top do -- if n == 42 then if n == 48 then - local zero = encode[0] - local res3 = result[r-3] - local res2 = result[r-2] - local res1 = result[r-1] - local res0 = result[r] - result[r-3] = zero - result[r-2] = zero +-- local zero = encode[0] +-- local res3 = result[r-3] +-- local res2 = result[r-2] +-- local res1 = result[r-1] +-- local res0 = result[r] +-- result[r-3] = zero +-- result[r-2] = zero r = r + 1 ; result[r] = chars[t] - r = r + 1 ; result[r] = zero - r = r + 1 ; result[r] = zero - r = r + 1 ; result[r] = res3 - r = r + 1 ; result[r] = res2 - r = r + 1 ; result[r] = res1 - r = r + 1 ; result[r] = res0 +-- r = r + 1 ; result[r] = zero +-- r = r + 1 ; result[r] = zero +-- r = r + 1 ; result[r] = res3 +-- r = r + 1 ; result[r] = res2 +-- r = r + 1 ; result[r] = res1 +-- r = r + 1 ; result[r] = res0 n = 1 else n = n + 1 diff --git a/tex/context/base/mkiv/font-dsp.lua b/tex/context/base/mkiv/font-dsp.lua index f5e12c0ef..80e623929 100644 --- a/tex/context/base/mkiv/font-dsp.lua +++ b/tex/context/base/mkiv/font-dsp.lua @@ -3089,9 +3089,11 @@ function readers.colr(f,fontdata,specification) local tableoffset = gotodatatable(f,fontdata,"colr",specification.glyphs) if tableoffset then local version = readushort(f) - if version ~= 0 then + if version == 0 or version == 1 then report("table version %a of %a is not supported (yet), maybe font %s is bad",version,"colr",fontdata.filename) return + else + -- both versions have this in common end if not fontdata.tables.cpal then report("color table %a in font %a has no mandate %a table","colr",fontdata.filename,"cpal") diff --git a/tex/context/base/mkiv/font-otr.lua b/tex/context/base/mkiv/font-otr.lua index 339de109f..0b4bb65bb 100644 --- a/tex/context/base/mkiv/font-otr.lua +++ b/tex/context/base/mkiv/font-otr.lua @@ -66,6 +66,7 @@ if not modules then modules = { } end modules ['font-otr'] = { -- require("char-ini") -- end +local number = number local next, type, tonumber, rawget = next, type, tonumber, rawget local byte, lower, char, gsub = string.byte, string.lower, string.char, string.gsub local fullstrip = string.fullstrip @@ -1683,7 +1684,7 @@ end function readers.cmap(f,fontdata,specification) local tableoffset = gotodatatable(f,fontdata,"cmap",specification.glyphs) if tableoffset then - local version = readushort(f) + local version = readushort(f) -- check later versions local noftables = readushort(f) local records = { } local unicodecid = false @@ -2425,25 +2426,29 @@ function readers.loadfont(filename,n,instance) nofsubfonts = fontdata.subfonts and #fontdata.subfonts or nil, }, resources = { - -- filename = fontdata.filename, - filename = filename, - private = privateoffset, - duplicates = fontdata.duplicates or { }, - features = fontdata.features or { }, -- we need to add these in the loader - sublookups = fontdata.sublookups or { }, -- we need to add these in the loader - marks = fontdata.marks or { }, -- we need to add these in the loader - markclasses = fontdata.markclasses or { }, -- we need to add these in the loader - marksets = fontdata.marksets or { }, -- we need to add these in the loader - sequences = fontdata.sequences or { }, -- we need to add these in the loader - variants = fontdata.variants, -- variant -> unicode -> glyph - version = getname(fontdata,"version"), - cidinfo = fontdata.cidinfo, - mathconstants = fontdata.mathconstants, - colorpalettes = fontdata.colorpalettes, - svgshapes = fontdata.svgshapes, - pngshapes = fontdata.pngshapes, - variabledata = fontdata.variabledata, - foundtables = fontdata.foundtables, + -- filename = fontdata.filename, + filename = filename, + private = privateoffset, + duplicates = fontdata.duplicates or { }, + features = fontdata.features or { }, -- we need to add these in the loader + sublookups = fontdata.sublookups or { }, -- we need to add these in the loader + marks = fontdata.marks or { }, -- we need to add these in the loader + markclasses = fontdata.markclasses or { }, -- we need to add these in the loader + marksets = fontdata.marksets or { }, -- we need to add these in the loader + sequences = fontdata.sequences or { }, -- we need to add these in the loader + variants = fontdata.variants, -- variant -> unicode -> glyph + version = getname(fontdata,"version"), + cidinfo = fontdata.cidinfo, + mathconstants = fontdata.mathconstants, + colorpalettes = fontdata.colorpalettes, + colorpaintdata = fontdata.colorpaintdata, + colorpaintlist = fontdata.colorpaintlist, + colorlinesdata = fontdata.colorlinesdata, + coloraffinedata = fontdata.coloraffinedata, + svgshapes = fontdata.svgshapes, + pngshapes = fontdata.pngshapes, + variabledata = fontdata.variabledata, + foundtables = fontdata.foundtables, }, } end diff --git a/tex/context/base/mkiv/font-ott.lua b/tex/context/base/mkiv/font-ott.lua index aa1defd6a..74fd58375 100644 --- a/tex/context/base/mkiv/font-ott.lua +++ b/tex/context/base/mkiv/font-ott.lua @@ -52,16 +52,20 @@ local scripts = allocate { ["cari"] = "carian", ["cham"] = "cham", ["cher"] = "cherokee", + ["chrs"] = "chorasmian", ["copt"] = "coptic", + ["cpmn"] = "cypro-minoan", ["cprt"] = "cypriot syllabary", ["cyrl"] = "cyrillic", ["dev2"] = "devanagari variant 2", ["deva"] = "devanagari", + ["diak"] = "dives akuru", ["dogr"] = "dogra", ["dsrt"] = "deseret", ["dupl"] = "duployan", ["egyp"] = "egyptian heiroglyphs", ["elba"] = "elbasan", + ["elym"] = "elymaic", ["ethi"] = "ethiopic", ["geor"] = "georgian", ["gjr2"] = "gujarati variant 2", @@ -81,6 +85,7 @@ local scripts = allocate { ["hebr"] = "hebrew", ["hluw"] = "anatolian hieroglyphs", ["hmng"] = "pahawh hmong", + ["hmnp"] = "nyiakeng puachue hmong", ["hung"] = "old hungarian", ["ital"] = "old italic", ["jamo"] = "hangul jamo", @@ -90,6 +95,7 @@ local scripts = allocate { ["khar"] = "kharosthi", ["khmr"] = "khmer", ["khoj"] = "khojki", + ["kits"] = "khitan small script", ["knd2"] = "kannada variant 2", ["knda"] = "kannada", ["kthi"] = "kaithi", @@ -123,6 +129,7 @@ local scripts = allocate { ["musc"] = "musical symbols", ["mym2"] = "myanmar variant 2", ["mymr"] = "myanmar", + ["nand"] = "nandinagari", ["narb"] = "old north arabian", ["nbat"] = "nabataean", ["newa"] = "newa", @@ -132,9 +139,10 @@ local scripts = allocate { ["olck"] = "ol chiki", ["orkh"] = "old turkic and orkhon runic", ["ory2"] = "odia variant 2", - ["orya"] = "oriya", + ["orya"] = "odia", ["osge"] = "osage", ["osma"] = "osmanya", + ["ougr"] = "old uyghur", ["palm"] = "palmyrene", ["pauc"] = "pau cin hau", ["perm"] = "old permic", @@ -166,7 +174,7 @@ local scripts = allocate { ["tagb"] = "tagbanwa", ["takr"] = "takri", ["tale"] = "tai le", - ["talu"] = "tai lu", + ["talu"] = "new tai lue", ["taml"] = "tamil", ["tang"] = "tangut", ["tavt"] = "tai viet", @@ -178,12 +186,16 @@ local scripts = allocate { ["thai"] = "thai", ["tibt"] = "tibetan", ["tirh"] = "tirhuta", + ["tnsa"] = "tangsa", ["tml2"] = "tamil variant 2", + ["toto"] = "toto", ["ugar"] = "ugaritic cuneiform", ["vai" ] = "vai", ["wara"] = "warang citi", + ["wcho"] = "wancho", ["xpeo"] = "old persian cuneiform", ["xsux"] = "sumero-akkadian cuneiform", + ["yezi"] = "yezidi", ["yi" ] = "yi", ["zanb"] = "zanabazar square", } @@ -199,6 +211,7 @@ local languages = allocate { ["agw" ] = "agaw", ["aio" ] = "aiton", ["aka" ] = "akan", + ["akb" ] = "batak angkola", ["als" ] = "alsatian", ["alt" ] = "altai", ["amh" ] = "amharic", @@ -211,6 +224,7 @@ local languages = allocate { ["asm" ] = "assamese", ["ast" ] = "asturian", ["ath" ] = "athapaskan", + ["avn" ] = "avatime", ["avr" ] = "avar", ["awa" ] = "awadhi", ["aym" ] = "aymara", @@ -256,8 +270,12 @@ local languages = allocate { ["brx" ] = "bodo", ["bsh" ] = "bashkir", ["bsk" ] = "burushaski", + ["bta" ] = "batak alas kluet", + ["btd" ] = "batak dairi (pakpak)", ["bti" ] = "beti", + ["btm" ] = "batak mandailing", ["bts" ] = "batak simalungun", + ["btx" ] = "batak karo", ["bug" ] = "bugis", ["byv" ] = "medumba", ["cak" ] = "kaqchikel", @@ -292,14 +310,16 @@ local languages = allocate { ["csl" ] = "church slavonic", ["csy" ] = "czech", ["ctg" ] = "chittagonian", + ["ctt" ] = "wayanad chetti", ["cuk" ] = "san blas kuna", + ["dag" ] = "dagbani", ["dan" ] = "danish", ["dar" ] = "dargwa", ["dax" ] = "dayi", ["dcr" ] = "woods cree", ["deu" ] = "german", - ["dgo" ] = "dogri", - ["dgr" ] = "dogri", + ["dgo" ] = "dogri (individual language)", + ["dgr" ] = "dogri (macro language)", ["dhg" ] = "dhangu", ["dhv" ] = "divehi (dhivehi, maldivian)", ["diq" ] = "dimli", @@ -371,13 +391,16 @@ local languages = allocate { ["guj" ] = "gujarati", ["guz" ] = "gusii", ["hai" ] = "haitian (haitian creole)", + ["hai0"] = "haida", ["hal" ] = "halam", ["har" ] = "harauti", ["hau" ] = "hausa", ["haw" ] = "hawaiian", ["hay" ] = "haya", ["haz" ] = "hazaragi", + ["hmz" ] = "hmong shuat", ["hbn" ] = "hammer-banna", + ["hei" ] = "heiltsuk", ["her" ] = "herero", ["hil" ] = "hiligaynon", ["hin" ] = "hindi", @@ -402,10 +425,12 @@ local languages = allocate { ["ind" ] = "indonesian", ["ing" ] = "ingush", ["inu" ] = "inuktitut", + ["inuk"] = "nunavik inuktitut", ["ipk" ] = "inupiat", ["ipph"] = "phonetic transcription—ipa conventions", ["iri" ] = "irish", ["irt" ] = "irish traditional", + ["uri" ] = "irula", ["isl" ] = "icelandic", ["ism" ] = "inari sami", ["ita" ] = "italian", @@ -425,6 +450,7 @@ local languages = allocate { ["kan" ] = "kannada", ["kar" ] = "karachay", ["kat" ] = "georgian", + ["kaw" ] = "kawi (old javanese)", ["kaz" ] = "kazakh", ["kde" ] = "makonde", ["kea" ] = "kabuverdianu (crioulo)", @@ -482,6 +508,7 @@ local languages = allocate { ["kur" ] = "kurdish", ["kuu" ] = "kurukh", ["kuy" ] = "kuy", + ["kwk" ] = "kwakʼwala", ["kyk" ] = "koryak", ["kyu" ] = "western kayah", ["lad" ] = "ladin", @@ -493,6 +520,7 @@ local languages = allocate { ["laz" ] = "laz", ["lcr" ] = "l-cree", ["ldk" ] = "ladakhi", + ["lef" ] = "lelemi", ["lez" ] = "lezgi", ["lij" ] = "ligurian", ["lim" ] = "limburgish", @@ -505,6 +533,7 @@ local languages = allocate { ["lmo" ] = "lombard", ["lmw" ] = "lomwe", ["lom" ] = "loma", + ["lpo" ] = "lipo", ["lrc" ] = "luri", ["lsb" ] = "lower sorbian", ["lsm" ] = "lule sami", @@ -521,7 +550,7 @@ local languages = allocate { ["mah" ] = "marshallese", ["maj" ] = "majang", ["mak" ] = "makhuwa", - ["mal" ] = "malayalam reformed", + ["mal" ] = "malayalam", ["mam" ] = "mam", ["man" ] = "mansi", ["map" ] = "mapudungun", @@ -556,6 +585,7 @@ local languages = allocate { ["mok" ] = "moksha", ["mol" ] = "moldavian", ["mon" ] = "mon", + ["mnw" ] = "thailand mon", ["mor" ] = "moroccan", ["mos" ] = "mossi", ["mri" ] = "maori", @@ -594,7 +624,7 @@ local languages = allocate { ["nor" ] = "norwegian", ["nov" ] = "novial", ["nsm" ] = "northern sami", - ["nso" ] = "sotho, northern", + ["nso" ] = "northern sotho", ["nta" ] = "northern tai", ["nto" ] = "esperanto", ["nym" ] = "nyamwezi", @@ -642,6 +672,7 @@ local languages = allocate { ["rbu" ] = "russian buriat", ["rcr" ] = "r-cree", ["rej" ] = "rejang", + ["rhg" ] = "rohingya", ["ria" ] = "riang", ["rif" ] = "tarifit", ["rit" ] = "ritarungo", @@ -666,6 +697,7 @@ local languages = allocate { ["scs" ] = "north slavey", ["sek" ] = "sekota", ["sel" ] = "selkup", + ["sfm" ] = "small flowery miao", ["sga" ] = "old irish", ["sgo" ] = "sango", ["sgs" ] = "samogitian", @@ -687,7 +719,7 @@ local languages = allocate { ["snk" ] = "soninke", ["sog" ] = "sodo gurage", ["sop" ] = "songe", - ["sot" ] = "sotho, southern", + ["sot" ] = "southern sotho", ["sqi" ] = "albanian", ["srb" ] = "serbian", ["srd" ] = "sardinian", @@ -728,7 +760,9 @@ local languages = allocate { ["tht" ] = "tahitian", ["tib" ] = "tibetan", ["tiv" ] = "tiv", + ["tj;" ] = "tai laing", ["tkm" ] = "turkmen", + ["tli" ] = "tlingit", ["tmh" ] = "tamashek", ["tmn" ] = "temne", ["tna" ] = "tswana", @@ -742,7 +776,7 @@ local languages = allocate { ["tsj" ] = "tshangla", ["tua" ] = "turoyo aramaic", ["tul" ] = "tulu", - ["tum" ] = "tulu", + ["tum" ] = "tumbuka", ["tuv" ] = "tuvin", ["tvl" ] = "tuvalu", ["twi" ] = "twi", @@ -764,6 +798,7 @@ local languages = allocate { ["wa" ] = "wa", ["wag" ] = "wagdi", ["war" ] = "waray-waray", + ["wci" ] = "waci gbe", ["wcr" ] = "west-cree", ["wel" ] = "welsh", ["wlf" ] = "wolof", @@ -775,17 +810,23 @@ local languages = allocate { ["xkf" ] = "khengkha", ["xog" ] = "soga", ["xpe" ] = "kpelle (liberia)", + ["xub" ] = "bette kuruma", + ["xuj" ] = "jennu kuruma", ["yak" ] = "sakha", ["yao" ] = "yao", ["yap" ] = "yapese", ["yba" ] = "yoruba", ["ycr" ] = "y-cree", + ["ygp" ] = "gepo", ["yic" ] = "yi classic", ["yim" ] = "yi modern", + ["yna" ] = "aluo", + ["ywq" ] = "wuding-luquan", ["zea" ] = "zealandic", ["zgh" ] = "standard morrocan tamazigh", ["zha" ] = "zhuang", ["zhh" ] = "chinese, hong kong sar", + ["zho" ] = "chinese traditional, macao", ["zhp" ] = "chinese phonetic", ["zhs" ] = "chinese simplified", ["zht" ] = "chinese traditional", @@ -794,7 +835,6 @@ local languages = allocate { ["zza" ] = "zazaki", } - local features = allocate { ["aalt"] = "access all alternates", ["abvf"] = "above-base forms", @@ -811,6 +851,7 @@ local features = allocate { ["case"] = "case-sensitive forms", ["ccmp"] = "glyph composition/decomposition", ["cfar"] = "conjunct form after ro", + ["chws"] = "contextual half-width spacing", ["cjct"] = "conjunct forms", ["clig"] = "contextual ligatures", ["cpct"] = "centered cjk punctuation", @@ -866,7 +907,7 @@ local features = allocate { ["nukt"] = "nukta forms", ["numr"] = "numerators", ["onum"] = "old style figures", - ["opbd"] = "optical bounds", + ["opbd"] = "optical bounds", -- funny, this is obsolete (too hard?) (and was recomended always true) ["ordn"] = "ordinals", ["ornm"] = "ornaments", ["palt"] = "proportional alternate width", @@ -887,8 +928,8 @@ local features = allocate { ["rtbd"] = "right bounds", ["rtla"] = "right-to-left alternates", ["rtlm"] = "right to left mirrored forms", - ["rvrn"] = "required variation alternates", ["ruby"] = "ruby notation forms", + ["rvrn"] = "required variation alternates", ["salt"] = "stylistic alternates", ["sinf"] = "scientific inferiors", ["size"] = "optical size", -- now stat table @@ -928,6 +969,7 @@ local features = allocate { ["unic"] = "unicase", ["valt"] = "alternate vertical metrics", ["vatu"] = "vattu variants", + ["vchw"] = "vertical contextual half-width spacing", ["vert"] = "vertical writing", ["vhal"] = "alternate vertical half metrics", ["vjmo"] = "vowel jamo forms", diff --git a/tex/context/base/mkiv/math-act.lua b/tex/context/base/mkiv/math-act.lua index a77fdc020..d56efbbfd 100644 --- a/tex/context/base/mkiv/math-act.lua +++ b/tex/context/base/mkiv/math-act.lua @@ -62,6 +62,9 @@ function mathematics.initializeparameters(target,original) if not mathparameters.SpaceBeforeScript then mathparameters.SpaceBeforeScript = mathparameters.SpaceAfterScript end + if not mathparameters.SubscriptShiftDownWithSuperscript then + mathparameters.SubscriptShiftDownWithSuperscript = mathparameters.SubscriptShiftDown * 1.5 + end target.mathparameters = mathparameters end end @@ -155,28 +158,67 @@ function mathematics.overloadparameters(target,original) if trace_defining then report_math("overloading math parameters in %a @ %p",target.properties.fullname,target.parameters.size) end + -- for name, value in next, parameters do + -- local tvalue = type(value) + -- if tvalue == "string" then + -- report_math("comment for math parameter %a: %s",name,value) + -- else + -- local oldvalue = mathparameters[name] + -- local newvalue = oldvalue + -- if oldvalue then + -- if tvalue == "number" then + -- newvalue = value + -- elseif tvalue == "function" then + -- newvalue = value(oldvalue,target,original) + -- elseif not tvalue then + -- newvalue = nil + -- end + -- if trace_defining and oldvalue ~= newvalue then + -- report_math("overloading math parameter %a: %S => %S",name,oldvalue,newvalue) + -- end + -- else + -- -- report_math("invalid math parameter %a",name) + -- end + -- mathparameters[name] = newvalue + -- end + -- end + for name, value in next, parameters do + local tvalue = type(value) + local oldvalue = mathparameters[name] + local newvalue = oldvalue + if tvalue == "number" then + newvalue = value + elseif tvalue == "string" then + -- delay till all set + elseif tvalue == "function" then + newvalue = value(oldvalue,target,original) + elseif not tvalue then + newvalue = nil + end + if trace_defining and oldvalue ~= newvalue then + report_math("overloading math parameter %a: %S => %S",name,oldvalue or 0,newvalue) + end + mathparameters[name] = newvalue + end for name, value in next, parameters do local tvalue = type(value) if tvalue == "string" then - report_math("comment for math parameter %a: %s",name,value) - else - local oldvalue = mathparameters[name] - local newvalue = oldvalue - if oldvalue then - if tvalue == "number" then - newvalue = value - elseif tvalue == "function" then - newvalue = value(oldvalue,target,original) - elseif not tvalue then - newvalue = nil - end - if trace_defining and oldvalue ~= newvalue then - report_math("overloading math parameter %a: %S => %S",name,oldvalue,newvalue) + local newvalue = mathparameters[value] + if not newvalue then + local code = loadstring("return " .. value,"","t",mathparameters) + if type(code) == "function" then + local okay, v = pcall(code) + if okay then + newvalue = v + end end - else - -- report_math("invalid math parameter %a",name) end - mathparameters[name] = newvalue + if newvalue then + -- split in number and string + mathparameters[name] = newvalue + elseif trace_defining then + report_math("ignoring math parameter %a: %S",name,value) + end end end end diff --git a/tex/context/base/mkiv/math-dim.lua b/tex/context/base/mkiv/math-dim.lua index 06b4bbd97..c46e16faf 100644 --- a/tex/context/base/mkiv/math-dim.lua +++ b/tex/context/base/mkiv/math-dim.lua @@ -132,7 +132,6 @@ end function mathematics.dimensions(dimens) -- beware, dimens get spoiled if dimens.SpaceAfterScript then - dimens.SubscriptShiftDownWithSuperscript = dimens.SubscriptShiftDown * 1.5 -- move this one return table.fastcopy(dimens), { } elseif dimens.AxisHeight or dimens.axis_height then local t = { } diff --git a/tex/context/base/mkiv/mult-prm.lua b/tex/context/base/mkiv/mult-prm.lua index 32b8aa440..891b795ad 100644 --- a/tex/context/base/mkiv/mult-prm.lua +++ b/tex/context/base/mkiv/mult-prm.lua @@ -266,6 +266,7 @@ return { "alltextstyles", "alluncrampedstyles", "allunsplitstyles", + "amcode", "atendofgroup", "atendofgrouped", "attribute", @@ -297,6 +298,7 @@ return { "boxymove", "boxyoffset", "catcodetable", + "cfcode", "clearmarks", "copymathatomrule", "copymathparent", @@ -305,6 +307,7 @@ return { "crampedscriptscriptstyle", "crampedscriptstyle", "crampedtextstyle", + "csactive", "csstring", "currentloopiterator", "currentloopnesting", @@ -328,6 +331,7 @@ return { "everytab", "exceptionpenalty", "expand", + "expandactive", "expandafterpars", "expandafterspaces", "expandcstoken", diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf Binary files differindex c5ec75d70..da390abd8 100644 --- a/tex/context/base/mkiv/status-files.pdf +++ b/tex/context/base/mkiv/status-files.pdf diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf Binary files differindex afc189b09..ffd8cb5f2 100644 --- a/tex/context/base/mkiv/status-lua.pdf +++ b/tex/context/base/mkiv/status-lua.pdf diff --git a/tex/context/base/mkxl/anch-box.mkxl b/tex/context/base/mkxl/anch-box.mkxl index 0dec78838..88809152e 100644 --- a/tex/context/base/mkxl/anch-box.mkxl +++ b/tex/context/base/mkxl/anch-box.mkxl @@ -93,7 +93,6 @@ {\setupmathfence[\c!leftsource=,\c!rightsource=]% \glet\m_anch_matrix_list\empty} - \defineboxanchorcontent [arrow] [\c!mp=mypos:boxanchor:arrow, diff --git a/tex/context/base/mkxl/catc-def.mkxl b/tex/context/base/mkxl/catc-def.mkxl index 896dd35d3..e24aa9cb3 100644 --- a/tex/context/base/mkxl/catc-def.mkxl +++ b/tex/context/base/mkxl/catc-def.mkxl @@ -24,13 +24,13 @@ \ifdefined\texcatcodes \else \newcatcodetable \texcatcodes \fi \ifdefined\luacatcodes \else \newcatcodetable \luacatcodes \fi \ifdefined\notcatcodes \else \newcatcodetable \notcatcodes \fi -\ifdefined\rlncatcodes \else \newcatcodetable \rlncatcodes \fi +\ifdefined\rlncatcodes \else \newcatcodetable \rlncatcodes \fi % readline, not for context \ifdefined\vrbcatcodes \else \newcatcodetable \vrbcatcodes \fi \ifdefined\prtcatcodes \else \newcatcodetable \prtcatcodes \fi \ifdefined\ctxcatcodes \else \newcatcodetable \ctxcatcodes \fi \ifdefined\txtcatcodes \else \newcatcodetable \txtcatcodes \fi -\ifdefined\tpacatcodes \else \newcatcodetable \tpacatcodes \fi % { } -\ifdefined\tpbcatcodes \else \newcatcodetable \tpbcatcodes \fi % < > +\ifdefined\tpacatcodes \else \newcatcodetable \tpacatcodes \fi % verbatim: { } +\ifdefined\tpbcatcodes \else \newcatcodetable \tpbcatcodes \fi % verbatim: < > \ifdefined\ctdcatcodes \else \newcatcodetable \ctdcatcodes \fi % context definitions \startcatcodetable \nilcatcodes @@ -254,6 +254,9 @@ \letcatcodecommand \ctxcatcodes \barasciicode \relax \letcatcodecommand \ctxcatcodes \tildeasciicode \relax +% \letcatcodecommand \prtcatcodes \barasciicode \relax +% \letcatcodecommand \prtcatcodes \tildeasciicode \relax + %D Because some characters have a special meaning, we provide shortcuts to their %D character representation. Some will be overloaded (which might change). @@ -273,4 +276,9 @@ \aliased\let\defaultcatcodetable\ctxcatcodes \popoverloadmode +\amcode \circumflexasciicode \superscriptcatcode +\amcode \underscoreasciicode \subscriptcatcode +\amcode \barasciicode \othercatcode +\amcode \tildeasciicode \othercatcode + \endinput diff --git a/tex/context/base/mkxl/catc-ini.mkxl b/tex/context/base/mkxl/catc-ini.mkxl index 4376acd83..0740e7e2a 100644 --- a/tex/context/base/mkxl/catc-ini.mkxl +++ b/tex/context/base/mkxl/catc-ini.mkxl @@ -39,18 +39,19 @@ %D We predefine some prefixes ahead of syst-aux and mult-sys. We reserve 8 slots for %D catcodes. (This active mess probably needs an update some day.) -\installsystemnamespace{catcodelet} % let : \let -\installsystemnamespace{catcodedef} % def : \def -\installsystemnamespace{catcodeued} % ued : \protected\def -\installsystemnamespace{catcodeget} % \meaning +% \installsystemnamespace{catcodelet} % let : \let +% \installsystemnamespace{catcodedef} % def : \def +% \installsystemnamespace{catcodeued} % ued : \protected\def +% \installsystemnamespace{catcodeget} % \meaning \installsystemnamespace{catcodetablet} \installsystemnamespace{catcodetablen} \newcount\c_syst_catcodes_n \c_syst_catcodes_n\zerocount % 0 = signal, so advance before allocate -\newcount\c_syst_catcodes_a -\newcount\c_syst_catcodes_b -\newcount\c_syst_catcodes_c + +% \newcount\c_syst_catcodes_a +% \newcount\c_syst_catcodes_b +% \newcount\c_syst_catcodes_c \permanent\protected\def\newcatcodetable#1% we could move the cctdefcounter to lua {\global\advance\c_syst_catcodes_n\plusone @@ -92,6 +93,8 @@ \aliased\let\permitcaretescape\permitcircumflexescape +\newconstant\defaultcatcodetable + % == % % \protected\def\startextendcatcodetable#1#2\stopextendcatcodetable @@ -104,99 +107,147 @@ %D The next command can be defined in a cleaner way in the MkIV way but we want %D to have a fast one with a minimal chance for interference. Do we still need %D this complex mechanism? Probably not. Future versions of \MKIV\ might only use -%D active characters for very special cases. - -\setnewconstant\c_syst_catcodes_hack\tildeasciicode - -%D Once a catcode is assigned, the next assignments will happen faster. However, -%D redefinitions probably happen seldom so it's sort of overkill. - -\permanent\protected\def\letcatcodecommand{\afterassignment\syst_catcodes_let_a\c_syst_catcodes_a} -\permanent\protected\def\defcatcodecommand{\afterassignment\syst_catcodes_def_a\c_syst_catcodes_a} -\permanent\protected\def\uedcatcodecommand{\afterassignment\syst_catcodes_ued_a\c_syst_catcodes_a} - -\def\syst_catcodes_let_a{\afterassignment\syst_catcodes_let_b\c_syst_catcodes_b} -\def\syst_catcodes_def_a{\afterassignment\syst_catcodes_def_b\c_syst_catcodes_b} -\def\syst_catcodes_ued_a{\afterassignment\syst_catcodes_ued_b\c_syst_catcodes_b} - -\def\syst_catcodes_let_b % each time - {\ifcsname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname - \expandafter\lastnamedcs - \else - \expandafter\syst_catcodes_let_c - \fi} - -\def\syst_catcodes_def_b % each time - {\ifcsname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname - \expandafter\lastnamedcs - \else - \expandafter\syst_catcodes_def_c - \fi} +%D active characters for very special cases. Older files demonstrate this old +%D hackery tilde abuse. -\def\syst_catcodes_ued_b % each time - {\ifcsname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname - \expandafter\lastnamedcs - \else - \expandafter\syst_catcodes_ued_c - \fi} - -\def\syst_catcodes_let_c % only first time - {\frozen\enforced\gdefcsname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname\expandafter - {\enforced\letcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}% - \syst_catcodes_reinstate_unexpanded - \csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname} - -\def\syst_catcodes_def_c % only first time (we could use \normalexpanded here) - {\frozen\enforced\gdefcsname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname - \expandafter##\expandafter1\expandafter - {\frozen\enforced\defcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}% - \syst_catcodes_reinstate_normal - \csname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname} - -\def\syst_catcodes_ued_c % only first time - {\frozen\enforced\gdefcsname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname - \expandafter##\expandafter1\expandafter - {\frozen\enforced\protected\defcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}% - \syst_catcodes_reinstate_unexpanded - \csname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname} - -\permanent\def\reinstatecatcodecommand{\afterassignment\syst_catcodes_reinstate_normal\c_syst_catcodes_b} +% %D Once a catcode is assigned, the next assignments will happen faster. However, +% %D redefinitions probably happen seldom so it's sort of overkill. We also need to +% %D take care of the initial (shared between catcode regfimes) binding. +% +% \permanent\protected\def\letcatcodecommand{\afterassignment\syst_catcodes_let_a\c_syst_catcodes_a} +% \permanent\protected\def\defcatcodecommand{\afterassignment\syst_catcodes_def_a\c_syst_catcodes_a} % obsolete +% \permanent\protected\def\uedcatcodecommand{\afterassignment\syst_catcodes_ued_a\c_syst_catcodes_a} % obsolete +% +% \def\syst_catcodes_let_a{\afterassignment\syst_catcodes_let_b\c_syst_catcodes_b} +% \def\syst_catcodes_def_a{\afterassignment\syst_catcodes_def_b\c_syst_catcodes_b} +% \def\syst_catcodes_ued_a{\afterassignment\syst_catcodes_ued_b\c_syst_catcodes_b} +% +% % The two step definition is used because we need to fetch the third argument. +% +% \def\syst_catcodes_let_b % each time +% {\ifcsname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname +% \expandafter\lastnamedcs +% \else +% \expandafter\syst_catcodes_let_c +% \fi} +% +% \def\syst_catcodes_def_b % each time +% {\ifcsname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname +% \expandafter\lastnamedcs +% \else +% \expandafter\syst_catcodes_def_c +% \fi} +% +% \def\syst_catcodes_ued_b % each time +% {\ifcsname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname +% \expandafter\lastnamedcs +% \else +% \expandafter\syst_catcodes_ued_c +% \fi} +% +% \def\syst_catcodes_let_c % only first time +% {\frozen\enforced\gdefcsname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname\expandafter +% {\expandafter\enforced\letcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}% +% \syst_catcodes_reinstate_unexpanded +% \csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname} +% +% \def\syst_catcodes_def_c % only first time (we could use \normalexpanded here) +% {\frozen\enforced\gdefcsname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname +% \expandafter##\expandafter1\expandafter +% {\expandafter\frozen\expandafter\enforced\defcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}% +% \syst_catcodes_reinstate_normal +% \csname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname} +% +% \def\syst_catcodes_ued_c % only first time +% {\frozen\enforced\gdefcsname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname +% \expandafter##\expandafter1\expandafter +% {\expandafter\frozen\expandafter\enforced\expandafter\protected\defcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}% +% \syst_catcodes_reinstate_unexpanded +% \csname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname} +% +% %D We can simplify this a bit (not that critical): +% +% \def\syst_catcodes_let_b +% {\afterassignment\syst_catcodes_let_c\let\m_syst_catcodes_temp} +% +% \def\syst_catcodes_let_c +% {\enforced\letcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname\m_syst_catcodes_temp +% \protected\edef\m_syst_catcodes_temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}% +% \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp} +% +% \def\syst_catcodes_let_c +% {\letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp} +% +% % not that much gain: +% +% \def\syst_catcodes_let_c % only first time +% {\expandafter\integerdef\csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname\c_syst_catcodes_b +% \letcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname\m_syst_catcodes_temp +% \protected\edef\m_syst_catcodes_temp{\noexpand\catcodecommand\csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}% +% \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp} %D This can be used when a direct definition has been done and the selector has been -%D lost. A problem is that \type {\next} needs to be unique (as it gets bound) (still?). +%D lost. I really need to get rid of this ... +% \permanent\def\reinstatecatcodecommand{\afterassignment\syst_catcodes_reinstate_normal\c_syst_catcodes_b} +% +% \let\m_syst_catcodes_temp\relax +% % \def\syst_catcodes_reinstate_normal -% {\begingroup -% \edef\temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}% -% \global\letcharcode\c_syst_catcodes_b\temp -% \endgroup} +% {\edef\m_syst_catcodes_temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}% +% \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp} % % \def\syst_catcodes_reinstate_unexpanded -% {\begingroup -% \protected\edef\temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}% -% \global\letcharcode\c_syst_catcodes_b\temp -% \endgroup} - -% I really need to get rid of this ... - -\let\m_syst_catcodes_temp\relax - -\def\syst_catcodes_reinstate_normal - {\edef\m_syst_catcodes_temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}% - \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp} - -\def\syst_catcodes_reinstate_unexpanded - {\protected\edef\m_syst_catcodes_temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}% - \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp} - -\newconstant\defaultcatcodetable - -\permanent\def\catcodecommand#1% - {\csname\??catcodeget\number - \ifcsname\??catcodeget\number\currentcatcodetable:\number#1\endcsname - \currentcatcodetable \else \defaultcatcodetable - \fi - :\number#1\endcsname} +% {\protected\edef\m_syst_catcodes_temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}% +% \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp} +% +% \permanent\def\catcodecommand#1% +% {\csname\??catcodeget\number +% \ifcsname\??catcodeget\number\currentcatcodetable:\number#1\endcsname +% \currentcatcodetable \else \defaultcatcodetable +% \fi +% :\number#1\endcsname} + +%D For now, will become just letcharcode: + +\permanent\protected\def\letcatcodecommand{\afterassignment\letcharcode\scratchcounter} + +%D \startbuffer +%D \def\foo{foo} +%D \start +%D \pushactivechar | \letcharcode124 \foo test||test\par +%D \popactivechar | test||test\par +%D \stop +%D \start +%D \pushactivecharcode124 \letcharcode124 \foo test||test\par +%D \popactivecharcode 124 test||test\par +%D \stop +%D \stopbuffer +%D +%D \typebuffer \getbuffer + +\permanent\protected\def\pushactivechar#1% + {\expandafter\let\expandafter\m_active\csname\csactive#1\endcsname + %\expandafter\let\expandafter\m_active\csname\Uchar"FFFF\Uchar\expandafter`\string#1\endcsname + \pushmacro\m_active} + +\permanent\protected\def\popactivechar#1% + {\popmacro\m_active + %\letcsname\Uchar"FFFF\Uchar\expandafter`\string#1\endcsname\m_active + \letcsname\csactive#1\endcsname\m_active} + +\permanent\protected\def\pushactivecharcode{\afterassignment\syst_active_push\integerdef\c_active_char_code} +\permanent\protected\def\popactivecharcode {\afterassignment\syst_active_pop\integerdef \c_active_char_code} + +\permanent\protected\def\syst_active_push + {\expandafter\let\expandafter\m_active\csname\csactive\Uchar\c_active_char_code\endcsname + %\expandafter\let\expandafter\m_active\csname\Uchar"FFFF\Uchar\c_active_char_code\endcsname + \pushmacro\m_active} + +\permanent\protected\def\syst_active_pop + {\popmacro\m_active + %\letcsname\Uchar"FFFF\Uchar\c_active_char_code\endcsname\m_active + \letcsname\csactive\Uchar\c_active_char_code\endcsname\m_active} %D \macros %D {restorecatcodes,pushcatcodetable,popcatcodetable} diff --git a/tex/context/base/mkxl/cont-new.mkxl b/tex/context/base/mkxl/cont-new.mkxl index 2d953077e..2ac4217cd 100644 --- a/tex/context/base/mkxl/cont-new.mkxl +++ b/tex/context/base/mkxl/cont-new.mkxl @@ -13,7 +13,7 @@ % \normalend % uncomment this to get the real base runtime -\newcontextversion{2022.11.18 13:15} +\newcontextversion{2022.12.01 12:38} %D This file is loaded at runtime, thereby providing an excellent place for hacks, %D patches, extensions and new features. There can be local overloads in cont-loc diff --git a/tex/context/base/mkxl/context.mkxl b/tex/context/base/mkxl/context.mkxl index 47a9ffb4b..1391f64e7 100644 --- a/tex/context/base/mkxl/context.mkxl +++ b/tex/context/base/mkxl/context.mkxl @@ -29,7 +29,7 @@ %D {YYYY.MM.DD HH:MM} format. \immutable\edef\contextformat {\jobname} -\immutable\edef\contextversion{2022.11.18 13:15} +\immutable\edef\contextversion{2022.12.01 12:38} %overloadmode 1 % check frozen / warning %overloadmode 2 % check frozen / error diff --git a/tex/context/base/mkxl/driv-shp.lmt b/tex/context/base/mkxl/driv-shp.lmt index 941365f59..7f74bd2e3 100644 --- a/tex/context/base/mkxl/driv-shp.lmt +++ b/tex/context/base/mkxl/driv-shp.lmt @@ -659,7 +659,6 @@ local hlist_out, vlist_out do if si then local box = si -- si[1] finalize(box) -- tricky: we might need to group --- print(getanchors(box)) if getid(box) == vlist_code then vlist_out(current,box) else @@ -717,11 +716,6 @@ local hlist_out, vlist_out do local function applyanchor(anchor,shift,anchor_h,anchor_v,width,height,depth) local h = 0 local v = 0 - if shift then - anchor = (anchor & 0xFFFF0000) >> 16 - else - anchor = (anchor & 0x0000FFFF) - end local a = anchor & 0x00FF local s = anchor & 0x0F00 if a == 0x02 then @@ -805,8 +799,12 @@ local hlist_out, vlist_out do -- we can encounter par, boundary and penalty nodes but a special -- iterator over content nodes won't save much for current, id, subtype in nextnode, current do +-- if id == nil then +-- print("bad node",current) +-- end if id == glyph_code then local char, font = isglyph(current) +-- if char then local x_offset, y_offset, left, right, raise = getoffsets(current) if x_offset ~= 0 or y_offset ~= 0 then if pos_r == righttoleft_code then @@ -823,6 +821,9 @@ local hlist_out, vlist_out do local wd = flush_character(current,font,char,false,true,pos_h,pos_v,pos_r) -- cur_h = cur_h + wd - right -- hm, no left here? cur_h = cur_h + wd -- see new tabulate alignment code +-- else +-- print("bad character",current,nuts.getfont(current),nuts.getchar(current)) +-- end elseif id == glue_code then -- local gluewidth = effectiveglue(current,this_box) local gluewidth = effectiveglue(current,this_box,true) @@ -975,13 +976,11 @@ local hlist_out, vlist_out do local boxdir = getdirection(current) or lefttoright_code local shift = getshift(current) local geometry, hasoffset, hasorientation, hasanchor = getgeometry(current,true) - local anchor, source, target, targetdata + local anchor, source, target, targetdata, s_anchor, t_anchor local anc_h, anc_v local usedorientation = false if hasanchor then - anchor, source, target = getanchors(current) --- print(getanchors(current)) --- if not anchor then anchor = 0 end + anchor, source, target, s_anchor, t_anchor = getanchors(current) end if hasorientation then local orientation, xoffset, yoffset, woffset, hoffset, doffset = getorientation(current) @@ -1074,8 +1073,10 @@ local hlist_out, vlist_out do end pos_v = targetdata[2] + anc_v if anchor and anchor > 0 then - pos_h, pos_v = applyanchor(anchor,true,pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) - pos_h, pos_v = applyanchor(anchor,false,pos_h,pos_v,width,height,depth) +-- pos_h, pos_v = applyanchor(anchor,true,t_anchor,pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) +-- pos_h, pos_v = applyanchor(anchor,false,s_anchor,pos_h,pos_v,width,height,depth) + pos_h, pos_v = applyanchor(t_anchor,true, pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) + pos_h, pos_v = applyanchor(s_anchor,false,pos_h,pos_v,width,height,depth) end ::process:: if source then @@ -1365,10 +1366,10 @@ local glueheight = effectiveglue(current,this_box,true) local boxdir = getdirection(current) or lefttoright_code local shift = getshift(current) local geometry, hasoffset, hasorientation, hasanchor = getgeometry(current,true) - local anchor, source, target, targetdata + local anchor, source, target, targetdata, s_anchor, t_anchor local usedorientation = false if hasanchor then - anchor, source, target = getanchors(current) + anchor, source, target, s_anchor, t_anchor = getanchors(current) end if hasorientation then local orientation, xoffset, yoffset, woffset, hoffset, doffset = getorientation(current) @@ -1469,8 +1470,10 @@ local glueheight = effectiveglue(current,this_box,true) goto process ::posdone:: if anchor and anchor > 0 then - pos_h, pos_v = applyanchor(anchor,true,pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) - pos_h, pos_v = applyanchor(anchor,false,pos_h,pos_v,width,height,depth) +-- pos_h, pos_v = applyanchor(anchor,true,t_anchor,pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) +-- pos_h, pos_v = applyanchor(anchor,false,s_anchor,pos_h,pos_v,width,height,depth) + pos_h, pos_v = applyanchor(t_anchor,true, pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) + pos_h, pos_v = applyanchor(s_anchor,false,pos_h,pos_v,width,height,depth) end ::process:: if source then diff --git a/tex/context/base/mkxl/driv-usr.lmt b/tex/context/base/mkxl/driv-usr.lmt index 04e7da5e6..0542228aa 100644 --- a/tex/context/base/mkxl/driv-usr.lmt +++ b/tex/context/base/mkxl/driv-usr.lmt @@ -125,11 +125,11 @@ cur_h = cur_h + effectiveglue(current,this_box,true) local boxdir = getdirection(current) or lefttoright_code local shift = getshift(current) local geometry, hasoffset, hasorientation, hasanchor = getgeometry(current,true) - local anchor, source, target, targetdata + local anchor, source, target, targetdata, s_anchor, t_anchor local anc_h, anc_v local usedorientation = false if hasanchor then - anchor, source, target = getanchors(current) + anchor, source, target, s_anchor, t_anchor = getanchors(current) end if hasorientation then local orientation, xoffset, yoffset, woffset, hoffset, doffset = getorientation(current) @@ -221,8 +221,8 @@ cur_h = cur_h + effectiveglue(current,this_box,true) end pos_v = targetdata[2] + anc_v if anchor and anchor > 0 then - pos_h, pos_v = applyanchor(anchor,true,pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) - pos_h, pos_v = applyanchor(anchor,false,pos_h,pos_v,width,height,depth) + pos_h, pos_v = applyanchor(t_anchor,true, pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) + pos_h, pos_v = applyanchor(s_anchor,false,pos_h,pos_v,width,height,depth) end ::process:: if source then @@ -359,10 +359,10 @@ cur_v = cur_v + effectiveglue(current,this_box,true) local boxdir = getdirection(current) or lefttoright_code local shift = getshift(current) local geometry, hasoffset, hasorientation, hasanchor = getgeometry(current,true) - local anchor, source, target, targetdata + local anchor, source, target, targetdata, s_anchor, t_anchor local usedorientation = false if hasanchor then - anchor, source, target = getanchors(current) + anchor, source, target, s_anchor, t_anchor = getanchors(current) end if hasorientation then local orientation, xoffset, yoffset, woffset, hoffset, doffset = getorientation(current) @@ -462,8 +462,8 @@ cur_v = cur_v + effectiveglue(current,this_box,true) goto process ::posdone:: if anchor and anchor > 0 then - pos_h, pos_v = applyanchor(anchor,true,pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) - pos_h, pos_v = applyanchor(anchor,false,pos_h,pos_v,width,height,depth) + pos_h, pos_v = applyanchor(t_anchor,true, pos_h,pos_v,targetdata[3],targetdata[4],targetdata[5]) + pos_h, pos_v = applyanchor(s_anchor,false,pos_h,pos_v,width,height,depth) end ::process:: if source then diff --git a/tex/context/base/mkxl/font-con.lmt b/tex/context/base/mkxl/font-con.lmt index daf106a8a..117e81a66 100644 --- a/tex/context/base/mkxl/font-con.lmt +++ b/tex/context/base/mkxl/font-con.lmt @@ -502,7 +502,12 @@ function constructors.scale(tfmdata,specification) local scaledheight = defaultheight * vdelta local scaleddepth = defaultdepth * vdelta -- - local textcontrol = properties.textcontrol + local textcontrol = properties.textcontrol or 0 + if targetproperties.mode == "base" then + textcontrol = textcontrol | 0x02 | 0x04 -- todo symbolic + elseif targetproperties.mode == "none" then + textcontrol = textcontrol | 0x08 -- todo symbolic + end targetproperties.textcontrol = textcontrol target.textcontrol = textcontrol -- diff --git a/tex/context/base/mkxl/font-dsp.lmt b/tex/context/base/mkxl/font-dsp.lmt new file mode 100644 index 000000000..28a1d4e3c --- /dev/null +++ b/tex/context/base/mkxl/font-dsp.lmt @@ -0,0 +1,4646 @@ +if not modules then modules = { } end modules ['font-dsp'] = { + version = 1.001, + optimize = true, + comment = "companion to font-ini.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- many 0,0 entry/exit + +-- This loader went through a few iterations. First I made a ff compatible one so +-- that we could do some basic checking. Also some verbosity was added (named +-- glyphs). Eventually all that was dropped for a context friendly format, simply +-- because keeping the different table models in sync too to much time. I have the +-- old file somewhere. A positive side effect is that we get an (upto) much smaller +-- smaller tma/tmc file. In the end the loader will be not much slower than the +-- c based ff one. + +-- Being binary encoded, an opentype is rather compact. When expanded into a Lua table +-- quite some memory can be used. This is very noticeable in the ff loader, which for +-- a good reason uses a verbose format. However, when we use that data we create a couple +-- of hashes. In the Lua loader we create these hashes directly, which save quite some +-- memory. +-- +-- We convert a font file only once and then cache it. Before creating the cached instance +-- packing takes place: common tables get shared. After (re)loading and unpacking we then +-- get a rather efficient internal representation of the font. In the new loader there is a +-- pitfall. Because we use some common coverage magic we put a bit more information in +-- the mark and cursive coverage tables than strickly needed: a reference to the coverage +-- itself. This permits a fast lookup of the second glyph involved. In the marks we +-- expand the class indicator to a class hash, in the cursive we use a placeholder that gets +-- a self reference. This means that we cannot pack these subtables unless we add a unique +-- id per entry (the same one per coverage) and that makes the tables larger. Because only a +-- few fonts benefit from this, I decided to not do this. Experiments demonstrated that it +-- only gives a few percent gain (on for instance husayni we can go from 845K to 828K +-- bytecode). Better stay conceptually clean than messy compact. + +-- When we can reduce all basic lookups to one step we might safe a bit in the processing +-- so then only chains are multiple. + +-- I used to flatten kerns here but that has been moved elsewhere because it polutes the code +-- here and can be done fast afterwards. One can even wonder if it makes sense to do it as we +-- pack anyway. In a similar fashion the unique placeholders in anchors in marks have been +-- removed because packing doesn't save much there anyway. + +-- Although we have a bit more efficient tables in the cached files, the internals are still +-- pretty similar. And although we have a slightly more direct coverage access the processing +-- of node lists is not noticeable faster for latin texts, but for arabic we gain some 10% +-- (and could probably gain a bit more). + +-- All this packing in the otf format is somewhat obsessive as nowadays 4K resolution +-- multi-gig videos pass through our networks and storage and memory is abundant. + +-- Although we use a few table readers there i sno real gain in there (apart from having +-- less code. After all there are often not that many demanding features. + +local next, type, tonumber = next, type, tonumber +local band = bit32.band +local extract = bit32.extract +local bor = bit32.bor +local lshift = bit32.lshift +local rshift = bit32.rshift +local gsub = string.gsub +local lower = string.lower +local sub = string.sub +local strip = string.strip +local tohash = table.tohash +local concat = table.concat +local copy = table.copy +local reversed = table.reversed +local sort = table.sort +local insert = table.insert +local round = math.round + +local settings_to_hash = utilities.parsers.settings_to_hash_colon_too +local setmetatableindex = table.setmetatableindex +local formatters = string.formatters +local sortedkeys = table.sortedkeys +local sortedhash = table.sortedhash +local sequenced = table.sequenced + +local report = logs.reporter("otf reader") + +local readers = fonts.handlers.otf.readers +local streamreader = readers.streamreader + +local setposition = streamreader.setposition +local getposition = streamreader.getposition +local readuinteger = streamreader.readcardinal1 +local readushort = streamreader.readcardinal2 +local readuoffset = streamreader.readcardinal3 +local readulong = streamreader.readcardinal4 +local readinteger = streamreader.readinteger1 +local readshort = streamreader.readinteger2 +local readstring = streamreader.readstring +local readtag = streamreader.readtag +local readbytes = streamreader.readbytes +local readfixed = streamreader.readfixed4 +local read2dot14 = streamreader.read2dot14 +local skipshort = streamreader.skipshort +local skipbytes = streamreader.skip +local readbytetable = streamreader.readbytetable +local readbyte = streamreader.readbyte +local readcardinaltable = streamreader.readcardinaltable +local readintegertable = streamreader.readintegertable +local readfword = readshort + +local short = 2 +local ushort = 2 +local uoffset = 3 +local ulong = 4 + +directives.register("fonts.streamreader",function() + + streamreader = utilities.streams + + setposition = streamreader.setposition + getposition = streamreader.getposition + readuinteger = streamreader.readcardinal1 + readushort = streamreader.readcardinal2 + readuoffset = streamreader.readcardinal3 + readulong = streamreader.readcardinal4 + readinteger = streamreader.readinteger1 + readshort = streamreader.readinteger2 + readstring = streamreader.readstring + readtag = streamreader.readtag + readbytes = streamreader.readbytes + readfixed = streamreader.readfixed4 + read2dot14 = streamreader.read2dot14 + skipshort = streamreader.skipshort + skipbytes = streamreader.skip + readbytetable = streamreader.readbytetable + readbyte = streamreader.readbyte + readcardinaltable = streamreader.readcardinaltable + readintegertable = streamreader.readintegertable + readfword = readshort + +end) + +local gsubhandlers = { } +local gposhandlers = { } + +readers.gsubhandlers = gsubhandlers +readers.gposhandlers = gposhandlers + +local helpers = readers.helpers +local gotodatatable = helpers.gotodatatable +local setvariabledata = helpers.setvariabledata + +local lookupidoffset = -1 -- will become 1 when we migrate (only -1 for comparign with old) + +local classes = { + "base", + "ligature", + "mark", + "component", +} + +local gsubtypes = { + "single", + "multiple", + "alternate", + "ligature", + "context", + "chainedcontext", + "extension", + "reversechainedcontextsingle", +} + +local gpostypes = { + "single", + "pair", + "cursive", + "marktobase", + "marktoligature", + "marktomark", + "context", + "chainedcontext", + "extension", +} + +local chaindirections = { + context = 0, + chainedcontext = 1, + reversechainedcontextsingle = -1, +} + +local function setmetrics(data,where,tag,d) + local w = data[where] + if w then + local v = w[tag] + if v then + -- it looks like some fonts set the value and not the delta + -- report("adding %s to %s.%s value %s",d,where,tag,v) + w[tag] = v + d + end + end +end + +local variabletags = { + hasc = function(data,d) setmetrics(data,"windowsmetrics","typoascender",d) end, + hdsc = function(data,d) setmetrics(data,"windowsmetrics","typodescender",d) end, + hlgp = function(data,d) setmetrics(data,"windowsmetrics","typolinegap",d) end, + hcla = function(data,d) setmetrics(data,"windowsmetrics","winascent",d) end, + hcld = function(data,d) setmetrics(data,"windowsmetrics","windescent",d) end, + vasc = function(data,d) setmetrics(data,"vhea not done","ascent",d) end, + vdsc = function(data,d) setmetrics(data,"vhea not done","descent",d) end, + vlgp = function(data,d) setmetrics(data,"vhea not done","linegap",d) end, + xhgt = function(data,d) setmetrics(data,"windowsmetrics","xheight",d) end, + cpht = function(data,d) setmetrics(data,"windowsmetrics","capheight",d) end, + sbxs = function(data,d) setmetrics(data,"windowsmetrics","subscriptxsize",d) end, + sbys = function(data,d) setmetrics(data,"windowsmetrics","subscriptysize",d) end, + sbxo = function(data,d) setmetrics(data,"windowsmetrics","subscriptxoffset",d) end, + sbyo = function(data,d) setmetrics(data,"windowsmetrics","subscriptyoffset",d) end, + spxs = function(data,d) setmetrics(data,"windowsmetrics","superscriptxsize",d) end, + spys = function(data,d) setmetrics(data,"windowsmetrics","superscriptysize",d) end, + spxo = function(data,d) setmetrics(data,"windowsmetrics","superscriptxoffset",d) end, + spyo = function(data,d) setmetrics(data,"windowsmetrics","superscriptyoffset",d) end, + strs = function(data,d) setmetrics(data,"windowsmetrics","strikeoutsize",d) end, + stro = function(data,d) setmetrics(data,"windowsmetrics","strikeoutpos",d) end, + unds = function(data,d) setmetrics(data,"postscript","underlineposition",d) end, + undo = function(data,d) setmetrics(data,"postscript","underlinethickness",d) end, +} + +local read_cardinal = { + streamreader.readcardinal1, + streamreader.readcardinal2, + streamreader.readcardinal3, + streamreader.readcardinal4, +} + +local read_integer = { + streamreader.readinteger1, + streamreader.readinteger2, + streamreader.readinteger3, + streamreader.readinteger4, +} + +-- Traditionally we use these unique names (so that we can flatten the lookup list +-- (we create subsets runtime) but I will adapt the old code to newer names. + +-- chainsub +-- reversesub + +local lookupnames = { + gsub = { + single = "gsub_single", + multiple = "gsub_multiple", + alternate = "gsub_alternate", + ligature = "gsub_ligature", + context = "gsub_context", + chainedcontext = "gsub_contextchain", + reversechainedcontextsingle = "gsub_reversecontextchain", -- reversesub + }, + gpos = { + single = "gpos_single", + pair = "gpos_pair", + cursive = "gpos_cursive", + marktobase = "gpos_mark2base", + marktoligature = "gpos_mark2ligature", + marktomark = "gpos_mark2mark", + context = "gpos_context", + chainedcontext = "gpos_contextchain", + } +} + +-- keep this as reference: +-- +-- local lookupbits = { +-- [0x0001] = "righttoleft", +-- [0x0002] = "ignorebaseglyphs", +-- [0x0004] = "ignoreligatures", +-- [0x0008] = "ignoremarks", +-- [0x0010] = "usemarkfilteringset", +-- [0x00E0] = "reserved", +-- [0xFF00] = "markattachmenttype", +-- } +-- +-- local lookupstate = setmetatableindex(function(t,k) +-- local v = { } +-- for kk, vv in next, lookupbits do +-- if band(k,kk) ~= 0 then +-- v[vv] = true +-- end +-- end +-- t[k] = v +-- return v +-- end) + +local lookupflags = setmetatableindex(function(t,k) + local v = { + band(k,0x0008) ~= 0 and true or false, -- ignoremarks + band(k,0x0004) ~= 0 and true or false, -- ignoreligatures + band(k,0x0002) ~= 0 and true or false, -- ignorebaseglyphs + band(k,0x0001) ~= 0 and true or false, -- r2l + } + t[k] = v + return v +end) + +-- Variation stores: it's not entirely clear if the regions are a shared +-- resource (it looks like they are). Anyway, we play safe and use a +-- share. + +-- values can be anything the min/max permits so we can either think of +-- real values of a fraction along the axis (probably easier) + +-- wght=400,wdth=100,ital=1 + +local function axistofactors(str) + local t = settings_to_hash(str) + for k, v in next, t do + t[k] = tonumber(v) or v -- this also normalizes numbers itself + end + return t +end + +local hash = table.setmetatableindex(function(t,k) + local v = sequenced(axistofactors(k),",") + t[k] = v + return v +end) + +helpers.normalizedaxishash = hash + +local cleanname = fonts.names and fonts.names.cleanname or function(name) + return name and (gsub(lower(name),"[^%a%d]","")) or nil +end + +helpers.cleanname = cleanname + +function helpers.normalizedaxis(str) + return hash[str] or str +end + +-- contradicting spec ... (signs) so i'll check it and fix it once we have +-- proper fonts + +local function getaxisscale(segments,minimum,default,maximum,user) + -- + -- returns the right values cf example in standard + -- + if not minimum or not default or not maximum then + return false + end + if user < minimum then + user = minimum + elseif user > maximum then + user = maximum + end + if user < default then + default = - (default - user) / (default - minimum) + elseif user > default then + default = (user - default) / (maximum - default) + else + default = 0 + end + if not segments then + return default + end + local e + for i=1,#segments do + local s = segments[i] + if type(s) ~= "number" then + report("using default axis scale") + return default + elseif s[1] >= default then + if s[2] == default then + return default + else + e = i + break + end + end + end + if e then + local b = segments[e-1] + local e = segments[e] + return b[2] + (e[2] - b[2]) * (default - b[1]) / (e[1] - b[1]) + else + return false + end +end + +local function getfactors(data,instancespec) + if instancespec == true then + -- take default + elseif type(instancespec) ~= "string" or instancespec == "" then + return + end + local variabledata = data.variabledata + if not variabledata then + return + end + local instances = variabledata.instances + local axis = variabledata.axis + local segments = variabledata.segments + if instances and axis then + local values + if instancespec == true then + -- first instance: + -- values = instances[1].values + -- axis defaults: + values = { } + for i=1,#axis do + values[i] = { + -- axis = axis[i].tag, + value = axis[i].default, + } + end + + else + for i=1,#instances do + local instance = instances[i] + if cleanname(instance.subfamily) == instancespec then + values = instance.values + break + end + end + end + if values then + local factors = { } + for i=1,#axis do + local a = axis[i] + factors[i] = getaxisscale(segments,a.minimum,a.default,a.maximum,values[i].value) + end + return factors + end + local values = axistofactors(hash[instancespec] or instancespec) + if values then + local factors = { } + for i=1,#axis do + local a = axis[i] + local d = a.default + factors[i] = getaxisscale(segments,a.minimum,d,a.maximum,values[a.name or a.tag] or d) + end + return factors + end + end +end + +local function getscales(regions,factors) + local scales = { } + for i=1,#regions do + local region = regions[i] + local s = 1 + for j=1,#region do + local axis = region[j] + local f = factors[j] + local start = axis.start + local peak = axis.peak + local stop = axis.stop + -- get rid of these tests, false flag + if start > peak or peak > stop then + -- * 1 + elseif start < 0 and stop > 0 and peak ~= 0 then + -- * 1 + elseif peak == 0 then + -- * 1 + elseif f < start or f > stop then + -- * 0 + s = 0 + break + elseif f < peak then + -- s = - s * (f - start) / (peak - start) + s = s * (f - start) / (peak - start) + elseif f > peak then + s = s * (stop - f) / (stop - peak) + else + -- * 1 + end + end + scales[i] = s + end + return scales +end + +helpers.getaxisscale = getaxisscale +helpers.getfactors = getfactors +helpers.getscales = getscales +helpers.axistofactors = axistofactors + +local function readvariationdata(f,storeoffset,factors) -- store + local position = getposition(f) + setposition(f,storeoffset) + -- header + local format = readushort(f) + local regionoffset = storeoffset + readulong(f) + local nofdeltadata = readushort(f) + local deltadata = readcardinaltable(f,nofdeltadata,ulong) + -- regions + setposition(f,regionoffset) + local nofaxis = readushort(f) + local nofregions = readushort(f) + local regions = { } + for i=1,nofregions do -- 0 + local t = { } + for i=1,nofaxis do + t[i] = { -- maybe no keys, just 1..3 + start = read2dot14(f), + peak = read2dot14(f), + stop = read2dot14(f), + } + end + regions[i] = t + end + -- deltas + -- if factors then + for i=1,nofdeltadata do + setposition(f,storeoffset+deltadata[i]) + local nofdeltasets = readushort(f) + local nofshorts = readushort(f) + local nofregions = readushort(f) + local usedregions = { } + local deltas = { } + for i=1,nofregions do + usedregions[i] = regions[readushort(f)+1] + end + -- we could test before and save a for + for i=1,nofdeltasets do + local t = readintegertable(f,nofshorts,short) + for i=nofshorts+1,nofregions do + t[i] = readinteger(f) + end + deltas[i] = t + end + deltadata[i] = { + regions = usedregions, + deltas = deltas, + scales = factors and getscales(usedregions,factors) or nil, + } + end + -- end + setposition(f,position) + return regions, deltadata +end + +helpers.readvariationdata = readvariationdata + +-- Beware: only use the simple variant if we don't set keys/values (otherwise too many entries). We +-- could also have a variant that applies a function but there is no real benefit in this. + +local function readcoverage(f,offset,simple) + setposition(f,offset) + local coverageformat = readushort(f) + if coverageformat == 1 then + local nofcoverage = readushort(f) + if simple then + -- often 1 or 2 + if nofcoverage == 1 then + return { readushort(f) } + elseif nofcoverage == 2 then + return { readushort(f), readushort(f) } + else + return readcardinaltable(f,nofcoverage,ushort) + end + elseif nofcoverage == 1 then + return { [readushort(f)] = 0 } + elseif nofcoverage == 2 then + return { [readushort(f)] = 0, [readushort(f)] = 1 } + else + local coverage = { } + for i=0,nofcoverage-1 do + coverage[readushort(f)] = i -- index in record + end + return coverage + end + elseif coverageformat == 2 then + local nofranges = readushort(f) + local coverage = { } + local n = simple and 1 or 0 -- needs checking + for i=1,nofranges do + local firstindex = readushort(f) + local lastindex = readushort(f) + local coverindex = readushort(f) + if simple then + for i=firstindex,lastindex do + coverage[n] = i + n = n + 1 + end + else + for i=firstindex,lastindex do + coverage[i] = n + n = n + 1 + end + end + end + return coverage + else + report("unknown coverage format %a ",coverageformat) + return { } + end +end + +local function readclassdef(f,offset,preset) + setposition(f,offset) + local classdefformat = readushort(f) + local classdef = { } + if type(preset) == "number" then + for k=0,preset-1 do + classdef[k] = 1 + end + end + if classdefformat == 1 then + local index = readushort(f) + local nofclassdef = readushort(f) + for i=1,nofclassdef do + classdef[index] = readushort(f) + 1 + index = index + 1 + end + elseif classdefformat == 2 then + local nofranges = readushort(f) + local n = 0 + for i=1,nofranges do + local firstindex = readushort(f) + local lastindex = readushort(f) + local class = readushort(f) + 1 + for i=firstindex,lastindex do + classdef[i] = class + end + end + else + report("unknown classdef format %a ",classdefformat) + end + if type(preset) == "table" then + for k in next, preset do + if not classdef[k] then + classdef[k] = 1 + end + end + end + return classdef +end + +local function classtocoverage(defs) + if defs then + local list = { } + for index, class in next, defs do + local c = list[class] + if c then + c[#c+1] = index + else + list[class] = { index } + end + end + return list + end +end + +-- extra readers + +local skips = { [0] = + 0, -- ---- + 1, -- ---x + 1, -- --y- + 2, -- --yx + 1, -- -h-- + 2, -- -h-x + 2, -- -hy- + 3, -- -hyx + 2, -- v--x + 2, -- v-y- + 3, -- v-yx + 2, -- vh-- + 3, -- vh-x + 3, -- vhy- + 4, -- vhyx +} + +-- We can assume that 0 is nothing and in fact we can start at 1 as +-- usual in Lua to make sure of that. + +local function readvariation(f,offset) + local p = getposition(f) + setposition(f,offset) + local outer = readushort(f) + local inner = readushort(f) + local format = readushort(f) + setposition(f,p) + if format == 0x8000 then + return outer, inner + end +end + +local function readposition(f,format,mainoffset,getdelta) + if format == 0 then + return false + end + -- a few happen often + if format == 0x04 then + local h = readshort(f) + if h == 0 then + return true -- all zero + else + return { 0, 0, h, 0 } + end + end + if format == 0x05 then + local x = readshort(f) + local h = readshort(f) + if x == 0 and h == 0 then + return true -- all zero + else + return { x, 0, h, 0 } + end + end + if format == 0x44 then + local h = readshort(f) + if getdelta then + local d = readshort(f) -- short or ushort + if d > 0 then + local outer, inner = readvariation(f,mainoffset+d) + if outer then + h = h + getdelta(outer,inner) + end + end + else + skipshort(f,1) + end + if h == 0 then + return true -- all zero + else + return { 0, 0, h, 0 } + end + end + -- + -- todo: + -- + -- if format == 0x55 then + -- local x = readshort(f) + -- local h = readshort(f) + -- .... + -- end + -- + local x = band(format,0x1) ~= 0 and readshort(f) or 0 -- x placement + local y = band(format,0x2) ~= 0 and readshort(f) or 0 -- y placement + local h = band(format,0x4) ~= 0 and readshort(f) or 0 -- h advance + local v = band(format,0x8) ~= 0 and readshort(f) or 0 -- v advance + if format >= 0x10 then + local X = band(format,0x10) ~= 0 and skipshort(f) or 0 + local Y = band(format,0x20) ~= 0 and skipshort(f) or 0 + local H = band(format,0x40) ~= 0 and skipshort(f) or 0 + local V = band(format,0x80) ~= 0 and skipshort(f) or 0 + local s = skips[extract(format,4,4)] + if s > 0 then + skipshort(f,s) + end + if getdelta then + if X > 0 then + local outer, inner = readvariation(f,mainoffset+X) + if outer then + x = x + getdelta(outer,inner) + end + end + if Y > 0 then + local outer, inner = readvariation(f,mainoffset+Y) + if outer then + y = y + getdelta(outer,inner) + end + end + if H > 0 then + local outer, inner = readvariation(f,mainoffset+H) + if outer then + h = h + getdelta(outer,inner) + end + end + if V > 0 then + local outer, inner = readvariation(f,mainoffset+V) + if outer then + v = v + getdelta(outer,inner) + end + end + end + return { x, y, h, v } + elseif x == 0 and y == 0 and h == 0 and v == 0 then + return true -- all zero + else + return { x, y, h, v } + end +end + +local function readanchor(f,offset,getdelta) -- maybe also ignore 0's as in pos + if not offset or offset == 0 then + return nil -- false + end + setposition(f,offset) + -- no need to skip as we position each + local format = readshort(f) -- 1: x y 2: x y index 3 x y X Y + local x = readshort(f) + local y = readshort(f) + if format == 3 then + if getdelta then + local X = readshort(f) + local Y = readshort(f) + if X > 0 then + local outer, inner = readvariation(f,offset+X) + if outer then + x = x + getdelta(outer,inner) + end + end + if Y > 0 then + local outer, inner = readvariation(f,offset+Y) + if outer then + y = y + getdelta(outer,inner) + end + end + else + skipshort(f,2) + end + return { x, y } -- , { xindex, yindex } + else + return { x, y } + end +end + +-- common handlers: inlining can be faster but we cache anyway +-- so we don't bother too much about speed here + +local function readfirst(f,offset) + if offset then + setposition(f,offset) + end + return { readushort(f) } +end + +-- quite often 0, 1, 2 + +local function readarray(f,offset) + if offset then + setposition(f,offset) + end + local n = readushort(f) + if n == 1 then + return { readushort(f) }, 1 + elseif n > 0 then + return readcardinaltable(f,n,ushort), n + end +end + +local function readcoveragearray(f,offset,t,simple) + if not t then + return nil + end + local n = #t + if n == 0 then + return nil + end + for i=1,n do + t[i] = readcoverage(f,offset+t[i],simple) + end + return t +end + +local function covered(subset,all) + local used, u + for i=1,#subset do + local s = subset[i] + if all[s] then + if used then + u = u + 1 + used[u] = s + else + u = 1 + used = { s } + end + end + end + return used +end + +-- We generalize the chained lookups so that we can do with only one handler +-- when processing them. + +-- pruned + +local function readlookuparray(f,noflookups,nofcurrent) + local lookups = { } + if noflookups > 0 then + local length = 0 + for i=1,noflookups do + local index = readushort(f) + 1 + if index > length then + length = index + end + local lookup = readushort(f) + 1 + local list = lookups[index] + if list then + list[#list+1] = lookup + else + lookups[index] = { lookup } + end + end + for index=1,length do + if not lookups[index] then + lookups[index] = false + end + end + -- if length > nofcurrent then + -- report("more lookups than currently matched characters") + -- end + end + return lookups +end + +-- not pruned +-- +-- local function readlookuparray(f,noflookups,nofcurrent) +-- local lookups = { } +-- for i=1,nofcurrent do +-- lookups[i] = false +-- end +-- for i=1,noflookups do +-- local index = readushort(f) + 1 +-- if index > nofcurrent then +-- report("more lookups than currently matched characters") +-- for i=nofcurrent+1,index-1 do +-- lookups[i] = false +-- end +-- nofcurrent = index +-- end +-- lookups[index] = readushort(f) + 1 +-- end +-- return lookups +-- end + +local function unchainedcontext(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,what) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + if subtype == 1 then + local coverage = readushort(f) + local subclasssets = readarray(f) + local rules = { } + if subclasssets then + coverage = readcoverage(f,tableoffset+coverage,true) + for i=1,#subclasssets do + local offset = subclasssets[i] + if offset > 0 then + local firstcoverage = coverage[i] + local rulesoffset = tableoffset + offset + local subclassrules = readarray(f,rulesoffset) + for rule=1,#subclassrules do + setposition(f,rulesoffset + subclassrules[rule]) + local nofcurrent = readushort(f) + local noflookups = readushort(f) + local current = { { firstcoverage } } + for i=2,nofcurrent do + current[i] = { readushort(f) } + end + local lookups = readlookuparray(f,noflookups,nofcurrent) + rules[#rules+1] = { + current = current, + lookups = lookups + } + end + end + end + else + report("empty subclassset in %a subtype %i","unchainedcontext",subtype) + end + return { + format = "glyphs", + rules = rules, + } + elseif subtype == 2 then + -- We expand the classes as later on we do a pack over the whole table so then we get + -- back efficiency. This way we can also apply the coverage to the first current. + local coverage = readushort(f) + local currentclassdef = readushort(f) + local subclasssets = readarray(f) + local rules = { } + if subclasssets then + coverage = readcoverage(f,tableoffset + coverage) + currentclassdef = readclassdef(f,tableoffset + currentclassdef,coverage) + local currentclasses = classtocoverage(currentclassdef,fontdata.glyphs) + for class=1,#subclasssets do + local offset = subclasssets[class] + if offset > 0 then + local firstcoverage = currentclasses[class] + if firstcoverage then + firstcoverage = covered(firstcoverage,coverage) -- bonus + if firstcoverage then + local rulesoffset = tableoffset + offset + local subclassrules = readarray(f,rulesoffset) + for rule=1,#subclassrules do + setposition(f,rulesoffset + subclassrules[rule]) + local nofcurrent = readushort(f) + local noflookups = readushort(f) + local current = { firstcoverage } + for i=2,nofcurrent do + current[i] = currentclasses[readushort(f) + 1] + end + local lookups = readlookuparray(f,noflookups,nofcurrent) + rules[#rules+1] = { + current = current, + lookups = lookups + } + end + else + report("no coverage") + end + else + report("no coverage class") + end + end + end + else + report("empty subclassset in %a subtype %i","unchainedcontext",subtype) + end + return { + format = "class", + rules = rules, + } + elseif subtype == 3 then + local nofglyphs = readushort(f) + local noflookups = readushort(f) + local current = readcardinaltable(f,nofglyphs,ushort) + local lookups = readlookuparray(f,noflookups,#current) + current = readcoveragearray(f,tableoffset,current,true) + return { + format = "coverage", + rules = { + { + current = current, + lookups = lookups, + } + } + } + else + report("unsupported subtype %a in %a %s",subtype,"unchainedcontext",what) + end +end + +-- todo: optimize for n=1 ? + +-- class index needs checking, probably no need for +1 + +local function chainedcontext(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,what) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + if subtype == 1 then + local coverage = readushort(f) + local subclasssets = readarray(f) + local rules = { } + if subclasssets then + coverage = readcoverage(f,tableoffset+coverage,true) + for i=1,#subclasssets do + local offset = subclasssets[i] + if offset > 0 then + local firstcoverage = coverage[i] + local rulesoffset = tableoffset + offset + local subclassrules = readarray(f,rulesoffset) + for rule=1,#subclassrules do + setposition(f,rulesoffset + subclassrules[rule]) + local nofbefore = readushort(f) + local before + if nofbefore > 0 then + before = { } + for i=1,nofbefore do + before[i] = { readushort(f) } + end + end + local nofcurrent = readushort(f) + local current = { { firstcoverage } } + for i=2,nofcurrent do + current[i] = { readushort(f) } + end + local nofafter = readushort(f) + local after + if nofafter > 0 then + after = { } + for i=1,nofafter do + after[i] = { readushort(f) } + end + end + local noflookups = readushort(f) + local lookups = readlookuparray(f,noflookups,nofcurrent) + rules[#rules+1] = { + before = before, + current = current, + after = after, + lookups = lookups, + } + end + end + end + else + report("empty subclassset in %a subtype %i","chainedcontext",subtype) + end + return { + format = "glyphs", + rules = rules, + } + elseif subtype == 2 then + local coverage = readushort(f) + local beforeclassdef = readushort(f) + local currentclassdef = readushort(f) + local afterclassdef = readushort(f) + local subclasssets = readarray(f) + local rules = { } + if subclasssets then + local coverage = readcoverage(f,tableoffset + coverage) + local beforeclassdef = readclassdef(f,tableoffset + beforeclassdef,nofglyphs) + local currentclassdef = readclassdef(f,tableoffset + currentclassdef,coverage) + local afterclassdef = readclassdef(f,tableoffset + afterclassdef,nofglyphs) + local beforeclasses = classtocoverage(beforeclassdef,fontdata.glyphs) + local currentclasses = classtocoverage(currentclassdef,fontdata.glyphs) + local afterclasses = classtocoverage(afterclassdef,fontdata.glyphs) + for class=1,#subclasssets do + local offset = subclasssets[class] + if offset > 0 then + local firstcoverage = currentclasses[class] + if firstcoverage then + firstcoverage = covered(firstcoverage,coverage) -- bonus + if firstcoverage then + local rulesoffset = tableoffset + offset + local subclassrules = readarray(f,rulesoffset) + for rule=1,#subclassrules do + -- watch out, in context we first get the counts and then the arrays while + -- here we get them mixed + setposition(f,rulesoffset + subclassrules[rule]) + local nofbefore = readushort(f) + local before + if nofbefore > 0 then + before = { } + for i=1,nofbefore do + before[i] = beforeclasses[readushort(f) + 1] + end + end + local nofcurrent = readushort(f) + local current = { firstcoverage } + for i=2,nofcurrent do + current[i] = currentclasses[readushort(f)+ 1] + end + local nofafter = readushort(f) + local after + if nofafter > 0 then + after = { } + for i=1,nofafter do + after[i] = afterclasses[readushort(f) + 1] + end + end + -- no sequence index here (so why in context as it saves nothing) + local noflookups = readushort(f) + local lookups = readlookuparray(f,noflookups,nofcurrent) + rules[#rules+1] = { + before = before, + current = current, + after = after, + lookups = lookups, + } + end + else + report("no coverage") + end + else + report("class is not covered") + end + end + end + else + report("empty subclassset in %a subtype %i","chainedcontext",subtype) + end + return { + format = "class", + rules = rules, + } + elseif subtype == 3 then + local before = readarray(f) + local current = readarray(f) + local after = readarray(f) + local noflookups = readushort(f) + local lookups = readlookuparray(f,noflookups,#current) + before = readcoveragearray(f,tableoffset,before,true) + current = readcoveragearray(f,tableoffset,current,true) + after = readcoveragearray(f,tableoffset,after,true) + return { + format = "coverage", + rules = { + { + before = before, + current = current, + after = after, + lookups = lookups, + } + } + } + else + report("unsupported subtype %a in %a %s",subtype,"chainedcontext",what) + end +end + +local function extension(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,types,handlers,what) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + if subtype == 1 then + local lookuptype = types[readushort(f)] + local faroffset = readulong(f) + local handler = handlers[lookuptype] + if handler then + -- maybe we can just pass one offset (or tableoffset first) + return handler(f,fontdata,lookupid,tableoffset + faroffset,0,glyphs,nofglyphs), lookuptype + else + report("no handler for lookuptype %a subtype %a in %s %s",lookuptype,subtype,what,"extension") + end + else + report("unsupported subtype %a in %s %s",subtype,what,"extension") + end +end + +-- gsub handlers + +function gsubhandlers.single(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + if subtype == 1 then + local coverage = readushort(f) + local delta = readshort(f) -- can be negative + local coverage = readcoverage(f,tableoffset+coverage) -- not simple as we need to set key/value anyway + for index in next, coverage do + local newindex = (index + delta) % 65536 -- modulo is new in 1.8.3 + if index > nofglyphs or newindex > nofglyphs then + report("invalid index in %s format %i: %i -> %i (max %i)","single",subtype,index,newindex,nofglyphs) + coverage[index] = nil + else + coverage[index] = newindex + end + end + return { + coverage = coverage + } + elseif subtype == 2 then -- in streamreader a seek and fetch is faster than a temp table + local coverage = readushort(f) + local nofreplacements = readushort(f) + local replacements = readcardinaltable(f,nofreplacements,ushort) + local coverage = readcoverage(f,tableoffset + coverage) -- not simple as we need to set key/value anyway + for index, newindex in next, coverage do + newindex = newindex + 1 + if index > nofglyphs or newindex > nofglyphs then + report("invalid index in %s format %i: %i -> %i (max %i)","single",subtype,index,newindex,nofglyphs) + coverage[index] = nil + else + coverage[index] = replacements[newindex] + end + end + return { + coverage = coverage + } + else + report("unsupported subtype %a in %a substitution",subtype,"single") + end +end + +-- we see coverage format 0x300 in some old ms fonts + +local function sethandler(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,what) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + if subtype == 1 then + local coverage = readushort(f) + local nofsequence = readushort(f) + local sequences = readcardinaltable(f,nofsequence,ushort) + for i=1,nofsequence do + setposition(f,tableoffset + sequences[i]) + sequences[i] = readcardinaltable(f,readushort(f),ushort) + end + local coverage = readcoverage(f,tableoffset + coverage) + for index, newindex in next, coverage do + newindex = newindex + 1 + if index > nofglyphs or newindex > nofglyphs then + report("invalid index in %s format %i: %i -> %i (max %i)",what,subtype,index,newindex,nofglyphs) + coverage[index] = nil + else + coverage[index] = sequences[newindex] + end + end + return { + coverage = coverage + } + else + report("unsupported subtype %a in %a substitution",subtype,what) + end +end + +function gsubhandlers.multiple(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return sethandler(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,"multiple") +end + +function gsubhandlers.alternate(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return sethandler(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,"alternate") +end + +function gsubhandlers.ligature(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + if subtype == 1 then + local coverage = readushort(f) + local nofsets = readushort(f) + local ligatures = readcardinaltable(f,nofsets,ushort) + for i=1,nofsets do + local offset = lookupoffset + offset + ligatures[i] + setposition(f,offset) + local n = readushort(f) + if n == 1 then + ligatures[i] = { offset + readushort(f) } + else + local l = { } + for i=1,n do + l[i] = offset + readushort(f) + end + ligatures[i] = l + end + end + local coverage = readcoverage(f,tableoffset + coverage) + for index, newindex in next, coverage do + local hash = { } + local ligatures = ligatures[newindex+1] + for i=1,#ligatures do + local offset = ligatures[i] + setposition(f,offset) + local lig = readushort(f) + local cnt = readushort(f) + local hsh = hash + for i=2,cnt do + local c = readushort(f) + local h = hsh[c] + if not h then + h = { } + hsh[c] = h + end + hsh = h + end + hsh.ligature = lig + end + coverage[index] = hash + end + return { + coverage = coverage + } + else + report("unsupported subtype %a in %a substitution",subtype,"ligature") + end +end + +function gsubhandlers.context(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return unchainedcontext(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,"substitution"), "context" +end + +function gsubhandlers.chainedcontext(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return chainedcontext(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,"substitution"), "chainedcontext" +end + +function gsubhandlers.extension(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return extension(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,gsubtypes,gsubhandlers,"substitution") +end + +function gsubhandlers.reversechainedcontextsingle(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + if subtype == 1 then -- NEEDS CHECKING + local current = readfirst(f) + local before = readarray(f) + local after = readarray(f) + local replacements = readarray(f) + current = readcoveragearray(f,tableoffset,current,true) + before = readcoveragearray(f,tableoffset,before,true) + after = readcoveragearray(f,tableoffset,after,true) + return { + format = "reversecoverage", -- reversesub + rules = { + { + before = before, + current = current, + after = after, + replacements = replacements, + } + } + }, "reversechainedcontextsingle" + else + report("unsupported subtype %a in %a substitution",subtype,"reversechainedcontextsingle") + end +end + +-- gpos handlers + +local function readpairsets(f,tableoffset,sets,format1,format2,mainoffset,getdelta) + local done = { } + for i=1,#sets do + local offset = sets[i] + local reused = done[offset] + if not reused then + offset = tableoffset + offset + setposition(f,offset) + local n = readushort(f) + reused = { } + for i=1,n do + reused[i] = { + readushort(f), -- second glyph id + readposition(f,format1,offset,getdelta), + readposition(f,format2,offset,getdelta), + } + end + done[offset] = reused + end + sets[i] = reused + end + return sets +end + +local function readpairclasssets(f,nofclasses1,nofclasses2,format1,format2,mainoffset,getdelta) + local classlist1 = { } + for i=1,nofclasses1 do + local classlist2 = { } + classlist1[i] = classlist2 + for j=1,nofclasses2 do + local one = readposition(f,format1,mainoffset,getdelta) + local two = readposition(f,format2,mainoffset,getdelta) + if one or two then + classlist2[j] = { one, two } + else + classlist2[j] = false + end + end + end + return classlist1 +end + +-- no real gain in kerns as we pack + +function gposhandlers.single(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + local getdelta = fontdata.temporary.getdelta + if subtype == 1 then + local coverage = readushort(f) + local format = readushort(f) + local value = readposition(f,format,tableoffset,getdelta) + local coverage = readcoverage(f,tableoffset+coverage) + for index, newindex in next, coverage do + coverage[index] = value -- will be packed and shared anyway + end + return { + format = "single", + coverage = coverage, + } + elseif subtype == 2 then + local coverage = readushort(f) + local format = readushort(f) + local nofvalues = readushort(f) + local values = { } + for i=1,nofvalues do + values[i] = readposition(f,format,tableoffset,getdelta) + end + local coverage = readcoverage(f,tableoffset+coverage) + for index, newindex in next, coverage do + coverage[index] = values[newindex+1] + end + return { + format = "single", + coverage = coverage, + } + else + report("unsupported subtype %a in %a positioning",subtype,"single") + end +end + +-- this needs checking! if no second pair then another advance over the list + +-- ValueFormat1 applies to the ValueRecord of the first glyph in each pair. ValueRecords for all first glyphs must use ValueFormat1. If ValueFormat1 is set to zero (0), the corresponding glyph has no ValueRecord and, therefore, should not be repositioned. +-- ValueFormat2 applies to the ValueRecord of the second glyph in each pair. ValueRecords for all second glyphs must use ValueFormat2. If ValueFormat2 is set to null, then the second glyph of the pair is the “next” glyph for which a lookup should be performed. + +-- local simple = { +-- [true] = { [true] = { true, true }, [false] = { true } }, +-- [false] = { [true] = { false, true }, [false] = { false } }, +-- } + +-- function gposhandlers.pair(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) +-- local tableoffset = lookupoffset + offset +-- setposition(f,tableoffset) +-- local subtype = readushort(f) +-- local getdelta = fontdata.temporary.getdelta +-- if subtype == 1 then +-- local coverage = readushort(f) +-- local format1 = readushort(f) +-- local format2 = readushort(f) +-- local sets = readarray(f) +-- sets = readpairsets(f,tableoffset,sets,format1,format2,mainoffset,getdelta) +-- coverage = readcoverage(f,tableoffset + coverage) +-- local shared = { } -- partial sparse, when set also needs to be handled in the packer +-- for index, newindex in next, coverage do +-- local set = sets[newindex+1] +-- local hash = { } +-- for i=1,#set do +-- local value = set[i] +-- if value then +-- local other = value[1] +-- if shared then +-- local s = shared[value] +-- if s == nil then +-- local first = value[2] +-- local second = value[3] +-- if first or second then +-- s = { first, second or nil } -- needs checking +-- else +-- s = false +-- end +-- shared[value] = s +-- end +-- hash[other] = s or nil +-- else +-- local first = value[2] +-- local second = value[3] +-- if first or second then +-- hash[other] = { first, second or nil } -- needs checking +-- else +-- hash[other] = nil -- what if set, maybe warning +-- end +-- end +-- end +-- end +-- coverage[index] = hash +-- end +-- return { +-- shared = shared and true or nil, +-- format = "pair", +-- coverage = coverage, +-- } +-- elseif subtype == 2 then +-- local coverage = readushort(f) +-- local format1 = readushort(f) +-- local format2 = readushort(f) +-- local classdef1 = readushort(f) +-- local classdef2 = readushort(f) +-- local nofclasses1 = readushort(f) -- incl class 0 +-- local nofclasses2 = readushort(f) -- incl class 0 +-- local classlist = readpairclasssets(f,nofclasses1,nofclasses2,format1,format2,tableoffset,getdelta) +-- coverage = readcoverage(f,tableoffset+coverage) +-- classdef1 = readclassdef(f,tableoffset+classdef1,coverage) +-- classdef2 = readclassdef(f,tableoffset+classdef2,nofglyphs) +-- local usedcoverage = { } +-- local shared = { } -- partial sparse, when set also needs to be handled in the packer +-- for g1, c1 in next, classdef1 do +-- if coverage[g1] then +-- local l1 = classlist[c1] +-- if l1 then +-- local hash = { } +-- for paired, class in next, classdef2 do +-- local offsets = l1[class] +-- if offsets then +-- local first = offsets[1] +-- local second = offsets[2] +-- if first or second then +-- if shared then +-- local s1 = shared[first] +-- if s1 == nil then +-- s1 = { } +-- shared[first] = s1 +-- end +-- local s2 = s1[second] +-- if s2 == nil then +-- s2 = { first, second or nil } +-- s1[second] = s2 +-- end +-- hash[paired] = s2 +-- else +-- hash[paired] = { first, second or nil } +-- end +-- else +-- -- upto the next lookup for this combination +-- end +-- end +-- end +-- usedcoverage[g1] = hash +-- end +-- end +-- end +-- return { +-- shared = shared and true or nil, +-- format = "pair", +-- coverage = usedcoverage, +-- } +-- elseif subtype == 3 then +-- report("yet unsupported subtype %a in %a positioning",subtype,"pair") +-- else +-- report("unsupported subtype %a in %a positioning",subtype,"pair") +-- end +-- end + +function gposhandlers.pair(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + local getdelta = fontdata.temporary.getdelta + if subtype == 1 then + local coverage = readushort(f) + local format1 = readushort(f) + local format2 = readushort(f) + local sets = readarray(f) + sets = readpairsets(f,tableoffset,sets,format1,format2,mainoffset,getdelta) + coverage = readcoverage(f,tableoffset + coverage) + local shared = { } -- partial sparse, when set also needs to be handled in the packer + for index, newindex in next, coverage do + local set = sets[newindex+1] + local hash = { } + for i=1,#set do + local value = set[i] + if value then + local other = value[1] + local share = shared[value] + if share == nil then + local first = value[2] + local second = value[3] + if first or second then + share = { first, second or nil } -- needs checking + else + share = false + end + shared[value] = share + end + hash[other] = share or nil -- really overload ? + end + end + coverage[index] = hash + end + return { + shared = shared and true or nil, + format = "pair", + coverage = coverage, + } + elseif subtype == 2 then + local coverage = readushort(f) + local format1 = readushort(f) + local format2 = readushort(f) + local classdef1 = readushort(f) + local classdef2 = readushort(f) + local nofclasses1 = readushort(f) -- incl class 0 + local nofclasses2 = readushort(f) -- incl class 0 + local classlist = readpairclasssets(f,nofclasses1,nofclasses2,format1,format2,tableoffset,getdelta) + coverage = readcoverage(f,tableoffset+coverage) + classdef1 = readclassdef(f,tableoffset+classdef1,coverage) + classdef2 = readclassdef(f,tableoffset+classdef2,nofglyphs) + local usedcoverage = { } + local shared = { } -- partial sparse, when set also needs to be handled in the packer + for g1, c1 in next, classdef1 do + if coverage[g1] then + local l1 = classlist[c1] + if l1 then + local hash = { } + for paired, class in next, classdef2 do + local offsets = l1[class] + if offsets then + local first = offsets[1] + local second = offsets[2] + if first or second then + local s1 = shared[first] + if s1 == nil then + s1 = { } + shared[first] = s1 + end + local s2 = s1[second] + if s2 == nil then + s2 = { first, second or nil } + s1[second] = s2 + end + hash[paired] = s2 + end + end + end + usedcoverage[g1] = hash + end + end + end + return { + shared = shared and true or nil, + format = "pair", + coverage = usedcoverage, + } + elseif subtype == 3 then + report("yet unsupported subtype %a in %a positioning",subtype,"pair") + else + report("unsupported subtype %a in %a positioning",subtype,"pair") + end +end + +function gposhandlers.cursive(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + local getdelta = fontdata.temporary.getdelta + if subtype == 1 then + local coverage = tableoffset + readushort(f) + local nofrecords = readushort(f) + local records = { } + for i=1,nofrecords do + local entry = readushort(f) + local exit = readushort(f) + records[i] = { + -- entry = entry ~= 0 and (tableoffset + entry) or false, + -- exit = exit ~= 0 and (tableoffset + exit ) or nil, + entry ~= 0 and (tableoffset + entry) or false, + exit ~= 0 and (tableoffset + exit ) or nil, + } + end + -- slot 1 will become hash after loading and it must be unique because we + -- pack the tables (packed we turn the cc-* into a zero) + local cc = (fontdata.temporary.cursivecount or 0) + 1 + fontdata.temporary.cursivecount = cc + cc = "cc-" .. cc + coverage = readcoverage(f,coverage) + for i=1,nofrecords do + local r = records[i] + records[i] = { + -- 1, + cc, + -- readanchor(f,r.entry,getdelta) or false, + -- readanchor(f,r.exit, getdelta) or nil, + readanchor(f,r[1],getdelta) or false, + readanchor(f,r[2],getdelta) or nil, + } + end + for index, newindex in next, coverage do + coverage[index] = records[newindex+1] + end + return { + coverage = coverage, + } + else + report("unsupported subtype %a in %a positioning",subtype,"cursive") + end +end + +local function handlemark(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,ligature) + local tableoffset = lookupoffset + offset + setposition(f,tableoffset) + local subtype = readushort(f) + local getdelta = fontdata.temporary.getdelta + if subtype == 1 then + -- we are one based, not zero + local markcoverage = tableoffset + readushort(f) + local basecoverage = tableoffset + readushort(f) + local nofclasses = readushort(f) + local markoffset = tableoffset + readushort(f) + local baseoffset = tableoffset + readushort(f) + -- + local markcoverage = readcoverage(f,markcoverage) + local basecoverage = readcoverage(f,basecoverage,true) -- TO BE CHECKED: true + -- + setposition(f,markoffset) + local markclasses = { } + local nofmarkclasses = readushort(f) + -- + local lastanchor = fontdata.lastanchor or 0 + local usedanchors = { } + -- + for i=1,nofmarkclasses do + local class = readushort(f) + 1 + local offset = readushort(f) + if offset == 0 then + markclasses[i] = false + else + markclasses[i] = { class, markoffset + offset } + end + usedanchors[class] = true + end + for i=1,nofmarkclasses do + local mc = markclasses[i] + if mc then + mc[2] = readanchor(f,mc[2],getdelta) + end + end + -- + setposition(f,baseoffset) + local nofbaserecords = readushort(f) + local baserecords = { } + -- + if ligature then + -- 3 components + -- 1 : class .. nofclasses -- NULL when empty + -- 2 : class .. nofclasses -- NULL when empty + -- 3 : class .. nofclasses -- NULL when empty + for i=1,nofbaserecords do -- here i is the class + local offset = readushort(f) + if offset == 0 then + baserecords[i] = false + else + baserecords[i] = baseoffset + offset + end + end + for i=1,nofbaserecords do + local recordoffset = baserecords[i] + if recordoffset then + setposition(f,recordoffset) + local nofcomponents = readushort(f) + local components = { } + for i=1,nofcomponents do + local classes = { } + for i=1,nofclasses do + local offset = readushort(f) + if offset ~= 0 then + classes[i] = recordoffset + offset + else + classes[i] = false + end + end + components[i] = classes + end + baserecords[i] = components + end + end + local baseclasses = { } -- setmetatableindex("table") + for i=1,nofclasses do + baseclasses[i] = { } + end + for i=1,nofbaserecords do + local components = baserecords[i] + if components then + local b = basecoverage[i] + for c=1,#components do + local classes = components[c] + if classes then + for i=1,nofclasses do + local anchor = readanchor(f,classes[i],getdelta) + local bclass = baseclasses[i] + local bentry = bclass[b] + if bentry then + bentry[c] = anchor + else + bclass[b]= { [c] = anchor } + end + end + end + end + end + end + for index, newindex in next, markcoverage do + markcoverage[index] = markclasses[newindex+1] or nil + end + return { + format = "ligature", + baseclasses = baseclasses, + coverage = markcoverage, + } + else + for i=1,nofbaserecords do + local r = { } + for j=1,nofclasses do + local offset = readushort(f) + if offset == 0 then + r[j] = false + else + r[j] = baseoffset + offset + end + end + baserecords[i] = r + end + local baseclasses = { } -- setmetatableindex("table") + for i=1,nofclasses do + baseclasses[i] = { } + end + for i=1,nofbaserecords do + local r = baserecords[i] + local b = basecoverage[i] + for j=1,nofclasses do + baseclasses[j][b] = readanchor(f,r[j],getdelta) + end + end + for index, newindex in next, markcoverage do + markcoverage[index] = markclasses[newindex+1] or nil + end + -- we could actually already calculate the displacement if we want + return { + format = "base", + baseclasses = baseclasses, + coverage = markcoverage, + } + end + else + report("unsupported subtype %a in",subtype) + end + +end + +function gposhandlers.marktobase(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return handlemark(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) +end + +function gposhandlers.marktoligature(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return handlemark(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,true) +end + +function gposhandlers.marktomark(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return handlemark(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) +end + +function gposhandlers.context(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return unchainedcontext(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,"positioning"), "context" +end + +function gposhandlers.chainedcontext(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return chainedcontext(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,"positioning"), "chainedcontext" +end + +function gposhandlers.extension(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs) + return extension(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofglyphs,gpostypes,gposhandlers,"positioning") +end + +-- main loader + +do + + local plugins = { } + + function plugins.size(f,fontdata,tableoffset,feature) + if fontdata.designsize then + -- yes, there are fonts with multiple size entries ... it probably relates + -- to the other two fields (menu entries in some language) + else + local function check(offset) + setposition(f,offset) + local designsize = readushort(f) + if designsize > 0 then -- we could also have a threshold + local fontstyleid = readushort(f) + local guimenuid = readushort(f) + local minsize = readushort(f) + local maxsize = readushort(f) + if minsize == 0 and maxsize == 0 and fontstyleid == 0 and guimenuid == 0 then + minsize = designsize + maxsize = designsize + end + if designsize >= minsize and designsize <= maxsize then + return minsize, maxsize, designsize + end + end + end + local minsize, maxsize, designsize = check(tableoffset+feature.offset+feature.parameters) + if not designsize then + -- some old adobe fonts have: tableoffset+feature.parameters and we could + -- use some heuristic but why bother ... this extra check will be removed + -- some day and/or when we run into an issue + minsize, maxsize, designsize = check(tableoffset+feature.parameters) + if designsize then + report("bad size feature in %a, falling back to wrong offset",fontdata.filename or "?") + else + report("bad size feature in %a,",fontdata.filename or "?") + end + end + if designsize then + fontdata.minsize = minsize + fontdata.maxsize = maxsize + fontdata.designsize = designsize + end + end + end + + -- function plugins.rvrn(f,fontdata,tableoffset,feature) + -- -- todo, at least a message + -- end + + -- feature order needs checking ... as we loop over a hash ... however, in the file + -- they are sorted so order is not that relevant + + local function reorderfeatures(fontdata,scripts,features) + local scriptlangs = { } + local featurehash = { } + local featureorder = { } + for script, languages in next, scripts do + for language, record in next, languages do + local hash = { } + local list = record.featureindices + for k=1,#list do + local index = list[k] + local feature = features[index] + local lookups = feature.lookups + local tag = feature.tag + if tag then + hash[tag] = true + end + if lookups then + for i=1,#lookups do + local lookup = lookups[i] + local o = featureorder[lookup] + if o then + local okay = true + for i=1,#o do + if o[i] == tag then + okay = false + break + end + end + if okay then + o[#o+1] = tag + end + else + featureorder[lookup] = { tag } + end + local f = featurehash[lookup] + if f then + local h = f[tag] + if h then + local s = h[script] + if s then + s[language] = true + else + h[script] = { [language] = true } + end + else + f[tag] = { [script] = { [language] = true } } + end + else + featurehash[lookup] = { [tag] = { [script] = { [language] = true } } } + end + -- + local h = scriptlangs[tag] + if h then + local s = h[script] + if s then + s[language] = true + else + h[script] = { [language] = true } + end + else + scriptlangs[tag] = { [script] = { [language] = true } } + end + end + end + end + end + end + return scriptlangs, featurehash, featureorder + end + + local function readscriplan(f,fontdata,scriptoffset) + setposition(f,scriptoffset) + local nofscripts = readushort(f) + local scripts = { } + for i=1,nofscripts do + scripts[readtag(f)] = scriptoffset + readushort(f) + end + -- script list -> language system info + local languagesystems = setmetatableindex("table") + for script, offset in next, scripts do + setposition(f,offset) + local defaultoffset = readushort(f) + local noflanguages = readushort(f) + local languages = { } + if defaultoffset > 0 then + languages.dflt = languagesystems[offset + defaultoffset] + end + for i=1,noflanguages do + local language = readtag(f) + local offset = offset + readushort(f) + languages[language] = languagesystems[offset] + end + scripts[script] = languages + end + -- script list -> language system info -> feature list + for offset, usedfeatures in next, languagesystems do + if offset > 0 then + setposition(f,offset) + local featureindices = { } + usedfeatures.featureindices = featureindices + usedfeatures.lookuporder = readushort(f) -- reserved, not used (yet) + usedfeatures.requiredindex = readushort(f) -- relates to required (can be 0xFFFF) + local noffeatures = readushort(f) + for i=1,noffeatures do + featureindices[i] = readushort(f) + 1 + end + end + end + return scripts + end + + local function readfeatures(f,fontdata,featureoffset) + setposition(f,featureoffset) + local features = { } + local noffeatures = readushort(f) + for i=1,noffeatures do + -- also shared? + features[i] = { + tag = readtag(f), + offset = readushort(f) + } + end + -- + for i=1,noffeatures do + local feature = features[i] + local offset = featureoffset+feature.offset + setposition(f,offset) + local parameters = readushort(f) -- feature.parameters + local noflookups = readushort(f) + if noflookups > 0 then +-- local lookups = { } +-- feature.lookups = lookups +-- for j=1,noflookups do +-- lookups[j] = readushort(f) + 1 +-- end + local lookups = readcardinaltable(f,noflookups,ushort) + feature.lookups = lookups + for j=1,noflookups do + lookups[j] = lookups[j] + 1 + end + end + if parameters > 0 then + feature.parameters = parameters + local plugin = plugins[feature.tag] + if plugin then + plugin(f,fontdata,featureoffset,feature) + end + end + end + return features + end + + local function readlookups(f,lookupoffset,lookuptypes,featurehash,featureorder) + setposition(f,lookupoffset) + local noflookups = readushort(f) + local lookups = readcardinaltable(f,noflookups,ushort) + for lookupid=1,noflookups do + local offset = lookups[lookupid] + setposition(f,lookupoffset+offset) + local subtables = { } + local typebits = readushort(f) + local flagbits = readushort(f) + local lookuptype = lookuptypes[typebits] + local lookupflags = lookupflags[flagbits] + local nofsubtables = readushort(f) + for j=1,nofsubtables do + subtables[j] = offset + readushort(f) -- we can probably put lookupoffset here + end + -- which one wins? + local markclass = band(flagbits,0x0010) ~= 0 -- usemarkfilteringset + if markclass then + markclass = readushort(f) -- + 1 + end + local markset = rshift(flagbits,8) + if markset > 0 then + markclass = markset -- + 1 + end + lookups[lookupid] = { + type = lookuptype, + -- chain = chaindirections[lookuptype] or nil, + flags = lookupflags, + name = lookupid, + subtables = subtables, + markclass = markclass, + features = featurehash[lookupid], -- not if extension + order = featureorder[lookupid], + } + end + return lookups + end + + local f_lookupname = formatters["%s_%s_%s"] + + local function resolvelookups(f,lookupoffset,fontdata,lookups,lookuptypes,lookuphandlers,what,tableoffset) + + local sequences = fontdata.sequences or { } + local sublookuplist = fontdata.sublookups or { } + fontdata.sequences = sequences + fontdata.sublookups = sublookuplist + local nofsublookups = #sublookuplist + local nofsequences = #sequences -- 0 + local lastsublookup = nofsublookups + local lastsequence = nofsequences + local lookupnames = lookupnames[what] + local sublookuphash = { } + local sublookupcheck = { } + local glyphs = fontdata.glyphs + local nofglyphs = fontdata.nofglyphs or #glyphs + local noflookups = #lookups + local lookupprefix = sub(what,2,2) -- g[s|p][ub|os] + -- + local usedlookups = false -- setmetatableindex("number") + -- + local allsteps = { } -- new per 2022-09-25 + + for lookupid=1,noflookups do + local lookup = lookups[lookupid] + local lookuptype = lookup.type + local subtables = lookup.subtables + local features = lookup.features + local handler = lookuphandlers[lookuptype] + if handler then + local nofsubtables = #subtables + local order = lookup.order + local flags = lookup.flags + -- this is expected in the font handler (faster checking) + if flags[1] then flags[1] = "mark" end + if flags[2] then flags[2] = "ligature" end + if flags[3] then flags[3] = "base" end + -- + local markclass = lookup.markclass + -- local chain = lookup.chain + if nofsubtables > 0 then + local steps = { } + local nofsteps = 0 + local oldtype = nil + for s=1,nofsubtables do + local step, lt = handler(f,fontdata,lookupid,lookupoffset,subtables[s],glyphs,nofglyphs) + if lt then + lookuptype = lt + if oldtype and lt ~= oldtype then + report("messy %s lookup type %a and %a",what,lookuptype,oldtype) + end + oldtype = lookuptype + end + if not step then + report("unsupported %s lookup type %a",what,lookuptype) + else + nofsteps = nofsteps + 1 + steps[nofsteps] = step + local rules = step.rules + if rules then + allsteps[#allsteps+1] = step -- new per 2022-09-25 + for i=1,#rules do + local rule = rules[i] + local before = rule.before + local current = rule.current + local after = rule.after + local replacements = rule.replacements + if before then + for i=1,#before do + before[i] = tohash(before[i]) + end + -- as with original ctx ff loader + rule.before = reversed(before) + end + if current then + if replacements then + -- We have a reverse lookup and therefore only one current entry. We might need + -- to reverse the order in the before and after lists so that needs checking. + local first = current[1] + local hash = { } + local repl = { } + for i=1,#first do + local c = first[i] + hash[c] = true + repl[c] = replacements[i] + end + rule.current = { hash } + rule.replacements = repl + else + for i=1,#current do + current[i] = tohash(current[i]) + end + end + else + -- weird lookup + end + if after then + for i=1,#after do + after[i] = tohash(after[i]) + end + end + if usedlookups then + local lookups = rule.lookups + if lookups then + for k, v in next, lookups do + if v then + for k, v in next, v do + usedlookups[v] = usedlookups[v] + 1 + end + end + end + end + end + end + end + end + end + if nofsteps ~= nofsubtables then + report("bogus subtables removed in %s lookup type %a",what,lookuptype) + end + lookuptype = lookupnames[lookuptype] or lookuptype + if features then + nofsequences = nofsequences + 1 + -- report("registering %i as sequence step %i",lookupid,nofsequences) + local l = { + index = nofsequences, + name = f_lookupname(lookupprefix,"s",lookupid+lookupidoffset), + steps = steps, + nofsteps = nofsteps, + type = lookuptype, + markclass = markclass or nil, + flags = flags, + -- chain = chain, + order = order, + features = features, + } + sequences[nofsequences] = l + lookup.done = l + else + nofsublookups = nofsublookups + 1 + -- report("registering %i as sublookup %i",lookupid,nofsublookups) + local l = { + index = nofsublookups, + name = f_lookupname(lookupprefix,"l",lookupid+lookupidoffset), + steps = steps, + nofsteps = nofsteps, + type = lookuptype, + markclass = markclass or nil, + flags = flags, + -- chain = chain, + } + sublookuplist[nofsublookups] = l + sublookuphash[lookupid] = nofsublookups + sublookupcheck[lookupid] = 0 + lookup.done = l + end + else + report("no subtables for lookup %a",lookupid) + end + else + report("no handler for lookup %a with type %a",lookupid,lookuptype) + end + end + + if usedlookups then + report("used %s lookups: % t",what,sortedkeys(usedlookups)) + end + + -- When we have a context, we have sublookups that resolve into lookups for which we need to + -- know the type. We split the main lookuptable in two parts: sequences (the main lookups) + -- and subtable lookups (simple specs with no features). We could keep them merged and might do + -- that once we only use this loader. Then we can also move the simple specs into the sequence. + -- After all, we pack afterwards. + + local reported = { } + + local function report_issue(i,what,step,kind) +-- if not reported[step] then + report("rule %i in step %i of %s has %s lookups",i,step,what,kind) +-- reported[name] = true +-- end + end + + -- for i=lastsequence+1,nofsequences do + -- local sequence = sequences[i] + -- local steps = sequence.steps + -- for i=1,#steps do + -- local step = steps[i] + + for s=1,#allsteps do -- new per 2022-09-25 + local step = allsteps[s] -- new per 2022-09-25 + local rules = step.rules + if rules then + for i=1,#rules do + local rule = rules[i] + local rlookups = rule.lookups + if not rlookups then + report_issue(i,what,s,"no") + elseif not next(rlookups) then + -- can be ok as it aborts a chain sequence + -- report_issue(i,what,s,"empty") + rule.lookups = nil + else + -- we can have holes in rlookups flagged false and we can have multiple lookups + -- applied (first time seen in seguemj) + local length = #rlookups + for index=1,length do + local lookuplist = rlookups[index] + if lookuplist then + local length = #lookuplist + local found = { } + local noffound = 0 + for index=1,length do + local lookupid = lookuplist[index] + if lookupid then + local h = sublookuphash[lookupid] + if not h then + -- here we have a lookup that is used independent as well + -- as in another one + local lookup = lookups[lookupid] + if lookup then + local d = lookup.done + if d then + nofsublookups = nofsublookups + 1 + -- report("registering %i as sublookup %i",lookupid,nofsublookups) + local l = { + index = nofsublookups, -- handy for tracing + name = f_lookupname(lookupprefix,"d",lookupid+lookupidoffset), + derived = true, -- handy for tracing + steps = d.steps, + nofsteps = d.nofsteps, + type = d.lookuptype or "gsub_single", -- todo: check type + markclass = d.markclass or nil, + flags = d.flags, + -- chain = d.chain, + } + sublookuplist[nofsublookups] = copy(l) -- we repack later + sublookuphash[lookupid] = nofsublookups + sublookupcheck[lookupid] = 1 + h = nofsublookups + else + report_issue(i,what,s,"missing") + rule.lookups = nil + break + end + else + report_issue(i,what,s,"bad") + rule.lookups = nil + break + end + else + sublookupcheck[lookupid] = sublookupcheck[lookupid] + 1 + end + if h then + noffound = noffound + 1 + found[noffound] = h + end + end + end + rlookups[index] = noffound > 0 and found or false + else + rlookups[index] = false + end + end + end + end + end + end + -- end -- new per 2022-09-25 + + for i, n in sortedhash(sublookupcheck) do + local l = lookups[i] + local t = l.type + if n == 0 and t ~= "extension" then + local d = l.done + report("%s lookup %s of type %a is not used",what,d and d.name or l.name,t) + end + end + + end + + local function loadvariations(f,fontdata,variationsoffset,lookuptypes,featurehash,featureorder) + setposition(f,variationsoffset) + local version = readulong(f) -- two times readushort + local nofrecords = readulong(f) + local records = { } + for i=1,nofrecords do + records[i] = { + conditions = readulong(f), + substitutions = readulong(f), + } + end + for i=1,nofrecords do + local record = records[i] + local offset = record.conditions + if offset == 0 then + record.condition = nil + record.matchtype = "always" + else + local offset = variationsoffset+offset + setposition(f,offset) + local nofconditions = readushort(f) + local conditions = { } + for i=1,nofconditions do + conditions[i] = offset + readulong(f) + end + record.conditions = conditions + record.matchtype = "condition" + end + end + for i=1,nofrecords do + local record = records[i] + if record.matchtype == "condition" then + local conditions = record.conditions + for i=1,#conditions do + setposition(f,conditions[i]) + conditions[i] = { + format = readushort(f), + axis = readushort(f), + minvalue = read2dot14(f), + maxvalue = read2dot14(f), + } + end + end + end + + for i=1,nofrecords do + local record = records[i] + local offset = record.substitutions + if offset == 0 then + record.substitutions = { } + else + setposition(f,variationsoffset + offset) + local version = readulong(f) + local nofsubstitutions = readushort(f) + local substitutions = { } + for i=1,nofsubstitutions do + substitutions[readushort(f)] = readulong(f) + end + for index, alternates in sortedhash(substitutions) do + if index == 0 then + record.substitutions = false + else + local tableoffset = variationsoffset + offset + alternates + setposition(f,tableoffset) + local parameters = readulong(f) -- feature parameters + local noflookups = readushort(f) + local lookups = readcardinaltable(f,noflookups,ushort) -- not sure what to do with these + -- todo : resolve to proper lookups + record.substitutions = lookups + end + end + end + end + setvariabledata(fontdata,"features",records) + end + + local function readscripts(f,fontdata,what,lookuptypes,lookuphandlers,lookupstoo) + local tableoffset = gotodatatable(f,fontdata,what,true) + if tableoffset then + local version = readulong(f) + local scriptoffset = tableoffset + readushort(f) + local featureoffset = tableoffset + readushort(f) + local lookupoffset = tableoffset + readushort(f) + -- MFK : Rubik-Regular.ttf : we need to delay adding the offset + -- local variationsoffset = version > 0x00010000 and (tableoffset + readulong(f)) or 0 + local variationsoffset = version > 0x00010000 and readulong(f) or 0 + if not scriptoffset then + return + end + local scripts = readscriplan(f,fontdata,scriptoffset) + local features = readfeatures(f,fontdata,featureoffset) + -- + local scriptlangs, featurehash, featureorder = reorderfeatures(fontdata,scripts,features) + -- + if fontdata.features then + fontdata.features[what] = scriptlangs + else + fontdata.features = { [what] = scriptlangs } + end + -- + if not lookupstoo then + return + end + -- + local lookups = readlookups(f,lookupoffset,lookuptypes,featurehash,featureorder) + -- + if lookups then + resolvelookups(f,lookupoffset,fontdata,lookups,lookuptypes,lookuphandlers,what,tableoffset) + end + -- + if variationsoffset > 0 then + -- loadvariations(f,fontdata,variationsoffset,lookuptypes,featurehash,featureorder) + loadvariations(f,fontdata,tableoffset + variationsoffset,lookuptypes,featurehash,featureorder) + end + end + end + + local function checkkerns(f,fontdata,specification) + local datatable = fontdata.tables.kern + if not datatable then + return -- no kerns + end + local features = fontdata.features + local gposfeatures = features and features.gpos + local name + if not gposfeatures or not gposfeatures.kern then + name = "kern" + elseif specification.globalkerns then + name = "globalkern" + else + report("ignoring global kern table, using gpos kern feature") + return + end + setposition(f,datatable.offset) + local version = readushort(f) + local noftables = readushort(f) + if noftables > 1 then + report("adding global kern table as gpos feature %a",name) + local kerns = setmetatableindex("table") + for i=1,noftables do + local version = readushort(f) + local length = readushort(f) + local coverage = readushort(f) + -- bit 8-15 of coverage: format 0 or 2 + local format = rshift(coverage,8) -- is this ok + if format == 0 then + local nofpairs = readushort(f) + local searchrange = readushort(f) + local entryselector = readushort(f) + local rangeshift = readushort(f) + for i=1,nofpairs do + kerns[readushort(f)][readushort(f)] = readfword(f) + end + elseif format == 2 then + -- apple specific so let's ignore it + else + -- not supported by ms + end + end + local feature = { dflt = { dflt = true } } + if not features then + fontdata.features = { gpos = { [name] = feature } } + elseif not gposfeatures then + fontdata.features.gpos = { [name] = feature } + else + gposfeatures[name] = feature + end + local sequences = fontdata.sequences + if not sequences then + sequences = { } + fontdata.sequences = sequences + end + local nofsequences = #sequences + 1 + sequences[nofsequences] = { + index = nofsequences, + name = name, + steps = { + { + coverage = kerns, + format = "kern", + }, + }, + nofsteps = 1, + type = "gpos_pair", + flags = { false, false, false, false }, + order = { name }, + features = { [name] = feature }, + } + else + report("ignoring empty kern table of feature %a",name) + end + end + + function readers.gsub(f,fontdata,specification) + if specification.details then + readscripts(f,fontdata,"gsub",gsubtypes,gsubhandlers,specification.lookups) + end + end + + function readers.gpos(f,fontdata,specification) + if specification.details then + readscripts(f,fontdata,"gpos",gpostypes,gposhandlers,specification.lookups) + if specification.lookups then + checkkerns(f,fontdata,specification) + end + end + end + +end + +function readers.gdef(f,fontdata,specification) + if not specification.glyphs then + return + end + local datatable = fontdata.tables.gdef + if datatable then + local tableoffset = datatable.offset + setposition(f,tableoffset) + local version = readulong(f) + local classoffset = readushort(f) + local attachmentoffset = readushort(f) -- used for bitmaps + local ligaturecarets = readushort(f) -- used in editors (maybe nice for tracing) + local markclassoffset = readushort(f) + local marksetsoffset = version >= 0x00010002 and readushort(f) or 0 + local varsetsoffset = version >= 0x00010003 and readulong(f) or 0 + local glyphs = fontdata.glyphs + local marks = { } + local markclasses = setmetatableindex("table") + local marksets = setmetatableindex("table") + fontdata.marks = marks + fontdata.markclasses = markclasses + fontdata.marksets = marksets + -- class definitions + if classoffset ~= 0 then + setposition(f,tableoffset + classoffset) + local classformat = readushort(f) + if classformat == 1 then + local firstindex = readushort(f) + local lastindex = firstindex + readushort(f) - 1 + for index=firstindex,lastindex do + local class = classes[readushort(f)] + if class == "mark" then + marks[index] = true + end + glyphs[index].class = class + end + elseif classformat == 2 then + local nofranges = readushort(f) + for i=1,nofranges do + local firstindex = readushort(f) + local lastindex = readushort(f) + local class = classes[readushort(f)] + if class then + for index=firstindex,lastindex do + glyphs[index].class = class + if class == "mark" then + marks[index] = true + end + end + end + end + end + end + -- mark classes + if markclassoffset ~= 0 then + setposition(f,tableoffset + markclassoffset) + local classformat = readushort(f) + if classformat == 1 then + local firstindex = readushort(f) + local lastindex = firstindex + readushort(f) - 1 + for index=firstindex,lastindex do + markclasses[readushort(f)][index] = true + end + elseif classformat == 2 then + local nofranges = readushort(f) + for i=1,nofranges do + local firstindex = readushort(f) + local lastindex = readushort(f) + local class = markclasses[readushort(f)] + for index=firstindex,lastindex do + class[index] = true + end + end + end + end + -- mark sets : todo: just make the same as class sets above + if marksetsoffset ~= 0 then + marksetsoffset = tableoffset + marksetsoffset + setposition(f,marksetsoffset) + local format = readushort(f) + if format == 1 then + local nofsets = readushort(f) + local sets = readcardinaltable(f,nofsets,ulong) + for i=1,nofsets do + local offset = sets[i] + if offset ~= 0 then + marksets[i] = readcoverage(f,marksetsoffset+offset) + end + end + end + end + + local factors = specification.factors + + if (specification.variable or factors) and varsetsoffset ~= 0 then + + local regions, deltas = readvariationdata(f,tableoffset+varsetsoffset,factors) + + -- setvariabledata(fontdata,"gregions",regions) + + if factors then + fontdata.temporary.getdelta = function(outer,inner) + local delta = deltas[outer+1] + if delta then + local d = delta.deltas[inner+1] + if d then + local scales = delta.scales + local dd = 0 + for i=1,#scales do + local di = d[i] + if di then + dd = dd + scales[i] * di + else + break + end + end + return round(dd) + end + end + return 0 + end + end + + end + end +end + +-- We keep this code here instead of font-otm.lua because we need coverage +-- helpers. Okay, these helpers could go to the main reader file some day. + +local function readmathvalue(f) + local v = readshort(f) + skipshort(f,1) -- offset to device table + return v +end + +local function readmathconstants(f,fontdata,offset) + setposition(f,offset) + fontdata.mathconstants = { + ScriptPercentScaleDown = readshort(f), + ScriptScriptPercentScaleDown = readshort(f), + DelimitedSubFormulaMinHeight = readushort(f), + DisplayOperatorMinHeight = readushort(f), + MathLeading = readmathvalue(f), + AxisHeight = readmathvalue(f), + AccentBaseHeight = readmathvalue(f), + FlattenedAccentBaseHeight = readmathvalue(f), + SubscriptShiftDown = readmathvalue(f), + SubscriptTopMax = readmathvalue(f), + SubscriptBaselineDropMin = readmathvalue(f), + SuperscriptShiftUp = readmathvalue(f), + SuperscriptShiftUpCramped = readmathvalue(f), + SuperscriptBottomMin = readmathvalue(f), + SuperscriptBaselineDropMax = readmathvalue(f), + SubSuperscriptGapMin = readmathvalue(f), + SuperscriptBottomMaxWithSubscript = readmathvalue(f), + SpaceAfterScript = readmathvalue(f), + UpperLimitGapMin = readmathvalue(f), + UpperLimitBaselineRiseMin = readmathvalue(f), + LowerLimitGapMin = readmathvalue(f), + LowerLimitBaselineDropMin = readmathvalue(f), + StackTopShiftUp = readmathvalue(f), + StackTopDisplayStyleShiftUp = readmathvalue(f), + StackBottomShiftDown = readmathvalue(f), + StackBottomDisplayStyleShiftDown = readmathvalue(f), + StackGapMin = readmathvalue(f), + StackDisplayStyleGapMin = readmathvalue(f), + StretchStackTopShiftUp = readmathvalue(f), + StretchStackBottomShiftDown = readmathvalue(f), + StretchStackGapAboveMin = readmathvalue(f), + StretchStackGapBelowMin = readmathvalue(f), + FractionNumeratorShiftUp = readmathvalue(f), + FractionNumeratorDisplayStyleShiftUp = readmathvalue(f), + FractionDenominatorShiftDown = readmathvalue(f), + FractionDenominatorDisplayStyleShiftDown = readmathvalue(f), + FractionNumeratorGapMin = readmathvalue(f), + FractionNumeratorDisplayStyleGapMin = readmathvalue(f), + FractionRuleThickness = readmathvalue(f), + FractionDenominatorGapMin = readmathvalue(f), + FractionDenominatorDisplayStyleGapMin = readmathvalue(f), + SkewedFractionHorizontalGap = readmathvalue(f), + SkewedFractionVerticalGap = readmathvalue(f), + OverbarVerticalGap = readmathvalue(f), + OverbarRuleThickness = readmathvalue(f), + OverbarExtraAscender = readmathvalue(f), + UnderbarVerticalGap = readmathvalue(f), + UnderbarRuleThickness = readmathvalue(f), + UnderbarExtraDescender = readmathvalue(f), + RadicalVerticalGap = readmathvalue(f), + RadicalDisplayStyleVerticalGap = readmathvalue(f), + RadicalRuleThickness = readmathvalue(f), + RadicalExtraAscender = readmathvalue(f), + RadicalKernBeforeDegree = readmathvalue(f), + RadicalKernAfterDegree = readmathvalue(f), + RadicalDegreeBottomRaisePercent = readshort(f), + } +end + +local function readmathglyphinfo(f,fontdata,offset) + setposition(f,offset) + local italics = readushort(f) + local accents = readushort(f) + local extensions = readushort(f) + local kerns = readushort(f) + local glyphs = fontdata.glyphs + if italics ~= 0 then + setposition(f,offset+italics) + local coverage = readushort(f) + local nofglyphs = readushort(f) + coverage = readcoverage(f,offset+italics+coverage,true) + setposition(f,offset+italics+4) + for i=1,nofglyphs do + local italic = readmathvalue(f) + if italic ~= 0 then + local glyph = glyphs[coverage[i]] + local math = glyph.math + if not math then + glyph.math = { italic = italic } + else + math.italic = italic + end + end + end + fontdata.hasitalics = true + end + if accents ~= 0 then + setposition(f,offset+accents) + local coverage = readushort(f) + local nofglyphs = readushort(f) + coverage = readcoverage(f,offset+accents+coverage,true) + setposition(f,offset+accents+4) + for i=1,nofglyphs do + local accent = readmathvalue(f) + if accent ~= 0 then + local glyph = glyphs[coverage[i]] + local math = glyph.math + if not math then + glyph.math = { accent = accent } + else + math.accent = accent -- will become math.topanchor + end + end + end + end + if extensions ~= 0 then + setposition(f,offset+extensions) + end + if kerns ~= 0 then + local kernoffset = offset + kerns + setposition(f,kernoffset) + local coverage = readushort(f) + local nofglyphs = readushort(f) + if nofglyphs > 0 then + local function get(offset) + setposition(f,kernoffset+offset) + local n = readushort(f) + if n == 0 then + local k = readmathvalue(f) + if k == 0 then + -- no need for it (happens sometimes) + else + return { { kern = k } } + end + else + local l = { } + for i=1,n do + l[i] = { height = readmathvalue(f) } + end + for i=1,n do + l[i].kern = readmathvalue(f) + end + l[n+1] = { kern = readmathvalue(f) } + return l + end + end + local kernsets = { } + for i=1,nofglyphs do + local topright = readushort(f) + local topleft = readushort(f) + local bottomright = readushort(f) + local bottomleft = readushort(f) + kernsets[i] = { + topright = topright ~= 0 and topright or nil, + topleft = topleft ~= 0 and topleft or nil, + bottomright = bottomright ~= 0 and bottomright or nil, + bottomleft = bottomleft ~= 0 and bottomleft or nil, + } + end + coverage = readcoverage(f,kernoffset+coverage,true) + for i=1,nofglyphs do + local kernset = kernsets[i] + if next(kernset) then + local k = kernset.topright if k then kernset.topright = get(k) end + local k = kernset.topleft if k then kernset.topleft = get(k) end + local k = kernset.bottomright if k then kernset.bottomright = get(k) end + local k = kernset.bottomleft if k then kernset.bottomleft = get(k) end + if next(kernset) then + local glyph = glyphs[coverage[i]] + local math = glyph.math + if math then + math.kerns = kernset + else + glyph.math = { kerns = kernset } + end + end + end + end + end + end +end + +local function readmathvariants(f,fontdata,offset) + setposition(f,offset) + local glyphs = fontdata.glyphs + local minoverlap = readushort(f) + local vcoverage = readushort(f) + local hcoverage = readushort(f) + local vnofglyphs = readushort(f) + local hnofglyphs = readushort(f) + local vconstruction = readcardinaltable(f,vnofglyphs,ushort) + local hconstruction = readcardinaltable(f,hnofglyphs,ushort) + + fontdata.mathconstants.MinConnectorOverlap = minoverlap + + -- variants[i] = { + -- glyph = readushort(f), + -- advance = readushort(f), + -- } + + local function get(offset,coverage,nofglyphs,construction,kvariants,kparts,kitalic,korientation,orientation) + if coverage ~= 0 and nofglyphs > 0 then + local coverage = readcoverage(f,offset+coverage,true) + for i=1,nofglyphs do + local c = construction[i] + if c ~= 0 then + local index = coverage[i] + local glyph = glyphs[index] + local math = glyph.math + setposition(f,offset+c) + local assembly = readushort(f) + local nofvariants = readushort(f) + if nofvariants > 0 then + local variants, v = nil, 0 + for i=1,nofvariants do + local variant = readushort(f) + if variant == index then + -- ignore + elseif variants then + v = v + 1 + variants[v] = variant + else + v = 1 + variants = { variant } + end + skipshort(f) + end + if not variants then + -- only self + elseif not math then + math = { [kvariants] = variants } + glyph.math = math + else + math[kvariants] = variants + end + end + if assembly ~= 0 then + setposition(f,offset + c + assembly) + local italic = readmathvalue(f) + local nofparts = readushort(f) + local parts = { } + for i=1,nofparts do + local p = { + glyph = readushort(f), + start = readushort(f), + ["end"] = readushort(f), + advance = readushort(f), + } + local flags = readushort(f) + if band(flags,0x0001) ~= 0 then + p.extender = 1 -- true + end + parts[i] = p + end + if not math then + math = { + [kparts] = parts + } + glyph.math = math + else + math[kparts] = parts + end + if italic and italic ~= 0 then + math[kitalic] = italic + end + if orientation then + math[korientation] = orientation + end + end + end + end + end + end + + -- if CONTEXTLMTXMODE and CONTEXTLMTXMODE > 0 then + get(offset,hcoverage,hnofglyphs,hconstruction,"variants","parts","partsitalic","partsorientation","horizontal") + get(offset,vcoverage,vnofglyphs,vconstruction,"variants","parts","partsitalic","partsorientation","vertical") + -- else + -- get(offset,vcoverage,vnofglyphs,vconstruction,"vvariants","vparts","vitalic") + -- get(offset,hcoverage,hnofglyphs,hconstruction,"hvariants","hparts","hitalic") + -- end +end + +function readers.math(f,fontdata,specification) + local tableoffset = gotodatatable(f,fontdata,"math",specification.glyphs) + if tableoffset then + local version = readulong(f) + -- if version ~= 0x00010000 then + -- report("table version %a of %a is not supported (yet), maybe font %s is bad",version,"math",fontdata.filename) + -- return + -- end + local constants = readushort(f) + local glyphinfo = readushort(f) + local variants = readushort(f) + if constants == 0 then + report("the math table of %a has no constants",fontdata.filename) + else + readmathconstants(f,fontdata,tableoffset+constants) + end + if glyphinfo ~= 0 then + readmathglyphinfo(f,fontdata,tableoffset+glyphinfo) + end + if variants ~= 0 then + readmathvariants(f,fontdata,tableoffset+variants) + end + end +end + +do + + -- format 1: PaintColrLayers + -- for each referenced child paint table, in bottom-up z-order: + -- call renderPaint() passing the child paint table + -- compose the returned graphic onto the surface using simplealpha blending + -- + -- format 2, 3: PaintSolid + -- paint the specified color onto the surface + -- + -- format 4, 5, 6, 7, 8, 9: PaintLinearGradient, PaintRadialGradient, PaintSweepGradient + -- paint the gradient onto the surface following the gradient algorithm + -- + -- format 10: PaintGlyph + -- apply the outline of the referenced glyph to the clip region + -- (take the intersection of clip regions—see Filling shapes) + -- call renderPaint() passing the child paint table + -- restore the previous clip region + -- + -- format 11: PaintColrGlyph + -- call renderPaint() passing the paint table referenced by the base glyph ID + -- + -- format 12 .. 31: Transform Translate Scale + -- apply the specified transform, compose the transform with the current transform + -- call renderPaint() passing the child paint table + -- restore the previous transform state + -- + -- format 32: PaintComposite + -- call renderPaint() passing the backdrop child paint table and save the result + -- call renderPaint() passing the source child paint table and save the result + -- compose the source and backdrop using the specified composite mode + -- compose the result of the above composition onto the surface using simple alpha blending + + local paintdata -- for the moment verbose, will be just indexed + local linesdata -- for the moment verbose, will be just indexed + local affinedata -- for the moment verbose, will be just indexed + + local function getpaintoffset(f,offset) + offset = offset + readuoffset(f) + return paintdata[offset] and offset or nil + end + + local function getlinesoffset(f,offset,var) + local offset = offset + readuoffset(f) + if linesdata[offset] == nil then + linesdata[offset] = var + end + return offset + end + + local function getaffineoffset(f,offset,var) + local offset = offset + readuoffset(f) + if affinedata[offset] == nil then + affinedata[offset] = var + end + return offset + end + + paintreaders = { + -- uint8 numLayers Number of offsets to paint tables to read from LayerList. + -- uint32 firstLayerIndex Index (base 0) into the LayerList. + [1] = function(f,format) + return { + format = format, + name = "PaintColrLayers", + count = readuinteger(f), + index = readulong(f), + list = false, + } + end, + -- uint16 paletteIndex Index for a CPAL palette entry. + -- F2DOT14 alpha Alpha value. + [2] = function(f,format) + return { + format = format, + name = "Paintsolid", + palette = readushort(f), + alpha = read2dot14(f), + } + end, + -- uint16 paletteIndex Index for a CPAL palette entry. + -- F2DOT14 alpha Alpha value. For variation, use varIndexBase + 0. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [3] = function(f,format) + return { + format = format, + name = "Paintsolid", + palette = readushort(f), + alpha = read2dot14(f), + varbase = readulong(f), + } + end, + -- Offset24 colorLineOffset Offset to VarColorLine table. + -- FWORD x0 Start point (p₀) x coordinate. For variation, use varIndexBase + 0. + -- FWORD y0 Start point (p₀) y coordinate. For variation, use varIndexBase + 1. + -- FWORD x1 End point (p₁) x coordinate. For variation, use varIndexBase + 2. + -- FWORD y1 End point (p₁) y coordinate. For variation, use varIndexBase + 3. + -- FWORD x2 Rotation point (p₂) x coordinate. For variation, use varIndexBase + 4. + -- FWORD y2 Rotation point (p₂) y coordinate. For variation, use varIndexBase + 5. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [4] = function(f,format,offset) + return { + format = format, + name = "PaintLinearGradient", + color = getlinesoffset(f,offset,false), + x0 = readfword(f), + y0 = readfword(f), + x1 = readfword(f), + y1 = readfword(f), + x2 = readfword(f), + y2 = readfword(f), + } + end, + [5] = function(f,format,offset) + return { + format = format, + name = "PaintLinearGradient", + color = getlinesoffset(f,offset,true), + x0 = readfword(f), + y0 = readfword(f), + x1 = readfword(f), + y1 = readfword(f), + x2 = readfword(f), + y2 = readfword(f), + varbase = readulong(f), + } + end, + -- Offset24 colorLineOffset Offset to VarColorLine table. + -- FWORD x0 Start circle center x coordinate. For variation, use varIndexBase + 0. + -- FWORD y0 Start circle center y coordinate. For variation, use varIndexBase + 1. + -- UFWORD radius0 Start circle radius. For variation, use varIndexBase + 2. + -- FWORD x1 End circle center x coordinate. For variation, use varIndexBase + 3. + -- FWORD y1 End circle center y coordinate. For variation, use varIndexBase + 4. + -- UFWORD radius1 End circle radius. For variation, use varIndexBase + 5. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [6] = function(f,format) + return { + format = format, + name = "PaintRadialGradient", + color = getlinesoffset(f,offset,false), + x0 = readfword(f), + y0 = readfword(f), + radius0 = readfword(f), + x1 = readfword(f), + y1 = readfword(f), + radius1 = readfword(f), + } + end, + [7] = function(f,format) + return { + format = format, + name = "PaintRadialGradient", + color = getlinesoffset(f,offset,true), + x0 = readfword(f), + y0 = readfword(f), + radius0 = readfword(f), + x1 = readfword(f), + y1 = readfword(f), + radius1 = readfword(f), + varbase = readulong(f), + } + end, + -- Offset24 colorLineOffset Offset to VarColorLine table. + -- FWORD centerX Center x coordinate. For variation, use varIndexBase + 0. + -- FWORD centerY Center y coordinate. For variation, use varIndexBase + 1. + -- F2DOT14 startAngle Start of the angular range of the gradient, 180° in counter-clockwise degrees per 1.0 of value. For variation, use varIndexBase + 2. + -- F2DOT14 endAngle End of the angular range of the gradient, 180° in counter-clockwise degrees per 1.0 of value. For variation, use varIndexBase + 3. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [8] = function(f,format) + return { + format = format, + name = "PaintSweepGradient", + color = getlinesoffset(f,offset,false), + centerx = readfword(f), + centery = readfword(f), + startangle = read2dot14(f), + endangle = read2dot14(f), + } + end, + [9] = function(f,format) + return { + format = format, + name = "PaintSweepGradient", + color = getlinesoffset(f,offset,true), + centerx = readfword(f), + centery = readfword(f), + startangle = read2dot14(f), + endangle = read2dot14(f), + varbase = readulong(f), + } + end, + -- Offset24 paintOffset Offset to a Paint table. + -- uint16 glyphID Glyph ID for the source outline. + [10] = function(f,format,offset) + return { + format = format, + name = "PaintGlyph", + paint = getpaintoffset(f,offset), + glyph = readushort(f), + } + end, + -- uint16 glyphID Glyph ID for a BaseGlyphList base glyph. + [11] = function(f,format) + return { + format = format, + name = "PaintColrGlyph", + glyph = readushort(f), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- Offset24 transformOffset Offset to an (Var)Affine2x3 table. + [12] = function(f,format,offset) + return { + format = format, + name = "PaintTransform", + affine = getaffineoffset(f,offset,false), + paint = getpaintoffset(f,offset), + } + end, + [13] = function(f,format,offset) + return { + format = format, + name = "PaintTransform", + affine = getaffineoffset(f,offset,true), + paint = getpaintoffset(f,offset), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- FWORD dx Translation in x direction. For variation, use varIndexBase + 0. + -- FWORD dy Translation in y direction. For variation, use varIndexBase + 1. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [14] = function(f,format,offset) + return { + format = format, + name = "PaintTranslate", + paint = getpaintoffset(f,offset), + dx = readfword(f), + dy = readfword(f), + } + end, + [15] = function(f,format,offset) + return { + format = format, + name = "PaintTranslate", + paint = getpaintoffset(f,offset), + dx = readfword(f), + dy = readfword(f), + varbase = readulong(f), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- F2DOT14 scaleX Scale factor in x direction. For variation, use varIndexBase + 0. + -- F2DOT14 scaleY Scale factor in y direction. For variation, use varIndexBase + 1. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [16] = function(f,format,offset) + return { + format = format, + name = "PaintScale", + paint = getpaintoffset(f,offset), + scalex = read2dot14(f), + scaley = read2dot14(f), + } + end, + [17] = function(f,format,offset) + return { + format = format, + name = "PaintScale", + paint = getpaintoffset(f,offset), + scalex = read2dot14(f), + scaley = read2dot14(f), + varbase = readulong(f), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- F2DOT14 scaleX Scale factor in x direction. For variation, use varIndexBase + 0. + -- F2DOT14 scaleY Scale factor in y direction. For variation, use varIndexBase + 1. + -- FWORD centerX x coordinate for the center of scaling. For variation, use varIndexBase + 2. + -- FWORD centerY y coordinate for the center of scaling. For variation, use varIndexBase + 3. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [18] = function(f,format,offset) + return { + format = format, + name = "PaintScale", + paint = getpaintoffset(f,offset), + scalex = read2dot14(f), + scaley = read2dot14(f), + centerx = readfword(f), + centery = readfword(f), + } + end, + [19] = function(f,format,offset) + return { + format = format, + name = "PaintScale", + paint = getpaintoffset(f,offset), + scalex = read2dot14(f), + scaley = read2dot14(f), + centerx = readfword(f), + centery = readfword(f), + varbase = readulong(f), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- F2DOT14 scale Scale factor in x and y directions. For variation, use varIndexBase + 0. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [20] = function(f,format,offset) + return { + format = format, + name = "PaintScale", + paint = getpaintoffset(f,offset), + scale = read2dot14(f), + } + end, + [21] = function(f,format,offset) + return { + format = format, + name = "PaintScale", + paint = getpaintoffset(f,offset), + scale = read2dot14(f), + varbase = readulong(f), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- F2DOT14 scale Scale factor in x and y directions. For variation, use varIndexBase + 0. + -- FWORD centerX x coordinate for the center of scaling. For variation, use varIndexBase + 1. + -- FWORD centerY y coordinate for the center of scaling. For variation, use varIndexBase + 2. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [22] = function(f,format,offset) + return { + format = format, + name = "PaintScale", + paint = getpaintoffset(f,offset), + scale = read2dot14(f), + centerx = readfword(f), + centery = readfword(f), + } + end, + [23] = function(f,format,offset) + return { + format = format, + name = "PaintScale", + paint = getpaintoffset(f,offset), + scale = read2dot14(f), + centerx = readfword(f), + centery = readfword(f), + varbase = readulong(f), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- F2DOT14 angle Rotation angle, 180° in counter-clockwise degrees per 1.0 of value. For variation, use varIndexBase + 0. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [24] = function(f,format,offset) + return { + format = format, + angle = read2dot14(f), + paint = getpaintoffset(f,offset), + name = "PaintRotate", + } + end, + [25] = function(f,format,offset) + return { + format = format, + name = "PaintRotate", + paint = getpaintoffset(f,offset), + angle = read2dot14(f), + varbase = readulong(f), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- F2DOT14 angle Rotation angle, 180° in counter-clockwise degrees per 1.0 of value. For variation, use varIndexBase + 0. + -- FWORD centerX x coordinate for the center of rotation. For variation, use varIndexBase + 1. + -- FWORD centerY y coordinate for the center of rotation. For variation, use varIndexBase + 2. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [26] = function(f,format,offset) + return { + format = format, + name = "PaintRotate", + paint = getpaintoffset(f,offset), + centerx = readfword(f), + centery = readfword(f), + } + end, + [27] = function(f,format,offset) + return { + format = format, + name = "PaintRotate", + paint = getpaintoffset(f,offset), + centerx = read2dot14(f), + centery = read2dot14(f), + varbase = readulong(f), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- F2DOT14 xSkewAngle Angle of skew in the direction of the x-axis, 180° in counter-clockwise degrees per 1.0 of value. For variation, use varIndexBase + 0. + -- F2DOT14 ySkewAngle Angle of skew in the direction of the y-axis, 180° in counter-clockwise degrees per 1.0 of value. For variation, use varIndexBase + 1. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [28] = function(f,format,offset) + return { + format = format, + name = "PaintSkew", + paint = getpaintoffset(f,offset), + xangle = read2dot14(f), + yangle = read2dot14(f), + } + end, + [29] = function(f,format,offset) + return { + format = format, + name = "PaintSkew", + paint = getpaintoffset(f,offset), + xangle = read2dot14(f), + yangle = read2dot14(f), + varbase = readulong(f), + } + end, + -- Offset24 paintOffset Offset to a Paint subtable. + -- F2DOT14 xSkewAngle Angle of skew in the direction of the x-axis, 180° in counter-clockwise degrees per 1.0 of value. For variation, use varIndexBase + 0. + -- F2DOT14 ySkewAngle Angle of skew in the direction of the y-axis, 180° in counter-clockwise degrees per 1.0 of value. For variation, use varIndexBase + 1. + -- FWORD centerX x coordinate for the center of rotation. For variation, use varIndexBase + 2. + -- FWORD centerY y coordinate for the center of rotation. For variation, use varIndexBase + 3. + -- uint32 varIndexBase Base index into DeltaSetIndexMap. + [30] = function(f,format,offset) + return { + format = format, + name = "PaintSkew", + paint = getpaintoffset(f,offset), + xangle = read2dot14(f), + yangle = read2dot14(f), + centerx = readfword(f), + centery = readfword(f), + } + end, + [31] = function(f,format,offset) + return { + format = format, + name = "PaintSkew", + paint = getpaintoffset(f,offset), + xangle = read2dot14(f), + yangle = read2dot14(f), + centerx = readfword(f), + centery = readfword(f), + varbase = readulong(f), + } + end, + -- Offset24 sourcePaintOffset Offset to a source Paint table. + -- uint8 compositeMode A CompositeMode enumeration value. + -- Offset24 backdropaintOffset Offset to a backdrop Paint table. + [32] = function(f,format,offset) + return { + format = format, + name = "PaintComposite", + source = getpaintoffset(f,offset), + mode = readuinteger(f), + backdrop = getpaintoffset(f,offset), + } + end, + } + + local unsupported = function() + return nil + end + + setmetatableindex(paintreaders,function(t,format) + report("unsupported colr type 2 paint format %i",format) + t[format] = unsupported + return unsupported + end) + + function readers.colr(f,fontdata,specification) + local tableoffset = gotodatatable(f,fontdata,"colr",specification.glyphs) + if tableoffset then + local version = readushort(f) + if version == 0 then + -- we're okay + elseif version == 1 then + report("table version %a of %a is %s supported for font %s",version,"colr","partially",fontdata.filename) + else + report("table version %a of %a is %s supported for font %s",version,"colr","not",fontdata.filename) + return + end + if not fontdata.tables.cpal then + report("color table %a in font %a has no mandate %a table","colr",fontdata.filename,"cpal") + fontdata.colorpalettes = { } + end + local glyphs = fontdata.glyphs + local nofglyphs = readushort(f) + local baseoffset = readulong(f) + local layeroffset = readulong(f) + local noflayers = readushort(f) + local glyphlistoffset = 0 + local layerlistoffset = 0 + local cliplistoffset = 0 + local varindexmapoffset = 0 + local variationoffset = 0 + if version == 1 then + glyphlistoffset = readulong(f) + layerlistoffset = readulong(f) + cliplistoffset = readulong(f) + varindexmapoffset = readulong(f) + variationoffset = readulong(f) + end + local layerrecords = { } + local maxclass = 0 + -- The special value 0xFFFF is foreground (but we index from 1). It + -- more looks like indices into a palette so 'class' is a better name + -- than 'palette'. + if layeroffset > 0 then + setposition(f,tableoffset + layeroffset) + for i=1,noflayers do + local slot = readushort(f) + local class = readushort(f) + if class < 0xFFFF then + class = class + 1 + if class > maxclass then + maxclass = class + end + end + layerrecords[i] = { + slot = slot, + class = class, + } + end + end + fontdata.maxcolorclass = maxclass + if baseoffset > 0 then + setposition(f,tableoffset + baseoffset) + for i=0,nofglyphs-1 do + local glyphindex = readushort(f) + local firstlayer = readushort(f) + local noflayers = readushort(f) + local t = { } + for i=1,noflayers do + t[i] = layerrecords[firstlayer+i] + end + glyphs[glyphindex].colors = t + end + end +if next(layerrecords) then + report("table version %a of %a is %s supported for font %s",version,"colr","partially",fontdata.filename) + return +end + -- if not (CONTEXTLMTXMODE and CONTEXTLMTXMODE > 0) then + -- return + -- end + if layerlistoffset > 0 and glyphlistoffset > 0 then + local layers = { } + local paints = { } + local count, offset + -- + setposition(f,tableoffset + layerlistoffset) + count = readulong(f) + -- layers = readcardinaltable(f,count,uoffset) + for i=1,count do -- zero ? + layers[i] = readulong(f) -- offsets to painttable + end + -- + offset = tableoffset + glyphlistoffset + setposition(f,offset) + count = readulong(f) + for i=1,count do + -- glyph index -> paintrecord + paints[readushort(f)] = readulong(f) -- paintrecord offset (32 formats) + end + paintdata = setmetatableindex(function(t,k) + setposition(f,k) + local format = readuinteger(f) + local v = paintreaders[format](f,format,k) + t[k] = v + return v + end) + linesdata = { } + affinedata = { } + for k, v in next, paints do + local o = offset + v + if paintdata[o] then + paints[k] = o -- first paint + end + end + -- expand format 1 + offset = tableoffset + layerlistoffset + for k, v in next, paints do + v = paintdata[v] + local format = v.format + if format == 1 then + -- name + local count = v.count + local index = v.index + local list = { } + v.count = nil + v.index = nil + v.list = list + for i=1,count do + local o = offset + layers[index+i] + if paintdata[o] then + list[i] = o + end + end + end + end + -- + if variationoffset > 0 then + local offsettostore = tableoffset + variationoffset + local factors = specification.factors + if factors then + local regions, deltas = readvariationdata(f,offsettostore,factors) + report("font %a has a colr variations, check it out",fontdata.filename) + -- inspect(regions) + -- inspect(deltas) + end + end + -- + -- It will take while before I finish this, also because there has never + -- been much demand for color fonts and there is no real incentive for + -- spending too much time on it. + -- + -- todo: cliplist, varbase, affine, deltas etc + -- + -- reindex tables, gives smaller files but optional because of tracing + -- + for k, v in next, linesdata do + setposition(f,k) + local extend = readuinteger(f) + local count = readushort(f) + local stops = { } + for i=1,count do + stops[i] = { + stop = read2dot14(f), + pallette = readushort(f), + alpha = read2dot14(f), + varbase = v and readulong(f) or nil, + } + end + linesdata[k] = { + extend = readuinteger(f), + stops = stops, + } + end + -- + for k, v in next, affinedata do + setposition(f,k) + affinedata[k] = { + xx = readfixed(f), + yx = readfixed(f), + xy = readfixed(f), + yy = readfixed(f), + dx = readfixed(f), + dy = readfixed(f), + } + end + -- + local function rehash(t) + local hash = { } + local data = { } + local n = 0 + for k, v in table.sortedhash(t) do + n = n + 1 + hash[k] = n + data[n] = v + end + return hash, data + end + -- + if true then + local phash, pdata = rehash(paintdata) + local lhash, ldata = rehash(linesdata) + local ahash, adata = rehash(affinedata) + for k, v in next, paintdata do + local c = v.color + if c then + v.color = lhash[c] + end + local a = v.affine + if a then + v.affine = ahash[a] + end + local p = v.paint + if p then + v.paint = phash[p] + goto done + end + local l = v.list + if l then + for i=1,#l do + l[i] = phash[l[i]] + end + goto done + end + local s = v.source + if s then + v.source = phash[s] + v.backdrop = phash[v.backdrop] + -- goto done + end + ::done:: + end + paintdata = pdata + linesdata = ldata + for k, v in next, paints do -- zero indexed + paints[k] = phash[v] + end + end + -- + if not next(layerrecords) then + for k, v in next, paints do + local paint = paintdata[v] + local format = paint.format + if format == 1 then + local list = paint.list + local done = { } + for i=1,#list do + local p = paintdata[list[i]] + local f = p.format + if f == 10 or f == 11 then + done[i] = { + slot = p.glyph, + class = i, + } + end + end + glyphs[k].colors = done + end + end + end +-- fontdata.colorpaintdata = paintdata +-- fontdata.colorpaintlist = paints +-- fontdata.colorlinesdata = linesdata +-- fontdata.coloraffinedata = affinedata + end + end + fontdata.hascolor = true + end + +end + +function readers.cpal(f,fontdata,specification) + local tableoffset = gotodatatable(f,fontdata,"cpal",specification.glyphs) + if tableoffset then + local version = readushort(f) + -- if version > 1 then + -- report("table version %a of %a is not supported (yet), maybe font %s is bad",version,"cpal",fontdata.filename) + -- return + -- end + local nofpaletteentries = readushort(f) + local nofpalettes = readushort(f) + local nofcolorrecords = readushort(f) + local firstcoloroffset = readulong(f) + local colorrecords = { } + local palettes = readcardinaltable(f,nofpalettes,ushort) + if version == 1 then + -- used for guis + local palettettypesoffset = readulong(f) + local palettelabelsoffset = readulong(f) + local paletteentryoffset = readulong(f) + end + setposition(f,tableoffset+firstcoloroffset) + for i=1,nofcolorrecords do + local b, g, r, a = readbytes(f,4) + colorrecords[i] = { + r, g, b, a ~= 255 and a or nil, + } + end + for i=1,nofpalettes do + local p = { } + local o = palettes[i] + for j=1,nofpaletteentries do + p[j] = colorrecords[o+j] + end + palettes[i] = p + end + fontdata.colorpalettes = palettes + end +end + +local compress = gzip and gzip.compress +local compressed = compress and gzip.compressed + +-- At some point I will delay loading and only store the offsets (in context lmtx +-- only). + +-- compressed = false + +function readers.svg(f,fontdata,specification) + local tableoffset = gotodatatable(f,fontdata,"svg",specification.glyphs) + if tableoffset then + local version = readushort(f) + -- if version ~= 0 then + -- report("table version %a of %a is not supported (yet), maybe font %s is bad",version,"svg",fontdata.filename) + -- return + -- end + local glyphs = fontdata.glyphs + local indexoffset = tableoffset + readulong(f) + local reserved = readulong(f) + setposition(f,indexoffset) + local nofentries = readushort(f) + local entries = { } + for i=1,nofentries do + entries[i] = { + first = readushort(f), + last = readushort(f), + offset = indexoffset + readulong(f), + length = readulong(f), + } + end + for i=1,nofentries do + local entry = entries[i] + setposition(f,entry.offset) + local data = readstring(f,entry.length) + if compressed and not compressed(data) then + data = compress(data) + end + entries[i] = { + first = entry.first, + last = entry.last, + data = data + } + end + fontdata.svgshapes = entries + end + fontdata.hascolor = true +end + +function readers.sbix(f,fontdata,specification) + local tableoffset = gotodatatable(f,fontdata,"sbix",specification.glyphs) + if tableoffset then + local version = readushort(f) + local flags = readushort(f) + local nofstrikes = readulong(f) + local strikes = { } + local nofglyphs = fontdata.nofglyphs + for i=1,nofstrikes do + strikes[i] = readulong(f) + end + local shapes = { } + local done = 0 + for i=1,nofstrikes do + local strikeoffset = strikes[i] + tableoffset + setposition(f,strikeoffset) + strikes[i] = { + ppem = readushort(f), + ppi = readushort(f), + offset = strikeoffset + } + end + -- highest first + sort(strikes,function(a,b) + if b.ppem == a.ppem then + return b.ppi < a.ppi + else + return b.ppem < a.ppem + end + end) + local glyphs = { } + -- local delayed = CONTEXTLMTXMODE and CONTEXTLMTXMODE > 0 or fonts.handlers.typethree + for i=1,nofstrikes do + local strike = strikes[i] + local strikeppem = strike.ppem + local strikeppi = strike.ppi + local strikeoffset = strike.offset + setposition(f,strikeoffset) + for i=0,nofglyphs do + glyphs[i] = readulong(f) + end + local glyphoffset = glyphs[0] + for i=0,nofglyphs-1 do + local nextoffset = glyphs[i+1] + if not shapes[i] then + local datasize = nextoffset - glyphoffset + if datasize > 0 then + setposition(f,strikeoffset + glyphoffset) + local x = readshort(f) + local y = readshort(f) + local tag = readtag(f) -- or just skip, we never needed it till now + local size = datasize - 8 + local data = nil + local offset = nil + -- if delayed then + offset = getposition(f) + -- else + -- data = readstring(f,size) + -- size = nil + -- end + shapes[i] = { + x = x, + y = y, + o = offset, + s = size, + data = data, + -- tag = tag, -- maybe for tracing + -- ppem = strikeppem, -- not used, for tracing + -- ppi = strikeppi, -- not used, for tracing + } + done = done + 1 + if done == nofglyphs then + break + end + end + end + glyphoffset = nextoffset + end + end + fontdata.pngshapes = shapes + end +end + +-- Another bitmap (so not that useful) format. But Luigi found a font that +-- has them , so ... + +do + + local function getmetrics(f) + return { + ascender = readinteger(f), + descender = readinteger(f), + widthmax = readuinteger(f), + caretslopedumerator = readinteger(f), + caretslopedenominator = readinteger(f), + caretoffset = readinteger(f), + minorigin = readinteger(f), + minadvance = readinteger(f), + maxbefore = readinteger(f), + minafter = readinteger(f), + pad1 = readinteger(f), + pad2 = readinteger(f), + } + end + + -- bad names + + local function getbigmetrics(f) + -- bigmetrics, maybe just skip 9 bytes + return { + height = readuinteger(f), + width = readuinteger(f), + horiBearingX = readinteger(f), + horiBearingY = readinteger(f), + horiAdvance = readuinteger(f), + vertBearingX = readinteger(f), + vertBearingY = readinteger(f), + vertAdvance = readuinteger(f), + } + end + + local function getsmallmetrics(f) + -- smallmetrics, maybe just skip 5 bytes + return { + height = readuinteger(f), + width = readuinteger(f), + bearingX = readinteger(f), + bearingY = readinteger(f), + advance = readuinteger(f), + } + end + + function readers.cblc(f,fontdata,specification) + -- should we delay this ? + local ctdttableoffset = gotodatatable(f,fontdata,"cbdt",specification.glyphs) + if not ctdttableoffset then + return + end + local cblctableoffset = gotodatatable(f,fontdata,"cblc",specification.glyphs) + if cblctableoffset then + local majorversion = readushort(f) + local minorversion = readushort(f) + local nofsizetables = readulong(f) + local sizetables = { } + local shapes = { } + local subtables = { } + for i=1,nofsizetables do + sizetables[i] = { + subtables = readulong(f), + indexsize = readulong(f), + nofsubtables = readulong(f), + colorref = readulong(f), + hormetrics = getmetrics(f), + vermetrics = getmetrics(f), + firstindex = readushort(f), + lastindex = readushort(f), + ppemx = readbyte(f), + ppemy = readbyte(f), + bitdepth = readbyte(f), + flags = readbyte(f), + } + end + sort(sizetables,function(a,b) + if b.ppemx == a.ppemx then + return b.bitdepth < a.bitdepth + else + return b.ppemx < a.ppemx + end + end) + for i=1,nofsizetables do + local s = sizetables[i] + local d = false + for j=s.firstindex,s.lastindex do + if not shapes[j] then + shapes[j] = i + d = true + end + end + if d then + s.used = true + end + end + for i=1,nofsizetables do + local s = sizetables[i] + if s.used then + local offset = s.subtables + setposition(f,cblctableoffset+offset) + for j=1,s.nofsubtables do + local firstindex = readushort(f) + local lastindex = readushort(f) + local tableoffset = readulong(f) + offset + for k=firstindex,lastindex do + if shapes[k] == i then + local s = subtables[tableoffset] + if not s then + s = { + firstindex = firstindex, + lastindex = lastindex, + } + subtables[tableoffset] = s + end + shapes[k] = s + end + end + end + end + end + + -- there is no need to sort in string stream but we have a nicer trace + -- if needed + + for offset, subtable in sortedhash(subtables) do + local tabletype = readushort(f) + subtable.format = readushort(f) + local baseoffset = readulong(f) + ctdttableoffset + local offsets = { } + local metrics = nil + if tabletype == 1 then + -- we have the usual one more to get the size + for i=subtable.firstindex,subtable.lastindex do + offsets[i] = readulong(f) + baseoffset + end + skipbytes(f,4) + elseif tabletype == 2 then + local size = readulong(f) + local done = baseoffset + metrics = getbigmetrics(f) + for i=subtable.firstindex,subtable.lastindex do + offsets[i] = done + done = done + size + end + elseif tabletype == 3 then + -- we have the usual one more to get the size + local n = subtable.lastindex - subtable.firstindex + 2 + for i=subtable.firstindex,subtable.lastindex do + offsets[i] = readushort(f) + baseoffset + end + if math.odd(n) then + skipbytes(f,4) + else + skipbytes(f,2) + end + elseif tabletype == 4 then + for i=1,readulong(f) do + offsets[readushort(f)] = readushort(f) + baseoffset + end + elseif tabletype == 5 then + local size = readulong(f) + local done = baseoffset + metrics = getbigmetrics(f) + local n = readulong(f) + for i=1,n do + offsets[readushort(f)] = done + done = done + size + end + if math.odd(n) then + skipbytes(f,2) + end + else + return -- unsupported format + end + subtable.offsets = offsets + subtable.metrics = metrics + end + + -- we only support a few sensible types ... there are hardly any fonts so + -- why are there so many variants ... not the best spec + + local default = { width = 0, height = 0 } + local glyphs = fontdata.glyphs + -- local delayed = CONTEXTLMTXMODE and CONTEXTLMTXMODE > 0 or fonts.handlers.typethree + + for index, subtable in sortedhash(shapes) do + if type(subtable) == "table" then + local data = nil + local size = nil + local metrics = default + local format = subtable.format + local offset = subtable.offsets[index] + setposition(f,offset) + if format == 17 then + metrics = getsmallmetrics(f) + size = true + elseif format == 18 then + metrics = getbigmetrics(f) + size = true + elseif format == 19 then + metrics = subtable.metrics + size = true + else + -- forget about it + end + if size then + size = readulong(f) + -- if delayed then + offset = getposition(f) + data = nil + -- else + -- offset = nil + -- data = readstring(f,size) + -- size = nil + -- end + else + offset = nil + end + local x = metrics.width + local y = metrics.height + shapes[index] = { + x = x, + y = y, + o = offset, + s = size, + data = data, + } + -- I'll look into this in more details when needed + -- as we can use the bearings to get better boxes. + local glyph = glyphs[index] + if not glyph.boundingbox then + local width = glyph.width + local height = width * y/x + glyph.boundingbox = { 0, 0, width, height } + end + else + shapes[index] = { + x = 0, + y = 0, + data = "", -- or just nil + } + end + end + + fontdata.pngshapes = shapes -- we cheat + end + end + + function readers.cbdt(f,fontdata,specification) + -- local tableoffset = gotodatatable(f,fontdata,"ctdt",specification.glyphs) + -- if tableoffset then + -- local majorversion = readushort(f) + -- local minorversion = readushort(f) + -- end + end + + -- function readers.ebdt(f,fontdata,specification) + -- if specification.glyphs then + -- end + -- end + + -- function readers.ebsc(f,fontdata,specification) + -- if specification.glyphs then + -- end + -- end + + -- function readers.eblc(f,fontdata,specification) + -- if specification.glyphs then + -- end + -- end + +end + +-- + AVAR : optional +-- + CFF2 : otf outlines +-- - CVAR : ttf hinting, not needed +-- + FVAR : the variations +-- + GVAR : ttf outline changes +-- + HVAR : horizontal changes +-- + MVAR : metric changes +-- + STAT : relations within fonts +-- * VVAR : vertical changes +-- +-- * BASE : extra baseline adjustments +-- - GASP : not needed +-- + GDEF : not needed (carets) +-- + GPOS : adapted device tables (needed?) +-- + GSUB : new table +-- + NAME : 25 added + +function readers.stat(f,fontdata,specification) + local tableoffset = gotodatatable(f,fontdata,"stat",true) -- specification.variable + if tableoffset then + local extras = fontdata.extras + local version = readulong(f) -- 0x00010000 + local axissize = readushort(f) + local nofaxis = readushort(f) + local axisoffset = readulong(f) + local nofvalues = readushort(f) + local valuesoffset = readulong(f) + local fallbackname = extras[readushort(f)] -- beta fonts mess up + local axis = { } + local values = { } + setposition(f,tableoffset+axisoffset) + for i=1,nofaxis do + local tag = readtag(f) + axis[i] = { + tag = tag, + name = lower(extras[readushort(f)] or tag), + ordering = readushort(f), -- maybe gaps + variants = { } + } + end + -- flags: + -- + -- 0x0001 : OlderSiblingFontAttribute + -- 0x0002 : ElidableAxisValueName + -- 0xFFFC : reservedFlags + -- + setposition(f,tableoffset+valuesoffset) + for i=1,nofvalues do + values[i] = readushort(f) + end + for i=1,nofvalues do + setposition(f,tableoffset + valuesoffset + values[i]) + local format = readushort(f) + local index = readushort(f) + 1 + local flags = readushort(f) + local name = lower(extras[readushort(f)] or "no name") + local value = readfixed(f) + local variant + if format == 1 then + variant = { + flags = flags, + name = name, + value = value, + } + elseif format == 2 then + variant = { + flags = flags, + name = name, + value = value, + minimum = readfixed(f), + maximum = readfixed(f), + } + elseif format == 3 then + variant = { + flags = flags, + name = name, + value = value, + link = readfixed(f), + } + end + insert(axis[index].variants,variant) + end + sort(axis,function(a,b) + return a.ordering < b.ordering + end) + for i=1,#axis do + local a = axis[i] + sort(a.variants,function(a,b) + return a.name < b.name + end) + a.ordering = nil + end + setvariabledata(fontdata,"designaxis",axis) + setvariabledata(fontdata,"fallbackname",fallbackname) + end +end + +-- The avar table is optional and used in combination with fvar. Given the +-- detailed explanation about bad values we expect the worst and do some +-- checking. + +function readers.avar(f,fontdata,specification) + local tableoffset = gotodatatable(f,fontdata,"avar",true) -- specification.variable + if tableoffset then + + local function collect() + local nofvalues = readushort(f) + local values = { } + local lastfrom = false + local lastto = false + for i=1,nofvalues do + local from = read2dot14(f) + local to = read2dot14(f) + if lastfrom and from <= lastfrom then + -- ignore + elseif lastto and to >= lastto then + -- ignore + else + values[#values+1] = { from, to } + lastfrom, lastto = from, to + end + end + nofvalues = #values + if nofvalues > 2 then + local some = values[1] + if some[1] == -1 and some[2] == -1 then + some = values[nofvalues] + if some[1] == 1 and some[2] == 1 then + for i=2,nofvalues-1 do + some = values[i] + if some[1] == 0 and some[2] == 0 then + return values + end + end + end + end + end + return false + end + + local version = readulong(f) -- 0x00010000 + local reserved = readushort(f) + local nofaxis = readushort(f) + local segments = { } + for i=1,nofaxis do + segments[i] = collect() + end + setvariabledata(fontdata,"segments",segments) + end +end + +function readers.fvar(f,fontdata,specification) + local tableoffset = gotodatatable(f,fontdata,"fvar",true) -- specification.variable or specification.instancenames + if tableoffset then + local version = readulong(f) -- 0x00010000 + local offsettoaxis = tableoffset + readushort(f) + local reserved = skipshort(f) + -- pair 1 + local nofaxis = readushort(f) + local sizeofaxis = readushort(f) + -- pair 2 + local nofinstances = readushort(f) + local sizeofinstances = readushort(f) + -- + local extras = fontdata.extras + local axis = { } + local instances = { } + -- + setposition(f,offsettoaxis) + -- + for i=1,nofaxis do + axis[i] = { + tag = readtag(f), -- ital opsz slnt wdth wght + minimum = readfixed(f), + default = readfixed(f), + maximum = readfixed(f), + flags = readushort(f), + name = lower(extras[readushort(f)] or "bad name"), + } + local n = sizeofaxis - 20 + if n > 0 then + skipbytes(f,n) + elseif n < 0 then + -- error + end + end + -- + local nofbytes = 2 + 2 + 2 + nofaxis * 4 + local readpsname = nofbytes <= sizeofinstances + local skippable = sizeofinstances - nofbytes + for i=1,nofinstances do + local subfamid = readushort(f) + local flags = readushort(f) -- 0, not used yet + local values = { } + for i=1,nofaxis do + values[i] = { + axis = axis[i].tag, + value = readfixed(f), + } + end + local psnameid = readpsname and readushort(f) or 0xFFFF + if subfamid == 2 or subfamid == 17 then + -- okay + elseif subfamid == 0xFFFF then + subfamid = nil + elseif subfamid <= 256 or subfamid >= 32768 then + subfamid = nil -- actually an error + end + if psnameid == 6 then + -- okay + elseif psnameid == 0xFFFF then + psnameid = nil + elseif psnameid <= 256 or psnameid >= 32768 then + psnameid = nil -- actually an error + end + instances[i] = { + -- flags = flags, + subfamily = extras[subfamid], + psname = psnameid and extras[psnameid] or nil, + values = values, + } + if skippable > 0 then + skipbytes(f,skippable) + end + end + setvariabledata(fontdata,"axis",axis) + setvariabledata(fontdata,"instances",instances) + end +end + +function readers.hvar(f,fontdata,specification) + local factors = specification.factors + if not factors then + return + end + local tableoffset = gotodatatable(f,fontdata,"hvar",specification.variable) + if not tableoffset then + report("no hvar table, expect problems due to messy widths") + return + end + + local version = readulong(f) -- 0x00010000 + local variationoffset = tableoffset + readulong(f) -- the store + local advanceoffset = tableoffset + readulong(f) + local lsboffset = tableoffset + readulong(f) + local rsboffset = tableoffset + readulong(f) + + local regions = { } + local variations = { } + local innerindex = { } -- size is mapcount + local outerindex = { } -- size is mapcount + local deltas = { } + + if variationoffset > 0 then + regions, deltas = readvariationdata(f,variationoffset,factors) + end + if not regions then + -- for now .. what to do ? + return + end + + if advanceoffset > 0 then + -- + -- innerIndexBitCountMask = 0x000F + -- mapEntrySizeMask = 0x0030 + -- reservedFlags = 0xFFC0 + -- + -- outerIndex = entry >> ((entryFormat & innerIndexBitCountMask) + 1) + -- innerIndex = entry & ((1 << ((entryFormat & innerIndexBitCountMask) + 1)) - 1) + -- + setposition(f,advanceoffset) + local format = readushort(f) -- todo: check + local mapcount = readushort(f) + local entrysize = rshift(band(format,0x0030),4) + 1 + local nofinnerbits = band(format,0x000F) + 1 -- n of inner bits + local innermask = lshift(1,nofinnerbits) - 1 + local readcardinal = read_cardinal[entrysize] -- 1 upto 4 bytes + for i=0,mapcount-1 do + local mapdata = readcardinal(f) + outerindex[i] = rshift(mapdata,nofinnerbits) + innerindex[i] = band(mapdata,innermask) + end + -- use last entry when no match i + setvariabledata(fontdata,"hvarwidths",true) + local glyphs = fontdata.glyphs + for i=0,fontdata.nofglyphs-1 do + local glyph = glyphs[i] + local width = glyph.width + if width then + local outer = outerindex[i] or 0 + local inner = innerindex[i] or i + if outer and inner then -- not needed + local delta = deltas[outer+1] + if delta then + local d = delta.deltas[inner+1] + if d then + local scales = delta.scales + local deltaw = 0 + for i=1,#scales do + local di = d[i] + if di then + deltaw = deltaw + scales[i] * di + else + break -- can't happen + end + end +-- report("index: %i, outer: %i, inner: %i, deltas: %|t, scales: %|t, width: %i, delta %i", +-- i,outer,inner,d,scales,width,round(deltaw)) + glyph.width = width + round(deltaw) + end + end + end + end + end + + end + + -- if lsboffset > 0 then + -- -- we don't use left side bearings + -- end + + -- if rsboffset > 0 then + -- -- we don't use right side bearings + -- end + + -- setvariabledata(fontdata,"hregions",regions) + +end + +function readers.vvar(f,fontdata,specification) + if not specification.variable then + return + end +end + +function readers.mvar(f,fontdata,specification) + local tableoffset = gotodatatable(f,fontdata,"mvar",specification.variable) + if tableoffset then + local version = readulong(f) -- 0x00010000 + local reserved = skipshort(f,1) + local recordsize = readushort(f) + local nofrecords = readushort(f) + local offsettostore = tableoffset + readushort(f) + local dimensions = { } + local factors = specification.factors + if factors then + local regions, deltas = readvariationdata(f,offsettostore,factors) + for i=1,nofrecords do + local tag = readtag(f) + local var = variabletags[tag] + if var then + local outer = readushort(f) + local inner = readushort(f) + local delta = deltas[outer+1] + if delta then + local d = delta.deltas[inner+1] + if d then + local scales = delta.scales + local dd = 0 + for i=1,#scales do + dd = dd + scales[i] * d[i] + end + var(fontdata,round(dd)) + end + end + else + skipshort(f,2) + end + if recordsize > 8 then -- 4 + 2 + 2 + skipbytes(recordsize-8) + end + end + end + -- setvariabledata(fontdata,"mregions",regions) + end +end diff --git a/tex/context/base/mkxl/font-imp-text.lmt b/tex/context/base/mkxl/font-imp-text.lmt index 3ca05eb04..16edb2ef4 100644 --- a/tex/context/base/mkxl/font-imp-text.lmt +++ b/tex/context/base/mkxl/font-imp-text.lmt @@ -13,7 +13,7 @@ local registerotffeature = fonts.handlers.otf.features.register local function initialize(tfmdata,value) if type(value) == "string" then - tfmdata.properties.textcontrol = tex.stringtocodesbitmap(value,tex.textcontrolcodes) + tfmdata.properties.textcontrol = (tfmdata.properties.textcontrol or 0) | tex.stringtocodesbitmap(value,tex.textcontrolcodes) end end diff --git a/tex/context/base/mkxl/font-lib.mklx b/tex/context/base/mkxl/font-lib.mklx index bb0eec7e6..d3a18f8b5 100644 --- a/tex/context/base/mkxl/font-lib.mklx +++ b/tex/context/base/mkxl/font-lib.mklx @@ -43,7 +43,7 @@ \registerctxluafile{font-web}{} % opentype fontloader \registerctxluafile{font-cff}{optimize} % quadratic outlines \registerctxluafile{font-ttf}{optimize} % cubic outlines -\registerctxluafile{font-dsp}{optimize} % ... for this one +\registerctxluafile{font-dsp}{autosuffix,optimize} \registerctxluafile{font-hsh}{autosuffix} % hashes used by context \registerctxluafile{font-vir}{} \registerctxluafile{font-vfc}{autosuffix} diff --git a/tex/context/base/mkxl/lang-mis.mkxl b/tex/context/base/mkxl/lang-mis.mkxl index 48c2fa1cd..374379a0b 100644 --- a/tex/context/base/mkxl/lang-mis.mkxl +++ b/tex/context/base/mkxl/lang-mis.mkxl @@ -119,27 +119,10 @@ %D \installdiscretionary | - %D \stoptyping %D -%D Some alternative definitions are: -%D -%D \startbuffer -%D \installdiscretionary * - -%D \installdiscretionary + - -%D \installdiscretionary / - -%D \installdiscretionary ~ - -%D \stopbuffer -%D -%D \typebuffer -%D -%D after which we can say: -%D -%D \start \getbuffer -%D \starttest -%D \test {test**test**test} -%D \test {test++test++test} -%D \test {test//test//test} -%D \test {test~~test~~test} -%D \stoptest -%D \stop +%D We used to have an installable mechanism but in the perspective of \MKIV\ and +%D especialy \LMTX\ it no longer makes sense to complicate the code, so from now on +%D we only deal with the active bar. Older code can be seen in the archives. It also +%D means that we now just hardcode the bar. We also deal with math differently. %D \macros %D {compoundhyphen} @@ -163,7 +146,6 @@ \ifdefined\postwordbreak\else \permanent\protected\def\postwordbreak {\penalty\zerocount \hskip\zeropoint\relax} \fi \ifdefined\hspaceamount \else \def\hspaceamount#1#2{.16667\emwidth} \fi % will be overloaded -%frozen\protected\def\permithyphenation{\ifhmode\prewordbreak\fi} % doesn't remove spaces \permanent\protected\def\permithyphenation{\ifhmode\wordboundary\fi} % doesn't remove spaces %D \macros @@ -173,7 +155,7 @@ %D In the previous macros we provided two hooks which can be used to support nested %D sub||sentences. In \CONTEXT\ these hooks are used to insert a small space when %D needed. - +%D %D The following piece of code is a torture test compound handling. The \type %D {\relax} before the \type {\ifmmode} is needed because of the alignment scanner %D (in \ETEX\ this problem is not present because there a protected macro is not @@ -186,22 +168,10 @@ %D \stopformula \installcorenamespace{discretionaryaction} -\installcorenamespace{discretionarytext} -\installcorenamespace{discretionarymath} -\installcorenamespace{discretionaryboth} \installcorenamespace{discretionarymode} -\permanent\protected\def\installdiscretionary#1#2% - {\edefcsname\??discretionarymath\detokenize{#1}\endcsname{\detokenize{#1}}% ? - \defcsname \??discretionarytext\detokenize{#1}\endcsname{#2}% - \defcsname \??discretionaryboth\detokenize{#1}\endcsname{\lang_discretionaries_command#1}% - \scratchcounter\expandafter`\detokenize{#1}% - \expandafter\uedcatcodecommand\expandafter\ctxcatcodes\expandafter\scratchcounter\csname\??discretionaryboth\detokenize{#1}\endcsname} - -\permanent\protected\def\handlemathmodediscretionary#1{\ifcsname\??discretionarymath\detokenize{#1}\endcsname\lastnamedcs} -\permanent\protected\def\handletextmodediscretionary#1{\ifcsname\??discretionarytext\detokenize{#1}\endcsname\lastnamedcs} - -\permanent\protected\def\installdiscretionaries#1#2{\writestatus\m!system{use \string \installdiscretionary}} % obsolete +\aliased\let\installdiscretionaries\gobbletwoarguments % this alias will go +\aliased\let\installdiscretionary \gobbletwoarguments % this alias will go \setnewconstant\discretionarymode\plusone @@ -209,22 +179,21 @@ \permanent\protected\def\obeydiscretionaries {\discretionarymode\plusone} \def\lang_discretionaries_command - {% if direct if, we need \relax for lookahead in math mode - \csname\??discretionarymode + {\begincsname\??discretionarymode \ifcase\discretionarymode n% - \orelse\ifmmode - m% \else t% \fi \endcsname} -\defcsname\??discretionarymode n\endcsname#1% - {\detokenize{#1}} +% \catcode\barasciicode\activecatcode +% \amcode \barasciicode\othercatcode + +\letcatcodecommand\ctxcatcodes\barasciicode\lang_discretionaries_command -%D The macro \type{\lang_discretionaries_check_before} takes care of loners like -%D \type{||word}, while it counterpart \type {\lang_discretionaries_check_after} is +%D The macro \type {\lang_discretionaries_check_before} takes care of loners like +%D \type {||word}, while it counterpart \type {\lang_discretionaries_check_after} is %D responsible for handling the comma. \newconditional\punctafterdiscretionary @@ -257,73 +226,58 @@ \ifx :\nexttoken \settrue \punctafterdiscretionary \orelse \ifx ;\nexttoken \settrue \punctafterdiscretionary \fi} -\letcsname\??discretionarymode m\endcsname\handlemathmodediscretionary +\edefcsname\??discretionarymode n\endcsname + {\detokenize{|}} -\defcsname\??discretionarymode t\endcsname#1% +\defcsname\??discretionarymode t\endcsname#1|% {\bgroup - \let\nextnextnext\egroup - \def\next##1#1% - {\def\next{\activedododotextmodediscretionary#1{##1}}% - \futurelet\nexttoken\next}% - \next} - -\let\lang_discretionaries_token \relax -\let\lang_discretionaries_action\relax + \def\next{\lang_discretionaries_handle{#1}}% + \futurelet\nexttoken\next}% -\permanent\protected\def\activedododotextmodediscretionary#1#2% - {\edef\lang_discretionaries_token{\detokenize{#2}}% - \def\lang_discretionaries_action{\handletextmodediscretionary{#1}}% +\permanent\protected\def\lang_discretionaries_handle#1% + {\edef\lang_discretionaries_token{\detokenize{#1}}% \lang_discretionaries_check_after \ifempty\lang_discretionaries_token - \ifx#1\nexttoken % takes care of ||| and +++ and ...... - \ifcsname\??discretionaryaction\string#1\endcsname + \ifx|\nexttoken % takes care of ||| + \ifcsname\??discretionaryaction\string|\endcsname \lastnamedcs \orelse\ifconditional\spaceafterdiscretionary - %\prewordbreak\hbox{\string#1}\relax - \wordboundary\hbox{\string#1}\relax + \wordboundary\hbox{\letterbar}\relax \orelse\ifconditional\punctafterdiscretionary - %\prewordbreak\hbox{\string#1}\relax - \wordboundary\hbox{\string#1}\wordboundary + \wordboundary\hbox{\letterbar}\wordboundary \else - %\prewordbreak\hbox{\string#1}\prewordbreak - \wordboundary\hbox{\string#1}\wordboundary + \wordboundary\hbox{\letterbar}\wordboundary \fi - \def\nextnextnext{\afterassignment\egroup\let\next=}% + \def\next{\afterassignment\egroup\let\next=}% \else \lang_discretionaries_check_before - % the next line has been changed (20050203) - % \prewordbreak\hbox{\lang_discretionaries_action\nexttoken}\postwordbreak - % but an hbox blocks a possible \discretionary \ifcsname\??discretionaryaction\endcsname \lastnamedcs \orelse\ifconditional\spaceafterdiscretionary - %\prewordbreak\lang_discretionaries_action\relax - \wordboundary\lang_discretionaries_action\relax + \wordboundary\defaultdiscretionaryhyphen\relax \orelse\ifconditional\punctafterdiscretionary - %\prewordbreak\lang_discretionaries_action\relax - \wordboundary\lang_discretionaries_action\relax + \wordboundary\defaultdiscretionaryhyphen\relax \else - %\prewordbreak\lang_discretionaries_action\prewordbreak - \wordboundary\lang_discretionaries_action\wordboundary + \wordboundary\defaultdiscretionaryhyphen\wordboundary \fi + \let\next\egroup \fi \orelse\ifcsname\??discretionaryaction\lang_discretionaries_token\endcsname \lastnamedcs + \let\next\egroup \else \lang_discretionaries_check_before \ifconditional\spaceafterdiscretionary - %\prewordbreak\hbox{#2}\relax - \wordboundary\hbox{#2}\relax + \wordboundary\hbox{#1}\relax \orelse\ifconditional\punctafterdiscretionary - %\prewordbreak\hbox{#2}\relax - \wordboundary\hbox{#2}\relax + \wordboundary\hbox{#1}\relax \else - %\prewordbreak\discretionary{\hbox{#2}}{}{\hbox{#2}}\postwordbreak - \wordboundary\discretionary{\hbox{#2}}{}{\hbox{#2}}\wordboundary - %\discretionary options \plusthree{\hbox{#2}}{}{\hbox{#2}}% + \wordboundary\discretionary{\hbox{#1}}{}{\hbox{#1}}\wordboundary + %\discretionary options \plusthree{\hbox{#1}}{}{\hbox{#1}}% \fi + \let\next\egroup \fi - \nextnextnext} + \next} %D \macros %D {directdiscretionary} @@ -332,7 +286,7 @@ %D use the more direct approach: \permanent\protected\def\directdiscretionary - {\csname\??discretionarymode + {\begincsname\??discretionarymode \ifcase\discretionarymode n% \else @@ -341,7 +295,7 @@ \endcsname} \permanent\protected\def\indirectdiscretionary - {\csname\??discretionarymode + {\begincsname\??discretionarymode \ifcase\discretionarymode n% \else @@ -350,16 +304,13 @@ \endcsname} \protected\defcsname\??discretionarymode d\endcsname#1% - {\edef\lang_discretionaries_token{\detokenize{#1}}% - \let\lang_discretionaries_action\compoundhyphen - \ifcsname\??discretionaryaction\lang_discretionaries_token\endcsname + {\ifcsname\??discretionaryaction\detokenize{#1}\endcsname \expandafter\lastnamedcs \else \expandafter\indirectdiscretionary \fi{#1}} \protected\defcsname\??discretionarymode i\endcsname#1% - %{\prewordbreak\discretionary{\hbox{#1}}{}{\hbox{#1}}\postwordbreak} {\wordboundary\discretionary{\hbox{#1}}{}{\hbox{#1}}\wordboundary} %{\discretionary options \plusthree{\hbox{#1}}{}{\hbox{#1}}} @@ -385,13 +336,10 @@ \def\lang_discretionaries_hyphen_like#1#2% {\ifconditional\spaceafterdiscretionary - %prewordbreak\hbox{#1}\relax \wordboundary\hbox{#1}\relax \orelse\ifconditional\punctafterdiscretionary - %prewordbreak\hbox{#1}\relax \wordboundary\hbox{#1}\relax \else - %\prewordbreak#2\postwordbreak % was prewordbreak \wordboundary#2\wordboundary \fi} @@ -409,78 +357,55 @@ \definetextmodediscretionary ( {\ifdim\lastskip>\zeropoint - %(\prewordbreak (\wordboundary \else - %\prewordbreak\discretionary{}{(-}{(}\prewordbreak \wordboundary\discretionary{}{(-}{(}\wordboundary %\discretionary options \plusthree{}{(-}{(}% \fi} \definetextmodediscretionary ~ - %{\prewordbreak\discretionary{-}{}{\thinspace}\postwordbreak} {\wordboundary\discretionary{-}{}{\thinspace}\wordboundary} %{\discretionary options \plusthree{-}{}{\thinspace}} \definetextmodediscretionary ' - %{\prewordbreak\discretionary{-}{}{'}\postwordbreak} {\wordboundary\discretionary{-}{}{'}\wordboundary} %{\discretionary options \plusthree{-}{}{'}} \definetextmodediscretionary ^ - %{\prewordbreak\discretionary{\hbox{\normalstartimath|\normalstopimath}}{}{\hbox{\normalstartimath|\normalstopimath}}% - % \postwordbreak} % bugged {\wordboundary \discretionary{\hbox{\normalstartimath|\normalstopimath}}{}{\hbox{\normalstartimath|\normalstopimath}}% \wordboundary} % bugged %{\discretionary options \plusthree{\hbox{\normalstartimath|\normalstopimath}}{}{\hbox{\normalstartimath|\normalstopimath}}} \definetextmodediscretionary < - %{\beginofsubsentence\prewordbreak\beginofsubsentencespacing {\beginofsubsentence\wordboundary\beginofsubsentencespacing \aftergroup\ignorespaces} % tricky, we need to go over the \nextnextnext \definetextmodediscretionary > {\removeunwantedspaces - %\endofsubsentencespacing\prewordbreak\endofsubsentence} \endofsubsentencespacing\wordboundary\endofsubsentence} \definetextmodediscretionary = {\removeunwantedspaces - %\prewordbreak\midsentence\prewordbreak \wordboundary\midsentence\wordboundary \aftergroup\ignorespaces} % french -%definetextmodediscretionary : {\removeunwantedspaces\prewordbreak\kern\hspaceamount\empty{:}:} -%definetextmodediscretionary ; {\removeunwantedspaces\prewordbreak\kern\hspaceamount\empty{;};} -%definetextmodediscretionary ? {\removeunwantedspaces\prewordbreak\kern\hspaceamount\empty{?}?} -%definetextmodediscretionary ! {\removeunwantedspaces\prewordbreak\kern\hspaceamount\empty{!}!} - \definetextmodediscretionary : {\removeunwantedspaces\wordboundary\kern\hspaceamount\empty{:}:} \definetextmodediscretionary ; {\removeunwantedspaces\wordboundary\kern\hspaceamount\empty{;};} \definetextmodediscretionary ? {\removeunwantedspaces\wordboundary\kern\hspaceamount\empty{?}?} \definetextmodediscretionary ! {\removeunwantedspaces\wordboundary\kern\hspaceamount\empty{!}!} -%definetextmodediscretionary * {\prewordbreak\discretionary{-}{}{\kern.05\emwidth}\prewordbreak} \definetextmodediscretionary * {\wordboundary\discretionary{-}{}{\kern.05\emwidth}\wordboundary} % spanish -%definetextmodediscretionary ?? {\prewordbreak\questiondown} -%definetextmodediscretionary !! {\prewordbreak\exclamdown} - \definetextmodediscretionary ?? {\wordboundary\questiondown} \definetextmodediscretionary !! {\wordboundary\exclamdown} -%D \installdiscretionary | + -%D \installdiscretionary + = - \permanent\protected\def\defaultdiscretionaryhyphen{\compoundhyphen} -\installdiscretionary | \defaultdiscretionaryhyphen % installs in ctx and prt will fall back on it - %D \macros %D {fakecompoundhyphen} %D @@ -547,9 +472,9 @@ %D so test\compound{}test can be used instead of test||test %D \stoptyping -\bgroup - - \catcode\barasciicode\activecatcode +% \bgroup +% +% \catcode\barasciicode\activecatcode \permanent\protected\gdef\compound#1{|#1|} @@ -557,7 +482,7 @@ \enforced\permanent\protected\def|#1|{\ifx#1\empty\empty-\else#1\fi}% \to \everysimplifycommands -\egroup +% \egroup %D Here we hook some code into the clean up mechanism needed for verbatim data. @@ -581,4 +506,6 @@ {\ifnum\posthyphenchar>\zerocount\char\posthyphenchar\fi}% {\ifnum\posthyphenchar>\zerocount\char\posthyphenchar\fi}} +% \setcatcodetable\prtcatcodes % because we activated the bar + \protect \endinput diff --git a/tex/context/base/mkxl/lang-tra.mkxl b/tex/context/base/mkxl/lang-tra.mkxl index a44ee4ba9..66a020a14 100644 --- a/tex/context/base/mkxl/lang-tra.mkxl +++ b/tex/context/base/mkxl/lang-tra.mkxl @@ -27,8 +27,10 @@ \appendtoks \disablediscretionaries - \let~=\lettertilde + % \let~=\lettertilde + \enforced\letcharcode\tildeasciicode\lettertilde % \let|=\letterbar + % \enforced\letcharcode\barasciicode\letterbar \to \everytransliterations \def\lang_transliteration_common diff --git a/tex/context/base/mkxl/lang-url.mkxl b/tex/context/base/mkxl/lang-url.mkxl index 6e38c95a1..87e64feaf 100644 --- a/tex/context/base/mkxl/lang-url.mkxl +++ b/tex/context/base/mkxl/lang-url.mkxl @@ -57,8 +57,10 @@ \enforced\let\{\letterleftbrace \enforced\let\}\letterrightbrace \enforced\let\|\letterbar - \enforced\let~=\lettertilde - \enforced\let|=\letterbar + % \enforced\let~=\lettertilde + % \enforced\let|=\letterbar + \enforced\letcharcode\tildeasciicode\lettertilde + \enforced\letcharcode\barasciicode \letterbar \enforced\let\Ux\eUx \to \everyhyphenatedurl diff --git a/tex/context/base/mkxl/math-act.lmt b/tex/context/base/mkxl/math-act.lmt index 2df708806..d4abed0ba 100644 --- a/tex/context/base/mkxl/math-act.lmt +++ b/tex/context/base/mkxl/math-act.lmt @@ -84,13 +84,13 @@ function mathematics.initializeparameters(target,original) mathparameters = mathematics.dimensions(mathparameters) -- -- if not mathparameters.MinConnectorOverlap then mathparameters.MinConnectorOverlap = undefined end - -- if not mathparameters.SubscriptShiftDownWithSuperscript then mathparameters.SubscriptShiftDownWithSuperscript = undefined end -- a tex one + if not mathparameters.SubscriptShiftDownWithSuperscript then mathparameters.SubscriptShiftDownWithSuperscript = mathparameters.SubscriptShiftDown * 1.5 end -- if not mathparameters.FractionDelimiterSize then mathparameters.FractionDelimiterSize = undefined end -- if not mathparameters.FractionDelimiterDisplayStyleSize then mathparameters.FractionDelimiterDisplayStyleSize = undefined end -- if not mathparameters.SkewedDelimiterTolerance then mathparameters.SkewedDelimiterTolerance = undefined end -- some more can be undefined: - if not mathparameters.PrimeRaisePercent then mathparameters.PrimeRaisePercent = 50 end - if not mathparameters.PrimeRaiseComposedPercent then mathparameters.PrimeRaiseComposedPercent = 25 end + if not mathparameters.PrimeRaisePercent then mathparameters.PrimeRaisePercent = 0 end + if not mathparameters.PrimeRaiseComposedPercent then mathparameters.PrimeRaiseComposedPercent = 0 end if not mathparameters.PrimeShiftUp then mathparameters.PrimeShiftUp = mathparameters.SuperscriptShiftUp end if not mathparameters.PrimeBaselineDropMax then mathparameters.PrimeBaselineDropMax = mathparameters.SuperscriptBaselineDropMax end if not mathparameters.PrimeShiftUpCramped then mathparameters.PrimeShiftUpCramped = mathparameters.SuperscriptShiftUpCramped end @@ -216,15 +216,15 @@ function mathematics.overloadparameters(target,original) local tvalue = type(value) if tvalue == "string" then local newvalue = mathparameters[value] - -- if not newvalue then - -- local code = loadstring("return " .. value,"","t",mathparameters) - -- if type(code) == "function" then - -- local okay, v = pcall(code) - -- if okay then - -- newvalue = v - -- end - -- end - -- end + if not newvalue then + local code = loadstring("return " .. value,"","t",mathparameters) + if type(code) == "function" then + local okay, v = pcall(code) + if okay then + newvalue = v + end + end + end if newvalue then -- split in number and string mathparameters[name] = newvalue @@ -421,6 +421,84 @@ local detail do end +-- This temporary tweak was used when we (MS & HH) were fixing the Latin Modern +-- parameters that relate to script placement. We started from the original cmr +-- ratios combined with the formal specification and ended up with the following +-- values. In the end we rejected this tweak and setteled for checking and fixing: +-- +-- SubscriptShiftDown +-- SubscriptShiftDownWithSuperscript +-- SuperscriptShiftUp +-- SuperscriptShiftUpCramped +-- +-- because it looks a bit arbitrary what values are set. We keep the code below +-- as documentation. + +-- do +-- -- modern +-- -- +-- -- local factors = { +-- -- scripts = { +-- -- SubscriptBaselineDropMin = 0.116, +-- -- SubscriptShiftDown = 0.348, +-- -- SubscriptShiftDownWithSuperscript = 0.573, +-- -- SubscriptTopMax = 0.800, +-- -- SuperscriptBaselineDropMax = 0.896, +-- -- SuperscriptBottomMaxWithSubscript = 0.800, +-- -- SuperscriptBottomMin = 0.250, +-- -- SuperscriptShiftUp = 0.958, +-- -- SuperscriptShiftUpCramped = 0.958, +-- -- } +-- -- } +-- +-- -- -- cambria +-- -- +-- -- local factors = { +-- -- scripts = { +-- -- SubscriptBaselineDropMin = 0.279, +-- -- SubscriptShiftDown = 0.364, +-- -- SubscriptShiftDownWithSuperscript = 0.547, +-- -- SubscriptTopMax = 0.662, +-- -- SuperscriptBaselineDropMax = 0.401, +-- -- SuperscriptBottomMaxWithSubscript = 0.667, +-- -- SuperscriptBottomMin = 0.208, +-- -- SuperscriptShiftUp = 0.654, +-- -- SuperscriptShiftUpCramped = 0.654, +-- -- } +-- -- } +-- +-- -- after some tests and inspection +-- -- +-- +-- local factors = { +-- scripts = { +-- SubscriptBaselineDropMin = 0.100, -- harmless but small (seldom triggered) +-- SubscriptShiftDown = 0.400, -- by inspection in several files +-- SubscriptShiftDownWithSuperscript = 0.400, -- as above +-- SubscriptTopMax = 0.800, -- Microsoft recommendation +-- SuperscriptBaselineDropMax = 0.100, -- see SubscriptBaselineDropMin +-- SuperscriptBottomMaxWithSubscript = 0.800, -- Microsoft recommendation +-- SuperscriptBottomMin = 0.250, -- Microsoft recommendation +-- SuperscriptShiftUp = 0.650, -- by inspection, but also a bit gamble +-- SuperscriptShiftUpCramped = 0.650, -- see above, non-TeX +-- } +-- } +-- +-- datasets.fixparameters = factors +-- +-- function mathtweaks.fixparameters(target,original,parameters) +-- local mathparameters = target.mathparameters +-- if mathparameters and next(mathparameters) then +-- local xheight = target.parameters.xheight +-- -- todo : options +-- for k, v in next, factors.scripts do +-- mathparameters[k] = v * xheight +-- end +-- end +-- end +-- +-- end + do local stepper = utilities.parsers.stepper @@ -819,150 +897,110 @@ end do - -- see changed hack in math-fbk - local nps = fonts.helpers.newprivateslot local list = { - { 0x2032, { nps("prime 0x2032 1"), nps("prime 0x2032 2"), nps("prime 0x2032 3") }, 1 }, - { 0x2033, { nps("prime 0x2033 1"), nps("prime 0x2033 2"), nps("prime 0x2033 3") }, 2 }, - { 0x2034, { nps("prime 0x2034 1"), nps("prime 0x2034 2"), nps("prime 0x2034 3") }, 3 }, - { 0x2057, { nps("prime 0x2057 1"), nps("prime 0x2057 2"), nps("prime 0x2057 3") }, 4 }, - { 0x2035, { nps("prime 0x2035 1"), nps("prime 0x2035 2"), nps("prime 0x2035 3") }, 1 }, - { 0x2036, { nps("prime 0x2036 1"), nps("prime 0x2036 2"), nps("prime 0x2036 3") }, 2 }, - { 0x2037, { nps("prime 0x2037 1"), nps("prime 0x2037 2"), nps("prime 0x2037 3") }, 3 }, + { 0x2032, nps("delimited ghost 0x2032"), 1 }, + { 0x2033, nps("delimited ghost 0x2033"), 2, 0x2032 }, + { 0x2034, nps("delimited ghost 0x2034"), 3, 0x2032 }, + { 0x2057, nps("delimited ghost 0x2057"), 4, 0x2032 }, + { 0x2035, nps("delimited ghost 0x2035"), 1 }, + { 0x2036, nps("delimited ghost 0x2036"), 2, 0x2035 }, + { 0x2037, nps("delimited ghost 0x2037"), 3, 0x2035 }, } - local function copytable(t) - return { - width = t.width, - height = t.height, - depth = t.depth, - index = t.index, - unicode = t.unicode, - } - end - - local lastprivate - - local function use(target,original,targetcharacters,charcode,private,newheight,scale,fake,keep,count,smaller) - if count == 1 then - lastprivate = private - end - if keep then - if trace_tweaking then - report_tweak("keeping prime %U",target,original,charcode) - end - return - end - if fake and count > 1 then - local olddata = targetcharacters[lastprivate] - if olddata then - -- todo: when keep - local oldheight = scale * olddata.height - local oldwidth = scale * olddata.width - local yoffset = (newheight - oldheight) / scale - local xoffset = fake * oldwidth - local newwidth = oldwidth + (count - 1) * xoffset - targetcharacters[charcode] = { - yoffset = yoffset, - width = newwidth, - height = newheight, - smaller = smaller, - unicode = olddata.unicode, - commands = { - { "offset", 0, 0, lastprivate, scale, scale }, - { "offset", xoffset, 0, lastprivate, scale, scale }, - count > 2 and { "offset", 2 * xoffset, 0, lastprivate, scale, scale } or nil, - count > 3 and { "offset", 3 * xoffset, 0, lastprivate, scale, scale } or nil, - }, - } - if trace_tweaking then - report_tweak("faking %U with %i primes",target,original,charcode,count) - end - return - end - end - local olddata = targetcharacters[private] - if olddata then - local oldheight = scale * olddata.height - local oldwidth = scale * olddata.width - local yoffset = (newheight - oldheight) / scale - targetcharacters[charcode] = { - yoffset = yoffset, - width = oldwidth, - height = newheight, - smaller = smaller, - unicode = olddata.unicode, - commands = { - { "offset", 0, 0, private, scale, scale } - }, - } - if trace_tweaking then - report_tweak("fixing prime %U",target,original,charcode) - end - return - end - if trace_tweaking then - report_tweak("unable to fix prime %U",target,original,charcode) - end - end + datasets.fixprimes = list function mathtweaks.fixprimes(target,original,parameters) - local targetcharacters = target.characters - local targetparameters = target.parameters - local originalcharacters = original.characters - local factor = parameters.factor or 0.85 - local scale = parameters.scale or 1 - local smaller = parameters.smaller - local fake = parameters.fake - local keep = parameters.keep and targetparameters.mathsize == 1 - local newheight = factor * target.mathparameters.AccentBaseHeight - local compact = targetparameters.textscale and true or false - -- - lastprivate = false - -- make virtual copies (just all of them, also for tracing) + local targetcharacters = target.characters + local factor = parameters.factor or 1 + local fake = tonumber(parameters.fake) for i=1,#list do - local entry = list[i] - local count = entry[3] - if not fake or count == 1 then - local c1 = entry[1] - local d1 = targetcharacters[c1] - if d1 then - local pc = entry[2] - local c2 = d1.smaller or c1 - local d2 = targetcharacters[c2] - local c3 = d2.smaller or c2 - local d3 = targetcharacters[c3] - if smaller then - d1 = d2 - d2 = d3 + local entry = list[i] + local unicode = entry[1] + local count = entry[3] + local used = fonts.handlers.otf.getsubstitution(target,unicode,"ssty",true,"math","dflt") or unicode + local data = targetcharacters[used] + if data then + targetcharacters[unicode] = data + local oldheight = data.height or 0 + local newheight = factor * oldheight + data.yoffset = newheight - (oldheight or 0) + data.height = newheight + data.smaller = nil + elseif not fake then + report_tweak("missing %i prime %U",target,original,count,unicode) + end + end + if fake then + for i=1,#list do + local entry = list[i] + local count = entry[3] + if count > 1 then + local unicode = entry[1] + local original = entry[4] + local data = targetcharacters[original] + if data then + local oldwidth = data.width + local xoffset = fake * oldwidth + local newwidth = oldwidth + (count - 1) * xoffset + targetcharacters[unicode] = { + width = newwidth, + height = data.height, + unicode = unicode, + commands = { + { "offset", 0, 0, original }, + { "offset", xoffset, 0, original }, + count > 2 and { "offset", 2 * xoffset, 0, original } or nil, + count > 3 and { "offset", 3 * xoffset, 0, original } or nil, + }, + } end - targetcharacters[pc[1]] = copytable(d1) - targetcharacters[pc[2]] = copytable(d2) - targetcharacters[pc[3]] = copytable(d3) end end end - -- replace + end + + function mathtweaks.addprimed(target,original,parameters) + local characters = target.characters for i=1,#list do - local entry = list[i] - local count = entry[3] - local c1 = entry[1] - local pc = entry[2] - local s1 = pc[1] - local d1 = targetcharacters[c1] - if compact and d1 then - local c2 = d1.smaller or c1 - local d2 = targetcharacters[c2] - local c3 = d2.smaller or c2 - local s2 = pc[2] - local s3 = pc[3] - use(target,original,targetcharacters,c1,s1,newheight,scale,fake,keep, count,c2) - use(target,original,targetcharacters,c2,s2,newheight,scale,fake,false,count,c3) - use(target,original,targetcharacters,c3,s3,newheight,scale,fake,false,count) - else - use(target,original,targetcharacters,c1,s1,newheight,scale,fake,keep,count) + local entry = list[i] + local basecode = entry[1] + local movecode = entry[2] + local basedata = characters[basecode] + if basedata then + local baseheight = basedata.height or 0 + local basewidth = basedata.width or 0 + local used = baseheight + local total = baseheight + characters[movecode] = { -- todo:share + width = basewidth, + height = used, + unicode = basecode, -- 0xFFFD or space or so + -- callback = "devirtualize", + commands = { + downcommand[used], + { "rule", used, 0 }, + }, + } + basedata.parts = { + { + advance = used, + ["end"] = used, + extender = 1, + glyph = movecode, + start = used, + }, + { + advance = total, + ["end"] = 0, + glyph = basecode, + start = total, + }, + } + if trace_tweaking then + report_tweak("primed %U added",target,original,basecode) + end end end end @@ -1384,7 +1422,7 @@ do local half = (italic/2) * factor c.topanchor = width + half c.bottomanchor = width - half - c.bottomright = - italic + c.bottomright = - italic * (parameters.icfactor or 1) if trace_tweaking then -- todo end @@ -2698,9 +2736,9 @@ do local nps = fonts.helpers.newprivateslot local list = { - { 0x302, nps("delimited right hat" ), nps("delimited ghost hat" ) }, - { 0x303, nps("delimited right tilde"), nps("delimited ghost tilde") }, - { 0x30C, nps("delimited right check"), nps("delimited ghost check") }, + { 0x0302, nps("delimited right hat"), nps("delimited ghost hat") }, + { 0x0303, nps("delimited right tilde"), nps("delimited ghost tilde") }, + { 0x030C, nps("delimited right check"), nps("delimited ghost check") }, } function mathtweaks.addfourier(target,original,parameters) diff --git a/tex/context/base/mkxl/math-ali.mkxl b/tex/context/base/mkxl/math-ali.mkxl index 8a29f4f9d..a2b09a084 100644 --- a/tex/context/base/mkxl/math-ali.mkxl +++ b/tex/context/base/mkxl/math-ali.mkxl @@ -2728,7 +2728,8 @@ \appendtoks \edef\p_threshold{\mathematicsparameter\c!threshold}% \maththreshold\ifcsname\??maththreshold\p_threshold\endcsname\lastnamedcs\else\p_threshold\fi\relax -\to \everymath % \everymathematics +% \to \everymath % \everymathematics +\to \everymathematics %D Here is simple alignment mechanism: diff --git a/tex/context/base/mkxl/math-def.mkxl b/tex/context/base/mkxl/math-def.mkxl index 16c65cf55..be33bac86 100644 --- a/tex/context/base/mkxl/math-def.mkxl +++ b/tex/context/base/mkxl/math-def.mkxl @@ -24,7 +24,7 @@ \popoverloadmode -\activatemathcharacters +% \activatemathcharacters %D The \mfunction macro is an alternative for \hbox with a controlable font switch. diff --git a/tex/context/base/mkxl/math-del.mklx b/tex/context/base/mkxl/math-del.mklx index cfb8dc5d2..0377f3507 100644 --- a/tex/context/base/mkxl/math-del.mklx +++ b/tex/context/base/mkxl/math-del.mklx @@ -36,6 +36,7 @@ \c!depth=\zeropoint, \c!strut=\v!height, \c!source=\zerocount, + \c!size=\zerocount, \c!left=\zerocount, \c!right=\zerocount, \c!leftmargin=\zeropoint, @@ -67,6 +68,7 @@ \s!depth \dimexpr\mathdelimitedparameter\c!depth+\mathdelimitedparameter\c!bottomoffset\relax \s!source\numexpr\namedboxanchor{\mathdelimitedparameter\c!source}\relax \s!style \normalmathstyle + \s!size \numexpr\namedboxanchor{\mathdelimitedparameter\c!size}\relax \zerocount \mathdelimitedparameter\c!left \zerocount \mathdelimitedparameter\c!right \relax} @@ -114,4 +116,12 @@ % $ \autofences \fourier{(z+\frac12)} + \courier{(z+\frac12)} + \xourier{(z+\frac12)} $ +\definemathdelimited [primed] [\c!right="2032,\c!size=1,\c!topoffset=.2\exheight] +\definemathdelimited [doubleprimed] [primed] [\c!right="2033] +\definemathdelimited [tripleprimed] [primed] [\c!right="2034] +\definemathdelimited [quadrupleprimed] [primed] [\c!right="2057] +\definemathdelimited [reverseprimed] [primed] [\c!right="2035] +\definemathdelimited [doublereverseprimed] [primed] [\c!right="2036] +\definemathdelimited [triplereverseprimed] [primed] [\c!right="2037] + \protect \endinput diff --git a/tex/context/base/mkxl/math-dim.lmt b/tex/context/base/mkxl/math-dim.lmt index 15eeb47b3..8f35b4bbd 100644 --- a/tex/context/base/mkxl/math-dim.lmt +++ b/tex/context/base/mkxl/math-dim.lmt @@ -137,7 +137,6 @@ end function mathematics.dimensions(dimens) -- beware, dimens get spoiled if dimens.SpaceAfterScript then - dimens.SubscriptShiftDownWithSuperscript = dimens.SubscriptShiftDown * 1.5 -- move this one return table.fastcopy(dimens), { } elseif dimens.AxisHeight or dimens.axisheight then local t = { } diff --git a/tex/context/base/mkxl/math-fen.mkxl b/tex/context/base/mkxl/math-fen.mkxl index d8cbf4e76..1efc6bbe6 100644 --- a/tex/context/base/mkxl/math-fen.mkxl +++ b/tex/context/base/mkxl/math-fen.mkxl @@ -569,10 +569,10 @@ \fi \math_fenced_force_size\bigmathdelimitervariant\c_math_fenced_level\relax} -\appendtoks - \enforced\aliased\let|\letterbar - %\enforced\aliased\let\bar\letterbar -\to \everymathematics +% \appendtoks +% \enforced\aliased\let|\letterbar +% %\enforced\aliased\let\bar\letterbar +% \to \everymathematics \definemathfence [parenthesis] [\c!left="0028,\c!right="0029] \definemathfence [bracket] [\c!left="005B,\c!right="005D] @@ -843,9 +843,9 @@ % % \installmathfencepair ⦗ \Linterval ⦘ \Rinterval -\appendtoks - \ignorediscretionaries % so $\mtext{a|b}$ works, this is ok because it's an \hbox -\to \everymathematics +% \appendtoks +% \ignorediscretionaries % so $\mtext{a|b}$ works, this is ok because it's an \hbox +% \to \everymathematics % We unofficial support some synonyms as we need them for some fuzzy web related math. @@ -1215,5 +1215,7 @@ \definemathfence [tupanddownarrows] [\v!text] [\c!left="2191,\c!right="2193] \definemathfence [tupdownarrows] [\v!text] [\c!left="21C5,\c!right=0] \definemathfence [tdownuparrows] [\v!text] [\c!middle="21F5] +\definemathfence [tuparrow] [\v!text] [\c!middle="2191] +\definemathfence [tdownarrow] [\v!text] [\c!middle="2193] \protect diff --git a/tex/context/base/mkxl/math-ini.mkxl b/tex/context/base/mkxl/math-ini.mkxl index 259fd360e..5fcd232a2 100644 --- a/tex/context/base/mkxl/math-ini.mkxl +++ b/tex/context/base/mkxl/math-ini.mkxl @@ -2100,23 +2100,33 @@ %D The result is as expected: the first line typesets ok, while the second one %D triggers an error message. -\setnewconstant\activemathcharcode "8000 - -\newtoks\activatedmathcharacters - -\permanent\protected\def\activatemathcharacter#1% - {\appendtoks - \global\mathcode#1=\activemathcharcode - \to \activatedmathcharacters} - -\permanent\def\activatemathcharacters - {\the\activatedmathcharacters} +% \setnewconstant\activemathcharcode "8000 +% +% \newtoks\activatedmathcharacters +% +% \permanent\protected\def\activatemathcharacter#1% +% {\appendtoks +% \global\mathcode#1=\activemathcharcode +% \to \activatedmathcharacters} +% +% \permanent\def\activatemathcharacters +% {\the\activatedmathcharacters} +% +% % beware, not runtime, so has to happen at format generation +% +% \activatemathcharacter\circumflexasciicode +% \activatemathcharacter\underscoreasciicode +% %activatemathcharacter\ampersandasciicode -% beware, not runtime, so has to happen at format generation +% % already done in catc-def.mkxl: +% +% \amcode \circumflexasciicode \superscriptcatcode +% \amcode \underscoreasciicode \subscriptcatcode +% \amcode \barasciicode \othercatcode +% \amcode \tildeasciicode \othercatcode -\activatemathcharacter\circumflexasciicode -\activatemathcharacter\underscoreasciicode -%activatemathcharacter\ampersandasciicode +% \Umathcode\circumflexasciicode="0 "0 \circumflexasciicode +% \Umathcode\underscoreasciicode="0 "0 \underscoreasciicode % \permanent\def\normalmathaligntab{&} % @@ -2200,27 +2210,24 @@ %D We also dropped the option to let ampersands be alignment tabs. That has never %D been a \CONTEXT\ feature|/|habit anyway. -\bgroup - - \catcode\underscoreasciicode\activecatcode - \catcode\circumflexasciicode\activecatcode - % \catcode\ampersandasciicode \activecatcode - - \aliased\glet\specialmathaligntab\normalmathaligntab - - \permanent\protected\gdef\obeymathcatcodes{% - \enforced\let _\normalsubscript - \enforced\let ^\normalsuperscript - % \enforced\def &\specialmathaligntab - } - - \doglobal\appendtoks - \enforced\let _\normalsubscript - \enforced\let ^\normalsuperscript - % \enforced\let &\specialmathaligntab - \to \everymathematics - -\egroup +% \bgroup +% +% \catcode\underscoreasciicode\activecatcode +% \catcode\circumflexasciicode\activecatcode +% +% \aliased\glet\specialmathaligntab\normalmathaligntab +% +% \permanent\protected\gdef\obeymathcatcodes{% +% \enforced\let _\normalsubscript +% \enforced\let ^\normalsuperscript +% } +% +% \doglobal\appendtoks +% \enforced\let _\normalsubscript +% \enforced\let ^\normalsuperscript +% \to \everymathematics +% +% \egroup %D We keep this, just for the fun of it: @@ -3772,10 +3779,10 @@ \permanent\protected\def\?{\mathtextauto{?}{?}} \popoverloadmode -\appendtoks - \reinstatecatcodecommand\barasciicode - \obeydiscretionaries -\to \everymathtext +% \appendtoks +% \reinstatecatcodecommand\barasciicode +% \obeydiscretionaries +% \to \everymathtext %D Safeguard against redefinitions: @@ -4452,19 +4459,47 @@ %D test \im {z + \mathaxisontop x + 2 - a = 1} test %D \stoptyping +%D \starttyping +%D $ x^2 + x_2 + x_2^2\mathaxisbelow\mathscriptbelow$\par +%D $ x^g + x_g + x_g^g\mathaxisbelow\mathscriptbelow$ +%D \stoptyping + \definesystemattribute[mathaxis][public] -\def\math_axis_inject#1% - {\mathatom \s!class \mathghostcode \s!unpack { - \showmakeup[strut]% todo: shortcut - \scratchdimen.5\Umathfractionrule\mathstyle - \srule - \s!height \dimexpr\Umathaxis\mathstyle+\scratchdimen\relax - \s!depth -\dimexpr\Umathaxis\mathstyle-\scratchdimen\relax - \s!attr \mathaxisattribute #1% - \relax}} - -\permanent\protected\def\mathaxisbelow{\math_axis_inject\plusone} -\permanent\protected\def\mathaxisontop{\math_axis_inject\plustwo} + +\def\math_axis_inject_indeed#1% + {\srule + \s!height \dimexpr\scratchdimentwo+\scratchdimenone\relax + \s!depth -\dimexpr\scratchdimentwo-\scratchdimenone\relax + \s!attr \mathaxisattribute#1% + \relax} + +\def\math_axis_inject#1#2% + {\mathatom \s!class \mathghostcode \s!unpack { + \showmakeup[strut]% todo: shortcut + \ifcase#1% + % nothing + \or + \scratchdimenone.5\Umathfractionrule\mathstyle + \scratchdimentwo\Umathaxis\mathstyle + \math_axis_inject_indeed#2% + \or + \scratchdimenone.25\Umathfractionrule\mathstyle + \scratchdimentwo\Umathsupshiftup\mathstyle + \math_axis_inject_indeed#2% + \scratchdimentwo-\Umathsubshiftdown\mathstyle + \math_axis_inject_indeed#2% + \scratchdimentwo-\Umathsubsupshiftdown\mathstyle + \math_axis_inject_indeed#2% + % \scratchdimenone.125\Umathfractionrule\mathstyle + \divide\scratchdimenone\plustwo + \scratchdimentwo\Umathprimeshiftup\mathstyle + \math_axis_inject_indeed#2% + \fi}} + +\permanent\protected\def\mathaxisbelow {\math_axis_inject\plusone\plusone} +\permanent\protected\def\mathaxisontop {\math_axis_inject\plusone\plustwo} +\permanent\protected\def\mathscriptbelow{\math_axis_inject\plustwo\plusone} +\permanent\protected\def\mathscriptontop{\math_axis_inject\plustwo\plustwo} \protect \endinput diff --git a/tex/context/base/mkxl/math-twk.mkxl b/tex/context/base/mkxl/math-twk.mkxl index 269c855c0..6ffb36818 100644 --- a/tex/context/base/mkxl/math-twk.mkxl +++ b/tex/context/base/mkxl/math-twk.mkxl @@ -86,8 +86,11 @@ \immutable\chardef\textminute"2032 \immutable\chardef\textsecond"2033 -\immutable\Umathchardef\mathminute\zerocount\zerocount\privatecharactercode{prime 0x2032 1} -\immutable\Umathchardef\mathsecond\zerocount\zerocount\privatecharactercode{prime 0x2033 1} +% \immutable\Umathchardef\mathminute\zerocount\zerocount\privatecharactercode{prime 0x2032 1} +% \immutable\Umathchardef\mathsecond\zerocount\zerocount\privatecharactercode{prime 0x2033 1} + +\immutable\Umathchardef\mathminute\zerocount\zerocount\textminute +\immutable\Umathchardef\mathsecond\zerocount\zerocount\textsecond \permanent\protected\def\minute{\iffontchar\font\textminute\textminute\else\mathminute\fi} \permanent\protected\def\second{\iffontchar\font\textsecond\textsecond\else\mathsecond\fi} diff --git a/tex/context/base/mkxl/node-fnt.lmt b/tex/context/base/mkxl/node-fnt.lmt index 26e1fc343..61a20c628 100644 --- a/tex/context/base/mkxl/node-fnt.lmt +++ b/tex/context/base/mkxl/node-fnt.lmt @@ -73,6 +73,7 @@ local wordboundary_code = boundarycodes.word local protectglyphs = nuts.protectglyphs local unprotectglyphs = nuts.unprotectglyphs +local protectglyphsnone = nuts.protectglyphsnone local setmetatableindex = table.setmetatableindex @@ -168,64 +169,428 @@ local function stop_trace(u,usedfonts,d,dynamicfonts,b,basefonts,r,redundant) report_fonts() end +-- This is the original handler and we keep it around as reference. It served us +-- well for quite a while. + +-- do +-- +-- local usedfonts +-- local dynamicfonts +-- local basefonts -- could be reused +-- local basefont +-- local prevfont +-- local prevdynamic +-- local variants +-- local redundant -- could be reused +-- local firstnone +-- local lastfont +-- local lastproc +-- local lastnone +-- +-- local d, u, b, r +-- +-- local function protectnone() +-- protectglyphs(firstnone,lastnone) +-- firstnone = nil +-- end +-- +-- local function setnone(n) +-- if firstnone then +-- protectnone() +-- end +-- if basefont then +-- basefont[2] = getprev(n) +-- basefont = false +-- end +-- if not firstnone then +-- firstnone = n +-- end +-- lastnone = n +-- end +-- +-- local function setbase(n) +-- if firstnone then +-- protectnone() +-- end +-- if force_basepass then +-- if basefont then +-- basefont[2] = getprev(n) +-- end +-- b = b + 1 +-- basefont = { n, false } +-- basefonts[b] = basefont +-- end +-- end +-- +-- local function setnode(n,font,dynamic) -- we could use prevfont and prevdynamic when we set then first +-- if firstnone then +-- protectnone() +-- end +-- if basefont then +-- basefont[2] = getprev(n) +-- basefont = false +-- end +-- if dynamic > 0 then +-- local used = dynamicfonts[font] +-- if not used then +-- used = { } +-- dynamicfonts[font] = used +-- end +-- if not used[dynamic] then +-- local fd = setfontdynamics[font] +-- if fd then +-- used[dynamic] = fd[dynamic] +-- d = d + 1 +-- end +-- end +-- else +-- local used = usedfonts[font] +-- if not used then +-- lastfont = font +-- lastproc = fontprocesses[font] +-- if lastproc then +-- usedfonts[font] = lastproc +-- u = u + 1 +-- end +-- end +-- end +-- end +-- +-- function handlers.characters(head,groupcode,direction) +-- -- either next or not, but definitely no already processed list +-- starttiming(nodes) +-- +-- usedfonts = { } +-- dynamicfonts = { } +-- basefonts = { } +-- basefont = nil +-- prevfont = nil +-- prevdynamic = 0 +-- variants = nil +-- redundant = nil +-- firstnone = nil +-- lastfont = nil +-- lastproc = nil +-- lastnone = nil +-- +-- local fontmode = nil -- base none or other +-- +-- d, u, b, r = 0, 0, 0, 0 +-- +-- if trace_fontrun then +-- start_trace(head) +-- end +-- +-- -- There is no gain in checking for a single glyph and then having a fast path. On the +-- -- metafun manual (with some 2500 single char lists) the difference is just noise. +-- +-- for n, char, font, dynamic in nextchar, head do +-- +-- if font ~= prevfont then +-- prevfont = font +-- fontmode = fontmodes[font] +-- if fontmode == "none" then +-- prevdynamic = 0 +-- variants = false +-- setnone(n) +-- elseif fontmode == "base" then +-- prevdynamic = 0 +-- variants = false +-- setbase(n) +-- else +-- -- local dynamic = getglyphdata(n) or 0 -- zero dynamic is reserved for fonts in context +-- prevdynamic = dynamic +-- variants = fontvariants[font] +-- setnode(n,font,dynamic) +-- end +-- elseif fontmode == "node" then +-- local dynamic = getglyphdata(n) or 0 -- zero dynamic is reserved for fonts in context +-- if dynamic ~= prevdynamic then +-- prevdynamic = dynamic +-- variants = fontvariants[font] +-- setnode(n,font,dynamic) +-- end +-- elseif firstnone then +-- lastnone = n +-- end +-- +-- if variants then +-- if (char >= 0xFE00 and char <= 0xFE0F) or (char >= 0xE0100 and char <= 0xE01EF) then +-- -- if variants and char >= 0xFE00 then +-- -- if char < 0xFE0F or (char >= 0xE0100 and char <= 0xE01EF) then +-- local hash = variants[char] +-- if hash then +-- local p = getprev(n) +-- if p then +-- local char = ischar(p) -- checked +-- local variant = hash[char] +-- if variant then +-- if trace_variants then +-- report_fonts("replacing %C by %C",char,variant) +-- end +-- setchar(p,variant) +-- if redundant then +-- r = r + 1 +-- redundant[r] = n +-- else +-- r = 1 +-- redundant = { n } +-- end +-- end +-- end +-- elseif keep_redundant then +-- -- go on, can be used for tracing +-- elseif redundant then +-- r = r + 1 +-- redundant[r] = n +-- else +-- r = 1 +-- redundant = { n } +-- end +-- end +-- end +-- +-- end +-- +-- if firstnone then +-- protectnone() +-- end +-- +-- if force_boundaryrun then +-- +-- -- we can inject wordboundaries and then let the hyphenator do its work +-- -- but we need to get rid of those nodes in order to build ligatures +-- -- and kern (a rather context thing) +-- +-- for b, subtype in nextboundary, head do +-- if subtype == wordboundary_code then +-- if redundant then +-- r = r + 1 +-- redundant[r] = b +-- else +-- r = 1 +-- redundant = { b } +-- end +-- end +-- end +-- +-- end +-- +-- if redundant then +-- for i=1,r do +-- local r = redundant[i] +-- local p, n = getboth(r) +-- if r == head then +-- head = n +-- setprev(n) +-- else +-- setlink(p,n) +-- end +-- if b > 0 then +-- for i=1,b do +-- local bi = basefonts[i] +-- local b1 = bi[1] +-- local b2 = bi[2] +-- if b1 == b2 then +-- if b1 == r then +-- bi[1] = false +-- bi[2] = false +-- end +-- elseif b1 == r then +-- bi[1] = n +-- elseif b2 == r then +-- bi[2] = p +-- end +-- end +-- end +-- flushnode(r) +-- end +-- end +-- +-- if force_discrun then +-- -- basefont is not supported in disc only runs ... it would mean a lot of +-- -- ranges .. we could try to run basemode as a separate processor run but not +-- -- for now (we can consider it when the new node code is tested +-- for disc in nextdisc, head do +-- -- doing only replace is good enough because pre and post are normally used +-- -- for hyphens and these come from fonts that part of the hyphenated word +-- local r = getreplace(disc) +-- if r then +-- local prevfont = nil +-- local prevdynamic = nil +-- local none = false +-- firstnone = nil +-- basefont = nil +-- for n, char, font, dynamic in nextchar, r do +-- -- local dynamic = getglyphdata(n) or 0 -- zero dynamic is reserved for fonts in context +-- if font ~= prevfont or dynamic ~= prevdynamic then +-- prevfont = font +-- prevdynamic = dynamic +-- local fontmode = fontmodes[font] +-- if fontmode == "none" then +-- setnone(n) +-- elseif fontmode == "base" then +-- -- so the replace gets an extra treatment ... so be it +-- setbase(n) +-- else +-- setnode(n,font,dynamic) +-- end +-- elseif firstnone then +-- -- lastnone = n +-- lastnone = nil +-- end +-- -- we assume one font for now (and if there are more and we get into issues then +-- -- we can always remove the break) +-- break +-- end +-- if firstnone then +-- protectnone() +-- end +-- end +-- end +-- +-- end +-- +-- if trace_fontrun then +-- stop_trace(u,usedfonts,d,dynamicfonts,b,basefonts,r,redundant) +-- end +-- +-- -- in context we always have at least 2 processors +-- if u == 0 then +-- -- skip +-- elseif u == 1 then +-- for i=1,#lastproc do +-- head = lastproc[i](head,lastfont,0,direction) +-- end +-- else +-- for font, processors in next, usedfonts do -- unordered +-- for i=1,#processors do +-- head = processors[i](head,font,0,direction,u) -- u triggers disc optimizer +-- end +-- end +-- end +-- +-- if d == 0 then +-- -- skip +-- elseif d == 1 then +-- local font, dynamics = next(dynamicfonts) +-- for dynamic, processors in next, dynamics do -- unordered, dynamic can switch in between +-- for i=1,#processors do +-- head = processors[i](head,font,dynamic,direction) +-- end +-- end +-- else +-- for font, dynamics in next, dynamicfonts do +-- for dynamic, processors in next, dynamics do -- unordered, dynamic can switch in between +-- for i=1,#processors do +-- head = processors[i](head,font,dynamic,direction,d) -- d triggers disc optimizer +-- end +-- end +-- end +-- end +-- if b == 0 then +-- -- skip +-- elseif b == 1 then +-- -- only one font +-- local range = basefonts[1] +-- local start = range[1] +-- local stop = range[2] +-- if (start or stop) and (start ~= stop) then +-- local front = head == start +-- if stop then +-- start = ligaturing(start,stop) +-- start = kerning(start,stop) +-- elseif start then -- safeguard +-- start = ligaturing(start) +-- start = kerning(start) +-- end +-- if front and head ~= start then +-- head = start +-- end +-- end +-- else +-- -- multiple fonts +-- for i=1,b do +-- local range = basefonts[i] +-- local start = range[1] +-- local stop = range[2] +-- if start then -- and start ~= stop but that seldom happens +-- local front = head == start +-- local prev = getprev(start) +-- local next = getnext(stop) +-- if stop then +-- start, stop = ligaturing(start,stop) +-- start, stop = kerning(start,stop) +-- else +-- start = ligaturing(start) +-- start = kerning(start) +-- end +-- -- is done automatically +-- if prev then +-- setlink(prev,start) +-- end +-- if next then +-- setlink(stop,next) +-- end +-- -- till here +-- if front and head ~= start then +-- head = start +-- end +-- end +-- end +-- end +-- +-- stoptiming(nodes) +-- +-- if trace_characters then +-- nodes.report(head) +-- end +-- +-- return head +-- end +-- +-- end + + +-- This variant uses less code but relies on the engine checking the textcontrol +-- flags: +-- +-- baseligatures : 0x02 +-- basekerns : 0x04 +-- noneprotected : 0x08 +-- +-- This permits one 'base' pass instead of multiple over ranges which is kind of +-- tricky because we then can have clashes when we process replace fields +-- independently. We can also protect 'none' in one go. It is actually not that +-- much faster (and in some cases it might even be slower). We can make the code +-- a bit leaner (no setbase and setnone). + do local usedfonts local dynamicfonts - local basefonts -- could be reused - local basefont local prevfont local prevdynamic local variants local redundant -- could be reused - local firstnone local lastfont local lastproc - local lastnone + local basedone + local nonedone local d, u, b, r - local function protectnone() - protectglyphs(firstnone,lastnone) - firstnone = nil - end - local function setnone(n) - if firstnone then - protectnone() - end - if basefont then - basefont[2] = getprev(n) - basefont = false - end - if not firstnone then - firstnone = n - end - lastnone = n + nonedone = true end local function setbase(n) - if firstnone then - protectnone() - end if force_basepass then - if basefont then - basefont[2] = getprev(n) - end - b = b + 1 - basefont = { n, false } - basefonts[b] = basefont + basedone = true end end local function setnode(n,font,dynamic) -- we could use prevfont and prevdynamic when we set then first - if firstnone then - protectnone() - end - if basefont then - basefont[2] = getprev(n) - basefont = false - end if dynamic > 0 then local used = dynamicfonts[font] if not used then @@ -258,24 +623,22 @@ do usedfonts = { } dynamicfonts = { } - basefonts = { } - basefont = nil prevfont = nil prevdynamic = 0 variants = nil redundant = nil - firstnone = nil lastfont = nil lastproc = nil - lastnone = nil + nonedone = nil + basedone = nil local fontmode = nil -- base none or other d, u, b, r = 0, 0, 0, 0 - if trace_fontrun then - start_trace(head) - end +-- if trace_fontrun then +-- start_trace(head) +-- end -- There is no gain in checking for a single glyph and then having a fast path. On the -- metafun manual (with some 2500 single char lists) the difference is just noise. @@ -306,8 +669,6 @@ do variants = fontvariants[font] setnode(n,font,dynamic) end - elseif firstnone then - lastnone = n end if variants then @@ -348,10 +709,6 @@ do end - if firstnone then - protectnone() - end - if force_boundaryrun then -- we can inject wordboundaries and then let the hyphenator do its work @@ -382,31 +739,13 @@ do else setlink(p,n) end - if b > 0 then - for i=1,b do - local bi = basefonts[i] - local b1 = bi[1] - local b2 = bi[2] - if b1 == b2 then - if b1 == r then - bi[1] = false - bi[2] = false - end - elseif b1 == r then - bi[1] = n - elseif b2 == r then - bi[2] = p - end - end - end flushnode(r) end end + -- todo: make this more clever + if force_discrun then - -- basefont is not supported in disc only runs ... it would mean a lot of - -- ranges .. we could try to run basemode as a separate processor run but not - -- for now (we can consider it when the new node code is tested for disc in nextdisc, head do -- doing only replace is good enough because pre and post are normally used -- for hyphens and these come from fonts that part of the hyphenated word @@ -415,8 +754,6 @@ do local prevfont = nil local prevdynamic = nil local none = false - firstnone = nil - basefont = nil for n, char, font, dynamic in nextchar, r do -- local dynamic = getglyphdata(n) or 0 -- zero dynamic is reserved for fonts in context if font ~= prevfont or dynamic ~= prevdynamic then @@ -426,29 +763,26 @@ do if fontmode == "none" then setnone(n) elseif fontmode == "base" then - -- so the replace gets an extra treatment ... so be it setbase(n) else setnode(n,font,dynamic) end - elseif firstnone then - -- lastnone = n - lastnone = nil end -- we assume one font for now (and if there are more and we get into issues then -- we can always remove the break) break end - if firstnone then - protectnone() - end end end end - if trace_fontrun then - stop_trace(u,usedfonts,d,dynamicfonts,b,basefonts,r,redundant) +-- if trace_fontrun then +-- stop_trace(u,usedfonts,d,dynamicfonts,b,basefonts,r,redundant) +-- end + + if nonedone then + protectglyphsnone(head) end -- in context we always have at least 2 processors @@ -465,6 +799,7 @@ do end end end + if d == 0 then -- skip elseif d == 1 then @@ -483,55 +818,13 @@ do end end end - if b == 0 then - -- skip - elseif b == 1 then - -- only one font - local range = basefonts[1] - local start = range[1] - local stop = range[2] - if (start or stop) and (start ~= stop) then - local front = head == start - if stop then - start = ligaturing(start,stop) - start = kerning(start,stop) - elseif start then -- safeguard - start = ligaturing(start) - start = kerning(start) - end - if front and head ~= start then - head = start - end - end - else - -- multiple fonts - for i=1,b do - local range = basefonts[i] - local start = range[1] - local stop = range[2] - if start then -- and start ~= stop but that seldom happens - local front = head == start - local prev = getprev(start) - local next = getnext(stop) - if stop then - start, stop = ligaturing(start,stop) - start, stop = kerning(start,stop) - else - start = ligaturing(start) - start = kerning(start) - end - -- is done automatically - if prev then - setlink(prev,start) - end - if next then - setlink(stop,next) - end - -- till here - if front and head ~= start then - head = start - end - end + + if basedone then + local start = head + start = ligaturing(start) + start = kerning(start) + if head ~= start then + head = start end end diff --git a/tex/context/base/mkxl/node-nut.lmt b/tex/context/base/mkxl/node-nut.lmt index 62b733125..38c182f8a 100644 --- a/tex/context/base/mkxl/node-nut.lmt +++ b/tex/context/base/mkxl/node-nut.lmt @@ -199,6 +199,7 @@ local nuts = { prependbeforehead = direct.prependbeforehead, protectglyph = direct.protectglyph, protectglyphs = direct.protectglyphs, + protectglyphsnone = direct.protectglyphsnone, protrusionskippable = direct.protrusionskippable, rangedimensions = direct.rangedimensions, remove = d_remove_node, diff --git a/tex/context/base/mkxl/spac-chr.mkxl b/tex/context/base/mkxl/spac-chr.mkxl index a7d339019..346bc2183 100644 --- a/tex/context/base/mkxl/spac-chr.mkxl +++ b/tex/context/base/mkxl/spac-chr.mkxl @@ -73,7 +73,11 @@ % unexpanded as otherwise we need to intercept / cleanup a lot +\pushoverloadmode + \protected\def~{\nobreakspace} +\popoverloadmode + \protect \endinput diff --git a/tex/context/base/mkxl/spac-hor.mkxl b/tex/context/base/mkxl/spac-hor.mkxl index 4c153931b..111bdc94c 100644 --- a/tex/context/base/mkxl/spac-hor.mkxl +++ b/tex/context/base/mkxl/spac-hor.mkxl @@ -523,10 +523,12 @@ \permanent\protected\def\fixedspaces {\letcatcodecommand \ctxcatcodes \tildeasciicode\fixedspace - \enforced\let~\fixedspace} % we need to renew it + %\enforced\let~\fixedspace} % we need to renew it + \enforced\letcharcode\tildeasciicode\fixedspace} % why this \appendtoks - \enforced\let~\space + %\enforced\let~\space + \enforced\letcharcode\tildeasciicode\space \enforced\let\ \space \to \everysimplifycommands diff --git a/tex/context/base/mkxl/tabl-tab.mkxl b/tex/context/base/mkxl/tabl-tab.mkxl index ad9bfdd45..815767e18 100644 --- a/tex/context/base/mkxl/tabl-tab.mkxl +++ b/tex/context/base/mkxl/tabl-tab.mkxl @@ -57,7 +57,8 @@ \dontcomplain \tabl_table_restore_lineskips \normalbaselines - \enforced\let~\fixedspace + % \enforced\let~\fixedspace +` \enforced\letcharcode\tildeasciicode\fixedspace % why \inhibitblank % \blank[\v!disable]% % added \the\everytableparbox} @@ -835,7 +836,7 @@ \bgroup \catcode\tildeasciicode\activecatcode \appendtoks - \catcode\barasciicode\activecatcode +% \catcode\barasciicode\activecatcode \enforced\protected\def ~{\kern.5em}% \enforced\protected\def\\{\ifhmode\space\else\par\fi}% \to \everytable @@ -1492,26 +1493,42 @@ % \noalign{\globalpopmacro\simpletableHL}% } -\bgroup \catcode\barasciicode\othercatcode +% \bgroup \catcode\barasciicode\othercatcode +% +% \gdef\tabl_table_second_stage[#1]% brr nested mess +% {\bgroup +% \tabl_table_use_bar +% \global\setfalse\tableactionstatepermitted +% \global\setfalse\hassometablehead +% \global\setfalse\hassometabletail +% \expanded{\doifelseinstring{|}{#1}} +% {\xdef\tabl_table_restart{\noexpand\tabl_table_restart_indeed{\noexpand\tabl_table_third_stage{#1}}}} +% {\ifcsname\??tabletemplate#1\endcsname +% \gdef\tabl_table_restart{\csname\??tabletemplate#1\endcsname}% +% \else +% \gdef\tabl_table_restart{\tabl_table_restart_indeed{\begincsname#1\endcsname}}% +% \fi}% +% \egroup +% \tabl_table_restart} +% +% \egroup -\gdef\tabl_table_second_stage[#1]% brr nested mess +\def\tabl_table_second_stage[#1]% brr nested mess {\bgroup \tabl_table_use_bar \global\setfalse\tableactionstatepermitted \global\setfalse\hassometablehead \global\setfalse\hassometabletail - \expanded{\doifelseinstring{|}{#1}} - {\xdef\tabl_table_restart{\noexpand\tabl_table_restart_indeed{\noexpand\tabl_table_third_stage{#1}}}} - {\ifcsname\??tabletemplate#1\endcsname - \gdef\tabl_table_restart{\csname\??tabletemplate#1\endcsname}% - \else - \gdef\tabl_table_restart{\tabl_table_restart_indeed{\begincsname#1\endcsname}}% - \fi}% + \normalexpanded{\noexpand\ifhastoks{|}{#1}}% + \xdef\tabl_table_restart{\noexpand\tabl_table_restart_indeed{\noexpand\tabl_table_third_stage{#1}}}% + \orelse\ifcsname\??tabletemplate#1\endcsname + \gdef\tabl_table_restart{\csname\??tabletemplate#1\endcsname}% + \else + \gdef\tabl_table_restart{\tabl_table_restart_indeed{\begincsname#1\endcsname}}% + \fi \egroup \tabl_table_restart} -\egroup - %D The third stage involves a lot of (re)sets, which we will explain later. \appendtoks @@ -1859,14 +1876,15 @@ \newcount\currenttablecolumn -% DWhile defining this macro we change the \CATCODE\ of \type{|}. When counting the -% Dbars, we use a non active representation of the bar, simply because we cannot be -% Dsure if the bar is active or not.\footnote{Normally it is, but \TABLE\ changes -% Dthe catcode when needed.} +%D While defining this macro we change the \CATCODE\ of \type{|}. When counting the +%D bars, we use a non active representation of the bar, simply because we cannot be +%D sure if the bar is active or not.\footnote{Normally it is, but \TABLE\ changes +%D the catcode when needed.} \bgroup - \catcode\barasciicode\othercatcode \permanent\gdef\tabl_table_bar{|} - \catcode\barasciicode\activecatcode\gdef\tabl_table_use_bar{\enforced\let|\tabl_table_bar} +% \catcode\barasciicode\othercatcode \permanent\gdef\tabl_table_bar{|} +% \catcode\barasciicode\activecatcode\gdef\tabl_table_use_bar{\enforced\let|\tabl_table_bar} +\glet\tabl_table_use_bar\relax \egroup \bgroup \catcode\barasciicode\othercatcode @@ -1874,7 +1892,7 @@ \gdef\tabl_tables_get_nofcolumns#1% todo: also divert this to lua as with tabulate {\bgroup \cleanupfeatures % needed ! - \tabl_table_use_bar + \tabl_table_use_bar % doesn't do anything as we don't treat #1 \egroup} \egroup diff --git a/tex/context/base/mkxl/typo-rub.mkxl b/tex/context/base/mkxl/typo-rub.mkxl index bf48d8fa1..79760c328 100644 --- a/tex/context/base/mkxl/typo-rub.mkxl +++ b/tex/context/base/mkxl/typo-rub.mkxl @@ -45,7 +45,8 @@ \enforced\let\ruby\noruby \edef\currentruby{#1}% \edef\p_location{\rubyparameter\c!location}% - \enforced\let|\relax + % \enforced\let|\relax + \enforced\letcharcode\barasciicode\relax \ifcsname\??rubyanalyze\p_location\endcsname \expandafter\lastnamedcs\else\expandafter\typo_ruby_analyze \fi{#2}{#3}% diff --git a/tex/context/fonts/mkiv/bonum-math.lfg b/tex/context/fonts/mkiv/bonum-math.lfg index dfce35d80..736b8a176 100644 --- a/tex/context/fonts/mkiv/bonum-math.lfg +++ b/tex/context/fonts/mkiv/bonum-math.lfg @@ -24,11 +24,16 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, DisplayOperatorMinHeight = 1900, -- 1250 in font - -- AccentSuperscriptDrop = 100, - -- AccentSuperscriptPercent = 20, - -- PrimeRaisePercent = 50, - PrimeRaiseComposedPercent = 0, - -- PrimeShiftUp = 0, + -- ScriptPercentScaleDown = 65, -- 77 in font + -- ScriptScriptPercentScaleDown = 50, -- 60 in font + SubscriptShiftDown = 201, -- 231 in font + SuperscriptShiftUp = 364, -- 334 in font + SubscriptShiftDownWithSuperscript = "1.4*SubscriptShiftDown", -- trial and error + -- PrimeRaisePercent = 0, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 0, -- set to 0 in math-act + PrimeShiftUp = "1.25*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.25*SuperscriptShiftUp", + -- PrimeBaselineDropMax = 0, }, tweaks = { @@ -41,7 +46,7 @@ return { tweak = "fixprimes", -- scale = 0.8, -- smaller = true, - factor = 1,--1.1 + factor = 0.77,--1.1 }, { tweak = "addmirrors", @@ -121,8 +126,8 @@ return { [0x0029] = { topright = -0.15, bottomright = -0.15 }, -- right parenthesis variants ["0x29.variants.*"] = { topright = -0.15, bottomright = -0.15 }, -- right parenthesis variants ["0x29.parts.top"] = { topright = -0.15, }, -- right parenthesis top - ["0x29.parts.bottom"] = { bottomright = -0.15 }, -- right parenthesis bottom -- radical - [0x221A] = { topright = 0.2, bottomright = 0.2 }, + ["0x29.parts.bottom"] = { bottomright = -0.15 }, -- right parenthesis bottom + [0x221A] = { topright = 0.2, bottomright = 0.2 }, -- radical ["0x221A.variants.*"] = { topright = 0.2, bottomright = 0.2 }, ["0x221A.parts.top"] = { topright = 0.2, }, ["0x221A.parts.bottom"] = { bottomright = 0.2 }, @@ -131,6 +136,8 @@ return { [0x27EB] = { topright = -0.1, bottomright = -0.1 }, ["0x27EB.variants.*"] = { topright = -0.2, bottomright = -0.2 }, -- + [0x1D465] = { bottomright = -0.05, }, -- italic x (ic is too large) + [0x00393] = { bottomright = -0.30, }, -- upright Gamma -- [0x222B] = integral_variants, ["0x222B.variants.*"] = integral_variants, ["0x222B.parts.top"] = integral_top, ["0x222B.parts.bottom"] = { bottomright = -0.20 }, -- int ["0x222C.parts.bottom"] = { bottomright = -0.15 }, -- iint @@ -420,6 +427,9 @@ return { tweak = "addfourier", variant = 1, }, + { + tweak = "addprimed", + }, -- this is the mkiv section { tweak = "emulatelmtx", diff --git a/tex/context/fonts/mkiv/cambria-math.lfg b/tex/context/fonts/mkiv/cambria-math.lfg index 97cea6674..e4791fc79 100644 --- a/tex/context/fonts/mkiv/cambria-math.lfg +++ b/tex/context/fonts/mkiv/cambria-math.lfg @@ -21,8 +21,10 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, DisplayOperatorMinHeight = 2800, -- 2500 in font - PrimeRaisePercent = 75, -- 50 default - -- PrimeRaiseComposedPercent = 25, -- 25 default + -- PrimeRaisePercent = 0, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 25, -- set to 0 in math-act + PrimeShiftUp = "1.25*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.25*SuperscriptShiftUp", }, tweaks = { aftercopying = { @@ -46,35 +48,28 @@ return { { tweak = "kerns", list = { - [0x2F] = { - topleft = -0.2, - -- bottomleft = 0, - -- topright = 0, - bottomright = -0.2, - }, - ["0x7D.parts.top"] = { - topright = -0.1, - }, - ["0x7D.parts.bottom"] = { - bottomright = -0.1, - }, - ["0x29.parts.top"] = { - topright = -0.2, - }, - ["0x29.parts.bottom"] = { - bottomright = -0.2, - }, - ["0x221A.parts.top"] = { - topright = 0.2, - }, - ["0x221A.parts.bottom"] = { - bottomright = 0.2, - }, - ["0x221A.variants.*"] = { - topright = 0.2, - bottomright = 0.2, - }, - + [0x002F] = { topleft = -0.2, bottomright = -0.2 }, + ["0x002F.variants.*"] = { topleft = -0.2, bottomright = -0.2 }, + -- No! + -- [0x0028] = { topleft = -0.1, bottomleft = -0.1 }, -- left parenthesis + -- [0x0029] = { topright = -0.1, bottomright = -0.1, all = true }, -- right parenthesis + -- + [0x007D] = { topright = -0.05, bottomright = -0.05 }, -- right brace variants + ["0x7D.variants.*"] = { topright = -0.10, bottomright = -0.10 }, -- right brace variants + ["0x7D.parts.top"] = { topright = -0.10, }, -- right brace top + ["0x7D.parts.bottom"] = { bottomright = -0.10 }, -- right brace bottom + [0x0029] = { topright = -0.15, bottomright = -0.15 }, -- right parenthesis variants + ["0x29.variants.*"] = { topright = -0.15, bottomright = -0.15 }, -- right parenthesis variants + ["0x29.parts.top"] = { topright = -0.15, }, -- right parenthesis top + ["0x29.parts.bottom"] = { bottomright = -0.15 }, -- right parenthesis bottom + [0x221A] = { topright = 0.2, bottomright = 0.2 }, -- radical + ["0x221A.variants.*"] = { topright = 0.2, bottomright = 0.2 }, + ["0x221A.parts.top"] = { topright = 0.2, }, + ["0x221A.parts.bottom"] = { bottomright = 0.2 }, + [0x27E9] = { topright = -0.1, bottomright = -0.1 }, -- angle + ["0x27E9.variants.*"] = { topright = -0.3, bottomright = -0.3 }, + [0x27EB] = { topright = -0.1, bottomright = -0.1 }, + ["0x27EB.variants.*"] = { topright = -0.2, bottomright = -0.2 }, -- Keep as example. not needed in cambria (after all it is the reference): [0x2A0C] = { bottomright = -0.1 }, -- iiiint does not have any ic @@ -156,9 +151,10 @@ return { { tweak = "fixprimes", - scale = 0.9, + -- scale = 1.0, -- smaller = true, - factor = 0.9, + factor = 0.83, + fake = 0.8, }, { tweak = "checkspacing", @@ -206,6 +202,9 @@ return { tweak = "addfourier", variant = 2, }, + { + tweak = "addprimed", + }, -- { -- the ldots are squareshaped and the cdots are circular -- tweak = "fixellipses", -- }, diff --git a/tex/context/fonts/mkiv/common-math.lfg b/tex/context/fonts/mkiv/common-math.lfg index 85915c26a..d2785b3b2 100644 --- a/tex/context/fonts/mkiv/common-math.lfg +++ b/tex/context/fonts/mkiv/common-math.lfg @@ -214,6 +214,7 @@ return { return { tweak = "movelimits", factor = parameters.factor or 1, + icfactor = parameters.icfactor or 1, list = mathematics.tweaks.subsets.integrals, } end, diff --git a/tex/context/fonts/mkiv/concrete-math.lfg b/tex/context/fonts/mkiv/concrete-math.lfg index c107cfdeb..d0fc5749f 100644 --- a/tex/context/fonts/mkiv/concrete-math.lfg +++ b/tex/context/fonts/mkiv/concrete-math.lfg @@ -14,12 +14,16 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, -- DisplayOperatorMinHeight = 1800, -- 1400 in font (one size) + -- PrimeRaisePercent = 0, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 25, -- set to 0 in math-act + PrimeShiftUp = "1.2*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.2*SuperscriptShiftUp", }, tweaks = { aftercopying = { { tweak = "fixprimes", - factor = 0.8, + factor = 0.92, }, -- there are circular variants -- { @@ -44,6 +48,43 @@ return { [0x002F] = { topleft = -0.2, bottomright = -0.2 }, }, }, + { + tweak = "kerns", + list = { + [0x002F] = { topleft = -0.2, bottomright = -0.2 }, + ["0x002F.variants.*"] = { topleft = -0.2, bottomright = -0.2 }, + -- No! + -- [0x0028] = { topleft = -0.1, bottomleft = -0.1 }, -- left parenthesis + -- [0x0029] = { topright = -0.1, bottomright = -0.1, all = true }, -- right parenthesis + -- + [0x007D] = { topright = -0.05, bottomright = -0.05 }, -- right brace variants + ["0x7D.variants.*"] = { topright = -0.25, bottomright = -0.25 }, -- right brace variants + ["0x7D.parts.top"] = { topright = -0.25, }, -- right brace top + ["0x7D.parts.bottom"] = { bottomright = -0.25 }, -- right brace bottom + [0x0029] = { topright = -0.15, bottomright = -0.15 }, -- right parenthesis variants + ["0x29.variants.*"] = { topright = -0.15, bottomright = -0.15 }, -- right parenthesis variants + ["0x29.parts.top"] = { topright = -0.15, }, -- right parenthesis top + ["0x29.parts.bottom"] = { bottomright = -0.15 }, -- right parenthesis bottom + [0x221A] = { topright = 0.2, bottomright = 0.2 }, -- radical + ["0x221A.variants.*"] = { topright = 0.2, bottomright = 0.2 }, + ["0x221A.parts.top"] = { topright = 0.2, }, + ["0x221A.parts.bottom"] = { bottomright = 0.2 }, + [0x27E9] = { topright = -0.1, bottomright = -0.1 }, -- angle + ["0x27E9.variants.*"] = { topright = -0.3, bottomright = -0.3 }, + [0x27EB] = { topright = -0.1, bottomright = -0.1 }, + ["0x27EB.variants.*"] = { topright = -0.2, bottomright = -0.2 }, + -- + [0x00393] = { bottomright = -0.30, }, -- upright Gamma + }, + }, + + { + tweak = "dimensions", + list = { + [0x00393] = { width = 0.875, }, -- upright Gamma + }, + }, + { tweak = "radicaldegreeanchors", @@ -79,6 +120,9 @@ return { variant = 2, }, { + tweak = "addprimed", + }, + { tweak = "setoptions", set = { "ignorekerndimensions" } }, diff --git a/tex/context/fonts/mkiv/dejavu-math.lfg b/tex/context/fonts/mkiv/dejavu-math.lfg index 728a5f2fc..af6a4bd55 100644 --- a/tex/context/fonts/mkiv/dejavu-math.lfg +++ b/tex/context/fonts/mkiv/dejavu-math.lfg @@ -24,8 +24,15 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, DisplayOperatorMinHeight = 1800, -- 1333 in font + -- ScriptPercentScaleDown = 70, -- 80 in font (set in typescript) + -- ScriptScriptPercentScaleDown = 55, -- 65 in font (set in typescript) + SubscriptShiftDown = 177, -- 277 in font + SuperscriptShiftUp = 428, -- 381 in font + SubscriptShiftDownWithSuperscript = "1.3*SubscriptShiftDown", -- trial and error -- PrimeRaisePercent = 50, -- 50 default - PrimeRaiseComposedPercent = 10, -- 25 default + -- PrimeRaiseComposedPercent = 10, -- 25 default + PrimeShiftUp = "1.4*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.4*SuperscriptShiftUp", }, tweaks = { aftercopying = { @@ -33,17 +40,9 @@ return { tweak = "version", expected = "Version 1.106", }, --- { --- tweak = "fixprimes", --- scale = 0.85, --- -- smaller = true, --- factor = 1, --- }, { tweak = "fixprimes", - scale = 1, - -- smaller = true, - factor = 0.8, + factor = 0.63, }, { tweak = "addmirrors", @@ -89,7 +88,8 @@ return { [0x27EB] = { topright = -0.1, bottomright = -0.1 }, ["0x27EB.variants.*"] = { topright = -0.2, bottomright = -0.2 }, -- - + [0x00393] = { bottomright = -0.30, }, -- upright Gamma + -- ["0x222B.parts.bottom"] = { bottomright = -0.20 }, -- int ["0x222C.parts.bottom"] = { bottomright = -0.15 }, -- iint ["0x222D.parts.bottom"] = { bottomright = -0.10 }, -- iiint @@ -197,6 +197,9 @@ return { tweak = "addfourier", variant = 1, }, + { + tweak = "addprimed", + }, -- this is the mkiv section { tweak = "emulatelmtx", diff --git a/tex/context/fonts/mkiv/ebgaramond-math.lfg b/tex/context/fonts/mkiv/ebgaramond-math.lfg index afdd66279..43bafadd1 100644 --- a/tex/context/fonts/mkiv/ebgaramond-math.lfg +++ b/tex/context/fonts/mkiv/ebgaramond-math.lfg @@ -22,9 +22,15 @@ return { -- RadicalRuleThickness = 50, -- 50 in font DelimiterPercent = 90, DelimiterShortfall = 400, + SubscriptShiftDown = 200, -- 250 in font + SuperscriptShiftUp = 400, -- 430 in font + SubscriptShiftDownWithSuperscript = "1.25*SubscriptShiftDown", -- trial and error + SubSuperscriptGapMin = 200, -- 250 in font -- DisplayOperatorMinHeight = 1800, -- 1300 in font (only one) - PrimeRaisePercent = 85, -- 50 default - -- PrimeRaiseComposedPercent = 25, -- 25 default + -- PrimeRaisePercent = 0, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 25, -- set to 0 in math-act + PrimeShiftUp = "1.2*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.2*SuperscriptShiftUp", }, tweaks = { aftercopying = { @@ -34,9 +40,9 @@ return { }, { tweak = "fixprimes", - scale = 0.95, + -- scale = 1, -- smaller = true, - factor = 0.95, + factor = 0.73, -- fake = 0.75, }, { @@ -232,6 +238,9 @@ return { tweak = "addfourier", variant = 2, }, + { + tweak = "addprimed", + }, -- this is the mkiv section { tweak = "emulatelmtx", diff --git a/tex/context/fonts/mkiv/erewhon-math.lfg b/tex/context/fonts/mkiv/erewhon-math.lfg index cf2e7f28c..0cb72d149 100644 --- a/tex/context/fonts/mkiv/erewhon-math.lfg +++ b/tex/context/fonts/mkiv/erewhon-math.lfg @@ -22,14 +22,19 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, -- DisplayOperatorMinHeight = 1800, -- 1300 in font (one size) + SubscriptShiftDown = 200, -- 220 in font + SuperscriptShiftUp = 400, -- 400 in font + SubscriptShiftDownWithSuperscript = "1.2*SubscriptShiftDown", -- unclear original value, trial and error + PrimeShiftUp = "1.1*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.1*SuperscriptShiftUp", }, tweaks = { aftercopying = { { tweak = "fixprimes", -- smaller = true, -- replace multiples - factor = 0.9, - scale = 1.0, + factor = 0.94 , + -- scale = 1.0, fake = 0.85, -- replace multiples with this width proportion }, { @@ -72,6 +77,7 @@ return { ["0x27E9.variants.*"] = { topright = -0.2, bottomright = -0.2 }, [0x27EB] = { topright = -0.1, bottomright = -0.1 }, ["0x27EB.variants.*"] = { topright = -0.2, bottomright = -0.2 }, + [0x00393] = { bottomright = -0.30, }, -- upright Gamma }, }, { @@ -114,6 +120,9 @@ return { variant = 2, }, { + tweak = "addprimed", + }, + { tweak = "setoptions", -- set = { "ignorekerndimensions" } }, diff --git a/tex/context/fonts/mkiv/kpfonts-math.lfg b/tex/context/fonts/mkiv/kpfonts-math.lfg index 385170aab..7cb968860 100644 --- a/tex/context/fonts/mkiv/kpfonts-math.lfg +++ b/tex/context/fonts/mkiv/kpfonts-math.lfg @@ -21,8 +21,11 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, -- DisplayOperatorMinHeight = 1800, -- 1500 in font (only one) - PrimeRaisePercent = 25, -- 50 default - -- PrimeRaiseComposedPercent = 25, -- 25 default + -- PrimeRaisePercent = 25, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 25, -- set to 0 in math-act + PrimeShiftUp = "1.15*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.15*SuperscriptShiftUp", + SubscriptShiftDownWithSuperscript = "1.2*SubscriptShiftDown", -- unclear original value, trial and error }, tweaks = { aftercopying = { @@ -54,7 +57,7 @@ return { }, { tweak = "fixprimes", - factor = 1.4, -- accent base height + factor = 0.86, -- accent base height smaller = true, -- replace multiples scale = 1, -- glyph scale fake = 0.8, -- replace multiples with this width proportion @@ -142,6 +145,9 @@ return { variant = 2, }, { + tweak = "addprimed", + }, + { tweak = "setoptions", set = { "ignorekerndimensions" } }, diff --git a/tex/context/fonts/mkiv/libertinus-math.lfg b/tex/context/fonts/mkiv/libertinus-math.lfg index 71863fb59..30f8f6456 100644 --- a/tex/context/fonts/mkiv/libertinus-math.lfg +++ b/tex/context/fonts/mkiv/libertinus-math.lfg @@ -4,7 +4,6 @@ -- fix-feature or just disable ssty, but we have to revert to a runtime fix. Lucky me that I didn't -- remove that hardly needed 'tweaks' mechanism yet. --- The italic x has a rectangular piece in its eye (redundant point) local common = fonts.goodies.load("common-math.lfg") local presets = common.mathematics.tweaks.presets @@ -34,8 +33,10 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, -- DisplayOperatorMinHeight = 1800, -- 1250 in font (only one) - PrimeRaisePercent = 70, -- 50 default - PrimeRaiseComposedPercent = 10, -- 25 default + -- PrimeRaisePercent = 70, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 10, -- set to 0 in math-act + PrimeShiftUp = "1.2*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.2*SuperscriptShiftUp", }, tweaks = { aftercopying = { @@ -88,7 +89,8 @@ return { ["0x27E9.variants.*"] = { topright = -0.2, bottomright = -0.2 }, [0x27EB] = { topright = -0.1, bottomright = -0.1 }, ["0x27EB.variants.*"] = { topright = -0.2, bottomright = -0.2 }, - + -- + [0x00393] = { bottomright = -0.30, }, -- upright Gamma }, }, -- Accents are a mess. We migrate the extensibles from the combiners to the base accent @@ -157,10 +159,10 @@ return { { tweak = "fixprimes", - scale = 0.9, + -- scale = 1, -- smaller = true, - factor = 0.9, - fake = 0.8, + factor = 0.92, + fake = 0.75, }, { tweak = "checkspacing", @@ -189,6 +191,9 @@ return { variant = 1, }, { + tweak = "addprimed", + }, + { -- needed for integrals (bad axis) and (less prominent) fences tweak = "setoptions", set = { "ignorekerndimensions" } diff --git a/tex/context/fonts/mkiv/lucida-math.lfg b/tex/context/fonts/mkiv/lucida-math.lfg index 6842d505b..f0ab84b8b 100644 --- a/tex/context/fonts/mkiv/lucida-math.lfg +++ b/tex/context/fonts/mkiv/lucida-math.lfg @@ -28,9 +28,11 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, -- DisplayOperatorMinHeight = 1800, -- 1300 in font (only one) - PrimeRaisePercent = 60, -- 50 default - PrimeRaiseComposedPercent = 50, -- 25 default - AxisHeight = 325, -- we keep the old fonts as they are (also for demos) + SuperscriptBottomMaxWithSubscript = 325, + -- PrimeRaisePercent = 60, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 50, -- set to 0 in math-act + PrimeShiftUp = "1.4*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.4*SuperscriptShiftUp", }, tweaks = { aftercopying = { @@ -217,12 +219,13 @@ return { }, }, { - tweak = "fixprimes", - factor = 0.8, -- accent base height - -- smaller = true, -- replace multiples - scale = 1.1, -- glyph scale - fake = 0.65, -- replace multiples with this width proportion - -- keep = true, -- keep the text size prime (aka minute) + tweak = "fixprimes", + -- factor = 0.75, -- accent base height + factor = 0.7, -- accent base height + -- smaller = true, -- replace multiples + -- scale = 1.1, -- glyph scale + fake = 0.65, -- replace multiples with this width proportion + -- keep = true, -- keep the text size prime (aka minute) }, { tweak = "checkspacing", @@ -245,6 +248,9 @@ return { variant = 1, }, { + tweak = "addprimed", + }, + { tweak = "addarrows", left = 0.1, right = 0.1, diff --git a/tex/context/fonts/mkiv/minion-math.lfg b/tex/context/fonts/mkiv/minion-math.lfg index 82b0b0c33..dfc2251ec 100644 --- a/tex/context/fonts/mkiv/minion-math.lfg +++ b/tex/context/fonts/mkiv/minion-math.lfg @@ -39,9 +39,12 @@ return { -- DisplayOperatorMinHeight = 1900, -- 1250 in font -- AccentSuperscriptDrop = 100, -- AccentSuperscriptPercent = 20, - PrimeRaisePercent = 50, -- 50 default - PrimeRaiseComposedPercent = 25, -- 25 default - -- PrimeShiftUp = 0, + SubscriptShiftDown = 200, -- 250 in font + SubscriptShiftDownWithSuperscript = "1.4*SubscriptShiftDown", -- 1.5* in math-act + -- PrimeRaisePercent = 50, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 25, -- set to 0 in math-act + PrimeShiftUp = "1.15*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.15*SuperscriptShiftUp", -- PrimeBaselineDropMax = 0, -- RadicalKernAfterExtensible = 100, -- 0 default -- RadicalKernBeforeExtensible = 100, -- 0 default @@ -65,6 +68,38 @@ return { presets.moveitalics { correct = true, letters = true }, presets.moveintegrals { factor = 1.3 }, presets.wipeitalics { }, + + { + tweak = "kerns", + list = { + [0x002F] = { topleft = -0.2, bottomright = -0.2 }, + ["0x002F.variants.*"] = { topleft = -0.2, bottomright = -0.2 }, + -- No! + -- [0x0028] = { topleft = -0.1, bottomleft = -0.1 }, -- left parenthesis + -- [0x0029] = { topright = -0.1, bottomright = -0.1, all = true }, -- right parenthesis + -- + [0x007D] = { topright = -0.05, bottomright = -0.05 }, -- right brace variants + ["0x7D.variants.*"] = { topright = -0.10, bottomright = -0.10 }, -- right brace variants + ["0x7D.parts.top"] = { topright = -0.10, }, -- right brace top + ["0x7D.parts.bottom"] = { bottomright = -0.10 }, -- right brace bottom + [0x0029] = { topright = -0.15, bottomright = -0.15 }, -- right parenthesis variants + ["0x29.variants.*"] = { topright = -0.15, bottomright = -0.15 }, -- right parenthesis variants + ["0x29.parts.top"] = { topright = -0.15, }, -- right parenthesis top + ["0x29.parts.bottom"] = { bottomright = -0.15 }, -- right parenthesis bottom + [0x221A] = { topright = 0.2, bottomright = 0.2 }, -- radical + ["0x221A.variants.*"] = { topright = 0.2, bottomright = 0.2 }, + ["0x221A.parts.top"] = { topright = 0.2, }, + ["0x221A.parts.bottom"] = { bottomright = 0.2 }, + [0x27E9] = { topright = -0.1, bottomright = -0.1 }, -- angle + ["0x27E9.variants.*"] = { topright = -0.3, bottomright = -0.3 }, + [0x27EB] = { topright = -0.1, bottomright = -0.1 }, + ["0x27EB.variants.*"] = { topright = -0.2, bottomright = -0.2 }, + -- Keep as example. not needed in cambria (after all it is the reference): + [0x2A0C] = { bottomright = -0.1 }, -- iiiint does not have any ic + }, + }, + + { tweak = "simplifykerns", }, @@ -96,8 +131,8 @@ return { }, { tweak = "fixprimes", - factor = 0.95, - scale = 0.9, + factor = 1, + -- scale = 0.9, }, -- { -- tweak = "checkspacing", @@ -115,6 +150,9 @@ return { tweak = "addfourier", variant = 1, }, + { + tweak = "addprimed", + }, -- this is the mkiv section { tweak = "emulatelmtx", diff --git a/tex/context/fonts/mkiv/modern-math.lfg b/tex/context/fonts/mkiv/modern-math.lfg index 509e5a7da..4c433702c 100644 --- a/tex/context/fonts/mkiv/modern-math.lfg +++ b/tex/context/fonts/mkiv/modern-math.lfg @@ -26,14 +26,22 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, -- DisplayOperatorMinHeight = 1800, -- 1300 in font (only one) - PrimeRaisePercent = 60, - PrimeRaiseComposedPercent = 15, - -- SubSuperscriptGapMin = 160, -- 160 in font - SuperscriptBaselineDropMax = 250, -- 250 in font - -- SuperscriptBottomMaxWithSubscript = 344, -- 344 in font - -- SuperscriptBottomMin = 108, -- 108 in font - SuperscriptShiftUp = 363, -- 363 in font - SuperscriptShiftUpCramped = 89, -- 289 in font + -- SubSuperscriptGapMin = 160, -- 160 in font (4*ruleheight) + -- SubscriptBaselineDropMin = 50, -- 200 in font (multiplied by 0.59999/2.39868) + SubscriptShiftDown = 150, -- 247 in font (multiplied to be consistent with cm) + SubscriptShiftDownWithSuperscript = 247, -- relates to the previous one (see math-act) + -- SubscriptTopMax = 344, -- 344 in font .8 exheight + -- SuperscriptBaselineDropMax = 386, -- 250 in font (multiplied by 4.6333/2.99 (values in cm/values in lm)) 0 means: align the baseline of the superscript at the (top) + SuperscriptBaselineDropMax = 0.6*431, -- 250 in font (multiplied by 4.6333/2.99 (values in cm/values in lm)) + SubscriptBaselineDropMin = 0.1*431, -- 200 in font + -- SuperscriptBottomMaxWithSubscript = 344, -- 344 in font .8 exheight + -- SuperscriptBottomMin = 108, -- 108 in font .25 exheight + SuperscriptShiftUp = 413, -- 363 in font (multiplied with 4.9547/4.3536, got 413) + SuperscriptShiftUpCramped = 413, -- 289 in font (no distinction, old TeX) + PrimeShiftUp = "1.1*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.1*SuperscriptShiftUp", + -- PrimeRaisePercent = 0, + -- PrimeRaiseComposedPercent = 25, }, tweaks = { aftercopying = { @@ -53,7 +61,7 @@ return { presets.fallbacks { }, presets.moveitalics { correct = true }, presets.moveitalics { correct = true, letters = true }, - presets.moveintegrals { factor = 1.4 }, + presets.moveintegrals { factor = 1.4, icfactor = 0.8 }, -- the icfactor is due to the very slanted integral. presets.wipeanchors { }, presets.wipeitalics { }, -- these will become moveanchors @@ -279,6 +287,8 @@ return { [0x1D4B4] = { bottomright = -0.18, }, -- script Y [0x1D4B5] = { bottomright = -0.05, }, -- script Z -- + [0x00393] = { bottomright = -0.20, }, -- upright Gamma + -- ["0x7D.parts.top"] = { topright = -0.25 }, -- right brace top ["0x7D.parts.bottom"] = { bottomright = -0.25 }, -- right brace bottom ["0x7D.variants.*"] = { topright = -0.25, bottomright = -0.25 }, -- right brace variants @@ -376,9 +386,9 @@ return { }, { tweak = "fixprimes", - factor = 1.05, + factor = 0.825, -- smaller = true, - scale = 0.9, + -- scale = 1, -- fake = 0.6, }, { @@ -411,6 +421,9 @@ return { variant = 1, }, { + tweak = "addprimed", + }, + { tweak = "addequals", }, -- { diff --git a/tex/context/fonts/mkiv/newcomputermodern-math.lfg b/tex/context/fonts/mkiv/newcomputermodern-math.lfg index 2e0c3169d..bf818b077 100644 --- a/tex/context/fonts/mkiv/newcomputermodern-math.lfg +++ b/tex/context/fonts/mkiv/newcomputermodern-math.lfg @@ -70,6 +70,19 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, -- DisplayOperatorMinHeight = 1800, -- 1300 in font (only one) + -- Borrow values from lm: + -- SubSuperscriptGapMin = 160, -- 160 in font (4*ruleheight) + SubscriptBaselineDropMin = 50, -- 200 in font (multiplied by 0.59999/2.39868) + SubscriptShiftDown = 150, -- 247 in font (multiplied to be consistent with cm) + SubscriptShiftDownWithSuperscript = 247, -- relates to the previous one (see math-act) + -- SubscriptTopMax = 344, -- 344 in font .8 exheight + SuperscriptBaselineDropMax = 386, -- 250 in font (multiplied by 4.6333/2.99 (values in cm/values in lm)) + -- SuperscriptBottomMaxWithSubscript = 344, -- 344 in font .8 exheight + -- SuperscriptBottomMin = 108, -- 108 in font .25 exheight + SuperscriptShiftUp = 413, -- 363 in font (multiplied with 4.9547/4.3536, got 413) + SuperscriptShiftUpCramped = 413, -- 289 in font (no distinction, old TeX) + PrimeShiftUp = "1.1*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.1*SuperscriptShiftUp", }, tweaks = { aftercopying = { @@ -83,13 +96,36 @@ return { -- nothing yet }, }, - -- { - -- tweak = "fixprimes", - -- factor = 0.85, - -- smaller = true, - -- scale = 0.70, - -- fake = 0.6, - -- }, + { + tweak = "fixprimes", + factor = 0.825, + smaller = true, + -- scale = 0.70, + fake = 0.6, + }, + { + tweak = "kerns", + list = { + [0x0002F] = { topleft = -0.2, bottomright = -0.2 }, -- solidus + -- + [0x00393] = { bottomright = -0.20, }, -- upright Gamma + -- + ["0x7D.parts.top"] = { topright = -0.25 }, -- right brace top + ["0x7D.parts.bottom"] = { bottomright = -0.25 }, -- right brace bottom + ["0x7D.variants.*"] = { topright = -0.25, bottomright = -0.25 }, -- right brace variants + ["0x29.parts.top"] = { topright = -0.3, }, -- right parenthesis top + ["0x29.parts.bottom"] = { bottomright = -0.3 }, -- right parenthesis bottom + ["0x29.variants.*"] = { topright = -0.25, bottomright = -0.25 }, -- right parenthesis variants + ["0x221A.parts.top"] = { topright = 0.2, }, -- right radical top + ["0x221A.parts.bottom"] = { bottomright = 0.2 }, -- right radical bottom + ["0x221A.variants.*"] = { topright = 0.2, bottomright = 0.2 }, -- right radical variants + [0x27E9] = { topright = -0.2, bottomright = -0.2 }, -- angles + ["0x27E9.variants.*"] = { topright = -0.3, bottomright = -0.3 }, + [0x27EB] = { topright = -0.2, bottomright = -0.2 }, + ["0x27EB.variants.*"] = { topright = -0.3, bottomright = -0.3 }, + + }, + }, { tweak = "checkspacing", }, @@ -105,6 +141,13 @@ return { { tweak = "addrules", }, + { + tweak = "addfourier", + variant = 1, + }, + { + tweak = "addprimed", + }, }, }, bigs = { diff --git a/tex/context/fonts/mkiv/pagella-math.lfg b/tex/context/fonts/mkiv/pagella-math.lfg index 8c60759ab..4df58ad3b 100644 --- a/tex/context/fonts/mkiv/pagella-math.lfg +++ b/tex/context/fonts/mkiv/pagella-math.lfg @@ -28,10 +28,15 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, DisplayOperatorMinHeight = 1800, -- 1500 in font - PrimeRaisePercent = 75, -- 50 default - PrimeRaiseComposedPercent = 10, -- 25 default -- RadicalKernAfterExtensible = 0, -- 0 default -- RadicalKernBeforeExtensible = 0, -- 0 default + SuperscriptShiftUp = 386, -- 354 in font + SubscriptShiftDown = 200, -- 232 in font + SubscriptShiftDownWithSuperscript = "1.4*SubscriptShiftDown", -- 1.5* in math-act + -- PrimeRaisePercent = 75, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 10, -- set to 0 in math-act + PrimeShiftUp = "1.3*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.3*SuperscriptShiftUp", }, tweaks = { aftercopying = { @@ -41,9 +46,10 @@ return { }, { tweak = "fixprimes", - factor = 0.9, + -- factor = 0.7, + factor = 0.8, -- smaller = true, - scale = 0.9, + -- scale = 1, -- fake = 0.65, }, { @@ -190,6 +196,8 @@ return { [0x27EB] = { topright = -0.2, bottomright = -0.2 }, ["0x27EB.variants.*"] = { topright = -0.3, bottomright = -0.3 }, -- + [0x00393] = { bottomright = -0.20, }, -- upright Gamma + -- ["0x222B.parts.bottom"] = { bottomright = -0.20 }, -- int ["0x222C.parts.bottom"] = { bottomright = -0.15 }, -- iint ["0x222D.parts.bottom"] = { bottomright = -0.10 }, -- iiint @@ -238,6 +246,9 @@ return { variant = 1, }, { + tweak = "addprimed", + }, + { tweak = "kernpairs", list = { -- beware: we kept the italic correction in spite of punctuation class diff --git a/tex/context/fonts/mkiv/schola-math.lfg b/tex/context/fonts/mkiv/schola-math.lfg index 8666690da..d1711bce0 100644 --- a/tex/context/fonts/mkiv/schola-math.lfg +++ b/tex/context/fonts/mkiv/schola-math.lfg @@ -25,8 +25,11 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, DisplayOperatorMinHeight = 1800, -- 1333 in font - -- PrimeRaisePercent = 50, -- 50 default - PrimeRaiseComposedPercent = 10, -- 25 default + -- PrimeRaisePercent = 50, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 10, -- set to 0 in math-act + PrimeShiftUp = "1.35*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.35*SuperscriptShiftUp", + -- Can be improved: Smaller script and scriptscript, modified SuperShiftUp and SubShiftDown, ... }, tweaks = { aftercopying = { @@ -36,9 +39,9 @@ return { }, { tweak = "fixprimes", - scale = 0.85, + -- scale = 0.95, -- smaller = true, - factor = 1, + factor = 0.74, }, { tweak = "addmirrors", @@ -176,6 +179,9 @@ return { -- scale = 1.25, variant = 1, }, + { + tweak = "addprimed", + }, -- this is the mkiv section { tweak = "emulatelmtx", diff --git a/tex/context/fonts/mkiv/stixtwo-math.lfg b/tex/context/fonts/mkiv/stixtwo-math.lfg index 8e5489830..dadc70067 100644 --- a/tex/context/fonts/mkiv/stixtwo-math.lfg +++ b/tex/context/fonts/mkiv/stixtwo-math.lfg @@ -42,8 +42,10 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, DisplayOperatorMinHeight = 1800, -- 1800 in font - PrimeRaisePercent = 75, -- 50 default - PrimeRaiseComposedPercent = 10, -- 25 default + -- PrimeRaisePercent = 75, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 10, -- set to 0 in math-act + PrimeShiftUp = "1.3*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.3*SuperscriptShiftUp", }, tweaks = { aftercopying = { @@ -53,9 +55,7 @@ return { }, { tweak = "fixprimes", - scale = 1, - -- smaller = true, - factor = 1, + factor = 0.84, }, { tweak = "addmirrors", @@ -82,11 +82,11 @@ return { presets.moderntocalligraphic { }, presets.eulertocalligraphic { }, presets.xitsarabic { rscale = 0.95 }, - presets.fallbacks { }, + presets.fallbacks { }, presets.moveitalics { correct = true }, - presets.moveitalics { correct = true, letters = true }, + presets.moveitalics { correct = true, letters = true }, presets.moveintegrals { }, - presets.wipeitalics { }, + presets.wipeitalics { }, { tweak = "simplifykerns", }, @@ -216,6 +216,9 @@ return { variant = 1, }, { + tweak = "addprimed", + }, + { tweak = "addparts", list = { [0x21F4] = { diff --git a/tex/context/fonts/mkiv/termes-math.lfg b/tex/context/fonts/mkiv/termes-math.lfg index cd4536b55..cff3090a9 100644 --- a/tex/context/fonts/mkiv/termes-math.lfg +++ b/tex/context/fonts/mkiv/termes-math.lfg @@ -24,8 +24,15 @@ return { DelimiterPercent = 90, DelimiterShortfall = 400, DisplayOperatorMinHeight = 1800, -- 1300 in font (only one) - PrimeRaisePercent = 60, -- 50 default - PrimeRaiseComposedPercent = 10, -- 25 default + -- ScriptPercentScaleDown = 70, -- 74 in font (set in typescript) + -- ScriptScriptPercentScaleDown = 50, -- 50 in font (set in typescript) + SuperscriptShiftUp = 310, -- 339 + SubscriptShiftDown = 200, -- 222 + SubscriptShiftDownWithSuperscript = "1.4*SubscriptShiftDown", -- 1.5* in math-act + -- PrimeRaisePercent = 60, -- set to 0 in math-act + -- PrimeRaiseComposedPercent = 10, -- set to 0 in math-act + PrimeShiftUp = "1.35*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.35*SuperscriptShiftUp", }, tweaks = { aftercopying = { @@ -35,9 +42,9 @@ return { }, { tweak = "fixprimes", - scale = 0.85, + -- scale = 0.9, -- smaller = true, - factor = 0.95, + factor = 0.76, }, { tweak = "addmirrors", @@ -71,6 +78,8 @@ return { [0x27EB] = { topright = -0.1, bottomright = -0.1 }, ["0x27EB.variants.*"] = { topright = -0.2, bottomright = -0.2 }, -- + [0x00393] = { bottomright = -0.20, }, -- upright Gamma + -- ["0x222B.parts.bottom"] = { bottomright = -0.20 }, -- int ["0x222C.parts.bottom"] = { bottomright = -0.15 }, -- iint ["0x222D.parts.bottom"] = { bottomright = -0.10 }, -- iiint diff --git a/tex/context/fonts/mkiv/type-imp-dejavu.mkiv b/tex/context/fonts/mkiv/type-imp-dejavu.mkiv index d6fe406fb..49b788a73 100644 --- a/tex/context/fonts/mkiv/type-imp-dejavu.mkiv +++ b/tex/context/fonts/mkiv/type-imp-dejavu.mkiv @@ -45,8 +45,8 @@ \stoptypescript \starttypescript [\s!math] [dejavu,dejavu-nt,dejavu-condensed] - \checkedmapfontsize[\typescripttwo][\s!script] [.80] - \checkedmapfontsize[\typescripttwo][\s!scriptscript][.65] + \checkedmapfontsize[\typescripttwo][\s!script] [.70]%.80 + \checkedmapfontsize[\typescripttwo][\s!scriptscript][.55]%.65 \stoptypescript \starttypescript [\s!math] [dejavu] [\s!name] diff --git a/tex/context/fonts/mkiv/type-imp-modernlatin.mkiv b/tex/context/fonts/mkiv/type-imp-modernlatin.mkiv index f62855a59..c10ffa54e 100644 --- a/tex/context/fonts/mkiv/type-imp-modernlatin.mkiv +++ b/tex/context/fonts/mkiv/type-imp-modernlatin.mkiv @@ -83,9 +83,9 @@ \stoptypescript \starttypescript [\s!math] [modernlatin,modern-latin,lessmodernlatin,less-modern-latin] - \loadfontgoodies[lm] - \definefontsynonym [MathRoman] [\v!file:latinmodern-math-regular.otf] [\s!features={\s!math\mathsizesuffix,lm-math,lm-math-regular,mathextra},\s!goodies=lm] - \definefontsynonym [MathRomanBold] [\v!file:latinmodern-math-regular.otf] [\s!features={\s!math\mathsizesuffix,lm-math,lm-math-bold,mathextra},\s!goodies=lm] + \loadfontgoodies[modern-math] + \definefontsynonym [MathRoman] [\v!file:latinmodern-math.otf] [\s!features={\s!math\mathsizesuffix,lm-math,lm-math-regular,mathextra},\s!goodies=modern-math] + \definefontsynonym [MathRomanBold] [\v!file:latinmodern-math.otf] [\s!features={\s!math\mathsizesuffix,lm-math,lm-math-bold,mathextra},\s!goodies=modern-math] \stoptypescript \starttypescript [modern-latin,modernlatin,less-modern-latin,lessmodernlatin] diff --git a/tex/context/fonts/mkiv/type-imp-newcomputermodern.mkiv b/tex/context/fonts/mkiv/type-imp-newcomputermodern.mkiv index 73f7c4dbf..65a84ec5f 100644 --- a/tex/context/fonts/mkiv/type-imp-newcomputermodern.mkiv +++ b/tex/context/fonts/mkiv/type-imp-newcomputermodern.mkiv @@ -49,8 +49,8 @@ \starttypescript [\s!math][newcomputermodern] [\s!name] % \loadfontgoodies[newcomputermodern-math] ,\s!goodies=newcomputermodern-math - \definefontsynonym[\s!MathRoman] [\s!file:newcmmath-regular][\s!features={\s!math\mathsizesuffix,mathextra},\s!goodies=lm] - \definefontsynonym[\s!MathRomanBold][\s!file:newcmmath-regular][\s!features={\s!math\mathsizesuffix,newcomputermodern-math-bold,mathextra},\s!goodies=lm] + \definefontsynonym[\s!MathRoman] [\s!file:newcmmath-regular][\s!features={\s!math\mathsizesuffix,mathextra},\s!goodies=newcomputermodern-math] + \definefontsynonym[\s!MathRomanBold][\s!file:newcmmath-regular][\s!features={\s!math\mathsizesuffix,newcomputermodern-math-bold,mathextra},\s!goodies=newcomputermodern-math] \stoptypescript \starttypescript [\s!serif] [newcomputermodern-book] [\s!name] @@ -79,8 +79,8 @@ \starttypescript [\s!math][newcomputermodern-book] [\s!name] % \loadfontgoodies[newcomputermodern-math] ,\s!goodies=newcomputermodern-math - \definefontsynonym[\s!MathRoman] [\s!file:newcmmath-book][\s!features={\s!math\mathsizesuffix,newcm-mathextra,mathextra},\s!goodies=lm] - \definefontsynonym[\s!MathRomanBold][\s!file:newcmmath-book][\s!features={\s!math\mathsizesuffix,newcm-mathextra,newcomputermodern-math-bold,mathextra},\s!goodies=lm] + \definefontsynonym[\s!MathRoman] [\s!file:newcmmath-book][\s!features={\s!math\mathsizesuffix,newcm-mathextra,mathextra},\s!goodies=newcomputermodern-math] + \definefontsynonym[\s!MathRomanBold][\s!file:newcmmath-book][\s!features={\s!math\mathsizesuffix,newcm-mathextra,newcomputermodern-math-bold,mathextra},\s!goodies=newcomputermodern-math] \stoptypescript \starttypescript[newcomputermodern,newcomputermodern-book] diff --git a/tex/context/fonts/mkiv/type-imp-texgyre.mkiv b/tex/context/fonts/mkiv/type-imp-texgyre.mkiv index 87009f11e..a3f3952c7 100644 --- a/tex/context/fonts/mkiv/type-imp-texgyre.mkiv +++ b/tex/context/fonts/mkiv/type-imp-texgyre.mkiv @@ -281,8 +281,8 @@ \starttypescriptcollection[texgyre-math-termes] \starttypescript [\s!math][termes,termes-nt,times][\s!all] - \checkedmapfontsize[\typescripttwo][\s!script] [.74] - \checkedmapfontsize[\typescripttwo][\s!scriptscript][.55] + \checkedmapfontsize[\typescripttwo][\s!script] [.70]%.74 + \checkedmapfontsize[\typescripttwo][\s!scriptscript][.50]%.55 \stoptypescript \starttypescript [\s!math][termes,times][\s!all] \definefontsynonym[\s!MathRoman] [file:texgyretermes-math.otf][\s!features={\s!math\mathsizesuffix,termes:mathextra,mathextra},\s!goodies=termes-math] @@ -335,8 +335,8 @@ \starttypescriptcollection[texgyre-math-bonum] \starttypescript [\s!math][bonum,bonum-nt,bookman][\s!all] - \checkedmapfontsize[\typescripttwo][\s!script] [.77] - \checkedmapfontsize[\typescripttwo][\s!scriptscript][.60] + \checkedmapfontsize[\typescripttwo][\s!script] [.70]%.77 + \checkedmapfontsize[\typescripttwo][\s!scriptscript][.55]%.60 \stoptypescript \starttypescript [\s!math][bonum,bookman][\s!all] \definefontsynonym[\s!MathRoman] [file:texgyrebonum-math.otf][\s!features={\s!math\mathsizesuffix,bonum:mathextra,mathextra},\s!goodies=bonum-math] diff --git a/tex/context/fonts/mkiv/xcharter-math.lfg b/tex/context/fonts/mkiv/xcharter-math.lfg index be09720d1..c91ea6fa7 100644 --- a/tex/context/fonts/mkiv/xcharter-math.lfg +++ b/tex/context/fonts/mkiv/xcharter-math.lfg @@ -22,13 +22,17 @@ return { -- DelimiterPercent = 90, -- DelimiterShortfall = 400, -- DisplayOperatorMinHeight = 1800, -- 1300 in font (one size) + SubscriptShiftDown = 200, -- 300 in font + SubscriptShiftDownWithSuperscript = "1.3*SubscriptShiftDown", -- 1.5* in math-act + PrimeShiftUp = "1.2*SuperscriptShiftUp", + PrimeShiftUpCramped = "1.2*SuperscriptShiftUp", }, tweaks = { aftercopying = { { tweak = "fixprimes", - factor = 0.8, - scale = 1, + factor = 0.92, + -- scale = 1, }, { tweak = "addmirrors", @@ -70,6 +74,8 @@ return { ["0x27E9.variants.*"] = { topright = -0.2, bottomright = -0.2 }, [0x27EB] = { topright = -0.1, bottomright = -0.1 }, ["0x27EB.variants.*"] = { topright = -0.2, bottomright = -0.2 }, + -- + [0x00393] = { bottomright = -0.20, }, -- upright Gamma }, }, { diff --git a/tex/context/modules/mkiv/s-fonts-variable.lua b/tex/context/modules/mkiv/s-fonts-variable.lua index b84b78ada..e77903665 100644 --- a/tex/context/modules/mkiv/s-fonts-variable.lua +++ b/tex/context/modules/mkiv/s-fonts-variable.lua @@ -232,14 +232,16 @@ function moduledata.fonts.variable.showvariations(specification) end context.stopsubject() - local sample = specification.sample + local sample = specification.sample + local features = specification.features or "default" for i=1,#collected do local instance = collected[i] context.startsubject { title = instance } + local fontspecification = "name:" .. instance .. "*" .. features context.start() - context.definedfont { "name:" .. instance .. "*default" } + context.definedfont { fontspecification } context.start() if show_glyphs then context.showglyphs() @@ -247,6 +249,7 @@ function moduledata.fonts.variable.showvariations(specification) if show_kerns then context.showfontkerns() end +-- print("using",fontspecification) if sample and sample ~= "" then context(sample) else diff --git a/tex/context/modules/mkiv/s-fonts-variable.mkiv b/tex/context/modules/mkiv/s-fonts-variable.mkiv index 43cb1d32c..d061179eb 100644 --- a/tex/context/modules/mkiv/s-fonts-variable.mkiv +++ b/tex/context/modules/mkiv/s-fonts-variable.mkiv @@ -132,4 +132,8 @@ % \showfontvariations[font=file:goldmansanscdvf_wght.ttf] % \stoptext +\showfontvariations + [font=file:nupuram-vf.otf,features=malayalam-two,sample=test നൂപുരം] +% [font=file:nupuram-vf.ttf,features=malayalam-two,sample=test നൂപുരം] + \stoptext diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index 376fa61db..eee1dc4e2 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua -- parent file : c:/data/develop/context/sources/luatex-fonts.lua --- merge date : 2022-11-18 13:15 +-- merge date : 2022-12-01 12:38 do -- begin closure to overcome local limits and interference @@ -11306,6 +11306,7 @@ if not modules then modules={} end modules ['font-otr']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local number=number local next,type,tonumber,rawget=next,type,tonumber,rawget local byte,lower,char,gsub=string.byte,string.lower,string.char,string.gsub local fullstrip=string.fullstrip @@ -12389,7 +12390,7 @@ end function readers.cmap(f,fontdata,specification) local tableoffset=gotodatatable(f,fontdata,"cmap",specification.glyphs) if tableoffset then - local version=readushort(f) + local version=readushort(f) local noftables=readushort(f) local records={} local unicodecid=false @@ -13050,6 +13051,10 @@ function readers.loadfont(filename,n,instance) cidinfo=fontdata.cidinfo, mathconstants=fontdata.mathconstants, colorpalettes=fontdata.colorpalettes, + colorpaintdata=fontdata.colorpaintdata, + colorpaintlist=fontdata.colorpaintlist, + colorlinesdata=fontdata.colorlinesdata, + coloraffinedata=fontdata.coloraffinedata, svgshapes=fontdata.svgshapes, pngshapes=fontdata.pngshapes, variabledata=fontdata.variabledata, @@ -14833,20 +14838,7 @@ end local n=0 for i=1,top do if n==48 then - local zero=encode[0] - local res3=result[r-3] - local res2=result[r-2] - local res1=result[r-1] - local res0=result[r] - result[r-3]=zero - result[r-2]=zero r=r+1;result[r]=chars[t] - r=r+1;result[r]=zero - r=r+1;result[r]=zero - r=r+1;result[r]=res3 - r=r+1;result[r]=res2 - r=r+1;result[r]=res1 - r=r+1;result[r]=res0 n=1 else n=n+1 @@ -19275,9 +19267,10 @@ function readers.colr(f,fontdata,specification) local tableoffset=gotodatatable(f,fontdata,"colr",specification.glyphs) if tableoffset then local version=readushort(f) - if version~=0 then + if version==0 or version==1 then report("table version %a of %a is not supported (yet), maybe font %s is bad",version,"colr",fontdata.filename) return + else end if not fontdata.tables.cpal then report("color table %a in font %a has no mandate %a table","colr",fontdata.filename,"cpal") @@ -20194,16 +20187,20 @@ local scripts=allocate { ["cari"]="carian", ["cham"]="cham", ["cher"]="cherokee", + ["chrs"]="chorasmian", ["copt"]="coptic", + ["cpmn"]="cypro-minoan", ["cprt"]="cypriot syllabary", ["cyrl"]="cyrillic", ["dev2"]="devanagari variant 2", ["deva"]="devanagari", + ["diak"]="dives akuru", ["dogr"]="dogra", ["dsrt"]="deseret", ["dupl"]="duployan", ["egyp"]="egyptian heiroglyphs", ["elba"]="elbasan", + ["elym"]="elymaic", ["ethi"]="ethiopic", ["geor"]="georgian", ["gjr2"]="gujarati variant 2", @@ -20223,6 +20220,7 @@ local scripts=allocate { ["hebr"]="hebrew", ["hluw"]="anatolian hieroglyphs", ["hmng"]="pahawh hmong", + ["hmnp"]="nyiakeng puachue hmong", ["hung"]="old hungarian", ["ital"]="old italic", ["jamo"]="hangul jamo", @@ -20232,6 +20230,7 @@ local scripts=allocate { ["khar"]="kharosthi", ["khmr"]="khmer", ["khoj"]="khojki", + ["kits"]="khitan small script", ["knd2"]="kannada variant 2", ["knda"]="kannada", ["kthi"]="kaithi", @@ -20265,6 +20264,7 @@ local scripts=allocate { ["musc"]="musical symbols", ["mym2"]="myanmar variant 2", ["mymr"]="myanmar", + ["nand"]="nandinagari", ["narb"]="old north arabian", ["nbat"]="nabataean", ["newa"]="newa", @@ -20274,9 +20274,10 @@ local scripts=allocate { ["olck"]="ol chiki", ["orkh"]="old turkic and orkhon runic", ["ory2"]="odia variant 2", - ["orya"]="oriya", + ["orya"]="odia", ["osge"]="osage", ["osma"]="osmanya", + ["ougr"]="old uyghur", ["palm"]="palmyrene", ["pauc"]="pau cin hau", ["perm"]="old permic", @@ -20308,7 +20309,7 @@ local scripts=allocate { ["tagb"]="tagbanwa", ["takr"]="takri", ["tale"]="tai le", - ["talu"]="tai lu", + ["talu"]="new tai lue", ["taml"]="tamil", ["tang"]="tangut", ["tavt"]="tai viet", @@ -20320,12 +20321,16 @@ local scripts=allocate { ["thai"]="thai", ["tibt"]="tibetan", ["tirh"]="tirhuta", + ["tnsa"]="tangsa", ["tml2"]="tamil variant 2", + ["toto"]="toto", ["ugar"]="ugaritic cuneiform", ["vai" ]="vai", ["wara"]="warang citi", + ["wcho"]="wancho", ["xpeo"]="old persian cuneiform", ["xsux"]="sumero-akkadian cuneiform", + ["yezi"]="yezidi", ["yi" ]="yi", ["zanb"]="zanabazar square", } @@ -20340,6 +20345,7 @@ local languages=allocate { ["agw" ]="agaw", ["aio" ]="aiton", ["aka" ]="akan", + ["akb" ]="batak angkola", ["als" ]="alsatian", ["alt" ]="altai", ["amh" ]="amharic", @@ -20352,6 +20358,7 @@ local languages=allocate { ["asm" ]="assamese", ["ast" ]="asturian", ["ath" ]="athapaskan", + ["avn" ]="avatime", ["avr" ]="avar", ["awa" ]="awadhi", ["aym" ]="aymara", @@ -20397,8 +20404,12 @@ local languages=allocate { ["brx" ]="bodo", ["bsh" ]="bashkir", ["bsk" ]="burushaski", + ["bta" ]="batak alas kluet", + ["btd" ]="batak dairi (pakpak)", ["bti" ]="beti", + ["btm" ]="batak mandailing", ["bts" ]="batak simalungun", + ["btx" ]="batak karo", ["bug" ]="bugis", ["byv" ]="medumba", ["cak" ]="kaqchikel", @@ -20433,14 +20444,16 @@ local languages=allocate { ["csl" ]="church slavonic", ["csy" ]="czech", ["ctg" ]="chittagonian", + ["ctt" ]="wayanad chetti", ["cuk" ]="san blas kuna", + ["dag" ]="dagbani", ["dan" ]="danish", ["dar" ]="dargwa", ["dax" ]="dayi", ["dcr" ]="woods cree", ["deu" ]="german", - ["dgo" ]="dogri", - ["dgr" ]="dogri", + ["dgo" ]="dogri (individual language)", + ["dgr" ]="dogri (macro language)", ["dhg" ]="dhangu", ["dhv" ]="divehi (dhivehi, maldivian)", ["diq" ]="dimli", @@ -20512,13 +20525,16 @@ local languages=allocate { ["guj" ]="gujarati", ["guz" ]="gusii", ["hai" ]="haitian (haitian creole)", + ["hai0"]="haida", ["hal" ]="halam", ["har" ]="harauti", ["hau" ]="hausa", ["haw" ]="hawaiian", ["hay" ]="haya", ["haz" ]="hazaragi", + ["hmz" ]="hmong shuat", ["hbn" ]="hammer-banna", + ["hei" ]="heiltsuk", ["her" ]="herero", ["hil" ]="hiligaynon", ["hin" ]="hindi", @@ -20543,10 +20559,12 @@ local languages=allocate { ["ind" ]="indonesian", ["ing" ]="ingush", ["inu" ]="inuktitut", + ["inuk"]="nunavik inuktitut", ["ipk" ]="inupiat", ["ipph"]="phonetic transcription—ipa conventions", ["iri" ]="irish", ["irt" ]="irish traditional", + ["uri" ]="irula", ["isl" ]="icelandic", ["ism" ]="inari sami", ["ita" ]="italian", @@ -20566,6 +20584,7 @@ local languages=allocate { ["kan" ]="kannada", ["kar" ]="karachay", ["kat" ]="georgian", + ["kaw" ]="kawi (old javanese)", ["kaz" ]="kazakh", ["kde" ]="makonde", ["kea" ]="kabuverdianu (crioulo)", @@ -20623,6 +20642,7 @@ local languages=allocate { ["kur" ]="kurdish", ["kuu" ]="kurukh", ["kuy" ]="kuy", + ["kwk" ]="kwakʼwala", ["kyk" ]="koryak", ["kyu" ]="western kayah", ["lad" ]="ladin", @@ -20634,6 +20654,7 @@ local languages=allocate { ["laz" ]="laz", ["lcr" ]="l-cree", ["ldk" ]="ladakhi", + ["lef" ]="lelemi", ["lez" ]="lezgi", ["lij" ]="ligurian", ["lim" ]="limburgish", @@ -20646,6 +20667,7 @@ local languages=allocate { ["lmo" ]="lombard", ["lmw" ]="lomwe", ["lom" ]="loma", + ["lpo" ]="lipo", ["lrc" ]="luri", ["lsb" ]="lower sorbian", ["lsm" ]="lule sami", @@ -20662,7 +20684,7 @@ local languages=allocate { ["mah" ]="marshallese", ["maj" ]="majang", ["mak" ]="makhuwa", - ["mal" ]="malayalam reformed", + ["mal" ]="malayalam", ["mam" ]="mam", ["man" ]="mansi", ["map" ]="mapudungun", @@ -20697,6 +20719,7 @@ local languages=allocate { ["mok" ]="moksha", ["mol" ]="moldavian", ["mon" ]="mon", + ["mnw" ]="thailand mon", ["mor" ]="moroccan", ["mos" ]="mossi", ["mri" ]="maori", @@ -20735,7 +20758,7 @@ local languages=allocate { ["nor" ]="norwegian", ["nov" ]="novial", ["nsm" ]="northern sami", - ["nso" ]="sotho, northern", + ["nso" ]="northern sotho", ["nta" ]="northern tai", ["nto" ]="esperanto", ["nym" ]="nyamwezi", @@ -20783,6 +20806,7 @@ local languages=allocate { ["rbu" ]="russian buriat", ["rcr" ]="r-cree", ["rej" ]="rejang", + ["rhg" ]="rohingya", ["ria" ]="riang", ["rif" ]="tarifit", ["rit" ]="ritarungo", @@ -20807,6 +20831,7 @@ local languages=allocate { ["scs" ]="north slavey", ["sek" ]="sekota", ["sel" ]="selkup", + ["sfm" ]="small flowery miao", ["sga" ]="old irish", ["sgo" ]="sango", ["sgs" ]="samogitian", @@ -20828,7 +20853,7 @@ local languages=allocate { ["snk" ]="soninke", ["sog" ]="sodo gurage", ["sop" ]="songe", - ["sot" ]="sotho, southern", + ["sot" ]="southern sotho", ["sqi" ]="albanian", ["srb" ]="serbian", ["srd" ]="sardinian", @@ -20869,7 +20894,9 @@ local languages=allocate { ["tht" ]="tahitian", ["tib" ]="tibetan", ["tiv" ]="tiv", + ["tj;" ]="tai laing", ["tkm" ]="turkmen", + ["tli" ]="tlingit", ["tmh" ]="tamashek", ["tmn" ]="temne", ["tna" ]="tswana", @@ -20883,7 +20910,7 @@ local languages=allocate { ["tsj" ]="tshangla", ["tua" ]="turoyo aramaic", ["tul" ]="tulu", - ["tum" ]="tulu", + ["tum" ]="tumbuka", ["tuv" ]="tuvin", ["tvl" ]="tuvalu", ["twi" ]="twi", @@ -20905,6 +20932,7 @@ local languages=allocate { ["wa" ]="wa", ["wag" ]="wagdi", ["war" ]="waray-waray", + ["wci" ]="waci gbe", ["wcr" ]="west-cree", ["wel" ]="welsh", ["wlf" ]="wolof", @@ -20916,17 +20944,23 @@ local languages=allocate { ["xkf" ]="khengkha", ["xog" ]="soga", ["xpe" ]="kpelle (liberia)", + ["xub" ]="bette kuruma", + ["xuj" ]="jennu kuruma", ["yak" ]="sakha", ["yao" ]="yao", ["yap" ]="yapese", ["yba" ]="yoruba", ["ycr" ]="y-cree", + ["ygp" ]="gepo", ["yic" ]="yi classic", ["yim" ]="yi modern", + ["yna" ]="aluo", + ["ywq" ]="wuding-luquan", ["zea" ]="zealandic", ["zgh" ]="standard morrocan tamazigh", ["zha" ]="zhuang", ["zhh" ]="chinese, hong kong sar", + ["zho" ]="chinese traditional, macao", ["zhp" ]="chinese phonetic", ["zhs" ]="chinese simplified", ["zht" ]="chinese traditional", @@ -20950,6 +20984,7 @@ local features=allocate { ["case"]="case-sensitive forms", ["ccmp"]="glyph composition/decomposition", ["cfar"]="conjunct form after ro", + ["chws"]="contextual half-width spacing", ["cjct"]="conjunct forms", ["clig"]="contextual ligatures", ["cpct"]="centered cjk punctuation", @@ -21026,8 +21061,8 @@ local features=allocate { ["rtbd"]="right bounds", ["rtla"]="right-to-left alternates", ["rtlm"]="right to left mirrored forms", - ["rvrn"]="required variation alternates", ["ruby"]="ruby notation forms", + ["rvrn"]="required variation alternates", ["salt"]="stylistic alternates", ["sinf"]="scientific inferiors", ["size"]="optical size", @@ -21047,6 +21082,7 @@ local features=allocate { ["unic"]="unicase", ["valt"]="alternate vertical metrics", ["vatu"]="vattu variants", + ["vchw"]="vertical contextual half-width spacing", ["vert"]="vertical writing", ["vhal"]="alternate vertical half metrics", ["vjmo"]="vowel jamo forms", |