From de678e954438110850d93c5f35a0d6bf14129901 Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Sun, 13 Jan 2013 23:10:00 +0100 Subject: beta 2013.01.13 23:10 --- .../scite/scite-context-data-interfaces.properties | 1580 +- .../scite/scite-context-data-metafun.properties | 14 +- .../scite/scite-context-data-metapost.properties | 24 +- .../data/scite/scite-context-data-tex.properties | 124 +- context/data/scite/scite-ctx.lua | 46 +- context/data/scite/scite-ctx.properties | 3 + scripts/context/lua/mtx-context.lua | 14 +- scripts/context/lua/mtxrun.lua | 29844 ++++++++----------- scripts/context/stubs/mswin/mtxrun.lua | 29844 ++++++++----------- scripts/context/stubs/unix/mtxrun | 29844 ++++++++----------- tex/context/base/bibl-tra.mkiv | 15 +- tex/context/base/char-def.lua | 29 +- tex/context/base/cont-new.mkii | 2 +- tex/context/base/cont-new.mkiv | 2 +- tex/context/base/context-version.pdf | Bin 0 -> 4126 bytes tex/context/base/context-version.png | Bin 40128 -> 39975 bytes tex/context/base/context.mkii | 2 +- tex/context/base/context.mkiv | 6 +- tex/context/base/data-ini.lua | 2 +- tex/context/base/data-lua.lua | 2 +- tex/context/base/data-res.lua | 2 +- tex/context/base/file-job.lua | 2 +- tex/context/base/font-con.lua | 2 + tex/context/base/font-otn.lua | 5 +- tex/context/base/l-file.lua | 33 +- tex/context/base/l-lpeg.lua | 45 +- tex/context/base/l-string.lua | 2 +- tex/context/base/luat-env.lua | 4 +- tex/context/base/luat-fmt.lua | 15 +- tex/context/base/math-arr.mkiv | 4 + tex/context/base/math-def.mkiv | 2 +- tex/context/base/meta-ini.mkiv | 15 +- tex/context/base/mlib-pdf.mkiv | 20 +- tex/context/base/mlib-pps.lua | 211 +- tex/context/base/node-aux.lua | 34 +- tex/context/base/node-fnt.lua | 4 +- tex/context/base/node-pro.lua | 43 +- tex/context/base/page-mix.mkiv | 7 + tex/context/base/scrp-ini.lua | 4 +- tex/context/base/spac-flr.mkiv | 112 + tex/context/base/spac-ver.lua | 42 +- tex/context/base/status-files.pdf | Bin 24749 -> 24750 bytes tex/context/base/status-lua.pdf | Bin 208079 -> 208234 bytes tex/context/base/trac-inf.lua | 9 + tex/context/base/type-ini.mkvi | 6 +- tex/context/base/util-mrg.lua | 109 +- tex/context/base/util-seq.lua | 11 +- tex/generic/context/luatex/luatex-fonts-merged.lua | 89 +- 48 files changed, 40358 insertions(+), 51871 deletions(-) create mode 100644 tex/context/base/context-version.pdf create mode 100644 tex/context/base/spac-flr.mkiv diff --git a/context/data/scite/scite-context-data-interfaces.properties b/context/data/scite/scite-context-data-interfaces.properties index ce6a3eba4..0c734e6e9 100644 --- a/context/data/scite/scite-context-data-interfaces.properties +++ b/context/data/scite/scite-context-data-interfaces.properties @@ -1,312 +1,167 @@ -keywordclass.context.en=\ -CAP Cap Caps Character \ -Characters MONTH Numbers Romannumerals WEEKDAY \ -WORD WORDS Word Words about \ -adaptlayout adding appendix arg at \ -atleftmargin atpage atrightmargin background backspace \ -blackrule blackrules blank bookmark bottomdistance \ -bottomheight bottomspace but button bypassblocks \ -cap chapter character characters chem \ -clip clonefield color colorbar colorvalue \ -column comment comparecolorgroup comparepalet completecombinedlist \ -completelistoffloats completelistofsorts completelistofsynonyms completepagenumber completeregister \ -component convertnumber copyfield correctwhitespace coupledocument \ -coupledregister couplemarking couplepage couplepaper coupleregister \ -crlf currentdate currentheadnumber cutspace date \ -decouplemarking decrementnumber define defineaccent defineblank \ -defineblock definebodyfont definebodyfontDEF definebodyfontREF definebodyfontenvironment \ -definebuffer definecharacter definecolor definecolorgroup definecolumnbreak \ -definecolumnset definecombination definecombinedlist definecommand defineconversion \ -definedescription definedfont defineenumeration definefield definefieldstack \ -definefiguresymbol definefloat definefont definefontfeature definefonthandling \ -definefontstyle definefontsynonym defineframed defineframedtext definehbox \ -definehead defineindentedtext defineinmargin defineinteractionmenu defineitemgroup \ -definelabel definelayer definelayout definelist definelogo \ -definemainfield definemakeup definemarking definemathalignment defineoutput \ -defineoverlay definepagebreak definepalet definepapersize defineparagraphs \ -defineplacement defineprofile defineprogram definerawfont definereference \ -definereferenceformat definereferencelist defineregister definerule definesection \ -definesectionblock definesorting definestartstop definestyle definesubfield \ -definesymbol definesynonyms definetabletemplate definetabulate definetext \ -definetextposition definetextvariable definetype definetypeface definetyping \ -defineversion description determineheadnumber determinelistcharacteristics determineregistercharacteristics \ -dimension disableinteractionmenu domicile donttest edgedistance \ -edgewidth emptylines enumeration environment externalfigure \ -fact field fieldstack fillinfield fillinline \ -fillinrules fillintext fitfield fixedspace fixedspaces \ -followprofile followprofileversion followversion footerdistance footerheight \ -footnote footnotetext forceblocks formulanumber fraction \ -framed framedtext from getbuffer getmarking \ -getnumber godown goto gotobox gotopage \ -graycolor greyvalue grid hairline head \ -headerdistance headerheight headlevel headnumber headsym \ -headtext hideblocks high hl immediatebetweenlist \ -immediatetolist in incrementnumber indentation indenting \ -inframed infull ininner inleft inleftedge \ -inleftmargin inline inmaframed inmargin inneredgedistance \ -inneredgewidth innermargindistance innermarginwidth inothermargin inouter \ -inright inrightedge inrightmargin installlanguage interactionbar \ -interactionbuttons interactionmenu item items its \ -keepblocks label labeling labels labeltext \ -language leftaligned leftedgedistance leftedgewidth leftmargindistance \ -leftmarginwidth leg linethickness listheight listlength \ -listsymbol listwidth loadsorts loadsynonyms logfields \ -lohi low maframed mainlanguage makeupheight \ -makeupwidth mapfontsize mar margindistance marginrule \ -margintext margintitle marginwidth marginword marking \ -markversion mathematics mediaeval menubutton midaligned \ -mirror month moveformula moveongrid movesidefloat \ -name navigating nextsection nocap nodimension \ -noheaderandfooterlines noindenting nolist nomarking nomoreblocks \ -nomorefiles nop nospace note notopandbottomlines \ -nowhitespace numberofsubpages numbers outeredgedistance outeredgewidth \ -outermargindistance outermarginwidth overbar overbars overstrike \ -overstrikes packed page pagedepth pagenumber \ -pageoffset pagereference paperheight paperwidth paragraph \ -part periods placebookmarks placecombinedlist placefloat \ -placefootnotes placeformula placeheadnumber placeheadtext placelegend \ -placelist placelistoffloats placelistofsorts placelistofsynonyms placelocalfootnotes \ -placelogos placeongrid placeontopofeachother placepagenumber placerawlist \ -placereferencelist placeregister placerule placesidebyside placesubformula \ -placetextvariable position positiontext printpaperheight printpaperwidth \ -processblocks processpage product program project \ -publication quotation quote ran redo \ -ref reference referral referraldate referring \ -register remark reservefloat reset resetmarking \ -resetnumber resettextcontent rightaligned rightedgedistance rightedgewidth \ -rightmargindistance rightmarginwidth romannumerals rotate scale \ -screen section seeregister selectblocks selectpaper \ -selectversion setnumber settextcontent settextvariable setupalign \ -setupanswerarea setuparranging setupbackground setupbackgrounds setupblackrules \ -setupblank setupblock setupbodyfont setupbodyfontenvironment setupbottom \ -setupbottomtexts setupbuffer setupbuttons setupcapitals setupcaption \ -setupcaptions setupclipping setupcolor setupcolors setupcolumns \ -setupcolumnset setupcolumnsetlines setupcolumnsetstart setupcombinations setupcombinedlist \ -setupcomment setupdescriptions setupenumerations setupexternalfigures setupfield \ -setupfields setupfillinlines setupfillinrules setupfloat setupfloats \ -setupfloatsplitting setupfonthandling setupfontsynonym setupfooter setupfootertexts \ -setupfootnotedefinition setupfootnotes setupforms setupformulae setupframed \ -setupframedtexts setuphead setupheader setupheadertexts setupheadnumber \ -setupheads setupheadtext setuphyphenmark setupindentedtext setupindenting \ -setupinmargin setupinteraction setupinteractionbar setupinteractionmenu setupinteractionscreen \ -setupinterlinespace setupinterlinespace2 setupitemgroup setupitemizations setupitems \ -setuplabeltext setuplanguage setuplayout setuplegend setuplinenumbering \ -setuplines setuplinewidth setuplist setuplistalternative setupmakeup \ -setupmarginblocks setupmarginrules setupmarking setupmathalignment setupnarrower \ -setupnumber setupnumbering setupoppositeplacing setupoutput setuppagecomment \ -setuppagenumber setuppagenumbering setuppagetransitions setuppalet setuppaper \ -setuppapersize setupparagraphnumbering setupparagraphs setupplacement setuppositioning \ -setupprofiles setupprograms setuppublications setupquote setupreferencelist \ -setupreferencing setupregister setuprotate setuprule setups \ -setupscreens setupsection setupsectionblock setupsorting setupspacing \ -setupstartstop setupstrut setupsubpagenumber setupsymbolset setupsynchronization \ -setupsynchronizationbar setupsynonyms setupsystem setuptab setuptables \ -setuptabulate setuptext setuptextposition setuptextrules setuptexttexts \ -setuptextvariable setupthinrules setuptolerance setuptop setuptoptexts \ -setuptype setuptyping setupunderbar setupurl setupversions \ -setupwhitespace showbodyfont showbodyfontenvironment showcolor showcolorgroup \ -showexternalfigures showfields showframe showgrid showlayout \ -showmakeup showpalet showprint showsetups showstruts \ -showsymbolset someline somewhere sort space \ -splitfloat startalignment startbackground startbuffer startcoding \ -startcolor startcolumnmakeup startcolumns startcolumnset startcombination \ -startcomment startcomponent startdescription startdocument startenumeration \ -startenvironment startfact startfigure startfloattext startformula \ -startframedtext startglobal starthiding startinteractionmenu startitemgroup \ -startlegend startline startlinecorrection startlinenumbering startlines \ -startlocal startlocalenvironment startlocalfootnotes startmakeup startmarginblock \ -startmarginrule startnamemakeup startnarrower startopposite startoverlay \ -startoverview startpacked startparagraph startpositioning startpostponing \ -startproduct startprofile startproject startquotation startraster \ -startregister startsymbolset startsynchronization starttable starttables \ -starttabulate starttext starttextrule starttyping startunpacked \ -startversion stopalignment stopbackground stopbuffer stopcoding \ -stopcolor stopcolumnmakeup stopcolumns stopcolumnset stopcombination \ -stopcomment stopcomponent stopdescription stopdocument stopenumeration \ -stopenvironment stopfact stopfigure stopfloattext stopformula \ -stopframedtext stopglobal stophiding stopinteractionmenu stopitemgroup \ -stoplegend stopline stoplinecorrection stoplinenumbering stoplines \ -stoplocal stoplocalenvironment stoplocalfootnotes stopmakeup stopmarginblock \ -stopmarginrule stopnamemakeup stopnarrower stopopposite stopoverlay \ -stopoverview stoppacked stopparagraph stoppositioning stoppostponing \ -stopproduct stopprofile stopproject stopquotation stopraster \ -stopsymbolset stopsynchronization stoptable stoptables stoptabulate \ -stoptext stoptextrule stoptyping stopunpacked stopversion \ -stretched sub subformulanumber subject subpagenumber \ -subsection subsubject subsubsection subsubsubject switchtobodyfont \ -switchtorawfont sym symbol symoffset synchronizationbar \ -synchronize synonym tab testcolumn testpage \ -tex textheight textreference textrule textvariable \ -textwidth thinrule thinrules title tooltip \ -topdistance topheight topspace totalnumberofpages translate \ -txt typ type typebuffer typefile \ -underbar underbars unitmeaning unknown useJSscripts \ -useURL useXMLfilter useblocks usecommands usedirectory \ -useencoding useexternaldocument useexternalfigure useexternalfile useexternalfiles \ -useexternalsoundtrack usemodule usemodules usereferences usespecials \ -usesymbols usetypescript usetypescriptfile useurl version \ -vl weekday whitespace wordright writebetweenlist \ -writetolist writetoreferencelist writetoregister - -keywordclass.context.ro=\ -CAP CUVANT CUVINTE Cap \ -Caps Cuvant Cuvinte KAP Kap \ -Kaps LUNA Litera Litere Numere \ -Numereromane ZIDINSAPTAMANA adapteazaaspect adubuffer adumarcaje \ -afiseazaaspect afiseazacampuri afiseazaculoare afiseazafiguriexterne afiseazafonttext \ -afiseazagrid afiseazagrupculoare afiseazamakeup afiseazamediufonttext afiseazapaleta \ -afiseazarama afiseazasetari afiseazasetsimboluri afiseazastruts afiseazatiparire \ -aliniat aliniatcentru aliniatdreapta aliniatstanga appendix \ -arg ascundeblocuri atleftmargin atrightmargin baraculoare \ -barainteractiune barasincronizare blanc but butoaneinteractiune \ -buton butonmeniu camp campumplere cap \ -chapter chem citat clip cloneazacamp \ -coloana comment comparagrupculoare comparapaleta completeazanumarpagina \ -completecombinedlist completelistoffloats completelistofsorts completelistofsynonyms completeregister \ -componenta convertestenumar copiazacamp corecteazaspatiualb coupledregister \ -crlf culoare culoaregri cupleazadocument cupleazamarcaje \ -cupleazaregistru cutspace cuvantdreapta cuvantmarginal data \ -datacurenta datareferit decrementnumber decupleazamarcaje definebodyfontDEF \ -definebodyfontREF definecolumnbreak definecolumnset definecombination definedfont \ -definefontfeature definefonthandling defineindentedtext defineinmargin defineitemgroup \ -definelayer definelayout definemathalignment definepagebreak defineplacement \ -defineste definesteaccent definesteantet definesteblanc definestebloc \ -definesteblocsectiune definestebuffer definestecamp definestecampprincipal definestecaracter \ -definestecomanda definesteconversie definesteculoare definestedescriere definestedimensiunehartie \ -definesteenumerare definesteeticheta definestefloat definestefont definestefontraw \ -definestefonttext definesteformatreferinte definestegrupculori definestehbox definesteinconjurare \ -definestelista definestelistacombinata definestelistareferinte definestelogo definestemakeup \ -definestemarcaje definestemediulfonttext definestemeniuinteractiune definesteoutput definesteoverlay \ -definestepaleta definesteparagraf definestepozitietext definesteprofil definesteprogram \ -definestereferinte definesteregistru definesterigla definestesablontabel definestesectiune \ -definestesimbol definestesimbolfigura definestesinonim definestesinonimfont definestesortare \ -definestestartstop definestestil definestestilfont definestestivacampuri definestesubcamp \ -definestetabulatori definestetext definestetexteinconjurate definestetextinconjurat definestetyping \ -definestevariabilatext definesteversiune definetype definetypeface description \ -despre determinacaracteristicilelistei determinacaracteristiciregistru determinanumartitlu dezactiveazameniuinteractiune \ -dimensiune din distantaantet distantacolt distantacoltdreapta \ -distantacoltstanga distantajos distantamargine distantamarginedreapta distantamarginestanga \ -distantasubsol distantasus domiciliu dute dutebox \ -dutepagina ecran el element emptylines \ -enumeration eticheta etichete fact faraaliniat \ -faradimensiune farafisiere faraliniiantetsisubsol faraliniisussijos faralista \ -faramarcaje faraspatiu faraspatiualb figuraexterna firdepar \ -folosesteURL folosestebloc folosestecodificarea folosestecomenzi folosestedirector \ -folosestedocumentextern folosestefiguraexterna folosestefisiereexterne folosestefisierextern folosestemodul \ -folosestemodule folosestemuzicaexterna folosestereferinte folosestescriptJS folosestesimboluri \ -folosestespeciale folosesteurl footnotetext forteazablocuri fractie \ -framed framedtext fundal gatablocuri getnumber \ -grid grosimelinie hartiedubla headsym hl \ -immediatebetweenlist immediatetolist impachetat impartefloat in \ -inalt inaltamargine inaltimeantet inaltimehartie inaltimehartieimprimanta \ -inaltimejos inaltimelista inaltimemakeup inaltimesubsol inaltimesus \ -inaltimetext indentation indreapta inframed ininner \ -injos inlinie inmaframed inmargineadreapta inmargineastanga \ -inneredgedistance inneredgewidth innermargindistance innermarginwidth inouter \ -inparteadreapta inparteastanga instalarelimba instanga intins \ -jos jossus kap la labeling \ -lapagina latimecoltdreapta latimecoltstanga latimecolturi latimehartie \ -latimehartieimprimanta latimelista latimemakeup latimemargine latimemarginedreapta \ -latimemarginestanga latimetext leg limba limbaprincipala \ -liniemargine linieneagra liniesubtire linieumplere liniinegre \ -liniisubtiri listsymbol litera litere loadsorts \ -loadsynonyms logcampuri luna lungimelista maframed \ -mapfontsize mar marcaje marcheazaversiune marginal \ -matematica mediaeval mediu meniuinteractiune minicitat \ -moveformula movesidefloat mutapegrid name navigating \ -necunoscut nextsection niveltitlu nocap nokap \ -nop nota notasubsol numarformula numarincrement \ -numarpagina numarsubformula numartitlu numartitlucurent numartotalpagini \ -numberofsubpages nume numere numereromane numeunitate \ -nutesta olinie outeredgedistance outeredgewidth outermargindistance \ -outermarginwidth overbar overbars overstrike overstrikes \ -pagedepth pageoffset pagina paginadubla paragraph \ -part pastreazablocuri pelung placefloat placeheadnumber \ -placeheadtext placelistoffloats placelistofsorts placelistofsynonyms placerawlist \ -placereferencelist plaseazapegrid plaseazasemnecarte potrivestecamp pozitie \ -pozitietext proceseazabloc proceseazapagina produs program \ -proiect publicatie puncte punedeasuprafiecareia punefatainfata \ -puneformula punelegenda punelista punelistacombinata punelogouri \ -punenotesubsol punenotesubsollocale punenumarpagina puneregistru punerigla \ -punesubformula punevariabilatext ran ref refa \ -referinta referintapagina referintatext referit referring \ -reflexie register remarca reservefloat reset \ -reseteazamarcaje resetnumber resettextcontent riglatext rigleumplere \ -roteste saripesteblocuri scala scriebuffer scrieinlista \ -scrieinlistareferinte scrieinregistru scrieintreliste section seeregister \ -selecteazablocuri selecteazahartie selecteazaversiune semncarte setarebarasincronizare \ -setareitemization setarelimba setareoutput setarepozitie setaresincronizare \ -setari seteazaaliniat seteazaalinierea seteazaantet seteazaaranjareapag \ -seteazaaspect seteazabarainteractiune seteazablanc seteazabloc seteazablocsectiune \ -seteazablocurimarginale seteazabuffer seteazabutoane seteazacamp seteazacampuri \ -seteazaclipping seteazacoloane seteazacombinari seteazacomentariu seteazacomentariupagina \ -seteazaculoare seteazaculori seteazadefinireanotasubsol seteazadescriere seteazadimensiunihartie \ -seteazaecrane seteazaecraninteractiune seteazaelemente seteazaenumerare seteazafiguriexterne \ -seteazafloat seteazafloats seteazafonttext seteazaformulare seteazaformule \ -seteazafundal seteazafundaluri seteazagrosimelinie seteazaimpartireafloat seteazainconjurat \ -seteazaingust seteazainteractiunea seteazajos seteazalegenda seteazalegendele \ -seteazaliniesilabe seteazaliniesubtire seteazalinii seteazaliniimargine seteazaliniinegre \ -seteazaliniiumplere seteazalista seteazalistacombinata seteazalistareferinte seteazamajuscule \ -seteazamakeup seteazamarcaje seteazamarginal seteazamediulfonttext seteazameniuinteractiune \ -seteazaminicitat seteazanotasubsol seteazanumarpagina seteazanumarsubpagina seteazanumartitlu \ -seteazanumerotare seteazanumerotarelinii seteazanumerotarepagina seteazanumerotareparagrafe seteazapaleta \ -seteazaparagrafe seteazaplasareaopozita seteazapozitietext seteazaprofile seteazaprograme \ -seteazapublicatii seteazareferinte seteazaregistru seteazarigla seteazarigletext \ -seteazarigleumplere seteazarotare seteazasectiune seteazasimbol seteazasinonime \ -seteazasistem seteazasortare seteazaspatiu seteazaspatiualb seteazaspatiuinterliniar \ -seteazastrut seteazasublinie seteazasubsol seteazasus seteazatab \ -seteazatabele seteazatabulatori seteazatext seteazatexteantet seteazatextejos \ -seteazatextesubsol seteazatextesus seteazatextetext seteazatexteticheta seteazatexttitlu \ -seteazatitlu seteazatitluri seteazatoleranta seteazatranzitiepagina seteazatype \ -seteazatyping seteazaurl seteazavariabilatext seteazaversiuni setnumber \ -settextcontent setupanswerarea setupcolumnset setupcolumnsetlines setupcolumnsetstart \ -setupfonthandling setupfontsynonym setupindentedtext setupinterlinespace2 setupitemgroup \ -setuplistalternative setupmathalignment setupnumber setuppaper setupplacement \ -setupstartstop setvariabilatext sim simbol sincronizeaza \ -sort spatiifixate spatiu spatiualb spatiufixat \ -spatiujos spatiuspate spatiusus startalignment startaliniere \ -startblocmarginal startbuffer startcitat startcodificare startcoloane \ -startcolumnmakeup startcolumns startcolumnset startcombinare startcombination \ -startcomment startcomponenta startcorectielinie startculoare startdescription \ -startdocument startenumeration startfact startfigura startfigure \ -startfloattext startformula startframedtext startfundal startglobal \ -starthiding startimpachetat startingust startitemgroup startlegend \ -startline startlinecorrection startlinenumbering startlines startlinie \ -startliniemargine startlinii startlocal startlocalenvironment startlocalfootnotes \ -startmakeup startmarginblock startmediu startmeniuinteractiune startnamemakeup \ -startnarrower startneimpachetat startnotesubsollocale startnumerotarelinii startopozit \ -startopposite startoverlay startoverview startparagraph startpositioning \ -startpostponing startpozitionare startprodus startprofil startprofile \ -startproiect startraster startregister startriglatext startsetsimboluri \ -startsincronizare startsymbolset startsynchronization starttabel starttabele \ -starttable starttables starttabulate starttext starttyping \ -startunpacked startversiune stivacampuri stopalignment stopaliniere \ -stopblobal stopblocmarginal stopbuffer stopcitat stopcodificare \ -stopcoloane stopcolumnmakeup stopcolumns stopcolumnset stopcombinare \ -stopcombination stopcomment stopcomponenta stopcorectielinie stopculoare \ -stopdescription stopdocument stopenumeration stopfact stopfigure \ -stopfloattext stopformula stopframedtext stopfundal stophiding \ -stopimpachetat stopingust stopitemgroup stoplegend stopline \ -stoplinecorrection stoplinenumbering stoplines stoplinie stopliniemargine \ -stoplinii stoplocal stoplocalenvironment stoplocalfootnotes stopmakeup \ -stopmarginblock stopmediu stopmeniuinteractiune stopnamemakeup stopnarrower \ -stopneimpachetat stopnotesubsollocale stopnumerotarelinii stopopozit stopopposite \ -stopoverlay stopoverview stopparagraph stoppositioning stoppostponing \ -stoppozitionare stopprodus stopprofil stopprofile stopproiect \ -stopraster stopriglatext stopsincronizare stopsymbolset stopsynchronization \ -stoptabel stoptabele stoptable stoptables stoptabulate \ -stoptext stoptyping stopunpacked stopversiune sub \ -subject subpagenumber subsection subsubject subsubsection \ -subsubsubject synonym tab testcolumn testpage \ -tex texteticheta textmarginal texttitlu textumplere \ -title titlu titlumarginal tooltip traduce \ -trecilafontraw trecilafonttext txt typ type \ -typefile underbar underbars undeva urmeazaprofil \ -urmeazaversiune urmeazaversiuneprofil useXMLfilter usedirectory usetypescript \ -usetypescriptfile valoareculoare valoaregri variabilatext versiune \ -vl zidinsaptamana - -keywordclass.context.pe=\ +keywordclass.context.fr=\ +CAP Cap Caps Caractere \ +Caracteres Chiffresromains JOURSEMAINE MOIS MOT \ +MOTS Mot Mots Numeros a \ +adaptedisposition affectenumero affectevariabletexte ajustechamp alaligne \ +alapage aligneadroite aligneagauche aligneaumilieu appendix \ +arg arriereplan atleftmargin atrightmargin baha \ +barrecouleur barreinteraction barresynchronisation bas bouton \ +boutonmenu boutonsinteraction but cacheblocs cap \ +caractere caracteres champ changepolicebrute changepolicecorps \ +chapter chem chiffresromains citation citer \ +clip clonechamp colonne comment commentaire \ +comparegroupecouleur comparepalette completecombinedlist completelistoffloats completelistofsorts \ +completelistofsynonyms completenumeropage completeregistre composant composeenalinea \ +concernant convertitnumero copitchamp corrigeespaceblanc couleur \ +couleurgrise coupledocument coupledregister couplemarquage couplepapier \ +coupleregistre crlf cutspace dactylographier dans \ +dansautremarge dansborddroit dansbordgauche dansdroite dansgauche \ +dansmarge dansmargedroite dansmargegauche date datecourante \ +daterecommandation de decouplemarquage decrementenumero definebodyfontDEF \ +definebodyfontREF definecombination definedfont definefontfeature definefonthandling \ +defineframed defineframedtext defineindentedtext defineitemgroup definemathalignment \ +defineplacement definetypeface definicaractere definit definitaccent \ +definitbloc definitblocsection definitbuffer definitcalque definitchamp \ +definitchampprincipal definitcommande definitconversion definitcouleur definitdactylo \ +definitdansmarge definitdemarrestoppe definitdescription definitdisposition definitenumeration \ +definitenvironnementpolicecorps definitetiquette definitflottant definitformatreference definitgroupecouleur \ +definithbox definitjeucolonne definitliste definitlisteimbriquee definitlistereference \ +definitlogo definitmakeup definitmarquage definitmenuinteraction definitnotepdp \ +definitpalette definitparagraphes definitpilechamp definitpolice definitpolicebrute \ +definitpolicecorps definitpositiontexte definitprofil definitprogramme definitreference \ +definitregistre definitregle definitrevetement definitsautdecolonne definitsautdepage \ +definitsection definitsortie definitsouschamp definitstyle definitstylepolice \ +definitsymbole definitsymbolefigure definitsynonymepolice definitsynonymes definittabulation \ +definittaillepapier definittete definittexte definittrametableau definittri \ +definittype definitvariabletexte definitversion definitvide demarrealignement \ +demarrearriereplan demarreblocmarge demarrecitation demarreciter demarrecodage \ +demarrecolonnes demarrecombinaison demarrecompoetroite demarrecomposant demarrecorrectionligne \ +demarrecouleur demarredegroupe demarredocument demarreenvironement demarrefigure \ +demarreglobal demarregroupe demarrejeucolonne demarrejeusymboles demarreligne \ +demarreligneregleetexte demarrelignes demarrelocal demarremakeup demarremargereglee \ +demarrenotespdplocales demarrenumerotationligne demarreopposition demarrepositionnement demarreproduit \ +demarreprofil demarreprojet demarreraster demarrerevetement demarresynchronisation \ +demarretableau demarretableaux demarretexte demarreversion demarrevuedensemble \ +deplaceformule deplacesurgrille description determinecaracteristiqueliste determinecaracteristiquesregistre \ +determinenumerotete dimension distancebord distanceborddroit distancebordgauche \ +distanceentete distanceinf distancemarge distancemargedroite distancemargegauche \ +distancepdp distancesup domicile echelle ecran \ +ecritdansliste ecritdanslistereference ecritentreliste ecritregistre el \ +element elements emptylines enumeration environement \ +espace espaceblanc espacefixe espaceinf espacesfixes \ +espacesup etiquette etiquettes etire fait \ +faitreference fichierdactylo figureexterne forceblocs fraction \ +framed framedtext gardeblocs getnumber grille \ +groupe haut hauteureditionpapier hauteurentete hauteurinf \ +hauteurliste hauteurmakeup hauteurpapier hauteurpdp hauteursup \ +hauteurtexte headsym hl immediatebetweenlist immediatetolist \ +inconnu incrementenumero indentation inframed infull \ +inhibemenuinteraction ininner inmframed inneredgedistance inneredgewidth \ +innermargindistance innermarginwidth inouter installelangue joursemaine \ +labeling labeltexte langue langueprincipale largeurbord \ +largeurborddroit largeurbordgauche largeureditionpapier largeurligne largeurliste \ +largeurmakeup largeurmarge largeurmargedroite largeurmargegauche largeurpapier \ +largeurtexte leg ligneh lignenoire ligneregleetexte \ +lignesnoires listesymbole llongueurliste loadsorts loadsynonyms \ +logchamp mapfontsize mar margereglee marquage \ +marquageversion marquepage mathematique mediaeval menuinteraction \ +mframed mois montrecadre montrechamps montrecouleur \ +montredisposition montreedition montreenvironnementpolicecorps montrefiguresexternes montregrille \ +montregroupecouleur montrejeusymboles montremakeup montrepalette montrepolicecorps \ +montrereglages montrestruts motdroit motmarge movesidefloat \ +name navigating nextsection niveautete nocap \ +nombredesouspages nombretotaldepages nommacro nop note \ +notepdp numeroformule numeropage numeros numerosousformule \ +numerotete numerotetecourant obtientmarquage oriente outeredgedistance \ +outeredgewidth outermargindistance outermarginwidth overbar overbars \ +overstrike overstrikes page pagedepth pagedouble \ +pageoffset paragraph part pasplusdeblocs pasplusdefichiers \ +periodes pilechamp placecoteacote placeflottant placeformule \ +placelegende placelesunsaudessusdesautres placeliste placelisteinmbriquee placelistereference \ +placelistoffloats placelistofsorts placelistofsynonyms placelogos placemarquespages \ +placenotespdp placenotespdplocales placenumeropage placenumerotete placerawlist \ +placeregistre placeregle placesousformule placesurgrille placetextetete \ +placevariabletexte position positionnetexte prendbuffer produit \ +programme projet publication qqpart ran \ +raz razmarquage raznumero recommandation ref \ +refait reference referencepage referencetexte reflete \ +register reglages reglealignement reglearrangement reglearriereplan \ +reglearriereplans reglebarreinteraction reglebarresynchronisation reglebloc regleblocmarge \ +regleblocsection regleboutons reglebuffer reglecapitales reglechamp \ +reglechamps regleclipping reglecolonnes reglecombinaisons reglecommentaire \ +reglecommentairepage reglecompoetroite reglecomposeenalinea reglecouleur reglecouleurs \ +regledactylo regledansmarge regledemarrestoppe regledescriptions regledisposition \ +regleecraninteraction regleecrans regleelements regleencadre regleentete \ +regleenumerations regleenvironnementpolicecorps regleepaisseurligne regleespaceblanc regleespacement \ +regleespacementinterligne reglefiguresexternes regleflottant regleflottants regleformulaires \ +regleformules reglegroupeselements regleinf regleinteraction regleintitule \ +regleintitules reglejeucolonne reglejeusymboles reglelabeltexte reglelangue \ +reglelegende reglelignes reglelignesnoires reglelignesreglestexte regleliste \ +reglelisteimbriquee reglelistereference reglemakeup reglemargereglee reglemarquage \ +reglemarquagehyphenation reglemenuinteraction reglenotepdp reglenumero reglenumeropage \ +reglenumerotation reglenumerotationligne reglenumerotationpage reglenumerotationparagraphe reglenumerotete \ +regleoriente reglepalette reglepapier regleparagraphes reglepdp \ +regleplacementopposition reglepolicecorps reglepositionnement reglepositiontexte regleprofils \ +regleprogrammes reglepublications reglereferencage regleregistre regleregle \ +regleremplitligne regleremplitlignesreglees reglesection regleseparationflottant reglesortie \ +reglesouslignage reglesousnumeropage reglestrut reglesup reglesynchronisation \ +reglesynonymes reglesysteme regletab regletableaux regletabulation \ +regletaillepapier regletete regletetes regletexte regletextesentete \ +regletextesinf regletextespdp regletextessup regletextestexte regletextetete \ +regletolerance regletraitsfins regletransitionspage regletri regletype \ +regleurl reglevariabletexte regleversions remplitchamp remplitligne \ +remplitlignesreglees remplittexte reservefloat resettextcontent retourarriere \ +sansalinea sansdimension sansespace sansespaceblanc sanslignesenteteetpdp \ +sanslignessupetinf sansliste sansmarquage sanstest sauteblocs \ +section seeregister selectionneblocs selectionnepapier selectionneversion \ +sensunite separeflottant settext setupanswerarea setupcolumnsetlines \ +setupcolumnsetstart setupfonthandling setupfontsynonym setupframedtexts setupindentedtext \ +setupinterlinespace2 setupitemgroup setuplistalternative setupmathalignment setupplacement \ +sort sousnumeropage startalignment startarriereplan startbuffer \ +startcitation startcolumnmakeup startcolumns startcombination startcomment \ +startcomposant startcouleur startdescription startdocument startenumeration \ +startenvironement startfait startfigure startfloattext startformula \ +startframedtext startgroupe starthiding startitemgroup startlegend \ +startligneregleetexte startline startlinecorrection startlinenumbering startlines \ +startlocal startlocalenvironment startlocalfootnotes startmakeup startmargereglee \ +startmarginblock startmenuinteraction startnamemakeup startnarrower startopposite \ +startoverlay startoverview startparagraph startpositioning startpostponing \ +startproduit startprofile startprojet startregister startsymbolset \ +startsynchronization starttable starttables starttabulate starttyping \ +startunpacked startversion stopalignment stoparriereplan stopbuffer \ +stopcitation stopcolumnmakeup stopcolumns stopcombination stopcomment \ +stopcompoetroite stopcomposant stopcouleur stopdescription stopdocument \ +stopenumeration stopenvironement stopfait stopfigure stopfloattext \ +stopformula stopframedtext stopgroupe stophiding stopitemgroup \ +stoplegend stopligneregleetexte stopline stoplinecorrection stoplinenumbering \ +stoplines stoplocal stoplocalenvironment stoplocalfootnotes stopmakeup \ +stopmargereglee stopmarginblock stopmenuinteraction stopnamemakeup stopnarrower \ +stopopposite stopoverlay stopoverview stopparagraph stoppealignement \ +stoppearriereplan stoppeblocmarge stoppecitation stoppecodage stoppecolonnes \ +stoppecombinaison stoppecomposant stoppecorrectionligne stoppecouleur stoppedegroupe \ +stoppedocument stoppeenvironement stoppeglobal stoppegroupe stoppejeucolonne \ +stoppeligne stoppeligneregleetexte stoppelignes stoppelocal stoppemakeup \ +stoppemargereglee stoppenotespdplocales stoppenumerotationligne stoppeopposition stoppepositionnement \ +stoppeproduit stoppeprofil stoppeprojet stopperaster stopperevetement \ +stoppesynchronisation stoppetableau stoppetableaux stoppetexte stoppeversion \ +stoppevuedensemble stoppositioning stoppostponing stopproduit stopprofile \ +stopprojet stopsymbolset stopsynchronization stoptable stoptables \ +stoptabulate stoptyping stopunpacked stopversion sub \ +subject subsection subsubject subsubsection subsubsubject \ +suggestion suivantprofil suivantversion suivantversionprofil sym \ +symbole synchronise synonym tab tapebuffer \ +testcolumn testpage tete tex textemarge \ +textenotepdp textetete title titremarge traduire \ +traiteblocs traitepage traitfin traitsfins txt \ +typ underbar underbars uneligne useXMLfilter \ +usedirectory usetypescript usetypescriptfile utiliseJSscripts utiliseURL \ +utiliseblocs utilisechemin utilisecommandes utilisedocumentexterne utiliseencodage \ +utilisefichierexterne utilisefichiersexternes utilisefigureexterne utilisemodule utilisemodules \ +utilisepsiteaudioexterne utilisereferences utilisespecialites utilisesymboles utiliseurl \ +va vaalaboite vaalapage vaenbas valeurcouleur \ +valeurgris variabletexte version vide vl + +keywordclass.context.pe=\ CAP Cap Caps Character \ Characters MONTH Numbers Romannumerals WEEKDAY \ WORD WORDS Word Words appendix \ @@ -469,328 +324,155 @@ useXMLfilter usedirectory useurl آفست‌صفحه آیتم \ کسر کشیده کلمه‌حاشیه کلمه‌راست گیره \ یادداشت یک‌جا یک‌خط -keywordclass.context.nl=\ -CAP Cap Caps Cijfers \ -KAP Kap Kaps Letter Letters \ -MAAND Romeins WEEKDAG WOORD WOORDEN \ -Woord Woorden aantalsubpaginas about achtergrond \ -appendix arg bepaalkopnummer bepaallijstkenmerken bepaalregisterkenmerken \ -betekenis binnenmargeafstand binnenmargebreedte binnenrandafstand binnenrandbreedte \ -blanko blokje blokjes blokkeerinteractiemenu bodemwit \ -bookmark bovenafstand bovenhoogte breuk buitenmargeafstand \ -buitenmargebreedte buitenrandafstand buitenrandbreedte but button \ -cap chapter chem cijfers citaat \ -citeer clip comment completecombinedlist completelistoffloats \ -completelistofsorts completelistofsynonyms converteernummer copieerveld corrigeerwitruimte \ -coupledregister crlf datum definebodyfontDEF definebodyfontREF \ -definedfont definefontfeature definefonthandling definerawfont definetypeface \ -definieer definieeraccent definieeralineas definieerbeeldmerk definieerblanko \ -definieerblok definieerbuffer definieercombinatie definieercommando definieerconversie \ -definieerfiguursymbool definieerfont definieerfontstijl definieerfontsynoniem definieerhbox \ -definieerhoofdveld definieeringesprongentext definieerinmarge definieerinteractiemenu definieeritemgroep \ -definieerkadertekst definieerkarakter definieerkleur definieerkleurgroep definieerkolomgroep \ -definieerkolomovergang definieerkop definieerkorps definieerkorpsomgeving definieerlayer \ -definieerlayout definieerletter definieerlijn definieerlijst definieermarkering \ -definieeromlijnd definieeropmaak definieeroverlay definieerpaginaovergang definieerpalet \ -definieerpapierformaat definieerplaats definieerplaatsblok definieerprofiel definieerprogramma \ -definieerreferentie definieerreferentieformaat definieerreferentielijst definieerregister definieersamengesteldelijst \ -definieersectie definieersectieblok definieersorteren definieerstartstop definieersubveld \ -definieersymbool definieersynoniemen definieertabelvorm definieertabulatie definieertekst \ -definieertekstpositie definieertekstvariabele definieertype definieertypen definieeruitvoer \ -definieerveld definieerveldstapel definieerversie definieerwiskundeuitlijnen description \ -dimensie directnaarlijst directtussenlijst doordefinieren doorlabelen \ -doornummeren dunnelijn dunnelijnen eenregel enumeration \ -ergens externfiguur forceerblokken formulenummer framedtext \ -gebruikJSscripts gebruikURL gebruikXMLfilter gebruikblokken gebruikcommandos \ -gebruikexterndocument gebruikexternefile gebruikexternefiles gebruikexternfiguur gebruikexterngeluidsfragment \ -gebruikgebied gebruikmodule gebruikmodules gebruikreferenties gebruikspecials \ -gebruiksymbolen gebruiktypescript gebruiktypescriptfile gebruikurl geenblokkenmeer \ -geenbovenenonderregels geendimensie geenfilesmeer geenhoofdenvoetregels geenlijst \ -geenmarkering geenspatie geentest geenwitruimte geg \ -grijskleur grijswaarde haalbuffer haalmarkering haalnummer \ -haarlijn handhaafblokken herhaal hl hoofdafstand \ -hoofdhoogte hoofdtaal hoog huidigedatum huidigekopnummer \ -in inanderemarge inbinnen inbuiten indentation \ -inlijnd inlinker inlinkermarge inlinkerrand inmarge \ -inrechter inrechtermarge inrechterrand inregel inspringen \ -installeertaal instellingen interactiebalk interactiebuttons interactiemenu \ -invullijnen invulregel invultekst invulveld inwilijnd \ -items its kantlijn kap kenmerk \ -kenmerkdatum kentekstvariabeletoe kleur kleurenbalk kleurwaarde \ -kloonveld kolom kop kopniveau kopnummer \ -koppeldocument koppelmarkering koppelpagina koppelpapier koppelregister \ -kopsym koptekst kopwit laag label \ -labeling labels labeltekst laho leg \ -legeregels letter letters lijndikte lijstbreedte \ -lijsthoogte lijstlengte lijstsymbool linkermargeafstand linkermargebreedte \ -linkerrandafstand linkerrandbreedte loadsorts loadsynonyms maand \ -mapfontsize mar margeafstand margebreedte margetekst \ -margetitel margewoord markeer markeerversie mediaeval \ -menubutton naam naar naarbox naarpagina \ -name navigerend nextsection nietinspringen nocap \ -nokap noot nop omgeving omlaag \ -omlijnd onbekend onderafstand onderdeel onderhoogte \ -ontkoppelmarkering op opelkaar oplinkermarge oppagina \ -oprechtermarge overbar overbars overstrike overstrikes \ -pagina paginadiepte paginanummer paginaoffset paginareferentie \ -papierbreedte papierhoogte paragraph part paslayoutaan \ -passeerblokken passendveld plaatsbeeldmerken plaatsbookmarks plaatsformule \ -plaatskopnummer plaatskoptekst plaatslegenda plaatslijn plaatslijst \ -plaatslokalevoetnoten plaatsnaastelkaar plaatsonderelkaar plaatsopgrid plaatspaginanummer \ -plaatsplaatsblok plaatsreferentielijst plaatsregister plaatsruwelijst plaatssamengesteldelijst \ -plaatssubformule plaatstekstvariabele plaatsvoetnoten placelistoffloats placelistofsorts \ -placelistofsynonyms positioneer positioneertekst printpapierbreedte printpapierhoogte \ -produkt programma projekt publicatie punten \ -ran randafstand randbreedte rechtermargeafstand rechtermargebreedte \ -rechterrandafstand rechterrandbreedte ref refereer referentie \ -regellinks regelmidden regelrechts register registreervelden \ -reservefloat reset resetmarkering resetnummer resettekstinhoud \ -resettextcontent romeins rooster roteer rugwit \ -schaal scherm schrijfnaarlijst schrijfnaarreferentielijst schrijfnaarregister \ -schrijftussenlijst section seeregister selecteerblokken selecteerpapier \ -selecteerversie setnummer setupfonthandling setupfontsynonym setupinterlinespace2 \ -setuplistalternative snijwit som sort spatie \ -spiegel splitsplaatsblok startachtergrond startalignment startbuffer \ -startcitaat startcodering startcolumns startcombinatie startcombination \ -startcomment startdescription startdocument startenumeration startfigure \ -startfiguur startfloattext startformula startframedtext startgeg \ -startglobaal starthiding startinteractiemenu startitemgroup startkantlijn \ -startkleur startkolomgroep startkolommen startkolomopmaak startlegend \ -startline startlinecorrection startlinenumbering startlines startlocal \ -startlocalenvironment startlocalfootnotes startlokaal startlokalevoetnoten startmakeup \ -startmargeblok startmarginblock startnaast startnamemakeup startnarrower \ -startomgeving startonderdeel startopelkaar startopmaak startopposite \ -startoverlay startoverview startoverzicht startparagraph startpositioneren \ -startpositioning startpostponing startprodukt startprofiel startprofile \ -startprojekt startraster startregel startregelcorrectie startregelnummeren \ -startregels startregister startsmaller startsymbolset startsymboolset \ -startsynchronisatie startsynchronization starttabel starttabellen starttable \ -starttables starttabulate starttekst starttekstlijn starttyping \ -startuitlijnen startunpacked startvanelkaar startversie stelachtergrondenin \ -stelachtergrondin stelalineasin stelantwoordgebiedin stelarrangerenin stelblankoin \ -stelblokin stelblokjesin stelblokkopjein stelblokkopjesin stelbovenin \ -stelboventekstenin stelbufferin stelbuttonsin stelciterenin stelclipin \ -stelcombinatiesin stelcommentaarin steldoordefinierenin steldoornummerenin steldunnelijnenin \ -stelexternefigurenin stelformulesin stelformulierenin stelhoofdin stelhoofdtekstenin \ -stelingesprongentextin stelinmargein stelinspringenin stelinteractiebalkin stelinteractiein \ -stelinteractiemenuin stelinteractieschermin stelinterliniein stelinvullijnenin stelinvulregelsin \ -stelitemgroepin stelitemsin stelkadertekstenin stelkantlijnin stelkapitalenin \ -stelkleurenin stelkleurin stelkolomgroepin stelkolomgroepregelsin stelkolomgroepstartin \ -stelkolommenin stelkopin stelkopnummerin stelkoppeltekenin stelkoppenin \ -stelkoptekstin stelkorpsin stelkorpsomgevingin stellabeltekstin stellayoutin \ -stellegendain stellijndiktein stellijnin stellijstin stelmargeblokkenin \ -stelmarkeringin stelnaastplaatsenin stelnummerenin stelnummerin stelomlijndin \ -stelonderin stelonderstrepenin stelondertekstenin stelopmaakin stelopsommingenin \ -stelpaginacommentaarin stelpaginanummerin stelpaginanummeringin stelpaginaovergangenin stelpaletin \ -stelpapierformaatin stelpapierin stelparagraafnummerenin stelplaatsblokin stelplaatsblokkenin \ -stelplaatsbloksplitsenin stelplaatsin stelpositionerenin stelprofielenin stelprogrammasin \ -stelpublicatiesin stelrastersin stelreferentielijstin stelrefererenin stelregelnummerenin \ -stelregelsin stelregisterin stelroterenin stelsamengesteldelijstin stelsectieblokin \ -stelsectiein stelsmallerin stelsorterenin stelspatieringin stelstartstopin \ -stelstrutin stelsubpaginanummerin stelsymboolsetin stelsynchronisatiebalkin stelsynchronisatiein \ -stelsynoniemenin stelsysteemin steltaalin steltabellenin steltabin \ -steltabulatiein steltekstin steltekstinhoudin steltekstlijnenin steltekstpositiein \ -stelteksttekstenin steltekstvariabelein steltolerantiein steltypein steltypenin \ -steluitlijnenin steluitvoerin stelurlin stelveldenin stelveldin \ -stelversiesin stelvoetin stelvoetnootdefinitiein stelvoetnotenin stelvoettekstenin \ -stelwiskundeuitlijnenin stelwitruimtein stopachtergrond stopalignment stopbuffer \ -stopcitaat stopcodering stopcolumns stopcombinatie stopcombination \ -stopcomment stopdescription stopdocument stopenumeration stopfigure \ -stopfloattext stopformula stopframedtext stopgeg stopglobaal \ -stophiding stopinteractiemenu stopitemgroup stopkantlijn stopkleur \ -stopkolomgroep stopkolommen stopkolomopmaak stoplegend stopline \ -stoplinecorrection stoplinenumbering stoplines stoplocal stoplocalenvironment \ -stoplocalfootnotes stoplokaal stoplokalevoetnoten stopmakeup stopmargeblok \ -stopmarginblock stopnaast stopnamemakeup stopnarrower stopomgeving \ -stoponderdeel stopopelkaar stopopmaak stopopposite stopoverlay \ -stopoverview stopoverzicht stopparagraph stoppositioneren stoppositioning \ -stoppostponing stopprodukt stopprofiel stopprofile stopprojekt \ -stopraster stopregel stopregelcorrectie stopregelnummeren stopregels \ -stopsmaller stopsymbolset stopsynchronisatie stopsynchronization stoptabel \ -stoptabellen stoptable stoptables stoptabulate stoptekst \ -stoptekstlijn stoptyping stopuitlijnen stopunpacked stopvanelkaar \ -stopversie sub subformulenummer subject subpaginanummer \ -subsection subsubject subsubsection subsubsubject suggestie \ -switchnaarkorps switchtorawfont sym symbool symoffset \ -synchronisatiebalk synchroniseer synonym taal tab \ -tekstbreedte teksthoogte tekstlijn tekstreferentie tekstvariabele \ -testkolom testpagina tex title toelichting \ -toonexternefiguren toongrid tooninstellingen toonkader toonkleur \ -toonkleurgroep toonkorps toonkorpsomgeving toonlayout toonopmaak \ -toonpalet toonprint toonstruts toonsymboolset toonvelden \ -totaalaantalpaginas txt typ type typebuffer \ -typefile uit uitgerekt underbar underbars \ -usecodering usedirectory vastespatie vastespaties veld \ -veldstapel verbergblokken vergelijkkleurgroep vergelijkpalet verhoognummer \ -verlaagnummer verplaatsformule verplaatsopgrid verplaatszijblok versie \ -vertaal verwerkblokken verwerkpagina vl voetafstand \ -voethoogte voetnoot voetnoottekst volgprofiel volgprofielversie \ -volgversie volledigepaginanummer volledigregister voluit weekdag \ -wilijnd wiskunde witruimte woonplaats woordrechts \ -zetbreedte zethoogte - -keywordclass.context.it=\ -CAP Cap Caps GIORNOSETTIMANA \ -Lettera Lettere MESE Numeri Numeriromani \ -PAROLA PAROLE Parola Parole accoppiacarta \ -accoppiadocumento accoppiamarcatura accoppiapagina accoppiaregistro adattacampo \ -adattalayout al allineacentro allineadestra allineasinistra \ -altezzacarta altezzacartastampa altezzacima altezzaelenco altezzafondo \ -altezzaintestazione altezzamakeup altezzapdp altezzatesto ambiente \ -ampiezzabordo ampiezzabordodestro ampiezzabordosinistro ampiezzacarta ampiezzacartastampa \ -ampiezzaelenco ampiezzamakeup ampiezzamargine ampiezzamarginedestro ampiezzamarginesinistro \ -ampiezzatesto ap apagina appendix arg \ -atleftmargin atrightmargin barracolori barrainterazione barrasincronizzazione \ -bastablocchi bastafile cambiaafontdeltesto campi camporiempimento \ -cap capello chapter chim circondato \ -citazione clip clonacampo colonna colore \ -coloregrigio comment commento completecombinedlist completelistoffloats \ -completelistofsorts completelistofsynonyms completeregister componenet confrontagruppocolori \ -confrontatavolozza convertinumero copiacampo correggispaziobianco coupledregister \ -crlf cutspace da daqualcheparte data \ -datadioggi datareferral decrementnumber definebodyfontDEF definebodyfontREF \ -definecolumnbreak definecombination definedfont definefontfeature definefonthandling \ -defineindentedtext defineinmargin defineitemgroup definelayer definemathalignment \ -definepagebreak defineplacement definetypeface definisci definisciaccento \ -definisciambientefontdeltesto definisciblocco definiscibloccosezione definiscibuffer definiscicampo \ -definiscicampoprincipale definiscicapoversi definiscicarattere definiscicolore definiscicomando \ -definisciconversione definiscidescrizione definiscidimensionicarta definiscielenco definiscielencocombinato \ -definiscienumerazione definiscietichetta definiscifigurasimbolo definiscifont definiscifontdeltesto \ -definiscifontgrezzo definisciformatoriferimento definiscigruppocolonne definiscigruppocolori definiscihbox \ -definisciincorniciato definisciiniziatermina definiscilayout definiscilinea definiscilistariferimenti \ -definiscilogo definiscimakeup definiscimarcatura definiscimenuinterazione definiscimodellotabella \ -definiscioggettomobile definisciordinamento definiscioutput definisciposizionetesto definisciprofilo \ -definisciprogramma definisciregistro definisciriferimento definiscirigovuoto definiscisezione \ -definiscisimbolo definiscisinonimi definiscisinonimofont definiscisottocampo definiscisovrapposizione \ -definiscistackcampi definiscistile definiscistilefont definiscitabulato definiscitavolozza \ -definiscitesta definiscitesto definiscitestoincorniciato definiscitype definiscityping \ -definiscivariabiletesto definisciversion description determinacaratteristicheregistro determinacarattersticheelenco \ -determinanumerotesta dimensione disabilitamenuinterazione distanzabordo distanzabordodestro \ -distanzabordosinistro distanzacima distanzafondo distanzaintestazione distanzamargine \ -distanzamarginedestro distanzamarginesinistro distanzapdp domicilio el \ -elaborablocchi elaborapagina elementi elemento emptylines \ -enumeration etichetta etichette fatto figuraesterna \ -fondo forzablocchi framedtext frazione getnumber \ -giornosettimana griglia headsym hl ignoto \ -immediatebetweenlist immediatetolist impaccato impostaallineamento impostaambientefontdeltesto \ -impostaampiezzariga impostabarrainterazione impostabarrasincronizzazione impostablocchimargine impostablocco \ -impostabloccosezione impostabuffer impostacampi impostacampo impostacapoversi \ -impostacaption impostacaptions impostacima impostaclippling impostacolonne \ -impostacolore impostacolori impostacombinazioni impostacommento impostacommentopagina \ -impostadefinizionenotepdp impostadescrizioni impostadimensionicarta impostaelementi impostaelencazioni \ -impostaelenco impostaelencocombinato impostaenumerazioni impostafigureesterne impostafondo \ -impostafontdeltesto impostaforms impostaformule impostagruppocolonne impostaincorniciato \ -impostainiziatermina impostainmargine impostainstestazione impostainterazione impostainterlinea \ -impostalayout impostalegenda impostalinea impostalineemargine impostalineenere \ -impostalineeriempimento impostalineesottili impostalineetesto impostalingua impostalistariferimenti \ -impostamaiuscole impostamakeup impostamarcatura impostamenuinterazione impostamenzione \ -impostanotepdp impostanumerazione impostanumerazionecapoversi impostanumerazionepagina impostanumerazionerighe \ -impostanumeropagina impostanumerosottopagina impostanumerotesta impostaoggettimobili impostaoggettomobile \ -impostaordinamento impostaoutput impostaparranging impostapdp impostapiustretto \ -impostaposizionamento impostaposizionamentoopposti impostaposizionetesto impostaprofili impostaprogrammi \ -impostapubblicazioni impostapulsanti impostaregistro impostarientro impostariferimento \ -impostarighe impostarigheriempimento impostarigovuoto impostarotazione impostaschermi \ -impostaschermointerazione impostasegnosillabazione impostasetsimboli impostasezione impostasfondi \ -impostasfondo impostasincronizzazione impostasinonimi impostasistema impostasottolinea \ -impostaspaziatura impostaspaziobianco impostaspezzamentooggettomobile impostastrut impostatab \ -impostatabelle impostatabulato impostatavolozza impostatesta impostateste \ -impostatesticima impostatestifondo impostatestiincorniciati impostatestiintestazioni impostatestipdp \ -impostatesto impostatestoetichette impostatestointestazioni impostatestotesti impostatolleranza \ -impostatransizionepagina impostatype impostatyping impostaurl impostavariabiletesto \ -impostaversioni impostazioni in inaltromargine incorniciato \ -incrementanumero indentation indestra ininner iniziaallineamento \ -iniziaambiente iniziabloccomargine iniziacitazione iniziacodifica iniziacolonne \ -iniziacolore iniziacombinazione iniziacomponente iniziacorrezioneriga iniziadocumento \ -iniziafigura iniziaglobale iniziagruppocolonne iniziaimpaccato inizialineamargine \ -inizialineatesto inizialocale iniziamakeup inizianotepdplocali inizianumerazionerighe \ -iniziaopposto iniziaoverview iniziapiustretto iniziaposizionamento iniziaprodotto \ -iniziaprofilo iniziaprogetto iniziaraster iniziariga iniziarighe \ -iniziasetsimboli iniziasfondo iniziasincronizzazione iniziasovrapposizione iniziatabella \ -iniziatabelle iniziatesto iniziaunpacked iniziaversione inlatodestro \ -inlatosinistro inmaframed inmargine inmarginedestro inmarginesinistro \ -inneredgedistance inneredgewidth innermargindistance innermarginwidth inouter \ -inriga insinistra installalingua intorno labeling \ -leg lettera lettere lineamargine lineanera \ -lineasottile lineatesto lineenere lineeriempimento lineesottili \ -lingua linguaprincipale listsymbol livellotesta loadsorts \ -loadsynonyms logcampi lunghezzaelenco maframed mapfontsize \ -mar marcatura marcaversione matematica mediaeval \ -menuinterattivo menzione mese mettielenco mettielencocombinato \ -mettifiancoafianco mettiformula mettiingriglia mettilegenda mettilinea \ -mettiloghi mettinotepdp mettinotepdplocali mettinumeropagina mettiregistro \ -mettisegnalibro mettisottoformula mettiunosullaltro mettivariabiletesto mostraambientefontdeltesto \ -mostracampi mostracolore mostracornice mostrafiguresterne mostrafontdeltesto \ -mostragriglia mostragruppocolori mostraimpostazioni mostralyout mostramakeup \ -mostrasetsimboli mostrastampa mostrastruts mostratavolozza movesidefloat \ -name nascondiblocchi navigating nextsection nientedimensioni \ -nienteelenco nientelineecimafondo nientelineintestazionepdp nientemarcatura nienterientro \ -nientespazio nientespaziobianco nocap nome nomeunita \ -nop nota notapdp notest numberofsubpages \ -numeri numeriromani numeroformula numeropagina numeropaginacompleto \ -numerosottoformula numerotesta numerotestacorrente numerototaledipagine outeredgedistance \ -outeredgewidth outermargindistance outermarginwidth overbar overbars \ -overstrike overstrikes pagedepth pageoffset pagina \ -paragraph paroladestra parolainmargine part passaafontgrezzo \ -ped pedap perlungo placefloat placelistoffloats \ -placelistofsorts placelistofsynonyms placerawlist placereferencelist posizionanumerotesta \ -posizionatesto posizionatestotesta posizione prendibuffer prendimarcatura \ -prodotto progetto programma pubblicazione pulsante \ -pulsantemenu pulsantinterazione punti qualcheriga ran \ -referral referring register reimposta reimpostamarcatura \ -reservefloat resetnumber resettextcontent rientro rif \ -rifai riferimento riferimentopagina riferimentotesto riflessione \ -rigariempimento rigovuoto ruota saltablocchi scala \ -schermo scrividentroelenco scriviinelenco scriviinlistariferimenti scriviinregistro \ -section seeregister segnalibro seguiprofilo seguiversione \ -seguiversioneprofilo selezionablocchi selezionacarta selezionaversione separamarcatura \ -setnumber settext setupanswerarea setupcolumnsetlines setupcolumnsetstart \ -setupfonthandling setupfontsynonym setupindentedtext setupinterlinespace2 setupitemgroup \ -setuplistalternative setupmathalignment setuppaper setupplacement setvariabiletesto \ -sfondo sim simbolo sincronizza sort \ -spazifissi spazio spaziobianco spaziocima spaziodietro \ -spaziofisso spaziofondo spessoreriga spezzaoggettomobile spostaagriglia \ -spostaformula stackcampi startalignment startambiente startbuffer \ -startcitazione startcolore startcolumnmakeup startcolumns startcombination \ -startcomment startcomponenet startdescription startdocument startenumeration \ -startfatto startfigure startfloattext startformula startframedtext \ -starthiding startimpaccato startitemgroup startlegend startline \ -startlineamargine startlineatesto startlinecorrection startlinenumbering startlines \ +keywordclass.context.en=\ +CAP Cap Caps Character \ +Characters MONTH Numbers Romannumerals WEEKDAY \ +WORD WORDS Word Words about \ +adaptlayout adding appendix arg at \ +atleftmargin atpage atrightmargin background backspace \ +blackrule blackrules blank bookmark bottomdistance \ +bottomheight bottomspace but button bypassblocks \ +cap chapter character characters chem \ +clip clonefield color colorbar colorvalue \ +column comment comparecolorgroup comparepalet completecombinedlist \ +completelistoffloats completelistofsorts completelistofsynonyms completepagenumber completeregister \ +component convertnumber copyfield correctwhitespace coupledocument \ +coupledregister couplemarking couplepage couplepaper coupleregister \ +crlf currentdate currentheadnumber cutspace date \ +decouplemarking decrementnumber define defineaccent defineblank \ +defineblock definebodyfont definebodyfontDEF definebodyfontREF definebodyfontenvironment \ +definebuffer definecharacter definecolor definecolorgroup definecolumnbreak \ +definecolumnset definecombination definecombinedlist definecommand defineconversion \ +definedescription definedfont defineenumeration definefield definefieldstack \ +definefiguresymbol definefloat definefont definefontfeature definefonthandling \ +definefontstyle definefontsynonym defineframed defineframedtext definehbox \ +definehead defineindentedtext defineinmargin defineinteractionmenu defineitemgroup \ +definelabel definelayer definelayout definelist definelogo \ +definemainfield definemakeup definemarking definemathalignment defineoutput \ +defineoverlay definepagebreak definepalet definepapersize defineparagraphs \ +defineplacement defineprofile defineprogram definerawfont definereference \ +definereferenceformat definereferencelist defineregister definerule definesection \ +definesectionblock definesorting definestartstop definestyle definesubfield \ +definesymbol definesynonyms definetabletemplate definetabulate definetext \ +definetextposition definetextvariable definetype definetypeface definetyping \ +defineversion description determineheadnumber determinelistcharacteristics determineregistercharacteristics \ +dimension disableinteractionmenu domicile donttest edgedistance \ +edgewidth emptylines enumeration environment externalfigure \ +fact field fieldstack fillinfield fillinline \ +fillinrules fillintext fitfield fixedspace fixedspaces \ +followprofile followprofileversion followversion footerdistance footerheight \ +footnote footnotetext forceblocks formulanumber fraction \ +framed framedtext from getbuffer getmarking \ +getnumber godown goto gotobox gotopage \ +graycolor greyvalue grid hairline head \ +headerdistance headerheight headlevel headnumber headsym \ +headtext hideblocks high hl immediatebetweenlist \ +immediatetolist in incrementnumber indentation indenting \ +inframed infull ininner inleft inleftedge \ +inleftmargin inline inmaframed inmargin inneredgedistance \ +inneredgewidth innermargindistance innermarginwidth inothermargin inouter \ +inright inrightedge inrightmargin installlanguage interactionbar \ +interactionbuttons interactionmenu item items its \ +keepblocks label labeling labels labeltext \ +language leftaligned leftedgedistance leftedgewidth leftmargindistance \ +leftmarginwidth leg linethickness listheight listlength \ +listsymbol listwidth loadsorts loadsynonyms logfields \ +lohi low maframed mainlanguage makeupheight \ +makeupwidth mapfontsize mar margindistance marginrule \ +margintext margintitle marginwidth marginword marking \ +markversion mathematics mediaeval menubutton midaligned \ +mirror month moveformula moveongrid movesidefloat \ +name navigating nextsection nocap nodimension \ +noheaderandfooterlines noindenting nolist nomarking nomoreblocks \ +nomorefiles nop nospace note notopandbottomlines \ +nowhitespace numberofsubpages numbers outeredgedistance outeredgewidth \ +outermargindistance outermarginwidth overbar overbars overstrike \ +overstrikes packed page pagedepth pagenumber \ +pageoffset pagereference paperheight paperwidth paragraph \ +part periods placebookmarks placecombinedlist placefloat \ +placefootnotes placeformula placeheadnumber placeheadtext placelegend \ +placelist placelistoffloats placelistofsorts placelistofsynonyms placelocalfootnotes \ +placelogos placeongrid placeontopofeachother placepagenumber placerawlist \ +placereferencelist placeregister placerule placesidebyside placesubformula \ +placetextvariable position positiontext printpaperheight printpaperwidth \ +processblocks processpage product program project \ +publication quotation quote ran redo \ +ref reference referral referraldate referring \ +register remark reservefloat reset resetmarking \ +resetnumber resettextcontent rightaligned rightedgedistance rightedgewidth \ +rightmargindistance rightmarginwidth romannumerals rotate scale \ +screen section seeregister selectblocks selectpaper \ +selectversion setnumber settextcontent settextvariable setupalign \ +setupanswerarea setuparranging setupbackground setupbackgrounds setupblackrules \ +setupblank setupblock setupbodyfont setupbodyfontenvironment setupbottom \ +setupbottomtexts setupbuffer setupbuttons setupcapitals setupcaption \ +setupcaptions setupclipping setupcolor setupcolors setupcolumns \ +setupcolumnset setupcolumnsetlines setupcolumnsetstart setupcombinations setupcombinedlist \ +setupcomment setupdescriptions setupenumerations setupexternalfigures setupfield \ +setupfields setupfillinlines setupfillinrules setupfloat setupfloats \ +setupfloatsplitting setupfonthandling setupfontsynonym setupfooter setupfootertexts \ +setupfootnotedefinition setupfootnotes setupforms setupformulae setupframed \ +setupframedtexts setuphead setupheader setupheadertexts setupheadnumber \ +setupheads setupheadtext setuphyphenmark setupindentedtext setupindenting \ +setupinmargin setupinteraction setupinteractionbar setupinteractionmenu setupinteractionscreen \ +setupinterlinespace setupinterlinespace2 setupitemgroup setupitemizations setupitems \ +setuplabeltext setuplanguage setuplayout setuplegend setuplinenumbering \ +setuplines setuplinewidth setuplist setuplistalternative setupmakeup \ +setupmarginblocks setupmarginrules setupmarking setupmathalignment setupnarrower \ +setupnumber setupnumbering setupoppositeplacing setupoutput setuppagecomment \ +setuppagenumber setuppagenumbering setuppagetransitions setuppalet setuppaper \ +setuppapersize setupparagraphnumbering setupparagraphs setupplacement setuppositioning \ +setupprofiles setupprograms setuppublications setupquote setupreferencelist \ +setupreferencing setupregister setuprotate setuprule setups \ +setupscreens setupsection setupsectionblock setupsorting setupspacing \ +setupstartstop setupstrut setupsubpagenumber setupsymbolset setupsynchronization \ +setupsynchronizationbar setupsynonyms setupsystem setuptab setuptables \ +setuptabulate setuptext setuptextposition setuptextrules setuptexttexts \ +setuptextvariable setupthinrules setuptolerance setuptop setuptoptexts \ +setuptype setuptyping setupunderbar setupurl setupversions \ +setupwhitespace showbodyfont showbodyfontenvironment showcolor showcolorgroup \ +showexternalfigures showfields showframe showgrid showlayout \ +showmakeup showpalet showprint showsetups showstruts \ +showsymbolset someline somewhere sort space \ +splitfloat startalignment startbackground startbuffer startcoding \ +startcolor startcolumnmakeup startcolumns startcolumnset startcombination \ +startcomment startcomponent startdescription startdocument startenumeration \ +startenvironment startfact startfigure startfloattext startformula \ +startframedtext startglobal starthiding startinteractionmenu startitemgroup \ +startlegend startline startlinecorrection startlinenumbering startlines \ startlocal startlocalenvironment startlocalfootnotes startmakeup startmarginblock \ -startmenuinterattivo startnamemakeup startnarrower startopposite startoverlay \ -startoverview startparagraph startpositioning startpostponing startprodotto \ -startprofile startprogetto startregister startsfondo startsymbolset \ -startsynchronization starttable starttables starttabulate starttyping \ -startunpacked startversione stirato stopalignment stopambiente \ -stopbuffer stopcitazione stopcolore stopcolumnmakeup stopcolumns \ -stopcombination stopcomment stopcomponenet stopdescription stopdocument \ -stopenumeration stopfatto stopfigure stopfloattext stopformula \ -stopframedtext stophiding stopimpaccato stopitemgroup stoplegend \ -stopline stoplineamargine stoplineatesto stoplinecorrection stoplinenumbering \ -stoplines stoplocal stoplocalenvironment stoplocalfootnotes stopmakeup \ -stopmarginblock stopmenuinterattivo stopnamemakeup stopnarrower stopopposite \ -stopoverlay stopoverview stopparagraph stoppositioning stoppostponing \ -stopprodotto stopprofile stopprogetto stopsfondo stopsymbolset \ -stopsynchronization stoptable stoptables stoptabulate stoptyping \ -stopunpacked stopversione sub subject subpagenumber \ -subsection subsubject subsubsection subsubsubject synonym \ -tab terminaallineamento terminaambiente terminabloccomargine terminacitazione \ -terminacodifica terminacolonne terminacolore terminacombinazione terminacomponente \ -terminacorrezioneriga terminadocumento terminaglobale terminagruppocolonne terminaimpaccato \ -terminalineamargine terminalineatesto terminalocale terminamakeup terminanotepdplocali \ -terminanumerazionerighe terminaopposto terminaoverview terminapiustretto terminaposizionamento \ -terminaprodotto terminaprofili terminaprogetto terminaraster terminariga \ -terminarighe terminasfondo terminasincronizzazione terminasovrapposizione terminatabella \ -terminatabelle terminatesto terminaunpacked terminaversioni testa \ -testcolumn testoetichetta testoinmargine testoinstestazioni testonotapdp \ -testoriempimento testpage tex tieniblocchi title \ -titoloinmargine tooltip traduci txt typ \ -type typebuffer typefile underbar underbars \ -usaJSscripts usaURL usablocco usacartella usacodifica \ -usacolonnasonoraesterna usacomandi usadocumentoesterno usafiguraesterna usafileesterni \ -usafileesterno usamoduli usamodulo usariferimenti usasimboli \ -usaspecialita usaurl useXMLfilter usedirectory usetypescript \ -usetypescriptfile vaia vaiabox vaiapagina vaigiu \ -valorecolore valoregrigio variabiletesto versione vl +startmarginrule startnamemakeup startnarrower startopposite startoverlay \ +startoverview startpacked startparagraph startpositioning startpostponing \ +startproduct startprofile startproject startquotation startraster \ +startregister startsymbolset startsynchronization starttable starttables \ +starttabulate starttext starttextrule starttyping startunpacked \ +startversion stopalignment stopbackground stopbuffer stopcoding \ +stopcolor stopcolumnmakeup stopcolumns stopcolumnset stopcombination \ +stopcomment stopcomponent stopdescription stopdocument stopenumeration \ +stopenvironment stopfact stopfigure stopfloattext stopformula \ +stopframedtext stopglobal stophiding stopinteractionmenu stopitemgroup \ +stoplegend stopline stoplinecorrection stoplinenumbering stoplines \ +stoplocal stoplocalenvironment stoplocalfootnotes stopmakeup stopmarginblock \ +stopmarginrule stopnamemakeup stopnarrower stopopposite stopoverlay \ +stopoverview stoppacked stopparagraph stoppositioning stoppostponing \ +stopproduct stopprofile stopproject stopquotation stopraster \ +stopsymbolset stopsynchronization stoptable stoptables stoptabulate \ +stoptext stoptextrule stoptyping stopunpacked stopversion \ +stretched sub subformulanumber subject subpagenumber \ +subsection subsubject subsubsection subsubsubject switchtobodyfont \ +switchtorawfont sym symbol symoffset synchronizationbar \ +synchronize synonym tab testcolumn testpage \ +tex textheight textreference textrule textvariable \ +textwidth thinrule thinrules title tooltip \ +topdistance topheight topspace totalnumberofpages translate \ +txt typ type typebuffer typefile \ +underbar underbars unitmeaning unknown useJSscripts \ +useURL useXMLfilter useblocks usecommands usedirectory \ +useencoding useexternaldocument useexternalfigure useexternalfile useexternalfiles \ +useexternalsoundtrack usemodule usemodules usereferences usespecials \ +usesymbols usetypescript usetypescriptfile useurl version \ +vl weekday whitespace wordright writebetweenlist \ +writetolist writetoreferencelist writetoregister keywordclass.context.de=\ Buchstabe Buchstaben CAP Cap \ @@ -951,168 +633,486 @@ zeigegitter zeigelayout zeigepalette zeigerahmen zeigestruts \ zeigeumbruch zentriert ziffern zitat zitieren \ zu zurbox zurseite zwischenraum -keywordclass.context.fr=\ -CAP Cap Caps Caractere \ -Caracteres Chiffresromains JOURSEMAINE MOIS MOT \ -MOTS Mot Mots Numeros a \ -adaptedisposition affectenumero affectevariabletexte ajustechamp alaligne \ -alapage aligneadroite aligneagauche aligneaumilieu appendix \ -arg arriereplan atleftmargin atrightmargin baha \ -barrecouleur barreinteraction barresynchronisation bas bouton \ -boutonmenu boutonsinteraction but cacheblocs cap \ -caractere caracteres champ changepolicebrute changepolicecorps \ -chapter chem chiffresromains citation citer \ -clip clonechamp colonne comment commentaire \ -comparegroupecouleur comparepalette completecombinedlist completelistoffloats completelistofsorts \ -completelistofsynonyms completenumeropage completeregistre composant composeenalinea \ -concernant convertitnumero copitchamp corrigeespaceblanc couleur \ -couleurgrise coupledocument coupledregister couplemarquage couplepapier \ -coupleregistre crlf cutspace dactylographier dans \ -dansautremarge dansborddroit dansbordgauche dansdroite dansgauche \ -dansmarge dansmargedroite dansmargegauche date datecourante \ -daterecommandation de decouplemarquage decrementenumero definebodyfontDEF \ -definebodyfontREF definecombination definedfont definefontfeature definefonthandling \ -defineframed defineframedtext defineindentedtext defineitemgroup definemathalignment \ -defineplacement definetypeface definicaractere definit definitaccent \ -definitbloc definitblocsection definitbuffer definitcalque definitchamp \ -definitchampprincipal definitcommande definitconversion definitcouleur definitdactylo \ -definitdansmarge definitdemarrestoppe definitdescription definitdisposition definitenumeration \ -definitenvironnementpolicecorps definitetiquette definitflottant definitformatreference definitgroupecouleur \ -definithbox definitjeucolonne definitliste definitlisteimbriquee definitlistereference \ -definitlogo definitmakeup definitmarquage definitmenuinteraction definitnotepdp \ -definitpalette definitparagraphes definitpilechamp definitpolice definitpolicebrute \ -definitpolicecorps definitpositiontexte definitprofil definitprogramme definitreference \ -definitregistre definitregle definitrevetement definitsautdecolonne definitsautdepage \ -definitsection definitsortie definitsouschamp definitstyle definitstylepolice \ -definitsymbole definitsymbolefigure definitsynonymepolice definitsynonymes definittabulation \ -definittaillepapier definittete definittexte definittrametableau definittri \ -definittype definitvariabletexte definitversion definitvide demarrealignement \ -demarrearriereplan demarreblocmarge demarrecitation demarreciter demarrecodage \ -demarrecolonnes demarrecombinaison demarrecompoetroite demarrecomposant demarrecorrectionligne \ -demarrecouleur demarredegroupe demarredocument demarreenvironement demarrefigure \ -demarreglobal demarregroupe demarrejeucolonne demarrejeusymboles demarreligne \ -demarreligneregleetexte demarrelignes demarrelocal demarremakeup demarremargereglee \ -demarrenotespdplocales demarrenumerotationligne demarreopposition demarrepositionnement demarreproduit \ -demarreprofil demarreprojet demarreraster demarrerevetement demarresynchronisation \ -demarretableau demarretableaux demarretexte demarreversion demarrevuedensemble \ -deplaceformule deplacesurgrille description determinecaracteristiqueliste determinecaracteristiquesregistre \ -determinenumerotete dimension distancebord distanceborddroit distancebordgauche \ -distanceentete distanceinf distancemarge distancemargedroite distancemargegauche \ -distancepdp distancesup domicile echelle ecran \ -ecritdansliste ecritdanslistereference ecritentreliste ecritregistre el \ -element elements emptylines enumeration environement \ -espace espaceblanc espacefixe espaceinf espacesfixes \ -espacesup etiquette etiquettes etire fait \ -faitreference fichierdactylo figureexterne forceblocs fraction \ -framed framedtext gardeblocs getnumber grille \ -groupe haut hauteureditionpapier hauteurentete hauteurinf \ -hauteurliste hauteurmakeup hauteurpapier hauteurpdp hauteursup \ -hauteurtexte headsym hl immediatebetweenlist immediatetolist \ -inconnu incrementenumero indentation inframed infull \ -inhibemenuinteraction ininner inmframed inneredgedistance inneredgewidth \ -innermargindistance innermarginwidth inouter installelangue joursemaine \ -labeling labeltexte langue langueprincipale largeurbord \ -largeurborddroit largeurbordgauche largeureditionpapier largeurligne largeurliste \ -largeurmakeup largeurmarge largeurmargedroite largeurmargegauche largeurpapier \ -largeurtexte leg ligneh lignenoire ligneregleetexte \ -lignesnoires listesymbole llongueurliste loadsorts loadsynonyms \ -logchamp mapfontsize mar margereglee marquage \ -marquageversion marquepage mathematique mediaeval menuinteraction \ -mframed mois montrecadre montrechamps montrecouleur \ -montredisposition montreedition montreenvironnementpolicecorps montrefiguresexternes montregrille \ -montregroupecouleur montrejeusymboles montremakeup montrepalette montrepolicecorps \ -montrereglages montrestruts motdroit motmarge movesidefloat \ -name navigating nextsection niveautete nocap \ -nombredesouspages nombretotaldepages nommacro nop note \ -notepdp numeroformule numeropage numeros numerosousformule \ -numerotete numerotetecourant obtientmarquage oriente outeredgedistance \ -outeredgewidth outermargindistance outermarginwidth overbar overbars \ -overstrike overstrikes page pagedepth pagedouble \ -pageoffset paragraph part pasplusdeblocs pasplusdefichiers \ -periodes pilechamp placecoteacote placeflottant placeformule \ -placelegende placelesunsaudessusdesautres placeliste placelisteinmbriquee placelistereference \ -placelistoffloats placelistofsorts placelistofsynonyms placelogos placemarquespages \ -placenotespdp placenotespdplocales placenumeropage placenumerotete placerawlist \ -placeregistre placeregle placesousformule placesurgrille placetextetete \ -placevariabletexte position positionnetexte prendbuffer produit \ -programme projet publication qqpart ran \ -raz razmarquage raznumero recommandation ref \ -refait reference referencepage referencetexte reflete \ -register reglages reglealignement reglearrangement reglearriereplan \ -reglearriereplans reglebarreinteraction reglebarresynchronisation reglebloc regleblocmarge \ -regleblocsection regleboutons reglebuffer reglecapitales reglechamp \ -reglechamps regleclipping reglecolonnes reglecombinaisons reglecommentaire \ -reglecommentairepage reglecompoetroite reglecomposeenalinea reglecouleur reglecouleurs \ -regledactylo regledansmarge regledemarrestoppe regledescriptions regledisposition \ -regleecraninteraction regleecrans regleelements regleencadre regleentete \ -regleenumerations regleenvironnementpolicecorps regleepaisseurligne regleespaceblanc regleespacement \ -regleespacementinterligne reglefiguresexternes regleflottant regleflottants regleformulaires \ -regleformules reglegroupeselements regleinf regleinteraction regleintitule \ -regleintitules reglejeucolonne reglejeusymboles reglelabeltexte reglelangue \ -reglelegende reglelignes reglelignesnoires reglelignesreglestexte regleliste \ -reglelisteimbriquee reglelistereference reglemakeup reglemargereglee reglemarquage \ -reglemarquagehyphenation reglemenuinteraction reglenotepdp reglenumero reglenumeropage \ -reglenumerotation reglenumerotationligne reglenumerotationpage reglenumerotationparagraphe reglenumerotete \ -regleoriente reglepalette reglepapier regleparagraphes reglepdp \ -regleplacementopposition reglepolicecorps reglepositionnement reglepositiontexte regleprofils \ -regleprogrammes reglepublications reglereferencage regleregistre regleregle \ -regleremplitligne regleremplitlignesreglees reglesection regleseparationflottant reglesortie \ -reglesouslignage reglesousnumeropage reglestrut reglesup reglesynchronisation \ -reglesynonymes reglesysteme regletab regletableaux regletabulation \ -regletaillepapier regletete regletetes regletexte regletextesentete \ -regletextesinf regletextespdp regletextessup regletextestexte regletextetete \ -regletolerance regletraitsfins regletransitionspage regletri regletype \ -regleurl reglevariabletexte regleversions remplitchamp remplitligne \ -remplitlignesreglees remplittexte reservefloat resettextcontent retourarriere \ -sansalinea sansdimension sansespace sansespaceblanc sanslignesenteteetpdp \ -sanslignessupetinf sansliste sansmarquage sanstest sauteblocs \ -section seeregister selectionneblocs selectionnepapier selectionneversion \ -sensunite separeflottant settext setupanswerarea setupcolumnsetlines \ -setupcolumnsetstart setupfonthandling setupfontsynonym setupframedtexts setupindentedtext \ -setupinterlinespace2 setupitemgroup setuplistalternative setupmathalignment setupplacement \ -sort sousnumeropage startalignment startarriereplan startbuffer \ -startcitation startcolumnmakeup startcolumns startcombination startcomment \ -startcomposant startcouleur startdescription startdocument startenumeration \ -startenvironement startfait startfigure startfloattext startformula \ -startframedtext startgroupe starthiding startitemgroup startlegend \ -startligneregleetexte startline startlinecorrection startlinenumbering startlines \ -startlocal startlocalenvironment startlocalfootnotes startmakeup startmargereglee \ -startmarginblock startmenuinteraction startnamemakeup startnarrower startopposite \ -startoverlay startoverview startparagraph startpositioning startpostponing \ -startproduit startprofile startprojet startregister startsymbolset \ +keywordclass.context.nl=\ +CAP Cap Caps Cijfers \ +KAP Kap Kaps Letter Letters \ +MAAND Romeins WEEKDAG WOORD WOORDEN \ +Woord Woorden aantalsubpaginas about achtergrond \ +appendix arg bepaalkopnummer bepaallijstkenmerken bepaalregisterkenmerken \ +betekenis binnenmargeafstand binnenmargebreedte binnenrandafstand binnenrandbreedte \ +blanko blokje blokjes blokkeerinteractiemenu bodemwit \ +bookmark bovenafstand bovenhoogte breuk buitenmargeafstand \ +buitenmargebreedte buitenrandafstand buitenrandbreedte but button \ +cap chapter chem cijfers citaat \ +citeer clip comment completecombinedlist completelistoffloats \ +completelistofsorts completelistofsynonyms converteernummer copieerveld corrigeerwitruimte \ +coupledregister crlf datum definebodyfontDEF definebodyfontREF \ +definedfont definefontfeature definefonthandling definerawfont definetypeface \ +definieer definieeraccent definieeralineas definieerbeeldmerk definieerblanko \ +definieerblok definieerbuffer definieercombinatie definieercommando definieerconversie \ +definieerfiguursymbool definieerfont definieerfontstijl definieerfontsynoniem definieerhbox \ +definieerhoofdveld definieeringesprongentext definieerinmarge definieerinteractiemenu definieeritemgroep \ +definieerkadertekst definieerkarakter definieerkleur definieerkleurgroep definieerkolomgroep \ +definieerkolomovergang definieerkop definieerkorps definieerkorpsomgeving definieerlayer \ +definieerlayout definieerletter definieerlijn definieerlijst definieermarkering \ +definieeromlijnd definieeropmaak definieeroverlay definieerpaginaovergang definieerpalet \ +definieerpapierformaat definieerplaats definieerplaatsblok definieerprofiel definieerprogramma \ +definieerreferentie definieerreferentieformaat definieerreferentielijst definieerregister definieersamengesteldelijst \ +definieersectie definieersectieblok definieersorteren definieerstartstop definieersubveld \ +definieersymbool definieersynoniemen definieertabelvorm definieertabulatie definieertekst \ +definieertekstpositie definieertekstvariabele definieertype definieertypen definieeruitvoer \ +definieerveld definieerveldstapel definieerversie definieerwiskundeuitlijnen description \ +dimensie directnaarlijst directtussenlijst doordefinieren doorlabelen \ +doornummeren dunnelijn dunnelijnen eenregel enumeration \ +ergens externfiguur forceerblokken formulenummer framedtext \ +gebruikJSscripts gebruikURL gebruikXMLfilter gebruikblokken gebruikcommandos \ +gebruikexterndocument gebruikexternefile gebruikexternefiles gebruikexternfiguur gebruikexterngeluidsfragment \ +gebruikgebied gebruikmodule gebruikmodules gebruikreferenties gebruikspecials \ +gebruiksymbolen gebruiktypescript gebruiktypescriptfile gebruikurl geenblokkenmeer \ +geenbovenenonderregels geendimensie geenfilesmeer geenhoofdenvoetregels geenlijst \ +geenmarkering geenspatie geentest geenwitruimte geg \ +grijskleur grijswaarde haalbuffer haalmarkering haalnummer \ +haarlijn handhaafblokken herhaal hl hoofdafstand \ +hoofdhoogte hoofdtaal hoog huidigedatum huidigekopnummer \ +in inanderemarge inbinnen inbuiten indentation \ +inlijnd inlinker inlinkermarge inlinkerrand inmarge \ +inrechter inrechtermarge inrechterrand inregel inspringen \ +installeertaal instellingen interactiebalk interactiebuttons interactiemenu \ +invullijnen invulregel invultekst invulveld inwilijnd \ +items its kantlijn kap kenmerk \ +kenmerkdatum kentekstvariabeletoe kleur kleurenbalk kleurwaarde \ +kloonveld kolom kop kopniveau kopnummer \ +koppeldocument koppelmarkering koppelpagina koppelpapier koppelregister \ +kopsym koptekst kopwit laag label \ +labeling labels labeltekst laho leg \ +legeregels letter letters lijndikte lijstbreedte \ +lijsthoogte lijstlengte lijstsymbool linkermargeafstand linkermargebreedte \ +linkerrandafstand linkerrandbreedte loadsorts loadsynonyms maand \ +mapfontsize mar margeafstand margebreedte margetekst \ +margetitel margewoord markeer markeerversie mediaeval \ +menubutton naam naar naarbox naarpagina \ +name navigerend nextsection nietinspringen nocap \ +nokap noot nop omgeving omlaag \ +omlijnd onbekend onderafstand onderdeel onderhoogte \ +ontkoppelmarkering op opelkaar oplinkermarge oppagina \ +oprechtermarge overbar overbars overstrike overstrikes \ +pagina paginadiepte paginanummer paginaoffset paginareferentie \ +papierbreedte papierhoogte paragraph part paslayoutaan \ +passeerblokken passendveld plaatsbeeldmerken plaatsbookmarks plaatsformule \ +plaatskopnummer plaatskoptekst plaatslegenda plaatslijn plaatslijst \ +plaatslokalevoetnoten plaatsnaastelkaar plaatsonderelkaar plaatsopgrid plaatspaginanummer \ +plaatsplaatsblok plaatsreferentielijst plaatsregister plaatsruwelijst plaatssamengesteldelijst \ +plaatssubformule plaatstekstvariabele plaatsvoetnoten placelistoffloats placelistofsorts \ +placelistofsynonyms positioneer positioneertekst printpapierbreedte printpapierhoogte \ +produkt programma projekt publicatie punten \ +ran randafstand randbreedte rechtermargeafstand rechtermargebreedte \ +rechterrandafstand rechterrandbreedte ref refereer referentie \ +regellinks regelmidden regelrechts register registreervelden \ +reservefloat reset resetmarkering resetnummer resettekstinhoud \ +resettextcontent romeins rooster roteer rugwit \ +schaal scherm schrijfnaarlijst schrijfnaarreferentielijst schrijfnaarregister \ +schrijftussenlijst section seeregister selecteerblokken selecteerpapier \ +selecteerversie setnummer setupfonthandling setupfontsynonym setupinterlinespace2 \ +setuplistalternative snijwit som sort spatie \ +spiegel splitsplaatsblok startachtergrond startalignment startbuffer \ +startcitaat startcodering startcolumns startcombinatie startcombination \ +startcomment startdescription startdocument startenumeration startfigure \ +startfiguur startfloattext startformula startframedtext startgeg \ +startglobaal starthiding startinteractiemenu startitemgroup startkantlijn \ +startkleur startkolomgroep startkolommen startkolomopmaak startlegend \ +startline startlinecorrection startlinenumbering startlines startlocal \ +startlocalenvironment startlocalfootnotes startlokaal startlokalevoetnoten startmakeup \ +startmargeblok startmarginblock startnaast startnamemakeup startnarrower \ +startomgeving startonderdeel startopelkaar startopmaak startopposite \ +startoverlay startoverview startoverzicht startparagraph startpositioneren \ +startpositioning startpostponing startprodukt startprofiel startprofile \ +startprojekt startraster startregel startregelcorrectie startregelnummeren \ +startregels startregister startsmaller startsymbolset startsymboolset \ +startsynchronisatie startsynchronization starttabel starttabellen starttable \ +starttables starttabulate starttekst starttekstlijn starttyping \ +startuitlijnen startunpacked startvanelkaar startversie stelachtergrondenin \ +stelachtergrondin stelalineasin stelantwoordgebiedin stelarrangerenin stelblankoin \ +stelblokin stelblokjesin stelblokkopjein stelblokkopjesin stelbovenin \ +stelboventekstenin stelbufferin stelbuttonsin stelciterenin stelclipin \ +stelcombinatiesin stelcommentaarin steldoordefinierenin steldoornummerenin steldunnelijnenin \ +stelexternefigurenin stelformulesin stelformulierenin stelhoofdin stelhoofdtekstenin \ +stelingesprongentextin stelinmargein stelinspringenin stelinteractiebalkin stelinteractiein \ +stelinteractiemenuin stelinteractieschermin stelinterliniein stelinvullijnenin stelinvulregelsin \ +stelitemgroepin stelitemsin stelkadertekstenin stelkantlijnin stelkapitalenin \ +stelkleurenin stelkleurin stelkolomgroepin stelkolomgroepregelsin stelkolomgroepstartin \ +stelkolommenin stelkopin stelkopnummerin stelkoppeltekenin stelkoppenin \ +stelkoptekstin stelkorpsin stelkorpsomgevingin stellabeltekstin stellayoutin \ +stellegendain stellijndiktein stellijnin stellijstin stelmargeblokkenin \ +stelmarkeringin stelnaastplaatsenin stelnummerenin stelnummerin stelomlijndin \ +stelonderin stelonderstrepenin stelondertekstenin stelopmaakin stelopsommingenin \ +stelpaginacommentaarin stelpaginanummerin stelpaginanummeringin stelpaginaovergangenin stelpaletin \ +stelpapierformaatin stelpapierin stelparagraafnummerenin stelplaatsblokin stelplaatsblokkenin \ +stelplaatsbloksplitsenin stelplaatsin stelpositionerenin stelprofielenin stelprogrammasin \ +stelpublicatiesin stelrastersin stelreferentielijstin stelrefererenin stelregelnummerenin \ +stelregelsin stelregisterin stelroterenin stelsamengesteldelijstin stelsectieblokin \ +stelsectiein stelsmallerin stelsorterenin stelspatieringin stelstartstopin \ +stelstrutin stelsubpaginanummerin stelsymboolsetin stelsynchronisatiebalkin stelsynchronisatiein \ +stelsynoniemenin stelsysteemin steltaalin steltabellenin steltabin \ +steltabulatiein steltekstin steltekstinhoudin steltekstlijnenin steltekstpositiein \ +stelteksttekstenin steltekstvariabelein steltolerantiein steltypein steltypenin \ +steluitlijnenin steluitvoerin stelurlin stelveldenin stelveldin \ +stelversiesin stelvoetin stelvoetnootdefinitiein stelvoetnotenin stelvoettekstenin \ +stelwiskundeuitlijnenin stelwitruimtein stopachtergrond stopalignment stopbuffer \ +stopcitaat stopcodering stopcolumns stopcombinatie stopcombination \ +stopcomment stopdescription stopdocument stopenumeration stopfigure \ +stopfloattext stopformula stopframedtext stopgeg stopglobaal \ +stophiding stopinteractiemenu stopitemgroup stopkantlijn stopkleur \ +stopkolomgroep stopkolommen stopkolomopmaak stoplegend stopline \ +stoplinecorrection stoplinenumbering stoplines stoplocal stoplocalenvironment \ +stoplocalfootnotes stoplokaal stoplokalevoetnoten stopmakeup stopmargeblok \ +stopmarginblock stopnaast stopnamemakeup stopnarrower stopomgeving \ +stoponderdeel stopopelkaar stopopmaak stopopposite stopoverlay \ +stopoverview stopoverzicht stopparagraph stoppositioneren stoppositioning \ +stoppostponing stopprodukt stopprofiel stopprofile stopprojekt \ +stopraster stopregel stopregelcorrectie stopregelnummeren stopregels \ +stopsmaller stopsymbolset stopsynchronisatie stopsynchronization stoptabel \ +stoptabellen stoptable stoptables stoptabulate stoptekst \ +stoptekstlijn stoptyping stopuitlijnen stopunpacked stopvanelkaar \ +stopversie sub subformulenummer subject subpaginanummer \ +subsection subsubject subsubsection subsubsubject suggestie \ +switchnaarkorps switchtorawfont sym symbool symoffset \ +synchronisatiebalk synchroniseer synonym taal tab \ +tekstbreedte teksthoogte tekstlijn tekstreferentie tekstvariabele \ +testkolom testpagina tex title toelichting \ +toonexternefiguren toongrid tooninstellingen toonkader toonkleur \ +toonkleurgroep toonkorps toonkorpsomgeving toonlayout toonopmaak \ +toonpalet toonprint toonstruts toonsymboolset toonvelden \ +totaalaantalpaginas txt typ type typebuffer \ +typefile uit uitgerekt underbar underbars \ +usecodering usedirectory vastespatie vastespaties veld \ +veldstapel verbergblokken vergelijkkleurgroep vergelijkpalet verhoognummer \ +verlaagnummer verplaatsformule verplaatsopgrid verplaatszijblok versie \ +vertaal verwerkblokken verwerkpagina vl voetafstand \ +voethoogte voetnoot voetnoottekst volgprofiel volgprofielversie \ +volgversie volledigepaginanummer volledigregister voluit weekdag \ +wilijnd wiskunde witruimte woonplaats woordrechts \ +zetbreedte zethoogte + +keywordclass.context.it=\ +CAP Cap Caps GIORNOSETTIMANA \ +Lettera Lettere MESE Numeri Numeriromani \ +PAROLA PAROLE Parola Parole accoppiacarta \ +accoppiadocumento accoppiamarcatura accoppiapagina accoppiaregistro adattacampo \ +adattalayout al allineacentro allineadestra allineasinistra \ +altezzacarta altezzacartastampa altezzacima altezzaelenco altezzafondo \ +altezzaintestazione altezzamakeup altezzapdp altezzatesto ambiente \ +ampiezzabordo ampiezzabordodestro ampiezzabordosinistro ampiezzacarta ampiezzacartastampa \ +ampiezzaelenco ampiezzamakeup ampiezzamargine ampiezzamarginedestro ampiezzamarginesinistro \ +ampiezzatesto ap apagina appendix arg \ +atleftmargin atrightmargin barracolori barrainterazione barrasincronizzazione \ +bastablocchi bastafile cambiaafontdeltesto campi camporiempimento \ +cap capello chapter chim circondato \ +citazione clip clonacampo colonna colore \ +coloregrigio comment commento completecombinedlist completelistoffloats \ +completelistofsorts completelistofsynonyms completeregister componenet confrontagruppocolori \ +confrontatavolozza convertinumero copiacampo correggispaziobianco coupledregister \ +crlf cutspace da daqualcheparte data \ +datadioggi datareferral decrementnumber definebodyfontDEF definebodyfontREF \ +definecolumnbreak definecombination definedfont definefontfeature definefonthandling \ +defineindentedtext defineinmargin defineitemgroup definelayer definemathalignment \ +definepagebreak defineplacement definetypeface definisci definisciaccento \ +definisciambientefontdeltesto definisciblocco definiscibloccosezione definiscibuffer definiscicampo \ +definiscicampoprincipale definiscicapoversi definiscicarattere definiscicolore definiscicomando \ +definisciconversione definiscidescrizione definiscidimensionicarta definiscielenco definiscielencocombinato \ +definiscienumerazione definiscietichetta definiscifigurasimbolo definiscifont definiscifontdeltesto \ +definiscifontgrezzo definisciformatoriferimento definiscigruppocolonne definiscigruppocolori definiscihbox \ +definisciincorniciato definisciiniziatermina definiscilayout definiscilinea definiscilistariferimenti \ +definiscilogo definiscimakeup definiscimarcatura definiscimenuinterazione definiscimodellotabella \ +definiscioggettomobile definisciordinamento definiscioutput definisciposizionetesto definisciprofilo \ +definisciprogramma definisciregistro definisciriferimento definiscirigovuoto definiscisezione \ +definiscisimbolo definiscisinonimi definiscisinonimofont definiscisottocampo definiscisovrapposizione \ +definiscistackcampi definiscistile definiscistilefont definiscitabulato definiscitavolozza \ +definiscitesta definiscitesto definiscitestoincorniciato definiscitype definiscityping \ +definiscivariabiletesto definisciversion description determinacaratteristicheregistro determinacarattersticheelenco \ +determinanumerotesta dimensione disabilitamenuinterazione distanzabordo distanzabordodestro \ +distanzabordosinistro distanzacima distanzafondo distanzaintestazione distanzamargine \ +distanzamarginedestro distanzamarginesinistro distanzapdp domicilio el \ +elaborablocchi elaborapagina elementi elemento emptylines \ +enumeration etichetta etichette fatto figuraesterna \ +fondo forzablocchi framedtext frazione getnumber \ +giornosettimana griglia headsym hl ignoto \ +immediatebetweenlist immediatetolist impaccato impostaallineamento impostaambientefontdeltesto \ +impostaampiezzariga impostabarrainterazione impostabarrasincronizzazione impostablocchimargine impostablocco \ +impostabloccosezione impostabuffer impostacampi impostacampo impostacapoversi \ +impostacaption impostacaptions impostacima impostaclippling impostacolonne \ +impostacolore impostacolori impostacombinazioni impostacommento impostacommentopagina \ +impostadefinizionenotepdp impostadescrizioni impostadimensionicarta impostaelementi impostaelencazioni \ +impostaelenco impostaelencocombinato impostaenumerazioni impostafigureesterne impostafondo \ +impostafontdeltesto impostaforms impostaformule impostagruppocolonne impostaincorniciato \ +impostainiziatermina impostainmargine impostainstestazione impostainterazione impostainterlinea \ +impostalayout impostalegenda impostalinea impostalineemargine impostalineenere \ +impostalineeriempimento impostalineesottili impostalineetesto impostalingua impostalistariferimenti \ +impostamaiuscole impostamakeup impostamarcatura impostamenuinterazione impostamenzione \ +impostanotepdp impostanumerazione impostanumerazionecapoversi impostanumerazionepagina impostanumerazionerighe \ +impostanumeropagina impostanumerosottopagina impostanumerotesta impostaoggettimobili impostaoggettomobile \ +impostaordinamento impostaoutput impostaparranging impostapdp impostapiustretto \ +impostaposizionamento impostaposizionamentoopposti impostaposizionetesto impostaprofili impostaprogrammi \ +impostapubblicazioni impostapulsanti impostaregistro impostarientro impostariferimento \ +impostarighe impostarigheriempimento impostarigovuoto impostarotazione impostaschermi \ +impostaschermointerazione impostasegnosillabazione impostasetsimboli impostasezione impostasfondi \ +impostasfondo impostasincronizzazione impostasinonimi impostasistema impostasottolinea \ +impostaspaziatura impostaspaziobianco impostaspezzamentooggettomobile impostastrut impostatab \ +impostatabelle impostatabulato impostatavolozza impostatesta impostateste \ +impostatesticima impostatestifondo impostatestiincorniciati impostatestiintestazioni impostatestipdp \ +impostatesto impostatestoetichette impostatestointestazioni impostatestotesti impostatolleranza \ +impostatransizionepagina impostatype impostatyping impostaurl impostavariabiletesto \ +impostaversioni impostazioni in inaltromargine incorniciato \ +incrementanumero indentation indestra ininner iniziaallineamento \ +iniziaambiente iniziabloccomargine iniziacitazione iniziacodifica iniziacolonne \ +iniziacolore iniziacombinazione iniziacomponente iniziacorrezioneriga iniziadocumento \ +iniziafigura iniziaglobale iniziagruppocolonne iniziaimpaccato inizialineamargine \ +inizialineatesto inizialocale iniziamakeup inizianotepdplocali inizianumerazionerighe \ +iniziaopposto iniziaoverview iniziapiustretto iniziaposizionamento iniziaprodotto \ +iniziaprofilo iniziaprogetto iniziaraster iniziariga iniziarighe \ +iniziasetsimboli iniziasfondo iniziasincronizzazione iniziasovrapposizione iniziatabella \ +iniziatabelle iniziatesto iniziaunpacked iniziaversione inlatodestro \ +inlatosinistro inmaframed inmargine inmarginedestro inmarginesinistro \ +inneredgedistance inneredgewidth innermargindistance innermarginwidth inouter \ +inriga insinistra installalingua intorno labeling \ +leg lettera lettere lineamargine lineanera \ +lineasottile lineatesto lineenere lineeriempimento lineesottili \ +lingua linguaprincipale listsymbol livellotesta loadsorts \ +loadsynonyms logcampi lunghezzaelenco maframed mapfontsize \ +mar marcatura marcaversione matematica mediaeval \ +menuinterattivo menzione mese mettielenco mettielencocombinato \ +mettifiancoafianco mettiformula mettiingriglia mettilegenda mettilinea \ +mettiloghi mettinotepdp mettinotepdplocali mettinumeropagina mettiregistro \ +mettisegnalibro mettisottoformula mettiunosullaltro mettivariabiletesto mostraambientefontdeltesto \ +mostracampi mostracolore mostracornice mostrafiguresterne mostrafontdeltesto \ +mostragriglia mostragruppocolori mostraimpostazioni mostralyout mostramakeup \ +mostrasetsimboli mostrastampa mostrastruts mostratavolozza movesidefloat \ +name nascondiblocchi navigating nextsection nientedimensioni \ +nienteelenco nientelineecimafondo nientelineintestazionepdp nientemarcatura nienterientro \ +nientespazio nientespaziobianco nocap nome nomeunita \ +nop nota notapdp notest numberofsubpages \ +numeri numeriromani numeroformula numeropagina numeropaginacompleto \ +numerosottoformula numerotesta numerotestacorrente numerototaledipagine outeredgedistance \ +outeredgewidth outermargindistance outermarginwidth overbar overbars \ +overstrike overstrikes pagedepth pageoffset pagina \ +paragraph paroladestra parolainmargine part passaafontgrezzo \ +ped pedap perlungo placefloat placelistoffloats \ +placelistofsorts placelistofsynonyms placerawlist placereferencelist posizionanumerotesta \ +posizionatesto posizionatestotesta posizione prendibuffer prendimarcatura \ +prodotto progetto programma pubblicazione pulsante \ +pulsantemenu pulsantinterazione punti qualcheriga ran \ +referral referring register reimposta reimpostamarcatura \ +reservefloat resetnumber resettextcontent rientro rif \ +rifai riferimento riferimentopagina riferimentotesto riflessione \ +rigariempimento rigovuoto ruota saltablocchi scala \ +schermo scrividentroelenco scriviinelenco scriviinlistariferimenti scriviinregistro \ +section seeregister segnalibro seguiprofilo seguiversione \ +seguiversioneprofilo selezionablocchi selezionacarta selezionaversione separamarcatura \ +setnumber settext setupanswerarea setupcolumnsetlines setupcolumnsetstart \ +setupfonthandling setupfontsynonym setupindentedtext setupinterlinespace2 setupitemgroup \ +setuplistalternative setupmathalignment setuppaper setupplacement setvariabiletesto \ +sfondo sim simbolo sincronizza sort \ +spazifissi spazio spaziobianco spaziocima spaziodietro \ +spaziofisso spaziofondo spessoreriga spezzaoggettomobile spostaagriglia \ +spostaformula stackcampi startalignment startambiente startbuffer \ +startcitazione startcolore startcolumnmakeup startcolumns startcombination \ +startcomment startcomponenet startdescription startdocument startenumeration \ +startfatto startfigure startfloattext startformula startframedtext \ +starthiding startimpaccato startitemgroup startlegend startline \ +startlineamargine startlineatesto startlinecorrection startlinenumbering startlines \ +startlocal startlocalenvironment startlocalfootnotes startmakeup startmarginblock \ +startmenuinterattivo startnamemakeup startnarrower startopposite startoverlay \ +startoverview startparagraph startpositioning startpostponing startprodotto \ +startprofile startprogetto startregister startsfondo startsymbolset \ startsynchronization starttable starttables starttabulate starttyping \ -startunpacked startversion stopalignment stoparriereplan stopbuffer \ -stopcitation stopcolumnmakeup stopcolumns stopcombination stopcomment \ -stopcompoetroite stopcomposant stopcouleur stopdescription stopdocument \ -stopenumeration stopenvironement stopfait stopfigure stopfloattext \ -stopformula stopframedtext stopgroupe stophiding stopitemgroup \ -stoplegend stopligneregleetexte stopline stoplinecorrection stoplinenumbering \ +startunpacked startversione stirato stopalignment stopambiente \ +stopbuffer stopcitazione stopcolore stopcolumnmakeup stopcolumns \ +stopcombination stopcomment stopcomponenet stopdescription stopdocument \ +stopenumeration stopfatto stopfigure stopfloattext stopformula \ +stopframedtext stophiding stopimpaccato stopitemgroup stoplegend \ +stopline stoplineamargine stoplineatesto stoplinecorrection stoplinenumbering \ stoplines stoplocal stoplocalenvironment stoplocalfootnotes stopmakeup \ -stopmargereglee stopmarginblock stopmenuinteraction stopnamemakeup stopnarrower \ -stopopposite stopoverlay stopoverview stopparagraph stoppealignement \ -stoppearriereplan stoppeblocmarge stoppecitation stoppecodage stoppecolonnes \ -stoppecombinaison stoppecomposant stoppecorrectionligne stoppecouleur stoppedegroupe \ -stoppedocument stoppeenvironement stoppeglobal stoppegroupe stoppejeucolonne \ -stoppeligne stoppeligneregleetexte stoppelignes stoppelocal stoppemakeup \ -stoppemargereglee stoppenotespdplocales stoppenumerotationligne stoppeopposition stoppepositionnement \ -stoppeproduit stoppeprofil stoppeprojet stopperaster stopperevetement \ -stoppesynchronisation stoppetableau stoppetableaux stoppetexte stoppeversion \ -stoppevuedensemble stoppositioning stoppostponing stopproduit stopprofile \ -stopprojet stopsymbolset stopsynchronization stoptable stoptables \ -stoptabulate stoptyping stopunpacked stopversion sub \ -subject subsection subsubject subsubsection subsubsubject \ -suggestion suivantprofil suivantversion suivantversionprofil sym \ -symbole synchronise synonym tab tapebuffer \ -testcolumn testpage tete tex textemarge \ -textenotepdp textetete title titremarge traduire \ -traiteblocs traitepage traitfin traitsfins txt \ -typ underbar underbars uneligne useXMLfilter \ -usedirectory usetypescript usetypescriptfile utiliseJSscripts utiliseURL \ -utiliseblocs utilisechemin utilisecommandes utilisedocumentexterne utiliseencodage \ -utilisefichierexterne utilisefichiersexternes utilisefigureexterne utilisemodule utilisemodules \ -utilisepsiteaudioexterne utilisereferences utilisespecialites utilisesymboles utiliseurl \ -va vaalaboite vaalapage vaenbas valeurcouleur \ -valeurgris variabletexte version vide vl +stopmarginblock stopmenuinterattivo stopnamemakeup stopnarrower stopopposite \ +stopoverlay stopoverview stopparagraph stoppositioning stoppostponing \ +stopprodotto stopprofile stopprogetto stopsfondo stopsymbolset \ +stopsynchronization stoptable stoptables stoptabulate stoptyping \ +stopunpacked stopversione sub subject subpagenumber \ +subsection subsubject subsubsection subsubsubject synonym \ +tab terminaallineamento terminaambiente terminabloccomargine terminacitazione \ +terminacodifica terminacolonne terminacolore terminacombinazione terminacomponente \ +terminacorrezioneriga terminadocumento terminaglobale terminagruppocolonne terminaimpaccato \ +terminalineamargine terminalineatesto terminalocale terminamakeup terminanotepdplocali \ +terminanumerazionerighe terminaopposto terminaoverview terminapiustretto terminaposizionamento \ +terminaprodotto terminaprofili terminaprogetto terminaraster terminariga \ +terminarighe terminasfondo terminasincronizzazione terminasovrapposizione terminatabella \ +terminatabelle terminatesto terminaunpacked terminaversioni testa \ +testcolumn testoetichetta testoinmargine testoinstestazioni testonotapdp \ +testoriempimento testpage tex tieniblocchi title \ +titoloinmargine tooltip traduci txt typ \ +type typebuffer typefile underbar underbars \ +usaJSscripts usaURL usablocco usacartella usacodifica \ +usacolonnasonoraesterna usacomandi usadocumentoesterno usafiguraesterna usafileesterni \ +usafileesterno usamoduli usamodulo usariferimenti usasimboli \ +usaspecialita usaurl useXMLfilter usedirectory usetypescript \ +usetypescriptfile vaia vaiabox vaiapagina vaigiu \ +valorecolore valoregrigio variabiletesto versione vl + +keywordclass.context.ro=\ +CAP CUVANT CUVINTE Cap \ +Caps Cuvant Cuvinte KAP Kap \ +Kaps LUNA Litera Litere Numere \ +Numereromane ZIDINSAPTAMANA adapteazaaspect adubuffer adumarcaje \ +afiseazaaspect afiseazacampuri afiseazaculoare afiseazafiguriexterne afiseazafonttext \ +afiseazagrid afiseazagrupculoare afiseazamakeup afiseazamediufonttext afiseazapaleta \ +afiseazarama afiseazasetari afiseazasetsimboluri afiseazastruts afiseazatiparire \ +aliniat aliniatcentru aliniatdreapta aliniatstanga appendix \ +arg ascundeblocuri atleftmargin atrightmargin baraculoare \ +barainteractiune barasincronizare blanc but butoaneinteractiune \ +buton butonmeniu camp campumplere cap \ +chapter chem citat clip cloneazacamp \ +coloana comment comparagrupculoare comparapaleta completeazanumarpagina \ +completecombinedlist completelistoffloats completelistofsorts completelistofsynonyms completeregister \ +componenta convertestenumar copiazacamp corecteazaspatiualb coupledregister \ +crlf culoare culoaregri cupleazadocument cupleazamarcaje \ +cupleazaregistru cutspace cuvantdreapta cuvantmarginal data \ +datacurenta datareferit decrementnumber decupleazamarcaje definebodyfontDEF \ +definebodyfontREF definecolumnbreak definecolumnset definecombination definedfont \ +definefontfeature definefonthandling defineindentedtext defineinmargin defineitemgroup \ +definelayer definelayout definemathalignment definepagebreak defineplacement \ +defineste definesteaccent definesteantet definesteblanc definestebloc \ +definesteblocsectiune definestebuffer definestecamp definestecampprincipal definestecaracter \ +definestecomanda definesteconversie definesteculoare definestedescriere definestedimensiunehartie \ +definesteenumerare definesteeticheta definestefloat definestefont definestefontraw \ +definestefonttext definesteformatreferinte definestegrupculori definestehbox definesteinconjurare \ +definestelista definestelistacombinata definestelistareferinte definestelogo definestemakeup \ +definestemarcaje definestemediulfonttext definestemeniuinteractiune definesteoutput definesteoverlay \ +definestepaleta definesteparagraf definestepozitietext definesteprofil definesteprogram \ +definestereferinte definesteregistru definesterigla definestesablontabel definestesectiune \ +definestesimbol definestesimbolfigura definestesinonim definestesinonimfont definestesortare \ +definestestartstop definestestil definestestilfont definestestivacampuri definestesubcamp \ +definestetabulatori definestetext definestetexteinconjurate definestetextinconjurat definestetyping \ +definestevariabilatext definesteversiune definetype definetypeface description \ +despre determinacaracteristicilelistei determinacaracteristiciregistru determinanumartitlu dezactiveazameniuinteractiune \ +dimensiune din distantaantet distantacolt distantacoltdreapta \ +distantacoltstanga distantajos distantamargine distantamarginedreapta distantamarginestanga \ +distantasubsol distantasus domiciliu dute dutebox \ +dutepagina ecran el element emptylines \ +enumeration eticheta etichete fact faraaliniat \ +faradimensiune farafisiere faraliniiantetsisubsol faraliniisussijos faralista \ +faramarcaje faraspatiu faraspatiualb figuraexterna firdepar \ +folosesteURL folosestebloc folosestecodificarea folosestecomenzi folosestedirector \ +folosestedocumentextern folosestefiguraexterna folosestefisiereexterne folosestefisierextern folosestemodul \ +folosestemodule folosestemuzicaexterna folosestereferinte folosestescriptJS folosestesimboluri \ +folosestespeciale folosesteurl footnotetext forteazablocuri fractie \ +framed framedtext fundal gatablocuri getnumber \ +grid grosimelinie hartiedubla headsym hl \ +immediatebetweenlist immediatetolist impachetat impartefloat in \ +inalt inaltamargine inaltimeantet inaltimehartie inaltimehartieimprimanta \ +inaltimejos inaltimelista inaltimemakeup inaltimesubsol inaltimesus \ +inaltimetext indentation indreapta inframed ininner \ +injos inlinie inmaframed inmargineadreapta inmargineastanga \ +inneredgedistance inneredgewidth innermargindistance innermarginwidth inouter \ +inparteadreapta inparteastanga instalarelimba instanga intins \ +jos jossus kap la labeling \ +lapagina latimecoltdreapta latimecoltstanga latimecolturi latimehartie \ +latimehartieimprimanta latimelista latimemakeup latimemargine latimemarginedreapta \ +latimemarginestanga latimetext leg limba limbaprincipala \ +liniemargine linieneagra liniesubtire linieumplere liniinegre \ +liniisubtiri listsymbol litera litere loadsorts \ +loadsynonyms logcampuri luna lungimelista maframed \ +mapfontsize mar marcaje marcheazaversiune marginal \ +matematica mediaeval mediu meniuinteractiune minicitat \ +moveformula movesidefloat mutapegrid name navigating \ +necunoscut nextsection niveltitlu nocap nokap \ +nop nota notasubsol numarformula numarincrement \ +numarpagina numarsubformula numartitlu numartitlucurent numartotalpagini \ +numberofsubpages nume numere numereromane numeunitate \ +nutesta olinie outeredgedistance outeredgewidth outermargindistance \ +outermarginwidth overbar overbars overstrike overstrikes \ +pagedepth pageoffset pagina paginadubla paragraph \ +part pastreazablocuri pelung placefloat placeheadnumber \ +placeheadtext placelistoffloats placelistofsorts placelistofsynonyms placerawlist \ +placereferencelist plaseazapegrid plaseazasemnecarte potrivestecamp pozitie \ +pozitietext proceseazabloc proceseazapagina produs program \ +proiect publicatie puncte punedeasuprafiecareia punefatainfata \ +puneformula punelegenda punelista punelistacombinata punelogouri \ +punenotesubsol punenotesubsollocale punenumarpagina puneregistru punerigla \ +punesubformula punevariabilatext ran ref refa \ +referinta referintapagina referintatext referit referring \ +reflexie register remarca reservefloat reset \ +reseteazamarcaje resetnumber resettextcontent riglatext rigleumplere \ +roteste saripesteblocuri scala scriebuffer scrieinlista \ +scrieinlistareferinte scrieinregistru scrieintreliste section seeregister \ +selecteazablocuri selecteazahartie selecteazaversiune semncarte setarebarasincronizare \ +setareitemization setarelimba setareoutput setarepozitie setaresincronizare \ +setari seteazaaliniat seteazaalinierea seteazaantet seteazaaranjareapag \ +seteazaaspect seteazabarainteractiune seteazablanc seteazabloc seteazablocsectiune \ +seteazablocurimarginale seteazabuffer seteazabutoane seteazacamp seteazacampuri \ +seteazaclipping seteazacoloane seteazacombinari seteazacomentariu seteazacomentariupagina \ +seteazaculoare seteazaculori seteazadefinireanotasubsol seteazadescriere seteazadimensiunihartie \ +seteazaecrane seteazaecraninteractiune seteazaelemente seteazaenumerare seteazafiguriexterne \ +seteazafloat seteazafloats seteazafonttext seteazaformulare seteazaformule \ +seteazafundal seteazafundaluri seteazagrosimelinie seteazaimpartireafloat seteazainconjurat \ +seteazaingust seteazainteractiunea seteazajos seteazalegenda seteazalegendele \ +seteazaliniesilabe seteazaliniesubtire seteazalinii seteazaliniimargine seteazaliniinegre \ +seteazaliniiumplere seteazalista seteazalistacombinata seteazalistareferinte seteazamajuscule \ +seteazamakeup seteazamarcaje seteazamarginal seteazamediulfonttext seteazameniuinteractiune \ +seteazaminicitat seteazanotasubsol seteazanumarpagina seteazanumarsubpagina seteazanumartitlu \ +seteazanumerotare seteazanumerotarelinii seteazanumerotarepagina seteazanumerotareparagrafe seteazapaleta \ +seteazaparagrafe seteazaplasareaopozita seteazapozitietext seteazaprofile seteazaprograme \ +seteazapublicatii seteazareferinte seteazaregistru seteazarigla seteazarigletext \ +seteazarigleumplere seteazarotare seteazasectiune seteazasimbol seteazasinonime \ +seteazasistem seteazasortare seteazaspatiu seteazaspatiualb seteazaspatiuinterliniar \ +seteazastrut seteazasublinie seteazasubsol seteazasus seteazatab \ +seteazatabele seteazatabulatori seteazatext seteazatexteantet seteazatextejos \ +seteazatextesubsol seteazatextesus seteazatextetext seteazatexteticheta seteazatexttitlu \ +seteazatitlu seteazatitluri seteazatoleranta seteazatranzitiepagina seteazatype \ +seteazatyping seteazaurl seteazavariabilatext seteazaversiuni setnumber \ +settextcontent setupanswerarea setupcolumnset setupcolumnsetlines setupcolumnsetstart \ +setupfonthandling setupfontsynonym setupindentedtext setupinterlinespace2 setupitemgroup \ +setuplistalternative setupmathalignment setupnumber setuppaper setupplacement \ +setupstartstop setvariabilatext sim simbol sincronizeaza \ +sort spatiifixate spatiu spatiualb spatiufixat \ +spatiujos spatiuspate spatiusus startalignment startaliniere \ +startblocmarginal startbuffer startcitat startcodificare startcoloane \ +startcolumnmakeup startcolumns startcolumnset startcombinare startcombination \ +startcomment startcomponenta startcorectielinie startculoare startdescription \ +startdocument startenumeration startfact startfigura startfigure \ +startfloattext startformula startframedtext startfundal startglobal \ +starthiding startimpachetat startingust startitemgroup startlegend \ +startline startlinecorrection startlinenumbering startlines startlinie \ +startliniemargine startlinii startlocal startlocalenvironment startlocalfootnotes \ +startmakeup startmarginblock startmediu startmeniuinteractiune startnamemakeup \ +startnarrower startneimpachetat startnotesubsollocale startnumerotarelinii startopozit \ +startopposite startoverlay startoverview startparagraph startpositioning \ +startpostponing startpozitionare startprodus startprofil startprofile \ +startproiect startraster startregister startriglatext startsetsimboluri \ +startsincronizare startsymbolset startsynchronization starttabel starttabele \ +starttable starttables starttabulate starttext starttyping \ +startunpacked startversiune stivacampuri stopalignment stopaliniere \ +stopblobal stopblocmarginal stopbuffer stopcitat stopcodificare \ +stopcoloane stopcolumnmakeup stopcolumns stopcolumnset stopcombinare \ +stopcombination stopcomment stopcomponenta stopcorectielinie stopculoare \ +stopdescription stopdocument stopenumeration stopfact stopfigure \ +stopfloattext stopformula stopframedtext stopfundal stophiding \ +stopimpachetat stopingust stopitemgroup stoplegend stopline \ +stoplinecorrection stoplinenumbering stoplines stoplinie stopliniemargine \ +stoplinii stoplocal stoplocalenvironment stoplocalfootnotes stopmakeup \ +stopmarginblock stopmediu stopmeniuinteractiune stopnamemakeup stopnarrower \ +stopneimpachetat stopnotesubsollocale stopnumerotarelinii stopopozit stopopposite \ +stopoverlay stopoverview stopparagraph stoppositioning stoppostponing \ +stoppozitionare stopprodus stopprofil stopprofile stopproiect \ +stopraster stopriglatext stopsincronizare stopsymbolset stopsynchronization \ +stoptabel stoptabele stoptable stoptables stoptabulate \ +stoptext stoptyping stopunpacked stopversiune sub \ +subject subpagenumber subsection subsubject subsubsection \ +subsubsubject synonym tab testcolumn testpage \ +tex texteticheta textmarginal texttitlu textumplere \ +title titlu titlumarginal tooltip traduce \ +trecilafontraw trecilafonttext txt typ type \ +typefile underbar underbars undeva urmeazaprofil \ +urmeazaversiune urmeazaversiuneprofil useXMLfilter usedirectory usetypescript \ +usetypescriptfile valoareculoare valoaregri variabilatext versiune \ +vl zidinsaptamana keywordclass.context.cs=\ CAP Cap Caps Cisla \ diff --git a/context/data/scite/scite-context-data-metafun.properties b/context/data/scite/scite-context-data-metafun.properties index c5aa4cbed..80cb13417 100644 --- a/context/data/scite/scite-context-data-metafun.properties +++ b/context/data/scite/scite-context-data-metafun.properties @@ -1,10 +1,3 @@ -keywordclass.metafun.internals=\ -nocolormodel greycolormodel graycolormodel rgbcolormodel \ -cmykcolormodel shadefactor textextoffset normaltransparent multiplytransparent \ -screentransparent overlaytransparent softlighttransparent hardlighttransparent colordodgetransparent \ -colorburntransparent darkentransparent lightentransparent differencetransparent exclusiontransparent \ -huetransparent saturationtransparent colortransparent luminositytransparent metapostversion - keywordclass.metafun.commands=\ sqr log ln exp \ inv pow pi radian tand \ @@ -52,3 +45,10 @@ drawpoints drawcontrolpoints drawcontrollines drawpointlabels drawlineoptions \ drawpointoptions drawcontroloptions drawlabeloptions draworiginoptions drawboundoptions \ drawpathoptions resetdrawoptions decorated redecorated undecorated +keywordclass.metafun.internals=\ +nocolormodel greycolormodel graycolormodel rgbcolormodel \ +cmykcolormodel shadefactor textextoffset normaltransparent multiplytransparent \ +screentransparent overlaytransparent softlighttransparent hardlighttransparent colordodgetransparent \ +colorburntransparent darkentransparent lightentransparent differencetransparent exclusiontransparent \ +huetransparent saturationtransparent colortransparent luminositytransparent metapostversion + diff --git a/context/data/scite/scite-context-data-metapost.properties b/context/data/scite/scite-context-data-metapost.properties index eda94babd..1f2d0ca1b 100644 --- a/context/data/scite/scite-context-data-metapost.properties +++ b/context/data/scite/scite-context-data-metapost.properties @@ -1,9 +1,6 @@ -keywordclass.metapost.internals=\ -mitered rounded beveled butt \ -squared eps epsilon infinity bboxmargin \ -ahlength ahangle labeloffset dotlabeldiam defaultpen \ -defaultscale join_radius pen_lft pen_rt pen_top \ -pen_bot +keywordclass.metapost.shortcuts=\ +.. ... -- --- \ +& keywordclass.metapost.commands=\ beginfig endfig rotatedaround reflectedabout \ @@ -36,9 +33,6 @@ yellow black white background graypart \ graycolor mm pt dd bp \ cm pc cc in -keywordclass.metapost.tex=\ -btex etex verbatimtex - keywordclass.metapost.primitives=\ charcode day linecap linejoin \ miterlimit month pausing prologues showstopping \ @@ -92,7 +86,13 @@ graycolor colormodel graypart dashpart penpart \ stroked filled textual clipped bounded \ expandafter -keywordclass.metapost.shortcuts=\ -.. ... -- --- \ -& +keywordclass.metapost.tex=\ +btex etex verbatimtex + +keywordclass.metapost.internals=\ +mitered rounded beveled butt \ +squared eps epsilon infinity bboxmargin \ +ahlength ahangle labeloffset dotlabeldiam defaultpen \ +defaultscale join_radius pen_lft pen_rt pen_top \ +pen_bot diff --git a/context/data/scite/scite-context-data-tex.properties b/context/data/scite/scite-context-data-tex.properties index 27a74c070..b754dd256 100644 --- a/context/data/scite/scite-context-data-tex.properties +++ b/context/data/scite/scite-context-data-tex.properties @@ -1,3 +1,65 @@ +keywordclass.tex.pdftex=\ +efcode expanded ifincsname ifpdfabsdim \ +ifpdfabsnum ifpdfprimitive leftmarginkern letterspacefont lpcode \ +pdfadjustspacing pdfannot pdfcatalog pdfcolorstack pdfcolorstackinit \ +pdfcompresslevel pdfcopyfont pdfcreationdate pdfdecimaldigits pdfdest \ +pdfdestmargin pdfdraftmode pdfeachlinedepth pdfeachlineheight pdfendlink \ +pdfendthread pdffirstlineheight pdffontattr pdffontexpand pdffontname \ +pdffontobjnum pdffontsize pdfgamma pdfgentounicode pdfglyphtounicode \ +pdfhorigin pdfignoreddimen pdfimageapplygamma pdfimagegamma pdfimagehicolor \ +pdfimageresolution pdfincludechars pdfinclusioncopyfonts pdfinclusionerrorlevel pdfinfo \ +pdfinsertht pdflastannot pdflastlinedepth pdflastlink pdflastobj \ +pdflastxform pdflastximage pdflastximagecolordepth pdflastximagepages pdflastxpos \ +pdflastypos pdflinkmargin pdfliteral pdfmapfile pdfmapline \ +pdfminorversion pdfnames pdfnoligatures pdfnormaldeviate pdfobj \ +pdfobjcompresslevel pdfoptionpdfminorversion pdfoutline pdfoutput pdfpageattr \ +pdfpagebox pdfpageheight pdfpageref pdfpageresources pdfpagesattr \ +pdfpagewidth pdfpkmode pdfpkresolution pdfprimitive pdfprotrudechars \ +pdfpxdimen pdfrandomseed pdfrefobj pdfrefxform pdfrefximage \ +pdfreplacefont pdfrestore pdfretval pdfsave pdfsavepos \ +pdfsetmatrix pdfsetrandomseed pdfstartlink pdfstartthread pdftexbanner \ +pdftexrevision pdftexversion pdfthread pdfthreadmargin pdftracingfonts \ +pdftrailer pdfuniformdeviate pdfuniqueresname pdfvorigin pdfxform \ +pdfxformattr pdfxformname pdfxformresources pdfximage pdfximagebbox \ +quitvmode rightmarginkern rpcode tagcode + +keywordclass.tex.luatex=\ +Udelcode Udelcodenum Udelimiter Udelimiterover \ +Udelimiterunder Umathaccent Umathaccents Umathaxis Umathbinbinspacing \ +Umathbinclosespacing Umathbininnerspacing Umathbinopenspacing Umathbinopspacing Umathbinordspacing \ +Umathbinpunctspacing Umathbinrelspacing Umathbotaccent Umathchar Umathchardef \ +Umathcharnum Umathclosebinspacing Umathcloseclosespacing Umathcloseinnerspacing Umathcloseopenspacing \ +Umathcloseopspacing Umathcloseordspacing Umathclosepunctspacing Umathcloserelspacing Umathcode \ +Umathcodenum Umathconnectoroverlapmin Umathfractiondelsize Umathfractiondenomdown Umathfractiondenomvgap \ +Umathfractionnumup Umathfractionnumvgap Umathfractionrule Umathinnerbinspacing Umathinnerclosespacing \ +Umathinnerinnerspacing Umathinneropenspacing Umathinneropspacing Umathinnerordspacing Umathinnerpunctspacing \ +Umathinnerrelspacing Umathlimitabovebgap Umathlimitabovekern Umathlimitabovevgap Umathlimitbelowbgap \ +Umathlimitbelowkern Umathlimitbelowvgap Umathopbinspacing Umathopclosespacing Umathopenbinspacing \ +Umathopenclosespacing Umathopeninnerspacing Umathopenopenspacing Umathopenopspacing Umathopenordspacing \ +Umathopenpunctspacing Umathopenrelspacing Umathoperatorsize Umathopinnerspacing Umathopopenspacing \ +Umathopopspacing Umathopordspacing Umathoppunctspacing Umathoprelspacing Umathordbinspacing \ +Umathordclosespacing Umathordinnerspacing Umathordopenspacing Umathordopspacing Umathordordspacing \ +Umathordpunctspacing Umathordrelspacing Umathoverbarkern Umathoverbarrule Umathoverbarvgap \ +Umathoverdelimiterbgap Umathoverdelimitervgap Umathpunctbinspacing Umathpunctclosespacing Umathpunctinnerspacing \ +Umathpunctopenspacing Umathpunctopspacing Umathpunctordspacing Umathpunctpunctspacing Umathpunctrelspacing \ +Umathquad Umathradicaldegreeafter Umathradicaldegreebefore Umathradicaldegreeraise Umathradicalkern \ +Umathradicalrule Umathradicalvgap Umathrelbinspacing Umathrelclosespacing Umathrelinnerspacing \ +Umathrelopenspacing Umathrelopspacing Umathrelordspacing Umathrelpunctspacing Umathrelrelspacing \ +Umathspaceafterscript Umathstackdenomdown Umathstacknumup Umathstackvgap Umathsubshiftdown \ +Umathsubshiftdrop Umathsubsupshiftdown Umathsubsupvgap Umathsubtopmax Umathsupbottommin \ +Umathsupshiftdrop Umathsupshiftup Umathsupsubbottommax Umathunderbarkern Umathunderbarrule \ +Umathunderbarvgap Umathunderdelimiterbgap Umathunderdelimitervgap Uoverdelimiter Uradical \ +Uroot Ustack Ustartdisplaymath Ustartmath Ustopdisplaymath \ +Ustopmath Usubscript Usuperscript Uunderdelimiter alignmark \ +aligntab attribute attributedef catcodetable clearmarks \ +crampeddisplaystyle crampedscriptscriptstyle crampedscriptstyle crampedtextstyle fontid \ +formatname gleaders ifabsdim ifabsnum ifprimitive \ +initcatcodetable latelua luaescapestring luastartup luatexdatestamp \ +luatexrevision luatexversion mathstyle nokerns noligs \ +outputbox pageleftoffset pagetopoffset postexhyphenchar posthyphenchar \ +preexhyphenchar prehyphenchar primitive savecatcodetable scantextokens \ +suppressfontnotfounderror suppressifcsnameerror suppresslongerror suppressoutererror synctex + keywordclass.tex.etex=\ botmarks clubpenalties currentgrouplevel currentgrouptype \ currentifbranch currentiflevel currentiftype detokenize dimexpr \ @@ -161,31 +223,6 @@ vrule vsize vskip vsplit vss \ vtop wd widowpenalties widowpenalty write \ xdef xleaders xspaceskip year -keywordclass.tex.pdftex=\ -efcode expanded ifincsname ifpdfabsdim \ -ifpdfabsnum ifpdfprimitive leftmarginkern letterspacefont lpcode \ -pdfadjustspacing pdfannot pdfcatalog pdfcolorstack pdfcolorstackinit \ -pdfcompresslevel pdfcopyfont pdfcreationdate pdfdecimaldigits pdfdest \ -pdfdestmargin pdfdraftmode pdfeachlinedepth pdfeachlineheight pdfendlink \ -pdfendthread pdffirstlineheight pdffontattr pdffontexpand pdffontname \ -pdffontobjnum pdffontsize pdfgamma pdfgentounicode pdfglyphtounicode \ -pdfhorigin pdfignoreddimen pdfimageapplygamma pdfimagegamma pdfimagehicolor \ -pdfimageresolution pdfincludechars pdfinclusioncopyfonts pdfinclusionerrorlevel pdfinfo \ -pdfinsertht pdflastannot pdflastlinedepth pdflastlink pdflastobj \ -pdflastxform pdflastximage pdflastximagecolordepth pdflastximagepages pdflastxpos \ -pdflastypos pdflinkmargin pdfliteral pdfmapfile pdfmapline \ -pdfminorversion pdfnames pdfnoligatures pdfnormaldeviate pdfobj \ -pdfobjcompresslevel pdfoptionpdfminorversion pdfoutline pdfoutput pdfpageattr \ -pdfpagebox pdfpageheight pdfpageref pdfpageresources pdfpagesattr \ -pdfpagewidth pdfpkmode pdfpkresolution pdfprimitive pdfprotrudechars \ -pdfpxdimen pdfrandomseed pdfrefobj pdfrefxform pdfrefximage \ -pdfreplacefont pdfrestore pdfretval pdfsave pdfsavepos \ -pdfsetmatrix pdfsetrandomseed pdfstartlink pdfstartthread pdftexbanner \ -pdftexrevision pdftexversion pdfthread pdfthreadmargin pdftracingfonts \ -pdftrailer pdfuniformdeviate pdfuniqueresname pdfvorigin pdfxform \ -pdfxformattr pdfxformname pdfxformresources pdfximage pdfximagebbox \ -quitvmode rightmarginkern rpcode tagcode - keywordclass.tex.aleph=\ AlephVersion Alephminorversion Alephrevision Alephversion \ Omegaminorversion Omegarevision Omegaversion boxdir pagebottomoffset \ @@ -199,40 +236,3 @@ omathaccent omathchar omathchardef omathcode oradical \ pagedir pageheight pagewidth pardir rightghost \ textdir -keywordclass.tex.luatex=\ -Udelcode Udelcodenum Udelimiter Udelimiterover \ -Udelimiterunder Umathaccent Umathaccents Umathaxis Umathbinbinspacing \ -Umathbinclosespacing Umathbininnerspacing Umathbinopenspacing Umathbinopspacing Umathbinordspacing \ -Umathbinpunctspacing Umathbinrelspacing Umathbotaccent Umathchar Umathchardef \ -Umathcharnum Umathclosebinspacing Umathcloseclosespacing Umathcloseinnerspacing Umathcloseopenspacing \ -Umathcloseopspacing Umathcloseordspacing Umathclosepunctspacing Umathcloserelspacing Umathcode \ -Umathcodenum Umathconnectoroverlapmin Umathfractiondelsize Umathfractiondenomdown Umathfractiondenomvgap \ -Umathfractionnumup Umathfractionnumvgap Umathfractionrule Umathinnerbinspacing Umathinnerclosespacing \ -Umathinnerinnerspacing Umathinneropenspacing Umathinneropspacing Umathinnerordspacing Umathinnerpunctspacing \ -Umathinnerrelspacing Umathlimitabovebgap Umathlimitabovekern Umathlimitabovevgap Umathlimitbelowbgap \ -Umathlimitbelowkern Umathlimitbelowvgap Umathopbinspacing Umathopclosespacing Umathopenbinspacing \ -Umathopenclosespacing Umathopeninnerspacing Umathopenopenspacing Umathopenopspacing Umathopenordspacing \ -Umathopenpunctspacing Umathopenrelspacing Umathoperatorsize Umathopinnerspacing Umathopopenspacing \ -Umathopopspacing Umathopordspacing Umathoppunctspacing Umathoprelspacing Umathordbinspacing \ -Umathordclosespacing Umathordinnerspacing Umathordopenspacing Umathordopspacing Umathordordspacing \ -Umathordpunctspacing Umathordrelspacing Umathoverbarkern Umathoverbarrule Umathoverbarvgap \ -Umathoverdelimiterbgap Umathoverdelimitervgap Umathpunctbinspacing Umathpunctclosespacing Umathpunctinnerspacing \ -Umathpunctopenspacing Umathpunctopspacing Umathpunctordspacing Umathpunctpunctspacing Umathpunctrelspacing \ -Umathquad Umathradicaldegreeafter Umathradicaldegreebefore Umathradicaldegreeraise Umathradicalkern \ -Umathradicalrule Umathradicalvgap Umathrelbinspacing Umathrelclosespacing Umathrelinnerspacing \ -Umathrelopenspacing Umathrelopspacing Umathrelordspacing Umathrelpunctspacing Umathrelrelspacing \ -Umathspaceafterscript Umathstackdenomdown Umathstacknumup Umathstackvgap Umathsubshiftdown \ -Umathsubshiftdrop Umathsubsupshiftdown Umathsubsupvgap Umathsubtopmax Umathsupbottommin \ -Umathsupshiftdrop Umathsupshiftup Umathsupsubbottommax Umathunderbarkern Umathunderbarrule \ -Umathunderbarvgap Umathunderdelimiterbgap Umathunderdelimitervgap Uoverdelimiter Uradical \ -Uroot Ustack Ustartdisplaymath Ustartmath Ustopdisplaymath \ -Ustopmath Usubscript Usuperscript Uunderdelimiter alignmark \ -aligntab attribute attributedef catcodetable clearmarks \ -crampeddisplaystyle crampedscriptscriptstyle crampedscriptstyle crampedtextstyle fontid \ -formatname gleaders ifabsdim ifabsnum ifprimitive \ -initcatcodetable latelua luaescapestring luastartup luatexdatestamp \ -luatexrevision luatexversion mathstyle nokerns noligs \ -outputbox pageleftoffset pagetopoffset postexhyphenchar posthyphenchar \ -preexhyphenchar prehyphenchar primitive savecatcodetable scantextokens \ -suppressfontnotfounderror suppressifcsnameerror suppresslongerror suppressoutererror synctex - diff --git a/context/data/scite/scite-ctx.lua b/context/data/scite/scite-ctx.lua index be3718043..546cac411 100644 --- a/context/data/scite/scite-ctx.lua +++ b/context/data/scite/scite-ctx.lua @@ -70,7 +70,7 @@ props = props or { } -- setmetatable(props,{ __index = function(k,v) props[k] = "unknown" return "unknown" end } ) -local byte, lower, upper, gsub, sub, find, rep, match, gmatch = string.byte, string.lower, string.upper, string.gsub, string.sub, string.find, string.rep, string.match, string.gmatch +local byte, lower, upper, gsub, sub, find, rep, match, gmatch, format = string.byte, string.lower, string.upper, string.gsub, string.sub, string.find, string.rep, string.match, string.gmatch, string.format local sort, concat = table.sort, table.concat local crlf = "\n" @@ -190,7 +190,7 @@ function extend_to_end() -- editor:LineEndExtend() does not work end function getfiletype() - local firstline = editor:GetLine(0) + local firstline = editor:GetLine(0) or "" if editor.Lexer == SCLEX_TEX then return 'tex' elseif editor.Lexer == SCLEX_XML then @@ -246,7 +246,7 @@ do print("\n- ctx.wraptext.length is set to " .. props['ctx.wraptext.length']) if props['ctx.helpinfo'] ~= '' then print("\n- key bindings:\n") - print((gsub(strip(props['ctx.helpinfo']),"%s*\|%s*","\n"))) + print((gsub(strip(props['ctx.helpinfo']),"%s*|%s*","\n"))) end print("\n- recognized first lines:\n") print("xml [^<%-][^<%-]+)([-\/])(%w%w+)","%1%3")) + editor:ReplaceSel(gsub(editor:GetSelText(),"(>[^<%-][^<%-]+)([-/])(%w%w+)","%1%3")) else - editor:ReplaceSel(gsub(editor:GetSelText(),"([^\|])([-\/]+)([^\|])","%1|%2|%3")) + editor:ReplaceSel(gsub(editor:GetSelText(),"([^|])([-/]+)([^|])","%1|%2|%3")) end end @@ -578,7 +578,7 @@ function check_text() -- obsolete, replaced by lexer if io.exists(filename) then traceln("loading " .. filename) for line in io.lines(filename) do - if not find(line,"^[\%\#\-]") then + if not find(line,"^[%#-]") then str = gsub(line,"%s*$", '') rawset(wordlist,str,true) worddone = worddone + 1 @@ -647,6 +647,34 @@ function reset_text() editor:SetStyling(editor.TextLength,INDIC_PLAIN) end +function add_text() + + local startposition = editor.SelectionStart + local endposition = editor.SelectionEnd + + if startposition == endposition then return end + + local selection = gsub(editor:GetSelText(), "%s*$", '') + + local n, sum = 0, 0 + for s in gmatch(selection,"[%d%.%,]+") do + s = gsub(s,",",".") + local m = tonumber(s) + if m then + n = n + 1 + sum = sum + m + traceln(format("%4i : %s",n,m)) + end + end + if n > 0 then + traceln("") + traceln(format("sum : %s",sum)) + else + traceln("no numbers selected") + end + +end + -- menu local menuactions = {} diff --git a/context/data/scite/scite-ctx.properties b/context/data/scite/scite-ctx.properties index 8b2651c0d..e7223d762 100644 --- a/context/data/scite/scite-ctx.properties +++ b/context/data/scite/scite-ctx.properties @@ -25,6 +25,7 @@ ctx.menulist.default=\ wrap=wrap_text|\ unwrap=unwrap_text|\ sort=sort_text|\ + add=add_text|\ check=check_text|\ reset=reset_text @@ -35,6 +36,7 @@ ctx.menulist.context=\ document=document_text|\ quote=quote_text|\ compound=compound_text|\ + add=add_text|\ check=check_text|\ reset=reset_text @@ -46,6 +48,7 @@ ctx.menulist.example=\ document=document_text|\ quote=quote_text|\ compound=compound_text|\ + add=add_text|\ check=check_text|\ reset=reset_text diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index e4b5fb302..e3aac141f 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -138,6 +138,7 @@ local application = logs.application { -- ["version"] = true, -- display version and exit -- ["luaonly"] = true, -- run a lua file, then exit -- ["luaconly"] = true, -- byte-compile a lua file, then exit +-- ["jiton"] = false, -- } local report = application.report @@ -147,6 +148,12 @@ scripts.context = scripts.context or { } -- for the moment here +if getargument("jit") then + -- bonus shortcut, we assume than --jit also indicates the engine + -- although --jit and --engine=luajittex are independent + setargument("engine","luajittex") +end + local engine_new = getargument("engine") or directives.value("system.engine") local engine_old = environment.ownbin @@ -656,7 +663,7 @@ function scripts.context.run(ctxdata,filename) ["fmt"] = formatfile, ["lua"] = scriptfile, ["jobname"] = jobname, - ["jiton"] = a_jit and true or nil, + ["jiton"] = a_jit and true or false, } -- if a_synctex then @@ -845,7 +852,7 @@ function scripts.context.pipe() -- still used? end local function make_mkiv_format(name,engine) - environment.make_format(name) + environment.make_format(name) -- jit is picked up later end local function make_mkii_format(name,engine) @@ -866,6 +873,9 @@ function scripts.context.make(name) end local list = (name and { name }) or (environment.files[1] and environment.files) or defaultformats local engine = getargument("engine") or "luatex" + if getargument("jit") then + engine = "luajittex" + end for i=1,#list do local name = list[i] name = formatofinterface[name] or name or "" diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 731f8ca21..4d5fd447d 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -54,112 +54,54 @@ if not modules then modules = { } end modules ['mtxrun'] = { do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +-- original size: 2319, stripped down to: 1038 + +if not modules then modules={} end modules ['l-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } - --- compatibility hacks ... try to avoid usage - -local major, minor = string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") - -_MAJORVERSION = tonumber(major) or 5 -_MINORVERSION = tonumber(minor) or 1 - --- basics: - +local major,minor=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") +_MAJORVERSION=tonumber(major) or 5 +_MINORVERSION=tonumber(minor) or 1 if loadstring then - - local loadnormal = load - - function load(first,...) - if type(first) == "string" then - return loadstring(first,...) - else - return loadnormal(first,...) - end + local loadnormal=load + function load(first,...) + if type(first)=="string" then + return loadstring(first,...) + else + return loadnormal(first,...) end - + end else - - loadstring = load - + loadstring=load end - --- table: - --- Starting with version 5.2 Lua no longer provide ipairs, which makes --- sense. As we already used the for loop and # in most places the --- impact on ConTeXt was not that large; the remaining ipairs already --- have been replaced. In a similar fashion we also hardly used pairs. --- --- Hm, actually ipairs was retained, but we no longer use it anyway. --- --- Just in case, we provide the fallbacks as discussed in Programming --- in Lua (http://www.lua.org/pil/7.3.html): - if not ipairs then - - -- for k, v in ipairs(t) do ... end - -- for k=1,#t do local v = t[k] ... end - - local function iterate(a,i) - i = i + 1 - local v = a[i] - if v ~= nil then - return i, v --, nil - end - end - - function ipairs(a) - return iterate, a, 0 + local function iterate(a,i) + i=i+1 + local v=a[i] + if v~=nil then + return i,v end - + end + function ipairs(a) + return iterate,a,0 + end end - if not pairs then - - -- for k, v in pairs(t) do ... end - -- for k, v in next, t do ... end - - function pairs(t) - return next, t -- , nil - end - + function pairs(t) + return next,t + end end - --- The unpack function has been moved to the table table, and for compatiility --- reasons we provide both now. - if not table.unpack then - - table.unpack = _G.unpack - + table.unpack=_G.unpack elseif not unpack then - - _G.unpack = table.unpack - + _G.unpack=table.unpack end - --- package: - --- if not package.seachers then --- --- package.searchers = package.loaders -- 5.2 --- --- elseif not package.loaders then --- --- package.loaders = package.searchers --- --- end - -if not package.loaders then -- brr, searchers is a special "loadlib function" userdata type - - package.loaders = package.searchers - +if not package.loaders then + package.loaders=package.searchers end @@ -167,7359 +109,3309 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-lpeg'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1) - --- move utf -> l-unicode --- move string -> l-string or keep it here - -local lpeg = require("lpeg") - --- tracing (only used when we encounter a problem in integration of lpeg in luatex) - --- some code will move to unicode and string - -local report = texio and texio.write_nl or print - --- local lpmatch = lpeg.match --- local lpprint = lpeg.print --- local lpp = lpeg.P --- local lpr = lpeg.R --- local lps = lpeg.S --- local lpc = lpeg.C --- local lpb = lpeg.B --- local lpv = lpeg.V --- local lpcf = lpeg.Cf --- local lpcb = lpeg.Cb --- local lpcg = lpeg.Cg --- local lpct = lpeg.Ct --- local lpcs = lpeg.Cs --- local lpcc = lpeg.Cc --- local lpcmt = lpeg.Cmt --- local lpcarg = lpeg.Carg - --- function lpeg.match(l,...) report("LPEG MATCH") lpprint(l) return lpmatch(l,...) end - --- function lpeg.P (l) local p = lpp (l) report("LPEG P =") lpprint(l) return p end --- function lpeg.R (l) local p = lpr (l) report("LPEG R =") lpprint(l) return p end --- function lpeg.S (l) local p = lps (l) report("LPEG S =") lpprint(l) return p end --- function lpeg.C (l) local p = lpc (l) report("LPEG C =") lpprint(l) return p end --- function lpeg.B (l) local p = lpb (l) report("LPEG B =") lpprint(l) return p end --- function lpeg.V (l) local p = lpv (l) report("LPEG V =") lpprint(l) return p end --- function lpeg.Cf (l) local p = lpcf (l) report("LPEG Cf =") lpprint(l) return p end --- function lpeg.Cb (l) local p = lpcb (l) report("LPEG Cb =") lpprint(l) return p end --- function lpeg.Cg (l) local p = lpcg (l) report("LPEG Cg =") lpprint(l) return p end --- function lpeg.Ct (l) local p = lpct (l) report("LPEG Ct =") lpprint(l) return p end --- function lpeg.Cs (l) local p = lpcs (l) report("LPEG Cs =") lpprint(l) return p end --- function lpeg.Cc (l) local p = lpcc (l) report("LPEG Cc =") lpprint(l) return p end --- function lpeg.Cmt (l) local p = lpcmt (l) report("LPEG Cmt =") lpprint(l) return p end --- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end - -local type, next = type, next -local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format - --- Beware, we predefine a bunch of patterns here and one reason for doing so --- is that we get consistent behaviour in some of the visualizers. - -lpeg.patterns = lpeg.patterns or { } -- so that we can share -local patterns = lpeg.patterns - -local P, R, S, V, Ct, C, Cs, Cc, Cp, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Cp, lpeg.Cmt -local lpegtype, lpegmatch = lpeg.type, lpeg.match - -local anything = P(1) -local endofstring = P(-1) -local alwaysmatched = P(true) - -patterns.anything = anything -patterns.endofstring = endofstring -patterns.beginofstring = alwaysmatched -patterns.alwaysmatched = alwaysmatched - -local digit, sign = R('09'), S('+-') -local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") -local newline = crlf + S("\r\n") -- cr + lf -local escaped = P("\\") * anything -local squote = P("'") -local dquote = P('"') -local space = P(" ") - -local utfbom_32_be = P('\000\000\254\255') -local utfbom_32_le = P('\255\254\000\000') -local utfbom_16_be = P('\255\254') -local utfbom_16_le = P('\254\255') -local utfbom_8 = P('\239\187\191') -local utfbom = utfbom_32_be + utfbom_32_le - + utfbom_16_be + utfbom_16_le - + utfbom_8 -local utftype = utfbom_32_be * Cc("utf-32-be") + utfbom_32_le * Cc("utf-32-le") - + utfbom_16_be * Cc("utf-16-be") + utfbom_16_le * Cc("utf-16-le") - + utfbom_8 * Cc("utf-8") + alwaysmatched * Cc("utf-8") -- assume utf8 -local utfoffset = utfbom_32_be * Cc(4) + utfbom_32_le * Cc(4) - + utfbom_16_be * Cc(2) + utfbom_16_le * Cc(2) - + utfbom_8 * Cc(3) + Cc(0) - -local utf8next = R("\128\191") - -patterns.utf8one = R("\000\127") -patterns.utf8two = R("\194\223") * utf8next -patterns.utf8three = R("\224\239") * utf8next * utf8next -patterns.utf8four = R("\240\244") * utf8next * utf8next * utf8next -patterns.utfbom = utfbom -patterns.utftype = utftype -patterns.utfoffset = utfoffset - -local utf8char = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four -local validutf8char = utf8char^0 * endofstring * Cc(true) + Cc(false) - -patterns.utf8 = utf8char -patterns.utf8char = utf8char -patterns.validutf8 = validutf8char -patterns.validutf8char = validutf8char - -local eol = S("\n\r") -local spacer = S(" \t\f\v") -- + char(0xc2, 0xa0) if we want utf (cf mail roberto) -local whitespace = eol + spacer -local nonspacer = 1 - spacer -local nonwhitespace = 1 - whitespace - -patterns.eol = eol -patterns.spacer = spacer -patterns.whitespace = whitespace -patterns.nonspacer = nonspacer -patterns.nonwhitespace = nonwhitespace - -local stripper = spacer^0 * C((spacer^0 * nonspacer^1)^0) -- from example by roberto - ------ collapser = Cs(spacer^0/"" * ((spacer^1 * P(-1) / "") + (spacer^1/" ") + P(1))^0) -local collapser = Cs(spacer^0/"" * nonspacer^0 * ((spacer^0/" " * nonspacer^1)^0)) - -patterns.stripper = stripper -patterns.collapser = collapser - -patterns.digit = digit -patterns.sign = sign -patterns.cardinal = sign^0 * digit^1 -patterns.integer = sign^0 * digit^1 -patterns.unsigned = digit^0 * P('.') * digit^1 -patterns.float = sign^0 * patterns.unsigned -patterns.cunsigned = digit^0 * P(',') * digit^1 -patterns.cfloat = sign^0 * patterns.cunsigned -patterns.number = patterns.float + patterns.integer -patterns.cnumber = patterns.cfloat + patterns.integer -patterns.oct = P("0") * R("07")^1 -patterns.octal = patterns.oct -patterns.HEX = P("0x") * R("09","AF")^1 -patterns.hex = P("0x") * R("09","af")^1 -patterns.hexadecimal = P("0x") * R("09","AF","af")^1 -patterns.lowercase = R("az") -patterns.uppercase = R("AZ") -patterns.letter = patterns.lowercase + patterns.uppercase -patterns.space = space -patterns.tab = P("\t") -patterns.spaceortab = patterns.space + patterns.tab -patterns.newline = newline -patterns.emptyline = newline^1 -patterns.equal = P("=") -patterns.comma = P(",") -patterns.commaspacer = P(",") * spacer^0 -patterns.period = P(".") -patterns.colon = P(":") -patterns.semicolon = P(";") -patterns.underscore = P("_") -patterns.escaped = escaped -patterns.squote = squote -patterns.dquote = dquote -patterns.nosquote = (escaped + (1-squote))^0 -patterns.nodquote = (escaped + (1-dquote))^0 -patterns.unsingle = (squote/"") * patterns.nosquote * (squote/"") -- will change to C in the middle -patterns.undouble = (dquote/"") * patterns.nodquote * (dquote/"") -- will change to C in the middle -patterns.unquoted = patterns.undouble + patterns.unsingle -- more often undouble -patterns.unspacer = ((patterns.spacer^1)/"")^0 - -patterns.singlequoted = squote * patterns.nosquote * squote -patterns.doublequoted = dquote * patterns.nodquote * dquote -patterns.quoted = patterns.doublequoted + patterns.singlequoted - -patterns.propername = R("AZ","az","__") * R("09","AZ","az", "__")^0 * P(-1) - -patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1 -patterns.beginline = #(1-newline) - -patterns.longtostring = Cs(whitespace^0/"" * nonwhitespace^0 * ((whitespace^0/" " * (patterns.quoted + nonwhitespace)^1)^0)) - -local function anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * V(1) } -end - -lpeg.anywhere = anywhere +-- original size: 25285, stripped down to: 13969 +if not modules then modules={} end modules ['l-lpeg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local lpeg=require("lpeg") +local report=texio and texio.write_nl or print +local type,next,tostring=type,next,tostring +local byte,char,gmatch,format=string.byte,string.char,string.gmatch,string.format +local floor=math.floor +lpeg.patterns=lpeg.patterns or {} +local patterns=lpeg.patterns +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt +local lpegtype,lpegmatch=lpeg.type,lpeg.match +local anything=P(1) +local endofstring=P(-1) +local alwaysmatched=P(true) +patterns.anything=anything +patterns.endofstring=endofstring +patterns.beginofstring=alwaysmatched +patterns.alwaysmatched=alwaysmatched +local digit,sign=R('09'),S('+-') +local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") +local newline=crlf+S("\r\n") +local escaped=P("\\")*anything +local squote=P("'") +local dquote=P('"') +local space=P(" ") +local utfbom_32_be=P('\000\000\254\255') +local utfbom_32_le=P('\255\254\000\000') +local utfbom_16_be=P('\255\254') +local utfbom_16_le=P('\254\255') +local utfbom_8=P('\239\187\191') +local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8 +local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8") +local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0) +local utf8next=R("\128\191") +patterns.utf8one=R("\000\127") +patterns.utf8two=R("\194\223")*utf8next +patterns.utf8three=R("\224\239")*utf8next*utf8next +patterns.utf8four=R("\240\244")*utf8next*utf8next*utf8next +patterns.utfbom=utfbom +patterns.utftype=utftype +patterns.utfoffset=utfoffset +local utf8char=patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four +local validutf8char=utf8char^0*endofstring*Cc(true)+Cc(false) +patterns.utf8=utf8char +patterns.utf8char=utf8char +patterns.validutf8=validutf8char +patterns.validutf8char=validutf8char +local eol=S("\n\r") +local spacer=S(" \t\f\v") +local whitespace=eol+spacer +local nonspacer=1-spacer +local nonwhitespace=1-whitespace +patterns.eol=eol +patterns.spacer=spacer +patterns.whitespace=whitespace +patterns.nonspacer=nonspacer +patterns.nonwhitespace=nonwhitespace +local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +patterns.stripper=stripper +patterns.collapser=collapser +patterns.digit=digit +patterns.sign=sign +patterns.cardinal=sign^0*digit^1 +patterns.integer=sign^0*digit^1 +patterns.unsigned=digit^0*P('.')*digit^1 +patterns.float=sign^0*patterns.unsigned +patterns.cunsigned=digit^0*P(',')*digit^1 +patterns.cfloat=sign^0*patterns.cunsigned +patterns.number=patterns.float+patterns.integer +patterns.cnumber=patterns.cfloat+patterns.integer +patterns.oct=P("0")*R("07")^1 +patterns.octal=patterns.oct +patterns.HEX=P("0x")*R("09","AF")^1 +patterns.hex=P("0x")*R("09","af")^1 +patterns.hexadecimal=P("0x")*R("09","AF","af")^1 +patterns.lowercase=R("az") +patterns.uppercase=R("AZ") +patterns.letter=patterns.lowercase+patterns.uppercase +patterns.space=space +patterns.tab=P("\t") +patterns.spaceortab=patterns.space+patterns.tab +patterns.newline=newline +patterns.emptyline=newline^1 +patterns.equal=P("=") +patterns.comma=P(",") +patterns.commaspacer=P(",")*spacer^0 +patterns.period=P(".") +patterns.colon=P(":") +patterns.semicolon=P(";") +patterns.underscore=P("_") +patterns.escaped=escaped +patterns.squote=squote +patterns.dquote=dquote +patterns.nosquote=(escaped+(1-squote))^0 +patterns.nodquote=(escaped+(1-dquote))^0 +patterns.unsingle=(squote/"")*patterns.nosquote*(squote/"") +patterns.undouble=(dquote/"")*patterns.nodquote*(dquote/"") +patterns.unquoted=patterns.undouble+patterns.unsingle +patterns.unspacer=((patterns.spacer^1)/"")^0 +patterns.singlequoted=squote*patterns.nosquote*squote +patterns.doublequoted=dquote*patterns.nodquote*dquote +patterns.quoted=patterns.doublequoted+patterns.singlequoted +patterns.propername=R("AZ","az","__")*R("09","AZ","az","__")^0*P(-1) +patterns.somecontent=(anything-newline-space)^1 +patterns.beginline=#(1-newline) +patterns.longtostring=Cs(whitespace^0/""*nonwhitespace^0*((whitespace^0/" "*(patterns.quoted+nonwhitespace)^1)^0)) +local function anywhere(pattern) + return P { P(pattern)+1*V(1) } +end +lpeg.anywhere=anywhere function lpeg.instringchecker(p) - p = anywhere(p) - return function(str) - return lpegmatch(p,str) and true or false - end + p=anywhere(p) + return function(str) + return lpegmatch(p,str) and true or false + end end - -function lpeg.splitter(pattern, action) - return (((1-P(pattern))^1)/action+1)^0 +function lpeg.splitter(pattern,action) + return (((1-P(pattern))^1)/action+1)^0 end - -function lpeg.tsplitter(pattern, action) - return Ct((((1-P(pattern))^1)/action+1)^0) +function lpeg.tsplitter(pattern,action) + return Ct((((1-P(pattern))^1)/action+1)^0) end - --- probleem: separator can be lpeg and that does not hash too well, but --- it's quite okay as the key is then not garbage collected - -local splitters_s, splitters_m, splitters_t = { }, { }, { } - +local splitters_s,splitters_m,splitters_t={},{},{} local function splitat(separator,single) - local splitter = (single and splitters_s[separator]) or splitters_m[separator] - if not splitter then - separator = P(separator) - local other = C((1 - separator)^0) - if single then - local any = anything - splitter = other * (separator * C(any^0) + "") -- ? - splitters_s[separator] = splitter - else - splitter = other * (separator * other)^0 - splitters_m[separator] = splitter - end + local splitter=(single and splitters_s[separator]) or splitters_m[separator] + if not splitter then + separator=P(separator) + local other=C((1-separator)^0) + if single then + local any=anything + splitter=other*(separator*C(any^0)+"") + splitters_s[separator]=splitter + else + splitter=other*(separator*other)^0 + splitters_m[separator]=splitter end - return splitter + end + return splitter end - local function tsplitat(separator) - local splitter = splitters_t[separator] - if not splitter then - splitter = Ct(splitat(separator)) - splitters_t[separator] = splitter - end - return splitter -end - -lpeg.splitat = splitat -lpeg.tsplitat = tsplitat - + local splitter=splitters_t[separator] + if not splitter then + splitter=Ct(splitat(separator)) + splitters_t[separator]=splitter + end + return splitter +end +lpeg.splitat=splitat +lpeg.tsplitat=tsplitat function string.splitup(str,separator) - if not separator then - separator = "," - end - return lpegmatch(splitters_m[separator] or splitat(separator),str) + if not separator then + separator="," + end + return lpegmatch(splitters_m[separator] or splitat(separator),str) end - --- local p = splitat("->",false) print(lpegmatch(p,"oeps->what->more")) -- oeps what more --- local p = splitat("->",true) print(lpegmatch(p,"oeps->what->more")) -- oeps what->more --- local p = splitat("->",false) print(lpegmatch(p,"oeps")) -- oeps --- local p = splitat("->",true) print(lpegmatch(p,"oeps")) -- oeps - -local cache = { } - +local cache={} function lpeg.split(separator,str) - local c = cache[separator] - if not c then - c = tsplitat(separator) - cache[separator] = c - end - return lpegmatch(c,str) + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c + end + return lpegmatch(c,str) end - function string.split(str,separator) - if separator then - local c = cache[separator] - if not c then - c = tsplitat(separator) - cache[separator] = c - end - return lpegmatch(c,str) - else - return { str } + if separator then + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c end -end - -local spacing = patterns.spacer^0 * newline -- sort of strip -local empty = spacing * Cc("") -local nonempty = Cs((1-spacing)^1) * spacing^-1 -local content = (empty + nonempty)^1 - -patterns.textline = content - -local linesplitter = tsplitat(newline) - -patterns.linesplitter = linesplitter - + return lpegmatch(c,str) + else + return { str } + end +end +local spacing=patterns.spacer^0*newline +local empty=spacing*Cc("") +local nonempty=Cs((1-spacing)^1)*spacing^-1 +local content=(empty+nonempty)^1 +patterns.textline=content +local linesplitter=tsplitat(newline) +patterns.linesplitter=linesplitter function string.splitlines(str) - return lpegmatch(linesplitter,str) + return lpegmatch(linesplitter,str) end - --- lpeg.splitters = cache -- no longer public - -local cache = { } - +local cache={} function lpeg.checkedsplit(separator,str) - local c = cache[separator] - if not c then - separator = P(separator) - local other = C((1 - separator)^1) - c = Ct(separator^0 * other * (separator^1 * other)^0) - cache[separator] = c - end - return lpegmatch(c,str) + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) end - function string.checkedsplit(str,separator) - local c = cache[separator] - if not c then - separator = P(separator) - local other = C((1 - separator)^1) - c = Ct(separator^0 * other * (separator^1 * other)^0) - cache[separator] = c - end - return lpegmatch(c,str) -end - --- from roberto's site: - -local function f2(s) local c1, c2 = byte(s,1,2) return c1 * 64 + c2 - 12416 end -local function f3(s) local c1, c2, c3 = byte(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end -local function f4(s) local c1, c2, c3, c4 = byte(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end - -local utf8byte = patterns.utf8one/byte + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 - -patterns.utf8byte = utf8byte - - - -local cache = { } - + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) +end +local function f2(s) local c1,c2=byte(s,1,2) return c1*64+c2-12416 end +local function f3(s) local c1,c2,c3=byte(s,1,3) return (c1*64+c2)*64+c3-925824 end +local function f4(s) local c1,c2,c3,c4=byte(s,1,4) return ((c1*64+c2)*64+c3)*64+c4-63447168 end +local utf8byte=patterns.utf8one/byte+patterns.utf8two/f2+patterns.utf8three/f3+patterns.utf8four/f4 +patterns.utf8byte=utf8byte +local cache={} function lpeg.stripper(str) - if type(str) == "string" then - local s = cache[str] - if not s then - s = Cs(((S(str)^1)/"" + 1)^0) - cache[str] = s - end - return s - else - return Cs(((str^1)/"" + 1)^0) + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs(((S(str)^1)/""+1)^0) + cache[str]=s end + return s + else + return Cs(((str^1)/""+1)^0) + end end - -local cache = { } - +local cache={} function lpeg.keeper(str) - if type(str) == "string" then - local s = cache[str] - if not s then - s = Cs((((1-S(str))^1)/"" + 1)^0) - cache[str] = s - end - return s + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs((((1-S(str))^1)/""+1)^0) + cache[str]=s + end + return s + else + return Cs((((1-str)^1)/""+1)^0) + end +end +function lpeg.frontstripper(str) + return (P(str)+P(true))*Cs(anything^0) +end +function lpeg.endstripper(str) + return Cs((1-P(str)*endofstring)^0) +end +function lpeg.replacer(one,two,makefunction,isutf) + local pattern + local u=isutf and utf8char or 1 + if type(one)=="table" then + local no=#one + local p=P(false) + if no==0 then + for k,v in next,one do + p=p+P(k)/v + end + pattern=Cs((p+u)^0) + elseif no==1 then + local o=one[1] + one,two=P(o[1]),o[2] + pattern=Cs((one/two+u)^0) + else + for i=1,no do + local o=one[i] + p=p+P(o[1])/o[2] + end + pattern=Cs((p+u)^0) + end + else + pattern=Cs((P(one)/(two or "")+u)^0) + end + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +function lpeg.finder(lst,makefunction) + local pattern + if type(lst)=="table" then + pattern=P(false) + if #lst==0 then + for k,v in next,lst do + pattern=pattern+P(k) + end else - return Cs((((1-str)^1)/"" + 1)^0) + for i=1,#lst do + pattern=pattern+P(lst[i]) + end end + else + pattern=P(lst) + end + pattern=(1-pattern)^0*pattern + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +local splitters_f,splitters_s={},{} +function lpeg.firstofsplit(separator) + local splitter=splitters_f[separator] + if not splitter then + separator=P(separator) + splitter=C((1-separator)^0) + splitters_f[separator]=splitter + end + return splitter +end +function lpeg.secondofsplit(separator) + local splitter=splitters_s[separator] + if not splitter then + separator=P(separator) + splitter=(1-separator)^0*separator*C(anything^0) + splitters_s[separator]=splitter + end + return splitter end - -function lpeg.frontstripper(str) -- or pattern (yet undocumented) - return (P(str) + P(true)) * Cs(anything^0) +function lpeg.balancer(left,right) + left,right=P(left),P(right) + return P { left*((1-left-right)+V(1))^0*right } end - -function lpeg.endstripper(str) -- or pattern (yet undocumented) - return Cs((1 - P(str) * endofstring)^0) +local nany=utf8char/"" +function lpeg.counter(pattern) + pattern=Cs((P(pattern)/" "+nany)^0) + return function(str) + return #lpegmatch(pattern,str) + end +end +local utfcharacters=utf and utf.characters or string.utfcharacters +local utfgmatch=unicode and unicode.utf8.gmatch +local utfchar=utf and utf.char or (unicode and unicode.utf8 and unicode.utf8.char) +lpeg.UP=lpeg.P +if utfcharacters then + function lpeg.US(str) + local p=P(false) + for uc in utfcharacters(str) do + p=p+P(uc) + end + return p + end +elseif utfgmatch then + function lpeg.US(str) + local p=P(false) + for uc in utfgmatch(str,".") do + p=p+P(uc) + end + return p + end +else + function lpeg.US(str) + local p=P(false) + local f=function(uc) + p=p+P(uc) + end + lpegmatch((utf8char/f)^0,str) + return p + end end - --- Just for fun I looked at the used bytecode and --- p = (p and p + pp) or pp gets one more (testset). - --- todo: cache when string - -function lpeg.replacer(one,two,makefunction,isutf) -- in principle we should sort the keys - local pattern - local u = isutf and utf8char or 1 - if type(one) == "table" then - local no = #one - local p = P(false) - if no == 0 then - for k, v in next, one do - p = p + P(k) / v - end - pattern = Cs((p + u)^0) - elseif no == 1 then - local o = one[1] - one, two = P(o[1]), o[2] - -- pattern = Cs(((1-one)^1 + one/two)^0) - pattern = Cs((one/two + u)^0) +local range=utf8byte*utf8byte+Cc(false) +function lpeg.UR(str,more) + local first,last + if type(str)=="number" then + first=str + last=more or first + else + first,last=lpegmatch(range,str) + if not last then + return P(str) + end + end + if first==last then + return P(str) + elseif utfchar and (last-first<8) then + local p=P(false) + for i=first,last do + p=p+P(utfchar(i)) + end + return p + else + local f=function(b) + return b>=first and b<=last + end + return utf8byte/f + end +end +function lpeg.is_lpeg(p) + return p and lpegtype(p)=="pattern" +end +function lpeg.oneof(list,...) + if type(list)~="table" then + list={ list,... } + end + local p=P(list[1]) + for l=2,#list do + p=p+P(list[l]) + end + return p +end +local sort=table.sort +local function copyindexed(old) + local new={} + for i=1,#old do + new[i]=old + end + return new +end +local function sortedkeys(tab) + local keys,s={},0 + for key,_ in next,tab do + s=s+1 + keys[s]=key + end + sort(keys) + return keys +end +function lpeg.append(list,pp,delayed,checked) + local p=pp + if #list>0 then + local keys=copyindexed(list) + sort(keys) + for i=#keys,1,-1 do + local k=keys[i] + if p then + p=P(k)+p + else + p=P(k) + end + end + elseif delayed then + local keys=sortedkeys(list) + if p then + for i=1,#keys,1 do + local k=keys[i] + local v=list[k] + p=P(k)/list+p + end + else + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)+p else - for i=1,no do - local o = one[i] - p = p + P(o[1]) / o[2] - end - pattern = Cs((p + u)^0) + p=P(k) end - else - pattern = Cs((P(one)/(two or "") + u)^0) + end + if p then + p=p/list + end end - if makefunction then - return function(str) - return lpegmatch(pattern,str) + elseif checked then + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + if k==v then + p=P(k)+p + else + p=P(k)/v+p end - else - return pattern - end -end - -function lpeg.finder(lst,makefunction) - local pattern - if type(lst) == "table" then - pattern = P(false) - if #lst == 0 then - for k, v in next, lst do - pattern = pattern + P(k) -- ignore key, so we can use a replacer table - end + else + if k==v then + p=P(k) else - for i=1,#lst do - pattern = pattern + P(lst[i]) - end + p=P(k)/v end - else - pattern = P(lst) + end end - pattern = (1-pattern)^0 * pattern - if makefunction then - return function(str) - return lpegmatch(pattern,str) - end - else - return pattern + else + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)/v+p + else + p=P(k)/v + end end + end + return p end - --- print(lpeg.match(lpeg.replacer("e","a"),"test test")) --- print(lpeg.match(lpeg.replacer{{"e","a"}},"test test")) --- print(lpeg.match(lpeg.replacer({ e = "a", t = "x" }),"test test")) - -local splitters_f, splitters_s = { }, { } - -function lpeg.firstofsplit(separator) -- always return value - local splitter = splitters_f[separator] - if not splitter then - separator = P(separator) - splitter = C((1 - separator)^0) - splitters_f[separator] = splitter +local function make(t) + local p + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + local v=t[k] + if not p then + if next(v) then + p=P(k)*make(v) + else + p=P(k) + end + else + if next(v) then + p=p+P(k)*make(v) + else + p=p+P(k) + end end - return splitter + end + return p end - -function lpeg.secondofsplit(separator) -- nil if not split - local splitter = splitters_s[separator] - if not splitter then - separator = P(separator) - splitter = (1 - separator)^0 * separator * C(anything^0) - splitters_s[separator] = splitter +function lpeg.utfchartabletopattern(list) + local tree={} + for i=1,#list do + local t=tree + for c in gmatch(list[i],".") do + if not t[c] then + t[c]={} + end + t=t[c] + end + end + return make(tree) +end +patterns.containseol=lpeg.finder(eol) +local function nextstep(n,step,result) + local m=n%step + local d=floor(n/step) + if d>0 then + local v=V(tostring(step)) + local s=result.start + for i=1,d do + if s then + s=v*s + else + s=v + end end - return splitter + result.start=s + end + if step>1 and result.start then + local v=V(tostring(step/2)) + result[tostring(step)]=v*v + end + if step>0 then + return nextstep(m,step/2,result) + else + return result + end end - -function lpeg.balancer(left,right) - left, right = P(left), P(right) - return P { left * ((1 - left - right) + V(1))^0 * right } +function lpeg.times(pattern,n) + return P(nextstep(n,2^16,{ "start",["1"]=pattern })) end --- print(1,lpegmatch(lpeg.firstofsplit(":"),"bc:de")) --- print(2,lpegmatch(lpeg.firstofsplit(":"),":de")) -- empty --- print(3,lpegmatch(lpeg.firstofsplit(":"),"bc")) --- print(4,lpegmatch(lpeg.secondofsplit(":"),"bc:de")) --- print(5,lpegmatch(lpeg.secondofsplit(":"),"bc:")) -- empty --- print(6,lpegmatch(lpeg.secondofsplit(":",""),"bc")) --- print(7,lpegmatch(lpeg.secondofsplit(":"),"bc")) --- print(9,lpegmatch(lpeg.secondofsplit(":","123"),"bc")) - --- -- slower: --- --- function lpeg.counter(pattern) --- local n, pattern = 0, (lpeg.P(pattern)/function() n = n + 1 end + lpeg.anything)^0 --- return function(str) n = 0 ; lpegmatch(pattern,str) ; return n end --- end - -local nany = utf8char/"" -function lpeg.counter(pattern) - pattern = Cs((P(pattern)/" " + nany)^0) - return function(str) - return #lpegmatch(pattern,str) - end -end +end -- of closure --- utf extensies +do -- create closure to overcome 200 locals limit -local utfcharacters = utf and utf.characters or string.utfcharacters -local utfgmatch = unicode and unicode.utf8.gmatch -local utfchar = utf and utf.char or (unicode and unicode.utf8 and unicode.utf8.char) +-- original size: 361, stripped down to: 322 -lpeg.UP = lpeg.P +if not modules then modules={} end modules ['l-functions']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +functions=functions or {} +function functions.dummy() end -if utfcharacters then - function lpeg.US(str) - local p = P(false) - for uc in utfcharacters(str) do - p = p + P(uc) - end - return p - end +end -- of closure +do -- create closure to overcome 200 locals limit -elseif utfgmatch then +-- original size: 5491, stripped down to: 2685 - function lpeg.US(str) - local p = P(false) - for uc in utfgmatch(str,".") do - p = p + P(uc) - end - return p - end +if not modules then modules={} end modules ['l-string']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local string=string +local sub,gmatch,format,char,byte,rep,lower=string.sub,string.gmatch,string.format,string.char,string.byte,string.rep,string.lower +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local P,S,C,Ct,Cc,Cs=lpeg.P,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cc,lpeg.Cs +local unquoted=patterns.squote*C(patterns.nosquote)*patterns.squote+patterns.dquote*C(patterns.nodquote)*patterns.dquote +function string.unquoted(str) + return lpegmatch(unquoted,str) or str +end +function string.quoted(str) + return format("%q",str) +end +function string.count(str,pattern) + local n=0 + for _ in gmatch(str,pattern) do + n=n+1 + end + return n +end +function string.limit(str,n,sentinel) + if #str>n then + sentinel=sentinel or "..." + return sub(str,1,(n-#sentinel))..sentinel + else + return str + end +end +local stripper=patterns.stripper +local collapser=patterns.collapser +local longtostring=patterns.longtostring +function string.strip(str) + return lpegmatch(stripper,str) or "" +end +function string.collapsespaces(str) + return lpegmatch(collapser,str) or "" +end +function string.longtostring(str) + return lpegmatch(longtostring,str) or "" +end +local pattern=P(" ")^0*P(-1) +function string.is_empty(str) + if str=="" then + return true + else + return lpegmatch(pattern,str) and true or false + end +end +local anything=patterns.anything +local allescapes=Cc("%")*S(".-+%?()[]*") +local someescapes=Cc("%")*S(".-+%()[]") +local matchescapes=Cc(".")*S("*?") +local pattern_a=Cs ((allescapes+anything )^0 ) +local pattern_b=Cs ((someescapes+matchescapes+anything )^0 ) +local pattern_c=Cs (Cc("^")*(someescapes+matchescapes+anything )^0*Cc("$") ) +function string.escapedpattern(str,simple) + return lpegmatch(simple and pattern_b or pattern_a,str) +end +function string.topattern(str,lowercase,strict) + if str=="" then + return ".*" + elseif strict then + str=lpegmatch(pattern_c,str) + else + str=lpegmatch(pattern_b,str) + end + if lowercase then + return lower(str) + else + return str + end +end +function string.valid(str,default) + return (type(str)=="string" and str~="" and str) or default or nil +end +string.itself=function(s) return s end +local pattern=Ct(C(1)^0) +function string.totable(str) + return lpegmatch(pattern,str) +end +local replacer=lpeg.replacer("@","%%") +function string.tformat(fmt,...) + return format(lpegmatch(replacer,fmt),...) +end +string.quote=string.quoted +string.unquote=string.unquoted -else - function lpeg.US(str) - local p = P(false) - local f = function(uc) - p = p + P(uc) - end - lpegmatch((utf8char/f)^0,str) - return p - end +end -- of closure -end +do -- create closure to overcome 200 locals limit -local range = utf8byte * utf8byte + Cc(false) -- utf8byte is already a capture +-- original size: 28973, stripped down to: 19400 -function lpeg.UR(str,more) - local first, last - if type(str) == "number" then - first = str - last = more or first +if not modules then modules={} end modules ['l-table']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring,tonumber,ipairs,select=type,next,tostring,tonumber,ipairs,select +local table,string=table,string +local concat,sort,insert,remove=table.concat,table.sort,table.insert,table.remove +local format,lower,dump=string.format,string.lower,string.dump +local getmetatable,setmetatable=getmetatable,setmetatable +local getinfo=debug.getinfo +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local floor=math.floor +local stripper=patterns.stripper +function table.strip(tab) + local lst,l={},0 + for i=1,#tab do + local s=lpegmatch(stripper,tab[i]) or "" + if s=="" then else - first, last = lpegmatch(range,str) - if not last then - return P(str) - end + l=l+1 + lst[l]=s end - if first == last then - return P(str) - elseif utfchar and (last - first < 8) then -- a somewhat arbitrary criterium - local p = P(false) - for i=first,last do - p = p + P(utfchar(i)) - end - return p -- nil when invalid range - else - local f = function(b) - return b >= first and b <= last - end - -- tricky, these nested captures - return utf8byte / f -- nil when invalid range + end + return lst +end +function table.keys(t) + if t then + local keys,k={},0 + for key,_ in next,t do + k=k+1 + keys[k]=key end + return keys + else + return {} + end end - --- print(lpeg.match(lpeg.Cs((C(lpeg.UR("αω"))/{ ["χ"] = "OEPS" })^0),"αωχαω")) - --- lpeg.print(lpeg.R("ab","cd","gh")) --- lpeg.print(lpeg.P("a","b","c")) --- lpeg.print(lpeg.S("a","b","c")) - --- print(lpeg.count("äáàa",lpeg.P("á") + lpeg.P("à"))) --- print(lpeg.count("äáàa",lpeg.UP("áà"))) --- print(lpeg.count("äáàa",lpeg.US("àá"))) --- print(lpeg.count("äáàa",lpeg.UR("aá"))) --- print(lpeg.count("äáàa",lpeg.UR("àá"))) --- print(lpeg.count("äáàa",lpeg.UR(0x0000,0xFFFF))) - -function lpeg.is_lpeg(p) - return p and lpegtype(p) == "pattern" -end - -function lpeg.oneof(list,...) -- lpeg.oneof("elseif","else","if","then") -- assume proper order - if type(list) ~= "table" then - list = { list, ... } - end - -- table.sort(list) -- longest match first - local p = P(list[1]) - for l=2,#list do - p = p + P(list[l]) - end - return p -end - --- For the moment here, but it might move to utilities. Beware, we need to --- have the longest keyword first, so 'aaa' comes beforte 'aa' which is why we --- loop back from the end cq. prepend. - -local sort = table.sort - -local function copyindexed(old) - local new = { } - for i=1,#old do - new[i] = old - end - return new +local function compare(a,b) + local ta,tb=type(a),type(b) + if ta==tb then + return a 0 then - local keys = copyindexed(list) - sort(keys) - for i=#keys,1,-1 do - local k = keys[i] - if p then - p = P(k) + p - else - p = P(k) - end - end - elseif delayed then -- hm, it looks like the lpeg parser resolves anyway - local keys = sortedkeys(list) - if p then - for i=1,#keys,1 do - local k = keys[i] - local v = list[k] - p = P(k)/list + p - end - else - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - p = P(k) + p - else - p = P(k) - end - end - if p then - p = p / list - end - end - elseif checked then - -- problem: substitution gives a capture - local keys = sortedkeys(list) - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - if k == v then - p = P(k) + p - else - p = P(k)/v + p - end - else - if k == v then - p = P(k) - else - p = P(k)/v - end - end + if tab then + local srt,category,s={},0,0 + for key,_ in next,tab do + s=s+1 + srt[s]=key + if category==3 then + else + local tkey=type(key) + if tkey=="string" then + category=(category==2 and 3) or 1 + elseif tkey=="number" then + category=(category==1 and 3) or 2 + else + category=3 end + end + end + if category==0 or category==3 then + sort(srt,compare) else - local keys = sortedkeys(list) - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - p = P(k)/v + p - else - p = P(k)/v - end - end + sort(srt) end - return p + return srt + else + return {} + end end - --- inspect(lpeg.append({ a = "1", aa = "1", aaa = "1" } ,nil,true)) --- inspect(lpeg.append({ ["degree celsius"] = "1", celsius = "1", degree = "1" } ,nil,true)) - --- function lpeg.exact_match(words,case_insensitive) --- local pattern = concat(words) --- if case_insensitive then --- local pattern = S(upper(characters)) + S(lower(characters)) --- local list = { } --- for i=1,#words do --- list[lower(words[i])] = true --- end --- return Cmt(pattern^1, function(_,i,s) --- return list[lower(s)] and i --- end) --- else --- local pattern = S(concat(words)) --- local list = { } --- for i=1,#words do --- list[words[i]] = true --- end --- return Cmt(pattern^1, function(_,i,s) --- return list[s] and i --- end) --- end --- end - --- experiment: - -local function make(t) - local p - local keys = sortedkeys(t) - for i=1,#keys do - local k = keys[i] - local v = t[k] - if not p then - if next(v) then - p = P(k) * make(v) - else - p = P(k) - end - else - if next(v) then - p = p + P(k) * make(v) - else - p = p + P(k) - end - end +local function sortedhashkeys(tab) + if tab then + local srt,s={},0 + for key,_ in next,tab do + if key then + s=s+1 + srt[s]=key + end end - return p + sort(srt) + return srt + else + return {} + end end - -function lpeg.utfchartabletopattern(list) -- goes to util-lpg - local tree = { } - for i=1,#list do - local t = tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c] = { } - end - t = t[c] - end +function table.allkeys(t) + local keys={} + for i=1,#t do + for k,v in next,t[i] do + keys[k]=true end - return make(tree) -end - --- inspect ( lpeg.utfchartabletopattern { --- utfchar(0x00A0), -- nbsp --- utfchar(0x2000), -- enquad --- utfchar(0x2001), -- emquad --- utfchar(0x2002), -- enspace --- utfchar(0x2003), -- emspace --- utfchar(0x2004), -- threeperemspace --- utfchar(0x2005), -- fourperemspace --- utfchar(0x2006), -- sixperemspace --- utfchar(0x2007), -- figurespace --- utfchar(0x2008), -- punctuationspace --- utfchar(0x2009), -- breakablethinspace --- utfchar(0x200A), -- hairspace --- utfchar(0x200B), -- zerowidthspace --- utfchar(0x202F), -- narrownobreakspace --- utfchar(0x205F), -- math thinspace --- } ) - --- a few handy ones: --- --- faster than find(str,"[\n\r]") when match and # > 7 and always faster when # > 3 - -patterns.containseol = lpeg.finder(eol) -- (1-eol)^0 * eol - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-functions'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -functions = functions or { } - -function functions.dummy() end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-string'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local string = string -local sub, gmatch, format, char, byte, rep, lower = string.sub, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower -local lpegmatch, patterns = lpeg.match, lpeg.patterns -local P, S, C, Ct, Cc, Cs = lpeg.P, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cs - --- Some functions are already defined in l-lpeg and maybe some from here will --- move there (unless we also expose caches). - --- if not string.split then --- --- function string.split(str,pattern) --- local t = { } --- if #str > 0 then --- local n = 1 --- for s in gmatch(str..pattern,"(.-)"..pattern) do --- t[n] = s --- n = n + 1 --- end --- end --- return t --- end --- --- end - --- function string.unquoted(str) --- return (gsub(str,"^([\"\'])(.*)%1$","%2")) -- interesting pattern --- end - -local unquoted = patterns.squote * C(patterns.nosquote) * patterns.squote - + patterns.dquote * C(patterns.nodquote) * patterns.dquote - -function string.unquoted(str) - return lpegmatch(unquoted,str) or str + end + return sortedkeys(keys) end - --- print(string.unquoted("test")) --- print(string.unquoted([["t\"est"]])) --- print(string.unquoted([["t\"est"x]])) --- print(string.unquoted("\'test\'")) --- print(string.unquoted('"test"')) --- print(string.unquoted('"test"')) - -function string.quoted(str) - return format("%q",str) -- always " +table.sortedkeys=sortedkeys +table.sortedhashkeys=sortedhashkeys +local function nothing() end +local function sortedhash(t) + if t then + local n,s=0,sortedkeys(t) + local function kv(s) + n=n+1 + local k=s[n] + return k,t[k] + end + return kv,s + else + return nothing + end +end +table.sortedhash=sortedhash +table.sortedpairs=sortedhash +function table.append(t,list) + local n=#t + for i=1,#list do + n=n+1 + t[n]=list[i] + end + return t +end +function table.prepend(t,list) + local nl=#list + local nt=nl+#t + for i=#t,1,-1 do + t[nt]=t[i] + nt=nt-1 + end + for i=1,#list do + t[i]=list[i] + end + return t +end +function table.merge(t,...) + t=t or {} + for i=1,select("#",...) do + for k,v in next,(select(i,...)) do + t[k]=v + end + end + return t end - -function string.count(str,pattern) -- variant 3 - local n = 0 - for _ in gmatch(str,pattern) do -- not for utf - n = n + 1 +function table.merged(...) + local t={} + for i=1,select("#",...) do + for k,v in next,(select(i,...)) do + t[k]=v end - return n + end + return t end - -function string.limit(str,n,sentinel) -- not utf proof - if #str > n then - sentinel = sentinel or "..." - return sub(str,1,(n-#sentinel)) .. sentinel - else - return str +function table.imerge(t,...) + local nt=#t + for i=1,select("#",...) do + local nst=select(i,...) + for j=1,#nst do + nt=nt+1 + t[nt]=nst[j] end + end + return t end - -local stripper = patterns.stripper -local collapser = patterns.collapser -local longtostring = patterns.longtostring - -function string.strip(str) - return lpegmatch(stripper,str) or "" -end - -function string.collapsespaces(str) - return lpegmatch(collapser,str) or "" -end - -function string.longtostring(str) - return lpegmatch(longtostring,str) or "" -end - --- function string.is_empty(str) --- return not find(str,"%S") --- end - -local pattern = P(" ")^0 * P(-1) - -function string.is_empty(str) - if str == "" then - return true - else - return lpegmatch(pattern,str) and true or false +function table.imerged(...) + local tmp,ntmp={},0 + for i=1,select("#",...) do + local nst=select(i,...) + for j=1,#nst do + ntmp=ntmp+1 + tmp[ntmp]=nst[j] + end + end + return tmp +end +local function fastcopy(old,metatabletoo) + if old then + local new={} + for k,v in next,old do + if type(v)=="table" then + new[k]=fastcopy(v,metatabletoo) + else + new[k]=v + end end -end - - --- if not string.escapedpattern then --- --- local patterns_escapes = { --- ["%"] = "%%", --- ["."] = "%.", --- ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", --- ["["] = "%[", ["]"] = "%]", --- ["("] = "%(", [")"] = "%)", --- -- ["{"] = "%{", ["}"] = "%}" --- -- ["^"] = "%^", ["$"] = "%$", --- } --- --- local simple_escapes = { --- ["-"] = "%-", --- ["."] = "%.", --- ["?"] = ".", --- ["*"] = ".*", --- } --- --- function string.escapedpattern(str,simple) --- return (gsub(str,".",simple and simple_escapes or patterns_escapes)) --- end --- --- function string.topattern(str,lowercase,strict) --- if str == "" then --- return ".*" --- else --- str = gsub(str,".",simple_escapes) --- if lowercase then --- str = lower(str) --- end --- if strict then --- return "^" .. str .. "$" --- else --- return str --- end --- end --- end --- --- end - ---- needs checking - -local anything = patterns.anything -local allescapes = Cc("%") * S(".-+%?()[]*") -- also {} and ^$ ? -local someescapes = Cc("%") * S(".-+%()[]") -- also {} and ^$ ? -local matchescapes = Cc(".") * S("*?") -- wildcard and single match - -local pattern_a = Cs ( ( allescapes + anything )^0 ) -local pattern_b = Cs ( ( someescapes + matchescapes + anything )^0 ) -local pattern_c = Cs ( Cc("^") * ( someescapes + matchescapes + anything )^0 * Cc("$") ) - -function string.escapedpattern(str,simple) - return lpegmatch(simple and pattern_b or pattern_a,str) -end - -function string.topattern(str,lowercase,strict) - if str == "" then - return ".*" - elseif strict then - str = lpegmatch(pattern_c,str) - else - str = lpegmatch(pattern_b,str) + if metatabletoo then + local mt=getmetatable(old) + if mt then + setmetatable(new,mt) + end end - if lowercase then - return lower(str) - else - return str + return new + else + return {} + end +end +local function copy(t,tables) + tables=tables or {} + local tcopy={} + if not tables[t] then + tables[t]=tcopy + end + for i,v in next,t do + if type(i)=="table" then + if tables[i] then + i=tables[i] + else + i=copy(i,tables) + end end + if type(v)~="table" then + tcopy[i]=v + elseif tables[v] then + tcopy[i]=tables[v] + else + tcopy[i]=copy(v,tables) + end + end + local mt=getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy +end +table.fastcopy=fastcopy +table.copy=copy +function table.derive(parent) + local child={} + if parent then + setmetatable(child,{ __index=parent }) + end + return child end - --- print(string.escapedpattern("12+34*.tex",false)) --- print(string.escapedpattern("12+34*.tex",true)) --- print(string.topattern ("12+34*.tex",false,false)) --- print(string.topattern ("12+34*.tex",false,true)) - -function string.valid(str,default) - return (type(str) == "string" and str ~= "" and str) or default or nil -end - --- handy fallback - -string.itself = function(s) return s end - --- also handy (see utf variant) - -local pattern = Ct(C(1)^0) -- string and not utf ! - -function string.totable(str) - return lpegmatch(pattern,str) +function table.tohash(t,value) + local h={} + if t then + if value==nil then value=true end + for _,v in next,t do + h[v]=value + end + end + return h end - --- handy from within tex: - -local replacer = lpeg.replacer("@","%%") -- Watch the escaped % in lpeg! - -function string.tformat(fmt,...) - return format(lpegmatch(replacer,fmt),...) +function table.fromhash(t) + local hsh,h={},0 + for k,v in next,t do + if v then + h=h+1 + hsh[h]=k + end + end + return hsh end - --- obsolete names: - -string.quote = string.quoted -string.unquote = string.unquoted - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-table'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +local noquotes,hexify,handle,reduce,compact,inline,functions +local reserved=table.tohash { + 'and','break','do','else','elseif','end','false','for','function','if', + 'in','local','nil','not','or','repeat','return','then','true','until','while', } - -local type, next, tostring, tonumber, ipairs, select = type, next, tostring, tonumber, ipairs, select -local table, string = table, string -local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, lower, dump = string.format, string.lower, string.dump -local getmetatable, setmetatable = getmetatable, setmetatable -local getinfo = debug.getinfo -local lpegmatch, patterns = lpeg.match, lpeg.patterns -local floor = math.floor - --- extra functions, some might go (when not used) - -local stripper = patterns.stripper - -function table.strip(tab) - local lst, l = { }, 0 - for i=1,#tab do - local s = lpegmatch(stripper,tab[i]) or "" - if s == "" then - -- skip this one +local function simple_table(t) + if #t>0 then + local n=0 + for _,v in next,t do + n=n+1 + end + if n==#t then + local tt,nt={},0 + for i=1,#t do + local v=t[i] + local tv=type(v) + if tv=="number" then + nt=nt+1 + if hexify then + tt[nt]=format("0x%04X",v) + else + tt[nt]=tostring(v) + end + elseif tv=="boolean" then + nt=nt+1 + tt[nt]=tostring(v) + elseif tv=="string" then + nt=nt+1 + tt[nt]=format("%q",v) else - l = l + 1 - lst[l] = s + tt=nil + break end + end + return tt end - return lst + end + return nil end - -function table.keys(t) - if t then - local keys, k = { }, 0 - for key, _ in next, t do - k = k + 1 - keys[k] = key - end - return keys +local propername=patterns.propername +local function dummy() end +local function do_serialize(root,name,depth,level,indexed) + if level>0 then + depth=depth.." " + if indexed then + handle(format("%s{",depth)) else - return { } + local tn=type(name) + if tn=="number" then + if hexify then + handle(format("%s[0x%04X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) + end + elseif tn=="string" then + if noquotes and not reserved[name] and lpegmatch(propername,name) then + handle(format("%s%s={",depth,name)) + else + handle(format("%s[%q]={",depth,name)) + end + elseif tn=="boolean" then + handle(format("%s[%s]={",depth,tostring(name))) + else + handle(format("%s{",depth)) + end end -end - -local function compare(a,b) - local ta, tb = type(a), type(b) -- needed, else 11 < 2 - if ta == tb then - return a < b - else - return tostring(a) < tostring(b) + end + if root and next(root) then + local first,last=nil,0 + if compact then + last=#root + for k=1,last do + if root[k]==nil then + last=k-1 + break + end + end + if last>0 then + first=1 + end end -end - -local function sortedkeys(tab) - if tab then - local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed - for key,_ in next, tab do - s = s + 1 - srt[s] = key - if category == 3 then - -- no further check + local sk=sortedkeys(root) + for i=1,#sk do + local k=sk[i] + local v=root[k] + local t,tk=type(v),type(k) + if compact and first and tk=="number" and k>=first and k<=last then + if t=="number" then + if hexify then + handle(format("%s 0x%04X,",depth,v)) + else + handle(format("%s %s,",depth,v)) + end + elseif t=="string" then + if reduce and tonumber(v) then + handle(format("%s %s,",depth,v)) + else + handle(format("%s %q,",depth,v)) + end + elseif t=="table" then + if not next(v) then + handle(format("%s {},",depth)) + elseif inline then + local st=simple_table(v) + if st then + handle(format("%s { %s },",depth,concat(st,", "))) else - local tkey = type(key) - if tkey == "string" then - category = (category == 2 and 3) or 1 - elseif tkey == "number" then - category = (category == 1 and 3) or 2 - else - category = 3 - end + do_serialize(v,k,depth,level+1,true) end + else + do_serialize(v,k,depth,level+1,true) + end + elseif t=="boolean" then + handle(format("%s %s,",depth,tostring(v))) + elseif t=="function" then + if functions then + handle(format('%s load(%q),',depth,dump(v))) + else + handle(format('%s "function",',depth)) + end + else + handle(format("%s %q,",depth,tostring(v))) + end + elseif k=="__p__" then + if false then + handle(format("%s __p__=nil,",depth)) end - if category == 0 or category == 3 then - sort(srt,compare) + elseif t=="number" then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif tk=="boolean" then + if hexify then + handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) + else + handle(format("%s [%s]=%s,",depth,tostring(k),v)) + end + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + if hexify then + handle(format("%s %s=0x%04X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) + end else - sort(srt) + if hexify then + handle(format("%s [%q]=0x%04X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end end - return srt - else - return { } - end -end - -local function sortedhashkeys(tab) -- fast one - if tab then - local srt, s = { }, 0 - for key,_ in next, tab do - if key then - s= s + 1 - srt[s] = key + elseif t=="string" then + if reduce and tonumber(v) then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) end - end - sort(srt) - return srt - else - return { } - end -end - -function table.allkeys(t) - local keys = { } - for i=1,#t do - for k, v in next, t[i] do - keys[k] = true - end - end - return sortedkeys(keys) -end - -table.sortedkeys = sortedkeys -table.sortedhashkeys = sortedhashkeys - -local function nothing() end - -local function sortedhash(t) - if t then - local n, s = 0, sortedkeys(t) -- the robust one - local function kv(s) - n = n + 1 - local k = s[n] - return k, t[k] - end - return kv, s - else - return nothing - end -end - -table.sortedhash = sortedhash -table.sortedpairs = sortedhash -- obsolete - -function table.append(t,list) - local n = #t - for i=1,#list do - n = n + 1 - t[n] = list[i] - end - return t -end - -function table.prepend(t, list) - local nl = #list - local nt = nl + #t - for i=#t,1,-1 do - t[nt] = t[i] - nt = nt - 1 - end - for i=1,#list do - t[i] = list[i] - end - return t -end - --- function table.merge(t, ...) -- first one is target --- t = t or { } --- local lst = { ... } --- for i=1,#lst do --- for k, v in next, lst[i] do --- t[k] = v --- end --- end --- return t --- end - -function table.merge(t, ...) -- first one is target - t = t or { } - for i=1,select("#",...) do - for k, v in next, (select(i,...)) do - t[k] = v - end - end - return t -end - --- function table.merged(...) --- local tmp, lst = { }, { ... } --- for i=1,#lst do --- for k, v in next, lst[i] do --- tmp[k] = v --- end --- end --- return tmp --- end - -function table.merged(...) - local t = { } - for i=1,select("#",...) do - for k, v in next, (select(i,...)) do - t[k] = v - end - end - return t -end - --- function table.imerge(t, ...) --- local lst, nt = { ... }, #t --- for i=1,#lst do --- local nst = lst[i] --- for j=1,#nst do --- nt = nt + 1 --- t[nt] = nst[j] --- end --- end --- return t --- end - -function table.imerge(t, ...) - local nt = #t - for i=1,select("#",...) do - local nst = select(i,...) - for j=1,#nst do - nt = nt + 1 - t[nt] = nst[j] - end - end - return t -end - --- function table.imerged(...) --- local tmp, ntmp, lst = { }, 0, {...} --- for i=1,#lst do --- local nst = lst[i] --- for j=1,#nst do --- ntmp = ntmp + 1 --- tmp[ntmp] = nst[j] --- end --- end --- return tmp --- end - -function table.imerged(...) - local tmp, ntmp = { }, 0 - for i=1,select("#",...) do - local nst = select(i,...) - for j=1,#nst do - ntmp = ntmp + 1 - tmp[ntmp] = nst[j] - end - end - return tmp -end - -local function fastcopy(old,metatabletoo) -- fast one - if old then - local new = { } - for k, v in next, old do - if type(v) == "table" then - new[k] = fastcopy(v,metatabletoo) -- was just table.copy + elseif tk=="boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%s,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end + else + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,v)) else - new[k] = v + handle(format("%s [%s]=%q,",depth,k,v)) end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,tostring(k),v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end end - if metatabletoo then - -- optional second arg - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) + elseif t=="table" then + if not next(v) then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]={},",depth,k)) + else + handle(format("%s [%s]={},",depth,k)) end - end - return new - else - return { } - end -end - --- todo : copy without metatable - -local function copy(t, tables) -- taken from lua wiki, slightly adapted - tables = tables or { } - local tcopy = {} - if not tables[t] then - tables[t] = tcopy - end - for i,v in next, t do -- brrr, what happens with sparse indexed - if type(i) == "table" then - if tables[i] then - i = tables[i] + elseif tk=="boolean" then + handle(format("%s [%s]={},",depth,tostring(k))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={},",depth,k)) + else + handle(format("%s [%q]={},",depth,k)) + end + elseif inline then + local st=simple_table(v) + if st then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif tk=="boolean" then + handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else - i = copy(i, tables) + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) end - end - if type(v) ~= "table" then - tcopy[i] = v - elseif tables[v] then - tcopy[i] = tables[v] + else + do_serialize(v,k,depth,level+1) + end else - tcopy[i] = copy(v, tables) - end - end - local mt = getmetatable(t) - if mt then - setmetatable(tcopy,mt) - end - return tcopy -end - -table.fastcopy = fastcopy -table.copy = copy - -function table.derive(parent) -- for the moment not public - local child = { } - if parent then - setmetatable(child,{ __index = parent }) - end - return child -end - -function table.tohash(t,value) - local h = { } - if t then - if value == nil then value = true end - for _, v in next, t do -- no ipairs here - h[v] = value - end - end - return h -end - -function table.fromhash(t) - local hsh, h = { }, 0 - for k, v in next, t do -- no ipairs here - if v then - h = h + 1 - hsh[h] = k - end - end - return hsh -end - -local noquotes, hexify, handle, reduce, compact, inline, functions - -local reserved = table.tohash { -- intercept a language inconvenience: no reserved words as key - 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', - 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', -} - -local function simple_table(t) - if #t > 0 then - local n = 0 - for _,v in next, t do - n = n + 1 - end - if n == #t then - local tt, nt = { }, 0 - for i=1,#t do - local v = t[i] - local tv = type(v) - if tv == "number" then - nt = nt + 1 - if hexify then - tt[nt] = format("0x%04X",v) - else - tt[nt] = tostring(v) -- tostring not needed - end - elseif tv == "boolean" then - nt = nt + 1 - tt[nt] = tostring(v) - elseif tv == "string" then - nt = nt + 1 - tt[nt] = format("%q",v) - else - tt = nil - break - end - end - return tt + do_serialize(v,k,depth,level+1) end - end - return nil -end - --- Because this is a core function of mkiv I moved some function calls --- inline. --- --- twice as fast in a test: --- --- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) - --- problem: there no good number_to_string converter with the best resolution - --- probably using .. is faster than format --- maybe split in a few cases (yes/no hexify) - --- todo: %g faster on numbers than %s - -local propername = patterns.propername -- was find(name,"^%a[%w%_]*$") - -local function dummy() end - -local function do_serialize(root,name,depth,level,indexed) - if level > 0 then - depth = depth .. " " - if indexed then - handle(format("%s{",depth)) + elseif t=="boolean" then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%s,",depth,k,tostring(v))) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%s,",depth,k,tostring(v))) else - local tn = type(name) - if tn == "number" then - if hexify then - handle(format("%s[0x%04X]={",depth,name)) - else - handle(format("%s[%s]={",depth,name)) - end - elseif tn == "string" then - if noquotes and not reserved[name] and lpegmatch(propername,name) then - handle(format("%s%s={",depth,name)) - else - handle(format("%s[%q]={",depth,name)) - end - elseif tn == "boolean" then - handle(format("%s[%s]={",depth,tostring(name))) - else - handle(format("%s{",depth)) - end - end - end - -- we could check for k (index) being number (cardinal) - if root and next(root) then - -- local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) - -- if compact then - -- -- NOT: for k=1,#root do (we need to quit at nil) - -- for k,v in ipairs(root) do -- can we use next? - -- if not first then first = k end - -- last = last + 1 - -- end - -- end - local first, last = nil, 0 - if compact then - last = #root - for k=1,last do - if root[k] == nil then - last = k - 1 - break - end - end - if last > 0 then - first = 1 - end + handle(format("%s [%q]=%s,",depth,k,tostring(v))) end - local sk = sortedkeys(root) - for i=1,#sk do - local k = sk[i] - local v = root[k] - -- circular - local t, tk = type(v), type(k) - if compact and first and tk == "number" and k >= first and k <= last then - if t == "number" then - if hexify then - handle(format("%s 0x%04X,",depth,v)) - else - handle(format("%s %s,",depth,v)) -- %.99g - end - elseif t == "string" then - if reduce and tonumber(v) then - handle(format("%s %s,",depth,v)) - else - handle(format("%s %q,",depth,v)) - end - elseif t == "table" then - if not next(v) then - handle(format("%s {},",depth)) - elseif inline then -- and #t > 0 - local st = simple_table(v) - if st then - handle(format("%s { %s },",depth,concat(st,", "))) - else - do_serialize(v,k,depth,level+1,true) - end - else - do_serialize(v,k,depth,level+1,true) - end - elseif t == "boolean" then - handle(format("%s %s,",depth,tostring(v))) - elseif t == "function" then - if functions then - handle(format('%s load(%q),',depth,dump(v))) - else - handle(format('%s "function",',depth)) - end - else - handle(format("%s %q,",depth,tostring(v))) - end - elseif k == "__p__" then -- parent - if false then - handle(format("%s __p__=nil,",depth)) - end - elseif t == "number" then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g - end - elseif tk == "boolean" then - if hexify then - handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) - else - handle(format("%s [%s]=%s,",depth,tostring(k),v)) -- %.99g - end - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - if hexify then - handle(format("%s %s=0x%04X,",depth,k,v)) - else - handle(format("%s %s=%s,",depth,k,v)) -- %.99g - end - else - if hexify then - handle(format("%s [%q]=0x%04X,",depth,k,v)) - else - handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g - end - end - elseif t == "string" then - if reduce and tonumber(v) then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%s,",depth,tostring(k),v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%s,",depth,k,v)) - else - handle(format("%s [%q]=%s,",depth,k,v)) - end - else - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,v)) - else - handle(format("%s [%s]=%q,",depth,k,v)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%q,",depth,tostring(k),v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%q,",depth,k,v)) - else - handle(format("%s [%q]=%q,",depth,k,v)) - end - end - elseif t == "table" then - if not next(v) then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]={},",depth,k)) - else - handle(format("%s [%s]={},",depth,k)) - end - elseif tk == "boolean" then - handle(format("%s [%s]={},",depth,tostring(k))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s={},",depth,k)) - else - handle(format("%s [%q]={},",depth,k)) - end - elseif inline then - local st = simple_table(v) - if st then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) - else - handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) - end - elseif tk == "boolean" then - handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s={ %s },",depth,k,concat(st,", "))) - else - handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) - end - else - do_serialize(v,k,depth,level+1) - end - else - do_serialize(v,k,depth,level+1) - end - elseif t == "boolean" then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) - else - handle(format("%s [%s]=%s,",depth,k,tostring(v))) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%s,",depth,k,tostring(v))) - else - handle(format("%s [%q]=%s,",depth,k,tostring(v))) - end - elseif t == "function" then - if functions then - local f = getinfo(v).what == "C" and dump(dummy) or dump(v) - -- local f = getinfo(v).what == "C" and dump(function(...) return v(...) end) or dump(v) - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=load(%q),",depth,k,f)) - else - handle(format("%s [%s]=load(%q),",depth,k,f)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=load(%q),",depth,tostring(k),f)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=load(%q),",depth,k,f)) - else - handle(format("%s [%q]=load(%q),",depth,k,f)) - end - end + elseif t=="function" then + if functions then + local f=getinfo(v).what=="C" and dump(dummy) or dump(v) + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=load(%q),",depth,k,f)) else - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) - else - handle(format("%s [%s]=%q,",depth,k,tostring(v))) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%q,",depth,k,tostring(v))) - else - handle(format("%s [%q]=%q,",depth,k,tostring(v))) - end + handle(format("%s [%s]=load(%q),",depth,k,f)) end - end - end - if level > 0 then - handle(format("%s},",depth)) - end -end - --- replacing handle by a direct t[#t+1] = ... (plus test) is not much --- faster (0.03 on 1.00 for zapfino.tma) - -local function serialize(_handle,root,name,specification) -- handle wins - local tname = type(name) - if type(specification) == "table" then - noquotes = specification.noquotes - hexify = specification.hexify - handle = _handle or specification.handle or print - reduce = specification.reduce or false - functions = specification.functions - compact = specification.compact - inline = specification.inline and compact - if functions == nil then - functions = true - end - if compact == nil then - compact = true - end - if inline == nil then - inline = compact - end - else - noquotes = false - hexify = false - handle = _handle or print - reduce = false - compact = true - inline = true - functions = true - end - if tname == "string" then - if name == "return" then - handle("return {") - else - handle(name .. "={") - end - elseif tname == "number" then - if hexify then - handle(format("[0x%04X]={",name)) - else - handle("[" .. name .. "]={") + elseif tk=="boolean" then + handle(format("%s [%s]=load(%q),",depth,tostring(k),f)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=load(%q),",depth,k,f)) + else + handle(format("%s [%q]=load(%q),",depth,k,f)) + end end - elseif tname == "boolean" then - if name then - handle("return {") + else + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%q,",depth,k,tostring(v))) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,tostring(v))) else - handle("{") + handle(format("%s [%q]=%q,",depth,k,tostring(v))) end + end + end + end + if level>0 then + handle(format("%s},",depth)) + end +end +local function serialize(_handle,root,name,specification) + local tname=type(name) + if type(specification)=="table" then + noquotes=specification.noquotes + hexify=specification.hexify + handle=_handle or specification.handle or print + reduce=specification.reduce or false + functions=specification.functions + compact=specification.compact + inline=specification.inline and compact + if functions==nil then + functions=true + end + if compact==nil then + compact=true + end + if inline==nil then + inline=compact + end + else + noquotes=false + hexify=false + handle=_handle or print + reduce=false + compact=true + inline=true + functions=true + end + if tname=="string" then + if name=="return" then + handle("return {") + else + handle(name.."={") + end + elseif tname=="number" then + if hexify then + handle(format("[0x%04X]={",name)) + else + handle("["..name.."]={") + end + elseif tname=="boolean" then + if name then + handle("return {") else - handle("t={") - end - if root then - -- The dummy access will initialize a table that has a delayed initialization - -- using a metatable. (maybe explicitly test for metatable) - if getmetatable(root) then -- todo: make this an option, maybe even per subtable - local dummy = root._w_h_a_t_e_v_e_r_ - root._w_h_a_t_e_v_e_r_ = nil - end - -- Let's forget about empty tables. - if next(root) then - do_serialize(root,name,"",0) - end + handle("{") end - handle("}") -end - --- name: --- --- true : return { } --- false : { } --- nil : t = { } --- string : string = { } --- "return" : return { } --- number : [number] = { } - -function table.serialize(root,name,specification) - local t, n = { }, 0 - local function flush(s) - n = n + 1 - t[n] = s + else + handle("t={") + end + if root then + if getmetatable(root) then + local dummy=root._w_h_a_t_e_v_e_r_ + root._w_h_a_t_e_v_e_r_=nil end - serialize(flush,root,name,specification) - return concat(t,"\n") -end - -table.tohandle = serialize - --- sometimes tables are real use (zapfino extra pro is some 85M) in which --- case a stepwise serialization is nice; actually, we could consider: --- --- for line in table.serializer(root,name,reduce,noquotes) do --- ...(line) --- end --- --- so this is on the todo list - -local maxtab = 2*1024 - -function table.tofile(filename,root,name,specification) - local f = io.open(filename,'w') - if f then - if maxtab > 1 then - local t, n = { }, 0 - local function flush(s) - n = n + 1 - t[n] = s - if n > maxtab then - f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice - t, n = { }, 0 -- we could recycle t if needed - end - end - serialize(flush,root,name,specification) - f:write(concat(t,"\n"),"\n") - else - local function flush(s) - f:write(s,"\n") - end - serialize(flush,root,name,specification) - end - f:close() - io.flush() + if next(root) then + do_serialize(root,name,"",0) end + end + handle("}") end - -local function flattened(t,f,depth) - if f == nil then - f = { } - depth = 0xFFFF - elseif tonumber(f) then - -- assume that only two arguments are given - depth = f - f = { } - elseif not depth then - depth = 0xFFFF - end - for k, v in next, t do - if type(k) ~= "number" then - if depth > 0 and type(v) == "table" then - flattened(v,f,depth-1) - else - f[k] = v - end - end - end - local n = #f - for k=1,#t do - local v = t[k] - if depth > 0 and type(v) == "table" then - flattened(v,f,depth-1) - n = #f - else - n = n + 1 - f[n] = v - end - end - return f -end - -table.flattened = flattened - -local function unnest(t,f) -- only used in mk, for old times sake - if not f then -- and only relevant for token lists - f = { } -- this one can become obsolete - end - for i=1,#t do - local v = t[i] - if type(v) == "table" then - if type(v[1]) == "table" then - unnest(v,f) - else - f[#f+1] = v - end - else - f[#f+1] = v - end - end - return f -end - -function table.unnest(t) -- bad name - return unnest(t) -end - -local function are_equal(a,b,n,m) -- indexed - if a and b and #a == #b then - n = n or 1 - m = m or #a - for i=n,m do - local ai, bi = a[i], b[i] - if ai==bi then - -- same - elseif type(ai) == "table" and type(bi) == "table" then - if not are_equal(ai,bi) then - return false - end - else - return false - end - end - return true - else - return false - end -end - -local function identical(a,b) -- assumes same structure - for ka, va in next, a do - local vb = b[ka] - if va == vb then - -- same - elseif type(va) == "table" and type(vb) == "table" then - if not identical(va,vb) then - return false - end - else - return false - end - end - return true -end - -table.identical = identical -table.are_equal = are_equal - --- maybe also make a combined one - -function table.compact(t) -- remove empty tables, assumes subtables - if t then - for k, v in next, t do - if not next(v) then -- no type checking - t[k] = nil - end - end - end -end - -function table.contains(t, v) - if t then - for i=1, #t do - if t[i] == v then - return i - end - end - end - return false -end - -function table.count(t) - local n = 0 - for k, v in next, t do - n = n + 1 - end - return n -end - -function table.swapped(t,s) -- hash - local n = { } - if s then - for k, v in next, s do - n[k] = v - end - end - for k, v in next, t do - n[v] = k - end - return n -end - -function table.mirrored(t) -- hash - local n = { } - for k, v in next, t do - n[v] = k - n[k] = v - end - return n -end - -function table.reversed(t) - if t then - local tt, tn = { }, #t - if tn > 0 then - local ttn = 0 - for i=tn,1,-1 do - ttn = ttn + 1 - tt[ttn] = t[i] - end - end - return tt - end -end - -function table.reverse(t) - if t then - local n = #t - for i=1,floor(n/2) do - local j = n - i + 1 - t[i], t[j] = t[j], t[i] - end - return t - end -end - -function table.sequenced(t,sep) -- hash only - if t then - local s, n = { }, 0 - for k, v in sortedhash(t) do - if simple then - if v == true then - n = n + 1 - s[n] = k - elseif v and v~= "" then - n = n + 1 - s[n] = k .. "=" .. tostring(v) - end - else - n = n + 1 - s[n] = k .. "=" .. tostring(v) - end - end - return concat(s, sep or " | ") - else - return "" - end -end - -function table.print(t,...) - if type(t) ~= "table" then - print(tostring(t)) - else - table.tohandle(print,t,...) - end -end - --- -- -- obsolete but we keep them for a while and might comment them later -- -- -- - --- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) - -function table.sub(t,i,j) - return { unpack(t,i,j) } -end - --- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) - -function table.is_empty(t) - return not t or not next(t) -end - -function table.has_one_entry(t) - return t and not next(t,next(t)) -end - --- new - -function table.loweredkeys(t) -- maybe utf - local l = { } - for k, v in next, t do - l[lower(k)] = v - end - return l -end - --- new, might move (maybe duplicate) - -function table.unique(old) - local hash = { } - local new = { } - local n = 0 - for i=1,#old do - local oi = old[i] - if not hash[oi] then - n = n + 1 - new[n] = oi - hash[oi] = true - end - end - return new -end - -function table.sorted(t,...) - sort(t,...) - return t -- still sorts in-place -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-io'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local io = io -local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format -local concat = table.concat -local floor = math.floor -local type = type - -if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator = "\\", ";" -else - io.fileseparator, io.pathseparator = "/" , ":" -end - -local function readall(f) - return f:read("*all") -end - --- The next one is upto 50% faster on large files and less memory consumption due --- to less intermediate large allocations. This phenomena was discussed on the --- luatex dev list. - -local function readall(f) - local size = f:seek("end") - if size == 0 then - return "" - elseif size < 1024*1024 then - f:seek("set",0) - return f:read('*all') - else - local done = f:seek("set",0) - if size < 1024*1024 then - step = 1024 * 1024 - elseif size > 16*1024*1024 then - step = 16*1024*1024 - else - step = floor(size/(1024*1024)) * 1024 * 1024 / 8 - end - local data = { } - while true do - local r = f:read(step) - if not r then - return concat(data) - else - data[#data+1] = r - end - end - end -end - -io.readall = readall - -function io.loaddata(filename,textmode) -- return nil if empty - local f = io.open(filename,(textmode and 'r') or 'rb') - if f then --- local data = f:read('*all') - local data = readall(f) - f:close() - if #data > 0 then - return data - end - end -end - -function io.savedata(filename,data,joiner) - local f = io.open(filename,"wb") - if f then - if type(data) == "table" then - f:write(concat(data,joiner or "")) - elseif type(data) == "function" then - data(f) - else - f:write(data or "") - end - f:close() - io.flush() - return true - else - return false - end -end - -function io.loadlines(filename,n) -- return nil if empty - local f = io.open(filename,'r') - if not f then - -- no file - elseif n then - local lines = { } - for i=1,n do - local line = f:read("*lines") - if line then - lines[#lines+1] = line - else - break - end - end - f:close() - lines = concat(lines,"\n") - if #lines > 0 then - return lines - end - else - local line = f:read("*line") or "" - f:close() - if #line > 0 then - return line - end - end -end - -function io.loadchunk(filename,n) - local f = io.open(filename,'rb') - if f then - local data = f:read(n or 1024) - f:close() - if #data > 0 then - return data - end - end -end - -function io.exists(filename) - local f = io.open(filename) - if f == nil then - return false - else - f:close() - return true - end -end - -function io.size(filename) - local f = io.open(filename) - if f == nil then - return 0 - else - local s = f:seek("end") - f:close() - return s - end -end - -function io.noflines(f) - if type(f) == "string" then - local f = io.open(filename) - if f then - local n = f and io.noflines(f) or 0 - f:close() - return n - else - return 0 - end - else - local n = 0 - for _ in f:lines() do - n = n + 1 - end - f:seek('set',0) - return n - end -end - -local nextchar = { - [ 4] = function(f) - return f:read(1,1,1,1) - end, - [ 2] = function(f) - return f:read(1,1) - end, - [ 1] = function(f) - return f:read(1) - end, - [-2] = function(f) - local a, b = f:read(1,1) - return b, a - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - return d, c, b, a - end -} - -function io.characters(f,n) - if f then - return nextchar[n or 1], f - end -end - -local nextbyte = { - [4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return byte(a), byte(b), byte(c), byte(d) - end - end, - [3] = function(f) - local a, b, c = f:read(1,1,1) - if b then - return byte(a), byte(b), byte(c) - end - end, - [2] = function(f) - local a, b = f:read(1,1) - if b then - return byte(a), byte(b) - end - end, - [1] = function (f) - local a = f:read(1) - if a then - return byte(a) - end - end, - [-2] = function (f) - local a, b = f:read(1,1) - if b then - return byte(b), byte(a) - end - end, - [-3] = function(f) - local a, b, c = f:read(1,1,1) - if b then - return byte(c), byte(b), byte(a) - end - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return byte(d), byte(c), byte(b), byte(a) - end - end -} - -function io.bytes(f,n) - if f then - return nextbyte[n or 1], f - else - return nil, nil - end -end - -function io.ask(question,default,options) - while true do - io.write(question) - if options then - io.write(format(" [%s]",concat(options,"|"))) - end - if default then - io.write(format(" [%s]",default)) - end - io.write(format(" ")) - io.flush() - local answer = io.read() - answer = gsub(answer,"^%s*(.*)%s*$","%1") - if answer == "" and default then - return default - elseif not options then - return answer - else - for k=1,#options do - if options[k] == answer then - return answer - end - end - local pattern = "^" .. answer - for k=1,#options do - local v = options[k] - if find(v,pattern) then - return v - end - end - end - end -end - -local function readnumber(f,n,m) - if m then - f:seek("set",n) - n = m - end - if n == 1 then - return byte(f:read(1)) - elseif n == 2 then - local a, b = byte(f:read(2),1,2) - return 256 * a + b - elseif n == 3 then - local a, b, c = byte(f:read(3),1,3) - return 256*256 * a + 256 * b + c - elseif n == 4 then - local a, b, c, d = byte(f:read(4),1,4) - return 256*256*256 * a + 256*256 * b + 256 * c + d - elseif n == 8 then - local a, b = readnumber(f,4), readnumber(f,4) - return 256 * a + b - elseif n == 12 then - local a, b, c = readnumber(f,4), readnumber(f,4), readnumber(f,4) - return 256*256 * a + 256 * b + c - elseif n == -2 then - local b, a = byte(f:read(2),1,2) - return 256*a + b - elseif n == -3 then - local c, b, a = byte(f:read(3),1,3) - return 256*256 * a + 256 * b + c - elseif n == -4 then - local d, c, b, a = byte(f:read(4),1,4) - return 256*256*256 * a + 256*256 * b + 256*c + d - elseif n == -8 then - local h, g, f, e, d, c, b, a = byte(f:read(8),1,8) - return 256*256*256*256*256*256*256 * a + - 256*256*256*256*256*256 * b + - 256*256*256*256*256 * c + - 256*256*256*256 * d + - 256*256*256 * e + - 256*256 * f + - 256 * g + - h - else - return 0 - end -end - -io.readnumber = readnumber - -function io.readstring(f,n,m) - if m then - f:seek("set",n) - n = m - end - local str = gsub(f:read(n),"\000","") - return str -end - --- - -if not io.i_limiter then function io.i_limiter() end end -- dummy so we can test safely -if not io.o_limiter then function io.o_limiter() end end -- dummy so we can test safely - --- This works quite ok: --- --- function io.piped(command,writer) --- local pipe = io.popen(command) --- -- for line in pipe:lines() do --- -- print(line) --- -- end --- while true do --- local line = pipe:read(1) --- if not line then --- break --- elseif line ~= "\n" then --- writer(line) --- end --- end --- return pipe:close() -- ok, status, (error)code --- end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-number'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module will be replaced when we have the bit library - -local tostring, tonumber = tostring, tonumber -local format, floor, match, rep = string.format, math.floor, string.match, string.rep -local concat, insert = table.concat, table.insert -local lpegmatch = lpeg.match - -number = number or { } -local number = number - -if bit32 then - - local btest, bor = bit32.btest, bit32.bor - - function number.bit(p) - return 2 ^ (p - 1) -- 1-based indexing - end - - number.hasbit = btest - number.setbit = bor - - function number.setbit(x,p) - return btest(x,p) and x or x + p - end - - function number.clearbit(x,p) - return btest(x,p) and x - p or x - end - -else - - -- http://ricilake.blogspot.com/2007/10/iterating-bits-in-lua.html - - function number.bit(p) - return 2 ^ (p - 1) -- 1-based indexing - end - - function number.hasbit(x, p) -- typical call: if hasbit(x, bit(3)) then ... - return x % (p + p) >= p - end - - function number.setbit(x, p) - return (x % (p + p) >= p) and x or x + p - end - - function number.clearbit(x, p) - return (x % (p + p) >= p) and x - p or x - end - -end - --- print(number.tobitstring(8)) --- print(number.tobitstring(14)) --- print(number.tobitstring(66)) --- print(number.tobitstring(0x00)) --- print(number.tobitstring(0xFF)) --- print(number.tobitstring(46260767936,4)) - -if bit32 then - - local bextract = bit32.extract - - local t = { - "0", "0", "0", "0", "0", "0", "0", "0", - "0", "0", "0", "0", "0", "0", "0", "0", - "0", "0", "0", "0", "0", "0", "0", "0", - "0", "0", "0", "0", "0", "0", "0", "0", - } - - function number.tobitstring(b,m) - -- if really needed we can speed this one up - -- because small numbers need less extraction - local n = 32 - for i=0,31 do - local v = bextract(b,i) - local k = 32 - i - if v == 1 then - n = k - t[k] = "1" - else - t[k] = "0" - end - end - if m then - m = 33 - m * 8 - if m < 1 then - m = 1 - end - return concat(t,"",m) - elseif n < 8 then - return concat(t) - elseif n < 16 then - return concat(t,"",9) - elseif n < 24 then - return concat(t,"",17) - else - return concat(t,"",25) - end - end - -else - - function number.tobitstring(n,m) - if n > 0 then - local t = { } - while n > 0 do - insert(t,1,n % 2 > 0 and 1 or 0) - n = floor(n/2) - end - local nn = 8 - #t % 8 - if nn > 0 and nn < 8 then - for i=1,nn do - insert(t,1,0) - end - end - if m then - m = m * 8 - #t - if m > 0 then - insert(t,1,rep("0",m)) - end - end - return concat(t) - elseif m then - rep("00000000",m) - else - return "00000000" - end - end - -end - -function number.valid(str,default) - return tonumber(str) or default or nil -end - -function number.toevenhex(n) - local s = format("%X",n) - if #s % 2 == 0 then - return s - else - return "0" .. s - end -end - --- a,b,c,d,e,f = number.toset(100101) --- --- function number.toset(n) --- return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") --- end --- --- -- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5% --- -- on --- --- for i=1,1000000 do --- local a,b,c,d,e,f,g,h = number.toset(12345678) --- local a,b,c,d = number.toset(1234) --- local a,b,c = number.toset(123) --- local a,b,c = number.toset("123") --- end - -local one = lpeg.C(1-lpeg.S('')/tonumber)^1 - -function number.toset(n) - return lpegmatch(one,tostring(n)) -end - --- function number.bits(n,zero) --- local t, i = { }, (zero and 0) or 1 --- while n > 0 do --- local m = n % 2 --- if m > 0 then --- insert(t,1,i) --- end --- n = floor(n/2) --- i = i + 1 --- end --- return t --- end --- --- -- a bit faster - -local function bits(n,i,...) - if n > 0 then - local m = n % 2 - local n = floor(n/2) - if m > 0 then - return bits(n, i+1, i, ...) - else - return bits(n, i+1, ...) - end - else - return ... - end -end - -function number.bits(n) - return { bits(n,1) } -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-set'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This will become obsolete when we have the bitset library embedded. - -set = set or { } - -local nums = { } -local tabs = { } -local concat = table.concat -local next, type = next, type - -set.create = table.tohash - -function set.tonumber(t) - if next(t) then - local s = "" - -- we could save mem by sorting, but it slows down - for k, v in next, t do - if v then - -- why bother about the leading space - s = s .. " " .. k - end - end - local n = nums[s] - if not n then - n = #tabs + 1 - tabs[n] = t - nums[s] = n - end - return n - else - return 0 - end -end - -function set.totable(n) - if n == 0 then - return { } - else - return tabs[n] or { } - end -end - -function set.tolist(n) - if n == 0 or not tabs[n] then - return "" - else - local t, n = { }, 0 - for k, v in next, tabs[n] do - if v then - n = n + 1 - t[n] = k - end - end - return concat(t," ") - end -end - -function set.contains(n,s) - if type(n) == "table" then - return n[s] - elseif n == 0 then - return false - else - local t = tabs[n] - return t and t[s] - end -end - - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-os'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This file deals with some operating system issues. Please don't bother me --- with the pros and cons of operating systems as they all have their flaws --- and benefits. Bashing one of them won't help solving problems and fixing --- bugs faster and is a waste of time and energy. --- --- path separators: / or \ ... we can use / everywhere --- suffixes : dll so exe ... no big deal --- quotes : we can use "" in most cases --- expansion : unless "" are used * might give side effects --- piping/threads : somewhat different for each os --- locations : specific user file locations and settings can change over time --- --- os.type : windows | unix (new, we already guessed os.platform) --- os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) --- os.platform : extended os.name with architecture - --- os.sleep() => socket.sleep() --- math.randomseed(tonumber(string.sub(string.reverse(tostring(math.floor(socket.gettime()*10000))),1,6))) - --- maybe build io.flush in os.execute - -local os = os -local date, time = os.date, os.time -local find, format, gsub, upper, gmatch = string.find, string.format, string.gsub, string.upper, string.gmatch -local concat = table.concat -local random, ceil, randomseed = math.random, math.ceil, math.randomseed -local rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring = rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring - --- The following code permits traversing the environment table, at least --- in luatex. Internally all environment names are uppercase. - --- The randomseed in Lua is not that random, although this depends on the operating system as well --- as the binary (Luatex is normally okay). But to be sure we set the seed anyway. - -math.initialseed = tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6)) - -randomseed(math.initialseed) - -if not os.__getenv__ then - - os.__getenv__ = os.getenv - os.__setenv__ = os.setenv - - if os.env then - - local osgetenv = os.getenv - local ossetenv = os.setenv - local osenv = os.env local _ = osenv.PATH -- initialize the table - - function os.setenv(k,v) - if v == nil then - v = "" - end - local K = upper(k) - osenv[K] = v - if type(v) == "table" then - v = concat(v,";") -- path - end - ossetenv(K,v) - end - - function os.getenv(k) - local K = upper(k) - local v = osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) - if v == "" then - return nil - else - return v - end - end - - else - - local ossetenv = os.setenv - local osgetenv = os.getenv - local osenv = { } - - function os.setenv(k,v) - if v == nil then - v = "" - end - local K = upper(k) - osenv[K] = v - end - - function os.getenv(k) - local K = upper(k) - local v = osenv[K] or osgetenv(K) or osgetenv(k) - if v == "" then - return nil - else - return v - end - end - - local function __index(t,k) - return os.getenv(k) - end - local function __newindex(t,k,v) - os.setenv(k,v) - end - - os.env = { } - - setmetatable(os.env, { __index = __index, __newindex = __newindex } ) - - end - -end - --- end of environment hack - -local execute, spawn, exec, iopopen, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.popen, io.flush - -function os.execute(...) ioflush() return execute(...) end -function os.spawn (...) ioflush() return spawn (...) end -function os.exec (...) ioflush() return exec (...) end -function io.popen (...) ioflush() return iopopen(...) end - -function os.resultof(command) - local handle = io.popen(command,"r") - return handle and handle:read("*all") or "" -end - -if not io.fileseparator then - if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" - else - io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" - end -end - -os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" -os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" - -if os.type == "windows" then - os.libsuffix, os.binsuffix, os.binsuffixes = 'dll', 'exe', { 'exe', 'cmd', 'bat' } -else - os.libsuffix, os.binsuffix, os.binsuffixes = 'so', '', { '' } -end - -local launchers = { - windows = "start %s", - macosx = "open %s", - unix = "$BROWSER %s &> /dev/null &", -} - -function os.launch(str) - os.execute(format(launchers[os.name] or launchers.unix,str)) -end - -if not os.times then - -- utime = user time - -- stime = system time - -- cutime = children user time - -- cstime = children system time - function os.times() - return { - utime = os.gettimeofday(), -- user - stime = 0, -- system - cutime = 0, -- children user - cstime = 0, -- children system - } - end -end - -os.gettimeofday = os.gettimeofday or os.clock - -local startuptime = os.gettimeofday() - -function os.runtime() - return os.gettimeofday() - startuptime -end - - --- no need for function anymore as we have more clever code and helpers now --- this metatable trickery might as well disappear - -os.resolvers = os.resolvers or { } -- will become private - -local resolvers = os.resolvers - -local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil -local osix = osmt.__index - -osmt.__index = function(t,k) - return (resolvers[k] or osix)(t,k) -end - -setmetatable(os,osmt) - --- we can use HOSTTYPE on some platforms - -local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" - -local function guess() - local architecture = os.resultof("uname -m") or "" - if architecture ~= "" then - return architecture - end - architecture = os.getenv("HOSTTYPE") or "" - if architecture ~= "" then - return architecture - end - return os.resultof("echo $HOSTTYPE") or "" -end - -if platform ~= "" then - - os.platform = platform - -elseif os.type == "windows" then - - -- we could set the variable directly, no function needed here - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" - if find(architecture,"AMD64") then - platform = "mswin-64" - else - platform = "mswin" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "linux" then - - function os.resolvers.platform(t,k) - -- we sometimes have HOSTTYPE set so let's check that first - local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "macosx" then - - --[[ - Identifying the architecture of OSX is quite a mess and this - is the best we can come up with. For some reason $HOSTTYPE is - a kind of pseudo environment variable, not known to the current - environment. And yes, uname cannot be trusted either, so there - is a change that you end up with a 32 bit run on a 64 bit system. - Also, some proper 64 bit intel macs are too cheap (low-end) and - therefore not permitted to run the 64 bit kernel. - ]]-- - - function os.resolvers.platform(t,k) - -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" - -- if architecture == "" then - -- architecture = os.resultof("echo $HOSTTYPE") or "" - -- end - local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" - if architecture == "" then - -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") - platform = "osx-intel" - elseif find(architecture,"i386") then - platform = "osx-intel" - elseif find(architecture,"x86_64") then - platform = "osx-64" - else - platform = "osx-ppc" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "sunos" then - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.resultof("uname -m") or "" - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "freebsd" then - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.resultof("uname -m") or "" - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "kfreebsd" then - - function os.resolvers.platform(t,k) - -- we sometimes have HOSTTYPE set so let's check that first - local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then - platform = "kfreebsd-amd64" - else - platform = "kfreebsd-i386" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -else - - -- platform = "linux" - -- os.setenv("MTX_PLATFORM",platform) - -- os.platform = platform - - function os.resolvers.platform(t,k) - local platform = "linux" - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -end - --- beware, we set the randomseed - --- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the --- version number as well as two reserved bits. All other bits are set using a random or pseudorandom --- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal --- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. --- --- as we don't call this function too often there is not so much risk on repetition - -local t = { 8, 9, "a", "b" } - -function os.uuid() - return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", - random(0xFFFF),random(0xFFFF), - random(0x0FFF), - t[ceil(random(4))] or 8,random(0x0FFF), - random(0xFFFF), - random(0xFFFF),random(0xFFFF),random(0xFFFF) - ) -end - -local d - -function os.timezone(delta) - d = d or tonumber(tonumber(date("%H")-date("!%H"))) - if delta then - if d > 0 then - return format("+%02i:00",d) - else - return format("-%02i:00",-d) - end - else - return 1 - end -end - -local timeformat = format("%%s%s",os.timezone(true)) -local dateformat = "!%Y-%m-%d %H:%M:%S" - -function os.fulltime(t,default) - t = tonumber(t) or 0 - if t > 0 then - -- valid time - elseif default then - return default - else - t = nil - end - return format(timeformat,date(dateformat,t)) -end - -local dateformat = "%Y-%m-%d %H:%M:%S" - -function os.localtime(t,default) - t = tonumber(t) or 0 - if t > 0 then - -- valid time - elseif default then - return default - else - t = nil - end - return date(dateformat,t) -end - -function os.converttime(t,default) - local t = tonumber(t) - if t and t > 0 then - return date(dateformat,t) - else - return default or "-" - end -end - -local memory = { } - -local function which(filename) - local fullname = memory[filename] - if fullname == nil then - local suffix = file.suffix(filename) - local suffixes = suffix == "" and os.binsuffixes or { suffix } - for directory in gmatch(os.getenv("PATH"),"[^" .. io.pathseparator .."]+") do - local df = file.join(directory,filename) - for i=1,#suffixes do - local dfs = file.addsuffix(df,suffixes[i]) - if io.exists(dfs) then - fullname = dfs - break - end - end - end - if not fullname then - fullname = false - end - memory[filename] = fullname - end - return fullname -end - -os.which = which -os.where = which - -function os.today() - return date("!*t") -- table with values -end - -function os.now() - return date("!%Y-%m-%d %H:%M:%S") -- 2011-12-04 14:59:12 -end - - --- print(os.which("inkscape.exe")) --- print(os.which("inkscape")) --- print(os.which("gs.exe")) --- print(os.which("ps2pdf")) - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-file'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- needs a cleanup - -file = file or { } -local file = file - -local insert, concat = table.insert, table.concat -local match = string.match -local lpegmatch = lpeg.match -local getcurrentdir, attributes = lfs.currentdir, lfs.attributes -local checkedsplit = string.checkedsplit - --- local patterns = file.patterns or { } --- file.patterns = patterns - -local P, R, S, C, Cs, Cp, Cc, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpeg.Cp, lpeg.Cc, lpeg.Ct - -local colon = P(":") -local period = P(".") -local periods = P("..") -local fwslash = P("/") -local bwslash = P("\\") -local slashes = S("\\/") -local noperiod = 1-period -local noslashes = 1-slashes -local name = noperiod^1 -local suffix = period/"" * (1-period-slashes)^1 * -1 - -local pattern = C((noslashes^0 * slashes^1)^1) - -local function pathpart(name,default) - return name and lpegmatch(pattern,name) or default or "" -end - -local pattern = (noslashes^0 * slashes)^1 * C(noslashes^1) * -1 - -local function basename(name) - return name and lpegmatch(pattern,name) or name -end - -local pattern = (noslashes^0 * slashes^1)^0 * Cs((1-suffix)^1) * suffix^0 - -local function nameonly(name) - return name and lpegmatch(pattern,name) or name -end - -local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * C(noperiod^1) * -1 - -local function suffixonly(name) - return name and lpegmatch(pattern,name) or "" -end - -file.pathpart = pathpart -file.basename = basename -file.nameonly = nameonly -file.suffixonly = suffixonly -file.suffix = suffixonly - -file.dirname = pathpart -- obsolete -file.extname = suffixonly -- obsolete - --- actually these are schemes - -local drive = C(R("az","AZ")) * colon -local path = C(((1-slashes)^0 * slashes)^0) -local suffix = period * C(P(1-period)^0 * P(-1)) -local base = C((1-suffix)^0) -local rest = C(P(1)^0) - -drive = drive + Cc("") -path = path + Cc("") -base = base + Cc("") -suffix = suffix + Cc("") - -local pattern_a = drive * path * base * suffix -local pattern_b = path * base * suffix -local pattern_c = C(drive * path) * C(base * suffix) -- trick: two extra captures -local pattern_d = path * rest - -function file.splitname(str,splitdrive) - if not str then - -- error - elseif splitdrive then - return lpegmatch(pattern_a,str) -- returns drive, path, base, suffix - else - return lpegmatch(pattern_b,str) -- returns path, base, suffix - end -end - -function file.splitbase(str) - return str and lpegmatch(pattern_d,str) -- returns path, base+suffix -end - -function file.nametotable(str,splitdrive) -- returns table - if str then - local path, drive, subpath, name, base, suffix = lpegmatch(pattern_c,str) - if splitdrive then - return { - path = path, - drive = drive, - subpath = subpath, - name = name, - base = base, - suffix = suffix, - } - else - return { - path = path, - name = name, - base = base, - suffix = suffix, - } - end - end -end - -local pattern = Cs(((period * noperiod^1 * -1)/"" + 1)^1) - -function file.removesuffix(name) - return name and lpegmatch(pattern,name) -end - --- local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * Cp() * noperiod^1 * -1 --- --- function file.addsuffix(name, suffix) --- local p = lpegmatch(pattern,name) --- if p then --- return name --- else --- return name .. "." .. suffix --- end --- end - -local suffix = period/"" * (1-period-slashes)^1 * -1 -local pattern = Cs((noslashes^0 * slashes^1)^0 * ((1-suffix)^1)) * Cs(suffix) - -function file.addsuffix(filename,suffix,criterium) - if not filename or not suffix or suffix == "" then - return filename - elseif criterium == true then - return filename .. "." .. suffix - elseif not criterium then - local n, s = lpegmatch(pattern,filename) - if not s or s == "" then - return filename .. "." .. suffix - else - return filename - end - else - local n, s = lpegmatch(pattern,filename) - if s and s ~= "" then - local t = type(criterium) - if t == "table" then - -- keep if in criterium - for i=1,#criterium do - if s == criterium[i] then - return filename - end - end - elseif t == "string" then - -- keep if criterium - if s == criterium then - return filename - end - end - end - return (n or filename) .. "." .. suffix - end -end - --- print("1 " .. file.addsuffix("name","new") .. " -> name.new") --- print("2 " .. file.addsuffix("name.old","new") .. " -> name.old") --- print("3 " .. file.addsuffix("name.old","new",true) .. " -> name.old.new") --- print("4 " .. file.addsuffix("name.old","new","new") .. " -> name.new") --- print("5 " .. file.addsuffix("name.old","new","old") .. " -> name.old") --- print("6 " .. file.addsuffix("name.old","new","foo") .. " -> name.new") --- print("7 " .. file.addsuffix("name.old","new",{"foo","bar"}) .. " -> name.new") --- print("8 " .. file.addsuffix("name.old","new",{"old","bar"}) .. " -> name.old") - -local suffix = period * (1-period-slashes)^1 * -1 -local pattern = Cs((1-suffix)^0) - -function file.replacesuffix(name,suffix) - if name and suffix and suffix ~= "" then - return lpegmatch(pattern,name) .. "." .. suffix - else - return name - end -end - --- - -local reslasher = lpeg.replacer(P("\\"),"/") - -function file.reslash(str) - return str and lpegmatch(reslasher,str) -end - --- We should be able to use: --- --- local writable = P(1) * P("w") * Cc(true) --- --- function file.is_writable(name) --- local a = attributes(name) or attributes(pathpart(name,".")) --- return a and lpegmatch(writable,a.permissions) or false --- end --- --- But after some testing Taco and I came up with the more robust --- variant: - -function file.is_writable(name) - if not name then - -- error - elseif lfs.isdir(name) then - name = name .. "/m_t_x_t_e_s_t.tmp" - local f = io.open(name,"wb") - if f then - f:close() - os.remove(name) - return true - end - elseif lfs.isfile(name) then - local f = io.open(name,"ab") - if f then - f:close() - return true - end - else - local f = io.open(name,"ab") - if f then - f:close() - os.remove(name) - return true - end - end - return false -end - -local readable = P("r") * Cc(true) - -function file.is_readable(name) - if name then - local a = attributes(name) - return a and lpegmatch(readable,a.permissions) or false - else - return false - end -end - -file.isreadable = file.is_readable -- depricated -file.iswritable = file.is_writable -- depricated - -function file.size(name) - if name then - local a = attributes(name) - return a and a.size or 0 - else - return 0 - end -end - -function file.splitpath(str,separator) -- string .. reslash is a bonus (we could do a direct split) - return str and checkedsplit(lpegmatch(reslasher,str),separator or io.pathseparator) -end - -function file.joinpath(tab,separator) -- table - return tab and concat(tab,separator or io.pathseparator) -- can have trailing // -end - -local stripper = Cs(P(fwslash)^0/"" * reslasher) -local isnetwork = fwslash * fwslash * (1-fwslash) + (1-fwslash-colon)^1 * colon -local isroot = fwslash^1 * -1 -local hasroot = fwslash^1 - -local deslasher = lpeg.replacer(S("\\/")^1,"/") - --- If we have a network or prefix then there is a change that we end up with two --- // in the middle ... we could prevent this if we (1) expand prefixes: and (2) --- split and rebuild as url. Of course we could assume no network paths (which --- makes sense) adn assume either mapped drives (windows) or mounts (unix) but --- then we still have to deal with urls ... anyhow, multiple // are never a real --- problem but just ugly. - -function file.join(...) - local lst = { ... } - local one = lst[1] - if lpegmatch(isnetwork,one) then - local two = lpegmatch(deslasher,concat(lst,"/",2)) - return one .. "/" .. two - elseif lpegmatch(isroot,one) then - local two = lpegmatch(deslasher,concat(lst,"/",2)) - if lpegmatch(hasroot,two) then - return two - else - return "/" .. two - end - elseif one == "" then - return lpegmatch(stripper,concat(lst,"/",2)) - else - return lpegmatch(deslasher,concat(lst,"/")) - end -end - --- print(file.join("c:/whatever","name")) --- print(file.join("//","/y")) --- print(file.join("/","/y")) --- print(file.join("","/y")) --- print(file.join("/x/","/y")) --- print(file.join("x/","/y")) --- print(file.join("http://","/y")) --- print(file.join("http://a","/y")) --- print(file.join("http:///a","/y")) --- print(file.join("//nas-1","/y")) - --- The previous one fails on "a.b/c" so Taco came up with a split based --- variant. After some skyping we got it sort of compatible with the old --- one. After that the anchoring to currentdir was added in a better way. --- Of course there are some optimizations too. Finally we had to deal with --- windows drive prefixes and things like sys://. Eventually gsubs and --- finds were replaced by lpegs. - -local drivespec = R("az","AZ")^1 * colon -local anchors = fwslash + drivespec -local untouched = periods + (1-period)^1 * P(-1) -local splitstarter = (Cs(drivespec * (bwslash/"/" + fwslash)^0) + Cc(false)) * Ct(lpeg.splitat(S("/\\")^1)) -local absolute = fwslash - -function file.collapsepath(str,anchor) - if not str then - return - end - if anchor and not lpegmatch(anchors,str) then - str = getcurrentdir() .. "/" .. str - end - if str == "" or str =="." then - return "." - elseif lpegmatch(untouched,str) then - return lpegmatch(reslasher,str) - end - local starter, oldelements = lpegmatch(splitstarter,str) - local newelements = { } - local i = #oldelements - while i > 0 do - local element = oldelements[i] - if element == '.' then - -- do nothing - elseif element == '..' then - local n = i - 1 - while n > 0 do - local element = oldelements[n] - if element ~= '..' and element ~= '.' then - oldelements[n] = '.' - break - else - n = n - 1 - end - end - if n < 1 then - insert(newelements,1,'..') - end - elseif element ~= "" then - insert(newelements,1,element) - end - i = i - 1 - end - if #newelements == 0 then - return starter or "." - elseif starter then - return starter .. concat(newelements, '/') - elseif lpegmatch(absolute,str) then - return "/" .. concat(newelements,'/') - else - return concat(newelements, '/') - end -end - --- local function test(str) --- print(string.format("%-20s %-15s %-15s",str,file.collapsepath(str),file.collapsepath(str,true))) --- end --- test("a/b.c/d") test("b.c/d") test("b.c/..") --- test("/") test("c:/..") test("sys://..") --- test("") test("./") test(".") test("..") test("./..") test("../..") --- test("a") test("./a") test("/a") test("a/../..") --- test("a/./b/..") test("a/aa/../b/bb") test("a/.././././b/..") test("a/./././b/..") --- test("a/b/c/../..") test("./a/b/c/../..") test("a/b/c/../..") - -local validchars = R("az","09","AZ","--","..") -local pattern_a = lpeg.replacer(1-validchars) -local pattern_a = Cs((validchars + P(1)/"-")^1) -local whatever = P("-")^0 / "" -local pattern_b = Cs(whatever * (1 - whatever * -1)^1) - -function file.robustname(str,strict) - if str then - str = lpegmatch(pattern_a,str) or str - if strict then - return lpegmatch(pattern_b,str) or str -- two step is cleaner (less backtracking) - else - return str - end - end -end - -file.readdata = io.loaddata -file.savedata = io.savedata - -function file.copy(oldname,newname) - if oldname and newname then - file.savedata(newname,io.loaddata(oldname)) - end -end - --- also rewrite previous - -local letter = R("az","AZ") + S("_-+") -local separator = P("://") - -local qualified = period^0 * fwslash - + letter * colon - + letter^1 * separator - + letter^1 * fwslash -local rootbased = fwslash - + letter * colon - -lpeg.patterns.qualified = qualified -lpeg.patterns.rootbased = rootbased - --- ./name ../name /name c: :// name/name - -function file.is_qualified_path(filename) - return filename and lpegmatch(qualified,filename) ~= nil -end - -function file.is_rootbased_path(filename) - return filename and lpegmatch(rootbased,filename) ~= nil -end - --- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end --- --- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } --- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } --- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } --- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } - --- -- maybe: --- --- if os.type == "windows" then --- local currentdir = getcurrentdir --- function getcurrentdir() --- return lpegmatch(reslasher,currentdir()) --- end --- end - --- for myself: - -function file.strip(name,dir) - if name then - local b, a = match(name,"^(.-)" .. dir .. "(.*)$") - return a ~= "" and a or name - end -end - --- local debuglist = { --- "pathpart", "basename", "nameonly", "suffixonly", "suffix", "dirname", "extname", --- "addsuffix", "removesuffix", "replacesuffix", "join", --- "strip","collapsepath", "joinpath", "splitpath", --- } - --- for i=1,#debuglist do --- local name = debuglist[i] --- local f = file[name] --- file[name] = function(...) --- print(name,f(...)) --- return f(...) --- end --- end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-md5'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This also provides file checksums and checkers. - -local md5, file = md5, file -local gsub, format, byte = string.gsub, string.format, string.byte - -local function convert(str,fmt) - return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) -end - -if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end -if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end -if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end - --- local P, Cs, lpegmatch = lpeg.P, lpeg.Cs,lpeg.match --- --- if not md5.HEX then --- local function remap(chr) return format("%02X",byte(chr)) end --- function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end --- end --- --- if not md5.hex then --- local function remap(chr) return format("%02x",byte(chr)) end --- function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end --- end --- --- if not md5.dec then --- local function remap(chr) return format("%03i",byte(chr)) end --- function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end --- end - --- if not md5.HEX then --- local pattern_HEX = Cs( ( P(1) / function(chr) return format("%02X",byte(chr)) end)^0 ) --- function md5.HEX(str) return lpegmatch(pattern_HEX,md5.sum(str)) end --- end --- --- if not md5.hex then --- local pattern_hex = Cs( ( P(1) / function(chr) return format("%02x",byte(chr)) end)^0 ) --- function md5.hex(str) return lpegmatch(pattern_hex,md5.sum(str)) end --- end --- --- if not md5.dec then --- local pattern_dec = Cs( ( P(1) / function(chr) return format("%02i",byte(chr)) end)^0 ) --- function md5.dec(str) return lpegmatch(pattern_dec,md5.sum(str)) end --- end - -function file.needsupdating(oldname,newname,threshold) -- size modification access change - local oldtime = lfs.attributes(oldname,"modification") - if oldtime then - local newtime = lfs.attributes(newname,"modification") - if not newtime then - return true -- no new file, so no updating needed - elseif newtime >= oldtime then - return false -- new file definitely needs updating - elseif oldtime - newtime < (threshold or 1) then - return false -- new file is probably still okay - else - return true -- new file has to be updated - end - else - return false -- no old file, so no updating needed - end -end - -file.needs_updating = file.needsupdating - -function file.syncmtimes(oldname,newname) - local oldtime = lfs.attributes(oldname,"modification") - if oldtime and lfs.isfile(newname) then - lfs.touch(newname,oldtime,oldtime) - end -end - -function file.checksum(name) - if md5 then - local data = io.loaddata(name) - if data then - return md5.HEX(data) - end - end - return nil -end - -function file.loadchecksum(name) - if md5 then - local data = io.loaddata(name .. ".md5") - return data and (gsub(data,"%s","")) - end - return nil -end - -function file.savechecksum(name,checksum) - if not checksum then checksum = file.checksum(name) end - if checksum then - io.savedata(name .. ".md5",checksum) - return checksum - end - return nil -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-url'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local char, format, byte = string.char, string.format, string.byte -local concat = table.concat -local tonumber, type = tonumber, type -local P, C, R, S, Cs, Cc, Ct, Cf, Cg, V = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.Cf, lpeg.Cg, lpeg.V -local lpegmatch, lpegpatterns, replacer = lpeg.match, lpeg.patterns, lpeg.replacer - --- from wikipedia: --- --- foo://username:password@example.com:8042/over/there/index.dtb?type=animal;name=narwhal#nose --- \_/ \_______________/ \_________/ \__/ \___/ \_/ \______________________/ \__/ --- | | | | | | | | --- | userinfo hostname port | | query fragment --- | \________________________________/\_____________|____|/ --- scheme | | | | --- | authority path | | --- | | | --- | path interpretable as filename --- | ___________|____________ | --- / \ / \ | --- urn:example:animal:ferret:nose interpretable as extension - -url = url or { } -local url = url - -local tochar = function(s) return char(tonumber(s,16)) end - -local colon = P(":") -local qmark = P("?") -local hash = P("#") -local slash = P("/") -local percent = P("%") -local endofstring = P(-1) - -local hexdigit = R("09","AF","af") -local plus = P("+") -local nothing = Cc("") -local escapedchar = (percent * C(hexdigit * hexdigit)) / tochar -local escaped = (plus / " ") + escapedchar - -local noslash = P("/") / "" - --- we assume schemes with more than 1 character (in order to avoid problems with windows disks) --- we also assume that when we have a scheme, we also have an authority --- --- maybe we should already split the query (better for unescaping as = & can be part of a value - -local schemestr = Cs((escaped+(1-colon-slash-qmark-hash))^2) -local authoritystr = Cs((escaped+(1- slash-qmark-hash))^0) -local pathstr = Cs((escaped+(1- qmark-hash))^0) ------ querystr = Cs((escaped+(1- hash))^0) -local querystr = Cs(( (1- hash))^0) -local fragmentstr = Cs((escaped+(1- endofstring))^0) - -local scheme = schemestr * colon + nothing -local authority = slash * slash * authoritystr + nothing -local path = slash * pathstr + nothing -local query = qmark * querystr + nothing -local fragment = hash * fragmentstr + nothing - -local validurl = scheme * authority * path * query * fragment -local parser = Ct(validurl) - -lpegpatterns.url = validurl -lpegpatterns.urlsplitter = parser - -local escapes = { } - -setmetatable(escapes, { __index = function(t,k) - local v = format("%%%02X",byte(k)) - t[k] = v - return v -end }) - -local escaper = Cs((R("09","AZ","az")^1 + P(" ")/"%%20" + S("-./_")^1 + P(1) / escapes)^0) -- space happens most -local unescaper = Cs((escapedchar + 1)^0) - -lpegpatterns.urlunescaped = escapedchar -lpegpatterns.urlescaper = escaper -lpegpatterns.urlunescaper = unescaper - --- todo: reconsider Ct as we can as well have five return values (saves a table) --- so we can have two parsers, one with and one without - -local function split(str) - return (type(str) == "string" and lpegmatch(parser,str)) or str -end - -local isscheme = schemestr * colon * slash * slash -- this test also assumes authority - -local function hasscheme(str) - if str then - local scheme = lpegmatch(isscheme,str) -- at least one character - return scheme ~= "" and scheme or false - else - return false - end -end - - --- todo: cache them - -local rootletter = R("az","AZ") - + S("_-+") -local separator = P("://") -local qualified = P(".")^0 * P("/") - + rootletter * P(":") - + rootletter^1 * separator - + rootletter^1 * P("/") -local rootbased = P("/") - + rootletter * P(":") - -local barswapper = replacer("|",":") -local backslashswapper = replacer("\\","/") - --- queries: - -local equal = P("=") -local amp = P("&") -local key = Cs(((escapedchar+1)-equal )^0) -local value = Cs(((escapedchar+1)-amp -endofstring)^0) - -local splitquery = Cf ( Ct("") * P { "sequence", - sequence = V("pair") * (amp * V("pair"))^0, - pair = Cg(key * equal * value), -}, rawset) - --- hasher - -local function hashed(str) -- not yet ok (/test?test) - if str == "" then - return { - scheme = "invalid", - original = str, - } - end - local s = split(str) - local rawscheme = s[1] - local rawquery = s[4] - local somescheme = rawscheme ~= "" - local somequery = rawquery ~= "" - if not somescheme and not somequery then - s = { - scheme = "file", - authority = "", - path = str, - query = "", - fragment = "", - original = str, - noscheme = true, - filename = str, - } - else -- not always a filename but handy anyway - local authority, path, filename = s[2], s[3] - if authority == "" then - filename = path - elseif path == "" then - filename = "" - else - filename = authority .. "/" .. path - end - s = { - scheme = rawscheme, - authority = authority, - path = path, - query = lpegmatch(unescaper,rawquery), -- unescaped, but possible conflict with & and = - queries = lpegmatch(splitquery,rawquery), -- split first and then unescaped - fragment = s[5], - original = str, - noscheme = false, - filename = filename, - } - end - return s -end - --- inspect(hashed("template://test")) - --- Here we assume: --- --- files: /// = relative --- files: //// = absolute (!) - - - -url.split = split -url.hasscheme = hasscheme -url.hashed = hashed - -function url.addscheme(str,scheme) -- no authority - if hasscheme(str) then - return str - elseif not scheme then - return "file:///" .. str - else - return scheme .. ":///" .. str - end -end - -function url.construct(hash) -- dodo: we need to escape ! - local fullurl, f = { }, 0 - local scheme, authority, path, query, fragment = hash.scheme, hash.authority, hash.path, hash.query, hash.fragment - if scheme and scheme ~= "" then - f = f + 1 ; fullurl[f] = scheme .. "://" - end - if authority and authority ~= "" then - f = f + 1 ; fullurl[f] = authority - end - if path and path ~= "" then - f = f + 1 ; fullurl[f] = "/" .. path - end - if query and query ~= "" then - f = f + 1 ; fullurl[f] = "?".. query - end - if fragment and fragment ~= "" then - f = f + 1 ; fullurl[f] = "#".. fragment - end - return lpegmatch(escaper,concat(fullurl)) -end - -local pattern = Cs(noslash * R("az","AZ") * (S(":|")/":") * noslash * P(1)^0) - -function url.filename(filename) - local spec = hashed(filename) - local path = spec.path - return (spec.scheme == "file" and path and lpegmatch(pattern,path)) or filename -end - --- print(url.filename("/c|/test")) --- print(url.filename("/c/test")) - -local function escapestring(str) - return lpegmatch(escaper,str) -end - -url.escape = escapestring - -function url.query(str) - if type(str) == "string" then - return lpegmatch(splitquery,str) or "" - else - return str - end -end - -function url.toquery(data) - local td = type(data) - if td == "string" then - return #str and escape(data) or nil -- beware of double escaping - elseif td == "table" then - if next(data) then - local t = { } - for k, v in next, data do - t[#t+1] = format("%s=%s",k,escapestring(v)) - end - return concat(t,"&") - end - else - -- nil is a signal that no query - end -end - --- /test/ | /test | test/ | test => test - -local pattern = Cs(noslash^0 * (1 - noslash * P(-1))^0) - -function url.barepath(path) - if not path or path == "" then - return "" - else - return lpegmatch(pattern,path) - end -end - --- print(url.barepath("/test"),url.barepath("test/"),url.barepath("/test/"),url.barepath("test")) --- print(url.barepath("/x/yz"),url.barepath("x/yz/"),url.barepath("/x/yz/"),url.barepath("x/yz")) - - - - - - - - - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-dir'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- dir.expandname will be merged with cleanpath and collapsepath - -local type, select = type, select -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub -local concat, insert, remove = table.concat, table.insert, table.remove -local lpegmatch = lpeg.match - -local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V - -dir = dir or { } -local dir = dir -local lfs = lfs - -local attributes = lfs.attributes -local walkdir = lfs.dir -local isdir = lfs.isdir -local isfile = lfs.isfile -local currentdir = lfs.currentdir - --- in case we load outside luatex - -if not isdir then - function isdir(name) - local a = attributes(name) - return a and a.mode == "directory" - end - lfs.isdir = isdir -end - -if not isfile then - function isfile(name) - local a = attributes(name) - return a and a.mode == "file" - end - lfs.isfile = isfile -end - --- handy - -function dir.current() - return (gsub(currentdir(),"\\","/")) -end - --- optimizing for no find (*) does not save time - - -local lfsisdir = isdir - -local function isdir(path) - path = gsub(path,"[/\\]+$","") - return lfsisdir(path) -end - -lfs.isdir = isdir - -local function globpattern(path,patt,recurse,action) - if path == "/" then - path = path .. "." - elseif not find(path,"/$") then - path = path .. '/' - end - if isdir(path) then -- lfs.isdir does not like trailing / - for name in walkdir(path) do -- lfs.dir accepts trailing / - local full = path .. name - local mode = attributes(full,'mode') - if mode == 'file' then - if find(full,patt) then - action(full) - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - globpattern(full,patt,recurse,action) - end - end - end -end - -dir.globpattern = globpattern - -local function collectpattern(path,patt,recurse,result) - local ok, scanner - result = result or { } - if path == "/" then - ok, scanner, first = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe - else - ok, scanner, first = xpcall(function() return walkdir(path) end, function() end) -- kepler safe - end - if ok and type(scanner) == "function" then - if not find(path,"/$") then path = path .. '/' end - for name in scanner, first do - local full = path .. name - local attr = attributes(full) - local mode = attr.mode - if mode == 'file' then - if find(full,patt) then - result[name] = attr - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - attr.list = collectpattern(full,patt,recurse) - result[name] = attr - end - end - end - return result -end - -dir.collectpattern = collectpattern - -local pattern = Ct { - [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), - [2] = C(((1-S("*?/"))^0 * P("/"))^0), - [3] = C(P(1)^0) -} - -local filter = Cs ( ( - P("**") / ".*" + - P("*") / "[^/]*" + - P("?") / "[^/]" + - P(".") / "%%." + - P("+") / "%%+" + - P("-") / "%%-" + - P(1) -)^0 ) - -local function glob(str,t) - if type(t) == "function" then - if type(str) == "table" then - for s=1,#str do - glob(str[s],t) - end - elseif isfile(str) then - t(str) - else - local split = lpegmatch(pattern,str) -- we could use the file splitter - if split then - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = lpegmatch(filter,start .. base) - globpattern(start,result,recurse,t) - end - end - else - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) - end - return t - elseif isfile(str) then - if t then - t[#t+1] = str - return t - else - return { str } - end - else - local split = lpegmatch(pattern,str) -- we could use the file splitter - if split then - local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = lpegmatch(filter,start .. base) - globpattern(start,result,recurse,action) - return t - else - return { } - end - end - end -end - -dir.glob = glob - - -local function globfiles(path,recurse,func,files) -- func == pattern or function - if type(func) == "string" then - local s = func - func = function(name) return find(name,s) end - end - files = files or { } - local noffiles = #files - for name in walkdir(path) do - if find(name,"^%.") then - --- skip - else - local mode = attributes(name,'mode') - if mode == "directory" then - if recurse then - globfiles(path .. "/" .. name,recurse,func,files) - end - elseif mode == "file" then - if not func or func(name) then - noffiles = noffiles + 1 - files[noffiles] = path .. "/" .. name - end - end - end - end - return files -end - -dir.globfiles = globfiles - --- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") --- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") --- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") --- t = dir.glob("f:/minimal/tex/**/*") --- print(dir.ls("f:/minimal/tex/**/*")) --- print(dir.ls("*.tex")) - -function dir.ls(pattern) - return concat(glob(pattern),"\n") -end - - -local make_indeed = true -- false - -local onwindows = os.type == "windows" or find(os.getenv("PATH"),";") - -if onwindows then - - function dir.mkdirs(...) - local str, pth = "", "" - for i=1,select("#",...) do - local s = select(i,...) - if s == "" then - -- skip - elseif str == "" then - str = s - else - str = str .. "/" .. s - end - end - local first, middle, last - local drive = false - first, middle, last = match(str,"^(//)(//*)(.*)$") - if first then - -- empty network path == local path - else - first, last = match(str,"^(//)/*(.-)$") - if first then - middle, last = match(str,"([^/]+)/+(.-)$") - if middle then - pth = "//" .. middle - else - pth = "//" .. last - last = "" - end - else - first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") - if first then - pth, drive = first .. middle, true - else - middle, last = match(str,"^(/*)(.-)$") - if not middle then - last = str - end - end - end - end - for s in gmatch(last,"[^/]+") do - if pth == "" then - pth = s - elseif drive then - pth, drive = pth .. s, false - else - pth = pth .. "/" .. s - end - if make_indeed and not isdir(pth) then - lfs.mkdir(pth) - end - end - return pth, (isdir(pth) == true) - end - - -else - - function dir.mkdirs(...) - local str, pth = "", "" - for i=1,select("#",...) do - local s = select(i,...) - if s and s ~= "" then -- we catch nil and false - if str ~= "" then - str = str .. "/" .. s - else - str = s - end - end - end - str = gsub(str,"/+","/") - if find(str,"^/") then - pth = "/" - for s in gmatch(str,"[^/]+") do - local first = (pth == "/") - if first then - pth = pth .. s - else - pth = pth .. "/" .. s - end - if make_indeed and not first and not isdir(pth) then - lfs.mkdir(pth) - end - end - else - pth = "." - for s in gmatch(str,"[^/]+") do - pth = pth .. "/" .. s - if make_indeed and not isdir(pth) then - lfs.mkdir(pth) - end - end - end - return pth, (isdir(pth) == true) - end - - -end - -dir.makedirs = dir.mkdirs - --- we can only define it here as it uses dir.current - -if onwindows then - - function dir.expandname(str) -- will be merged with cleanpath and collapsepath - local first, nothing, last = match(str,"^(//)(//*)(.*)$") - if first then - first = dir.current() .. "/" - end - if not first then - first, last = match(str,"^(//)/*(.*)$") - end - if not first then - first, last = match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d = currentdir() - if lfs.chdir(first) then - first = dir.current() - end - lfs.chdir(d) - end - end - if not first then - first, last = dir.current(), str - end - last = gsub(last,"//","/") - last = gsub(last,"/%./","/") - last = gsub(last,"^/*","") - first = gsub(first,"/*$","") - if last == "" or last == "." then - return first - else - return first .. "/" .. last - end - end - -else - - function dir.expandname(str) -- will be merged with cleanpath and collapsepath - if not find(str,"^/") then - str = currentdir() .. "/" .. str - end - str = gsub(str,"//","/") - str = gsub(str,"/%./","/") - str = gsub(str,"(.)/%.$","%1") - return str - end - -end - -file.expandname = dir.expandname -- for convenience - -local stack = { } - -function dir.push(newdir) - insert(stack,lfs.currentdir()) -end - -function dir.pop() - local d = remove(stack) - if d then - lfs.chdir(d) - end - return d -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-boolean'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local type, tonumber = type, tonumber - -boolean = boolean or { } -local boolean = boolean - -function boolean.tonumber(b) - if b then return 1 else return 0 end -- test and return or return -end - -function toboolean(str,tolerant) - if str == nil then - return false - elseif str == false then - return false - elseif str == true then - return true - elseif str == "true" then - return true - elseif str == "false" then - return false - elseif not tolerant then - return false - elseif str == 0 then - return false - elseif (tonumber(str) or 0) > 0 then - return true - else - return str == "yes" or str == "on" or str == "t" - end -end - -string.toboolean = toboolean - -function string.booleanstring(str) - if str == nil then - return false - elseif str == false then - return false - elseif str == true then - return true - elseif str == "true" then - return true - elseif str == "false" then - return false - elseif str == 0 then - return false - elseif (tonumber(str) or 0) > 0 then - return true - else - return str == "yes" or str == "on" or str == "t" - end -end - -function string.is_boolean(str,default) - if type(str) == "string" then - if str == "true" or str == "yes" or str == "on" or str == "t" then - return true - elseif str == "false" or str == "no" or str == "off" or str == "f" then - return false - end - end - return default -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-unicode'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module will be reorganized - --- todo: utf.sub replacement (used in syst-aux) - --- we put these in the utf namespace: - -utf = utf or (unicode and unicode.utf8) or { } - -utf.characters = utf.characters or string.utfcharacters -utf.values = utf.values or string.utfvalues - --- string.utfvalues --- string.utfcharacters --- string.characters --- string.characterpairs --- string.bytes --- string.bytepairs - -local type = type -local char, byte, format, sub = string.char, string.byte, string.format, string.sub -local concat = table.concat -local P, C, R, Cs, Ct, Cmt, Cc, Carg = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Carg -local lpegmatch, patterns = lpeg.match, lpeg.patterns - -local bytepairs = string.bytepairs - -local finder = lpeg.finder -local replacer = lpeg.replacer - -local utfvalues = utf.values -local utfgmatch = utf.gmatch -- not always present - -local p_utftype = patterns.utftype -local p_utfoffset = patterns.utfoffset -local p_utf8char = patterns.utf8char -local p_utf8byte = patterns.utf8byte -local p_utfbom = patterns.utfbom -local p_newline = patterns.newline -local p_whitespace = patterns.whitespace - -if not unicode then - - unicode = { utf = utf } -- for a while - -end - -if not utf.char then - - local floor, char = math.floor, string.char - - function utf.char(n) - if n < 0x80 then - -- 0aaaaaaa : 0x80 - return char(n) - elseif n < 0x800 then - -- 110bbbaa : 0xC0 : n >> 6 - -- 10aaaaaa : 0x80 : n & 0x3F - return char( - 0xC0 + floor(n/0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x10000 then - -- 1110bbbb : 0xE0 : n >> 12 - -- 10bbbbaa : 0x80 : (n >> 6) & 0x3F - -- 10aaaaaa : 0x80 : n & 0x3F - return char( - 0xE0 + floor(n/0x1000), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x200000 then - -- 11110ccc : 0xF0 : n >> 18 - -- 10ccbbbb : 0x80 : (n >> 12) & 0x3F - -- 10bbbbaa : 0x80 : (n >> 6) & 0x3F - -- 10aaaaaa : 0x80 : n & 0x3F - -- dddd : ccccc - 1 - return char( - 0xF0 + floor(n/0x40000), - 0x80 + (floor(n/0x1000) % 0x40), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - else - return "" - end - end - -end - -if not utf.byte then - - local utf8byte = patterns.utf8byte - - function utf.byte(c) - return lpegmatch(utf8byte,c) - end - -end - -local utfchar, utfbyte = utf.char, utf.byte - --- As we want to get rid of the (unmaintained) utf library we implement our own --- variants (in due time an independent module): - -function utf.filetype(data) - return data and lpegmatch(p_utftype,data) or "unknown" -end - -local toentities = Cs ( - ( - patterns.utf8one - + ( - patterns.utf8two - + patterns.utf8three - + patterns.utf8four - ) / function(s) local b = utfbyte(s) if b < 127 then return s else return format("&#%X;",b) end end - )^0 -) - -patterns.toentities = toentities - -function utf.toentities(str) - return lpegmatch(toentities,str) -end - - - - -local one = P(1) -local two = C(1) * C(1) -local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1) - --- actually one of them is already utf ... sort of useless this one - --- function utf.char(n) --- if n < 0x80 then --- return char(n) --- elseif n < 0x800 then --- return char( --- 0xC0 + floor(n/0x40), --- 0x80 + (n % 0x40) --- ) --- elseif n < 0x10000 then --- return char( --- 0xE0 + floor(n/0x1000), --- 0x80 + (floor(n/0x40) % 0x40), --- 0x80 + (n % 0x40) --- ) --- elseif n < 0x40000 then --- return char( --- 0xF0 + floor(n/0x40000), --- 0x80 + floor(n/0x1000), --- 0x80 + (floor(n/0x40) % 0x40), --- 0x80 + (n % 0x40) --- ) --- else --- -- return char( --- -- 0xF1 + floor(n/0x1000000), --- -- 0x80 + floor(n/0x40000), --- -- 0x80 + floor(n/0x1000), --- -- 0x80 + (floor(n/0x40) % 0x40), --- -- 0x80 + (n % 0x40) --- -- ) --- return "?" --- end --- end --- --- merge into: - -local pattern = P("\254\255") * Cs( ( - four / function(a,b,c,d) - local ab = 0xFF * byte(a) + byte(b) - local cd = 0xFF * byte(c) + byte(d) - return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) - end - + two / function(a,b) - return utfchar(byte(a)*256 + byte(b)) - end - + one - )^1 ) - + P("\255\254") * Cs( ( - four / function(b,a,d,c) - local ab = 0xFF * byte(a) + byte(b) - local cd = 0xFF * byte(c) + byte(d) - return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) - end - + two / function(b,a) - return utfchar(byte(a)*256 + byte(b)) - end - + one - )^1 ) - -function string.toutf(s) -- in string namespace - return lpegmatch(pattern,s) or s -- todo: utf32 -end - -local validatedutf = Cs ( - ( - patterns.utf8one - + patterns.utf8two - + patterns.utf8three - + patterns.utf8four - + P(1) / "�" - )^0 -) - -patterns.validatedutf = validatedutf - -function utf.is_valid(str) - return type(str) == "string" and lpegmatch(validatedutf,str) or false -end - -if not utf.len then - - -- -- alternative 1: 0.77 - -- - -- local utfcharcounter = utfbom^-1 * Cs((p_utf8char/'!')^0) - -- - -- function utf.len(str) - -- return #lpegmatch(utfcharcounter,str or "") - -- end - -- - -- -- alternative 2: 1.70 - -- - -- local n = 0 - -- - -- local utfcharcounter = utfbom^-1 * (p_utf8char/function() n = n + 1 end)^0 -- slow - -- - -- function utf.length(str) - -- n = 0 - -- lpegmatch(utfcharcounter,str or "") - -- return n - -- end - -- - -- -- alternative 3: 0.24 (native unicode.utf8.len: 0.047) - - -- local n = 0 - -- - -- -- local utfcharcounter = lpeg.patterns.utfbom^-1 * P ( ( Cp() * ( - -- -- patterns.utf8one ^1 * Cc(1) - -- -- + patterns.utf8two ^1 * Cc(2) - -- -- + patterns.utf8three^1 * Cc(3) - -- -- + patterns.utf8four ^1 * Cc(4) ) * Cp() / function(f,d,t) n = n + (t - f)/d end - -- -- )^0 ) -- just as many captures as below - -- - -- -- local utfcharcounter = lpeg.patterns.utfbom^-1 * P ( ( - -- -- (Cmt(patterns.utf8one ^1,function(_,_,s) n = n + #s return true end)) - -- -- + (Cmt(patterns.utf8two ^1,function(_,_,s) n = n + #s/2 return true end)) - -- -- + (Cmt(patterns.utf8three^1,function(_,_,s) n = n + #s/3 return true end)) - -- -- + (Cmt(patterns.utf8four ^1,function(_,_,s) n = n + #s/4 return true end)) - -- -- )^0 ) -- not interesting as it creates strings but sometimes faster - -- - -- -- The best so far: - -- - -- local utfcharcounter = utfbom^-1 * P ( ( - -- Cp() * (patterns.utf8one )^1 * Cp() / function(f,t) n = n + t - f end - -- + Cp() * (patterns.utf8two )^1 * Cp() / function(f,t) n = n + (t - f)/2 end - -- + Cp() * (patterns.utf8three)^1 * Cp() / function(f,t) n = n + (t - f)/3 end - -- + Cp() * (patterns.utf8four )^1 * Cp() / function(f,t) n = n + (t - f)/4 end - -- )^0 ) - - -- function utf.len(str) - -- n = 0 - -- lpegmatch(utfcharcounter,str or "") - -- return n - -- end - - local n, f = 0, 1 - - local utfcharcounter = patterns.utfbom^-1 * Cmt ( - Cc(1) * patterns.utf8one ^1 - + Cc(2) * patterns.utf8two ^1 - + Cc(3) * patterns.utf8three^1 - + Cc(4) * patterns.utf8four ^1, - function(_,t,d) -- due to Cc no string captures, so faster - n = n + (t - f)/d - f = t - return true - end - )^0 - - function utf.len(str) - n, f = 0, 1 - lpegmatch(utfcharcounter,str or "") - return n - end - -end - -utf.length = utf.len - -if not utf.sub then - - -- inefficient as lpeg just copies ^n - - -- local function sub(str,start,stop) - -- local pattern = p_utf8char^-(start-1) * C(p_utf8char^-(stop-start+1)) - -- inspect(pattern) - -- return lpegmatch(pattern,str) or "" - -- end - - -- local b, e, n, first, last = 0, 0, 0, 0, 0 - -- - -- local function slide(s,p) - -- n = n + 1 - -- if n == first then - -- b = p - -- if not last then - -- return nil - -- end - -- end - -- if n == last then - -- e = p - -- return nil - -- else - -- return p - -- end - -- end - -- - -- local pattern = Cmt(p_utf8char,slide)^0 - -- - -- function utf.sub(str,start,stop) -- todo: from the end - -- if not start then - -- return str - -- end - -- b, e, n, first, last = 0, 0, 0, start, stop - -- lpegmatch(pattern,str) - -- if not stop then - -- return sub(str,b) - -- else - -- return sub(str,b,e-1) - -- end - -- end - - -- print(utf.sub("Hans Hagen is my name")) - -- print(utf.sub("Hans Hagen is my name",5)) - -- print(utf.sub("Hans Hagen is my name",5,10)) - - local utflength = utf.length - - -- also negative indices, upto 10 times slower than a c variant - - local b, e, n, first, last = 0, 0, 0, 0, 0 - - local function slide_zero(s,p) - n = n + 1 - if n >= last then - e = p - 1 - else - return p - end - end - - local function slide_one(s,p) - n = n + 1 - if n == first then - b = p - end - if n >= last then - e = p - 1 - else - return p - end - end - - local function slide_two(s,p) - n = n + 1 - if n == first then - b = p - else - return true - end - end - - local pattern_zero = Cmt(p_utf8char,slide_zero)^0 - local pattern_one = Cmt(p_utf8char,slide_one )^0 - local pattern_two = Cmt(p_utf8char,slide_two )^0 - - function utf.sub(str,start,stop) - if not start then - return str - end - if start == 0 then - start = 1 - end - if not stop then - if start < 0 then - local l = utflength(str) -- we can inline this function if needed - start = l + start - else - start = start - 1 - end - b, n, first = 0, 0, start - lpegmatch(pattern_two,str) - if n >= first then - return sub(str,b) - else - return "" - end - end - if start < 0 or stop < 0 then - local l = utf.length(str) - if start < 0 then - start = l + start - if start <= 0 then - start = 1 - else - start = start + 1 - end - end - if stop < 0 then - stop = l + stop - if stop == 0 then - stop = 1 - else - stop = stop + 1 - end - end - end - if start > stop then - return "" - elseif start > 1 then - b, e, n, first, last = 0, 0, 0, start - 1, stop - lpegmatch(pattern_one,str) - if n >= first and e == 0 then - e = #str - end - return sub(str,b,e) - else - b, e, n, last = 1, 0, 0, stop - lpegmatch(pattern_zero,str) - if e == 0 then - e = #str - end - return sub(str,b,e) - end - end - - -- local n = 100000 - -- local str = string.rep("123456àáâãäå",100) - -- - -- for i=-15,15,1 do - -- for j=-15,15,1 do - -- if utf.xsub(str,i,j) ~= utf.sub(str,i,j) then - -- print("error",i,j,"l>"..utf.xsub(str,i,j),"s>"..utf.sub(str,i,j)) - -- end - -- end - -- if utf.xsub(str,i) ~= utf.sub(str,i) then - -- print("error",i,"l>"..utf.xsub(str,i),"s>"..utf.sub(str,i)) - -- end - -- end - - -- print(" 1, 7",utf.xsub(str, 1, 7),utf.sub(str, 1, 7)) - -- print(" 0, 7",utf.xsub(str, 0, 7),utf.sub(str, 0, 7)) - -- print(" 0, 9",utf.xsub(str, 0, 9),utf.sub(str, 0, 9)) - -- print(" 4 ",utf.xsub(str, 4 ),utf.sub(str, 4 )) - -- print(" 0 ",utf.xsub(str, 0 ),utf.sub(str, 0 )) - -- print(" 0, 0",utf.xsub(str, 0, 0),utf.sub(str, 0, 0)) - -- print(" 4, 4",utf.xsub(str, 4, 4),utf.sub(str, 4, 4)) - -- print(" 4, 0",utf.xsub(str, 4, 0),utf.sub(str, 4, 0)) - -- print("-3, 0",utf.xsub(str,-3, 0),utf.sub(str,-3, 0)) - -- print(" 0,-3",utf.xsub(str, 0,-3),utf.sub(str, 0,-3)) - -- print(" 5,-3",utf.xsub(str,-5,-3),utf.sub(str,-5,-3)) - -- print("-3 ",utf.xsub(str,-3 ),utf.sub(str,-3 )) - -end - --- a replacement for simple gsubs: - -function utf.remapper(mapping) - local pattern = Cs((p_utf8char/mapping)^0) - return function(str) - if not str or str == "" then - return "" - else - return lpegmatch(pattern,str) - end - end, pattern -end - --- local remap = utf.remapper { a = 'd', b = "c", c = "b", d = "a" } --- print(remap("abcd 1234 abcd")) - --- - -function utf.replacer(t) -- no precheck, always string builder - local r = replacer(t,false,false,true) - return function(str) - return lpegmatch(r,str) - end -end - -function utf.subtituter(t) -- with precheck and no building if no match - local f = finder (t) - local r = replacer(t,false,false,true) - return function(str) - local i = lpegmatch(f,str) - if not i then - return str - elseif i > #str then - return str - else - -- return sub(str,1,i-2) .. lpegmatch(r,str,i-1) -- slower - return lpegmatch(r,str) - end - end -end - --- inspect(utf.split("a b c d")) --- inspect(utf.split("a b c d",true)) - -local utflinesplitter = p_utfbom^-1 * lpeg.tsplitat(p_newline) -local utfcharsplitter_ows = p_utfbom^-1 * Ct(C(p_utf8char)^0) -local utfcharsplitter_iws = p_utfbom^-1 * Ct((p_whitespace^1 + C(p_utf8char))^0) -local utfcharsplitter_raw = Ct(C(p_utf8char)^0) - -patterns.utflinesplitter = utflinesplitter - -function utf.splitlines(str) - return lpegmatch(utflinesplitter,str or "") -end - -function utf.split(str,ignorewhitespace) -- new - if ignorewhitespace then - return lpegmatch(utfcharsplitter_iws,str or "") - else - return lpegmatch(utfcharsplitter_ows,str or "") - end -end - -function utf.totable(str) -- keeps bom - return lpegmatch(utfcharsplitter_raw,str) -end - --- 0 EF BB BF UTF-8 --- 1 FF FE UTF-16-little-endian --- 2 FE FF UTF-16-big-endian --- 3 FF FE 00 00 UTF-32-little-endian --- 4 00 00 FE FF UTF-32-big-endian --- --- \000 fails in <= 5.0 but is valid in >=5.1 where %z is depricated - --- utf.name = { --- [0] = 'utf-8', --- [1] = 'utf-16-le', --- [2] = 'utf-16-be', --- [3] = 'utf-32-le', --- [4] = 'utf-32-be' --- } --- --- function utf.magic(f) --- local str = f:read(4) --- if not str then --- f:seek('set') --- return 0 --- -- elseif find(str,"^%z%z\254\255") then -- depricated --- -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged --- elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) --- return 4 --- -- elseif find(str,"^\255\254%z%z") then -- depricated --- -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged --- elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) --- return 3 --- elseif find(str,"^\254\255") then --- f:seek('set',2) --- return 2 --- elseif find(str,"^\255\254") then --- f:seek('set',2) --- return 1 --- elseif find(str,"^\239\187\191") then --- f:seek('set',3) --- return 0 --- else --- f:seek('set') --- return 0 --- end --- end - -function utf.magic(f) -- not used - local str = f:read(4) or "" - local off = lpegmatch(p_utfoffset,str) - if off < 4 then - f:seek('set',off) - end - return lpegmatch(p_utftype,str) -end - -local function utf16_to_utf8_be(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, 0 - for left, right in bytepairs(t[i]) do - if right then - local now = 256*left + right - if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong - more = 0 - r = r + 1 - result[r] = utfchar(now) - elseif now >= 0xD800 and now <= 0xDBFF then - more = now - else - r = r + 1 - result[r] = utfchar(now) - end - end - end - t[i] = concat(result,"",1,r) -- we reused tmp, hence t - end - return t -end - -local function utf16_to_utf8_le(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, 0 - for left, right in bytepairs(t[i]) do - if right then - local now = 256*right + left - if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong - more = 0 - r = r + 1 - result[r] = utfchar(now) - elseif now >= 0xD800 and now <= 0xDBFF then - more = now - else - r = r + 1 - result[r] = utfchar(now) - end - end - end - t[i] = concat(result,"",1,r) -- we reused tmp, hence t - end - return t -end - -local function utf32_to_utf8_be(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, -1 - for a,b in bytepairs(t[i]) do - if a and b then - if more < 0 then - more = 256*256*256*a + 256*256*b - else - r = r + 1 - result[t] = utfchar(more + 256*a + b) - more = -1 - end - else - break - end - end - t[i] = concat(result,"",1,r) - end - return t -end - -local function utf32_to_utf8_le(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, -1 - for a,b in bytepairs(t[i]) do - if a and b then - if more < 0 then - more = 256*b + a - else - r = r + 1 - result[t] = utfchar(more + 256*256*256*b + 256*256*a) - more = -1 - end - else - break - end - end - t[i] = concat(result,"",1,r) - end - return t -end - -utf.utf32_to_utf8_be = utf32_to_utf8_be -utf.utf32_to_utf8_le = utf32_to_utf8_le -utf.utf16_to_utf8_be = utf16_to_utf8_be -utf.utf16_to_utf8_le = utf16_to_utf8_le - -function utf.utf8_to_utf8(t) - return type(t) == "string" and lpegmatch(utflinesplitter,t) or t -end - -function utf.utf16_to_utf8(t,endian) - return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t -end - -function utf.utf32_to_utf8(t,endian) - return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t -end - -local function little(c) - local b = byte(c) - if b < 0x10000 then - return char(b%256,b/256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1%256,b1/256,b2%256,b2/256) - end -end - -local function big(c) - local b = byte(c) - if b < 0x10000 then - return char(b/256,b%256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1/256,b1%256,b2/256,b2%256) - end -end - --- function utf.utf8_to_utf16(str,littleendian) --- if littleendian then --- return char(255,254) .. utfgsub(str,".",little) --- else --- return char(254,255) .. utfgsub(str,".",big) --- end --- end - -local _, l_remap = utf.remapper(little) -local _, b_remap = utf.remapper(big) - -function utf.utf8_to_utf16(str,littleendian) - if littleendian then - return char(255,254) .. lpegmatch(l_remap,str) - else - return char(254,255) .. lpegmatch(b_remap,str) - end -end - --- function utf.tocodes(str,separator) -- can be sped up with an lpeg --- local t, n = { }, 0 --- for u in utfvalues(str) do --- n = n + 1 --- t[n] = format("0x%04X",u) --- end --- return concat(t,separator or " ") --- end - -local pattern = Cs ( - (p_utf8byte / function(unicode ) return format( "0x%04X", unicode) end) * - (p_utf8byte * Carg(1) / function(unicode,separator) return format("%s0x%04X",separator,unicode) end)^0 -) - -function utf.tocodes(str,separator) - return lpegmatch(pattern,str,1,separator or " ") -end - -function utf.ustring(s) - return format("U+%05X",type(s) == "number" and s or utfbyte(s)) -end - -function utf.xstring(s) - return format("0x%05X",type(s) == "number" and s or utfbyte(s)) -end - --- - -local p_nany = p_utf8char / "" - -if utfgmatch then - - function utf.count(str,what) - if type(what) == "string" then - local n = 0 - for _ in utfgmatch(str,what) do - n = n + 1 - end - return n - else -- 4 times slower but still faster than / function - return #lpegmatch(Cs((P(what)/" " + p_nany)^0),str) - end - end - -else - - local cache = { } - - function utf.count(str,what) - if type(what) == "string" then - local p = cache[what] - if not p then - p = Cs((P(what)/" " + p_nany)^0) - cache[p] = p - end - return #lpegmatch(p,str) - else -- 4 times slower but still faster than / function - return #lpegmatch(Cs((P(what)/" " + p_nany)^0),str) - end - end - -end - --- maybe also register as string.utf* - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-math'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan - -if not math.round then - function math.round(x) return floor(x + 0.5) end -end - -if not math.div then - function math.div(n,m) return floor(n/m) end -end - -if not math.mod then - function math.mod(n,m) return n % m end -end - -local pipi = 2*math.pi/360 - -if not math.sind then - function math.sind(d) return sin(d*pipi) end - function math.cosd(d) return cos(d*pipi) end - function math.tand(d) return tan(d*pipi) end -end - -if not math.odd then - function math.odd (n) return n % 2 ~= 0 end - function math.even(n) return n % 2 == 0 end -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-tab'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or {} -utilities.tables = utilities.tables or { } -local tables = utilities.tables - -local format, gmatch, rep, gsub = string.format, string.gmatch, string.rep, string.gsub -local concat, insert, remove = table.concat, table.insert, table.remove -local setmetatable, getmetatable, tonumber, tostring = setmetatable, getmetatable, tonumber, tostring -local type, next, rawset, tonumber, load, select = type, next, rawset, tonumber, load, select -local lpegmatch, P, Cs = lpeg.match, lpeg.P, lpeg.Cs -local serialize = table.serialize - -local splitter = lpeg.tsplitat(".") - -function tables.definetable(target,nofirst,nolast) -- defines undefined tables - local composed, shortcut, t = nil, nil, { } - local snippets = lpegmatch(splitter,target) - for i=1,#snippets - (nolast and 1 or 0) do - local name = snippets[i] - if composed then - composed = shortcut .. "." .. name - shortcut = shortcut .. "_" .. name - t[#t+1] = format("local %s = %s if not %s then %s = { } %s = %s end",shortcut,composed,shortcut,shortcut,composed,shortcut) - else - composed = name - shortcut = name - if not nofirst then - t[#t+1] = format("%s = %s or { }",composed,composed) - end - end - end - if nolast then - composed = shortcut .. "." .. snippets[#snippets] - end - return concat(t,"\n"), composed -end - --- local t = tables.definedtable("a","b","c","d") - -function tables.definedtable(...) - local t = _G - for i=1,select("#",...) do - local li = select(i,...) - local tl = t[li] - if not tl then - tl = { } - t[li] = tl - end - t = tl - end - return t -end - -function tables.accesstable(target,root) - local t = root or _G - for name in gmatch(target,"([^%.]+)") do - t = t[name] - if not t then - return - end - end - return t -end - -function tables.migratetable(target,v,root) - local t = root or _G - local names = string.split(target,".") - for i=1,#names-1 do - local name = names[i] - t[name] = t[name] or { } - t = t[name] - if not t then - return - end - end - t[names[#names]] = v -end - -function tables.removevalue(t,value) -- todo: n - if value then - for i=1,#t do - if t[i] == value then - remove(t,i) - -- remove all, so no: return - end - end - end -end - -function tables.insertbeforevalue(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i,extra) - return - end - end - insert(t,1,extra) -end - -function tables.insertaftervalue(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i+1,extra) - return - end - end - insert(t,#t+1,extra) -end - --- experimental - -local function toxml(t,d,result,step) - for k, v in table.sortedpairs(t) do - if type(v) == "table" then - if type(k) == "number" then - result[#result+1] = format("%s",d,k) - toxml(v,d..step,result,step) - result[#result+1] = format("%s",d,k) - else - result[#result+1] = format("%s<%s>",d,k) - toxml(v,d..step,result,step) - result[#result+1] = format("%s",d,k) - end - elseif type(k) == "number" then - result[#result+1] = format("%s%s",d,k,v,k) - else - result[#result+1] = format("%s<%s>%s",d,k,tostring(v),k) - end - end -end - -function table.toxml(t,name,nobanner,indent,spaces) - local noroot = name == false - local result = (nobanner or noroot) and { } or { "" } - local indent = rep(" ",indent or 0) - local spaces = rep(" ",spaces or 1) - if noroot then - toxml( t, inndent, result, spaces) - else - toxml( { [name or "root"] = t }, indent, result, spaces) - end - return concat(result,"\n") -end - --- also experimental - --- encapsulate(table,utilities.tables) --- encapsulate(table,utilities.tables,true) --- encapsulate(table,true) - -function tables.encapsulate(core,capsule,protect) - if type(capsule) ~= "table" then - protect = true - capsule = { } - end - for key, value in next, core do - if capsule[key] then - print(format("\ninvalid inheritance '%s' in '%s': %s",key,tostring(core))) - os.exit() - else - capsule[key] = value - end - end - if protect then - for key, value in next, core do - core[key] = nil - end - setmetatable(core, { - __index = capsule, - __newindex = function(t,key,value) - if capsule[key] then - print(format("\ninvalid overload '%s' in '%s'",key,tostring(core))) - os.exit() - else - rawset(t,key,value) - end - end - } ) - end -end - -local function fastserialize(t,r,outer) -- no mixes - r[#r+1] = "{" - local n = #t - if n > 0 then - for i=1,n do - local v = t[i] - local tv = type(v) - if tv == "string" then - r[#r+1] = format("%q,",v) - elseif tv == "number" then - r[#r+1] = format("%s,",v) - elseif tv == "table" then - fastserialize(v,r) - elseif tv == "boolean" then - r[#r+1] = format("%s,",tostring(v)) - end - end - else - for k, v in next, t do - local tv = type(v) - if tv == "string" then - r[#r+1] = format("[%q]=%q,",k,v) - elseif tv == "number" then - r[#r+1] = format("[%q]=%s,",k,v) - elseif tv == "table" then - r[#r+1] = format("[%q]=",k) - fastserialize(v,r) - elseif tv == "boolean" then - r[#r+1] = format("[%q]=%s,",k,tostring(v)) - end - end - end - if outer then - r[#r+1] = "}" - else - r[#r+1] = "}," - end - return r -end - --- local f_hashed_string = formatters["[%q]=%q,"] --- local f_hashed_number = formatters["[%q]=%s,"] --- local f_hashed_table = formatters["[%q]="] --- local f_hashed_true = formatters["[%q]=true,"] --- local f_hashed_false = formatters["[%q]=false,"] --- --- local f_indexed_string = formatters["%q,"] --- local f_indexed_number = formatters["%s,"] --- ----- f_indexed_true = formatters["true,"] --- ----- f_indexed_false = formatters["false,"] --- --- local function fastserialize(t,r,outer) -- no mixes --- r[#r+1] = "{" --- local n = #t --- if n > 0 then --- for i=1,n do --- local v = t[i] --- local tv = type(v) --- if tv == "string" then --- r[#r+1] = f_indexed_string(v) --- elseif tv == "number" then --- r[#r+1] = f_indexed_number(v) --- elseif tv == "table" then --- fastserialize(v,r) --- elseif tv == "boolean" then --- -- r[#r+1] = v and f_indexed_true(k) or f_indexed_false(k) --- r[#r+1] = v and "true," or "false," --- end --- end --- else --- for k, v in next, t do --- local tv = type(v) --- if tv == "string" then --- r[#r+1] = f_hashed_string(k,v) --- elseif tv == "number" then --- r[#r+1] = f_hashed_number(k,v) --- elseif tv == "table" then --- r[#r+1] = f_hashed_table(k) --- fastserialize(v,r) --- elseif tv == "boolean" then --- r[#r+1] = v and f_hashed_true(k) or f_hashed_false(k) --- end --- end --- end --- if outer then --- r[#r+1] = "}" --- else --- r[#r+1] = "}," --- end --- return r --- end - -function table.fastserialize(t,prefix) -- so prefix should contain the = - return concat(fastserialize(t,{ prefix or "return" },true)) -end - -function table.deserialize(str) - if not str or str == "" then - return - end - local code = load(str) - if not code then - return - end - code = code() - if not code then - return - end - return code -end - --- inspect(table.fastserialize { a = 1, b = { 4, { 5, 6 } }, c = { d = 7, e = 'f"g\nh' } }) - -function table.load(filename) - if filename then - local t = io.loaddata(filename) - if t and t ~= "" then - t = load(t) - if type(t) == "function" then - t = t() - if type(t) == "table" then - return t - end - end - end - end -end - -function table.save(filename,t,n,...) - io.savedata(filename,serialize(t,n == nil and true or n,...)) -end - -local function slowdrop(t) - local r = { } - local l = { } - for i=1,#t do - local ti = t[i] - local j = 0 - for k, v in next, ti do - j = j + 1 - l[j] = format("%s=%q",k,v) - end - r[i] = format(" {%s},\n",concat(l)) - end - return format("return {\n%s}",concat(r)) -end - -local function fastdrop(t) - local r = { "return {\n" } - for i=1,#t do - local ti = t[i] - r[#r+1] = " {" - for k, v in next, ti do - r[#r+1] = format("%s=%q",k,v) - end - r[#r+1] = "},\n" - end - r[#r+1] = "}" - return concat(r) -end - -function table.drop(t,slow) - if #t == 0 then - return "return { }" - elseif slow == true then - return slowdrop(t) -- less memory - else - return fastdrop(t) -- some 15% faster - end -end - -function table.autokey(t,k) - local v = { } - t[k] = v - return v -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-sto'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local setmetatable, getmetatable = setmetatable, getmetatable - -utilities = utilities or { } -utilities.storage = utilities.storage or { } -local storage = utilities.storage - -local report = texio and texio.write_nl or print - -function storage.mark(t) - if not t then - report("fatal error: storage cannot be marked") - return -- os.exit() - end - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) - end - m.__storage__ = true - return t -end - -function storage.allocate(t) - t = t or { } - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) - end - m.__storage__ = true - return t -end - -function storage.marked(t) - local m = getmetatable(t) - return m and m.__storage__ -end - -function storage.checked(t) - if not t then - report("fatal error: storage has not been allocated") - return -- os.exit() - end - return t -end - --- function utilities.storage.delay(parent,name,filename) --- local m = getmetatable(parent) --- m.__list[name] = filename --- end --- --- function utilities.storage.predefine(parent) --- local list = { } --- local m = getmetatable(parent) or { --- __list = list, --- __index = function(t,k) --- local l = require(list[k]) --- t[k] = l --- return l --- end --- } --- setmetatable(parent,m) --- end --- --- bla = { } --- utilities.storage.predefine(bla) --- utilities.storage.delay(bla,"test","oepsoeps") --- local t = bla.test --- table.print(t) --- print(t.a) - -function storage.setinitializer(data,initialize) - local m = getmetatable(data) or { } - m.__index = function(data,k) - m.__index = nil -- so that we can access the entries during initializing - initialize() - return data[k] - end - setmetatable(data, m) -end - -local keyisvalue = { __index = function(t,k) - t[k] = k - return k -end } - -function storage.sparse(t) - t = t or { } - setmetatable(t,keyisvalue) - return t -end - --- table namespace ? - -local function f_empty () return "" end -- t,k -local function f_self (t,k) t[k] = k return k end -local function f_table (t,k) local v = { } t[k] = v return v end -local function f_ignore() end -- t,k,v - -local t_empty = { __index = f_empty } -local t_self = { __index = f_self } -local t_table = { __index = f_table } -local t_ignore = { __newindex = f_ignore } - -function table.setmetatableindex(t,f) - local m = getmetatable(t) - if m then - if f == "empty" then - m.__index = f_empty - elseif f == "key" then - m.__index = f_self - elseif f == "table" then - m.__index = f_table - else - m.__index = f - end - else - if f == "empty" then - setmetatable(t, t_empty) - elseif f == "key" then - setmetatable(t, t_self) - elseif f == "table" then - setmetatable(t, t_table) - else - setmetatable(t,{ __index = f }) - end - end - return t -end - -function table.setmetatablenewindex(t,f) - local m = getmetatable(t) - if m then - if f == "ignore" then - m.__newindex = f_ignore - else - m.__newindex = f - end - else - if f == "ignore" then - setmetatable(t, t_ignore) - else - setmetatable(t,{ __newindex = f }) - end - end - return t -end - -function table.setmetatablecall(t,f) - local m = getmetatable(t) - if m then - m.__call = f - else - setmetatable(t,{ __call = f }) - end - return t -end - -function table.setmetatablekey(t,key,value) - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) - end - m[key] = value - return t -end - -function table.getmetatablekey(t,key,value) - local m = getmetatable(t) - return m and m[key] -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-str'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or {} -utilities.strings = utilities.strings or { } -local strings = utilities.strings - -local load = load -local format, gsub, rep, sub = string.format, string.gsub, string.rep, string.sub -local concat = table.concat -local P, V, C, S, R, Ct, Cs, Cp, Carg = lpeg.P, lpeg.V, lpeg.C, lpeg.S, lpeg.R, lpeg.Ct, lpeg.Cs, lpeg.Cp, lpeg.Carg -local patterns, lpegmatch = lpeg.patterns, lpeg.match -local utfchar, utfbyte = utf.char, utf.byte -local setmetatableindex = table.setmetatableindex --- - -local stripper = patterns.stripzeros - -local function points(n) - return (not n or n == 0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) -end - -local function basepoints(n) - return (not n or n == 0) and "0bp" or lpegmatch(stripper,format("%.5fbp", n*(7200/7227)/65536)) -end - -number.points = points -number.basepoints = basepoints - --- str = " \n \ntest \n test\ntest " --- print("["..string.gsub(string.collapsecrlf(str),"\n","+").."]") - -local rubish = patterns.spaceortab^0 * patterns.newline -local anyrubish = patterns.spaceortab + patterns.newline -local anything = patterns.anything -local stripped = (patterns.spaceortab^1 / "") * patterns.newline -local leading = rubish^0 / "" -local trailing = (anyrubish^1 * patterns.endofstring) / "" -local redundant = rubish^3 / "\n" - -local pattern = Cs(leading * (trailing + redundant + stripped + anything)^0) - -function strings.collapsecrlf(str) - return lpegmatch(pattern,str) -end - --- The following functions might end up in another namespace. - -local repeaters = { } -- watch how we also moved the -1 in depth-1 to the creator - -function strings.newrepeater(str,offset) - offset = offset or 0 - local s = repeaters[str] - if not s then - s = { } - repeaters[str] = s - end - local t = s[offset] - if t then - return t - end - t = { } - setmetatableindex(t, function(t,k) - if not k then - return "" - end - local n = k + offset - local s = n > 0 and rep(str,n) or "" - t[k] = s - return s - end) - s[offset] = t - return t -end - --- local dashes = strings.newrepeater("--",-1) --- print(dashes[2],dashes[3],dashes[1]) - -local extra, tab, start = 0, 0, 4, 0 - -local nspaces = strings.newrepeater(" ") - -local pattern = - Carg(1) / function(t) - extra, tab, start = 0, t or 7, 1 - end - * Cs(( - Cp() * patterns.tab / function(position) - local current = (position - start + 1) + extra - local spaces = tab-(current-1) % tab - if spaces > 0 then - extra = extra + spaces - 1 - return nspaces[spaces] -- rep(" ",spaces) - else - return "" - end - end - + patterns.newline * Cp() / function(position) - extra, start = 0, position - end - + patterns.anything - )^1) - -function strings.tabtospace(str,tab) - return lpegmatch(pattern,str,1,tab or 7) -end - --- local t = { --- "1234567123456712345671234567", --- "\tb\tc", --- "a\tb\tc", --- "aa\tbb\tcc", --- "aaa\tbbb\tccc", --- "aaaa\tbbbb\tcccc", --- "aaaaa\tbbbbb\tccccc", --- "aaaaaa\tbbbbbb\tcccccc\n aaaaaa\tbbbbbb\tcccccc", --- "one\n two\nxxx three\nxx four\nx five\nsix", --- } --- for k=1,#t do --- print(strings.tabtospace(t[k])) --- end - -function strings.striplong(str) -- strips all leading spaces - str = gsub(str,"^%s*","") - str = gsub(str,"[\n\r]+ *","\n") - return str -end - --- local template = string.striplong([[ --- aaaa --- bb --- cccccc --- ]]) - -function strings.nice(str) - str = gsub(str,"[:%-+_]+"," ") -- maybe more - return str -end - --- Work in progress. Interesting is that compared to the built-in this --- is faster in luatex than in luajittex where we have a comparable speed. - -local n = 0 - --- we are somewhat sloppy in parsing prefixes as it's not that critical --- --- this does not work out ok: --- --- function fnc(...) -- 1,2,3 --- print(...,...,...) -- 1,1,1,2,3 --- end - -local prefix_any = C((S("+- .") + R("09"))^0) -local prefix_tab = C((1-R("az","AZ","09","%%"))^0) - --- we've split all cases as then we can optimize them (let's omit the fuzzy u) - -local format_s = function(f) - n = n + 1 - if f and f ~= "" then - return format("format('%%%ss',(select(%s,...)))",f,n) - else - return format("(select(%s,...))",n) - end -end - -local format_q = function() - n = n + 1 - return format("format('%%q',(select(%s,...)))",n) -- maybe an own lpeg -end - -local format_i = function(f) - n = n + 1 - if f and f ~= "" then - return format("format('%%%si',(select(%s,...)))",f,n) - else - return format("(select(%s,...))",n) - end -end - -local format_d = format_i - -local format_f = function(f) - n = n + 1 - return format("format('%%%sf',(select(%s,...)))",f,n) -end - -local format_g = function(f) - n = n + 1 - return format("format('%%%sg',(select(%s,...)))",f,n) -end - -local format_G = function(f) - n = n + 1 - return format("format('%%%sG',(select(%s,...)))",f,n) -end - -local format_e = function(f) - n = n + 1 - return format("format('%%%se',(select(%s,...)))",f,n) -end - -local format_E = function(f) - n = n + 1 - return format("format('%%%sE',(select(%s,...)))",f,n) -end - -local format_x = function(f) - n = n + 1 - return format("format('%%%sx',(select(%s,...)))",f,n) -end - -local format_X = function(f) - n = n + 1 - return format("format('%%%sX',(select(%s,...)))",f,n) -end - -local format_o = function(f) - n = n + 1 - return format("format('%%%so',(select(%s,...)))",f,n) -end - -local format_c = function() - n = n + 1 - return format("utfchar((select(%s,...)))",n) -end - -local format_r = function(f) - n = n + 1 - return format("format('%%%s.0f',(select(%s,...)))",f,n) -end - -local format_v = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +function table.serialize(root,name,specification) + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + end + serialize(flush,root,name,specification) + return concat(t,"\n") +end +table.tohandle=serialize +local maxtab=2*1024 +function table.tofile(filename,root,name,specification) + local f=io.open(filename,'w') + if f then + if maxtab>1 then + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + if n>maxtab then + f:write(concat(t,"\n"),"\n") + t,n={},0 + end + end + serialize(flush,root,name,specification) + f:write(concat(t,"\n"),"\n") else - return format("format('0x%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) + local function flush(s) + f:write(s,"\n") + end + serialize(flush,root,name,specification) end + f:close() + io.flush() + end end - -local format_V = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +local function flattened(t,f,depth) + if f==nil then + f={} + depth=0xFFFF + elseif tonumber(f) then + depth=f + f={} + elseif not depth then + depth=0xFFFF + end + for k,v in next,t do + if type(k)~="number" then + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + else + f[k]=v + end + end + end + local n=#f + for k=1,#t do + local v=t[k] + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + n=#f + else + n=n+1 + f[n]=v + end + end + return f +end +table.flattened=flattened +local function unnest(t,f) + if not f then + f={} + end + for i=1,#t do + local v=t[i] + if type(v)=="table" then + if type(v[1])=="table" then + unnest(v,f) + else + f[#f+1]=v + end else - return format("format('0x%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) + f[#f+1]=v end + end + return f end - -local format_u = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +function table.unnest(t) + return unnest(t) +end +local function are_equal(a,b,n,m) + if a and b and #a==#b then + n=n or 1 + m=m or #a + for i=n,m do + local ai,bi=a[i],b[i] + if ai==bi then + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end + else + return false + end + end + return true + else + return false + end +end +local function identical(a,b) + for ka,va in next,a do + local vb=b[ka] + if va==vb then + elseif type(va)=="table" and type(vb)=="table" then + if not identical(va,vb) then + return false + end else - return format("format('u+%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) + return false end + end + return true end - -local format_U = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) - else - return format("format('U+%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +table.identical=identical +table.are_equal=are_equal +function table.compact(t) + if t then + for k,v in next,t do + if not next(v) then + t[k]=nil + end end + end end - -local format_p = function() - n = n + 1 - return format("points((select(%s,...)))",n) +function table.contains(t,v) + if t then + for i=1,#t do + if t[i]==v then + return i + end + end + end + return false end - -local format_b = function() - n = n + 1 - return format("basepoints((select(%s,...)))",n) +function table.count(t) + local n=0 + for k,v in next,t do + n=n+1 + end + return n +end +function table.swapped(t,s) + local n={} + if s then + for k,v in next,s do + n[k]=v + end + end + for k,v in next,t do + n[v]=k + end + return n +end +function table.mirrored(t) + local n={} + for k,v in next,t do + n[v]=k + n[k]=v + end + return n end - -local format_t = function(f) - n = n + 1 - if f and f ~= "" then - return format("concat((select(%s,...)),%q)",n,f) - else - return format("concat((select(%s,...)))",n) +function table.reversed(t) + if t then + local tt,tn={},#t + if tn>0 then + local ttn=0 + for i=tn,1,-1 do + ttn=ttn+1 + tt[ttn]=t[i] + end end + return tt + end end - -local format_l = function() - n = n + 1 - return format("(select(%s,...) and 'true' or 'false')",n) +function table.reverse(t) + if t then + local n=#t + for i=1,floor(n/2) do + local j=n-i+1 + t[i],t[j]=t[j],t[i] + end + return t + end +end +function table.sequenced(t,sep) + if t then + local s,n={},0 + for k,v in sortedhash(t) do + if simple then + if v==true then + n=n+1 + s[n]=k + elseif v and v~="" then + n=n+1 + s[n]=k.."="..tostring(v) + end + else + n=n+1 + s[n]=k.."="..tostring(v) + end + end + return concat(s,sep or " | ") + else + return "" + end end - -local format_a = function(s) - return format("%q",s) +function table.print(t,...) + if type(t)~="table" then + print(tostring(t)) + else + table.tohandle(print,t,...) + end end - -local builder = Ct { "start", - start = (P("%") * ( - V("s") + V("q") - + V("i") + V("d") - + V("f") + V("g") + V("G") + V("e") + V("E") - + V("x") + V("X") + V("o") - -- - + V("c") - -- - + V("r") - + V("v") + V("V") + V("u") + V("U") - + V("p") + V("b") - + V("t") - + V("l") - ) - + V("a") - )^0, - -- - ["s"] = (prefix_any * P("s")) / format_s, -- %s => regular %s (string) - ["q"] = (prefix_any * P("q")) / format_q, -- %q => regular %q (quoted string) - ["i"] = (prefix_any * P("i")) / format_i, -- %i => regular %i (integer) - ["d"] = (prefix_any * P("d")) / format_d, -- %d => regular %d (integer) - ["f"] = (prefix_any * P("f")) / format_f, -- %f => regular %f (float) - ["g"] = (prefix_any * P("g")) / format_g, -- %g => regular %g (float) - ["G"] = (prefix_any * P("G")) / format_G, -- %G => regular %G (float) - ["e"] = (prefix_any * P("e")) / format_e, -- %e => regular %e (float) - ["E"] = (prefix_any * P("E")) / format_E, -- %E => regular %E (float) - ["x"] = (prefix_any * P("x")) / format_x, -- %x => regular %x (hexadecimal) - ["X"] = (prefix_any * P("X")) / format_X, -- %X => regular %X (HEXADECIMAL) - ["o"] = (prefix_any * P("o")) / format_o, -- %o => regular %o (octal) - -- - ["c"] = (prefix_any * P("c")) / format_c, -- %c => utf character (extension to regular) - -- - ["r"] = (prefix_any * P("r")) / format_r, -- %r => round - ["v"] = (prefix_any * P("v")) / format_v, -- %v => 0x0a1b2 (when - no 0x) - ["V"] = (prefix_any * P("V")) / format_V, -- %V => 0x0A1B2 (when - no 0x) - ["u"] = (prefix_any * P("u")) / format_u, -- %u => u+0a1b2 (when - no u+) - ["U"] = (prefix_any * P("U")) / format_U, -- %U => U+0A1B2 (when - no U+) - ["p"] = (prefix_any * P("p")) / format_p, -- %p => 12.345pt / maybe: P (and more units) - ["b"] = (prefix_any * P("b")) / format_b, -- %b => 12.342bp / maybe: B (and more units) - ["t"] = (prefix_tab * P("t")) / format_t, -- %t => concat - ["l"] = (prefix_tab * P("l")) / format_l, -- %l => boolean - -- - ["a"] = Cs(((1-P("%"))^1 + P("%%")/"%%")^1) / format_a, -- %a => text (including %%) -} - --- we can be clever and only alias what is needed - -local template = [[ -local format = string.format -local concat = table.concat -local points = number.points -local basepoints = number.basepoints -local utfchar = utf.char -local utfbyte = utf.byte -return function(...) - return %s +function table.sub(t,i,j) + return { unpack(t,i,j) } end -]] - -local function make(t,str) - n = 0 - local p = lpegmatch(builder,str) --- inspect(p) - local c = format(template,concat(p,"..")) --- inspect(c) - formatter = load(c)() - t[str] = formatter - return formatter +function table.is_empty(t) + return not t or not next(t) end - -local formatters = string.formatters or { } -string.formatters = formatters - -setmetatableindex(formatters,make) - -function string.makeformatter(str) - return formatters[str] +function table.has_one_entry(t) + return t and not next(t,next(t)) end - -function string.formatter(str,...) - return formatters[str](...) +function table.loweredkeys(t) + local l={} + for k,v in next,t do + l[lower(k)]=v + end + return l +end +function table.unique(old) + local hash={} + local new={} + local n=0 + for i=1,#old do + local oi=old[i] + if not hash[oi] then + n=n+1 + new[n]=oi + hash[oi]=true + end + end + return new +end +function table.sorted(t,...) + sort(t,...) + return t end - --- local p1 = "%s test %f done %p and %c and %V or %+t or %%" --- local p2 = "%s test %f done %s and %s and 0x%05X or %s or %%" --- --- local t = { 1,2,3,4 } --- local r = "" --- --- local format, formatter, formatters = string.format, string.formatter, string.formatters --- local utfchar, utfbyte, concat, points = utf.char, utf.byte, table.concat, number.points --- --- local c = os.clock() --- local f = formatters[p1] --- for i=1,500000 do --- -- r = formatters[p1]("hans",123.45,123.45,123,"a",t) --- r = formatter(p1,"hans",123.45,123.45,123,"a",t) --- -- r = f("hans",123.45,123.45,123,"a",t) --- end --- print(os.clock()-c,r) --- --- local c = os.clock() --- for i=1,500000 do --- r = format(p2,"hans",123.45,points(123.45),utfchar(123),utfbyte("a"),concat(t,"+")) --- end --- print(os.clock()-c,r) - --- local f = format --- function string.format(fmt,...) --- print(fmt,...) --- return f(fmt,...) --- end end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-mrg'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- hm, quite unreadable - -local gsub, format = string.gsub, string.format -local concat = table.concat -local type, next = type, next +-- original size: 8723, stripped down to: 6325 -utilities = utilities or {} -local merger = utilities.merger or { } -utilities.merger = merger -utilities.report = logs and logs.reporter("system") or print - -merger.strip_comment = true - -local m_begin_merge = "begin library merge" -local m_end_merge = "end library merge" -local m_begin_closure = "do -- create closure to overcome 200 locals limit" -local m_end_closure = "end -- of closure" - -local m_pattern = - "%c+" .. - "%-%-%s+" .. m_begin_merge .. - "%c+(.-)%c+" .. - "%-%-%s+" .. m_end_merge .. - "%c+" - -local m_format = - "\n\n-- " .. m_begin_merge .. - "\n%s\n" .. - "-- " .. m_end_merge .. "\n\n" - -local m_faked = - "-- " .. "created merged file" .. "\n\n" .. - "-- " .. m_begin_merge .. "\n\n" .. - "-- " .. m_end_merge .. "\n\n" - -local function self_fake() - return m_faked +if not modules then modules={} end modules ['l-io']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local io=io +local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format +local concat=table.concat +local floor=math.floor +local type=type +if string.find(os.getenv("PATH"),";") then + io.fileseparator,io.pathseparator="\\",";" +else + io.fileseparator,io.pathseparator="/",":" end - -local function self_nothing() +local function readall(f) + return f:read("*all") +end +local function readall(f) + local size=f:seek("end") + if size==0 then return "" + elseif size<1024*1024 then + f:seek("set",0) + return f:read('*all') + else + local done=f:seek("set",0) + if size<1024*1024 then + step=1024*1024 + elseif size>16*1024*1024 then + step=16*1024*1024 + else + step=floor(size/(1024*1024))*1024*1024/8 + end + local data={} + while true do + local r=f:read(step) + if not r then + return concat(data) + else + data[#data+1]=r + end + end + end end - -local function self_load(name) - local data = io.loaddata(name) or "" - if data == "" then - utilities.report("merge: unknown file %s",name) +io.readall=readall +function io.loaddata(filename,textmode) + local f=io.open(filename,(textmode and 'r') or 'rb') + if f then + local data=readall(f) + f:close() + if #data>0 then + return data + end + end +end +function io.savedata(filename,data,joiner) + local f=io.open(filename,"wb") + if f then + if type(data)=="table" then + f:write(concat(data,joiner or "")) + elseif type(data)=="function" then + data(f) else - utilities.report("merge: inserting %s",name) + f:write(data or "") + end + f:close() + io.flush() + return true + else + return false + end +end +function io.loadlines(filename,n) + local f=io.open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[#lines+1]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines end - return data or "" + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end + end end - -local function self_save(name, data) - if data ~= "" then - if merger.strip_comment then - local n = #data - -- saves some 20K .. scite comments - data = gsub(data,"%-%-~[^\n\r]*[\r\n]","") - -- saves some 20K .. ldx comments - data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-ldx%]%]%-%-","") - utilities.report("merge: %s bytes of comment stripped, %s bytes of code left",n-#data,#data) - end - io.savedata(name,data) - utilities.report("merge: saving %s",name) +function io.loadchunk(filename,n) + local f=io.open(filename,'rb') + if f then + local data=f:read(n or 1024) + f:close() + if #data>0 then + return data end + end end - -local function self_swap(data,code) - return data ~= "" and (gsub(data,m_pattern, function() return format(m_format,code) end, 1)) or "" +function io.exists(filename) + local f=io.open(filename) + if f==nil then + return false + else + f:close() + return true + end end - -local function self_libs(libs,list) - local result, f, frozen, foundpath = { }, nil, false, nil - result[#result+1] = "\n" - if type(libs) == 'string' then libs = { libs } end - if type(list) == 'string' then list = { list } end - for i=1,#libs do - local lib = libs[i] - for j=1,#list do - local pth = gsub(list[j],"\\","/") -- file.clean_path - utilities.report("merge: checking library path %s",pth) - local name = pth .. "/" .. lib - if lfs.isfile(name) then - foundpath = pth - end - end - if foundpath then break end - end - if foundpath then - utilities.report("merge: using library path %s",foundpath) - local right, wrong = { }, { } - for i=1,#libs do - local lib = libs[i] - local fullname = foundpath .. "/" .. lib - if lfs.isfile(fullname) then - utilities.report("merge: using library %s",fullname) - right[#right+1] = lib - result[#result+1] = m_begin_closure - result[#result+1] = io.loaddata(fullname,true) - result[#result+1] = m_end_closure - else - utilities.report("merge: skipping library %s",fullname) - wrong[#wrong+1] = lib - end - end - if #right > 0 then - utilities.report("merge: used libraries: %s",concat(right," ")) - end - if #wrong > 0 then - utilities.report("merge: skipped libraries: %s",concat(wrong," ")) - end +function io.size(filename) + local f=io.open(filename) + if f==nil then + return 0 + else + local s=f:seek("end") + f:close() + return s + end +end +function io.noflines(f) + if type(f)=="string" then + local f=io.open(filename) + if f then + local n=f and io.noflines(f) or 0 + f:close() + return n else - utilities.report("merge: no valid library path found") + return 0 + end + else + local n=0 + for _ in f:lines() do + n=n+1 end - return concat(result, "\n\n") + f:seek('set',0) + return n + end +end +local nextchar={ + [ 4]=function(f) + return f:read(1,1,1,1) + end, + [ 2]=function(f) + return f:read(1,1) + end, + [ 1]=function(f) + return f:read(1) + end, + [-2]=function(f) + local a,b=f:read(1,1) + return b,a + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + return d,c,b,a + end +} +function io.characters(f,n) + if f then + return nextchar[n or 1],f + end end - -function merger.selfcreate(libs,list,target) - if target then - self_save(target,self_swap(self_fake(),self_libs(libs,list))) +local nextbyte={ + [4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(a),byte(b),byte(c),byte(d) + end + end, + [3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(a),byte(b),byte(c) + end + end, + [2]=function(f) + local a,b=f:read(1,1) + if b then + return byte(a),byte(b) + end + end, + [1]=function (f) + local a=f:read(1) + if a then + return byte(a) + end + end, + [-2]=function (f) + local a,b=f:read(1,1) + if b then + return byte(b),byte(a) + end + end, + [-3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(c),byte(b),byte(a) + end + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(d),byte(c),byte(b),byte(a) end + end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1],f + else + return nil,nil + end end - -function merger.selfmerge(name,libs,list,target) - self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) +function io.ask(question,default,options) + while true do + io.write(question) + if options then + io.write(format(" [%s]",concat(options,"|"))) + end + if default then + io.write(format(" [%s]",default)) + end + io.write(format(" ")) + io.flush() + local answer=io.read() + answer=gsub(answer,"^%s*(.*)%s*$","%1") + if answer=="" and default then + return default + elseif not options then + return answer + else + for k=1,#options do + if options[k]==answer then + return answer + end + end + local pattern="^"..answer + for k=1,#options do + local v=options[k] + if find(v,pattern) then + return v + end + end + end + end end - -function merger.selfclean(name) - self_save(name,self_swap(self_load(name),self_nothing())) +local function readnumber(f,n,m) + if m then + f:seek("set",n) + n=m + end + if n==1 then + return byte(f:read(1)) + elseif n==2 then + local a,b=byte(f:read(2),1,2) + return 256*a+b + elseif n==3 then + local a,b,c=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==4 then + local a,b,c,d=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==8 then + local a,b=readnumber(f,4),readnumber(f,4) + return 256*a+b + elseif n==12 then + local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) + return 256*256*a+256*b+c + elseif n==-2 then + local b,a=byte(f:read(2),1,2) + return 256*a+b + elseif n==-3 then + local c,b,a=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==-4 then + local d,c,b,a=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==-8 then + local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) + return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + else + return 0 + end end +io.readnumber=readnumber +function io.readstring(f,n,m) + if m then + f:seek("set",n) + n=m + end + local str=gsub(f:read(n),"\000","") + return str +end +if not io.i_limiter then function io.i_limiter() end end +if not io.o_limiter then function io.o_limiter() end end end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - comment = "the strip code is written by Peter Cawley", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +-- original size: 4851, stripped down to: 2828 -local rep, sub, byte, dump, format = string.rep, string.sub, string.byte, string.dump, string.format -local load, loadfile, type = load, loadfile, type - -utilities = utilities or {} -utilities.lua = utilities.lua or { } -local luautilities = utilities.lua - -utilities.report = logs and logs.reporter("system") or print -- can be overloaded later - -local tracestripping = false -local forcestupidcompile = true -- use internal bytecode compiler -luautilities.stripcode = true -- support stripping when asked for -luautilities.alwaysstripcode = false -- saves 1 meg on 7 meg compressed format file (2012.08.12) -luautilities.nofstrippedchunks = 0 -luautilities.nofstrippedbytes = 0 -local strippedchunks = { } -- allocate() -luautilities.strippedchunks = strippedchunks - -luautilities.suffixes = { - tma = "tma", - tmc = jit and "tmb" or "tmc", - lua = "lua", - luc = jit and "lub" or "luc", - lui = "lui", - luv = "luv", - luj = "luj", - tua = "tua", - tuc = "tuc", +if not modules then modules={} end modules ['l-number']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } - -local function fatalerror(name) - utilities.report(format("fatal error in %q",name or "unknown")) +local tostring,tonumber=tostring,tonumber +local format,floor,match,rep=string.format,math.floor,string.match,string.rep +local concat,insert=table.concat,table.insert +local lpegmatch=lpeg.match +number=number or {} +local number=number +if bit32 then + local btest,bor=bit32.btest,bit32.bor + function number.bit(p) + return 2^(p-1) + end + number.hasbit=btest + number.setbit=bor + function number.setbit(x,p) + return btest(x,p) and x or x+p + end + function number.clearbit(x,p) + return btest(x,p) and x-p or x + end +else + function number.bit(p) + return 2^(p-1) + end + function number.hasbit(x,p) + return x%(p+p)>=p + end + function number.setbit(x,p) + return (x%(p+p)>=p) and x or x+p + end + function number.clearbit(x,p) + return (x%(p+p)>=p) and x-p or x + end end - -if jit or status.luatex_version >= 74 then - - local function register(name) - if tracestripping then - utilities.report("stripped bytecode: %s",name or "unknown") - end - strippedchunks[#strippedchunks+1] = name - luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1 - end - - local function stupidcompile(luafile,lucfile,strip) - local code = io.loaddata(luafile) - if code and code ~= "" then - code = load(code) - if code then - code = dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) - if code and code ~= "" then - register(name) - io.savedata(lucfile,code) - return true, 0 - end - else - fatalerror() - end - else - fatalerror() - end - return false, 0 - end - - -- quite subtle ... doing this wrong incidentally can give more bytes - - function luautilities.loadedluacode(fullname,forcestrip,name) - -- quite subtle ... doing this wrong incidentally can give more bytes - name = name or fullname - local code = loadfile(fullname) - if code then - code() - end - if forcestrip and luautilities.stripcode then - if type(forcestrip) == "function" then - forcestrip = forcestrip(fullname) - end - if forcestrip or luautilities.alwaysstripcode then - register(name) - return load(dump(code,true)), 0 - else - return code, 0 - end - elseif luautilities.alwaysstripcode then - register(name) - return load(dump(code,true)), 0 - else - return code, 0 - end +if bit32 then + local bextract=bit32.extract + local t={ + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + } + function number.tobitstring(b,m) + local n=32 + for i=0,31 do + local v=bextract(b,i) + local k=32-i + if v==1 then + n=k + t[k]="1" + else + t[k]="0" + end end - - function luautilities.strippedloadstring(code,forcestrip,name) -- not executed - if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then - code = load(code) - if not code then - fatalerror(name) - end - register(name) - code = dump(code,true) - end - return load(code), 0 - end - - function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true - utilities.report("lua: compiling %s into %s",luafile,lucfile) - os.remove(lucfile) - local done = stupidcompile(luafile,lucfile,strip ~= false) - if done then - utilities.report("lua: %s dumped into %s (stripped)",luafile,lucfile) - if cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - utilities.report("lua: removing %s",luafile) - os.remove(luafile) - end - end - return done + if m then + m=33-m*8 + if m<1 then + m=1 + end + return concat(t,"",m) + elseif n<8 then + return concat(t) + elseif n<16 then + return concat(t,"",9) + elseif n<24 then + return concat(t,"",17) + else + return concat(t,"",25) end - + end else - - -- The next function was posted by Peter Cawley on the lua list and strips line - -- number information etc. from the bytecode data blob. We only apply this trick - -- when we store data tables. Stripping makes the compressed format file about - -- 1MB smaller (and uncompressed we save at least 6MB). - -- - -- You can consider this feature an experiment, so it might disappear. There is - -- no noticeable gain in runtime although the memory footprint should be somewhat - -- smaller (and the file system has a bit less to deal with). - -- - -- Begin of borrowed code ... works for Lua 5.1 which LuaTeX currently uses ... - - local function register(name,before,after) - local delta = before - after - if tracestripping then - utilities.report("stripped bytecode: %s, before %s, after %s, delta %s",name or "unknown",before,after,delta) + function number.tobitstring(n,m) + if n>0 then + local t={} + while n>0 do + insert(t,1,n%2>0 and 1 or 0) + n=floor(n/2) + end + local nn=8-#t%8 + if nn>0 and nn<8 then + for i=1,nn do + insert(t,1,0) end - strippedchunks[#strippedchunks+1] = name - luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1 - luautilities.nofstrippedbytes = luautilities.nofstrippedbytes + delta - return delta - end - - local strip_code_pc - - if _MAJORVERSION == 5 and _MINORVERSION == 1 then - - strip_code_pc = function(dump,name) - local before = #dump - local version, format, endian, int, size, ins, num = byte(dump,5,11) - local subint - if endian == 1 then - subint = function(dump, i, l) - local val = 0 - for n = l, 1, -1 do - val = val * 256 + byte(dump,i + n - 1) - end - return val, i + l - end - else - subint = function(dump, i, l) - local val = 0 - for n = 1, l, 1 do - val = val * 256 + byte(dump,i + n - 1) - end - return val, i + l - end - end - local strip_function - strip_function = function(dump) - local count, offset = subint(dump, 1, size) - local stripped, dirty = rep("\0", size), offset + count - offset = offset + count + int * 2 + 4 - offset = offset + int + subint(dump, offset, int) * ins - count, offset = subint(dump, offset, int) - for n = 1, count do - local t - t, offset = subint(dump, offset, 1) - if t == 1 then - offset = offset + 1 - elseif t == 4 then - offset = offset + size + subint(dump, offset, size) - elseif t == 3 then - offset = offset + num - end - end - count, offset = subint(dump, offset, int) - stripped = stripped .. sub(dump,dirty, offset - 1) - for n = 1, count do - local proto, off = strip_function(sub(dump,offset, -1)) - stripped, offset = stripped .. proto, offset + off - 1 - end - offset = offset + subint(dump, offset, int) * int + int - count, offset = subint(dump, offset, int) - for n = 1, count do - offset = offset + subint(dump, offset, size) + size + int * 2 - end - count, offset = subint(dump, offset, int) - for n = 1, count do - offset = offset + subint(dump, offset, size) + size - end - stripped = stripped .. rep("\0", int * 3) - return stripped, offset - end - dump = sub(dump,1,12) .. strip_function(sub(dump,13,-1)) - local after = #dump - local delta = register(name,before,after) - return dump, delta + end + if m then + m=m*8-#t + if m>0 then + insert(t,1,rep("0",m)) end - + end + return concat(t) + elseif m then + rep("00000000",m) else - - strip_code_pc = function(dump,name) - return dump, 0 - end - - end - - -- ... end of borrowed code. - - -- quite subtle ... doing this wrong incidentally can give more bytes - - function luautilities.loadedluacode(fullname,forcestrip,name) - -- quite subtle ... doing this wrong incidentally can give more bytes - name = name or fullname - local code = loadfile(fullname) - if code then - code() - end - if forcestrip and luautilities.stripcode then - if type(forcestrip) == "function" then - forcestrip = forcestrip(fullname) - end - if forcestrip then - local code, n = strip_code_pc(dump(code),name) - return load(code), n - elseif luautilities.alwaysstripcode then - return load(strip_code_pc(dump(code),name)) - else - return code, 0 - end - elseif luautilities.alwaysstripcode then - return load(strip_code_pc(dump(code),name)) - else - return code, 0 - end - end - - function luautilities.strippedloadstring(code,forcestrip,name) -- not executed - local n = 0 - if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then - code = load(code) - if not code then - fatalerror(name) - end - code, n = strip_code_pc(dump(code),name) - end - return load(code), n - end - - local function stupidcompile(luafile,lucfile,strip) - local code = io.loaddata(luafile) - local n = 0 - if code and code ~= "" then - code = load(code) - if not code then - fatalerror() - end - code = dump(code) - if strip then - code, n = strip_code_pc(code,luautilities.stripcode or luautilities.alwaysstripcode,luafile) -- last one is reported - end - if code and code ~= "" then - io.savedata(lucfile,code) - end - end - return n + return "00000000" end - - local luac_normal = "texluac -o %q %q" - local luac_strip = "texluac -s -o %q %q" - - function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true - utilities.report("lua: compiling %s into %s",luafile,lucfile) - os.remove(lucfile) - local done = false - if strip ~= false then - strip = true - end - if forcestupidcompile then - fallback = true - elseif strip then - done = os.spawn(format(luac_strip, lucfile,luafile)) == 0 - else - done = os.spawn(format(luac_normal,lucfile,luafile)) == 0 - end - if not done and fallback then - local n = stupidcompile(luafile,lucfile,strip) - if n > 0 then - utilities.report("lua: %s dumped into %s (%i bytes stripped)",luafile,lucfile,n) - else - utilities.report("lua: %s dumped into %s (unstripped)",luafile,lucfile) - end - cleanup = false -- better see how bad it is - done = true -- hm - end - if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - utilities.report("lua: removing %s",luafile) - os.remove(luafile) - end - return done + end +end +function number.valid(str,default) + return tonumber(str) or default or nil +end +function number.toevenhex(n) + local s=format("%X",n) + if #s%2==0 then + return s + else + return "0"..s + end +end +local one=lpeg.C(1-lpeg.S('')/tonumber)^1 +function number.toset(n) + return lpegmatch(one,tostring(n)) +end +local function bits(n,i,...) + if n>0 then + local m=n%2 + local n=floor(n/2) + if m>0 then + return bits(n,i+1,i,...) + else + return bits(n,i+1,...) end - + else + return... + end +end +function number.bits(n) + return { bits(n,1) } end - --- local getmetatable, type = getmetatable, type --- --- local types = { } --- --- function luautilities.registerdatatype(d,name) --- types[getmetatable(d)] = name --- end --- --- function luautilities.datatype(d) --- local t = type(d) --- if t == "userdata" then --- local m = getmetatable(d) --- return m and types[m] or "userdata" --- else --- return t --- end --- end --- --- luautilities.registerdatatype(lpeg.P("!"),"lpeg") --- --- print(luautilities.datatype(lpeg.P("oeps"))) end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-prs'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +-- original size: 1923, stripped down to: 1133 -local lpeg, table, string = lpeg, table, string -local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf, Cp = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.Cp -local lpegmatch, patterns = lpeg.match, lpeg.patterns -local concat, format, gmatch, find = table.concat, string.format, string.gmatch, string.find -local tostring, type, next, rawset = tostring, type, next, rawset - -utilities = utilities or {} -utilities.parsers = utilities.parsers or { } -local parsers = utilities.parsers -parsers.patterns = parsers.patterns or { } - -local setmetatableindex = table.setmetatableindex -local sortedhash = table.sortedhash - --- we share some patterns - -local digit = R("09") -local space = P(' ') -local equal = P("=") -local comma = P(",") -local lbrace = P("{") -local rbrace = P("}") -local lparent = P("(") -local rparent = P(")") -local period = S(".") -local punctuation = S(".,:;") -local spacer = patterns.spacer -local whitespace = patterns.whitespace -local newline = patterns.newline -local anything = patterns.anything -local endofstring = patterns.endofstring - -local nobrace = 1 - ( lbrace + rbrace ) -local noparent = 1 - ( lparent + rparent) - --- we could use a Cf Cg construct - -local escape, left, right = P("\\"), P('{'), P('}') - -patterns.balanced = P { - [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, - [2] = left * V(1) * right +if not modules then modules={} end modules ['l-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } +set=set or {} +local nums={} +local tabs={} +local concat=table.concat +local next,type=next,type +set.create=table.tohash +function set.tonumber(t) + if next(t) then + local s="" + for k,v in next,t do + if v then + s=s.." "..k + end + end + local n=nums[s] + if not n then + n=#tabs+1 + tabs[n]=t + nums[s]=n + end + return n + else + return 0 + end +end +function set.totable(n) + if n==0 then + return {} + else + return tabs[n] or {} + end +end +function set.tolist(n) + if n==0 or not tabs[n] then + return "" + else + local t,n={},0 + for k,v in next,tabs[n] do + if v then + n=n+1 + t[n]=k + end + end + return concat(t," ") + end +end +function set.contains(n,s) + if type(n)=="table" then + return n[s] + elseif n==0 then + return false + else + local t=tabs[n] + return t and t[s] + end +end -local nestedbraces = P { lbrace * (nobrace + V(1))^0 * rbrace } -local nestedparents = P { lparent * (noparent + V(1))^0 * rparent } -local spaces = space^0 -local argument = Cs((lbrace/"") * ((nobrace + nestedbraces)^0) * (rbrace/"")) -local content = (1-endofstring)^0 - -patterns.nestedbraces = nestedbraces -- no capture -patterns.nestedparents = nestedparents -- no capture -patterns.nested = nestedbraces -- no capture -patterns.argument = argument -- argument after e.g. = -patterns.content = content -- rest after e.g = - -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) + C((nestedbraces + (1-comma))^0) - -local key = C((1-equal-comma)^1) -local pattern_a = (space+comma)^0 * (key * equal * value + key * C("")) -local pattern_c = (space+comma)^0 * (key * equal * value) - -local key = C((1-space-equal-comma)^1) -local pattern_b = spaces * comma^0 * spaces * (key * ((spaces * equal * spaces * value) + C(""))) - --- "a=1, b=2, c=3, d={a{b,c}d}, e=12345, f=xx{a{b,c}d}xx, g={}" : outer {} removes, leading spaces ignored - --- todo: rewrite to fold etc --- --- parse = lpeg.Cf(lpeg.Carg(1) * lpeg.Cg(key * equal * value) * separator^0,rawset)^0 -- lpeg.match(parse,"...",1,hash) - -local hash = { } -local function set(key,value) - hash[key] = value -end +end -- of closure -local pattern_a_s = (pattern_a/set)^1 -local pattern_b_s = (pattern_b/set)^1 -local pattern_c_s = (pattern_c/set)^1 +do -- create closure to overcome 200 locals limit -parsers.patterns.settings_to_hash_a = pattern_a_s -parsers.patterns.settings_to_hash_b = pattern_b_s -parsers.patterns.settings_to_hash_c = pattern_c_s +-- original size: 13731, stripped down to: 8450 -function parsers.make_settings_to_hash_pattern(set,how) - if how == "strict" then - return (pattern_c/set)^1 - elseif how == "tolerant" then - return (pattern_b/set)^1 - else - return (pattern_a/set)^1 +if not modules then modules={} end modules ['l-os']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local os=os +local date,time=os.date,os.time +local find,format,gsub,upper,gmatch=string.find,string.format,string.gsub,string.upper,string.gmatch +local concat=table.concat +local random,ceil,randomseed=math.random,math.ceil,math.randomseed +local rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring=rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring +math.initialseed=tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6)) +randomseed(math.initialseed) +if not os.__getenv__ then + os.__getenv__=os.getenv + os.__setenv__=os.setenv + if os.env then + local osgetenv=os.getenv + local ossetenv=os.setenv + local osenv=os.env local _=osenv.PATH + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + if type(v)=="table" then + v=concat(v,";") + end + ossetenv(K,v) end -end - -function parsers.settings_to_hash(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_a_s,str) - return hash - else - return { } + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + else + local ossetenv=os.setenv + local osgetenv=os.getenv + local osenv={} + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + end + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + local function __index(t,k) + return os.getenv(k) + end + local function __newindex(t,k,v) + os.setenv(k,v) end + os.env={} + setmetatable(os.env,{ __index=__index,__newindex=__newindex } ) + end end - -function parsers.settings_to_hash_tolerant(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_b_s,str) - return hash +local execute,spawn,exec,iopopen,ioflush=os.execute,os.spawn or os.execute,os.exec or os.execute,io.popen,io.flush +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end +function io.popen (...) ioflush() return iopopen(...) end +function os.resultof(command) + local handle=io.popen(command,"r") + return handle and handle:read("*all") or "" +end +if not io.fileseparator then + if find(os.getenv("PATH"),";") then + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + else + io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" + end +end +os.type=os.type or (io.pathseparator==";" and "windows") or "unix" +os.name=os.name or (os.type=="windows" and "mswin" ) or "linux" +if os.type=="windows" then + os.libsuffix,os.binsuffix,os.binsuffixes='dll','exe',{ 'exe','cmd','bat' } +else + os.libsuffix,os.binsuffix,os.binsuffixes='so','',{ '' } +end +local launchers={ + windows="start %s", + macosx="open %s", + unix="$BROWSER %s &> /dev/null &", +} +function os.launch(str) + os.execute(format(launchers[os.name] or launchers.unix,str)) +end +if not os.times then + function os.times() + return { + utime=os.gettimeofday(), + stime=0, + cutime=0, + cstime=0, + } + end +end +os.gettimeofday=os.gettimeofday or os.clock +local startuptime=os.gettimeofday() +function os.runtime() + return os.gettimeofday()-startuptime +end +os.resolvers=os.resolvers or {} +local resolvers=os.resolvers +local osmt=getmetatable(os) or { __index=function(t,k) t[k]="unset" return "unset" end } +local osix=osmt.__index +osmt.__index=function(t,k) + return (resolvers[k] or osix)(t,k) +end +setmetatable(os,osmt) +local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" +local function guess() + local architecture=os.resultof("uname -m") or "" + if architecture~="" then + return architecture + end + architecture=os.getenv("HOSTTYPE") or "" + if architecture~="" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end +if platform~="" then + os.platform=platform +elseif os.type=="windows" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform="mswin-64" + else + platform="mswin" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="linux" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform="linux-64" + elseif find(architecture,"ppc") then + platform="linux-ppc" + else + platform="linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="macosx" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + if architecture=="" then + platform="osx-intel" + elseif find(architecture,"i386") then + platform="osx-intel" + elseif find(architecture,"x86_64") then + platform="osx-64" + else + platform="osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="sunos" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform="solaris-sparc" + else + platform="solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="freebsd" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform="freebsd-amd64" + else + platform="freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="kfreebsd" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform="kfreebsd-amd64" + else + platform="kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +else + function os.resolvers.platform(t,k) + local platform="linux" + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +end +local t={ 8,9,"a","b" } +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end +local d +function os.timezone(delta) + d=d or tonumber(tonumber(date("%H")-date("!%H"))) + if delta then + if d>0 then + return format("+%02i:00",d) else - return { } + return format("-%02i:00",-d) end + else + return 1 + end end - -function parsers.settings_to_hash_strict(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_c_s,str) - return next(hash) and hash - else - return nil +local timeformat=format("%%s%s",os.timezone(true)) +local dateformat="!%Y-%m-%d %H:%M:%S" +function os.fulltime(t,default) + t=tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=nil + end + return format(timeformat,date(dateformat,t)) +end +local dateformat="%Y-%m-%d %H:%M:%S" +function os.localtime(t,default) + t=tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=nil + end + return date(dateformat,t) +end +function os.converttime(t,default) + local t=tonumber(t) + if t and t>0 then + return date(dateformat,t) + else + return default or "-" + end +end +local memory={} +local function which(filename) + local fullname=memory[filename] + if fullname==nil then + local suffix=file.suffix(filename) + local suffixes=suffix=="" and os.binsuffixes or { suffix } + for directory in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local df=file.join(directory,filename) + for i=1,#suffixes do + local dfs=file.addsuffix(df,suffixes[i]) + if io.exists(dfs) then + fullname=dfs + break + end + end + end + if not fullname then + fullname=false end + memory[filename]=fullname + end + return fullname +end +os.which=which +os.where=which +function os.today() + return date("!*t") +end +function os.now() + return date("!%Y-%m-%d %H:%M:%S") end -local separator = comma * space^0 -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) - + C((nestedbraces + (1-comma))^0) -local pattern = spaces * Ct(value*(separator*value)^0) --- "aap, {noot}, mies" : outer {} removes, leading spaces ignored +end -- of closure -parsers.patterns.settings_to_array = pattern +do -- create closure to overcome 200 locals limit --- we could use a weak table as cache +-- original size: 15501, stripped down to: 8354 -function parsers.settings_to_array(str,strict) - if not str or str == "" then - return { } - elseif strict then - if find(str,"{") then - return lpegmatch(pattern,str) - else - return { str } +if not modules then modules={} end modules ['l-file']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +file=file or {} +local file=file +local insert,concat=table.insert,table.concat +local match=string.match +local lpegmatch=lpeg.match +local getcurrentdir,attributes=lfs.currentdir,lfs.attributes +local checkedsplit=string.checkedsplit +local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local colon=P(":") +local period=P(".") +local periods=P("..") +local fwslash=P("/") +local bwslash=P("\\") +local slashes=S("\\/") +local noperiod=1-period +local noslashes=1-slashes +local name=noperiod^1 +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=C((1-(slashes^1*noslashes^1*-1))^1)*P(1) +local function pathpart(name,default) + return name and lpegmatch(pattern,name) or default or "" +end +local pattern=(noslashes^0*slashes)^1*C(noslashes^1)*-1 +local function basename(name) + return name and lpegmatch(pattern,name) or name +end +local pattern=(noslashes^0*slashes^1)^0*Cs((1-suffix)^1)*suffix^0 +local function nameonly(name) + return name and lpegmatch(pattern,name) or name +end +local pattern=(noslashes^0*slashes)^0*(noperiod^1*period)^1*C(noperiod^1)*-1 +local function suffixonly(name) + return name and lpegmatch(pattern,name) or "" +end +file.pathpart=pathpart +file.basename=basename +file.nameonly=nameonly +file.suffixonly=suffixonly +file.suffix=suffixonly +file.dirname=pathpart +file.extname=suffixonly +local drive=C(R("az","AZ"))*colon +local path=C((noslashes^0*slashes)^0) +local suffix=period*C(P(1-period)^0*P(-1)) +local base=C((1-suffix)^0) +local rest=C(P(1)^0) +drive=drive+Cc("") +path=path+Cc("") +base=base+Cc("") +suffix=suffix+Cc("") +local pattern_a=drive*path*base*suffix +local pattern_b=path*base*suffix +local pattern_c=C(drive*path)*C(base*suffix) +local pattern_d=path*rest +function file.splitname(str,splitdrive) + if not str then + elseif splitdrive then + return lpegmatch(pattern_a,str) + else + return lpegmatch(pattern_b,str) + end +end +function file.splitbase(str) + return str and lpegmatch(pattern_d,str) +end +function file.nametotable(str,splitdrive) + if str then + local path,drive,subpath,name,base,suffix=lpegmatch(pattern_c,str) + if splitdrive then + return { + path=path, + drive=drive, + subpath=subpath, + name=name, + base=base, + suffix=suffix, + } + else + return { + path=path, + name=name, + base=base, + suffix=suffix, + } + end + end +end +local pattern=Cs(((period*noperiod^1*-1)/""+1)^1) +function file.removesuffix(name) + return name and lpegmatch(pattern,name) +end +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=Cs((noslashes^0*slashes^1)^0*((1-suffix)^1))*Cs(suffix) +function file.addsuffix(filename,suffix,criterium) + if not filename or not suffix or suffix=="" then + return filename + elseif criterium==true then + return filename.."."..suffix + elseif not criterium then + local n,s=lpegmatch(pattern,filename) + if not s or s=="" then + return filename.."."..suffix + else + return filename + end + else + local n,s=lpegmatch(pattern,filename) + if s and s~="" then + local t=type(criterium) + if t=="table" then + for i=1,#criterium do + if s==criterium[i] then + return filename + end end - else - return lpegmatch(pattern,str) + elseif t=="string" then + if s==criterium then + return filename + end + end end + return (n or filename).."."..suffix + end end - -local function set(t,v) - t[#t+1] = v +local suffix=period*(1-period-slashes)^1*-1 +local pattern=Cs((1-suffix)^0) +function file.replacesuffix(name,suffix) + if name and suffix and suffix~="" then + return lpegmatch(pattern,name).."."..suffix + else + return name + end end - -local value = P(Carg(1)*value) / set -local pattern = value*(separator*value)^0 * Carg(1) - -function parsers.add_settings_to_array(t,str) - return lpegmatch(pattern,str,nil,t) +local reslasher=lpeg.replacer(P("\\"),"/") +function file.reslash(str) + return str and lpegmatch(reslasher,str) end - -function parsers.hash_to_string(h,separator,yes,no,strict,omit) - if h then - local t, tn, s = { }, 0, table.sortedkeys(h) - omit = omit and table.tohash(omit) - for i=1,#s do - local key = s[i] - if not omit or not omit[key] then - local value = h[key] - if type(value) == "boolean" then - if yes and no then - if value then - tn = tn + 1 - t[tn] = key .. '=' .. yes - elseif not strict then - tn = tn + 1 - t[tn] = key .. '=' .. no - end - elseif value or not strict then - tn = tn + 1 - t[tn] = key .. '=' .. tostring(value) - end - else - tn = tn + 1 - t[tn] = key .. '=' .. value - end - end - end - return concat(t,separator or ",") - else - return "" +function file.is_writable(name) + if not name then + elseif lfs.isdir(name) then + name=name.."/m_t_x_t_e_s_t.tmp" + local f=io.open(name,"wb") + if f then + f:close() + os.remove(name) + return true + end + elseif lfs.isfile(name) then + local f=io.open(name,"ab") + if f then + f:close() + return true + end + else + local f=io.open(name,"ab") + if f then + f:close() + os.remove(name) + return true end + end + return false end - -function parsers.array_to_string(a,separator) - if a then - return concat(a,separator or ",") +local readable=P("r")*Cc(true) +function file.is_readable(name) + if name then + local a=attributes(name) + return a and lpegmatch(readable,a.permissions) or false + else + return false + end +end +file.isreadable=file.is_readable +file.iswritable=file.is_writable +function file.size(name) + if name then + local a=attributes(name) + return a and a.size or 0 + else + return 0 + end +end +function file.splitpath(str,separator) + return str and checkedsplit(lpegmatch(reslasher,str),separator or io.pathseparator) +end +function file.joinpath(tab,separator) + return tab and concat(tab,separator or io.pathseparator) +end +local stripper=Cs(P(fwslash)^0/""*reslasher) +local isnetwork=fwslash*fwslash*(1-fwslash)+(1-fwslash-colon)^1*colon +local isroot=fwslash^1*-1 +local hasroot=fwslash^1 +local deslasher=lpeg.replacer(S("\\/")^1,"/") +function file.join(...) + local lst={... } + local one=lst[1] + if lpegmatch(isnetwork,one) then + local two=lpegmatch(deslasher,concat(lst,"/",2)) + return one.."/"..two + elseif lpegmatch(isroot,one) then + local two=lpegmatch(deslasher,concat(lst,"/",2)) + if lpegmatch(hasroot,two) then + return two + else + return "/"..two + end + elseif one=="" then + return lpegmatch(stripper,concat(lst,"/",2)) + else + return lpegmatch(deslasher,concat(lst,"/")) + end +end +local drivespec=R("az","AZ")^1*colon +local anchors=fwslash+drivespec +local untouched=periods+(1-period)^1*P(-1) +local splitstarter=(Cs(drivespec*(bwslash/"/"+fwslash)^0)+Cc(false))*Ct(lpeg.splitat(S("/\\")^1)) +local absolute=fwslash +function file.collapsepath(str,anchor) + if not str then + return + end + if anchor and not lpegmatch(anchors,str) then + str=getcurrentdir().."/"..str + end + if str=="" or str=="." then + return "." + elseif lpegmatch(untouched,str) then + return lpegmatch(reslasher,str) + end + local starter,oldelements=lpegmatch(splitstarter,str) + local newelements={} + local i=#oldelements + while i>0 do + local element=oldelements[i] + if element=='.' then + elseif element=='..' then + local n=i-1 + while n>0 do + local element=oldelements[n] + if element~='..' and element~='.' then + oldelements[n]='.' + break + else + n=n-1 + end + end + if n<1 then + insert(newelements,1,'..') + end + elseif element~="" then + insert(newelements,1,element) + end + i=i-1 + end + if #newelements==0 then + return starter or "." + elseif starter then + return starter..concat(newelements,'/') + elseif lpegmatch(absolute,str) then + return "/"..concat(newelements,'/') + else + return concat(newelements,'/') + end +end +local validchars=R("az","09","AZ","--","..") +local pattern_a=lpeg.replacer(1-validchars) +local pattern_a=Cs((validchars+P(1)/"-")^1) +local whatever=P("-")^0/"" +local pattern_b=Cs(whatever*(1-whatever*-1)^1) +function file.robustname(str,strict) + if str then + str=lpegmatch(pattern_a,str) or str + if strict then + return lpegmatch(pattern_b,str) or str else - return "" + return str end + end end - -function parsers.settings_to_set(str,t) -- tohash? -- todo: lpeg -- duplicate anyway - t = t or { } --- for s in gmatch(str,"%s*([^, ]+)") do -- space added - for s in gmatch(str,"[^, ]+") do -- space added - t[s] = true - end - return t +file.readdata=io.loaddata +file.savedata=io.savedata +function file.copy(oldname,newname) + if oldname and newname then + file.savedata(newname,io.loaddata(oldname)) + end +end +local letter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=period^0*fwslash+letter*colon+letter^1*separator+letter^1*fwslash +local rootbased=fwslash+letter*colon +lpeg.patterns.qualified=qualified +lpeg.patterns.rootbased=rootbased +function file.is_qualified_path(filename) + return filename and lpegmatch(qualified,filename)~=nil end - -function parsers.simple_hash_to_string(h, separator) - local t, tn = { }, 0 - for k, v in sortedhash(h) do - if v then - tn = tn + 1 - t[tn] = k - end - end - return concat(t,separator or ",") +function file.is_rootbased_path(filename) + return filename and lpegmatch(rootbased,filename)~=nil +end +function file.strip(name,dir) + if name then + local b,a=match(name,"^(.-)"..dir.."(.*)$") + return a~="" and a or name + end end --- for chem (currently one level) -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) - + C(digit^1 * lparent * (noparent + nestedparents)^1 * rparent) - + C((nestedbraces + (1-comma))^1) -local pattern_a = spaces * Ct(value*(separator*value)^0) +end -- of closure -local function repeater(n,str) - if not n then - return str - else - local s = lpegmatch(pattern_a,str) - if n == 1 then - return unpack(s) - else - local t, tn = { }, 0 - for i=1,n do - for j=1,#s do - tn = tn + 1 - t[tn] = s[j] - end - end - return unpack(t) - end +do -- create closure to overcome 200 locals limit + +-- original size: 3445, stripped down to: 1803 + +if not modules then modules={} end modules ['l-md5']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local md5,file=md5,file +local gsub,format,byte=string.gsub,string.format,string.byte +local function convert(str,fmt) + return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +end +if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end +if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end +if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end +function file.needsupdating(oldname,newname,threshold) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime then + local newtime=lfs.attributes(newname,"modification") + if not newtime then + return true + elseif newtime>=oldtime then + return false + elseif oldtime-newtime<(threshold or 1) then + return false + else + return true + end + else + return false + end +end +file.needs_updating=file.needsupdating +function file.syncmtimes(oldname,newname) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime and lfs.isfile(newname) then + lfs.touch(newname,oldtime,oldtime) + end +end +function file.checksum(name) + if md5 then + local data=io.loaddata(name) + if data then + return md5.HEX(data) end + end + return nil +end +function file.loadchecksum(name) + if md5 then + local data=io.loaddata(name..".md5") + return data and (gsub(data,"%s","")) + end + return nil +end +function file.savechecksum(name,checksum) + if not checksum then checksum=file.checksum(name) end + if checksum then + io.savedata(name..".md5",checksum) + return checksum + end + return nil end -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) - + (C(digit^1)/tonumber * lparent * Cs((noparent + nestedparents)^1) * rparent) / repeater - + C((nestedbraces + (1-comma))^1) -local pattern_b = spaces * Ct(value*(separator*value)^0) -function parsers.settings_to_array_with_repeat(str,expand) -- beware: "" => { } - if expand then - return lpegmatch(pattern_b,str) or { } - else - return lpegmatch(pattern_a,str) or { } +end -- of closure + +do -- create closure to overcome 200 locals limit + +-- original size: 11806, stripped down to: 5417 + +if not modules then modules={} end modules ['l-url']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local char,format,byte=string.char,string.format,string.byte +local concat=table.concat +local tonumber,type=tonumber,type +local P,C,R,S,Cs,Cc,Ct,Cf,Cg,V=lpeg.P,lpeg.C,lpeg.R,lpeg.S,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.Cf,lpeg.Cg,lpeg.V +local lpegmatch,lpegpatterns,replacer=lpeg.match,lpeg.patterns,lpeg.replacer +url=url or {} +local url=url +local tochar=function(s) return char(tonumber(s,16)) end +local colon=P(":") +local qmark=P("?") +local hash=P("#") +local slash=P("/") +local percent=P("%") +local endofstring=P(-1) +local hexdigit=R("09","AF","af") +local plus=P("+") +local nothing=Cc("") +local escapedchar=(percent*C(hexdigit*hexdigit))/tochar +local escaped=(plus/" ")+escapedchar +local noslash=P("/")/"" +local schemestr=Cs((escaped+(1-colon-slash-qmark-hash))^2) +local authoritystr=Cs((escaped+(1- slash-qmark-hash))^0) +local pathstr=Cs((escaped+(1- qmark-hash))^0) +local querystr=Cs(((1- hash))^0) +local fragmentstr=Cs((escaped+(1- endofstring))^0) +local scheme=schemestr*colon+nothing +local authority=slash*slash*authoritystr+nothing +local path=slash*pathstr+nothing +local query=qmark*querystr+nothing +local fragment=hash*fragmentstr+nothing +local validurl=scheme*authority*path*query*fragment +local parser=Ct(validurl) +lpegpatterns.url=validurl +lpegpatterns.urlsplitter=parser +local escapes={} +setmetatable(escapes,{ __index=function(t,k) + local v=format("%%%02X",byte(k)) + t[k]=v + return v +end }) +local escaper=Cs((R("09","AZ","az")^1+P(" ")/"%%20"+S("-./_")^1+P(1)/escapes)^0) +local unescaper=Cs((escapedchar+1)^0) +lpegpatterns.urlunescaped=escapedchar +lpegpatterns.urlescaper=escaper +lpegpatterns.urlunescaper=unescaper +local function split(str) + return (type(str)=="string" and lpegmatch(parser,str)) or str +end +local isscheme=schemestr*colon*slash*slash +local function hasscheme(str) + if str then + local scheme=lpegmatch(isscheme,str) + return scheme~="" and scheme or false + else + return false + end +end +local rootletter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=P(".")^0*P("/")+rootletter*P(":")+rootletter^1*separator+rootletter^1*P("/") +local rootbased=P("/")+rootletter*P(":") +local barswapper=replacer("|",":") +local backslashswapper=replacer("\\","/") +local equal=P("=") +local amp=P("&") +local key=Cs(((escapedchar+1)-equal )^0) +local value=Cs(((escapedchar+1)-amp -endofstring)^0) +local splitquery=Cf (Ct("")*P { "sequence", + sequence=V("pair")*(amp*V("pair"))^0, + pair=Cg(key*equal*value), +},rawset) +local function hashed(str) + if str=="" then + return { + scheme="invalid", + original=str, + } + end + local s=split(str) + local rawscheme=s[1] + local rawquery=s[4] + local somescheme=rawscheme~="" + local somequery=rawquery~="" + if not somescheme and not somequery then + s={ + scheme="file", + authority="", + path=str, + query="", + fragment="", + original=str, + noscheme=true, + filename=str, + } + else + local authority,path,filename=s[2],s[3] + if authority=="" then + filename=path + elseif path=="" then + filename="" + else + filename=authority.."/"..path + end + s={ + scheme=rawscheme, + authority=authority, + path=path, + query=lpegmatch(unescaper,rawquery), + queries=lpegmatch(splitquery,rawquery), + fragment=s[5], + original=str, + noscheme=false, + filename=filename, + } + end + return s +end +url.split=split +url.hasscheme=hasscheme +url.hashed=hashed +function url.addscheme(str,scheme) + if hasscheme(str) then + return str + elseif not scheme then + return "file:///"..str + else + return scheme..":///"..str + end +end +function url.construct(hash) + local fullurl,f={},0 + local scheme,authority,path,query,fragment=hash.scheme,hash.authority,hash.path,hash.query,hash.fragment + if scheme and scheme~="" then + f=f+1;fullurl[f]=scheme.."://" + end + if authority and authority~="" then + f=f+1;fullurl[f]=authority + end + if path and path~="" then + f=f+1;fullurl[f]="/"..path + end + if query and query~="" then + f=f+1;fullurl[f]="?"..query + end + if fragment and fragment~="" then + f=f+1;fullurl[f]="#"..fragment + end + return lpegmatch(escaper,concat(fullurl)) +end +local pattern=Cs(noslash*R("az","AZ")*(S(":|")/":")*noslash*P(1)^0) +function url.filename(filename) + local spec=hashed(filename) + local path=spec.path + return (spec.scheme=="file" and path and lpegmatch(pattern,path)) or filename +end +local function escapestring(str) + return lpegmatch(escaper,str) +end +url.escape=escapestring +function url.query(str) + if type(str)=="string" then + return lpegmatch(splitquery,str) or "" + else + return str + end +end +function url.toquery(data) + local td=type(data) + if td=="string" then + return #str and escape(data) or nil + elseif td=="table" then + if next(data) then + local t={} + for k,v in next,data do + t[#t+1]=format("%s=%s",k,escapestring(v)) + end + return concat(t,"&") end + else + end +end +local pattern=Cs(noslash^0*(1-noslash*P(-1))^0) +function url.barepath(path) + if not path or path=="" then + return "" + else + return lpegmatch(pattern,path) + end end --- -local value = lbrace * C((nobrace + nestedbraces)^0) * rbrace -local pattern = Ct((space + value)^0) +end -- of closure -function parsers.arguments_to_table(str) - return lpegmatch(pattern,str) -end +do -- create closure to overcome 200 locals limit --- temporary here (unoptimized) +-- original size: 13035, stripped down to: 8133 -function parsers.getparameters(self,class,parentclass,settings) - local sc = self[class] - if not sc then - sc = { } - self[class] = sc - if parentclass then - local sp = self[parentclass] - if not sp then - sp = { } - self[parentclass] = sp - end - setmetatableindex(sc,sp) - end +if not modules then modules={} end modules ['l-dir']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,select=type,select +local find,gmatch,match,gsub=string.find,string.gmatch,string.match,string.gsub +local concat,insert,remove=table.concat,table.insert,table.remove +local lpegmatch=lpeg.match +local P,S,R,C,Cc,Cs,Ct,Cv,V=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Cv,lpeg.V +dir=dir or {} +local dir=dir +local lfs=lfs +local attributes=lfs.attributes +local walkdir=lfs.dir +local isdir=lfs.isdir +local isfile=lfs.isfile +local currentdir=lfs.currentdir +if not isdir then + function isdir(name) + local a=attributes(name) + return a and a.mode=="directory" + end + lfs.isdir=isdir +end +if not isfile then + function isfile(name) + local a=attributes(name) + return a and a.mode=="file" + end + lfs.isfile=isfile +end +function dir.current() + return (gsub(currentdir(),"\\","/")) +end +local lfsisdir=isdir +local function isdir(path) + path=gsub(path,"[/\\]+$","") + return lfsisdir(path) +end +lfs.isdir=isdir +local function globpattern(path,patt,recurse,action) + if path=="/" then + path=path.."." + elseif not find(path,"/$") then + path=path..'/' + end + if isdir(path) then + for name in walkdir(path) do + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if find(full,patt) then + action(full) + end + elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then + globpattern(full,patt,recurse,action) + end end - parsers.settings_to_hash(settings,sc) + end end - -function parsers.listitem(str) - return gmatch(str,"[^, ]+") +dir.globpattern=globpattern +local function collectpattern(path,patt,recurse,result) + local ok,scanner + result=result or {} + if path=="/" then + ok,scanner,first=xpcall(function() return walkdir(path..".") end,function() end) + else + ok,scanner,first=xpcall(function() return walkdir(path) end,function() end) + end + if ok and type(scanner)=="function" then + if not find(path,"/$") then path=path..'/' end + for name in scanner,first do + local full=path..name + local attr=attributes(full) + local mode=attr.mode + if mode=='file' then + if find(full,patt) then + result[name]=attr + end + elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then + attr.list=collectpattern(full,patt,recurse) + result[name]=attr + end + end + end + return result end - --- - -local pattern = Cs { "start", - start = V("one") + V("two") + V("three"), - rest = (Cc(",") * V("thousand"))^0 * (P(".") + endofstring) * anything^0, - thousand = digit * digit * digit, - one = digit * V("rest"), - two = digit * digit * V("rest"), - three = V("thousand") * V("rest"), +dir.collectpattern=collectpattern +local pattern=Ct { + [1]=(C(P(".")+P("/")^1)+C(R("az","AZ")*P(":")*P("/")^0)+Cc("./"))*V(2)*V(3), + [2]=C(((1-S("*?/"))^0*P("/"))^0), + [3]=C(P(1)^0) } - -patterns.splitthousands = pattern -- maybe better in the parsers namespace ? - -function parsers.splitthousands(str) - return lpegmatch(pattern,str) or str -end - --- print(parsers.splitthousands("11111111111.11")) - -local optionalwhitespace = whitespace^0 - -patterns.words = Ct((Cs((1-punctuation-whitespace)^1) + anything)^1) -patterns.sentences = Ct((optionalwhitespace * Cs((1-period)^0 * period))^1) -patterns.paragraphs = Ct((optionalwhitespace * Cs((whitespace^1*endofstring/"" + 1 - (spacer^0*newline*newline))^1))^1) - --- local str = " Word1 word2. \n Word3 word4. \n\n Word5 word6.\n " --- inspect(lpegmatch(patterns.paragraphs,str)) --- inspect(lpegmatch(patterns.sentences,str)) --- inspect(lpegmatch(patterns.words,str)) - --- handy for k="v" [, ] k="v" - -local dquote = P('"') -local equal = P('=') -local escape = P('\\') -local separator = S(' ,') - -local key = C((1-equal)^1) -local value = dquote * C((1-dquote-escape*dquote)^0) * dquote - -local pattern = Cf(Ct("") * Cg(key * equal * value) * separator^0,rawset)^0 - -parsers.patterns.keq_to_hash_c = pattern - -function parsers.keq_to_hash(str) - if str and str ~= "" then - return lpegmatch(pattern,str) +local filter=Cs (( + P("**")/".*"+P("*")/"[^/]*"+P("?")/"[^/]"+P(".")/"%%."+P("+")/"%%+"+P("-")/"%%-"+P(1) +)^0 ) +local function glob(str,t) + if type(t)=="function" then + if type(str)=="table" then + for s=1,#str do + glob(str[s],t) + end + elseif isfile(str) then + t(str) + else + local split=lpegmatch(pattern,str) + if split then + local root,path,base=split[1],split[2],split[3] + local recurse=find(base,"%*%*") + local start=root..path + local result=lpegmatch(filter,start..base) + globpattern(start,result,recurse,t) + end + end + else + if type(str)=="table" then + local t=t or {} + for s=1,#str do + glob(str[s],t) + end + return t + elseif isfile(str) then + if t then + t[#t+1]=str + return t + else + return { str } + end else - return { } + local split=lpegmatch(pattern,str) + if split then + local t=t or {} + local action=action or function(name) t[#t+1]=name end + local root,path,base=split[1],split[2],split[3] + local recurse=find(base,"%*%*") + local start=root..path + local result=lpegmatch(filter,start..base) + globpattern(start,result,recurse,action) + return t + else + return {} + end + end + end +end +dir.glob=glob +local function globfiles(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if recurse then + globfiles(path.."/"..name,recurse,func,files) + end + elseif mode=="file" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name + end + end end + end + return files end - --- inspect(lpeg.match(pattern,[[key="value"]])) - -local defaultspecification = { separator = ",", quote = '"' } - --- this version accepts multiple separators and quotes as used in the --- database module - -function parsers.csvsplitter(specification) - specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification - local separator = specification.separator - local quotechar = specification.quote - local separator = S(separator ~= "" and separator or ",") - local whatever = C((1 - separator - newline)^0) - if quotechar and quotechar ~= "" then - local quotedata = nil - for chr in gmatch(quotechar,".") do - local quotechar = P(chr) - local quoteword = quotechar * C((1 - quotechar)^0) * quotechar - if quotedata then - quotedata = quotedata + quoteword - else - quotedata = quoteword - end +dir.globfiles=globfiles +function dir.ls(pattern) + return concat(glob(pattern),"\n") +end +local make_indeed=true +local onwindows=os.type=="windows" or find(os.getenv("PATH"),";") +if onwindows then + function dir.mkdirs(...) + local str,pth="","" + for i=1,select("#",...) do + local s=select(i,...) + if s=="" then + elseif str=="" then + str=s + else + str=str.."/"..s + end + end + local first,middle,last + local drive=false + first,middle,last=match(str,"^(//)(//*)(.*)$") + if first then + else + first,last=match(str,"^(//)/*(.-)$") + if first then + middle,last=match(str,"([^/]+)/+(.-)$") + if middle then + pth="//"..middle + else + pth="//"..last + last="" end - whatever = quotedata + whatever + else + first,middle,last=match(str,"^([a-zA-Z]:)(/*)(.-)$") + if first then + pth,drive=first..middle,true + else + middle,last=match(str,"^(/*)(.-)$") + if not middle then + last=str + end + end + end end - local parser = Ct((Ct(whatever * (separator * whatever)^0) * S("\n\r"))^0 ) - return function(data) - return lpegmatch(parser,data) + for s in gmatch(last,"[^/]+") do + if pth=="" then + pth=s + elseif drive then + pth,drive=pth..s,false + else + pth=pth.."/"..s + end + if make_indeed and not isdir(pth) then + lfs.mkdir(pth) + end end -end - --- and this is a slightly patched version of a version posted by Philipp Gesang - --- local mycsvsplitter = utilities.parsers.rfc4180splitter() --- --- local crap = [[ --- first,second,third,fourth --- "1","2","3","4" --- "a","b","c","d" --- "foo","bar""baz","boogie","xyzzy" --- ]] --- --- local list, names = mycsvsplitter(crap,true) inspect(list) inspect(names) --- local list, names = mycsvsplitter(crap) inspect(list) inspect(names) - -function parsers.rfc4180splitter(specification) - specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification - local separator = specification.separator --> rfc: COMMA - local quotechar = P(specification.quote) --> DQUOTE - local dquotechar = quotechar * quotechar --> 2DQUOTE - / specification.quote - local separator = S(separator ~= "" and separator or ",") - local escaped = quotechar - * Cs((dquotechar + (1 - quotechar))^0) - * quotechar - local non_escaped = C((1 - quotechar - newline - separator)^1) - local field = escaped + non_escaped - local record = Ct((field * separator^-1)^1) - local headerline = record * Cp() - local wholeblob = Ct((newline^-1 * record)^0) - return function(data,getheader) - if getheader then - local header, position = lpegmatch(headerline,data) - local data = lpegmatch(wholeblob,data,position) - return data, header + return pth,(isdir(pth)==true) + end +else + function dir.mkdirs(...) + local str,pth="","" + for i=1,select("#",...) do + local s=select(i,...) + if s and s~="" then + if str~="" then + str=str.."/"..s else - return lpegmatch(wholeblob,data) + str=s end + end end -end - --- utilities.parsers.stepper("1,7-",9,function(i) print(">>>",i) end) --- utilities.parsers.stepper("1-3,7,8,9") --- utilities.parsers.stepper("1-3,6,7",function(i) print(">>>",i) end) --- utilities.parsers.stepper(" 1 : 3, ,7 ") --- utilities.parsers.stepper("1:4,9:13,24:*",30) - -local function ranger(first,last,n,action) - if not first then - -- forget about it - elseif last == true then - for i=first,n or first do - action(i) + str=gsub(str,"/+","/") + if find(str,"^/") then + pth="/" + for s in gmatch(str,"[^/]+") do + local first=(pth=="/") + if first then + pth=pth..s + else + pth=pth.."/"..s end - elseif last then - for i=first,last do - action(i) + if make_indeed and not first and not isdir(pth) then + lfs.mkdir(pth) end + end else - action(first) + pth="." + for s in gmatch(str,"[^/]+") do + pth=pth.."/"..s + if make_indeed and not isdir(pth) then + lfs.mkdir(pth) + end + end end + return pth,(isdir(pth)==true) + end end - -local cardinal = patterns.cardinal / tonumber -local spacers = patterns.spacer^0 -local endofstring = patterns.endofstring - -local stepper = spacers * ( C(cardinal) * ( spacers * S(":-") * spacers * ( C(cardinal) + Cc(true) ) + Cc(false) ) - * Carg(1) * Carg(2) / ranger * S(", ")^0 )^1 - -local stepper = spacers * ( C(cardinal) * ( spacers * S(":-") * spacers * ( C(cardinal) + (P("*") + endofstring) * Cc(true) ) + Cc(false) ) - * Carg(1) * Carg(2) / ranger * S(", ")^0 )^1 * endofstring -- we're sort of strict (could do without endofstring) - -function utilities.parsers.stepper(str,n,action) - if type(n) == "function" then - lpegmatch(stepper,str,1,false,n or print) +dir.makedirs=dir.mkdirs +if onwindows then + function dir.expandname(str) + local first,nothing,last=match(str,"^(//)(//*)(.*)$") + if first then + first=dir.current().."/" + end + if not first then + first,last=match(str,"^(//)/*(.*)$") + end + if not first then + first,last=match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d=currentdir() + if lfs.chdir(first) then + first=dir.current() + end + lfs.chdir(d) + end + end + if not first then + first,last=dir.current(),str + end + last=gsub(last,"//","/") + last=gsub(last,"/%./","/") + last=gsub(last,"^/*","") + first=gsub(first,"/*$","") + if last=="" or last=="." then + return first else - lpegmatch(stepper,str,1,n,action or print) + return first.."/"..last + end + end +else + function dir.expandname(str) + if not find(str,"^/") then + str=currentdir().."/"..str end + str=gsub(str,"//","/") + str=gsub(str,"/%./","/") + str=gsub(str,"(.)/%.$","%1") + return str + end +end +file.expandname=dir.expandname +local stack={} +function dir.push(newdir) + insert(stack,lfs.currentdir()) +end +function dir.pop() + local d=remove(stack) + if d then + lfs.chdir(d) + end + return d end @@ -7527,95 +3419,71 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-fmt'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or { } -utilities.formatters = utilities.formatters or { } -local formatters = utilities.formatters +-- original size: 1822, stripped down to: 1544 -local concat, format = table.concat, string.format -local tostring, type = tostring, type -local strip = string.strip - -local P, R, Cs = lpeg.P, lpeg.R, lpeg.Cs -local lpegmatch = lpeg.match - --- temporary here - -local digit = R("09") -local period = P(".") -local zero = P("0") -local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") -local number = digit^1 * (case_1 + case_2) -local stripper = Cs((number + 1)^0) - - -lpeg.patterns.stripzeros = stripper - -function formatters.stripzeros(str) - return lpegmatch(stripper,str) +if not modules then modules={} end modules ['l-boolean']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,tonumber=type,tonumber +boolean=boolean or {} +local boolean=boolean +function boolean.tonumber(b) + if b then return 1 else return 0 end end - -function formatters.formatcolumns(result,between) - if result and #result > 0 then - between = between or " " - local widths, numbers = { }, { } - local first = result[1] - local n = #first - for i=1,n do - widths[i] = 0 - end - for i=1,#result do - local r = result[i] - for j=1,n do - local rj = r[j] - local tj = type(rj) - if tj == "number" then - numbers[j] = true - end - if tj ~= "string" then - rj = tostring(rj) - r[j] = rj - end - local w = #rj - if w > widths[j] then - widths[j] = w - end - end - end - for i=1,n do - local w = widths[i] - if numbers[i] then - if w > 80 then - widths[i] = "%s" .. between - else - widths[i] = "%0" .. w .. "i" .. between - end - else - if w > 80 then - widths[i] = "%s" .. between - elseif w > 0 then - widths[i] = "%-" .. w .. "s" .. between - else - widths[i] = "%s" - end - end - end - local template = strip(concat(widths)) - for i=1,#result do - local str = format(template,unpack(result[i])) - result[i] = strip(str) - end +function toboolean(str,tolerant) + if str==nil then + return false + elseif str==false then + return false + elseif str==true then + return true + elseif str=="true" then + return true + elseif str=="false" then + return false + elseif not tolerant then + return false + elseif str==0 then + return false + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +string.toboolean=toboolean +function string.booleanstring(str) + if str==nil then + return false + elseif str==false then + return false + elseif str==true then + return true + elseif str=="true" then + return true + elseif str=="false" then + return false + elseif str==0 then + return false + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +function string.is_boolean(str,default) + if type(str)=="string" then + if str=="true" or str=="yes" or str=="on" or str=="t" then + return true + elseif str=="false" or str=="no" or str=="off" or str=="f" then + return false end - return result + end + return default end @@ -7623,136 +3491,468 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-deb'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- the tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) - -local debug = require "debug" - -local getinfo = debug.getinfo -local type, next, tostring = type, next, tostring -local format, find = string.format, string.find -local is_boolean = string.is_boolean - -utilities = utilities or { } -utilities.debugger = utilities.debugger or { } -local debugger = utilities.debugger +-- original size: 24092, stripped down to: 11311 -local counters = { } -local names = { } - --- one - -local function hook() - local f = getinfo(2) -- "nS" - if f then - local n = "unknown" - if f.what == "C" then - n = f.name or '' - if not names[n] then - names[n] = format("%42s",n) - end +if not modules then modules={} end modules ['l-unicode']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utf=utf or (unicode and unicode.utf8) or {} +utf.characters=utf.characters or string.utfcharacters +utf.values=utf.values or string.utfvalues +local type=type +local char,byte,format,sub=string.char,string.byte,string.format,string.sub +local concat=table.concat +local P,C,R,Cs,Ct,Cmt,Cc,Carg=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Ct,lpeg.Cmt,lpeg.Cc,lpeg.Carg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local bytepairs=string.bytepairs +local finder=lpeg.finder +local replacer=lpeg.replacer +local utfvalues=utf.values +local utfgmatch=utf.gmatch +local p_utftype=patterns.utftype +local p_utfoffset=patterns.utfoffset +local p_utf8char=patterns.utf8char +local p_utf8byte=patterns.utf8byte +local p_utfbom=patterns.utfbom +local p_newline=patterns.newline +local p_whitespace=patterns.whitespace +if not unicode then + unicode={ utf=utf } +end +if not utf.char then + local floor,char=math.floor,string.char + function utf.char(n) + if n<0x80 then + return char(n) + elseif n<0x800 then + return char( + 0xC0+floor(n/0x40), + 0x80+(n%0x40) + ) + elseif n<0x10000 then + return char( + 0xE0+floor(n/0x1000), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) + elseif n<0x200000 then + return char( + 0xF0+floor(n/0x40000), + 0x80+(floor(n/0x1000)%0x40), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) + else + return "" + end + end +end +if not utf.byte then + local utf8byte=patterns.utf8byte + function utf.byte(c) + return lpegmatch(utf8byte,c) + end +end +local utfchar,utfbyte=utf.char,utf.byte +function utf.filetype(data) + return data and lpegmatch(p_utftype,data) or "unknown" +end +local toentities=Cs ( + ( + patterns.utf8one+( + patterns.utf8two+patterns.utf8three+patterns.utf8four + )/function(s) local b=utfbyte(s) if b<127 then return s else return format("&#%X;",b) end end + )^0 +) +patterns.toentities=toentities +function utf.toentities(str) + return lpegmatch(toentities,str) +end +local one=P(1) +local two=C(1)*C(1) +local four=C(R(utfchar(0xD8),utfchar(0xFF)))*C(1)*C(1)*C(1) +local pattern=P("\254\255")*Cs(( + four/function(a,b,c,d) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(a,b) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 )+P("\255\254")*Cs(( + four/function(b,a,d,c) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(b,a) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 ) +function string.toutf(s) + return lpegmatch(pattern,s) or s +end +local validatedutf=Cs ( + ( + patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four+P(1)/"�" + )^0 +) +patterns.validatedutf=validatedutf +function utf.is_valid(str) + return type(str)=="string" and lpegmatch(validatedutf,str) or false +end +if not utf.len then + local n,f=0,1 + local utfcharcounter=patterns.utfbom^-1*Cmt ( + Cc(1)*patterns.utf8one^1+Cc(2)*patterns.utf8two^1+Cc(3)*patterns.utf8three^1+Cc(4)*patterns.utf8four^1, + function(_,t,d) + n=n+(t-f)/d + f=t + return true + end + )^0 + function utf.len(str) + n,f=0,1 + lpegmatch(utfcharcounter,str or "") + return n + end +end +utf.length=utf.len +if not utf.sub then + local utflength=utf.length + local b,e,n,first,last=0,0,0,0,0 + local function slide_zero(s,p) + n=n+1 + if n>=last then + e=p-1 + else + return p + end + end + local function slide_one(s,p) + n=n+1 + if n==first then + b=p + end + if n>=last then + e=p-1 + else + return p + end + end + local function slide_two(s,p) + n=n+1 + if n==first then + b=p + else + return true + end + end + local pattern_zero=Cmt(p_utf8char,slide_zero)^0 + local pattern_one=Cmt(p_utf8char,slide_one )^0 + local pattern_two=Cmt(p_utf8char,slide_two )^0 + function utf.sub(str,start,stop) + if not start then + return str + end + if start==0 then + start=1 + end + if not stop then + if start<0 then + local l=utflength(str) + start=l+start + else + start=start-1 + end + b,n,first=0,0,start + lpegmatch(pattern_two,str) + if n>=first then + return sub(str,b) + else + return "" + end + end + if start<0 or stop<0 then + local l=utf.length(str) + if start<0 then + start=l+start + if start<=0 then + start=1 else - -- source short_src linedefined what name namewhat nups func - n = f.name or f.namewhat or f.what - if not n or n == "" then - n = "?" - end - if not names[n] then - names[n] = format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") - end + start=start+1 end - counters[n] = (counters[n] or 0) + 1 - end -end - -function debugger.showstats(printer,threshold) -- hm, something has changed, rubish now - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - local dataset = { } - for name, count in next, counters do - dataset[#dataset+1] = { name, count } - end - table.sort(dataset,function(a,b) return a[2] == b[2] and b[1] > a[1] or a[2] > b[2] end) - for i=1,#dataset do - local d = dataset[i] - local name = d[1] - local count = d[2] - if count > threshold and not find(name,"for generator") then -- move up - printer(format("%8i %s\n", count, names[name])) - total = total + count + end + if stop<0 then + stop=l+stop + if stop==0 then + stop=1 + else + stop=stop+1 end - grandtotal = grandtotal + count - functions = functions + 1 + end + end + if start>stop then + return "" + elseif start>1 then + b,e,n,first,last=0,0,0,start-1,stop + lpegmatch(pattern_one,str) + if n>=first and e==0 then + e=#str + end + return sub(str,b,e) + else + b,e,n,last=1,0,0,stop + lpegmatch(pattern_zero,str) + if e==0 then + e=#str + end + return sub(str,b,e) end - printer("\n") - printer(format("functions : % 10i\n", functions)) - printer(format("total : % 10i\n", total)) - printer(format("grand total: % 10i\n", grandtotal)) - printer(format("threshold : % 10i\n", threshold)) + end end - -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() +function utf.remapper(mapping) + local pattern=Cs((p_utf8char/mapping)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern +end +function utf.replacer(t) + local r=replacer(t,false,false,true) + return function(str) + return lpegmatch(r,str) + end +end +function utf.subtituter(t) + local f=finder (t) + local r=replacer(t,false,false,true) + return function(str) + local i=lpegmatch(f,str) + if not i then + return str + elseif i>#str then + return str + else + return lpegmatch(r,str) + end + end +end +local utflinesplitter=p_utfbom^-1*lpeg.tsplitat(p_newline) +local utfcharsplitter_ows=p_utfbom^-1*Ct(C(p_utf8char)^0) +local utfcharsplitter_iws=p_utfbom^-1*Ct((p_whitespace^1+C(p_utf8char))^0) +local utfcharsplitter_raw=Ct(C(p_utf8char)^0) +patterns.utflinesplitter=utflinesplitter +function utf.splitlines(str) + return lpegmatch(utflinesplitter,str or "") +end +function utf.split(str,ignorewhitespace) + if ignorewhitespace then + return lpegmatch(utfcharsplitter_iws,str or "") + else + return lpegmatch(utfcharsplitter_ows,str or "") + end +end +function utf.totable(str) + return lpegmatch(utfcharsplitter_raw,str) +end +function utf.magic(f) + local str=f:read(4) or "" + local off=lpegmatch(p_utfoffset,str) + if off<4 then + f:seek('set',off) + end + return lpegmatch(p_utftype,str) +end +local function utf16_to_utf8_be(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,0 + for left,right in bytepairs(t[i]) do + if right then + local now=256*left+right + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + r=r+1 + result[r]=utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + else + r=r+1 + result[r]=utfchar(now) + end + end end + t[i]=concat(result,"",1,r) + end + return t end - -function debugger.enable() - debug.sethook(hook,"c") +local function utf16_to_utf8_le(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,0 + for left,right in bytepairs(t[i]) do + if right then + local now=256*right+left + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + r=r+1 + result[r]=utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + else + r=r+1 + result[r]=utfchar(now) + end + end + end + t[i]=concat(result,"",1,r) + end + return t +end +local function utf32_to_utf8_be(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,-1 + for a,b in bytepairs(t[i]) do + if a and b then + if more<0 then + more=256*256*256*a+256*256*b + else + r=r+1 + result[t]=utfchar(more+256*a+b) + more=-1 + end + else + break + end + end + t[i]=concat(result,"",1,r) + end + return t +end +local function utf32_to_utf8_le(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,-1 + for a,b in bytepairs(t[i]) do + if a and b then + if more<0 then + more=256*b+a + else + r=r+1 + result[t]=utfchar(more+256*256*256*b+256*256*a) + more=-1 + end + else + break + end + end + t[i]=concat(result,"",1,r) + end + return t +end +utf.utf32_to_utf8_be=utf32_to_utf8_be +utf.utf32_to_utf8_le=utf32_to_utf8_le +utf.utf16_to_utf8_be=utf16_to_utf8_be +utf.utf16_to_utf8_le=utf16_to_utf8_le +function utf.utf8_to_utf8(t) + return type(t)=="string" and lpegmatch(utflinesplitter,t) or t +end +function utf.utf16_to_utf8(t,endian) + return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +end +function utf.utf32_to_utf8(t,endian) + return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +end +local function little(c) + local b=byte(c) + if b<0x10000 then + return char(b%256,b/256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end +end +local function big(c) + local b=byte(c) + if b<0x10000 then + return char(b/256,b%256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end +end +local _,l_remap=utf.remapper(little) +local _,b_remap=utf.remapper(big) +function utf.utf8_to_utf16(str,littleendian) + if littleendian then + return char(255,254)..lpegmatch(l_remap,str) + else + return char(254,255)..lpegmatch(b_remap,str) + end +end +local pattern=Cs ( + (p_utf8byte/function(unicode ) return format("0x%04X",unicode) end)*(p_utf8byte*Carg(1)/function(unicode,separator) return format("%s0x%04X",separator,unicode) end)^0 +) +function utf.tocodes(str,separator) + return lpegmatch(pattern,str,1,separator or " ") end - -function debugger.disable() - debug.sethook() +function utf.ustring(s) + return format("U+%05X",type(s)=="number" and s or utfbyte(s)) end - - - - - -local is_node = node and node.is_node -local is_lpeg = lpeg and lpeg.type - -function inspect(i) -- global function - local ti = type(i) - if ti == "table" then - table.print(i,"table") - elseif is_node and is_node(i) then - table.print(nodes.astable(i),tostring(i)) - elseif is_lpeg and is_lpeg(i) then - lpeg.print(i) - else - print(tostring(i)) - end - return i -- so that we can inline the inspect +function utf.xstring(s) + return format("0x%05X",type(s)=="number" and s or utfbyte(s)) end - --- from the lua book: - -function traceback() - local level = 1 - while true do - local info = debug.getinfo(level, "Sl") - if not info then - break - elseif info.what == "C" then - print(format("%3i : C function",level)) - else - print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) - end - level = level + 1 +local p_nany=p_utf8char/"" +if utfgmatch then + function utf.count(str,what) + if type(what)=="string" then + local n=0 + for _ in utfgmatch(str,what) do + n=n+1 + end + return n + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) + end + end +else + local cache={} + function utf.count(str,what) + if type(what)=="string" then + local p=cache[what] + if not p then + p=Cs((P(what)/" "+p_nany)^0) + cache[p]=p + end + return #lpegmatch(p,str) + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) end + end end @@ -7760,198 +3960,473 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-inf'] = { - version = 1.001, - comment = "companion to trac-inf.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- As we want to protect the global tables, we no longer store the timing --- in the tables themselves but in a hidden timers table so that we don't --- get warnings about assignments. This is more efficient than using rawset --- and rawget. +-- original size: 915, stripped down to: 836 -local format, lower = string.format, string.lower -local clock = os.gettimeofday or os.clock -- should go in environment -local write_nl = texio and texio.write_nl or print +if not modules then modules={} end modules ['l-math']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local floor,sin,cos,tan=math.floor,math.sin,math.cos,math.tan +if not math.round then + function math.round(x) return floor(x+0.5) end +end +if not math.div then + function math.div(n,m) return floor(n/m) end +end +if not math.mod then + function math.mod(n,m) return n%m end +end +local pipi=2*math.pi/360 +if not math.sind then + function math.sind(d) return sin(d*pipi) end + function math.cosd(d) return cos(d*pipi) end + function math.tand(d) return tan(d*pipi) end +end +if not math.odd then + function math.odd (n) return n%2~=0 end + function math.even(n) return n%2==0 end +end -statistics = statistics or { } -local statistics = statistics -statistics.enable = true -statistics.threshold = 0.01 +end -- of closure -local statusinfo, n, registered, timers = { }, 0, { }, { } +do -- create closure to overcome 200 locals limit -table.setmetatableindex(timers,function(t,k) - local v = { timing = 0, loadtime = 0 } - t[k] = v - return v -end) +-- original size: 10334, stripped down to: 6756 -local function hastiming(instance) - return instance and timers[instance] +if not modules then modules={} end modules ['util-tab']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.tables=utilities.tables or {} +local tables=utilities.tables +local format,gmatch,rep,gsub=string.format,string.gmatch,string.rep,string.gsub +local concat,insert,remove=table.concat,table.insert,table.remove +local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local type,next,rawset,tonumber,load,select=type,next,rawset,tonumber,load,select +local lpegmatch,P,Cs=lpeg.match,lpeg.P,lpeg.Cs +local serialize=table.serialize +local splitter=lpeg.tsplitat(".") +function tables.definetable(target,nofirst,nolast) + local composed,shortcut,t=nil,nil,{} + local snippets=lpegmatch(splitter,target) + for i=1,#snippets-(nolast and 1 or 0) do + local name=snippets[i] + if composed then + composed=shortcut.."."..name + shortcut=shortcut.."_"..name + t[#t+1]=format("local %s = %s if not %s then %s = { } %s = %s end",shortcut,composed,shortcut,shortcut,composed,shortcut) + else + composed=name + shortcut=name + if not nofirst then + t[#t+1]=format("%s = %s or { }",composed,composed) + end + end + end + if nolast then + composed=shortcut.."."..snippets[#snippets] + end + return concat(t,"\n"),composed end - -local function resettiming(instance) - timers[instance or "notimer"] = { timing = 0, loadtime = 0 } +function tables.definedtable(...) + local t=_G + for i=1,select("#",...) do + local li=select(i,...) + local tl=t[li] + if not tl then + tl={} + t[li]=tl + end + t=tl + end + return t end - -local function starttiming(instance) - local timer = timers[instance or "notimer"] - local it = timer.timing or 0 - if it == 0 then - timer.starttime = clock() - if not timer.loadtime then - timer.loadtime = 0 - end +function tables.accesstable(target,root) + local t=root or _G + for name in gmatch(target,"([^%.]+)") do + t=t[name] + if not t then + return end - timer.timing = it + 1 + end + return t end - -local function stoptiming(instance, report) - local timer = timers[instance or "notimer"] - local it = timer.timing - if it > 1 then - timer.timing = it - 1 - else - local starttime = timer.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - timer.stoptime = stoptime - timer.loadtime = timer.loadtime + loadtime - if report then - statistics.report("load time %0.3f",loadtime) - end - timer.timing = 0 - return loadtime - end +function tables.migratetable(target,v,root) + local t=root or _G + local names=string.split(target,".") + for i=1,#names-1 do + local name=names[i] + t[name]=t[name] or {} + t=t[name] + if not t then + return end - return 0 + end + t[names[#names]]=v end - -local function elapsedtime(instance) - local timer = timers[instance or "notimer"] - return format("%0.3f",timer and timer.loadtime or 0) +function tables.removevalue(t,value) + if value then + for i=1,#t do + if t[i]==value then + remove(t,i) + end + end + end end - -local function elapsedindeed(instance) - local timer = timers[instance or "notimer"] - return (timer and timer.loadtime or 0) > statistics.threshold +function tables.insertbeforevalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) end - -local function elapsedseconds(instance,rest) -- returns nil if 0 seconds - if elapsedindeed(instance) then - return format("%s seconds %s", elapsedtime(instance),rest or "") +function tables.insertaftervalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i+1,extra) + return end + end + insert(t,#t+1,extra) end - -statistics.hastiming = hastiming -statistics.resettiming = resettiming -statistics.starttiming = starttiming -statistics.stoptiming = stoptiming -statistics.elapsedtime = elapsedtime -statistics.elapsedindeed = elapsedindeed -statistics.elapsedseconds = elapsedseconds - --- general function .. we might split this module - -function statistics.register(tag,fnc) - if statistics.enable and type(fnc) == "function" then - local rt = registered[tag] or (#statusinfo + 1) - statusinfo[rt] = { tag, fnc } - registered[tag] = rt - if #tag > n then n = #tag end +local function toxml(t,d,result,step) + for k,v in table.sortedpairs(t) do + if type(v)=="table" then + if type(k)=="number" then + result[#result+1]=format("%s",d,k) + toxml(v,d..step,result,step) + result[#result+1]=format("%s",d,k) + else + result[#result+1]=format("%s<%s>",d,k) + toxml(v,d..step,result,step) + result[#result+1]=format("%s",d,k) + end + elseif type(k)=="number" then + result[#result+1]=format("%s%s",d,k,v,k) + else + result[#result+1]=format("%s<%s>%s",d,k,tostring(v),k) end + end end - -function statistics.show(reporter) - if statistics.enable then - if not reporter then reporter = function(tag,data,n) write_nl(tag .. " " .. data) end end - -- this code will move - local register = statistics.register - register("luatex banner", function() - return lower(status.banner) - end) - register("control sequences", function() - return format("%s of %s + %s", status.cs_count, status.hash_size,status.hash_extra) - end) - register("callbacks", function() - local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 - return format("%s direct, %s indirect, %s total", total-indirect, indirect, total) - end) - collectgarbage("collect") - register("current memory usage", statistics.memused) - register("runtime",statistics.runtime) - for i=1,#statusinfo do - local s = statusinfo[i] - local r = s[2]() - if r then - reporter(s[1],r,n) - end +function table.toxml(t,name,nobanner,indent,spaces) + local noroot=name==false + local result=(nobanner or noroot) and {} or { "" } + local indent=rep(" ",indent or 0) + local spaces=rep(" ",spaces or 1) + if noroot then + toxml(t,inndent,result,spaces) + else + toxml({ [name or "root"]=t },indent,result,spaces) + end + return concat(result,"\n") +end +function tables.encapsulate(core,capsule,protect) + if type(capsule)~="table" then + protect=true + capsule={} + end + for key,value in next,core do + if capsule[key] then + print(format("\ninvalid inheritance '%s' in '%s': %s",key,tostring(core))) + os.exit() + else + capsule[key]=value + end + end + if protect then + for key,value in next,core do + core[key]=nil + end + setmetatable(core,{ + __index=capsule, + __newindex=function(t,key,value) + if capsule[key] then + print(format("\ninvalid overload '%s' in '%s'",key,tostring(core))) + os.exit() + else + rawset(t,key,value) end - write_nl("") -- final newline - statistics.enable = false + end + } ) + end +end +local function fastserialize(t,r,outer) + r[#r+1]="{" + local n=#t + if n>0 then + for i=1,n do + local v=t[i] + local tv=type(v) + if tv=="string" then + r[#r+1]=format("%q,",v) + elseif tv=="number" then + r[#r+1]=format("%s,",v) + elseif tv=="table" then + fastserialize(v,r) + elseif tv=="boolean" then + r[#r+1]=format("%s,",tostring(v)) + end + end + else + for k,v in next,t do + local tv=type(v) + if tv=="string" then + r[#r+1]=format("[%q]=%q,",k,v) + elseif tv=="number" then + r[#r+1]=format("[%q]=%s,",k,v) + elseif tv=="table" then + r[#r+1]=format("[%q]=",k) + fastserialize(v,r) + elseif tv=="boolean" then + r[#r+1]=format("[%q]=%s,",k,tostring(v)) + end end + end + if outer then + r[#r+1]="}" + else + r[#r+1]="}," + end + return r end - -local template, report_statistics, nn = nil, nil, 0 -- we only calcute it once - -function statistics.showjobstat(tag,data,n) - if not logs then - -- sorry - elseif type(data) == "table" then - for i=1,#data do - statistics.showjobstat(tag,data[i],n) - end - else - if not template or n > nn then - template, n = format("%%-%ss - %%s",n), nn - report_statistics = logs.reporter("mkiv lua stats") +function table.fastserialize(t,prefix) + return concat(fastserialize(t,{ prefix or "return" },true)) +end +function table.deserialize(str) + if not str or str=="" then + return + end + local code=load(str) + if not code then + return + end + code=code() + if not code then + return + end + return code +end +function table.load(filename) + if filename then + local t=io.loaddata(filename) + if t and t~="" then + t=load(t) + if type(t)=="function" then + t=t() + if type(t)=="table" then + return t end - report_statistics(format(template,tag,data)) + end end + end end - -function statistics.memused() -- no math.round yet -) - local round = math.round or math.floor - return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) +function table.save(filename,t,n,...) + io.savedata(filename,serialize(t,n==nil and true or n,...)) end - -starttiming(statistics) - -function statistics.formatruntime(runtime) -- indirect so it can be overloaded and - return format("%s seconds", runtime) -- indeed that happens in cure-uti.lua +local function slowdrop(t) + local r={} + local l={} + for i=1,#t do + local ti=t[i] + local j=0 + for k,v in next,ti do + j=j+1 + l[j]=format("%s=%q",k,v) + end + r[i]=format(" {%s},\n",concat(l)) + end + return format("return {\n%s}",concat(r)) end - -function statistics.runtime() - stoptiming(statistics) - return statistics.formatruntime(elapsedtime(statistics)) +local function fastdrop(t) + local r={ "return {\n" } + for i=1,#t do + local ti=t[i] + r[#r+1]=" {" + for k,v in next,ti do + r[#r+1]=format("%s=%q",k,v) + end + r[#r+1]="},\n" + end + r[#r+1]="}" + return concat(r) end - -function statistics.timed(action,report) - report = report or logs.reporter("system") - starttiming("run") - action() - stoptiming("run") - report("total runtime: %s",elapsedtime("run")) +function table.drop(t,slow) + if #t==0 then + return "return { }" + elseif slow==true then + return slowdrop(t) + else + return fastdrop(t) + end +end +function table.autokey(t,k) + local v={} + t[k]=v + return v end --- where, not really the best spot for this: -commands = commands or { } +end -- of closure + +do -- create closure to overcome 200 locals limit -function commands.resettimer(name) - resettiming(name or "whatever") - starttiming(name or "whatever") -end +-- original size: 4270, stripped down to: 2989 -function commands.elapsedtime(name) - stoptiming(name or "whatever") - context(elapsedtime(name or "whatever")) +if not modules then modules={} end modules ['util-sto']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local setmetatable,getmetatable=setmetatable,getmetatable +utilities=utilities or {} +utilities.storage=utilities.storage or {} +local storage=utilities.storage +local report=texio and texio.write_nl or print +function storage.mark(t) + if not t then + report("fatal error: storage cannot be marked") + return + end + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.allocate(t) + t=t or {} + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.marked(t) + local m=getmetatable(t) + return m and m.__storage__ +end +function storage.checked(t) + if not t then + report("fatal error: storage has not been allocated") + return + end + return t +end +function storage.setinitializer(data,initialize) + local m=getmetatable(data) or {} + m.__index=function(data,k) + m.__index=nil + initialize() + return data[k] + end + setmetatable(data,m) +end +local keyisvalue={ __index=function(t,k) + t[k]=k + return k +end } +function storage.sparse(t) + t=t or {} + setmetatable(t,keyisvalue) + return t +end +local function f_empty () return "" end +local function f_self (t,k) t[k]=k return k end +local function f_table (t,k) local v={} t[k]=v return v end +local function f_ignore() end +local t_empty={ __index=f_empty } +local t_self={ __index=f_self } +local t_table={ __index=f_table } +local t_ignore={ __newindex=f_ignore } +function table.setmetatableindex(t,f) + local m=getmetatable(t) + if m then + if f=="empty" then + m.__index=f_empty + elseif f=="key" then + m.__index=f_self + elseif f=="table" then + m.__index=f_table + else + m.__index=f + end + else + if f=="empty" then + setmetatable(t,t_empty) + elseif f=="key" then + setmetatable(t,t_self) + elseif f=="table" then + setmetatable(t,t_table) + else + setmetatable(t,{ __index=f }) + end + end + return t +end +function table.setmetatablenewindex(t,f) + local m=getmetatable(t) + if m then + if f=="ignore" then + m.__newindex=f_ignore + else + m.__newindex=f + end + else + if f=="ignore" then + setmetatable(t,t_ignore) + else + setmetatable(t,{ __newindex=f }) + end + end + return t +end +function table.setmetatablecall(t,f) + local m=getmetatable(t) + if m then + m.__call=f + else + setmetatable(t,{ __call=f }) + end + return t +end +function table.setmetatablekey(t,key,value) + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m[key]=value + return t +end +function table.getmetatablekey(t,key,value) + local m=getmetatable(t) + return m and m[key] end @@ -7959,382 +4434,1165 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-set'] = { -- might become util-set.lua - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +-- original size: 11610, stripped down to: 7440 + +if not modules then modules={} end modules ['util-str']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.strings=utilities.strings or {} +local strings=utilities.strings +local load=load +local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local concat=table.concat +local P,V,C,S,R,Ct,Cs,Cp,Carg=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg +local patterns,lpegmatch=lpeg.patterns,lpeg.match +local utfchar,utfbyte=utf.char,utf.byte +local setmetatableindex=table.setmetatableindex +local stripper=patterns.stripzeros +local function points(n) + return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) +end +local function basepoints(n) + return (not n or n==0) and "0bp" or lpegmatch(stripper,format("%.5fbp",n*(7200/7227)/65536)) +end +number.points=points +number.basepoints=basepoints +local rubish=patterns.spaceortab^0*patterns.newline +local anyrubish=patterns.spaceortab+patterns.newline +local anything=patterns.anything +local stripped=(patterns.spaceortab^1/"")*patterns.newline +local leading=rubish^0/"" +local trailing=(anyrubish^1*patterns.endofstring)/"" +local redundant=rubish^3/"\n" +local pattern=Cs(leading*(trailing+redundant+stripped+anything)^0) +function strings.collapsecrlf(str) + return lpegmatch(pattern,str) +end +local repeaters={} +function strings.newrepeater(str,offset) + offset=offset or 0 + local s=repeaters[str] + if not s then + s={} + repeaters[str]=s + end + local t=s[offset] + if t then + return t + end + t={} + setmetatableindex(t,function(t,k) + if not k then + return "" + end + local n=k+offset + local s=n>0 and rep(str,n) or "" + t[k]=s + return s + end) + s[offset]=t + return t +end +local extra,tab,start=0,0,4,0 +local nspaces=strings.newrepeater(" ") +local pattern=Carg(1)/function(t) + extra,tab,start=0,t or 7,1 + end*Cs(( + Cp()*patterns.tab/function(position) + local current=(position-start+1)+extra + local spaces=tab-(current-1)%tab + if spaces>0 then + extra=extra+spaces-1 + return nspaces[spaces] + else + return "" + end + end+patterns.newline*Cp()/function(position) + extra,start=0,position + end+patterns.anything + )^1) +function strings.tabtospace(str,tab) + return lpegmatch(pattern,str,1,tab or 7) +end +function strings.striplong(str) + str=gsub(str,"^%s*","") + str=gsub(str,"[\n\r]+ *","\n") + return str +end +function strings.nice(str) + str=gsub(str,"[:%-+_]+"," ") + return str +end +local n=0 +local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_tab=C((1-R("az","AZ","09","%%"))^0) +local format_s=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%ss',(select(%s,...)))",f,n) + else + return format("(select(%s,...))",n) + end +end +local format_q=function() + n=n+1 + return format("format('%%q',(select(%s,...)))",n) +end +local format_i=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%si',(select(%s,...)))",f,n) + else + return format("(select(%s,...))",n) + end +end +local format_d=format_i +local format_f=function(f) + n=n+1 + return format("format('%%%sf',(select(%s,...)))",f,n) +end +local format_g=function(f) + n=n+1 + return format("format('%%%sg',(select(%s,...)))",f,n) +end +local format_G=function(f) + n=n+1 + return format("format('%%%sG',(select(%s,...)))",f,n) +end +local format_e=function(f) + n=n+1 + return format("format('%%%se',(select(%s,...)))",f,n) +end +local format_E=function(f) + n=n+1 + return format("format('%%%sE',(select(%s,...)))",f,n) +end +local format_x=function(f) + n=n+1 + return format("format('%%%sx',(select(%s,...)))",f,n) +end +local format_X=function(f) + n=n+1 + return format("format('%%%sX',(select(%s,...)))",f,n) +end +local format_o=function(f) + n=n+1 + return format("format('%%%so',(select(%s,...)))",f,n) +end +local format_c=function() + n=n+1 + return format("utfchar((select(%s,...)))",n) +end +local format_r=function(f) + n=n+1 + return format("format('%%%s.0f',(select(%s,...)))",f,n) +end +local format_v=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('0x%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_V=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('0x%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_u=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('u+%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_U=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('U+%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_p=function() + n=n+1 + return format("points((select(%s,...)))",n) +end +local format_b=function() + n=n+1 + return format("basepoints((select(%s,...)))",n) +end +local format_t=function(f) + n=n+1 + if f and f~="" then + return format("concat((select(%s,...)),%q)",n,f) + else + return format("concat((select(%s,...)))",n) + end +end +local format_l=function() + n=n+1 + return format("(select(%s,...) and 'true' or 'false')",n) +end +local format_a=function(s) + return format("%q",s) +end +local builder=Ct { "start", + start=(P("%")*( + V("s")+V("q")+V("i")+V("d")+V("f")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") ++V("c") ++V("r")+V("v")+V("V")+V("u")+V("U")+V("p")+V("b")+V("t")+V("l") + )+V("a") + )^0, + ["s"]=(prefix_any*P("s"))/format_s, + ["q"]=(prefix_any*P("q"))/format_q, + ["i"]=(prefix_any*P("i"))/format_i, + ["d"]=(prefix_any*P("d"))/format_d, + ["f"]=(prefix_any*P("f"))/format_f, + ["g"]=(prefix_any*P("g"))/format_g, + ["G"]=(prefix_any*P("G"))/format_G, + ["e"]=(prefix_any*P("e"))/format_e, + ["E"]=(prefix_any*P("E"))/format_E, + ["x"]=(prefix_any*P("x"))/format_x, + ["X"]=(prefix_any*P("X"))/format_X, + ["o"]=(prefix_any*P("o"))/format_o, + ["c"]=(prefix_any*P("c"))/format_c, + ["r"]=(prefix_any*P("r"))/format_r, + ["v"]=(prefix_any*P("v"))/format_v, + ["V"]=(prefix_any*P("V"))/format_V, + ["u"]=(prefix_any*P("u"))/format_u, + ["U"]=(prefix_any*P("U"))/format_U, + ["p"]=(prefix_any*P("p"))/format_p, + ["b"]=(prefix_any*P("b"))/format_b, + ["t"]=(prefix_tab*P("t"))/format_t, + ["l"]=(prefix_tab*P("l"))/format_l, + ["a"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_a, } +local template=[[ +local format = string.format +local concat = table.concat +local points = number.points +local basepoints = number.basepoints +local utfchar = utf.char +local utfbyte = utf.byte +return function(...) + return %s +end +]] +local function make(t,str) + n=0 + local p=lpegmatch(builder,str) + local c=format(template,concat(p,"..")) + formatter=load(c)() + t[str]=formatter + return formatter +end +local formatters=string.formatters or {} +string.formatters=formatters +setmetatableindex(formatters,make) +function string.makeformatter(str) + return formatters[str] +end +function string.formatter(str,...) + return formatters[str](...) +end --- maybe this should be util-set.lua -local type, next, tostring = type, next, tostring -local concat = table.concat -local format, find, lower, gsub, topattern = string.format, string.find, string.lower, string.gsub, string.topattern -local is_boolean = string.is_boolean -local settings_to_hash = utilities.parsers.settings_to_hash -local allocate = utilities.storage.allocate - -utilities = utilities or { } -local utilities = utilities -utilities.setters = utilities.setters or { } -local setters = utilities.setters - -local data = { } -- maybe just local - --- We can initialize from the cnf file. This is sort of tricky as --- later defined setters also need to be initialized then. If set --- this way, we need to ensure that they are not reset later on. - -local trace_initialize = false -- only for testing during development - -function setters.initialize(filename,name,values) -- filename only for diagnostics - local setter = data[name] - if setter then - frozen = true -- don't permitoverload --- trace_initialize = true - local data = setter.data - if data then - for key, newvalue in next, values do - local newvalue = is_boolean(newvalue,newvalue) - local functions = data[key] - if functions then - local oldvalue = functions.value - if functions.frozen then - if trace_initialize then - setter.report("%s: %q is frozen to %q",filename,key,tostring(oldvalue)) - end - elseif #functions > 0 and not oldvalue then --- elseif #functions > 0 and oldvalue == nil then - if trace_initialize then - setter.report("%s: %q is set to %q",filename,key,tostring(newvalue)) - end - for i=1,#functions do - functions[i](newvalue) - end - functions.value = newvalue - functions.frozen = functions.frozen or frozen - else - if trace_initialize then - setter.report("%s: %q is kept as %q",filename,key,tostring(oldvalue)) - end - end - else - -- we do a simple preregistration i.e. not in the - -- list as it might be an obsolete entry - functions = { default = newvalue, frozen = frozen } - data[key] = functions - if trace_initialize then - setter.report("%s: %q default to %q",filename,key,tostring(newvalue)) - end - end - end - return true - end +end -- of closure + +do -- create closure to overcome 200 locals limit + +-- original size: 7050, stripped down to: 5641 + +if not modules then modules={} end modules ['util-mrg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gsub,format=string.gsub,string.format +local concat=table.concat +local type,next=type,next +utilities=utilities or {} +local merger=utilities.merger or {} +utilities.merger=merger +utilities.report=logs and logs.reporter("system") or print +merger.strip_comment=true +local m_begin_merge="begin library merge" +local m_end_merge="end library merge" +local m_begin_closure="do -- create closure to overcome 200 locals limit" +local m_end_closure="end -- of closure" +local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" +local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" +local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" +local m_report=[[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] +local function self_fake() + return m_faked +end +local function self_nothing() + return "" +end +local function self_load(name) + local data=io.loaddata(name) or "" + if data=="" then + utilities.report("merge: unknown file %s",name) + else + utilities.report("merge: inserting %s",name) + end + return data or "" +end +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local equals=P("=")^0 +local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 +local close=P("]")*C(equals)*P("]") +local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) +local longstring=open*(1-closeeq)^0*close +local space=patterns.space +local eol=patterns.newline +local quoted=patterns.quoted +local emptyline=space^0*eol +local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") +local operator2=S("*+/") +local operator3=S("-") +local separator=S(",;") +local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" +local strings=quoted +local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" +local longstr=longstring +local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" +local pack=((eol+space)^0/"")*operator1*((eol+space)^0/"")+((eol+space)^0/"")*operator2*((space)^0/"")+((eol+space)^1/"")*operator3*((space)^1/"")+((space)^0/"")*separator*((space)^0/"") +local lines=emptyline^2/"\n" +local spaces=(space*space)/" " +local compact=Cs (( + ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 +)^1 ) +local strip=Cs((emptyline^2/"\n"+1)^0) +local function self_compact(data) + if merger.strip_comment then + local before=#data + data=lpeg.match(compact,data) + data=lpeg.match(strip,data) + local after=#data + local delta=before-after + utilities.report("merge: %s bytes compacted to %s (%s bytes stripped)",before,after,delta) + data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + return data,delta + else + return data,0 + end +end +local function self_save(name,data) + if data~="" then + io.savedata(name,data) + utilities.report("merge: saving %s bytes in %s",#data,name) + end +end +local function self_swap(data,code) + return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +end +local function self_libs(libs,list) + local result,f,frozen,foundpath={},nil,false,nil + result[#result+1]="\n" + if type(libs)=='string' then libs={ libs } end + if type(list)=='string' then list={ list } end + for i=1,#libs do + local lib=libs[i] + for j=1,#list do + local pth=gsub(list[j],"\\","/") + utilities.report("merge: checking library path %s",pth) + local name=pth.."/"..lib + if lfs.isfile(name) then + foundpath=pth + end end + if foundpath then break end + end + if foundpath then + utilities.report("merge: using library path %s",foundpath) + local right,wrong,original,stripped={},{},0,0 + for i=1,#libs do + local lib=libs[i] + local fullname=foundpath.."/"..lib + if lfs.isfile(fullname) then + utilities.report("merge: using library %s",fullname) + local data=io.loaddata(fullname,true) + original=original+#data + local data,delta=self_compact(data) + right[#right+1]=lib + result[#result+1]=m_begin_closure + result[#result+1]=data + result[#result+1]=m_end_closure + stripped=stripped+delta + else + utilities.report("merge: skipping library %s",fullname) + wrong[#wrong+1]=lib + end + end + right=#right>0 and concat(right," ") or "-" + wrong=#wrong>0 and concat(wrong," ") or "-" + utilities.report("merge: used libraries: %s",right) + utilities.report("merge: skipped libraries: %s",wrong) + utilities.report("merge: original bytes: %s",original) + utilities.report("merge: stripped bytes: %s",stripped) + result[#result+1]=format(m_report,right,wrong,original,stripped) + else + utilities.report("merge: no valid library path found") + end + return concat(result,"\n\n") +end +function merger.selfcreate(libs,list,target) + if target then + self_save(target,self_swap(self_fake(),self_libs(libs,list))) + end +end +function merger.selfmerge(name,libs,list,target) + self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) +end +function merger.selfclean(name) + self_save(name,self_swap(self_load(name),self_nothing())) end --- user interface code -local function set(t,what,newvalue) - local data = t.data - if not data.frozen then - local done = t.done - if type(what) == "string" then - what = settings_to_hash(what) -- inefficient but ok +end -- of closure + +do -- create closure to overcome 200 locals limit + +-- original size: 12211, stripped down to: 8441 + +if not modules then modules={} end modules ['util-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + comment="the strip code is written by Peter Cawley", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local rep,sub,byte,dump,format=string.rep,string.sub,string.byte,string.dump,string.format +local load,loadfile,type=load,loadfile,type +utilities=utilities or {} +utilities.lua=utilities.lua or {} +local luautilities=utilities.lua +utilities.report=logs and logs.reporter("system") or print +local tracestripping=false +local forcestupidcompile=true +luautilities.stripcode=true +luautilities.alwaysstripcode=false +luautilities.nofstrippedchunks=0 +luautilities.nofstrippedbytes=0 +local strippedchunks={} +luautilities.strippedchunks=strippedchunks +luautilities.suffixes={ + tma="tma", + tmc=jit and "tmb" or "tmc", + lua="lua", + luc=jit and "lub" or "luc", + lui="lui", + luv="luv", + luj="luj", + tua="tua", + tuc="tuc", +} +local function fatalerror(name) + utilities.report(format("fatal error in %q",name or "unknown")) +end +if jit or status.luatex_version>=74 then + local function register(name) + if tracestripping then + utilities.report("stripped bytecode: %s",name or "unknown") + end + strippedchunks[#strippedchunks+1]=name + luautilities.nofstrippedchunks=luautilities.nofstrippedchunks+1 + end + local function stupidcompile(luafile,lucfile,strip) + local code=io.loaddata(luafile) + if code and code~="" then + code=load(code) + if code then + code=dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) + if code and code~="" then + register(name) + io.savedata(lucfile,code) + return true,0 + end + else + fatalerror() + end + else + fatalerror() + end + return false,0 + end + function luautilities.loadedluacode(fullname,forcestrip,name) + name=name or fullname + local code=loadfile(fullname) + if code then + code() + end + if forcestrip and luautilities.stripcode then + if type(forcestrip)=="function" then + forcestrip=forcestrip(fullname) + end + if forcestrip or luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end + elseif luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end + end + function luautilities.strippedloadstring(code,forcestrip,name) + if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then + code=load(code) + if not code then + fatalerror(name) + end + register(name) + code=dump(code,true) + end + return load(code),0 + end + function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) + utilities.report("lua: compiling %s into %s",luafile,lucfile) + os.remove(lucfile) + local done=stupidcompile(luafile,lucfile,strip~=false) + if done then + utilities.report("lua: %s dumped into %s (stripped)",luafile,lucfile) + if cleanup==true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + utilities.report("lua: removing %s",luafile) + os.remove(luafile) + end + end + return done + end +else + local function register(name,before,after) + local delta=before-after + if tracestripping then + utilities.report("stripped bytecode: %s, before %s, after %s, delta %s",name or "unknown",before,after,delta) + end + strippedchunks[#strippedchunks+1]=name + luautilities.nofstrippedchunks=luautilities.nofstrippedchunks+1 + luautilities.nofstrippedbytes=luautilities.nofstrippedbytes+delta + return delta + end + local strip_code_pc + if _MAJORVERSION==5 and _MINORVERSION==1 then + strip_code_pc=function(dump,name) + local before=#dump + local version,format,endian,int,size,ins,num=byte(dump,5,11) + local subint + if endian==1 then + subint=function(dump,i,l) + local val=0 + for n=l,1,-1 do + val=val*256+byte(dump,i+n-1) + end + return val,i+l + end + else + subint=function(dump,i,l) + local val=0 + for n=1,l,1 do + val=val*256+byte(dump,i+n-1) + end + return val,i+l end - if type(what) ~= "table" then - return + end + local strip_function + strip_function=function(dump) + local count,offset=subint(dump,1,size) + local stripped,dirty=rep("\0",size),offset+count + offset=offset+count+int*2+4 + offset=offset+int+subint(dump,offset,int)*ins + count,offset=subint(dump,offset,int) + for n=1,count do + local t + t,offset=subint(dump,offset,1) + if t==1 then + offset=offset+1 + elseif t==4 then + offset=offset+size+subint(dump,offset,size) + elseif t==3 then + offset=offset+num + end end - if not done then -- catch ... why not set? - done = { } - t.done = done + count,offset=subint(dump,offset,int) + stripped=stripped..sub(dump,dirty,offset-1) + for n=1,count do + local proto,off=strip_function(sub(dump,offset,-1)) + stripped,offset=stripped..proto,offset+off-1 end - for w, value in next, what do - if value == "" then - value = newvalue - elseif not value then - value = false -- catch nil - else - value = is_boolean(value,value) - end - w = topattern(w,true,true) - for name, functions in next, data do - if done[name] then - -- prevent recursion due to wildcards - elseif find(name,w) then - done[name] = true - for i=1,#functions do - functions[i](value) - end - functions.value = value - end - end + offset=offset+subint(dump,offset,int)*int+int + count,offset=subint(dump,offset,int) + for n=1,count do + offset=offset+subint(dump,offset,size)+size+int*2 end - end -end - -local function reset(t) - local data = t.data - if not data.frozen then - for name, functions in next, data do - for i=1,#functions do - functions[i](false) - end - functions.value = false + count,offset=subint(dump,offset,int) + for n=1,count do + offset=offset+subint(dump,offset,size)+size end + stripped=stripped..rep("\0",int*3) + return stripped,offset + end + dump=sub(dump,1,12)..strip_function(sub(dump,13,-1)) + local after=#dump + local delta=register(name,before,after) + return dump,delta + end + else + strip_code_pc=function(dump,name) + return dump,0 + end + end + function luautilities.loadedluacode(fullname,forcestrip,name) + name=name or fullname + local code=loadfile(fullname) + if code then + code() + end + if forcestrip and luautilities.stripcode then + if type(forcestrip)=="function" then + forcestrip=forcestrip(fullname) + end + if forcestrip then + local code,n=strip_code_pc(dump(code),name) + return load(code),n + elseif luautilities.alwaysstripcode then + return load(strip_code_pc(dump(code),name)) + else + return code,0 + end + elseif luautilities.alwaysstripcode then + return load(strip_code_pc(dump(code),name)) + else + return code,0 + end + end + function luautilities.strippedloadstring(code,forcestrip,name) + local n=0 + if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then + code=load(code) + if not code then + fatalerror(name) + end + code,n=strip_code_pc(dump(code),name) + end + return load(code),n + end + local function stupidcompile(luafile,lucfile,strip) + local code=io.loaddata(luafile) + local n=0 + if code and code~="" then + code=load(code) + if not code then + fatalerror() + end + code=dump(code) + if strip then + code,n=strip_code_pc(code,luautilities.stripcode or luautilities.alwaysstripcode,luafile) + end + if code and code~="" then + io.savedata(lucfile,code) + end + end + return n + end + local luac_normal="texluac -o %q %q" + local luac_strip="texluac -s -o %q %q" + function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) + utilities.report("lua: compiling %s into %s",luafile,lucfile) + os.remove(lucfile) + local done=false + if strip~=false then + strip=true + end + if forcestupidcompile then + fallback=true + elseif strip then + done=os.spawn(format(luac_strip,lucfile,luafile))==0 + else + done=os.spawn(format(luac_normal,lucfile,luafile))==0 + end + if not done and fallback then + local n=stupidcompile(luafile,lucfile,strip) + if n>0 then + utilities.report("lua: %s dumped into %s (%i bytes stripped)",luafile,lucfile,n) + else + utilities.report("lua: %s dumped into %s (unstripped)",luafile,lucfile) + end + cleanup=false + done=true end + if done and cleanup==true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + utilities.report("lua: removing %s",luafile) + os.remove(luafile) + end + return done + end end -local function enable(t,what) - set(t,what,true) -end -local function disable(t,what) - local data = t.data - if not what or what == "" then - t.done = { } - reset(t) - else - set(t,what,false) - end -end +end -- of closure -function setters.register(t,what,...) - local data = t.data - what = lower(what) - local functions = data[what] - if not functions then - functions = { } - data[what] = functions - if trace_initialize then - t.report("defining %s",what) - end - end - local default = functions.default -- can be set from cnf file - for i=1,select("#",...) do - local fnc = select(i,...) - local typ = type(fnc) - if typ == "string" then - if trace_initialize then - t.report("coupling %s to %s",what,fnc) - end - local s = fnc -- else wrong reference - fnc = function(value) set(t,s,value) end - elseif typ ~= "function" then - fnc = nil - end - if fnc then - functions[#functions+1] = fnc - -- default: set at command line or in cnf file - -- value : set in tex run (needed when loading runtime) - local value = functions.value or default - if value ~= nil then - fnc(value) - functions.value = value - end - end - end - return false -- so we can use it in an assignment -end +do -- create closure to overcome 200 locals limit -function setters.enable(t,what) - local e = t.enable - t.enable, t.done = enable, { } - enable(t,what) - t.enable, t.done = e, { } -end +-- original size: 14514, stripped down to: 10374 -function setters.disable(t,what) - local e = t.disable - t.disable, t.done = disable, { } - disable(t,what) - t.disable, t.done = e, { } +if not modules then modules={} end modules ['util-prs']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local lpeg,table,string=lpeg,table,string +local P,R,V,S,C,Ct,Cs,Carg,Cc,Cg,Cf,Cp=lpeg.P,lpeg.R,lpeg.V,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cs,lpeg.Carg,lpeg.Cc,lpeg.Cg,lpeg.Cf,lpeg.Cp +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local concat,format,gmatch,find=table.concat,string.format,string.gmatch,string.find +local tostring,type,next,rawset=tostring,type,next,rawset +utilities=utilities or {} +utilities.parsers=utilities.parsers or {} +local parsers=utilities.parsers +parsers.patterns=parsers.patterns or {} +local setmetatableindex=table.setmetatableindex +local sortedhash=table.sortedhash +local digit=R("09") +local space=P(' ') +local equal=P("=") +local comma=P(",") +local lbrace=P("{") +local rbrace=P("}") +local lparent=P("(") +local rparent=P(")") +local period=S(".") +local punctuation=S(".,:;") +local spacer=patterns.spacer +local whitespace=patterns.whitespace +local newline=patterns.newline +local anything=patterns.anything +local endofstring=patterns.endofstring +local nobrace=1-(lbrace+rbrace ) +local noparent=1-(lparent+rparent) +local escape,left,right=P("\\"),P('{'),P('}') +patterns.balanced=P { + [1]=((escape*(left+right))+(1-(left+right))+V(2))^0, + [2]=left*V(1)*right +} +local nestedbraces=P { lbrace*(nobrace+V(1))^0*rbrace } +local nestedparents=P { lparent*(noparent+V(1))^0*rparent } +local spaces=space^0 +local argument=Cs((lbrace/"")*((nobrace+nestedbraces)^0)*(rbrace/"")) +local content=(1-endofstring)^0 +patterns.nestedbraces=nestedbraces +patterns.nestedparents=nestedparents +patterns.nested=nestedbraces +patterns.argument=argument +patterns.content=content +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local key=C((1-equal-comma)^1) +local pattern_a=(space+comma)^0*(key*equal*value+key*C("")) +local pattern_c=(space+comma)^0*(key*equal*value) +local key=C((1-space-equal-comma)^1) +local pattern_b=spaces*comma^0*spaces*(key*((spaces*equal*spaces*value)+C(""))) +local hash={} +local function set(key,value) + hash[key]=value +end +local pattern_a_s=(pattern_a/set)^1 +local pattern_b_s=(pattern_b/set)^1 +local pattern_c_s=(pattern_c/set)^1 +parsers.patterns.settings_to_hash_a=pattern_a_s +parsers.patterns.settings_to_hash_b=pattern_b_s +parsers.patterns.settings_to_hash_c=pattern_c_s +function parsers.make_settings_to_hash_pattern(set,how) + if how=="strict" then + return (pattern_c/set)^1 + elseif how=="tolerant" then + return (pattern_b/set)^1 + else + return (pattern_a/set)^1 + end end - -function setters.reset(t) - t.done = { } - reset(t) +function parsers.settings_to_hash(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_a_s,str) + return hash + else + return {} + end end - -function setters.list(t) -- pattern - local list = table.sortedkeys(t.data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what +function parsers.settings_to_hash_tolerant(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_b_s,str) + return hash + else + return {} + end +end +function parsers.settings_to_hash_strict(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_c_s,str) + return next(hash) and hash + else + return nil + end +end +local separator=comma*space^0 +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local pattern=spaces*Ct(value*(separator*value)^0) +parsers.patterns.settings_to_array=pattern +function parsers.settings_to_array(str,strict) + if not str or str=="" then + return {} + elseif strict then + if find(str,"{") then + return lpegmatch(pattern,str) + else + return { str } + end + else + return lpegmatch(pattern,str) + end +end +local function set(t,v) + t[#t+1]=v +end +local value=P(Carg(1)*value)/set +local pattern=value*(separator*value)^0*Carg(1) +function parsers.add_settings_to_array(t,str) + return lpegmatch(pattern,str,nil,t) +end +function parsers.hash_to_string(h,separator,yes,no,strict,omit) + if h then + local t,tn,s={},0,table.sortedkeys(h) + omit=omit and table.tohash(omit) + for i=1,#s do + local key=s[i] + if not omit or not omit[key] then + local value=h[key] + if type(value)=="boolean" then + if yes and no then + if value then + tn=tn+1 + t[tn]=key..'='..yes + elseif not strict then + tn=tn+1 + t[tn]=key..'='..no + end + elseif value or not strict then + tn=tn+1 + t[tn]=key..'='..tostring(value) + end else - user[#user+1] = what + tn=tn+1 + t[tn]=key..'='..value end + end end - return user, system + return concat(t,separator or ",") + else + return "" + end end - -function setters.show(t) - local category = t.name - local list = setters.list(t) - t.report() - for k=1,#list do - local name = list[k] - local functions = t.data[name] - if functions then - local value, default, modules = functions.value, functions.default, #functions - value = value == nil and "unset" or tostring(value) - default = default == nil and "unset" or tostring(default) - t.report("%-50s modules: %2i default: %-12s value: %-12s",name,modules,default,value) - end +function parsers.array_to_string(a,separator) + if a then + return concat(a,separator or ",") + else + return "" + end +end +function parsers.settings_to_set(str,t) + t=t or {} + for s in gmatch(str,"[^, ]+") do + t[s]=true + end + return t +end +function parsers.simple_hash_to_string(h,separator) + local t,tn={},0 + for k,v in sortedhash(h) do + if v then + tn=tn+1 + t[tn]=k end - t.report() + end + return concat(t,separator or ",") end - --- we could have used a bit of oo and the trackers:enable syntax but --- there is already a lot of code around using the singular tracker - --- we could make this into a module but we also want the rest avaliable - -local enable, disable, register, list, show = setters.enable, setters.disable, setters.register, setters.list, setters.show - -local write_nl = texio and texio.write_nl or print - -local function report(setter,...) - local report = logs and logs.report - if report then - report(setter.name,...) - else -- fallback, as this module is loaded before the logger - write_nl(format("%-15s : %s\n",setter.name,format(...))) +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) +local pattern_a=spaces*Ct(value*(separator*value)^0) +local function repeater(n,str) + if not n then + return str + else + local s=lpegmatch(pattern_a,str) + if n==1 then + return unpack(s) + else + local t,tn={},0 + for i=1,n do + for j=1,#s do + tn=tn+1 + t[tn]=s[j] + end + end + return unpack(t) end + end end - -local function default(setter,name) - local d = setter.data[name] - return d and d.default +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+(C(digit^1)/tonumber*lparent*Cs((noparent+nestedparents)^1)*rparent)/repeater+C((nestedbraces+(1-comma))^1) +local pattern_b=spaces*Ct(value*(separator*value)^0) +function parsers.settings_to_array_with_repeat(str,expand) + if expand then + return lpegmatch(pattern_b,str) or {} + else + return lpegmatch(pattern_a,str) or {} + end end - -local function value(setter,name) - local d = setter.data[name] - return d and (d.value or d.default) -end - -function setters.new(name) -- we could use foo:bar syntax (but not used that often) - local setter -- we need to access it in setter itself - setter = { - data = allocate(), -- indexed, but also default and value fields - name = name, - report = function(...) report (setter,...) end, - enable = function(...) enable (setter,...) end, - disable = function(...) disable (setter,...) end, - register = function(...) register(setter,...) end, - list = function(...) list (setter,...) end, - show = function(...) show (setter,...) end, - default = function(...) return default (setter,...) end, - value = function(...) return value (setter,...) end, - } - data[name] = setter - return setter +local value=lbrace*C((nobrace+nestedbraces)^0)*rbrace +local pattern=Ct((space+value)^0) +function parsers.arguments_to_table(str) + return lpegmatch(pattern,str) end - -trackers = setters.new("trackers") -directives = setters.new("directives") -experiments = setters.new("experiments") - -local t_enable, t_disable, t_report = trackers .enable, trackers .disable, trackers .report -local d_enable, d_disable, d_report = directives .enable, directives .disable, directives .report -local e_enable, e_disable, e_report = experiments.enable, experiments.disable, experiments.report - --- nice trick: we overload two of the directives related functions with variants that --- do tracing (itself using a tracker) .. proof of concept - -local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) -local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) - -function directives.enable(...) - if trace_directives then - d_report("enabling: %s",concat({...}," ")) +function parsers.getparameters(self,class,parentclass,settings) + local sc=self[class] + if not sc then + sc={} + self[class]=sc + if parentclass then + local sp=self[parentclass] + if not sp then + sp={} + self[parentclass]=sp + end + setmetatableindex(sc,sp) end - d_enable(...) + end + parsers.settings_to_hash(settings,sc) end - -function directives.disable(...) - if trace_directives then - d_report("disabling: %s",concat({...}," ")) - end - d_disable(...) +function parsers.listitem(str) + return gmatch(str,"[^, ]+") +end +local pattern=Cs { "start", + start=V("one")+V("two")+V("three"), + rest=(Cc(",")*V("thousand"))^0*(P(".")+endofstring)*anything^0, + thousand=digit*digit*digit, + one=digit*V("rest"), + two=digit*digit*V("rest"), + three=V("thousand")*V("rest"), +} +patterns.splitthousands=pattern +function parsers.splitthousands(str) + return lpegmatch(pattern,str) or str +end +local optionalwhitespace=whitespace^0 +patterns.words=Ct((Cs((1-punctuation-whitespace)^1)+anything)^1) +patterns.sentences=Ct((optionalwhitespace*Cs((1-period)^0*period))^1) +patterns.paragraphs=Ct((optionalwhitespace*Cs((whitespace^1*endofstring/""+1-(spacer^0*newline*newline))^1))^1) +local dquote=P('"') +local equal=P('=') +local escape=P('\\') +local separator=S(' ,') +local key=C((1-equal)^1) +local value=dquote*C((1-dquote-escape*dquote)^0)*dquote +local pattern=Cf(Ct("")*Cg(key*equal*value)*separator^0,rawset)^0 +parsers.patterns.keq_to_hash_c=pattern +function parsers.keq_to_hash(str) + if str and str~="" then + return lpegmatch(pattern,str) + else + return {} + end end - -function experiments.enable(...) - if trace_experiments then - e_report("enabling: %s",concat({...}," ")) +local defaultspecification={ separator=",",quote='"' } +function parsers.csvsplitter(specification) + specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=specification.quote + local separator=S(separator~="" and separator or ",") + local whatever=C((1-separator-newline)^0) + if quotechar and quotechar~="" then + local quotedata=nil + for chr in gmatch(quotechar,".") do + local quotechar=P(chr) + local quoteword=quotechar*C((1-quotechar)^0)*quotechar + if quotedata then + quotedata=quotedata+quoteword + else + quotedata=quoteword + end end - e_enable(...) + whatever=quotedata+whatever + end + local parser=Ct((Ct(whatever*(separator*whatever)^0)*S("\n\r"))^0 ) + return function(data) + return lpegmatch(parser,data) + end end - -function experiments.disable(...) - if trace_experiments then - e_report("disabling: %s",concat({...}," ")) - end - e_disable(...) +function parsers.rfc4180splitter(specification) + specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=P(specification.quote) + local dquotechar=quotechar*quotechar +/specification.quote + local separator=S(separator~="" and separator or ",") + local escaped=quotechar*Cs((dquotechar+(1-quotechar))^0)*quotechar + local non_escaped=C((1-quotechar-newline-separator)^1) + local field=escaped+non_escaped + local record=Ct((field*separator^-1)^1) + local headerline=record*Cp() + local wholeblob=Ct((newline^-1*record)^0) + return function(data,getheader) + if getheader then + local header,position=lpegmatch(headerline,data) + local data=lpegmatch(wholeblob,data,position) + return data,header + else + return lpegmatch(wholeblob,data) + end + end +end +local function ranger(first,last,n,action) + if not first then + elseif last==true then + for i=first,n or first do + action(i) + end + elseif last then + for i=first,last do + action(i) + end + else + action(first) + end +end +local cardinal=patterns.cardinal/tonumber +local spacers=patterns.spacer^0 +local endofstring=patterns.endofstring +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1 +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+(P("*")+endofstring)*Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1*endofstring +function utilities.parsers.stepper(str,n,action) + if type(n)=="function" then + lpegmatch(stepper,str,1,false,n or print) + else + lpegmatch(stepper,str,1,n,action or print) + end end --- a useful example - -directives.register("system.nostatistics", function(v) - statistics.enable = not v -end) - -directives.register("system.nolibraries", function(v) - libraries = nil -- we discard this tracing for security -end) - --- experiment -if environment then +end -- of closure - -- The engineflags are known earlier than environment.arguments but maybe we - -- need to handle them both as the later are parsed differently. The c: prefix - -- is used by mtx-context to isolate the flags from those that concern luatex. +do -- create closure to overcome 200 locals limit - local engineflags = environment.engineflags +-- original size: 3006, stripped down to: 2072 - if engineflags then - local list = engineflags["c:trackers"] or engineflags["trackers"] - if type(list) == "string" then - setters.initialize("commandline flags","trackers",settings_to_hash(list)) - -- t_enable(list) - end - local list = engineflags["c:directives"] or engineflags["directives"] - if type(list) == "string" then - setters.initialize("commandline flags","directives", settings_to_hash(list)) - -- d_enable(list) +if not modules then modules={} end modules ['util-fmt']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.formatters=utilities.formatters or {} +local formatters=utilities.formatters +local concat,format=table.concat,string.format +local tostring,type=tostring,type +local strip=string.strip +local P,R,Cs=lpeg.P,lpeg.R,lpeg.Cs +local lpegmatch=lpeg.match +local digit=R("09") +local period=P(".") +local zero=P("0") +local trailingzeros=zero^0*-digit +local case_1=period*trailingzeros/"" +local case_2=period*(digit-trailingzeros)^1*(trailingzeros/"") +local number=digit^1*(case_1+case_2) +local stripper=Cs((number+1)^0) +lpeg.patterns.stripzeros=stripper +function formatters.stripzeros(str) + return lpegmatch(stripper,str) +end +function formatters.formatcolumns(result,between) + if result and #result>0 then + between=between or " " + local widths,numbers={},{} + local first=result[1] + local n=#first + for i=1,n do + widths[i]=0 + end + for i=1,#result do + local r=result[i] + for j=1,n do + local rj=r[j] + local tj=type(rj) + if tj=="number" then + numbers[j]=true + end + if tj~="string" then + rj=tostring(rj) + r[j]=rj + end + local w=#rj + if w>widths[j] then + widths[j]=w end + end end - -end - --- here - -if texconfig then - - -- this happens too late in ini mode but that is no problem - - local function set(k,v) - v = tonumber(v) - if v then - texconfig[k] = v + for i=1,n do + local w=widths[i] + if numbers[i] then + if w>80 then + widths[i]="%s"..between + else + widths[i]="%0"..w.."i"..between + end + else + if w>80 then + widths[i]="%s"..between + elseif w>0 then + widths[i]="%-"..w.."s"..between + else + widths[i]="%s" end + end end - - directives.register("luatex.expanddepth", function(v) set("expand_depth",v) end) - directives.register("luatex.hashextra", function(v) set("hash_extra",v) end) - directives.register("luatex.nestsize", function(v) set("nest_size",v) end) - directives.register("luatex.maxinopen", function(v) set("max_in_open",v) end) - directives.register("luatex.maxprintline", function(v) set("max_print_line",v) end) - directives.register("luatex.maxstrings", function(v) set("max_strings",v) end) - directives.register("luatex.paramsize", function(v) set("param_size",v) end) - directives.register("luatex.savesize", function(v) set("save_size",v) end) - directives.register("luatex.stacksize", function(v) set("stack_size",v) end) - + local template=strip(concat(widths)) + for i=1,#result do + local str=format(template,unpack(result[i])) + result[i]=strip(str) + end + end + return result end @@ -8342,1234 +5600,1558 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-log'] = { - version = 1.001, - comment = "companion to trac-log.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- todo: less categories, more subcategories (e.g. nodes) - - -local write_nl, write = texio and texio.write_nl or print, texio and texio.write or io.write -local format, gmatch, find = string.format, string.gmatch, string.find -local concat, insert, remove = table.concat, table.insert, table.remove -local topattern = string.topattern -local texcount = tex and tex.count -local next, type, select = next, type, select - -local setmetatableindex = table.setmetatableindex -local formatters = string.formatters - - - -logs = logs or { } -local logs = logs - -local moreinfo = [[ -More information about ConTeXt and the tools that come with it can be found at: - -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] - --- basic loggers - -local function ignore() end - -setmetatableindex(logs, function(t,k) t[k] = ignore ; return ignore end) - -local report, subreport, status, settarget, setformats, settranslations +-- original size: 4118, stripped down to: 2901 -local direct, subdirect, writer, pushtarget, poptarget - -if tex and (tex.jobname or tex.formatname) then +if not modules then modules={} end modules ['util-deb']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local debug=require "debug" +local getinfo=debug.getinfo +local type,next,tostring=type,next,tostring +local format,find=string.format,string.find +local is_boolean=string.is_boolean +utilities=utilities or {} +utilities.debugger=utilities.debugger or {} +local debugger=utilities.debugger +local counters={} +local names={} +local function hook() + local f=getinfo(2) + if f then + local n="unknown" + if f.what=="C" then + n=f.name or '' + if not names[n] then + names[n]=format("%42s",n) + end + else + n=f.name or f.namewhat or f.what + if not n or n=="" then + n="?" + end + if not names[n] then + names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + end + counters[n]=(counters[n] or 0)+1 + end +end +function debugger.showstats(printer,threshold) + printer=printer or texio.write or print + threshold=threshold or 0 + local total,grandtotal,functions=0,0,0 + local dataset={} + for name,count in next,counters do + dataset[#dataset+1]={ name,count } + end + table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + for i=1,#dataset do + local d=dataset[i] + local name=d[1] + local count=d[2] + if count>threshold and not find(name,"for generator") then + printer(format("%8i %s\n",count,names[name])) + total=total+count + end + grandtotal=grandtotal+count + functions=functions+1 + end + printer("\n") + printer(format("functions : % 10i\n",functions)) + printer(format("total : % 10i\n",total)) + printer(format("grand total: % 10i\n",grandtotal)) + printer(format("threshold : % 10i\n",threshold)) +end +function debugger.savestats(filename,threshold) + local f=io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end +function debugger.enable() + debug.sethook(hook,"c") +end +function debugger.disable() + debug.sethook() +end +local is_node=node and node.is_node +local is_lpeg=lpeg and lpeg.type +function inspect(i) + local ti=type(i) + if ti=="table" then + table.print(i,"table") + elseif is_node and is_node(i) then + table.print(nodes.astable(i),tostring(i)) + elseif is_lpeg and is_lpeg(i) then + lpeg.print(i) + else + print(tostring(i)) + end + return i +end +function traceback() + local level=1 + while true do + local info=debug.getinfo(level,"Sl") + if not info then + break + elseif info.what=="C" then + print(format("%3i : C function",level)) + else + print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) + end + level=level+1 + end +end - local valueiskey = { __index = function(t,k) t[k] = k return k end } -- will be helper - local target = "term and log" +end -- of closure - logs.flush = io.flush +do -- create closure to overcome 200 locals limit - local formats = { } setmetatable(formats, valueiskey) - local translations = { } setmetatable(translations,valueiskey) +-- original size: 6209, stripped down to: 4958 - writer = function(...) - write_nl(target,...) +if not modules then modules={} end modules ['trac-inf']={ + version=1.001, + comment="companion to trac-inf.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower=string.format,string.lower +local concat=table.concat +local clock=os.gettimeofday or os.clock +local write_nl=texio and texio.write_nl or print +statistics=statistics or {} +local statistics=statistics +statistics.enable=true +statistics.threshold=0.01 +local statusinfo,n,registered,timers={},0,{},{} +table.setmetatableindex(timers,function(t,k) + local v={ timing=0,loadtime=0 } + t[k]=v + return v +end) +local function hastiming(instance) + return instance and timers[instance] +end +local function resettiming(instance) + timers[instance or "notimer"]={ timing=0,loadtime=0 } +end +local function starttiming(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing or 0 + if it==0 then + timer.starttime=clock() + if not timer.loadtime then + timer.loadtime=0 + end + end + timer.timing=it+1 +end +local function stoptiming(instance,report) + local timer=timers[instance or "notimer"] + local it=timer.timing + if it>1 then + timer.timing=it-1 + else + local starttime=timer.starttime + if starttime then + local stoptime=clock() + local loadtime=stoptime-starttime + timer.stoptime=stoptime + timer.loadtime=timer.loadtime+loadtime + if report then + statistics.report("load time %0.3f",loadtime) + end + timer.timing=0 + return loadtime end - - newline = function() - write_nl(target,"\n") + end + return 0 +end +local function elapsedtime(instance) + local timer=timers[instance or "notimer"] + return format("%0.3f",timer and timer.loadtime or 0) +end +local function elapsedindeed(instance) + local timer=timers[instance or "notimer"] + return (timer and timer.loadtime or 0)>statistics.threshold +end +local function elapsedseconds(instance,rest) + if elapsedindeed(instance) then + return format("%s seconds %s",elapsedtime(instance),rest or "") + end +end +statistics.hastiming=hastiming +statistics.resettiming=resettiming +statistics.starttiming=starttiming +statistics.stoptiming=stoptiming +statistics.elapsedtime=elapsedtime +statistics.elapsedindeed=elapsedindeed +statistics.elapsedseconds=elapsedseconds +function statistics.register(tag,fnc) + if statistics.enable and type(fnc)=="function" then + local rt=registered[tag] or (#statusinfo+1) + statusinfo[rt]={ tag,fnc } + registered[tag]=rt + if #tag>n then n=#tag end + end +end +function statistics.show(reporter) + if statistics.enable then + if not reporter then reporter=function(tag,data,n) write_nl(tag.." "..data) end end + local register=statistics.register + register("luatex banner",function() + return lower(status.banner) + end) + register("control sequences",function() + return format("%s of %s + %s",status.cs_count,status.hash_size,status.hash_extra) + end) + register("callbacks",function() + local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 + return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) + end) + if jit then + local status={ jit.status() } + if status[1] then + register("luajit status",function() + return concat(status," ",2) + end) + end end - - local f_one = formatters["%-15s > %s\n"] - local f_two = formatters["%-15s >\n"] - - report = function(a,b,c,...) - if c then - write_nl(target,f_one(translations[a],format(formats[b],c,...))) - elseif b then - write_nl(target,f_one(translations[a],formats[b])) - elseif a then - write_nl(target,f_two(translations[a])) - else - write_nl(target,"\n") - end + collectgarbage("collect") + register("current memory usage",statistics.memused) + register("runtime",statistics.runtime) + for i=1,#statusinfo do + local s=statusinfo[i] + local r=s[2]() + if r then + reporter(s[1],r,n) + end end - - local f_one = formatters["%-15s > %s"] - local f_two = formatters["%-15s >"] - - direct = function(a,b,c,...) - if c then - return f_one(translations[a],format(formats[b],c,...)) - elseif b then - return f_one(translations[a],formats[b]) - elseif a then - return f_two(translations[a]) - else - return "" - end + write_nl("") + statistics.enable=false + end +end +local template,report_statistics,nn=nil,nil,0 +function statistics.showjobstat(tag,data,n) + if not logs then + elseif type(data)=="table" then + for i=1,#data do + statistics.showjobstat(tag,data[i],n) end - - local f_one = formatters["%-15s > %s > %s\n"] - local f_two = formatters["%-15s > %s >\n"] - - subreport = function(a,s,b,c,...) - if c then - write_nl(target,f_one(translations[a],translations[s],format(formats[b],c,...))) - elseif b then - write_nl(target,f_one(translations[a],translations[s],formats[b])) - elseif a then - write_nl(target,f_two(translations[a],translations[s])) - else - write_nl(target,"\n") - end + else + if not template or n>nn then + template,n=format("%%-%ss - %%s",n),nn + report_statistics=logs.reporter("mkiv lua stats") end + report_statistics(format(template,tag,data)) + end +end +function statistics.memused() + local round=math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000),round(status.luastate_bytes/1000000)) +end +starttiming(statistics) +function statistics.formatruntime(runtime) + return format("%s seconds",runtime) +end +function statistics.runtime() + stoptiming(statistics) + return statistics.formatruntime(elapsedtime(statistics)) +end +function statistics.timed(action,report) + report=report or logs.reporter("system") + starttiming("run") + action() + stoptiming("run") + report("total runtime: %s",elapsedtime("run")) +end +commands=commands or {} +function commands.resettimer(name) + resettiming(name or "whatever") + starttiming(name or "whatever") +end +function commands.elapsedtime(name) + stoptiming(name or "whatever") + context(elapsedtime(name or "whatever")) +end - local f_one = formatters["%-15s > %s > %s"] - local f_two = formatters["%-15s > %s >"] - - subdirect = function(a,s,b,c,...) - if c then - return f_one(translations[a],translations[s],format(formats[b],c,...)) - elseif b then - return f_one(translations[a],translations[s],formats[b]) - elseif a then - return f_two(translations[a],translations[s]) - else - return "" - end - end - local f_one = formatters["%-15s : %s\n"] - local f_two = formatters["%-15s :\n"] +end -- of closure - status = function(a,b,c,...) - if c then - write_nl(target,f_one(translations[a],format(formats[b],c,...))) - elseif b then - write_nl(target,f_one(translations[a],formats[b])) - elseif a then - write_nl(target,f_two(translations[a])) - else - write_nl(target,"\n") - end - end +do -- create closure to overcome 200 locals limit - local targets = { - logfile = "log", - log = "log", - file = "log", - console = "term", - terminal = "term", - both = "term and log", - } +-- original size: 12560, stripped down to: 8979 - settarget = function(whereto) - target = targets[whereto or "both"] or targets.both - if target == "term" or target == "term and log" then - logs.flush = io.flush +if not modules then modules={} end modules ['trac-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring=type,next,tostring +local concat=table.concat +local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local is_boolean=string.is_boolean +local settings_to_hash=utilities.parsers.settings_to_hash +local allocate=utilities.storage.allocate +utilities=utilities or {} +local utilities=utilities +utilities.setters=utilities.setters or {} +local setters=utilities.setters +local data={} +local trace_initialize=false +function setters.initialize(filename,name,values) + local setter=data[name] + if setter then + frozen=true + local data=setter.data + if data then + for key,newvalue in next,values do + local newvalue=is_boolean(newvalue,newvalue) + local functions=data[key] + if functions then + local oldvalue=functions.value + if functions.frozen then + if trace_initialize then + setter.report("%s: %q is frozen to %q",filename,key,tostring(oldvalue)) + end + elseif #functions>0 and not oldvalue then + if trace_initialize then + setter.report("%s: %q is set to %q",filename,key,tostring(newvalue)) + end + for i=1,#functions do + functions[i](newvalue) + end + functions.value=newvalue + functions.frozen=functions.frozen or frozen + else + if trace_initialize then + setter.report("%s: %q is kept as %q",filename,key,tostring(oldvalue)) + end + end else - logs.flush = ignore + functions={ default=newvalue,frozen=frozen } + data[key]=functions + if trace_initialize then + setter.report("%s: %q default to %q",filename,key,tostring(newvalue)) + end end + end + return true end - - local stack = { } - - pushtarget = function(newtarget) - insert(stack,target) - settarget(newtarget) - end - - poptarget = function() - if #stack > 0 then - settarget(remove(stack)) + end +end +local function set(t,what,newvalue) + local data=t.data + if not data.frozen then + local done=t.done + if type(what)=="string" then + what=settings_to_hash(what) + end + if type(what)~="table" then + return + end + if not done then + done={} + t.done=done + end + for w,value in next,what do + if value=="" then + value=newvalue + elseif not value then + value=false + else + value=is_boolean(value,value) + end + w=topattern(w,true,true) + for name,functions in next,data do + if done[name] then + elseif find(name,w) then + done[name]=true + for i=1,#functions do + functions[i](value) + end + functions.value=value end + end end - - setformats = function(f) - formats = f + end +end +local function reset(t) + local data=t.data + if not data.frozen then + for name,functions in next,data do + for i=1,#functions do + functions[i](false) + end + functions.value=false end - - settranslations = function(t) - translations = t + end +end +local function enable(t,what) + set(t,what,true) +end +local function disable(t,what) + local data=t.data + if not what or what=="" then + t.done={} + reset(t) + else + set(t,what,false) + end +end +function setters.register(t,what,...) + local data=t.data + what=lower(what) + local functions=data[what] + if not functions then + functions={} + data[what]=functions + if trace_initialize then + t.report("defining %s",what) + end + end + local default=functions.default + for i=1,select("#",...) do + local fnc=select(i,...) + local typ=type(fnc) + if typ=="string" then + if trace_initialize then + t.report("coupling %s to %s",what,fnc) + end + local s=fnc + fnc=function(value) set(t,s,value) end + elseif typ~="function" then + fnc=nil + end + if fnc then + functions[#functions+1]=fnc + local value=functions.value or default + if value~=nil then + fnc(value) + functions.value=value + end end - -else - - logs.flush = ignore - - writer = write_nl - - newline = function() - write_nl("\n") + end + return false +end +function setters.enable(t,what) + local e=t.enable + t.enable,t.done=enable,{} + enable(t,what) + t.enable,t.done=e,{} +end +function setters.disable(t,what) + local e=t.disable + t.disable,t.done=disable,{} + disable(t,what) + t.disable,t.done=e,{} +end +function setters.reset(t) + t.done={} + reset(t) +end +function setters.list(t) + local list=table.sortedkeys(t.data) + local user,system={},{} + for l=1,#list do + local what=list[l] + if find(what,"^%*") then + system[#system+1]=what + else + user[#user+1]=what end - - local f_one = formatters["%-15s | %s"] - local f_two = formatters["%-15s |"] - - report = function(a,b,c,...) - if c then - write_nl(f_one(a,format(b,c,...))) - elseif b then - write_nl(f_one(a,b)) - elseif a then - write_nl(f_two(a)) - else - write_nl("") - end + end + return user,system +end +function setters.show(t) + local category=t.name + local list=setters.list(t) + t.report() + for k=1,#list do + local name=list[k] + local functions=t.data[name] + if functions then + local value,default,modules=functions.value,functions.default,#functions + value=value==nil and "unset" or tostring(value) + default=default==nil and "unset" or tostring(default) + t.report("%-50s modules: %2i default: %-12s value: %-12s",name,modules,default,value) + end + end + t.report() +end +local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show +local write_nl=texio and texio.write_nl or print +local function report(setter,...) + local report=logs and logs.report + if report then + report(setter.name,...) + else + write_nl(format("%-15s : %s\n",setter.name,format(...))) + end +end +local function default(setter,name) + local d=setter.data[name] + return d and d.default +end +local function value(setter,name) + local d=setter.data[name] + return d and (d.value or d.default) +end +function setters.new(name) + local setter + setter={ + data=allocate(), + name=name, + report=function(...) report (setter,...) end, + enable=function(...) enable (setter,...) end, + disable=function(...) disable (setter,...) end, + register=function(...) register(setter,...) end, + list=function(...) list (setter,...) end, + show=function(...) show (setter,...) end, + default=function(...) return default (setter,...) end, + value=function(...) return value (setter,...) end, + } + data[name]=setter + return setter +end +trackers=setters.new("trackers") +directives=setters.new("directives") +experiments=setters.new("experiments") +local t_enable,t_disable,t_report=trackers .enable,trackers .disable,trackers .report +local d_enable,d_disable,d_report=directives .enable,directives .disable,directives .report +local e_enable,e_disable,e_report=experiments.enable,experiments.disable,experiments.report +local trace_directives=false local trace_directives=false trackers.register("system.directives",function(v) trace_directives=v end) +local trace_experiments=false local trace_experiments=false trackers.register("system.experiments",function(v) trace_experiments=v end) +function directives.enable(...) + if trace_directives then + d_report("enabling: %s",concat({...}," ")) + end + d_enable(...) +end +function directives.disable(...) + if trace_directives then + d_report("disabling: %s",concat({...}," ")) + end + d_disable(...) +end +function experiments.enable(...) + if trace_experiments then + e_report("enabling: %s",concat({...}," ")) + end + e_enable(...) +end +function experiments.disable(...) + if trace_experiments then + e_report("disabling: %s",concat({...}," ")) + end + e_disable(...) +end +directives.register("system.nostatistics",function(v) + statistics.enable=not v +end) +directives.register("system.nolibraries",function(v) + libraries=nil +end) +if environment then + local engineflags=environment.engineflags + if engineflags then + local list=engineflags["c:trackers"] or engineflags["trackers"] + if type(list)=="string" then + setters.initialize("commandline flags","trackers",settings_to_hash(list)) end - - local f_one = formatters["%-15s | %s | %s"] - local f_two = formatters["%-15s | %s |"] - - subreport = function(a,sub,b,c,...) - if c then - write_nl(f_one(a,sub,format(b,c,...))) - elseif b then - write_nl(f_one(a,sub,b)) - elseif a then - write_nl(f_two(a,sub)) - else - write_nl("") - end + local list=engineflags["c:directives"] or engineflags["directives"] + if type(list)=="string" then + setters.initialize("commandline flags","directives",settings_to_hash(list)) end - - local f_one = formatters["%-15s : %s\n"] - local f_two = formatters["%-15s :\n"] - - status = function(a,b,c,...) -- not to be used in lua anyway - if c then - write_nl(f_one(a,format(b,c,...))) - elseif b then - write_nl(f_one(a,b)) -- b can have %'s - elseif a then - write_nl(f_two(a)) - else - write_nl("\n") - end + end +end +if texconfig then + local function set(k,v) + v=tonumber(v) + if v then + texconfig[k]=v end - - direct = ignore - subdirect = ignore - - settarget = ignore - pushtarget = ignore - poptarget = ignore - setformats = ignore - settranslations = ignore - + end + directives.register("luatex.expanddepth",function(v) set("expand_depth",v) end) + directives.register("luatex.hashextra",function(v) set("hash_extra",v) end) + directives.register("luatex.nestsize",function(v) set("nest_size",v) end) + directives.register("luatex.maxinopen",function(v) set("max_in_open",v) end) + directives.register("luatex.maxprintline",function(v) set("max_print_line",v) end) + directives.register("luatex.maxstrings",function(v) set("max_strings",v) end) + directives.register("luatex.paramsize",function(v) set("param_size",v) end) + directives.register("luatex.savesize",function(v) set("save_size",v) end) + directives.register("luatex.stacksize",function(v) set("stack_size",v) end) end -logs.report = report -logs.subreport = subreport -logs.status = status -logs.settarget = settarget -logs.pushtarget = pushtarget -logs.poptarget = poptarget -logs.setformats = setformats -logs.settranslations = settranslations -logs.direct = direct -logs.subdirect = subdirect -logs.writer = writer -logs.newline = newline - --- installer +end -- of closure --- todo: renew (un) locks when a new one is added and wildcard +do -- create closure to overcome 200 locals limit -local data, states = { }, nil +-- original size: 17885, stripped down to: 13242 +if not modules then modules={} end modules ['trac-log']={ + version=1.001, + comment="companion to trac-log.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local write_nl,write=texio and texio.write_nl or print,texio and texio.write or io.write +local format,gmatch,find=string.format,string.gmatch,string.find +local concat,insert,remove=table.concat,table.insert,table.remove +local topattern=string.topattern +local texcount=tex and tex.count +local next,type,select=next,type,select +local setmetatableindex=table.setmetatableindex +local formatters=string.formatters +logs=logs or {} +local logs=logs +local moreinfo=[[ +More information about ConTeXt and the tools that come with it can be found at: +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] +local function ignore() end +setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) +local report,subreport,status,settarget,setformats,settranslations +local direct,subdirect,writer,pushtarget,poptarget +if tex and (tex.jobname or tex.formatname) then + local valueiskey={ __index=function(t,k) t[k]=k return k end } + local target="term and log" + logs.flush=io.flush + local formats={} setmetatable(formats,valueiskey) + local translations={} setmetatable(translations,valueiskey) + writer=function(...) + write_nl(target,...) + end + newline=function() + write_nl(target,"\n") + end + local f_one=formatters["%-15s > %s\n"] + local f_two=formatters["%-15s >\n"] + report=function(a,b,c,...) + if c then + write_nl(target,f_one(translations[a],format(formats[b],c,...))) + elseif b then + write_nl(target,f_one(translations[a],formats[b])) + elseif a then + write_nl(target,f_two(translations[a])) + else + write_nl(target,"\n") + end + end + local f_one=formatters["%-15s > %s"] + local f_two=formatters["%-15s >"] + direct=function(a,b,c,...) + if c then + return f_one(translations[a],format(formats[b],c,...)) + elseif b then + return f_one(translations[a],formats[b]) + elseif a then + return f_two(translations[a]) + else + return "" + end + end + local f_one=formatters["%-15s > %s > %s\n"] + local f_two=formatters["%-15s > %s >\n"] + subreport=function(a,s,b,c,...) + if c then + write_nl(target,f_one(translations[a],translations[s],format(formats[b],c,...))) + elseif b then + write_nl(target,f_one(translations[a],translations[s],formats[b])) + elseif a then + write_nl(target,f_two(translations[a],translations[s])) + else + write_nl(target,"\n") + end + end + local f_one=formatters["%-15s > %s > %s"] + local f_two=formatters["%-15s > %s >"] + subdirect=function(a,s,b,c,...) + if c then + return f_one(translations[a],translations[s],format(formats[b],c,...)) + elseif b then + return f_one(translations[a],translations[s],formats[b]) + elseif a then + return f_two(translations[a],translations[s]) + else + return "" + end + end + local f_one=formatters["%-15s : %s\n"] + local f_two=formatters["%-15s :\n"] + status=function(a,b,c,...) + if c then + write_nl(target,f_one(translations[a],format(formats[b],c,...))) + elseif b then + write_nl(target,f_one(translations[a],formats[b])) + elseif a then + write_nl(target,f_two(translations[a])) + else + write_nl(target,"\n") + end + end + local targets={ + logfile="log", + log="log", + file="log", + console="term", + terminal="term", + both="term and log", + } + settarget=function(whereto) + target=targets[whereto or "both"] or targets.both + if target=="term" or target=="term and log" then + logs.flush=io.flush + else + logs.flush=ignore + end + end + local stack={} + pushtarget=function(newtarget) + insert(stack,target) + settarget(newtarget) + end + poptarget=function() + if #stack>0 then + settarget(remove(stack)) + end + end + setformats=function(f) + formats=f + end + settranslations=function(t) + translations=t + end +else + logs.flush=ignore + writer=write_nl + newline=function() + write_nl("\n") + end + local f_one=formatters["%-15s | %s"] + local f_two=formatters["%-15s |"] + report=function(a,b,c,...) + if c then + write_nl(f_one(a,format(b,c,...))) + elseif b then + write_nl(f_one(a,b)) + elseif a then + write_nl(f_two(a)) + else + write_nl("") + end + end + local f_one=formatters["%-15s | %s | %s"] + local f_two=formatters["%-15s | %s |"] + subreport=function(a,sub,b,c,...) + if c then + write_nl(f_one(a,sub,format(b,c,...))) + elseif b then + write_nl(f_one(a,sub,b)) + elseif a then + write_nl(f_two(a,sub)) + else + write_nl("") + end + end + local f_one=formatters["%-15s : %s\n"] + local f_two=formatters["%-15s :\n"] + status=function(a,b,c,...) + if c then + write_nl(f_one(a,format(b,c,...))) + elseif b then + write_nl(f_one(a,b)) + elseif a then + write_nl(f_two(a)) + else + write_nl("\n") + end + end + direct=ignore + subdirect=ignore + settarget=ignore + pushtarget=ignore + poptarget=ignore + setformats=ignore + settranslations=ignore +end +logs.report=report +logs.subreport=subreport +logs.status=status +logs.settarget=settarget +logs.pushtarget=pushtarget +logs.poptarget=poptarget +logs.setformats=setformats +logs.settranslations=settranslations +logs.direct=direct +logs.subdirect=subdirect +logs.writer=writer +logs.newline=newline +local data,states={},nil function logs.reporter(category,subcategory) - local logger = data[category] - if not logger then - local state = false - if states == true then - state = true - elseif type(states) == "table" then - for c, _ in next, states do - if find(category,c) then - state = true - break - end - end + local logger=data[category] + if not logger then + local state=false + if states==true then + state=true + elseif type(states)=="table" then + for c,_ in next,states do + if find(category,c) then + state=true + break end - logger = { - reporters = { }, - state = state, - } - data[category] = logger - end - local reporter = logger.reporters[subcategory or "default"] - if not reporter then - if subcategory then - reporter = function(...) - if not logger.state then - subreport(category,subcategory,...) - end - end - logger.reporters[subcategory] = reporter - else - local tag = category - reporter = function(...) - if not logger.state then - report(category,...) - end - end - logger.reporters.default = reporter + end + end + logger={ + reporters={}, + state=state, + } + data[category]=logger + end + local reporter=logger.reporters[subcategory or "default"] + if not reporter then + if subcategory then + reporter=function(...) + if not logger.state then + subreport(category,subcategory,...) end + end + logger.reporters[subcategory]=reporter + else + local tag=category + reporter=function(...) + if not logger.state then + report(category,...) + end + end + logger.reporters.default=reporter end - return reporter + end + return reporter end - -logs.new = logs.reporter -- for old times sake - --- context specicific: this ends up in the macro stream - -local ctxreport = logs.writer - +logs.new=logs.reporter +local ctxreport=logs.writer function logs.setmessenger(m) - ctxreport = m + ctxreport=m end - function logs.messenger(category,subcategory) - -- we need to avoid catcode mess (todo: fast context) - if subcategory then - return function(...) - ctxreport(subdirect(category,subcategory,...)) - end - else - return function(...) - ctxreport(direct(category,...)) - end + if subcategory then + return function(...) + ctxreport(subdirect(category,subcategory,...)) + end + else + return function(...) + ctxreport(direct(category,...)) end + end end - --- so far - local function setblocked(category,value) - if category == true then - -- lock all - category, value = "*", true - elseif category == false then - -- unlock all - category, value = "*", false - elseif value == nil then - -- lock selective - value = true - end - if category == "*" then - states = value - for k, v in next, data do - v.state = value - end - else - states = utilities.parsers.settings_to_hash(category) - for c, _ in next, states do - if data[c] then - v.state = value - else - c = topattern(c,true,true) - for k, v in next, data do - if find(k,c) then - v.state = value - end - end - end + if category==true then + category,value="*",true + elseif category==false then + category,value="*",false + elseif value==nil then + value=true + end + if category=="*" then + states=value + for k,v in next,data do + v.state=value + end + else + states=utilities.parsers.settings_to_hash(category) + for c,_ in next,states do + if data[c] then + v.state=value + else + c=topattern(c,true,true) + for k,v in next,data do + if find(k,c) then + v.state=value + end end + end end + end end - function logs.disable(category,value) - setblocked(category,value == nil and true or value) + setblocked(category,value==nil and true or value) end - function logs.enable(category) - setblocked(category,false) + setblocked(category,false) end - function logs.categories() - return table.sortedkeys(data) + return table.sortedkeys(data) end - function logs.show() - local n, c, s, max = 0, 0, 0, 0 - for category, v in table.sortedpairs(data) do - n = n + 1 - local state = v.state - local reporters = v.reporters - local nc = #category - if nc > c then - c = nc - end - for subcategory, _ in next, reporters do - local ns = #subcategory - if ns > c then - s = ns - end - local m = nc + ns - if m > max then - max = m - end - end - local subcategories = concat(table.sortedkeys(reporters),", ") - if state == true then - state = "disabled" - elseif state == false then - state = "enabled" - else - state = "unknown" - end - -- no new here - report("logging","category: '%s', subcategories: '%s', state: '%s'",category,subcategories,state) + local n,c,s,max=0,0,0,0 + for category,v in table.sortedpairs(data) do + n=n+1 + local state=v.state + local reporters=v.reporters + local nc=#category + if nc>c then + c=nc + end + for subcategory,_ in next,reporters do + local ns=#subcategory + if ns>c then + s=ns + end + local m=nc+ns + if m>max then + max=m + end end - report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) + local subcategories=concat(table.sortedkeys(reporters),", ") + if state==true then + state="disabled" + elseif state==false then + state="enabled" + else + state="unknown" + end + report("logging","category: '%s', subcategories: '%s', state: '%s'",category,subcategories,state) + end + report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) end - -directives.register("logs.blocked", function(v) - setblocked(v,true) +directives.register("logs.blocked",function(v) + setblocked(v,true) end) - -directives.register("logs.target", function(v) - settarget(v) +directives.register("logs.target",function(v) + settarget(v) end) - --- tex specific loggers (might move elsewhere) - -local report_pages = logs.reporter("pages") -- not needed but saves checking when we grep for it - -local real, user, sub - +local report_pages=logs.reporter("pages") +local real,user,sub function logs.start_page_number() - real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno --- real, user, sub = 0, 0, 0 -end - -local timing = false -local starttime = nil -local lasttime = nil - -trackers.register("pages.timing", function(v) -- only for myself (diagnostics) - starttime = os.clock() - timing = true + real,user,sub=texcount.realpageno,texcount.userpageno,texcount.subpageno +end +local timing=false +local starttime=nil +local lasttime=nil +trackers.register("pages.timing",function(v) + starttime=os.clock() + timing=true end) - -function logs.stop_page_number() -- the first page can includes the initialization so we omit this in average - if timing then - local elapsed, average - local stoptime = os.clock() - if not lasttime or real < 2 then - elapsed = stoptime - average = stoptime - starttime = stoptime - else - elapsed = stoptime - lasttime - average = (stoptime - starttime) / (real - 1) - end - lasttime = stoptime - if real > 0 then - if user > 0 then - if sub > 0 then - report_pages("flushing realpage %s, userpage %s, subpage %s, time %0.04f / %0.04f",real,user,sub,elapsed,average) - else - report_pages("flushing realpage %s, userpage %s, time %0.04f / %0.04f",real,user,elapsed,average) - end - else - report_pages("flushing realpage %s, time %0.04f / %0.04f",real,elapsed,average) - end - else - report_pages("flushing page, time %0.04f / %0.04f",elapsed,average) - end +function logs.stop_page_number() + if timing then + local elapsed,average + local stoptime=os.clock() + if not lasttime or real<2 then + elapsed=stoptime + average=stoptime + starttime=stoptime + else + elapsed=stoptime-lasttime + average=(stoptime-starttime)/(real-1) + end + lasttime=stoptime + if real>0 then + if user>0 then + if sub>0 then + report_pages("flushing realpage %s, userpage %s, subpage %s, time %0.04f / %0.04f",real,user,sub,elapsed,average) + else + report_pages("flushing realpage %s, userpage %s, time %0.04f / %0.04f",real,user,elapsed,average) + end + else + report_pages("flushing realpage %s, time %0.04f / %0.04f",real,elapsed,average) + end else - if real > 0 then - if user > 0 then - if sub > 0 then - report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) - else - report_pages("flushing realpage %s, userpage %s",real,user) - end - else - report_pages("flushing realpage %s",real) - end + report_pages("flushing page, time %0.04f / %0.04f",elapsed,average) + end + else + if real>0 then + if user>0 then + if sub>0 then + report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - report_pages("flushing page") + report_pages("flushing realpage %s, userpage %s",real,user) end + else + report_pages("flushing realpage %s",real) + end + else + report_pages("flushing page") end - logs.flush() + end + logs.flush() end - -logs.report_job_stat = statistics and statistics.showjobstat - -local report_files = logs.reporter("files") - -local nesting = 0 -local verbose = false -local hasscheme = url.hasscheme - --- we don't have show_open and show_close callbacks yet - +logs.report_job_stat=statistics and statistics.showjobstat +local report_files=logs.reporter("files") +local nesting=0 +local verbose=false +local hasscheme=url.hasscheme function logs.show_open(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- nesting = nesting + 1 - -- report_files("level %s, opening %s",nesting,name) - -- else - -- write(format("(%s",name)) -- tex adds a space - -- end - -- end end - function logs.show_close(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, closing %s",nesting,name) - -- nesting = nesting - 1 - -- else - -- write(")") -- tex adds a space - -- end - -- end end - function logs.show_load(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, loading %s",nesting+1,name) - -- else - -- write(format("(%s)",name)) - -- end - -- end end - --- there may be scripts out there using this: - -local simple = logs.reporter("comment") - -logs.simple = simple -logs.simpleline = simple - --- obsolete - -function logs.setprogram () end -- obsolete -function logs.extendbanner() end -- obsolete -function logs.reportlines () end -- obsolete -function logs.reportbanner() end -- obsolete -function logs.reportline () end -- obsolete -function logs.simplelines () end -- obsolete -function logs.help () end -- obsolete - --- applications - +local simple=logs.reporter("comment") +logs.simple=simple +logs.simpleline=simple +function logs.setprogram () end +function logs.extendbanner() end +function logs.reportlines () end +function logs.reportbanner() end +function logs.reportline () end +function logs.simplelines () end +function logs.help () end local function reportlines(t,str) - if str then - for line in gmatch(str,"(.-)[\n\r]") do - t.report(line) - end + if str then + for line in gmatch(str,"(.-)[\n\r]") do + t.report(line) end + end end - local function reportbanner(t) - local banner = t.banner - if banner then - t.report(banner) - t.report() - end + local banner=t.banner + if banner then + t.report(banner) + t.report() + end end - local function reportversion(t) - local banner = t.banner - if banner then - t.report(banner) - end + local banner=t.banner + if banner then + t.report(banner) + end end - local function reporthelp(t,...) - local helpinfo = t.helpinfo - if type(helpinfo) == "string" then - reportlines(t,helpinfo) - elseif type(helpinfo) == "table" then - local n = select("#",...) - for i=1,n do - reportlines(t,t.helpinfo[select(i,...)]) - if i < n then - t.report() - end - end + local helpinfo=t.helpinfo + if type(helpinfo)=="string" then + reportlines(t,helpinfo) + elseif type(helpinfo)=="table" then + local n=select("#",...) + for i=1,n do + reportlines(t,t.helpinfo[select(i,...)]) + if i %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) - for i=1,10 do - local f = io.open(whereto,"a") -- we can consider keepint the file open - if f then - f:write(message) - f:close() - break - else - sleep(0.1) - end + local message=format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) + for i=1,10 do + local f=io.open(whereto,"a") + if f then + f:write(message) + f:close() + break + else + sleep(0.1) end + end end - -local report_system = logs.reporter("system","logs") - +local report_system=logs.reporter("system","logs") function logs.obsolete(old,new) - local o = loadstring("return " .. new)() - if type(o) == "function" then - return function(...) - report_system("function %s is obsolete, use %s",old,new) - loadstring(old .. "=" .. new .. " return ".. old)()(...) - end - elseif type(o) == "table" then - local t, m = { }, { } - m.__index = function(t,k) - report_system("table %s is obsolete, use %s",old,new) - m.__index, m.__newindex = o, o - return o[k] - end - m.__newindex = function(t,k,v) - report_system("table %s is obsolete, use %s",old,new) - m.__index, m.__newindex = o, o - o[k] = v - end - if libraries then - libraries.obsolete[old] = t -- true - end - setmetatable(t,m) - return t - end + local o=loadstring("return "..new)() + if type(o)=="function" then + return function(...) + report_system("function %s is obsolete, use %s",old,new) + loadstring(old.."="..new.." return "..old)()(...) + end + elseif type(o)=="table" then + local t,m={},{} + m.__index=function(t,k) + report_system("table %s is obsolete, use %s",old,new) + m.__index,m.__newindex=o,o + return o[k] + end + m.__newindex=function(t,k,v) + report_system("table %s is obsolete, use %s",old,new) + m.__index,m.__newindex=o,o + o[k]=v + end + if libraries then + libraries.obsolete[old]=t + end + setmetatable(t,m) + return t + end end - if utilities then - utilities.report = report_system + utilities.report=report_system end - if tex and tex.error then - function logs.texerrormessage(...) -- for the moment we put this function here - tex.error(format(...), { }) - end + function logs.texerrormessage(...) + tex.error(format(...),{}) + end else - function logs.texerrormessage(...) - print(format(...)) - end + function logs.texerrormessage(...) + print(format(...)) + end end +io.stdout:setvbuf('no') +io.stderr:setvbuf('no') --- do we still need io.flush then? - -io.stdout:setvbuf('no') -io.stderr:setvbuf('no') - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-pro'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local getmetatable, setmetatable, rawset, type = getmetatable, setmetatable, rawset, type - --- The protection implemented here is probably not that tight but good enough to catch --- problems due to naive usage. --- --- There's a more extensive version (trac-xxx.lua) that supports nesting. --- --- This will change when we have _ENV in lua 5.2+ - -local trace_namespaces = false trackers.register("system.namespaces", function(v) trace_namespaces = v end) -local report_system = logs.reporter("system","protection") +end -- of closure -namespaces = namespaces or { } -local namespaces = namespaces +do -- create closure to overcome 200 locals limit -local registered = { } +-- original size: 5789, stripped down to: 3469 +if not modules then modules={} end modules ['trac-pro']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local getmetatable,setmetatable,rawset,type=getmetatable,setmetatable,rawset,type +local trace_namespaces=false trackers.register("system.namespaces",function(v) trace_namespaces=v end) +local report_system=logs.reporter("system","protection") +namespaces=namespaces or {} +local namespaces=namespaces +local registered={} local function report_index(k,name) - if trace_namespaces then - report_system("reference to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) - else - report_system("reference to '%s' in protected namespace '%s'",k,name) - end + if trace_namespaces then + report_system("reference to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) + else + report_system("reference to '%s' in protected namespace '%s'",k,name) + end end - local function report_newindex(k,name) - if trace_namespaces then - report_system("assignment to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) - else - report_system("assignment to '%s' in protected namespace '%s'",k,name) - end + if trace_namespaces then + report_system("assignment to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) + else + report_system("assignment to '%s' in protected namespace '%s'",k,name) + end end - local function register(name) - local data = name == "global" and _G or _G[name] - if not data then - return -- error - end - registered[name] = data - local m = getmetatable(data) - if not m then - m = { } - setmetatable(data,m) - end - local index, newindex = { }, { } - m.__saved__index = m.__index - m.__no__index = function(t,k) - if not index[k] then - index[k] = true - report_index(k,name) - end - return nil - end - m.__saved__newindex = m.__newindex - m.__no__newindex = function(t,k,v) - if not newindex[k] then - newindex[k] = true - report_newindex(k,name) - end - rawset(t,k,v) + local data=name=="global" and _G or _G[name] + if not data then + return + end + registered[name]=data + local m=getmetatable(data) + if not m then + m={} + setmetatable(data,m) + end + local index,newindex={},{} + m.__saved__index=m.__index + m.__no__index=function(t,k) + if not index[k] then + index[k]=true + report_index(k,name) end - m.__protection__depth = 0 -end - -local function private(name) -- maybe save name - local data = registered[name] + return nil + end + m.__saved__newindex=m.__newindex + m.__no__newindex=function(t,k,v) + if not newindex[k] then + newindex[k]=true + report_newindex(k,name) + end + rawset(t,k,v) + end + m.__protection__depth=0 +end +local function private(name) + local data=registered[name] + if not data then + data=_G[name] if not data then - data = _G[name] - if not data then - data = { } - _G[name] = data - end - register(name) + data={} + _G[name]=data end - return data + register(name) + end + return data end - local function protect(name) - local data = registered[name] - if not data then - return - end - local m = getmetatable(data) - local pd = m.__protection__depth - if pd > 0 then - m.__protection__depth = pd + 1 - else - m.__save_d_index, m.__saved__newindex = m.__index, m.__newindex - m.__index, m.__newindex = m.__no__index, m.__no__newindex - m.__protection__depth = 1 - end + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>0 then + m.__protection__depth=pd+1 + else + m.__save_d_index,m.__saved__newindex=m.__index,m.__newindex + m.__index,m.__newindex=m.__no__index,m.__no__newindex + m.__protection__depth=1 + end end - local function unprotect(name) - local data = registered[name] - if not data then - return - end - local m = getmetatable(data) - local pd = m.__protection__depth - if pd > 1 then - m.__protection__depth = pd - 1 - else - m.__index, m.__newindex = m.__saved__index, m.__saved__newindex - m.__protection__depth = 0 - end + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>1 then + m.__protection__depth=pd-1 + else + m.__index,m.__newindex=m.__saved__index,m.__saved__newindex + m.__protection__depth=0 + end end - local function protectall() - for name, _ in next, registered do - if name ~= "global" then - protect(name) - end + for name,_ in next,registered do + if name~="global" then + protect(name) end + end end - local function unprotectall() - for name, _ in next, registered do - if name ~= "global" then - unprotect(name) - end - end -end - -namespaces.register = register -- register when defined -namespaces.private = private -- allocate and register if needed -namespaces.protect = protect -namespaces.unprotect = unprotect -namespaces.protectall = protectall -namespaces.unprotectall = unprotectall - -namespaces.private("namespaces") registered = { } register("global") -- unreachable - -directives.register("system.protect", function(v) - if v then - protectall() - else - unprotectall() - end + for name,_ in next,registered do + if name~="global" then + unprotect(name) + end + end +end +namespaces.register=register +namespaces.private=private +namespaces.protect=protect +namespaces.unprotect=unprotect +namespaces.protectall=protectall +namespaces.unprotectall=unprotectall +namespaces.private("namespaces") registered={} register("global") +directives.register("system.protect",function(v) + if v then + protectall() + else + unprotectall() + end end) - -directives.register("system.checkglobals", function(v) - if v then - report_system("enabling global namespace guard") - protect("global") - else - report_system("disabling global namespace guard") - unprotect("global") - end +directives.register("system.checkglobals",function(v) + if v then + report_system("enabling global namespace guard") + protect("global") + else + report_system("disabling global namespace guard") + unprotect("global") + end end) --- dummy section (will go to luat-dum.lua) - - - - - - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-env'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- A former version provided functionality for non embeded core --- scripts i.e. runtime library loading. Given the amount of --- Lua code we use now, this no longer makes sense. Much of this --- evolved before bytecode arrays were available and so a lot of --- code has disappeared already. - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_lua = logs.reporter("resolvers","lua") - -local allocate, mark = utilities.storage.allocate, utilities.storage.mark - -local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find -local unquoted, quoted = string.unquoted, string.quoted -local concat, insert, remove = table.concat, table.insert, table.remove -local loadedluacode = utilities.lua.loadedluacode -local luasuffixes = utilities.lua.suffixes - -environment = environment or { } -local environment = environment - --- precautions - -os.setlocale(nil,nil) -- useless feature and even dangerous in luatex +-- original size: 12260, stripped down to: 8100 +if not modules then modules={} end modules ['luat-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_lua=logs.reporter("resolvers","lua") +local allocate,mark=utilities.storage.allocate,utilities.storage.mark +local format,sub,match,gsub,find=string.format,string.sub,string.match,string.gsub,string.find +local unquoted,quoted=string.unquoted,string.quoted +local concat,insert,remove=table.concat,table.insert,table.remove +local loadedluacode=utilities.lua.loadedluacode +local luasuffixes=utilities.lua.suffixes +environment=environment or {} +local environment=environment +os.setlocale(nil,nil) function os.setlocale() - -- no way you can mess with it end - --- dirty tricks (we will replace the texlua call by luatex --luaonly) - -local validengines = allocate { - ["luatex"] = true, - ["luajittex"] = true, - -- ["luatex.exe"] = true, - -- ["luajittex.exe"] = true, +local validengines=allocate { + ["luatex"]=true, + ["luajittex"]=true, } - -local basicengines = allocate { - ["luatex"] = "luatex", - ["texlua"] = "luatex", - ["texluac"] = "luatex", - ["luajittex"] = "luajittex", - ["texluajit"] = "luajittex", - -- ["texlua.exe"] = "luatex", - -- ["texluajit.exe"] = "luajittex", +local basicengines=allocate { + ["luatex"]="luatex", + ["texlua"]="luatex", + ["texluac"]="luatex", + ["luajittex"]="luajittex", + ["texluajit"]="luajittex", } - -environment.validengines = validengines -environment.basicengines = basicengines - -if arg and validengines[file.removesuffix(arg[0])] and arg[1] == "--luaonly" then - arg[-1] = arg[0] - arg[ 0] = arg[2] - for k=3,#arg do - arg[k-2] = arg[k] - end - remove(arg) -- last - remove(arg) -- pre-last +environment.validengines=validengines +environment.basicengines=basicengines +if arg and validengines[file.removesuffix(arg[0])] and arg[1]=="--luaonly" then + arg[-1]=arg[0] + arg[ 0]=arg[2] + for k=3,#arg do + arg[k-2]=arg[k] + end + remove(arg) + remove(arg) end - --- This is an ugly hack but it permits symlinking a script (say 'context') to 'mtxrun' as in: --- --- ln -s /opt/minimals/tex/texmf-linux-64/bin/mtxrun context --- --- The special mapping hack is needed because 'luatools' boils down to 'mtxrun --script base' --- but it's unlikely that there will be more of this - do - - local originalzero = file.basename(arg[0]) - local specialmapping = { luatools == "base" } - - if originalzero ~= "mtxrun" and originalzero ~= "mtxrun.lua" then - arg[0] = specialmapping[originalzero] or originalzero - insert(arg,0,"--script") - insert(arg,0,"mtxrun") - end - -end - --- environment - -environment.arguments = allocate() -environment.files = allocate() -environment.sortedflags = nil - -local mt = { - __index = function(_,k) - if k == "version" then - local version = tex.toks and tex.toks.contextversiontoks - if version and version ~= "" then - rawset(environment,"version",version) - return version - else - return "unknown" - end - elseif k == "jobname" or k == "formatname" then - local name = tex and tex[k] - if name or name== "" then - rawset(environment,k,name) - return name - else - return "unknown" - end - elseif k == "outputfilename" then - local name = environment.jobname - rawset(environment,k,name) - return name - end + local originalzero=file.basename(arg[0]) + local specialmapping={ luatools=="base" } + if originalzero~="mtxrun" and originalzero~="mtxrun.lua" then + arg[0]=specialmapping[originalzero] or originalzero + insert(arg,0,"--script") + insert(arg,0,"mtxrun") + end +end +environment.arguments=allocate() +environment.files=allocate() +environment.sortedflags=nil +local mt={ + __index=function(_,k) + if k=="version" then + local version=tex.toks and tex.toks.contextversiontoks + if version and version~="" then + rawset(environment,"version",version) + return version + else + return "unknown" + end + elseif k=="jobname" or k=="formatname" then + local name=tex and tex[k] + if name or name=="" then + rawset(environment,k,name) + return name + else + return "unknown" + end + elseif k=="outputfilename" then + local name=environment.jobname + rawset(environment,k,name) + return name end + end } - setmetatable(environment,mt) - --- context specific arguments (in order not to confuse the engine) - function environment.initializearguments(arg) - local arguments, files = { }, { } - environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index=1,#arg do - local argument = arg[index] - if index > 0 then - local flag, value = match(argument,"^%-+(.-)=(.-)$") - if flag then - flag = gsub(flag,"^c:","") - arguments[flag] = unquoted(value or "") - else - flag = match(argument,"^%-+(.+)") - if flag then - flag = gsub(flag,"^c:","") - arguments[flag] = true - else - files[#files+1] = argument - end - end + local arguments,files={},{} + environment.arguments,environment.files,environment.sortedflags=arguments,files,nil + for index=1,#arg do + local argument=arg[index] + if index>0 then + local flag,value=match(argument,"^%-+(.-)=(.-)$") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=unquoted(value or "") + else + flag=match(argument,"^%-+(.+)") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=true + else + files[#files+1]=argument end + end end - environment.ownname = file.reslash(environment.ownname or arg[0] or 'unknown.lua') + end + environment.ownname=file.reslash(environment.ownname or arg[0] or 'unknown.lua') end - function environment.setargument(name,value) - environment.arguments[name] = value + environment.arguments[name]=value end - --- todo: defaults, better checks e.g on type (boolean versus string) --- --- tricky: too many hits when we support partials unless we add --- a registration of arguments so from now on we have 'partial' - function environment.getargument(name,partial) - local arguments, sortedflags = environment.arguments, environment.sortedflags - if arguments[name] then - return arguments[name] - elseif partial then - if not sortedflags then - sortedflags = allocate(table.sortedkeys(arguments)) - for k=1,#sortedflags do - sortedflags[k] = "^" .. sortedflags[k] - end - environment.sortedflags = sortedflags - end - -- example of potential clash: ^mode ^modefile - for k=1,#sortedflags do - local v = sortedflags[k] - if find(name,v) then - return arguments[sub(v,2,#v)] - end - end + local arguments,sortedflags=environment.arguments,environment.sortedflags + if arguments[name] then + return arguments[name] + elseif partial then + if not sortedflags then + sortedflags=allocate(table.sortedkeys(arguments)) + for k=1,#sortedflags do + sortedflags[k]="^"..sortedflags[k] + end + environment.sortedflags=sortedflags end - return nil + for k=1,#sortedflags do + local v=sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] + end + end + end + return nil end - -environment.argument = environment.getargument - -function environment.splitarguments(separator) -- rather special, cut-off before separator - local done, before, after = false, { }, { } - local originalarguments = environment.originalarguments - for k=1,#originalarguments do - local v = originalarguments[k] - if not done and v == separator then - done = true - elseif done then - after[#after+1] = v - else - before[#before+1] = v - end +environment.argument=environment.getargument +function environment.splitarguments(separator) + local done,before,after=false,{},{} + local originalarguments=environment.originalarguments + for k=1,#originalarguments do + local v=originalarguments[k] + if not done and v==separator then + done=true + elseif done then + after[#after+1]=v + else + before[#before+1]=v end - return before, after + end + return before,after end - function environment.reconstructcommandline(arg,noquote) - arg = arg or environment.originalarguments - if noquote and #arg == 1 then - -- we could just do: return unquoted(resolvers.resolve(arg[i])) - local a = arg[1] - a = resolvers.resolve(a) - a = unquoted(a) - return a - elseif #arg > 0 then - local result = { } - for i=1,#arg do - -- we could just do: result[#result+1] = format("%q",unquoted(resolvers.resolve(arg[i]))) - local a = arg[i] - a = resolvers.resolve(a) - a = unquoted(a) - a = gsub(a,'"','\\"') -- tricky - if find(a," ") then - result[#result+1] = quoted(a) - else - result[#result+1] = a - end - end - return concat(result," ") - else - return "" + arg=arg or environment.originalarguments + if noquote and #arg==1 then + local a=arg[1] + a=resolvers.resolve(a) + a=unquoted(a) + return a + elseif #arg>0 then + local result={} + for i=1,#arg do + local a=arg[i] + a=resolvers.resolve(a) + a=unquoted(a) + a=gsub(a,'"','\\"') + if find(a," ") then + result[#result+1]=quoted(a) + else + result[#result+1]=a + end end + return concat(result," ") + else + return "" + end end - --- -- to be tested: --- --- function environment.reconstructcommandline(arg,noquote) --- arg = arg or environment.originalarguments --- if noquote and #arg == 1 then --- return unquoted(resolvers.resolve(arg[1])) --- elseif #arg > 0 then --- local result = { } --- for i=1,#arg do --- result[#result+1] = format("%q",unquoted(resolvers.resolve(arg[i]))) -- always quote --- end --- return concat(result," ") --- else --- return "" --- end --- end - if arg then - - -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) - local newarg, instring = { }, false - - for index=1,#arg do - local argument = arg[index] - if find(argument,"^\"") then - newarg[#newarg+1] = gsub(argument,"^\"","") - if not find(argument,"\"$") then - instring = true - end - elseif find(argument,"\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") - instring = false - elseif instring then - newarg[#newarg] = newarg[#newarg] .. " " .. argument - else - newarg[#newarg+1] = argument - end - end - for i=1,-5,-1 do - newarg[i] = arg[i] - end - - environment.initializearguments(newarg) - - environment.originalarguments = mark(newarg) - environment.rawarguments = mark(arg) - - arg = { } -- prevent duplicate handling - + local newarg,instring={},false + for index=1,#arg do + local argument=arg[index] + if find(argument,"^\"") then + newarg[#newarg+1]=gsub(argument,"^\"","") + if not find(argument,"\"$") then + instring=true + end + elseif find(argument,"\"$") then + newarg[#newarg]=newarg[#newarg].." "..gsub(argument,"\"$","") + instring=false + elseif instring then + newarg[#newarg]=newarg[#newarg].." "..argument + else + newarg[#newarg+1]=argument + end + end + for i=1,-5,-1 do + newarg[i]=arg[i] + end + environment.initializearguments(newarg) + environment.originalarguments=mark(newarg) + environment.rawarguments=mark(arg) + arg={} end - --- weird place ... depends on a not yet loaded module - function environment.texfile(filename) - return resolvers.findfile(filename,'tex') + return resolvers.findfile(filename,'tex') +end +function environment.luafile(filename) + local resolved=resolvers.findfile(filename,'tex') or "" + if resolved~="" then + return resolved + end + resolved=resolvers.findfile(filename,'texmfscripts') or "" + if resolved~="" then + return resolved + end + return resolvers.findfile(filename,'luatexlibs') or "" +end +local stripindeed=false directives.register("system.compile.strip",function(v) stripindeed=v end) +local function strippable(filename) + if stripindeed then + local modu=modules[file.nameonly(filename)] + return modu and modu.dataonly + else + return false + end end - -function environment.luafile(filename) -- needs checking - local resolved = resolvers.findfile(filename,'tex') or "" - if resolved ~= "" then - return resolved - end - resolved = resolvers.findfile(filename,'texmfscripts') or "" - if resolved ~= "" then - return resolved +function environment.luafilechunk(filename,silent) + filename=file.replacesuffix(filename,"lua") + local fullname=environment.luafile(filename) + if fullname and fullname~="" then + local data=loadedluacode(fullname,strippable,filename) + if trace_locating then + report_lua("loading file %s%s",fullname,not data and " failed" or "") + elseif not silent then + texio.write("<",data and "+ " or "- ",fullname,">") end - return resolvers.findfile(filename,'luatexlibs') or "" -end - --- local function checkstrip(filename) --- local modu = modules[file.nameonly(filename)] --- return modu and modu.dataonly --- end - -local stripindeed = false directives.register("system.compile.strip", function(v) stripindeed = v end) - -local function strippable(filename) - if stripindeed then - local modu = modules[file.nameonly(filename)] - return modu and modu.dataonly - else - return false + return data + else + if trace_locating then + report_lua("unknown file %s",filename) end -end - -function environment.luafilechunk(filename,silent) -- used for loading lua bytecode in the format - filename = file.replacesuffix(filename, "lua") - local fullname = environment.luafile(filename) - if fullname and fullname ~= "" then - local data = loadedluacode(fullname,strippable,filename) - if trace_locating then - report_lua("loading file %s%s", fullname, not data and " failed" or "") - elseif not silent then - texio.write("<",data and "+ " or "- ",fullname,">") - end - return data - else + return nil + end +end +function environment.loadluafile(filename,version) + local lucname,luaname,chunk + local basename=file.removesuffix(filename) + if basename==filename then + luaname=file.addsuffix(basename,luasuffixes.lua) + lucname=file.addsuffix(basename,luasuffixes.luc) + else + luaname=basename + lucname=nil + end + local fullname=(lucname and environment.luafile(lucname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %s",fullname) + end + chunk=loadfile(fullname) + end + if chunk then + assert(chunk)() + if version then + local v=version + if modules and modules[filename] then + v=modules[filename].version + elseif versions and versions[filename] then + v=versions[filename] + end + if v==version then + return true + else if trace_locating then - report_lua("unknown file %s", filename) + report_lua("version mismatch for %s: lua=%s, luc=%s",filename,v,version) end - return nil - end -end - --- the next ones can use the previous ones / combine - -function environment.loadluafile(filename, version) - local lucname, luaname, chunk - local basename = file.removesuffix(filename) - if basename == filename then - luaname = fiule.addsuffix(basename,luasuffixes.lua) - lucname = fiule.addsuffix(basename,luasuffixes.luc) + environment.loadluafile(filename) + end else - luaname = basename -- forced suffix - lucname = nil + return true end - -- when not overloaded by explicit suffix we look for a luc file first - local fullname = (lucname and environment.luafile(lucname)) or "" - if fullname ~= "" then - if trace_locating then - report_lua("loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - end - if chunk then - assert(chunk)() - if version then - -- we check of the version number of this chunk matches - local v = version -- can be nil - if modules and modules[filename] then - v = modules[filename].version -- new method - elseif versions and versions[filename] then - v = versions[filename] -- old method - end - if v == version then - return true - else - if trace_locating then - report_lua("version mismatch for %s: lua=%s, luc=%s", filename, v, version) - end - environment.loadluafile(filename) - end - else - return true - end + end + fullname=(luaname and environment.luafile(luaname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %s",fullname) end - fullname = (luaname and environment.luafile(luaname)) or "" - if fullname ~= "" then - if trace_locating then - report_lua("loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - if not chunk then - if trace_locating then - report_lua("unknown file %s", filename) - end - else - assert(chunk)() - return true - end + chunk=loadfile(fullname) + if not chunk then + if trace_locating then + report_lua("unknown file %s",filename) + end + else + assert(chunk)() + return true end - return false + end + return false end @@ -9577,1226 +7159,978 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-tab'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc --- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the --- trouble - --- todo: when serializing optionally remap named entities to hex (if known in char-ent.lua) --- maybe when letter -> utf, else name .. then we need an option to the serializer .. a bit --- of work so we delay this till we cleanup - -local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) - -local report_xml = logs and logs.reporter("xml","core") or function(...) print(format(...)) end - - - -xml = xml or { } -local xml = xml - - -local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber -local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub -local utfchar = utf.char -local lpegmatch = lpeg.match -local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs - - - -xml.xmlns = xml.xmlns or { } - -local check = P(false) -local parse = check - +-- original size: 42438, stripped down to: 26556 - -function xml.registerns(namespace, pattern) -- pattern can be an lpeg - check = check + C(P(lower(pattern))) / namespace - parse = P { P(check) + 1 * V(1) } +if not modules then modules={} end modules ['lxml-tab']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_entities=false trackers.register("xml.entities",function(v) trace_entities=v end) +local report_xml=logs and logs.reporter("xml","core") or function(...) print(format(...)) end +xml=xml or {} +local xml=xml +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,setmetatable,getmetatable,tonumber=type,next,setmetatable,getmetatable,tonumber +local format,lower,find,match,gsub=string.format,string.lower,string.find,string.match,string.gsub +local utfchar=utf.char +local lpegmatch=lpeg.match +local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs +xml.xmlns=xml.xmlns or {} +local check=P(false) +local parse=check +function xml.registerns(namespace,pattern) + check=check+C(P(lower(pattern)))/namespace + parse=P { P(check)+1*V(1) } end - - - function xml.checkns(namespace,url) - local ns = lpegmatch(parse,lower(url)) - if ns and namespace ~= ns then - xml.xmlns[namespace] = ns - end + local ns=lpegmatch(parse,lower(url)) + if ns and namespace~=ns then + xml.xmlns[namespace]=ns + end end - - - function xml.resolvens(url) - return lpegmatch(parse,lower(url)) or "" -end - - - - - --- not just one big nested table capture (lpeg overflow) - -local nsremap, resolvens = xml.xmlns, xml.resolvens - -local stack = { } -local top = { } -local dt = { } -local at = { } -local xmlns = { } -local errorstr = nil -local entities = { } -local strip = false -local cleanup = false -local utfize = false -local resolve_predefined = false -local unify_predefined = false - -local dcache = { } -local hcache = { } -local acache = { } - -local mt = { } - -local function initialize_mt(root) - mt = { __index = root } -- will be redefined later -end - -function xml.setproperty(root,k,v) - getmetatable(root).__index[k] = v -end - -function xml.checkerror(top,toclose) - return "" -- can be set -end - -local function add_attribute(namespace,tag,value) - if cleanup and #value > 0 then - value = cleanup(value) -- new - end - if tag == "xmlns" then - xmlns[#xmlns+1] = resolvens(value) - at[tag] = value - elseif namespace == "" then - at[tag] = value - elseif namespace == "xmlns" then - xml.checkns(tag,value) - at["xmlns:" .. tag] = value - else - -- for the moment this way: - at[namespace .. ":" .. tag] = value - end + return lpegmatch(parse,lower(url)) or "" +end +local nsremap,resolvens=xml.xmlns,xml.resolvens +local stack={} +local top={} +local dt={} +local at={} +local xmlns={} +local errorstr=nil +local entities={} +local strip=false +local cleanup=false +local utfize=false +local resolve_predefined=false +local unify_predefined=false +local dcache={} +local hcache={} +local acache={} +local mt={} +local function initialize_mt(root) + mt={ __index=root } end - -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } +function xml.setproperty(root,k,v) + getmetatable(root).__index[k]=v end - -local function add_begin(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } - setmetatable(top, mt) - dt = top.dt - stack[#stack+1] = top - at = { } +function xml.checkerror(top,toclose) + return "" end - -local function add_end(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local toclose = remove(stack) - top = stack[#stack] - if #stack < 1 then - errorstr = format("nothing to close with %s %s", tag, xml.checkerror(top,toclose) or "") - elseif toclose.tg ~= tag then -- no namespace check - errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.checkerror(top,toclose) or "") - end - dt = top.dt - dt[#dt+1] = toclose - -- dt[0] = top -- nasty circular reference when serializing table - if toclose.at.xmlns then - remove(xmlns) - end +local function add_attribute(namespace,tag,value) + if cleanup and #value>0 then + value=cleanup(value) + end + if tag=="xmlns" then + xmlns[#xmlns+1]=resolvens(value) + at[tag]=value + elseif namespace=="" then + at[tag]=value + elseif namespace=="xmlns" then + xml.checkns(tag,value) + at["xmlns:"..tag]=value + else + at[namespace..":"..tag]=value + end +end +local function add_empty(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=(namespace=="" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top=stack[#stack] + dt=top.dt + local t={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=top } + dt[#dt+1]=t + setmetatable(t,mt) + if at.xmlns then + remove(xmlns) + end + at={} +end +local function add_begin(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=(namespace=="" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=stack[#stack] } + setmetatable(top,mt) + dt=top.dt + stack[#stack+1]=top + at={} +end +local function add_end(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local toclose=remove(stack) + top=stack[#stack] + if #stack<1 then + errorstr=format("nothing to close with %s %s",tag,xml.checkerror(top,toclose) or "") + elseif toclose.tg~=tag then + errorstr=format("unable to close %s with %s %s",toclose.tg,tag,xml.checkerror(top,toclose) or "") + end + dt=top.dt + dt[#dt+1]=toclose + if toclose.at.xmlns then + remove(xmlns) + end end - local function add_text(text) - if cleanup and #text > 0 then - dt[#dt+1] = cleanup(text) - else - dt[#dt+1] = text - end + if cleanup and #text>0 then + dt[#dt+1]=cleanup(text) + else + dt[#dt+1]=text + end +end +local function add_special(what,spacing,text) + if #spacing>0 then + dt[#dt+1]=spacing + end + if strip and (what=="@cm@" or what=="@dt@") then + else + dt[#dt+1]={ special=true,ns="",tg=what,dt={ text } } + end end - -local function add_special(what, spacing, text) - if #spacing > 0 then - dt[#dt+1] = spacing - end - if strip and (what == "@cm@" or what == "@dt@") then - -- forget it - else - dt[#dt+1] = { special=true, ns="", tg=what, dt={ text } } - end -end - local function set_message(txt) - errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") + errorstr="garbage at the end of the file: "..gsub(txt,"([ \n\r\t]*)","") end - -local reported_attribute_errors = { } - +local reported_attribute_errors={} local function attribute_value_error(str) - if not reported_attribute_errors[str] then - report_xml("invalid attribute value: %q",str) - reported_attribute_errors[str] = true - at._error_ = str - end - return str + if not reported_attribute_errors[str] then + report_xml("invalid attribute value: %q",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str end - local function attribute_specification_error(str) - if not reported_attribute_errors[str] then - report_xml("invalid attribute specification: %q",str) - reported_attribute_errors[str] = true - at._error_ = str - end - return str -end - -xml.placeholders = { - unknown_dec_entity = function(str) return (str == "" and "&error;") or format("&%s;",str) end, - unknown_hex_entity = function(str) return format("&#x%s;",str) end, - unknown_any_entity = function(str) return format("&#x%s;",str) end, + if not reported_attribute_errors[str] then + report_xml("invalid attribute specification: %q",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str +end +xml.placeholders={ + unknown_dec_entity=function(str) return (str=="" and "&error;") or format("&%s;",str) end, + unknown_hex_entity=function(str) return format("&#x%s;",str) end, + unknown_any_entity=function(str) return format("&#x%s;",str) end, } - -local placeholders = xml.placeholders - +local placeholders=xml.placeholders local function fromhex(s) - local n = tonumber(s,16) - if n then - return utfchar(n) - else - return format("h:%s",s), true - end + local n=tonumber(s,16) + if n then + return utfchar(n) + else + return format("h:%s",s),true + end end - local function fromdec(s) - local n = tonumber(s) - if n then - return utfchar(n) - else - return format("d:%s",s), true - end -end - --- one level expansion (simple case), no checking done - -local rest = (1-P(";"))^0 -local many = P(1)^0 - -local parsedentity = - P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + - (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) - --- parsing in the xml file - -local predefined_unified = { - [38] = "&", - [42] = """, - [47] = "'", - [74] = "<", - [76] = ">", + local n=tonumber(s) + if n then + return utfchar(n) + else + return format("d:%s",s),true + end +end +local rest=(1-P(";"))^0 +local many=P(1)^0 +local parsedentity=P("&")*(P("#x")*(rest/fromhex)+P("#")*(rest/fromdec))*P(";")*P(-1)+(P("#x")*(many/fromhex)+P("#")*(many/fromdec)) +local predefined_unified={ + [38]="&", + [42]=""", + [47]="'", + [74]="<", + [76]=">", } - -local predefined_simplified = { - [38] = "&", amp = "&", - [42] = '"', quot = '"', - [47] = "'", apos = "'", - [74] = "<", lt = "<", - [76] = ">", gt = ">", -} - -local nofprivates = 0xF0000 -- shared but seldom used - -local privates_u = { -- unescaped - [ [[&]] ] = "&", - [ [["]] ] = """, - [ [[']] ] = "'", - [ [[<]] ] = "<", - [ [[>]] ] = ">", +local predefined_simplified={ + [38]="&",amp="&", + [42]='"',quot='"', + [47]="'",apos="'", + [74]="<",lt="<", + [76]=">",gt=">", } - -local privates_p = { +local nofprivates=0xF0000 +local privates_u={ + [ [[&]] ]="&", + [ [["]] ]=""", + [ [[']] ]="'", + [ [[<]] ]="<", + [ [[>]] ]=">", } - -local privates_n = { - -- keeps track of defined ones +local privates_p={} +local privates_n={ } - -local escaped = utf.remapper(privates_u) - +local escaped=utf.remapper(privates_u) local function unescaped(s) - local p = privates_n[s] - if not p then - nofprivates = nofprivates + 1 - p = utfchar(nofprivates) - privates_n[s] = p - s = "&" .. s .. ";" -- todo: use char-ent to map to hex - privates_u[p] = s - privates_p[p] = s - end - return p -end - -local unprivatized = utf.remapper(privates_p) - -xml.privatetoken = unescaped -xml.unprivatized = unprivatized -xml.privatecodes = privates_n - + local p=privates_n[s] + if not p then + nofprivates=nofprivates+1 + p=utfchar(nofprivates) + privates_n[s]=p + s="&"..s..";" + privates_u[p]=s + privates_p[p]=s + end + return p +end +local unprivatized=utf.remapper(privates_p) +xml.privatetoken=unescaped +xml.unprivatized=unprivatized +xml.privatecodes=privates_n local function handle_hex_entity(str) - local h = hcache[str] - if not h then - local n = tonumber(str,16) - h = unify_predefined and predefined_unified[n] - if h then - if trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %s",str,h) - end - elseif utfize then - h = (n and utfchar(n)) or xml.unknown_hex_entity(str) or "" - if not n then - report_xml("utfize, ignoring hex entity &#x%s;",str) - elseif trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %s",str,h) - end - else - if trace_entities then - report_xml("found entity &#x%s;",str) - end - h = "&#x" .. str .. ";" - end - hcache[str] = h + local h=hcache[str] + if not h then + local n=tonumber(str,16) + h=unify_predefined and predefined_unified[n] + if h then + if trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %s",str,h) + end + elseif utfize then + h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" + if not n then + report_xml("utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %s",str,h) + end + else + if trace_entities then + report_xml("found entity &#x%s;",str) + end + h="&#x"..str..";" end - return h + hcache[str]=h + end + return h end - local function handle_dec_entity(str) - local d = dcache[str] - if not d then - local n = tonumber(str) - d = unify_predefined and predefined_unified[n] - if d then + local d=dcache[str] + if not d then + local n=tonumber(str) + d=unify_predefined and predefined_unified[n] + if d then + if trace_entities then + report_xml("utfize, converting dec entity &#%s; into %s",str,d) + end + elseif utfize then + d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" + if not n then + report_xml("utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + report_xml("utfize, converting dec entity &#%s; into %s",str,d) + end + else + if trace_entities then + report_xml("found entity &#%s;",str) + end + d="&#"..str..";" + end + dcache[str]=d + end + return d +end +xml.parsedentitylpeg=parsedentity +local function handle_any_entity(str) + if resolve then + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + if trace_entities then + report_xml("resolved entity &%s; -> %s (predefined)",str,a) + end + else + if type(resolve)=="function" then + a=resolve(str) or entities[str] + else + a=entities[str] + end + if a then + if type(a)=="function" then if trace_entities then - report_xml("utfize, converting dec entity &#%s; into %s",str,d) - end - elseif utfize then - d = (n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" - if not n then - report_xml("utfize, ignoring dec entity &#%s;",str) - elseif trace_entities then - report_xml("utfize, converting dec entity &#%s; into %s",str,d) + report_xml("expanding entity &%s; (function)",str) end + a=a(str) or "" + end + a=lpegmatch(parsedentity,a) or a + if trace_entities then + report_xml("resolved entity &%s; -> %s (internal)",str,a) + end else + local unknown_any_entity=placeholders.unknown_any_entity + if unknown_any_entity then + a=unknown_any_entity(str) or "" + end + if a then if trace_entities then - report_xml("found entity &#%s;",str) - end - d = "&#" .. str .. ";" - end - dcache[str] = d - end - return d -end - -xml.parsedentitylpeg = parsedentity - -local function handle_any_entity(str) - if resolve then - local a = acache[str] -- per instance ! todo - if not a then - a = resolve_predefined and predefined_simplified[str] - if a then - if trace_entities then - report_xml("resolved entity &%s; -> %s (predefined)",str,a) - end - else - if type(resolve) == "function" then - a = resolve(str) or entities[str] - else - a = entities[str] - end - if a then - if type(a) == "function" then - if trace_entities then - report_xml("expanding entity &%s; (function)",str) - end - a = a(str) or "" - end - a = lpegmatch(parsedentity,a) or a -- for nested - if trace_entities then - report_xml("resolved entity &%s; -> %s (internal)",str,a) - end - else - local unknown_any_entity = placeholders.unknown_any_entity - if unknown_any_entity then - a = unknown_any_entity(str) or "" - end - if a then - if trace_entities then - report_xml("resolved entity &%s; -> %s (external)",str,a) - end - else - if trace_entities then - report_xml("keeping entity &%s;",str) - end - if str == "" then - a = "&error;" - else - a = "&" .. str .. ";" - end - end - end + report_xml("resolved entity &%s; -> %s (external)",str,a) end - acache[str] = a - elseif trace_entities then - if not acache[str] then - report_xml("converting entity &%s; into %s",str,a) - acache[str] = a + else + if trace_entities then + report_xml("keeping entity &%s;",str) end - end - return a - else - local a = acache[str] - if not a then - a = resolve_predefined and predefined_simplified[str] - if a then - -- one of the predefined - acache[str] = a - if trace_entities then - report_xml("entity &%s; becomes %s",str,tostring(a)) - end - elseif str == "" then - if trace_entities then - report_xml("invalid entity &%s;",str) - end - a = "&error;" - acache[str] = a + if str=="" then + a="&error;" else - if trace_entities then - report_xml("entity &%s; is made private",str) - end - -- a = "&" .. str .. ";" - a = unescaped(str) - acache[str] = a + a="&"..str..";" end + end end - return a + end + acache[str]=a + elseif trace_entities then + if not acache[str] then + report_xml("converting entity &%s; into %s",str,a) + acache[str]=a + end + end + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %s",str,tostring(a)) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a="&error;" + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a + end end + return a + end end - local function handle_end_entity(chr) - report_xml("error in entity, %q found instead of ';'",chr) -end - -local space = S(' \r\n\t') -local open = P('<') -local close = P('>') -local squote = S("'") -local dquote = S('"') -local equal = P('=') -local slash = P('/') -local colon = P(':') -local semicolon = P(';') -local ampersand = P('&') -local valid = R('az', 'AZ', '09') + S('_-.') -local name_yes = C(valid^1) * colon * C(valid^1) -local name_nop = C(P(true)) * C(valid^1) -local name = name_yes + name_nop -local utfbom = lpeg.patterns.utfbom -- no capture -local spacing = C(space^0) - ------ entitycontent = (1-open-semicolon)^0 -local anyentitycontent = (1-open-semicolon-space-close)^0 -local hexentitycontent = R("AF","af","09")^0 -local decentitycontent = R("09")^0 -local parsedentity = P("#")/"" * ( - P("x")/"" * (hexentitycontent/handle_hex_entity) + - (decentitycontent/handle_dec_entity) - ) + (anyentitycontent/handle_any_entity) -local entity = ampersand/"" * parsedentity * ( (semicolon/"") + #(P(1)/handle_end_entity)) - -local text_unparsed = C((1-open)^1) -local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) - -local somespace = space^1 -local optionalspace = space^0 - ------ value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value -local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value - -local endofattributes = slash * close + close -- recovery of flacky html -local whatever = space * name * optionalspace * equal ------ wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error ------ wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error ------ wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error -local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error - -local attributevalue = value + wrongvalue - -local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute ------ attributes = (attribute)^0 - -local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 - -local parsedtext = text_parsed / add_text -local unparsedtext = text_unparsed / add_text -local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example - -local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty -local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin -local endelement = (spacing * open * slash * name * optionalspace * close) / add_end - -local begincomment = open * P("!--") -local endcomment = P("--") * close -local begininstruction = open * P("?") -local endinstruction = P("?") * close -local begincdata = open * P("![CDATA[") -local endcdata = P("]]") * close - -local someinstruction = C((1 - endinstruction)^0) -local somecomment = C((1 - endcomment )^0) -local somecdata = C((1 - endcdata )^0) - -local function normalentity(k,v ) entities[k] = v end -local function systementity(k,v,n) entities[k] = v end -local function publicentity(k,v,n) entities[k] = v end - --- todo: separate dtd parser - -local begindoctype = open * P("!DOCTYPE") -local enddoctype = close -local beginset = P("[") -local endset = P("]") -local doctypename = C((1-somespace-close)^0) -local elementdoctype = optionalspace * P("') +local squote=S("'") +local dquote=S('"') +local equal=P('=') +local slash=P('/') +local colon=P(':') +local semicolon=P(';') +local ampersand=P('&') +local valid=R('az','AZ','09')+S('_-.') +local name_yes=C(valid^1)*colon*C(valid^1) +local name_nop=C(P(true))*C(valid^1) +local name=name_yes+name_nop +local utfbom=lpeg.patterns.utfbom +local spacing=C(space^0) +local anyentitycontent=(1-open-semicolon-space-close)^0 +local hexentitycontent=R("AF","af","09")^0 +local decentitycontent=R("09")^0 +local parsedentity=P("#")/""*( + P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) + )+(anyentitycontent/handle_any_entity) +local entity=ampersand/""*parsedentity*((semicolon/"")+#(P(1)/handle_end_entity)) +local text_unparsed=C((1-open)^1) +local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) +local somespace=space^1 +local optionalspace=space^0 +local value=(squote*Cs((entity+(1-squote))^0)*squote)+(dquote*Cs((entity+(1-dquote))^0)*dquote) +local endofattributes=slash*close+close +local whatever=space*name*optionalspace*equal +local wrongvalue=Cs(P(entity+(1-space-endofattributes))^1)/attribute_value_error +local attributevalue=value+wrongvalue +local attribute=(somespace*name*optionalspace*equal*optionalspace*attributevalue)/add_attribute +local attributes=(attribute+somespace^-1*(((1-endofattributes)^1)/attribute_specification_error))^0 +local parsedtext=text_parsed/add_text +local unparsedtext=text_unparsed/add_text +local balanced=P { "["*((1-S"[]")+V(1))^0*"]" } +local emptyelement=(spacing*open*name*attributes*optionalspace*slash*close)/add_empty +local beginelement=(spacing*open*name*attributes*optionalspace*close)/add_begin +local endelement=(spacing*open*slash*name*optionalspace*close)/add_end +local begincomment=open*P("!--") +local endcomment=P("--")*close +local begininstruction=open*P("?") +local endinstruction=P("?")*close +local begincdata=open*P("![CDATA[") +local endcdata=P("]]")*close +local someinstruction=C((1-endinstruction)^0) +local somecomment=C((1-endcomment )^0) +local somecdata=C((1-endcdata )^0) +local function normalentity(k,v ) entities[k]=v end +local function systementity(k,v,n) entities[k]=v end +local function publicentity(k,v,n) entities[k]=v end +local begindoctype=open*P("!DOCTYPE") +local enddoctype=close +local beginset=P("[") +local endset=P("]") +local doctypename=C((1-somespace-close)^0) +local elementdoctype=optionalspace*P(" & - cleanup = settings.text_cleanup - entities = settings.entities or { } - -- - if utfize == nil then - settings.utfize_entities = true - utfize = true - end - if resolve_predefined == nil then - settings.resolve_predefined_entities = true - resolve_predefined = true - end - -- - -- - stack, top, at, xmlns, errorstr = { }, { }, { }, { }, nil - acache, hcache, dcache = { }, { }, { } -- not stored - reported_attribute_errors = { } - if settings.parent_root then - mt = getmetatable(settings.parent_root) - else - initialize_mt(top) - end - stack[#stack+1] = top - top.dt = { } - dt = top.dt - if not data or data == "" then - errorstr = "empty xml file" - elseif utfize or resolve then - if lpegmatch(grammar_parsed_text,data) then - errorstr = "" - else - errorstr = "invalid xml file - parsed text" - end - elseif type(data) == "string" then - if lpegmatch(grammar_unparsed_text,data) then - errorstr = "" - else - errorstr = "invalid xml file - unparsed text" - end - else - errorstr = "invalid xml file - no text at all" - end - local result - if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={ }, er = true } } } - setmetatable(stack, mt) - local errorhandler = settings.error_handler - if errorhandler == false then - -- no error message - else - errorhandler = errorhandler or xml.errorhandler - if errorhandler then - local currentresource = settings.currentresource - if currentresource and currentresource ~= "" then - xml.errorhandler(format("load error in [%s]: %s",currentresource,errorstr)) - else - xml.errorhandler(format("load error: %s",errorstr)) - end - end - end - else - result = stack[1] - end - if not settings.no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={ }, entities = entities, settings = settings } - setmetatable(result, mt) - local rdt = result.dt - for k=1,#rdt do - local v = rdt[k] - if type(v) == "table" and not v.special then -- always table -) - result.ri = k -- rootindex - v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! - break - end +local function _xmlconvert_(data,settings) + settings=settings or {} + strip=settings.strip_cm_and_dt + utfize=settings.utfize_entities + resolve=settings.resolve_entities + resolve_predefined=settings.resolve_predefined_entities + unify_predefined=settings.unify_predefined_entities + cleanup=settings.text_cleanup + entities=settings.entities or {} + if utfize==nil then + settings.utfize_entities=true + utfize=true + end + if resolve_predefined==nil then + settings.resolve_predefined_entities=true + resolve_predefined=true + end + stack,top,at,xmlns,errorstr={},{},{},{},nil + acache,hcache,dcache={},{},{} + reported_attribute_errors={} + if settings.parent_root then + mt=getmetatable(settings.parent_root) + else + initialize_mt(top) + end + stack[#stack+1]=top + top.dt={} + dt=top.dt + if not data or data=="" then + errorstr="empty xml file" + elseif utfize or resolve then + if lpegmatch(grammar_parsed_text,data) then + errorstr="" + else + errorstr="invalid xml file - parsed text" + end + elseif type(data)=="string" then + if lpegmatch(grammar_unparsed_text,data) then + errorstr="" + else + errorstr="invalid xml file - unparsed text" + end + else + errorstr="invalid xml file - no text at all" + end + local result + if errorstr and errorstr~="" then + result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } + setmetatable(stack,mt) + local errorhandler=settings.error_handler + if errorhandler==false then + else + errorhandler=errorhandler or xml.errorhandler + if errorhandler then + local currentresource=settings.currentresource + if currentresource and currentresource~="" then + xml.errorhandler(format("load error in [%s]: %s",currentresource,errorstr)) + else + xml.errorhandler(format("load error: %s",errorstr)) end + end end - if errorstr and errorstr ~= "" then - result.error = true + else + result=stack[1] + end + if not settings.no_root then + result={ special=true,ns="",tg='@rt@',dt=result.dt,at={},entities=entities,settings=settings } + setmetatable(result,mt) + local rdt=result.dt + for k=1,#rdt do + local v=rdt[k] + if type(v)=="table" and not v.special then + result.ri=k + v.__p__=result + break + end end - result.statistics = { - entities = { - decimals = dcache, - hexadecimals = hcache, - names = acache, - } + end + if errorstr and errorstr~="" then + result.error=true + end + result.statistics={ + entities={ + decimals=dcache, + hexadecimals=hcache, + names=acache, } - strip, utfize, resolve, resolve_predefined = nil, nil, nil, nil - unify_predefined, cleanup, entities = nil, nil, nil - stack, top, at, xmlns, errorstr = nil, nil, nil, nil, nil - acache, hcache, dcache = nil, nil, nil - reported_attribute_errors, mt, errorhandler = nil, nil, nil - return result + } + strip,utfize,resolve,resolve_predefined=nil,nil,nil,nil + unify_predefined,cleanup,entities=nil,nil,nil + stack,top,at,xmlns,errorstr=nil,nil,nil,nil,nil + acache,hcache,dcache=nil,nil,nil + reported_attribute_errors,mt,errorhandler=nil,nil,nil + return result end - --- Because we can have a crash (stack issues) with faulty xml, we wrap this one --- in a protector: - function xmlconvert(data,settings) - local ok, result = pcall(function() return _xmlconvert_(data,settings) end) - if ok then - return result - else - return _xmlconvert_("",settings) - end -end - -xml.convert = xmlconvert - -function xml.inheritedconvert(data,xmldata) -- xmldata is parent - local settings = xmldata.settings - if settings then - settings.parent_root = xmldata -- to be tested - end - -- settings.no_root = true - local xc = xmlconvert(data,settings) -- hm, we might need to locate settings - -- xc.settings = nil - -- xc.entities = nil - -- xc.special = nil - -- xc.ri = nil - -- print(xc.tg) - return xc + local ok,result=pcall(function() return _xmlconvert_(data,settings) end) + if ok then + return result + else + return _xmlconvert_("",settings) + end +end +xml.convert=xmlconvert +function xml.inheritedconvert(data,xmldata) + local settings=xmldata.settings + if settings then + settings.parent_root=xmldata + end + local xc=xmlconvert(data,settings) + return xc end - - - function xml.is_valid(root) - return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er + return root and root.dt and root.dt[1] and type(root.dt[1])=="table" and not root.dt[1].er end - function xml.package(tag,attributes,data) - local ns, tg = match(tag,"^(.-):?([^:]+)$") - local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } - setmetatable(t, mt) - return t + local ns,tg=match(tag,"^(.-):?([^:]+)$") + local t={ ns=ns,tg=tg,dt=data or "",at=attributes or {} } + setmetatable(t,mt) + return t end - function xml.is_valid(root) - return root and not root.error + return root and not root.error end - -xml.errorhandler = report_xml - - - +xml.errorhandler=report_xml function xml.load(filename,settings) - local data = "" - if type(filename) == "string" then - -- local data = io.loaddata(filename) - -todo: check type in io.loaddata - local f = io.open(filename,'r') -- why not 'rb' - if f then - data = f:read("*all") -- io.readall(f) ... only makes sense for large files - f:close() - end - elseif filename then -- filehandle - data = filename:read("*all") -- io.readall(f) ... only makes sense for large files - end - if settings then - settings.currentresource = filename - local result = xmlconvert(data,settings) - settings.currentresource = nil - return result - else - return xmlconvert(data,{ currentresource = filename }) - end + local data="" + if type(filename)=="string" then + local f=io.open(filename,'r') + if f then + data=f:read("*all") + f:close() + end + elseif filename then + data=filename:read("*all") + end + if settings then + settings.currentresource=filename + local result=xmlconvert(data,settings) + settings.currentresource=nil + return result + else + return xmlconvert(data,{ currentresource=filename }) + end end - - - -local no_root = { no_root = true } - +local no_root={ no_root=true } function xml.toxml(data) - if type(data) == "string" then - local root = { xmlconvert(data,no_root) } - return (#root > 1 and root) or root[1] - else - return data - end + if type(data)=="string" then + local root={ xmlconvert(data,no_root) } + return (#root>1 and root) or root[1] + else + return data + end end - - - local function copy(old,tables) - if old then - tables = tables or { } - local new = { } - if not tables[old] then - tables[old] = new - end - for k,v in next, old do - new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) - end - return new - else - return { } + if old then + tables=tables or {} + local new={} + if not tables[old] then + tables[old]=new end -end - -xml.copy = copy - - - --- todo: add when not present - -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt = root.dt - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then - return - end - end - insert(dt, 1, { special = true, ns = "", tg = "@pi@", dt = { "xml version='1.0' standalone='yes'" } } ) - insert(dt, 2, "\n" ) + for k,v in next,old do + new[k]=(type(v)=="table" and (tables[v] or copy(v,tables))) or v end -end - - - --- new experimental reorganized serialize - -local function verbose_element(e,handlers) -- options - local handle = handlers.handle - local serialize = handlers.serialize - local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } - if ats then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,escaped(v)) - end + local mt=getmetatable(old) + if mt then + setmetatable(new,mt) end - if ern and trace_entities and ern ~= ens then - ens = ern + return new + else + return {} + end +end +xml.copy=copy +function xml.checkbom(root) + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" and find(v.dt[1],"xml.*version=") then + return + end end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - handle("<",ens,":",etg," ",concat(ats," "),">") - else - handle("<",ens,":",etg,">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - handle(escaped(e)) - else - serialize(e,handlers) - end - end - handle("") + insert(dt,1,{ special=true,ns="",tg="@pi@",dt={ "xml version='1.0' standalone='yes'" } } ) + insert(dt,2,"\n" ) + end +end +local function verbose_element(e,handlers) + local handle=handlers.handle + local serialize=handlers.serialize + local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn + local ats=eat and next(eat) and {} + if ats then + for k,v in next,eat do + ats[#ats+1]=format('%s=%q',k,escaped(v)) + end + end + if ern and trace_entities and ern~=ens then + ens=ern + end + if ens~="" then + if edt and #edt>0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) else - if ats then - handle("<",ens,":",etg," ",concat(ats," "),"/>") - else - handle("<",ens,":",etg,"/>") - end + serialize(e,handlers) end + end + handle("") else - if edt and #edt > 0 then - if ats then - handle("<",etg," ",concat(ats," "),">") - else - handle("<",etg,">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - handle(escaped(e)) -- option: hexify escaped entities - else - serialize(e,handlers) - end - end - handle("") + if ats then + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") + end + end + else + if edt and #edt>0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) else - if ats then - handle("<",etg," ",concat(ats," "),"/>") - else - handle("<",etg,"/>") - end + serialize(e,handlers) end + end + handle("") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") + else + handle("<",etg,"/>") + end end + end end - local function verbose_pi(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_comment(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_cdata(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_doctype(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_root(e,handlers) - handlers.serialize(e.dt,handlers) + handlers.serialize(e.dt,handlers) end - local function verbose_text(e,handlers) - handlers.handle(escaped(e)) + handlers.handle(escaped(e)) end - local function verbose_document(e,handlers) - local serialize = handlers.serialize - local functions = handlers.functions - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - functions["@tx@"](ei,handlers) - else - serialize(ei,handlers) - end + local serialize=handlers.serialize + local functions=handlers.functions + for i=1,#e do + local ei=e[i] + if type(ei)=="string" then + functions["@tx@"](ei,handlers) + else + serialize(ei,handlers) end + end end - local function serialize(e,handlers,...) - local initialize = handlers.initialize - local finalize = handlers.finalize - local functions = handlers.functions - if initialize then - local state = initialize(...) - if not state == true then - return state - end - end - local etg = e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - -- elseif type(e) == "string" then - -- functions["@tx@"](e,handlers) - else - functions["@dc@"](e,handlers) -- dc ? - end - if finalize then - return finalize() - end + local initialize=handlers.initialize + local finalize=handlers.finalize + local functions=handlers.functions + if initialize then + local state=initialize(...) + if not state==true then + return state + end + end + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end - local function xserialize(e,handlers) - local functions = handlers.functions - local etg = e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - -- elseif type(e) == "string" then - -- functions["@tx@"](e,handlers) - else - functions["@dc@"](e,handlers) - end -end - -local handlers = { } - + local functions=handlers.functions + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end +end +local handlers={} local function newhandlers(settings) - local t = table.copy(handlers.verbose or { }) -- merge - if settings then - for k,v in next, settings do - if type(v) == "table" then - local tk = t[k] if not tk then tk = { } t[k] = tk end - for kk,vv in next, v do - tk[kk] = vv - end - else - t[k] = v - end - end - if settings.name then - handlers[settings.name] = t - end + local t=table.copy(handlers.verbose or {}) + if settings then + for k,v in next,settings do + if type(v)=="table" then + local tk=t[k] if not tk then tk={} t[k]=tk end + for kk,vv in next,v do + tk[kk]=vv + end + else + t[k]=v + end end - utilities.storage.mark(t) - return t + if settings.name then + handlers[settings.name]=t + end + end + utilities.storage.mark(t) + return t end - -local nofunction = function() end - +local nofunction=function() end function xml.sethandlersfunction(handler,name,fnc) - handler.functions[name] = fnc or nofunction + handler.functions[name]=fnc or nofunction end - function xml.gethandlersfunction(handler,name) - return handler.functions[name] + return handler.functions[name] end - function xml.gethandlers(name) - return handlers[name] + return handlers[name] end - newhandlers { - name = "verbose", - initialize = false, -- faster than nil and mt lookup - finalize = false, -- faster than nil and mt lookup - serialize = xserialize, - handle = print, - functions = { - ["@dc@"] = verbose_document, - ["@dt@"] = verbose_doctype, - ["@rt@"] = verbose_root, - ["@el@"] = verbose_element, - ["@pi@"] = verbose_pi, - ["@cm@"] = verbose_comment, - ["@cd@"] = verbose_cdata, - ["@tx@"] = verbose_text, - } + name="verbose", + initialize=false, + finalize=false, + serialize=xserialize, + handle=print, + functions={ + ["@dc@"]=verbose_document, + ["@dt@"]=verbose_doctype, + ["@rt@"]=verbose_root, + ["@el@"]=verbose_element, + ["@pi@"]=verbose_pi, + ["@cm@"]=verbose_comment, + ["@cd@"]=verbose_cdata, + ["@tx@"]=verbose_text, + } } - - - --- maybe this will move to lxml-xml - local result - -local xmlfilehandler = newhandlers { - name = "file", - initialize = function(name) - result = io.open(name,"wb") - return result - end, - finalize = function() - result:close() - return true - end, - handle = function(...) - result:write(...) - end, +local xmlfilehandler=newhandlers { + name="file", + initialize=function(name) + result=io.open(name,"wb") + return result + end, + finalize=function() + result:close() + return true + end, + handle=function(...) + result:write(...) + end, } - --- no checking on writeability here but not faster either --- --- local xmlfilehandler = newhandlers { --- initialize = function(name) --- io.output(name,"wb") --- return true --- end, --- finalize = function() --- io.close() --- return true --- end, --- handle = io.write, --- } - function xml.save(root,name) - serialize(root,xmlfilehandler,name) + serialize(root,xmlfilehandler,name) end - local result - -local xmlstringhandler = newhandlers { - name = "string", - initialize = function() - result = { } - return result - end, - finalize = function() - return concat(result) - end, - handle = function(...) - result[#result+1] = concat { ... } - end, +local xmlstringhandler=newhandlers { + name="string", + initialize=function() + result={} + return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, } - -local function xmltostring(root) -- 25% overhead due to collecting - if not root then - return "" - elseif type(root) == 'string' then - return root - else -- if next(root) then -- next is faster than type (and >0 test) - return serialize(root,xmlstringhandler) or "" - end +local function xmltostring(root) + if not root then + return "" + elseif type(root)=='string' then + return root + else + return serialize(root,xmlstringhandler) or "" + end end - -local function __tostring(root) -- inline - return (root and xmltostring(root)) or "" +local function __tostring(root) + return (root and xmltostring(root)) or "" end - -initialize_mt = function(root) -- redefinition - mt = { __tostring = __tostring, __index = root } +initialize_mt=function(root) + mt={ __tostring=__tostring,__index=root } end - -xml.defaulthandlers = handlers -xml.newhandlers = newhandlers -xml.serialize = serialize -xml.tostring = xmltostring - - - +xml.defaulthandlers=handlers +xml.newhandlers=newhandlers +xml.serialize=serialize +xml.tostring=xmltostring local function xmlstring(e,handle) - if not handle or (e.special and e.tg ~= "@rt@") then - -- nothing - elseif e.tg then - local edt = e.dt - if edt then - for i=1,#edt do - xmlstring(edt[i],handle) - end - end - else - handle(e) + if not handle or (e.special and e.tg~="@rt@") then + elseif e.tg then + local edt=e.dt + if edt then + for i=1,#edt do + xmlstring(edt[i],handle) + end end + else + handle(e) + end end - -xml.string = xmlstring - - - - +xml.string=xmlstring function xml.settings(e) - while e do - local s = e.settings - if s then - return s - else - e = e.__p__ - end + while e do + local s=e.settings + if s then + return s + else + e=e.__p__ end - return nil + end + return nil end - function xml.root(e) - local r = e - while e do - e = e.__p__ - if e then - r = e - end + local r=e + while e do + e=e.__p__ + if e then + r=e end - return r + end + return r end - function xml.parent(root) - return root.__p__ + return root.__p__ end - function xml.body(root) - return root.ri and root.dt[root.ri] or root -- not ok yet + return root.ri and root.dt[root.ri] or root end - function xml.name(root) - if not root then - return "" - end - local ns = root.ns - local tg = root.tg - if ns == "" then - return tg - else - return ns .. ":" .. tg - end + if not root then + return "" + end + local ns=root.ns + local tg=root.tg + if ns=="" then + return tg + else + return ns..":"..tg + end end - - - function xml.erase(dt,k) - if dt then - if k then - dt[k] = "" - else for k=1,#dt do - dt[1] = { "" } - end end - end + if dt then + if k then + dt[k]="" + else for k=1,#dt do + dt[1]={ "" } + end end + end end - - - function xml.assign(dt,k,root) - if dt and k then - dt[k] = type(root) == "table" and xml.body(root) or root - return dt[k] - else - return xml.body(root) - end + if dt and k then + dt[k]=type(root)=="table" and xml.body(root) or root + return dt[k] + else + return xml.body(root) + end +end +function xml.tocdata(e,wrapper) + local whatever=type(e)=="table" and xmltostring(e.dt) or e or "" + if wrapper then + whatever=format("<%s>%s",wrapper,whatever,wrapper) + end + local t={ special=true,ns="",tg="@cd@",at={},rn="",dt={ whatever },__p__=e } + setmetatable(t,getmetatable(e)) + e.dt={ t } end - --- the following helpers may move - - - -function xml.tocdata(e,wrapper) -- a few more in the aux module - local whatever = type(e) == "table" and xmltostring(e.dt) or e or "" - if wrapper then - whatever = format("<%s>%s",wrapper,whatever,wrapper) - end - local t = { special = true, ns = "", tg = "@cd@", at = { }, rn = "", dt = { whatever }, __p__ = e } - setmetatable(t,getmetatable(e)) - e.dt = { t } -end - function xml.makestandalone(root) - if root.ri then - local dt = root.dt - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi@" then - local txt = v.dt[1] - if find(txt,"xml.*version=") then - v.dt[1] = txt .. " standalone='yes'" - break - end - end + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" then + local txt=v.dt[1] + if find(txt,"xml.*version=") then + v.dt[1]=txt.." standalone='yes'" + break end + end end - return root + end + return root end - function xml.kind(e) - local dt = e and e.dt - if dt then - local n = #dt - if n == 1 then - local d = dt[1] - if d.special then - local tg = d.tg - if tg == "@cd@" then - return "cdata" - elseif tg == "@cm" then - return "comment" - elseif tg == "@pi@" then - return "instruction" - elseif tg == "@dt@" then - return "declaration" - end - elseif type(d) == "string" then - return "text" - end - return "element" - elseif n > 0 then - return "mixed" - end + local dt=e and e.dt + if dt then + local n=#dt + if n==1 then + local d=dt[1] + if d.special then + local tg=d.tg + if tg=="@cd@" then + return "cdata" + elseif tg=="@cm" then + return "comment" + elseif tg=="@pi@" then + return "instruction" + elseif tg=="@dt@" then + return "declaration" + end + elseif type(d)=="string" then + return "text" + end + return "element" + elseif n>0 then + return "mixed" end - return "empty" + end + return "empty" end @@ -10804,1308 +8138,1036 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-lpt'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- e.ni is only valid after a filter run --- todo: B/C/[get first match] - -local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, tonumber, tostring, setmetatable, load, select = type, next, tonumber, tostring, setmetatable, load, select -local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns - -local setmetatableindex = table.setmetatableindex - --- beware, this is not xpath ... e.g. position is different (currently) and --- we have reverse-sibling as reversed preceding sibling - - - - - -local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end -local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end -local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end - -local report_lpath = logs.reporter("xml","lpath") - - - -local xml = xml - -local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end -local lpathcached = 0 function xml.lpathcached() return lpathcached end - -xml.functions = xml.functions or { } -- internal -local functions = xml.functions - -xml.expressions = xml.expressions or { } -- in expressions -local expressions = xml.expressions - -xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) -local finalizers = xml.finalizers - -xml.specialhandler = xml.specialhandler or { } -local specialhandler = xml.specialhandler - -lpegpatterns.xml = lpegpatterns.xml or { } -local xmlpatterns = lpegpatterns.xml - -finalizers.xml = finalizers.xml or { } -finalizers.tex = finalizers.tex or { } - -local function fallback (t, name) - local fn = finalizers[name] - if fn then - t[name] = fn - else - report_lpath("unknown sub finalizer '%s'",tostring(name)) - fn = function() end - end - return fn -end - -setmetatableindex(finalizers.xml, fallback) -setmetatableindex(finalizers.tex, fallback) - -xml.defaultprotocol = "xml" - --- as xsl does not follow xpath completely here we will also --- be more liberal especially with regards to the use of | and --- the rootpath: --- --- test : all 'test' under current --- /test : 'test' relative to current --- a|b|c : set of names --- (a|b|c) : idem --- ! : not --- --- after all, we're not doing transformations but filtering. in --- addition we provide filter functions (last bit) --- --- todo: optimizer --- --- .. : parent --- * : all kids --- / : anchor here --- // : /**/ --- ** : all in between --- --- so far we had (more practical as we don't transform) --- --- {/test} : kids 'test' under current node --- {test} : any kid with tag 'test' --- {//test} : same as above - --- evaluator (needs to be redone, for the moment copied) +-- original size: 47510, stripped down to: 30425 --- todo: apply_axis(list,notable) and collection vs single - -local apply_axis = { } - -apply_axis['root'] = function(list) - local collected = { } - for l=1,#list do - local ll = list[l] - local rt = ll - while ll do - ll = ll.__p__ - if ll then - rt = ll - end - end - collected[l] = rt +if not modules then modules={} end modules ['lxml-lpt']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,tonumber,tostring,setmetatable,load,select=type,next,tonumber,tostring,setmetatable,load,select +local format,upper,lower,gmatch,gsub,find,rep=string.format,string.upper,string.lower,string.gmatch,string.gsub,string.find,string.rep +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local setmetatableindex=table.setmetatableindex +local trace_lpath=false if trackers then trackers.register("xml.path",function(v) trace_lpath=v end) end +local trace_lparse=false if trackers then trackers.register("xml.parse",function(v) trace_lparse=v end) end +local trace_lprofile=false if trackers then trackers.register("xml.profile",function(v) trace_lpath=v trace_lparse=v trace_lprofile=v end) end +local report_lpath=logs.reporter("xml","lpath") +local xml=xml +local lpathcalls=0 function xml.lpathcalls () return lpathcalls end +local lpathcached=0 function xml.lpathcached() return lpathcached end +xml.functions=xml.functions or {} +local functions=xml.functions +xml.expressions=xml.expressions or {} +local expressions=xml.expressions +xml.finalizers=xml.finalizers or {} +local finalizers=xml.finalizers +xml.specialhandler=xml.specialhandler or {} +local specialhandler=xml.specialhandler +lpegpatterns.xml=lpegpatterns.xml or {} +local xmlpatterns=lpegpatterns.xml +finalizers.xml=finalizers.xml or {} +finalizers.tex=finalizers.tex or {} +local function fallback (t,name) + local fn=finalizers[name] + if fn then + t[name]=fn + else + report_lpath("unknown sub finalizer '%s'",tostring(name)) + fn=function() end + end + return fn +end +setmetatableindex(finalizers.xml,fallback) +setmetatableindex(finalizers.tex,fallback) +xml.defaultprotocol="xml" +local apply_axis={} +apply_axis['root']=function(list) + local collected={} + for l=1,#list do + local ll=list[l] + local rt=ll + while ll do + ll=ll.__p__ + if ll then + rt=ll + end end - return collected -end - -apply_axis['self'] = function(list) - return list -end - -apply_axis['child'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local dt = ll.dt - if dt then -- weird that this is needed - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - end - end - ll.en = en + collected[l]=rt + end + return collected +end +apply_axis['self']=function(list) + return list +end +apply_axis['child']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local dt=ll.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en end + end + ll.en=en end - return collected + end + return collected end - local function collect(list,collected,c) - local dt = list.dt - if dt then - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - c = collect(dk,collected,c) - end - end - list.en = en + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end end - return c + list.en=en + end + return c end - -apply_axis['descendant'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - c = collect(list[l],collected,c) - end - return collected +apply_axis['descendant']=function(list) + local collected,c={},0 + for l=1,#list do + c=collect(list[l],collected,c) + end + return collected end - local function collect(list,collected,c) - local dt = list.dt - if dt then - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - c = collect(dk,collected,c) - end - end - list.en = en - end - return c -end -apply_axis['descendant-or-self'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - if ll.special ~= true then -- catch double root - c = c + 1 - collected[c] = ll - end - c = collect(ll,collected,c) - end - return collected -end - -apply_axis['ancestor'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - while ll do - ll = ll.__p__ - if ll then - c = c + 1 - collected[c] = ll - end - end + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end end - return collected -end - -apply_axis['ancestor-or-self'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - c = c + 1 - collected[c] = ll - while ll do - ll = ll.__p__ - if ll then - c = c + 1 - collected[c] = ll - end - end + list.en=en + end + return c +end +apply_axis['descendant-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + if ll.special~=true then + c=c+1 + collected[c]=ll + end + c=collect(ll,collected,c) + end + return collected +end +apply_axis['ancestor']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end end - return collected -end - -apply_axis['parent'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local pl = list[l].__p__ - if pl then - c = c + 1 - collected[c] = pl - end + end + return collected +end +apply_axis['ancestor-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + c=c+1 + collected[c]=ll + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end end - return collected -end - -apply_axis['attribute'] = function(list) - return { } -end - -apply_axis['namespace'] = function(list) - return { } -end - -apply_axis['following'] = function(list) -- incomplete - return { } -end - -apply_axis['preceding'] = function(list) -- incomplete - return { } -end - -apply_axis['following-sibling'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=ll.ni+1,#d do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end + end + return collected +end +apply_axis['parent']=function(list) + local collected,c={},0 + for l=1,#list do + local pl=list[l].__p__ + if pl then + c=c+1 + collected[c]=pl + end + end + return collected +end +apply_axis['attribute']=function(list) + return {} +end +apply_axis['namespace']=function(list) + return {} +end +apply_axis['following']=function(list) + return {} +end +apply_axis['preceding']=function(list) + return {} +end +apply_axis['following-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni+1,#d do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end end - return collected -end - -apply_axis['preceding-sibling'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=1,ll.ni-1 do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end + end + return collected +end +apply_axis['preceding-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=1,ll.ni-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end end - return collected -end - -apply_axis['reverse-sibling'] = function(list) -- reverse preceding - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=ll.ni-1,1,-1 do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end + end + return collected +end +apply_axis['reverse-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni-1,1,-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end end - return collected + end + return collected end - -apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] -apply_axis['auto-descendant'] = apply_axis['descendant'] -apply_axis['auto-child'] = apply_axis['child'] -apply_axis['auto-self'] = apply_axis['self'] -apply_axis['initial-child'] = apply_axis['child'] - +apply_axis['auto-descendant-or-self']=apply_axis['descendant-or-self'] +apply_axis['auto-descendant']=apply_axis['descendant'] +apply_axis['auto-child']=apply_axis['child'] +apply_axis['auto-self']=apply_axis['self'] +apply_axis['initial-child']=apply_axis['child'] local function apply_nodes(list,directive,nodes) - -- todo: nodes[1] etc ... negated node name in set ... when needed - -- ... currently ignored - local maxn = #nodes - if maxn == 3 then --optimized loop - local nns, ntg = nodes[2], nodes[3] - if not nns and not ntg then -- wildcard + local maxn=#nodes + if maxn==3 then + local nns,ntg=nodes[2],nodes[3] + if not nns and not ntg then + if directive then + return list + else + return {} + end + else + local collected,c,m,p={},0,0,nil + if not nns then + for l=1,#list do + local ll=list[l] + local ltg=ll.tg + if ltg then if directive then - return list - else - return { } + if ntg==ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif ntg~=ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end - else - local collected, c, m, p = { }, 0, 0, nil - if not nns then -- only check tag - for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - if directive then - if ntg == ltg then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif ntg ~= ltg then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - elseif not ntg then -- only check namespace - for l=1,#list do - local ll = list[l] - local lns = ll.rn or ll.ns - if lns then - if directive then - if lns == nns then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif lns ~= nns then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - else -- check both - for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - local lns = ll.rn or ll.ns - local ok = ltg == ntg and lns == nns - if directive then - if ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif not ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end + end + end + elseif not ntg then + for l=1,#list do + local ll=list[l] + local lns=ll.rn or ll.ns + if lns then + if directive then + if lns==nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif lns~=nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end - return collected + end end - else - local collected, c, m, p = { }, 0, 0, nil + else for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - local lns = ll.rn or ll.ns - local ok = false - for n=1,maxn,3 do - local nns, ntg = nodes[n+1], nodes[n+2] - ok = (not ntg or ltg == ntg) and (not nns or lns == nns) - if ok then - break - end - end - if directive then - if ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif not ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=ltg==ntg and lns==nns + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end + end end - return collected + end + return collected end -end - -local quit_expression = false - -local function apply_expression(list,expression,order) - local collected, c = { }, 0 - quit_expression = false + else + local collected,c,m,p={},0,0,nil for l=1,#list do - local ll = list[l] - if expression(list,ll,l,order) then -- nasty, order alleen valid als n=1 - c = c + 1 - collected[c] = ll - end - if quit_expression then + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=false + for n=1,maxn,3 do + local nns,ntg=nodes[n+1],nodes[n+2] + ok=(not ntg or ltg==ntg) and (not nns or lns==nns) + if ok then break + end + end + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end + end end return collected + end end - -local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb - -local spaces = S(" \n\r\t\f")^0 -local lp_space = S(" \n\r\t\f") -local lp_any = P(1) -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_or = P("|") / " or " -local lp_and = P("&") / " and " - -local lp_builtin = P ( - P("text") / "(ll.dt[1] or '')" + -- fragile - P("content") / "ll.dt" + - -- P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + - P("name") / "((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)" + - P("tag") / "ll.tg" + - P("position") / "l" + -- is element in finalizer - P("firstindex") / "1" + - P("lastindex") / "(#ll.__p__.dt or 1)" + - P("firstelement") / "1" + - P("lastelement") / "(ll.__p__.en or 1)" + - P("first") / "1" + - P("last") / "#list" + - P("rootposition") / "order" + - P("order") / "order" + - P("element") / "(ll.ei or 1)" + - P("index") / "(ll.ni or 1)" + - P("match") / "(ll.mi or 1)" + - -- P("namespace") / "ll.ns" + - P("ns") / "ll.ns" - ) * ((spaces * P("(") * spaces * P(")"))/"") - --- for the moment we keep namespaces with attributes - -local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * ((R("az","AZ") + S("-_:"))^1) * Cc("'])") - --- lp_fastpos_p = (P("+")^0 * R("09")^1 * P(-1)) / function(s) return "l==" .. s end --- lp_fastpos_n = (P("-") * R("09")^1 * P(-1)) / function(s) return "(" .. s .. "<0 and (#list+".. s .. "==l))" end - -lp_fastpos_p = P("+")^0 * R("09")^1 * P(-1) / "l==%0" -lp_fastpos_n = P("-") * R("09")^1 * P(-1) / "(%0<0 and (#list+%0==l))" - -local lp_fastpos = lp_fastpos_n + lp_fastpos_p - -local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") - --- local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling --- return t .. "(" --- end - --- local lp_lua_function = (R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / "%0(" -local lp_lua_function = Cs((R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(")) / "%0" - -local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling - if expressions[t] then - return "expr." .. t .. "(" - else - return "expr.error(" - end -end - -local lparent = P("(") -local rparent = P(")") -local noparent = 1 - (lparent+rparent) -local nested = P{lparent * (noparent + V(1))^0 * rparent} -local value = P(lparent * C((noparent + nested)^0) * rparent) -- P{"("*C(((1-S("()"))+V(1))^0)*")"} - -local lp_child = Cc("expr.child(ll,'") * R("az","AZ","--","__")^1 * Cc("')") -local lp_number = S("+-") * R("09")^1 -local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") -local lp_content = (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) - +local quit_expression=false +local function apply_expression(list,expression,order) + local collected,c={},0 + quit_expression=false + for l=1,#list do + local ll=list[l] + if expression(list,ll,l,order) then + c=c+1 + collected[c]=ll + end + if quit_expression then + break + end + end + return collected +end +local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb +local spaces=S(" \n\r\t\f")^0 +local lp_space=S(" \n\r\t\f") +local lp_any=P(1) +local lp_noequal=P("!=")/"~="+P("<=")+P(">=")+P("==") +local lp_doequal=P("=")/"==" +local lp_or=P("|")/" or " +local lp_and=P("&")/" and " +local lp_builtin=P ( + P("text")/"(ll.dt[1] or '')"+ + P("content")/"ll.dt"+ + P("name")/"((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)"+P("tag")/"ll.tg"+P("position")/"l"+ + P("firstindex")/"1"+P("lastindex")/"(#ll.__p__.dt or 1)"+P("firstelement")/"1"+P("lastelement")/"(ll.__p__.en or 1)"+P("first")/"1"+P("last")/"#list"+P("rootposition")/"order"+P("order")/"order"+P("element")/"(ll.ei or 1)"+P("index")/"(ll.ni or 1)"+P("match")/"(ll.mi or 1)"+ + P("ns")/"ll.ns" + )*((spaces*P("(")*spaces*P(")"))/"") +local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") +lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" +lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" +local lp_fastpos=lp_fastpos_n+lp_fastpos_p +local lp_reserved=C("and")+C("or")+C("not")+C("div")+C("mod")+C("true")+C("false") +local lp_lua_function=Cs((R("az","AZ","__")^1*(P(".")*R("az","AZ","__")^1)^1)*("("))/"%0" +local lp_function=C(R("az","AZ","__")^1)*P("(")/function(t) + if expressions[t] then + return "expr."..t.."(" + else + return "expr.error(" + end +end +local lparent=P("(") +local rparent=P(")") +local noparent=1-(lparent+rparent) +local nested=P{lparent*(noparent+V(1))^0*rparent} +local value=P(lparent*C((noparent+nested)^0)*rparent) +local lp_child=Cc("expr.child(ll,'")*R("az","AZ","--","__")^1*Cc("')") +local lp_number=S("+-")*R("09")^1 +local lp_string=Cc("'")*R("az","AZ","--","__")^1*Cc("'") +local lp_content=(P("'")*(1-P("'"))^0*P("'")+P('"')*(1-P('"'))^0*P('"')) local cleaner - -local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) - if expressions[t] then - s = s and s ~= "" and lpegmatch(cleaner,s) - if s and s ~= "" then - return "expr." .. t .. "(ll," .. s ..")" - else - return "expr." .. t .. "(ll)" - end - else - return "expr.error(" .. t .. ")" - end -end - -local content = - lp_builtin + - lp_attribute + - lp_special + - lp_noequal + lp_doequal + - lp_or + lp_and + - lp_reserved + - lp_lua_function + lp_function + - lp_content + -- too fragile - lp_child + - lp_any - -local converter = Cs ( - lp_fastpos + (P { lparent * (V(1))^0 * rparent + content } )^0 +local lp_special=(C(P("name")+P("text")+P("tag")+P("count")+P("child")))*value/function(t,s) + if expressions[t] then + s=s and s~="" and lpegmatch(cleaner,s) + if s and s~="" then + return "expr."..t.."(ll,"..s..")" + else + return "expr."..t.."(ll)" + end + else + return "expr.error("..t..")" + end +end +local content=lp_builtin+lp_attribute+lp_special+lp_noequal+lp_doequal+lp_or+lp_and+lp_reserved+lp_lua_function+lp_function+lp_content+ + lp_child+lp_any +local converter=Cs ( + lp_fastpos+(P { lparent*(V(1))^0*rparent+content } )^0 ) - -cleaner = Cs ( ( - lp_reserved + - lp_number + - lp_string + -1 )^1 ) - - - -local template_e = [[ +cleaner=Cs (( + lp_reserved+lp_number+lp_string+1 )^1 ) +local template_e=[[ local expr = xml.expressions return function(list,ll,l,order) return %s end ]] - -local template_f_y = [[ +local template_f_y=[[ local finalizer = xml.finalizers['%s']['%s'] return function(collection) return finalizer(collection,%s) end ]] - -local template_f_n = [[ +local template_f_n=[[ return xml.finalizers['%s']['%s'] ]] - --- - -local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } -local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } -local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } -local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } -local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } -local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } -local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } -local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } -local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } -local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } -local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } -local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } -local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } -local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } -local register_reverse_sibling = { kind = "axis", axis = "reverse-sibling" } -- , apply = apply_axis["reverse-sibling"] } - -local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } -local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } -local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } -local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } - -local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } - -local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } - -local skip = { } - +local register_self={ kind="axis",axis="self" } +local register_parent={ kind="axis",axis="parent" } +local register_descendant={ kind="axis",axis="descendant" } +local register_child={ kind="axis",axis="child" } +local register_descendant_or_self={ kind="axis",axis="descendant-or-self" } +local register_root={ kind="axis",axis="root" } +local register_ancestor={ kind="axis",axis="ancestor" } +local register_ancestor_or_self={ kind="axis",axis="ancestor-or-self" } +local register_attribute={ kind="axis",axis="attribute" } +local register_namespace={ kind="axis",axis="namespace" } +local register_following={ kind="axis",axis="following" } +local register_following_sibling={ kind="axis",axis="following-sibling" } +local register_preceding={ kind="axis",axis="preceding" } +local register_preceding_sibling={ kind="axis",axis="preceding-sibling" } +local register_reverse_sibling={ kind="axis",axis="reverse-sibling" } +local register_auto_descendant_or_self={ kind="axis",axis="auto-descendant-or-self" } +local register_auto_descendant={ kind="axis",axis="auto-descendant" } +local register_auto_self={ kind="axis",axis="auto-self" } +local register_auto_child={ kind="axis",axis="auto-child" } +local register_initial_child={ kind="axis",axis="initial-child" } +local register_all_nodes={ kind="nodes",nodetest=true,nodes={ true,false,false } } +local skip={} local function errorrunner_e(str,cnv) - if not skip[str] then - report_lpath("error in expression: %s => %s",str,cnv) - skip[str] = cnv or str - end - return false + if not skip[str] then + report_lpath("error in expression: %s => %s",str,cnv) + skip[str]=cnv or str + end + return false end local function errorrunner_f(str,arg) - report_lpath("error in finalizer: %s(%s)",str,arg or "") - return false + report_lpath("error in finalizer: %s(%s)",str,arg or "") + return false end - local function register_nodes(nodetest,nodes) - return { kind = "nodes", nodetest = nodetest, nodes = nodes } + return { kind="nodes",nodetest=nodetest,nodes=nodes } end - local function register_expression(expression) - local converted = lpegmatch(converter,expression) - local runner = load(format(template_e,converted)) - runner = (runner and runner()) or function() errorrunner_e(expression,converted) end - return { kind = "expression", expression = expression, converted = converted, evaluator = runner } + local converted=lpegmatch(converter,expression) + local runner=load(format(template_e,converted)) + runner=(runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind="expression",expression=expression,converted=converted,evaluator=runner } end - local function register_finalizer(protocol,name,arguments) - local runner - if arguments and arguments ~= "" then - runner = load(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) - else - runner = load(format(template_f_n,protocol or xml.defaultprotocol,name)) - end - runner = (runner and runner()) or function() errorrunner_f(name,arguments) end - return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } -end - -local expression = P { "ex", - ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", - sq = "'" * (1 - S("'"))^0 * "'", - dq = '"' * (1 - S('"'))^0 * '"', + local runner + if arguments and arguments~="" then + runner=load(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) + else + runner=load(format(template_f_n,protocol or xml.defaultprotocol,name)) + end + runner=(runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind="finalizer",name=name,arguments=arguments,finalizer=runner } +end +local expression=P { "ex", + ex="["*C((V("sq")+V("dq")+(1-S("[]"))+V("ex"))^0)*"]", + sq="'"*(1-S("'"))^0*"'", + dq='"'*(1-S('"'))^0*'"', } - -local arguments = P { "ar", - ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", - nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, - sq = P("'") * (1 - P("'"))^0 * P("'"), - dq = P('"') * (1 - P('"'))^0 * P('"'), +local arguments=P { "ar", + ar="("*Cs((V("sq")+V("dq")+V("nq")+P(1-P(")")))^0)*")", + nq=((1-S("),'\""))^1)/function(s) return format("%q",s) end, + sq=P("'")*(1-P("'"))^0*P("'"), + dq=P('"')*(1-P('"'))^0*P('"'), } - --- todo: better arg parser - local function register_error(str) - return { kind = "error", error = format("unparsed: %s",str) } -end - --- there is a difference in * and /*/ and so we need to catch a few special cases - -local special_1 = P("*") * Cc(register_auto_descendant) * Cc(register_all_nodes) -- last one not needed -local special_2 = P("/") * Cc(register_auto_self) -local special_3 = P("") * Cc(register_auto_self) - -local no_nextcolon = P(-1) + #(1-P(":")) -- newer lpeg needs the P(-1) -local no_nextlparent = P(-1) + #(1-P("(")) -- newer lpeg needs the P(-1) - -local pathparser = Ct { "patterns", -- can be made a bit faster by moving some patterns outside - - patterns = spaces * V("protocol") * spaces * ( - ( V("special") * spaces * P(-1) ) + - ( V("initial") * spaces * V("step") * spaces * (P("/") * spaces * V("step") * spaces)^0 ) - ), - - protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), - - -- the / is needed for // as descendant or self is somewhat special - -- step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, - step = ((V("shortcuts") + P("/") + V("axis")) * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, - - axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + - V("descendant_or_self") + V("following_sibling") + V("following") + - V("reverse_sibling") + V("preceding_sibling") + V("preceding") + V("ancestor_or_self") + - #(1-P(-1)) * Cc(register_auto_child), - - special = special_1 + special_2 + special_3, - - initial = (P("/") * spaces * Cc(register_initial_child))^-1, - - error = (P(1)^1) / register_error, - - shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), - - shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, - - s_descendant_or_self = (P("***/") + P("/")) * Cc(register_descendant_or_self), --- *** is a bonus - s_descendant = P("**") * Cc(register_descendant), - s_child = P("*") * no_nextcolon * Cc(register_child ), - s_parent = P("..") * Cc(register_parent ), - s_self = P("." ) * Cc(register_self ), - s_root = P("^^") * Cc(register_root ), - s_ancestor = P("^") * Cc(register_ancestor ), - - descendant = P("descendant::") * Cc(register_descendant ), - child = P("child::") * Cc(register_child ), - parent = P("parent::") * Cc(register_parent ), - self = P("self::") * Cc(register_self ), - root = P('root::') * Cc(register_root ), - ancestor = P('ancestor::') * Cc(register_ancestor ), - descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), - ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), - -- attribute = P('attribute::') * Cc(register_attribute ), - -- namespace = P('namespace::') * Cc(register_namespace ), - following = P('following::') * Cc(register_following ), - following_sibling = P('following-sibling::') * Cc(register_following_sibling ), - preceding = P('preceding::') * Cc(register_preceding ), - preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), - reverse_sibling = P('reverse-sibling::') * Cc(register_reverse_sibling ), - - nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, - - expressions = expression / register_expression, - - letters = R("az")^1, - name = (1-S("/[]()|:*!"))^1, -- make inline - negate = P("!") * Cc(false), - - nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), - nodetest = V("negate") + Cc(true), - nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), - wildnodename = (C(V("name")) + P("*") * Cc(false)) * no_nextlparent, - nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, - - finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, - + return { kind="error",error=format("unparsed: %s",str) } +end +local special_1=P("*")*Cc(register_auto_descendant)*Cc(register_all_nodes) +local special_2=P("/")*Cc(register_auto_self) +local special_3=P("")*Cc(register_auto_self) +local no_nextcolon=P(-1)+#(1-P(":")) +local no_nextlparent=P(-1)+#(1-P("(")) +local pathparser=Ct { "patterns", + patterns=spaces*V("protocol")*spaces*( + (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) + ), + protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), + step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + axis=V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), + special=special_1+special_2+special_3, + initial=(P("/")*spaces*Cc(register_initial_child))^-1, + error=(P(1)^1)/register_error, + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, + s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), + s_descendant=P("**")*Cc(register_descendant), + s_child=P("*")*no_nextcolon*Cc(register_child ), + s_parent=P("..")*Cc(register_parent ), + s_self=P("." )*Cc(register_self ), + s_root=P("^^")*Cc(register_root ), + s_ancestor=P("^")*Cc(register_ancestor ), + descendant=P("descendant::")*Cc(register_descendant ), + child=P("child::")*Cc(register_child ), + parent=P("parent::")*Cc(register_parent ), + self=P("self::")*Cc(register_self ), + root=P('root::')*Cc(register_root ), + ancestor=P('ancestor::')*Cc(register_ancestor ), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), + following=P('following::')*Cc(register_following ), + following_sibling=P('following-sibling::')*Cc(register_following_sibling ), + preceding=P('preceding::')*Cc(register_preceding ), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), + nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, + expressions=expression/register_expression, + letters=R("az")^1, + name=(1-S("/[]()|:*!"))^1, + negate=P("!")*Cc(false), + nodefunction=V("negate")+P("not")*Cc(false)+Cc(true), + nodetest=V("negate")+Cc(true), + nodename=(V("negate")+Cc(true))*spaces*((V("wildnodename")*P(":")*V("wildnodename"))+(Cc(false)*V("wildnodename"))), + wildnodename=(C(V("name"))+P("*")*Cc(false))*no_nextlparent, + nodeset=spaces*Ct(V("nodename")*(spaces*P("|")*spaces*V("nodename"))^0)*spaces, + finalizer=(Cb("protocol")*P("/")^-1*C(V("name"))*arguments*P(-1))/register_finalizer, } - -xmlpatterns.pathparser = pathparser - -local cache = { } - +xmlpatterns.pathparser=pathparser +local cache={} local function nodesettostring(set,nodetest) - local t = { } - for i=1,#set,3 do - local directive, ns, tg = set[i], set[i+1], set[i+2] - if not ns or ns == "" then ns = "*" end - if not tg or tg == "" then tg = "*" end - tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) - t[i] = (directive and tg) or format("not(%s)",tg) - end - if nodetest == false then - return format("not(%s)",concat(t,"|")) - else - return concat(t,"|") - end + local t={} + for i=1,#set,3 do + local directive,ns,tg=set[i],set[i+1],set[i+2] + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + tg=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + t[i]=(directive and tg) or format("not(%s)",tg) + end + if nodetest==false then + return format("not(%s)",concat(t,"|")) + else + return concat(t,"|") + end end - local function tagstostring(list) - if #list == 0 then - return "no elements" - else - local t = { } - for i=1, #list do - local li = list[i] - local ns, tg = li.ns, li.tg - if not ns or ns == "" then ns = "*" end - if not tg or tg == "" then tg = "*" end - t[i] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) - end - return concat(t," ") - end -end - -xml.nodesettostring = nodesettostring - -local lpath -- we have a harmless kind of circular reference - -local lshowoptions = { functions = false } - + if #list==0 then + return "no elements" + else + local t={} + for i=1,#list do + local li=list[i] + local ns,tg=li.ns,li.tg + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + t[i]=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end +end +xml.nodesettostring=nodesettostring +local lpath +local lshowoptions={ functions=false } local function lshow(parsed) - if type(parsed) == "string" then - parsed = lpath(parsed) - end - report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, - table.serialize(parsed,false,lshowoptions)) + if type(parsed)=="string" then + parsed=lpath(parsed) + end + report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, + table.serialize(parsed,false,lshowoptions)) end - -xml.lshow = lshow - +xml.lshow=lshow local function add_comment(p,str) - local pc = p.comment - if not pc then - p.comment = { str } - else - pc[#pc+1] = str - end -end - -lpath = function (pattern) -- the gain of caching is rather minimal - lpathcalls = lpathcalls + 1 - if type(pattern) == "table" then - return pattern - else - local parsed = cache[pattern] - if parsed then - lpathcached = lpathcached + 1 - else - parsed = lpegmatch(pathparser,pattern) - if parsed then - parsed.pattern = pattern - local np = #parsed - if np == 0 then - parsed = { pattern = pattern, register_self, state = "parsing error" } - report_lpath("parsing error in '%s'",pattern) - lshow(parsed) - else - -- we could have done this with a more complex parser but this - -- is cleaner - local pi = parsed[1] - if pi.axis == "auto-child" then - if false then - add_comment(parsed, "auto-child replaced by auto-descendant-or-self") - parsed[1] = register_auto_descendant_or_self - else - add_comment(parsed, "auto-child replaced by auto-descendant") - parsed[1] = register_auto_descendant - end - elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then - add_comment(parsed, "initial-child removed") -- we could also make it a auto-self - remove(parsed,1) - end - local np = #parsed -- can have changed - if np > 1 then - local pnp = parsed[np] - if pnp.kind == "nodes" and pnp.nodetest == true then - local nodes = pnp.nodes - if nodes[1] == true and nodes[2] == false and nodes[3] == false then - add_comment(parsed, "redundant final wildcard filter removed") - remove(parsed,np) - end - end - end - end + local pc=p.comment + if not pc then + p.comment={ str } + else + pc[#pc+1]=str + end +end +lpath=function (pattern) + lpathcalls=lpathcalls+1 + if type(pattern)=="table" then + return pattern + else + local parsed=cache[pattern] + if parsed then + lpathcached=lpathcached+1 + else + parsed=lpegmatch(pathparser,pattern) + if parsed then + parsed.pattern=pattern + local np=#parsed + if np==0 then + parsed={ pattern=pattern,register_self,state="parsing error" } + report_lpath("parsing error in '%s'",pattern) + lshow(parsed) + else + local pi=parsed[1] + if pi.axis=="auto-child" then + if false then + add_comment(parsed,"auto-child replaced by auto-descendant-or-self") + parsed[1]=register_auto_descendant_or_self else - parsed = { pattern = pattern } + add_comment(parsed,"auto-child replaced by auto-descendant") + parsed[1]=register_auto_descendant end - cache[pattern] = parsed - if trace_lparse and not trace_lprofile then - lshow(parsed) + elseif pi.axis=="initial-child" and np>1 and parsed[2].axis then + add_comment(parsed,"initial-child removed") + remove(parsed,1) + end + local np=#parsed + if np>1 then + local pnp=parsed[np] + if pnp.kind=="nodes" and pnp.nodetest==true then + local nodes=pnp.nodes + if nodes[1]==true and nodes[2]==false and nodes[3]==false then + add_comment(parsed,"redundant final wildcard filter removed") + remove(parsed,np) + end end + end end - return parsed + else + parsed={ pattern=pattern } + end + cache[pattern]=parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) + end end + return parsed + end end - -xml.lpath = lpath - --- we can move all calls inline and then merge the trace back --- technically we can combine axis and the next nodes which is --- what we did before but this a bit cleaner (but slower too) --- but interesting is that it's not that much faster when we --- go inline --- --- beware: we need to return a collection even when we filter --- else the (simple) cache gets messed up - --- caching found lookups saves not that much (max .1 sec on a 8 sec run) --- and it also messes up finalizers - --- watch out: when there is a finalizer, it's always called as there --- can be cases that a finalizer returns (or does) something in case --- there is no match; an example of this is count() - -local profiled = { } xml.profiled = profiled - +xml.lpath=lpath +local profiled={} xml.profiled=profiled local function profiled_apply(list,parsed,nofparsed,order) - local p = profiled[parsed.pattern] - if p then - p.tested = p.tested + 1 - else - p = { tested = 1, matched = 0, finalized = 0 } - profiled[parsed.pattern] = p - end - local collected = list - for i=1,nofparsed do - local pi = parsed[i] - local kind = pi.kind - if kind == "axis" then - collected = apply_axis[pi.axis](collected) - elseif kind == "nodes" then - collected = apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind == "expression" then - collected = apply_expression(collected,pi.evaluator,order) - elseif kind == "finalizer" then - collected = pi.finalizer(collected) -- no check on # here - p.matched = p.matched + 1 - p.finalized = p.finalized + 1 - return collected - end - if not collected or #collected == 0 then - local pn = i < nofparsed and parsed[nofparsed] - if pn and pn.kind == "finalizer" then - collected = pn.finalizer(collected) - p.finalized = p.finalized + 1 - return collected - end - return nil - end - end - if collected then - p.matched = p.matched + 1 + local p=profiled[parsed.pattern] + if p then + p.tested=p.tested+1 + else + p={ tested=1,matched=0,finalized=0 } + profiled[parsed.pattern]=p + end + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + p.matched=p.matched+1 + p.finalized=p.finalized+1 + return collected + end + if not collected or #collected==0 then + local pn=i %s",(collected and #collected) or 0,pi.expression,pi.converted) - elseif kind == "finalizer" then - collected = pi.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") - return collected - end - if not collected or #collected == 0 then - local pn = i < nofparsed and parsed[nofparsed] - if pn and pn.kind == "finalizer" then - collected = pn.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") - return collected - end - return nil - end + if trace_lparse then + lshow(parsed) + end + report_lpath("collecting: %s",parsed.pattern) + report_lpath("root tags : %s",tagstostring(list)) + report_lpath("order : %s",order or "unset") + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected==0 then + local pn=i oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps oeps oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -xmlpatterns.escaped = escaped -xmlpatterns.unescaped = unescaped -xmlpatterns.cleansed = cleansed - -function xml.escaped (str) return lpegmatch(escaped,str) end + end +end +function xml.stripleadingspaces(dk,d,k) + if d and k then + local dkm=d[k-1] + if dkm and type(dkm)=="string" then + local s=match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") + end + end +end +local normal=(1-S("<&>"))^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local escaped=Cs(normal*(special*normal)^0) +local normal=(1-S"&")^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local unescaped=Cs(normal*(special*normal)^0) +local cleansed=Cs(((P("<")*(1-P(">"))^0*P(">"))/""+1)^0) +xmlpatterns.escaped=escaped +xmlpatterns.unescaped=unescaped +xmlpatterns.cleansed=cleansed +function xml.escaped (str) return lpegmatch(escaped,str) end function xml.unescaped(str) return lpegmatch(unescaped,str) end -function xml.cleansed (str) return lpegmatch(cleansed,str) end - --- this might move - +function xml.cleansed (str) return lpegmatch(cleansed,str) end function xml.fillin(root,pattern,str,check) - local e = xml.first(root,pattern) - if e then - local n = #e.dt - if not check or n == 0 or (n == 1 and e.dt[1] == "") then - e.dt = { str } - end + local e=xml.first(root,pattern) + if e then + local n=#e.dt + if not check or n==0 or (n==1 and e.dt[1]=="") then + e.dt={ str } end + end end @@ -12210,800 +9242,690 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-aux'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- not all functions here make sense anymore vbut we keep them for --- compatibility reasons - -local trace_manipulations = false trackers.register("lxml.manipulations", function(v) trace_manipulations = v end) - -local report_xml = logs.reporter("xml") - -local xml = xml - -local xmlconvert, xmlcopy, xmlname = xml.convert, xml.copy, xml.name -local xmlinheritedconvert = xml.inheritedconvert -local xmlapplylpath = xml.applylpath -local xmlfilter = xml.filter - -local type, setmetatable, getmetatable = type, setmetatable, getmetatable -local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat -local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip -local utfbyte = utf.byte +-- original size: 23813, stripped down to: 16826 +if not modules then modules={} end modules ['lxml-aux']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_manipulations=false trackers.register("lxml.manipulations",function(v) trace_manipulations=v end) +local report_xml=logs.reporter("xml") +local xml=xml +local xmlconvert,xmlcopy,xmlname=xml.convert,xml.copy,xml.name +local xmlinheritedconvert=xml.inheritedconvert +local xmlapplylpath=xml.applylpath +local xmlfilter=xml.filter +local type,setmetatable,getmetatable=type,setmetatable,getmetatable +local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat +local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip +local utfbyte=utf.byte local function report(what,pattern,c,e) - report_xml("%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) + report_xml("%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end - local function withelements(e,handle,depth) - if e and handle then - local edt = e.dt - if edt then - depth = depth or 0 - for i=1,#edt do - local e = edt[i] - if type(e) == "table" then - handle(e,depth) - withelements(e,handle,depth+1) - end - end + if e and handle then + local edt=e.dt + if edt then + depth=depth or 0 + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + handle(e,depth) + withelements(e,handle,depth+1) end + end end + end end - -xml.withelements = withelements - -function xml.withelement(e,n,handle) -- slow - if e and n ~= 0 and handle then - local edt = e.dt - if edt then - if n > 0 then - for i=1,#edt do - local ei = edt[i] - if type(ei) == "table" then - if n == 1 then - handle(ei) - return - else - n = n - 1 - end - end - end - elseif n < 0 then - for i=#edt,1,-1 do - local ei = edt[i] - if type(ei) == "table" then - if n == -1 then - handle(ei) - return - else - n = n + 1 - end - end - end +xml.withelements=withelements +function xml.withelement(e,n,handle) + if e and n~=0 and handle then + local edt=e.dt + if edt then + if n>0 then + for i=1,#edt do + local ei=edt[i] + if type(ei)=="table" then + if n==1 then + handle(ei) + return + else + n=n-1 + end + end + end + elseif n<0 then + for i=#edt,1,-1 do + local ei=edt[i] + if type(ei)=="table" then + if n==-1 then + handle(ei) + return + else + n=n+1 end + end end + end end + end end - function xml.each(root,pattern,handle,reverse) - local collected = xmlapplylpath(root,pattern) - if collected then - if reverse then - for c=#collected,1,-1 do - handle(collected[c]) - end - else - for c=1,#collected do - handle(collected[c]) - end - end - return collected + local collected=xmlapplylpath(root,pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end end + return collected + end end - function xml.processattributes(root,pattern,handle) - local collected = xmlapplylpath(root,pattern) - if collected and handle then - for c=1,#collected do - handle(collected[c].at) - end + local collected=xmlapplylpath(root,pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) end - return collected + end + return collected end - - - --- are these still needed -> lxml-cmp.lua - -function xml.collect(root, pattern) - return xmlapplylpath(root,pattern) +function xml.collect(root,pattern) + return xmlapplylpath(root,pattern) end - -function xml.collecttexts(root, pattern, flatten) -- todo: variant with handle - local collected = xmlapplylpath(root,pattern) - if collected and flatten then - local xmltostring = xml.tostring - for c=1,#collected do - collected[c] = xmltostring(collected[c].dt) - end +function xml.collecttexts(root,pattern,flatten) + local collected=xmlapplylpath(root,pattern) + if collected and flatten then + local xmltostring=xml.tostring + for c=1,#collected do + collected[c]=xmltostring(collected[c].dt) end - return collected or { } + end + return collected or {} end - -function xml.collect_tags(root, pattern, nonamespace) - local collected = xmlapplylpath(root,pattern) - if collected then - local t, n = { }, 0 - for c=1,#collected do - local e = collected[c] - local ns, tg = e.ns, e.tg - n = n + 1 - if nonamespace then - t[n] = tg - elseif ns == "" then - t[n] = tg - else - t[n] = ns .. ":" .. tg - end - end - return t +function xml.collect_tags(root,pattern,nonamespace) + local collected=xmlapplylpath(root,pattern) + if collected then + local t,n={},0 + for c=1,#collected do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace then + t[n]=tg + elseif ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg + end end + return t + end end - - - -local no_root = { no_root = true } - +local no_root={ no_root=true } local function redo_ni(d) - for k=1,#d do - local dk = d[k] - if type(dk) == "table" then - dk.ni = k - end + for k=1,#d do + local dk=d[k] + if type(dk)=="table" then + dk.ni=k end + end end - local function xmltoelement(whatever,root) - if not whatever then - return nil - end - local element - if type(whatever) == "string" then - element = xmlinheritedconvert(whatever,root) -- beware, not really a root - else - element = whatever -- we assume a table - end - if element.error then - return whatever -- string - end - if element then - end - return element -end - -xml.toelement = xmltoelement - + if not whatever then + return nil + end + local element + if type(whatever)=="string" then + element=xmlinheritedconvert(whatever,root) + else + element=whatever + end + if element.error then + return whatever + end + if element then + end + return element +end +xml.toelement=xmltoelement local function copiedelement(element,newparent) - if type(element) == "string" then - return element - else - element = xmlcopy(element).dt - if newparent and type(element) == "table" then - element.__p__ = newparent - end - return element + if type(element)=="string" then + return element + else + element=xmlcopy(element).dt + if newparent and type(element)=="table" then + element.__p__=newparent end + return element + end end - function xml.delete(root,pattern) - if not pattern or pattern == "" then - local p = root.__p__ + if not pattern or pattern=="" then + local p=root.__p__ + if p then + if trace_manipulations then + report('deleting',"--",c,root) + end + local d=p.dt + remove(d,root.ni) + redo_ni(d) + end + else + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ if p then - if trace_manipulations then - report('deleting',"--",c,root) - end - local d = p.dt - remove(d,root.ni) - redo_ni(d) -- can be made faster and inlined - end - else - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local p = e.__p__ - if p then - if trace_manipulations then - report('deleting',pattern,c,e) - end - local d = p.dt - remove(d,e.ni) - redo_ni(d) -- can be made faster and inlined - end - end + if trace_manipulations then + report('deleting',pattern,c,e) + end + local d=p.dt + remove(d,e.ni) + redo_ni(d) end + end end + end end - function xml.replace(root,pattern,whatever) - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local p = e.__p__ - if p then - if trace_manipulations then - report('replacing',pattern,c,e) - end - local d = p.dt - d[e.ni] = copiedelement(element,p) - redo_ni(d) -- probably not needed - end - end + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ + if p then + if trace_manipulations then + report('replacing',pattern,c,e) + end + local d=p.dt + d[e.ni]=copiedelement(element,p) + redo_ni(d) + end end + end end - local function wrap(e,wrapper) - local t = { - rn = e.rn, - tg = e.tg, - ns = e.ns, - at = e.at, - dt = e.dt, - __p__ = e, - } - setmetatable(t,getmetatable(e)) - e.rn = wrapper.rn or e.rn or "" - e.tg = wrapper.tg or e.tg or "" - e.ns = wrapper.ns or e.ns or "" - e.at = fastcopy(wrapper.at) - e.dt = { t } + local t={ + rn=e.rn, + tg=e.tg, + ns=e.ns, + at=e.at, + dt=e.dt, + __p__=e, + } + setmetatable(t,getmetatable(e)) + e.rn=wrapper.rn or e.rn or "" + e.tg=wrapper.tg or e.tg or "" + e.ns=wrapper.ns or e.ns or "" + e.at=fastcopy(wrapper.at) + e.dt={ t } end - function xml.wrap(root,pattern,whatever) - if whatever then - local wrapper = xmltoelement(whatever,root) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - if trace_manipulations then - report('wrapping',pattern,c,e) - end - wrap(e,wrapper) - end + if whatever then + local wrapper=xmltoelement(whatever,root) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if trace_manipulations then + report('wrapping',pattern,c,e) end - else - wrap(root,xmltoelement(pattern)) + wrap(e,wrapper) + end end + else + wrap(root,xmltoelement(pattern)) + end end - local function inject_element(root,pattern,whatever,prepend) - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - local function inject_e(e) - local r = e.__p__ - local d, k, rri = r.dt, e.ni, r.ri - local edt = (rri and d[rri].dt) or (d and d[k] and d[k].dt) - if edt then - local be, af - local cp = copiedelement(element,e) - if prepend then - be, af = cp, edt - else - be, af = edt, cp - end - local bn = #be - for i=1,#af do - bn = bn + 1 - be[bn] = af[i] - end - if rri then - r.dt[rri].dt = be - else - d[k].dt = be - end - redo_ni(d) - end - end - if not collected then - -- nothing - elseif collected.tg then - -- first or so - inject_e(collected) - else - for c=1,#collected do - inject_e(collected[c]) - end + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function inject_e(e) + local r=e.__p__ + local d,k,rri=r.dt,e.ni,r.ri + local edt=(rri and d[rri].dt) or (d and d[k] and d[k].dt) + if edt then + local be,af + local cp=copiedelement(element,e) + if prepend then + be,af=cp,edt + else + be,af=edt,cp + end + local bn=#be + for i=1,#af do + bn=bn+1 + be[bn]=af[i] + end + if rri then + r.dt[rri].dt=be + else + d[k].dt=be + end + redo_ni(d) end -end - -local function insert_element(root,pattern,whatever,before) -- todo: element als functie - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - local function insert_e(e) - local r = e.__p__ - local d, k = r.dt, e.ni - if not before then - k = k + 1 - end - insert(d,k,copiedelement(element,r)) - redo_ni(d) - end - if not collected then - -- nothing - elseif collected.tg then - -- first or so - insert_e(collected) - else - for c=1,#collected do - insert_e(collected[c]) - end + end + if not collected then + elseif collected.tg then + inject_e(collected) + else + for c=1,#collected do + inject_e(collected[c]) + end + end +end +local function insert_element(root,pattern,whatever,before) + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function insert_e(e) + local r=e.__p__ + local d,k=r.dt,e.ni + if not before then + k=k+1 + end + insert(d,k,copiedelement(element,r)) + redo_ni(d) + end + if not collected then + elseif collected.tg then + insert_e(collected) + else + for c=1,#collected do + insert_e(collected[c]) end + end end - -xml.insert_element = insert_element -xml.insertafter = insert_element -xml.insertbefore = function(r,p,e) insert_element(r,p,e,true) end -xml.injectafter = inject_element -xml.injectbefore = function(r,p,e) inject_element(r,p,e,true) end - +xml.insert_element=insert_element +xml.insertafter=insert_element +xml.insertbefore=function(r,p,e) insert_element(r,p,e,true) end +xml.injectafter=inject_element +xml.injectbefore=function(r,p,e) inject_element(r,p,e,true) end local function include(xmldata,pattern,attribute,recursive,loaddata) - -- parse="text" (default: xml), encoding="" (todo) - -- attribute = attribute or 'href' - pattern = pattern or 'include' - loaddata = loaddata or io.loaddata - local collected = xmlapplylpath(xmldata,pattern) - if collected then - for c=1,#collected do - local ek = collected[c] - local name = nil - local ekdt = ek.dt - local ekat = ek.at - local epdt = ek.__p__.dt - if not attribute or attribute == "" then - name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- check, probably always tab or str - end - if not name then - for a in gmatch(attribute or "href","([^|]+)") do - name = ekat[a] - if name then break end - end - end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - epdt[ek.ni] = "" -- xml.empty(d,k) - elseif ekat["parse"] == "text" then - -- for the moment hard coded - epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) - else - local xi = xmlinheritedconvert(data,xmldata) - if not xi then - epdt[ek.ni] = "" -- xml.empty(d,k) - else - if recursive then - include(xi,pattern,attribute,recursive,loaddata) - end - epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) - end - end + pattern=pattern or 'include' + loaddata=loaddata or io.loaddata + local collected=xmlapplylpath(xmldata,pattern) + if collected then + for c=1,#collected do + local ek=collected[c] + local name=nil + local ekdt=ek.dt + local ekat=ek.at + local epdt=ek.__p__.dt + if not attribute or attribute=="" then + name=(type(ekdt)=="table" and ekdt[1]) or ekdt + end + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name=ekat[a] + if name then break end end + end + local data=(name and name~="" and loaddata(name)) or "" + if data=="" then + epdt[ek.ni]="" + elseif ekat["parse"]=="text" then + epdt[ek.ni]=xml.escaped(data) + else + local xi=xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni]="" + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni]=xml.body(xi) + end + end end + end end - -xml.include = include - +xml.include=include local function stripelement(e,nolines,anywhere) - local edt = e.dt - if edt then - if anywhere then - local t, n = { }, 0 - for e=1,#edt do - local str = edt[e] - if type(str) ~= "string" then - n = n + 1 - t[n] = str - elseif str ~= "" then - -- todo: lpeg for each case - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s*(.-)%s*$","%1") - if str ~= "" then - n = n + 1 - t[n] = str - end - end - end - e.dt = t + local edt=e.dt + if edt then + if anywhere then + local t,n={},0 + for e=1,#edt do + local str=edt[e] + if type(str)~="string" then + n=n+1 + t[n]=str + elseif str~="" then + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"^%s*(.-)%s*$","%1") + if str~="" then + n=n+1 + t[n]=str + end + end + end + e.dt=t + else + if #edt>0 then + local str=edt[1] + if type(str)~="string" then + elseif str=="" then + remove(edt,1) else - -- we can assume a regular sparse xml table with no successive strings - -- otherwise we should use a while loop - if #edt > 0 then - -- strip front - local str = edt[1] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt,1) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s+","") - if str == "" then - remove(edt,1) - else - edt[1] = str - end - end - end - local nedt = #edt - if nedt > 0 then - -- strip end - local str = edt[nedt] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"%s+$","") - if str == "" then - remove(edt) - else - edt[nedt] = str - end - end - end + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"^%s+","") + if str=="" then + remove(edt,1) + else + edt[1]=str + end + end + end + local nedt=#edt + if nedt>0 then + local str=edt[nedt] + if type(str)~="string" then + elseif str=="" then + remove(edt) + else + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"%s+$","") + if str=="" then + remove(edt) + else + edt[nedt]=str + end end + end end - return e -- convenient + end + return e end - -xml.stripelement = stripelement - -function xml.strip(root,pattern,nolines,anywhere) -- strips all leading and trailing spacing - local collected = xmlapplylpath(root,pattern) -- beware, indices no longer are valid now - if collected then - for i=1,#collected do - stripelement(collected[i],nolines,anywhere) - end +xml.stripelement=stripelement +function xml.strip(root,pattern,nolines,anywhere) + local collected=xmlapplylpath(root,pattern) + if collected then + for i=1,#collected do + stripelement(collected[i],nolines,anywhere) end + end end - -local function renamespace(root, oldspace, newspace) -- fast variant - local ndt = #root.dt - for i=1,ndt or 0 do - local e = root[i] - if type(e) == "table" then - if e.ns == oldspace then - e.ns = newspace - if e.rn then - e.rn = newspace - end - end - local edt = e.dt - if edt then - renamespace(edt, oldspace, newspace) - end +local function renamespace(root,oldspace,newspace) + local ndt=#root.dt + for i=1,ndt or 0 do + local e=root[i] + if type(e)=="table" then + if e.ns==oldspace then + e.ns=newspace + if e.rn then + e.rn=newspace end + end + local edt=e.dt + if edt then + renamespace(edt,oldspace,newspace) + end end + end end - -xml.renamespace = renamespace - -function xml.remaptag(root, pattern, newtg) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - collected[c].tg = newtg - end +xml.renamespace=renamespace +function xml.remaptag(root,pattern,newtg) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].tg=newtg end + end end - -function xml.remapnamespace(root, pattern, newns) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - collected[c].ns = newns - end +function xml.remapnamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].ns=newns end + end end - -function xml.checknamespace(root, pattern, newns) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - if (not e.rn or e.rn == "") and e.ns == "" then - e.rn = newns - end - end +function xml.checknamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if (not e.rn or e.rn=="") and e.ns=="" then + e.rn=newns + end end + end end - -function xml.remapname(root, pattern, newtg, newns, newrn) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - e.tg, e.ns, e.rn = newtg, newns, newrn - end +function xml.remapname(root,pattern,newtg,newns,newrn) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + e.tg,e.ns,e.rn=newtg,newns,newrn end + end end - - - function xml.cdatatotext(e) - local dt = e.dt - if #dt == 1 then - local first = dt[1] - if first.tg == "@cd@" then - e.dt = first.dt - end - else - -- maybe option - end -end - --- local x = xml.convert("123") --- xml.texttocdata(xml.first(x,"a")) --- print(x) -- 23]]> - -function xml.texttocdata(e) -- could be a finalizer - local dt = e.dt - local s = xml.tostring(dt) -- no shortcut? - e.tg = "@cd@" - e.special = true - e.ns = "" - e.rn = "" - e.dt = { s } - e.at = nil -end - --- local x = xml.convert("123") --- xml.tocdata(xml.first(x,"a")) --- print(x) -- 123]]> - -function xml.elementtocdata(e) -- could be a finalizer - local dt = e.dt - local s = xml.tostring(e) -- no shortcut? - e.tg = "@cd@" - e.special = true - e.ns = "" - e.rn = "" - e.dt = { s } - e.at = nil -end - -xml.builtinentities = table.tohash { "amp", "quot", "apos", "lt", "gt" } -- used often so share - -local entities = characters and characters.entities or nil -local builtinentities = xml.builtinentities - -function xml.addentitiesdoctype(root,option) -- we could also have a 'resolve' i.e. inline hex - if not entities then - require("char-ent") - entities = characters.entities - end - if entities and root and root.tg == "@rt@" and root.statistics then - local list = { } - local hexify = option == "hexadecimal" - for k, v in table.sortedhash(root.statistics.entities.names) do - if not builtinentities[k] then - local e = entities[k] - if not e then - e = format("[%s]",k) - elseif hexify then - e = format("&#%05X;",utfbyte(k)) - end - list[#list+1] = format(" ",k,e) - end - end - local dt = root.dt - local n = dt[1].tg == "@pi@" and 2 or 1 - if #list > 0 then - insert(dt, n, { "\n" }) - insert(dt, n, { - tg = "@dt@", -- beware, doctype is unparsed - dt = { format("Something [\n%s\n] ",concat(list)) }, - ns = "", - special = true, - }) - insert(dt, n, { "\n\n" }) - else - -- insert(dt, n, { table.serialize(root.statistics) }) - end + local dt=e.dt + if #dt==1 then + local first=dt[1] + if first.tg=="@cd@" then + e.dt=first.dt + end + else + end +end +function xml.texttocdata(e) + local dt=e.dt + local s=xml.tostring(dt) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +function xml.elementtocdata(e) + local dt=e.dt + local s=xml.tostring(e) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +xml.builtinentities=table.tohash { "amp","quot","apos","lt","gt" } +local entities=characters and characters.entities or nil +local builtinentities=xml.builtinentities +function xml.addentitiesdoctype(root,option) + if not entities then + require("char-ent") + entities=characters.entities + end + if entities and root and root.tg=="@rt@" and root.statistics then + local list={} + local hexify=option=="hexadecimal" + for k,v in table.sortedhash(root.statistics.entities.names) do + if not builtinentities[k] then + local e=entities[k] + if not e then + e=format("[%s]",k) + elseif hexify then + e=format("&#%05X;",utfbyte(k)) + end + list[#list+1]=format(" ",k,e) + end end -end - --- local str = [==[ --- --- --- test   test { test --- --- --- ]==] --- --- local x = xml.convert(str) --- xml.addentitiesdoctype(x,"hexadecimal") --- print(x) - - - -xml.all = xml.each -xml.insert = xml.insertafter -xml.inject = xml.injectafter -xml.after = xml.insertafter -xml.before = xml.insertbefore -xml.process = xml.each - --- obsolete - -xml.obsolete = xml.obsolete or { } -local obsolete = xml.obsolete - -xml.strip_whitespace = xml.strip obsolete.strip_whitespace = xml.strip -xml.collect_elements = xml.collect obsolete.collect_elements = xml.collect -xml.delete_element = xml.delete obsolete.delete_element = xml.delete -xml.replace_element = xml.replace obsolete.replace_element = xml.replacet -xml.each_element = xml.each obsolete.each_element = xml.each -xml.process_elements = xml.process obsolete.process_elements = xml.process -xml.insert_element_after = xml.insertafter obsolete.insert_element_after = xml.insertafter -xml.insert_element_before = xml.insertbefore obsolete.insert_element_before = xml.insertbefore -xml.inject_element_after = xml.injectafter obsolete.inject_element_after = xml.injectafter -xml.inject_element_before = xml.injectbefore obsolete.inject_element_before = xml.injectbefore -xml.process_attributes = xml.processattributes obsolete.process_attributes = xml.processattributes -xml.collect_texts = xml.collecttexts obsolete.collect_texts = xml.collecttexts -xml.inject_element = xml.inject obsolete.inject_element = xml.inject -xml.remap_tag = xml.remaptag obsolete.remap_tag = xml.remaptag -xml.remap_name = xml.remapname obsolete.remap_name = xml.remapname -xml.remap_namespace = xml.remapnamespace obsolete.remap_namespace = xml.remapnamespace - --- new (probably ok) - + local dt=root.dt + local n=dt[1].tg=="@pi@" and 2 or 1 + if #list>0 then + insert(dt,n,{ "\n" }) + insert(dt,n,{ + tg="@dt@", + dt={ format("Something [\n%s\n] ",concat(list)) }, + ns="", + special=true, + }) + insert(dt,n,{ "\n\n" }) + else + end + end +end +xml.all=xml.each +xml.insert=xml.insertafter +xml.inject=xml.injectafter +xml.after=xml.insertafter +xml.before=xml.insertbefore +xml.process=xml.each +xml.obsolete=xml.obsolete or {} +local obsolete=xml.obsolete +xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip +xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect +xml.delete_element=xml.delete obsolete.delete_element=xml.delete +xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.each_element=xml.each obsolete.each_element=xml.each +xml.process_elements=xml.process obsolete.process_elements=xml.process +xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter +xml.insert_element_before=xml.insertbefore obsolete.insert_element_before=xml.insertbefore +xml.inject_element_after=xml.injectafter obsolete.inject_element_after=xml.injectafter +xml.inject_element_before=xml.injectbefore obsolete.inject_element_before=xml.injectbefore +xml.process_attributes=xml.processattributes obsolete.process_attributes=xml.processattributes +xml.collect_texts=xml.collecttexts obsolete.collect_texts=xml.collecttexts +xml.inject_element=xml.inject obsolete.inject_element=xml.inject +xml.remap_tag=xml.remaptag obsolete.remap_tag=xml.remaptag +xml.remap_name=xml.remapname obsolete.remap_name=xml.remapname +xml.remap_namespace=xml.remapnamespace obsolete.remap_namespace=xml.remapnamespace function xml.cdata(e) - if e then - local dt = e.dt - if dt and #dt == 1 then - local first = dt[1] - return first.tg == "@cd@" and first.dt[1] or "" - end + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" end - return "" + end + return "" end - function xml.finalizers.xml.cdata(collected) - if collected then - local e = collected[1] - if e then - local dt = e.dt - if dt and #dt == 1 then - local first = dt[1] - return first.tg == "@cd@" and first.dt[1] or "" - end - end + if collected then + local e=collected[1] + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" + end end - return "" -end - -function xml.insertcomment(e,str,n) -- also insertcdata - table.insert(e.dt,n or 1,{ - tg = "@cm@", - ns = "", - special = true, - at = { }, - dt = { str }, - }) -end - -function xml.setcdata(e,str) -- also setcomment - e.dt = { { - tg = "@cd@", - ns = "", - special = true, - at = { }, - dt = { str }, - } } + end + return "" +end +function xml.insertcomment(e,str,n) + table.insert(e.dt,n or 1,{ + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.setcdata(e,str) + e.dt={ { + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + } } end - --- maybe helpers like this will move to an autoloader - function xml.separate(x,pattern) - local collected = xmlapplylpath(x,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local d = e.dt - if d == x then - report_xml("warning: xml.separate changes root") - x = d - end - local t, n = { "\n" }, 1 - local i, nd = 1, #d - while i <= nd do - while i <= nd do - local di = d[i] - if type(di) == "string" then - if di == "\n" or find(di,"^%s+$") then -- first test is speedup - i = i + 1 - else - d[i] = strip(di) - break - end - else - break - end - end - if i > nd then - break - end - t[n+1] = "\n" - t[n+2] = d[i] - t[n+3] = "\n" - n = n + 3 - i = i + 1 + local collected=xmlapplylpath(x,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local d=e.dt + if d==x then + report_xml("warning: xml.separate changes root") + x=d + end + local t,n={ "\n" },1 + local i,nd=1,#d + while i<=nd do + while i<=nd do + local di=d[i] + if type(di)=="string" then + if di=="\n" or find(di,"^%s+$") then + i=i+1 + else + d[i]=strip(di) + break end - t[n+1] = "\n" - setmetatable(t,getmetatable(d)) - e.dt = t + else + break + end + end + if i>nd then + break end + t[n+1]="\n" + t[n+2]=d[i] + t[n+3]="\n" + n=n+3 + i=i+1 + end + t[n+1]="\n" + setmetatable(t,getmetatable(d)) + e.dt=t end - return x + end + return x end - --- - -local helpers = xml.helpers or { } -xml.helpers = helpers - +local helpers=xml.helpers or {} +xml.helpers=helpers local function normal(e,action) - local edt = e.dt - if edt then - for i=1,#edt do - local str = edt[i] - if type(str) == "string" and str ~= "" then - edt[i] = action(str) - end - end + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)=="string" and str~="" then + edt[i]=action(str) + end end + end end - local function recurse(e,action) - local edt = e.dt - if edt then - for i=1,#edt do - local str = edt[i] - if type(str) ~= "string" then - recurse(str,action,recursive) - elseif str ~= "" then - edt[i] = action(str) - end - end + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)~="string" then + recurse(str,action,recursive) + elseif str~="" then + edt[i]=action(str) + end end + end end - function helpers.recursetext(collected,action,recursive) - if recursive then - for i=1,#collected do - recurse(collected[i],action) - end - else - for i=1,#collected do - normal(collected[i],action) - end + if recursive then + for i=1,#collected do + recurse(collected[i],action) + end + else + for i=1,#collected do + normal(collected[i],action) end + end end @@ -13011,450 +9933,375 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-xml'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local concat = table.concat -local find, lower, upper = string.find, string.lower, string.upper - -local xml = xml +-- original size: 10274, stripped down to: 7538 -local finalizers = xml.finalizers.xml -local xmlfilter = xml.filter -- we could inline this one for speed -local xmltostring = xml.tostring -local xmlserialize = xml.serialize -local xmlcollected = xml.collected -local xmlnewhandlers = xml.newhandlers - -local function first(collected) -- wrong ? - return collected and collected[1] +if not modules then modules={} end modules ['lxml-xml']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat=table.concat +local find,lower,upper=string.find,string.lower,string.upper +local xml=xml +local finalizers=xml.finalizers.xml +local xmlfilter=xml.filter +local xmltostring=xml.tostring +local xmlserialize=xml.serialize +local xmlcollected=xml.collected +local xmlnewhandlers=xml.newhandlers +local function first(collected) + return collected and collected[1] end - local function last(collected) - return collected and collected[#collected] + return collected and collected[#collected] end - local function all(collected) - return collected + return collected end - --- local function reverse(collected) --- if collected then --- local nc = #collected --- if nc > 0 then --- local reversed, r = { }, 0 --- for c=nc,1,-1 do --- r = r + 1 --- reversed[r] = collected[c] --- end --- return reversed --- else --- return collected --- end --- end --- end - -local reverse = table.reversed - +local reverse=table.reversed local function attribute(collected,name) - if collected and #collected > 0 then - local at = collected[1].at - return at and at[name] - end + if collected and #collected>0 then + local at=collected[1].at + return at and at[name] + end end - local function att(id,name) - local at = id.at - return at and at[name] + local at=id.at + return at and at[name] end - local function count(collected) - return collected and #collected or 0 + return collected and #collected or 0 end - local function position(collected,n) - if not collected then - return 0 - end - local nc = #collected - if nc == 0 then - return 0 - end - n = tonumber(n) or 0 - if n < 0 then - return collected[nc + n + 1] - elseif n > 0 then - return collected[n] - else - return collected[1].mi or 0 - end + if not collected then + return 0 + end + local nc=#collected + if nc==0 then + return 0 + end + n=tonumber(n) or 0 + if n<0 then + return collected[nc+n+1] + elseif n>0 then + return collected[n] + else + return collected[1].mi or 0 + end end - local function match(collected) - return collected and #collected > 0 and collected[1].mi or 0 -- match + return collected and #collected>0 and collected[1].mi or 0 end - local function index(collected) - return collected and #collected > 0 and collected[1].ni or 0 -- 0 is new + return collected and #collected>0 and collected[1].ni or 0 end - local function attributes(collected,arguments) - if collected and #collected > 0 then - local at = collected[1].at - if arguments then - return at[arguments] - elseif next(at) then - return at -- all of them + if collected and #collected>0 then + local at=collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at + end + end +end +local function chainattribute(collected,arguments) + if collected and #collected>0 then + local e=collected[1] + while e do + local at=e.at + if at then + local a=at[arguments] + if a then + return a end + else + break + end + e=e.__p__ end + end + return "" end - -local function chainattribute(collected,arguments) -- todo: optional levels - if collected and #collected > 0 then - local e = collected[1] - while e do - local at = e.at - if at then - local a = at[arguments] - if a then - return a - end - else - break -- error - end - e = e.__p__ - end - end +local function raw(collected) + if collected and #collected>0 then + local e=collected[1] or collected + return e and xmltostring(e) or "" + else return "" + end end - -local function raw(collected) -- hybrid (not much different from text so it might go) - if collected and #collected > 0 then - local e = collected[1] or collected - return e and xmltostring(e) or "" -- only first as we cannot concat function - else - return "" - end -end - --- - -local xmltexthandler = xmlnewhandlers { - name = "string", - initialize = function() - result = { } - return result - end, - finalize = function() - return concat(result) - end, - handle = function(...) - result[#result+1] = concat { ... } - end, - escape = false, +local xmltexthandler=xmlnewhandlers { + name="string", + initialize=function() + result={} + return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, + escape=false, } - local function xmltotext(root) - local dt = root.dt - if not dt then - return "" - end - local nt = #dt -- string or table - if nt == 0 then - return "" - elseif nt == 1 and type(dt[1]) == "string" then - return dt[1] -- no escaping of " ' < > & - else - return xmlserialize(root,xmltexthandler) or "" - end -end - --- - -local function text(collected) -- hybrid - if collected then -- no # test here ! - local e = collected[1] or collected -- why fallback to element, how about cdata - return e and xmltotext(e) or "" - else - return "" - end + local dt=root.dt + if not dt then + return "" + end + local nt=#dt + if nt==0 then + return "" + elseif nt==1 and type(dt[1])=="string" then + return dt[1] + else + return xmlserialize(root,xmltexthandler) or "" + end +end +local function text(collected) + if collected then + local e=collected[1] or collected + return e and xmltotext(e) or "" + else + return "" + end end - local function texts(collected) - if not collected then - return { } -- why no nil - end - local nc = #collected - if nc == 0 then - return { } -- why no nil - end - local t, n = { }, 0 - for c=1,nc do - local e = collected[c] - if e and e.dt then - n = n + 1 - t[n] = e.dt - end - end - return t + if not collected then + return {} + end + local nc=#collected + if nc==0 then + return {} + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + if e and e.dt then + n=n+1 + t[n]=e.dt + end + end + return t end - local function tag(collected,n) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local c - if n == 0 or not n then - c = collected[1] - elseif n > 1 then - c = collected[n] - else - c = collected[nc-n+1] - end - return c and c.tg + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + return c and c.tg end - local function name(collected,n) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local c - if n == 0 or not n then - c = collected[1] - elseif n > 1 then - c = collected[n] - else - c = collected[nc-n+1] - end - if not c then - -- sorry - elseif c.ns == "" then - return c.tg - else - return c.ns .. ":" .. c.tg - end + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + if not c then + elseif c.ns=="" then + return c.tg + else + return c.ns..":"..c.tg + end end - local function tags(collected,nonamespace) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local t, n = { }, 0 - for c=1,nc do - local e = collected[c] - local ns, tg = e.ns, e.tg - n = n + 1 - if nonamespace or ns == "" then - t[n] = tg - else - t[n] = ns .. ":" .. tg - end + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace or ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg end - return t + end + return t end - local function empty(collected,spacesonly) - if not collected then - return true - end - local nc = #collected - if nc == 0 then - return true - end - for c=1,nc do - local e = collected[c] - if e then - local edt = e.dt - if edt then - local n = #edt - if n == 1 then - local edk = edt[1] - local typ = type(edk) - if typ == "table" then - return false - elseif edk ~= "" then - return false - elseif spacesonly and not find(edk,"%S") then - return false - end - elseif n > 1 then - return false - end - end + if not collected then + return true + end + local nc=#collected + if nc==0 then + return true + end + for c=1,nc do + local e=collected[c] + if e then + local edt=e.dt + if edt then + local n=#edt + if n==1 then + local edk=edt[1] + local typ=type(edk) + if typ=="table" then + return false + elseif edk~="" then + return false + elseif spacesonly and not find(edk,"%S") then + return false + end + elseif n>1 then + return false end + end end - return true -end - -finalizers.first = first -finalizers.last = last -finalizers.all = all -finalizers.reverse = reverse -finalizers.elements = all -finalizers.default = all -finalizers.attribute = attribute -finalizers.att = att -finalizers.count = count -finalizers.position = position -finalizers.match = match -finalizers.index = index -finalizers.attributes = attributes -finalizers.chainattribute = chainattribute -finalizers.text = text -finalizers.texts = texts -finalizers.tag = tag -finalizers.name = name -finalizers.tags = tags -finalizers.empty = empty - --- shortcuts -- we could support xmlfilter(id,pattern,first) - + end + return true +end +finalizers.first=first +finalizers.last=last +finalizers.all=all +finalizers.reverse=reverse +finalizers.elements=all +finalizers.default=all +finalizers.attribute=attribute +finalizers.att=att +finalizers.count=count +finalizers.position=position +finalizers.match=match +finalizers.index=index +finalizers.attributes=attributes +finalizers.chainattribute=chainattribute +finalizers.text=text +finalizers.texts=texts +finalizers.tag=tag +finalizers.name=name +finalizers.tags=tags +finalizers.empty=empty function xml.first(id,pattern) - return first(xmlfilter(id,pattern)) + return first(xmlfilter(id,pattern)) end - function xml.last(id,pattern) - return last(xmlfilter(id,pattern)) + return last(xmlfilter(id,pattern)) end - function xml.count(id,pattern) - return count(xmlfilter(id,pattern)) + return count(xmlfilter(id,pattern)) end - function xml.attribute(id,pattern,a,default) - return attribute(xmlfilter(id,pattern),a,default) + return attribute(xmlfilter(id,pattern),a,default) end - function xml.raw(id,pattern) - if pattern then - return raw(xmlfilter(id,pattern)) - else - return raw(id) - end -end - -function xml.text(id,pattern) -- brrr either content or element (when cdata) - if pattern then - -- return text(xmlfilter(id,pattern)) - local collected = xmlfilter(id,pattern) - return collected and #collected > 0 and xmltotext(collected[1]) or "" - elseif id then - -- return text(id) - return xmltotext(id) or "" - else - return "" - end + if pattern then + return raw(xmlfilter(id,pattern)) + else + return raw(id) + end +end +function xml.text(id,pattern) + if pattern then + local collected=xmlfilter(id,pattern) + return collected and #collected>0 and xmltotext(collected[1]) or "" + elseif id then + return xmltotext(id) or "" + else + return "" + end end - -xml.content = text - --- - -function xml.position(id,pattern,n) -- element - return position(xmlfilter(id,pattern),n) +xml.content=text +function xml.position(id,pattern,n) + return position(xmlfilter(id,pattern),n) end - -function xml.match(id,pattern) -- number - return match(xmlfilter(id,pattern)) +function xml.match(id,pattern) + return match(xmlfilter(id,pattern)) end - function xml.empty(id,pattern,spacesonly) - return empty(xmlfilter(id,pattern),spacesonly) + return empty(xmlfilter(id,pattern),spacesonly) end - -xml.all = xml.filter -xml.index = xml.position -xml.found = xml.filter - --- a nice one: - +xml.all=xml.filter +xml.index=xml.position +xml.found=xml.filter local function totable(x) - local t = { } - for e in xmlcollected(x[1] or x,"/*") do - t[e.tg] = xmltostring(e.dt) or "" - end - return next(t) and t or nil -end - -xml.table = totable -finalizers.table = totable - + local t={} + for e in xmlcollected(x[1] or x,"/*") do + t[e.tg]=xmltostring(e.dt) or "" + end + return next(t) and t or nil +end +xml.table=totable +finalizers.table=totable local function textonly(e,t) - if e then - local edt = e.dt - if edt then - for i=1,#edt do - local e = edt[i] - if type(e) == "table" then - textonly(e,t) - else - t[#t+1] = e - end - end + if e then + local edt=e.dt + if edt then + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + textonly(e,t) + else + t[#t+1]=e end + end end - return t + end + return t end - -function xml.textonly(e) -- no pattern - return concat(textonly(e,{})) +function xml.textonly(e) + return concat(textonly(e,{})) end - --- - --- local x = xml.convert("123") --- xml.filter(x,"**/lowerall()") print(x) --- xml.filter(x,"**/upperall()") print(x) - function finalizers.lowerall(collected) - for c=1,#collected do - local e = collected[c] - if not e.special then - e.tg = lower(e.tg) - local eat = e.at - if eat then - local t = { } - for k,v in next, eat do - t[lower(k)] = v - end - e.at = t - end - end + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=lower(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[lower(k)]=v + end + e.at=t + end end + end end - function finalizers.upperall(collected) - for c=1,#collected do - local e = collected[c] - if not e.special then - e.tg = upper(e.tg) - local eat = e.at - if eat then - local t = { } - for k,v in next, eat do - t[upper(k)] = v - end - e.at = t - end - end + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=upper(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[upper(k)]=v + end + e.at=t + end end + end end @@ -13462,237 +10309,159 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-ini'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - -local gsub, find, gmatch, char = string.gsub, string.find, string.gmatch, string.char -local next, type = next, type - -local filedirname, filebasename, filejoin = file.dirname, file.basename, file.join - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_initialization = logs.reporter("resolvers","initialization") +-- original size: 7894, stripped down to: 5497 -local ostype, osname, ossetenv, osgetenv = os.type, os.name, os.setenv, os.getenv - --- The code here used to be part of a data-res but for convenience --- we now split it over multiple files. As this file is now the --- starting point we introduce resolvers here. - -resolvers = resolvers or { } -local resolvers = resolvers - --- We don't want the kpse library to kick in. Also, we want to be able to --- execute programs. Control over execution is implemented later. - -texconfig.kpse_init = false -texconfig.shell_escape = 't' - -if kpse and kpse.default_texmfcnf then - local default_texmfcnf = kpse.default_texmfcnf() - -- looks more like context: - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTODIR","selfautodir:") - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTOPARENT","selfautoparent:") - default_texmfcnf = gsub(default_texmfcnf,"$HOME","home:") - -- - environment.default_texmfcnf = default_texmfcnf -end - -kpse = { original = kpse } - -setmetatable(kpse, { - __index = function(kp,name) - report_initialization("fatal error: kpse library is accessed (key: %s)",name) - os.exit() - end +if not modules then modules={} end modules ['data-ini']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char +local next,type=next,type +local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_initialization=logs.reporter("resolvers","initialization") +local ostype,osname,ossetenv,osgetenv=os.type,os.name,os.setenv,os.getenv +resolvers=resolvers or {} +local resolvers=resolvers +texconfig.kpse_init=false +texconfig.shell_escape='t' +if not (environment and environment.default_texmfcnf) and kpse and kpse.default_texmfcnf then + local default_texmfcnf=kpse.default_texmfcnf() + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTODIR","selfautodir:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOPARENT","selfautoparent:") + default_texmfcnf=gsub(default_texmfcnf,"$HOME","home:") + environment.default_texmfcnf=default_texmfcnf +end +kpse={ original=kpse } +setmetatable(kpse,{ + __index=function(kp,name) + report_initialization("fatal error: kpse library is accessed (key: %s)",name) + os.exit() + end } ) - --- First we check a couple of environment variables. Some might be --- set already but we need then later on. We start with the system --- font path. - do - - local osfontdir = osgetenv("OSFONTDIR") - - if osfontdir and osfontdir ~= "" then - -- ok - elseif osname == "windows" then - ossetenv("OSFONTDIR","c:/windows/fonts//") - elseif osname == "macosx" then - ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") - end - + local osfontdir=osgetenv("OSFONTDIR") + if osfontdir and osfontdir~="" then + elseif osname=="windows" then + ossetenv("OSFONTDIR","c:/windows/fonts//") + elseif osname=="macosx" then + ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + end end - --- Next comes the user's home path. We need this as later on we have --- to replace ~ with its value. - do - - local homedir = osgetenv(ostype == "windows" and 'USERPROFILE' or 'HOME') or '' - - if not homedir or homedir == "" then - homedir = char(127) -- we need a value, later we wil trigger on it - end - - homedir = file.collapsepath(homedir) - - ossetenv("HOME", homedir) -- can be used in unix cnf files - ossetenv("USERPROFILE",homedir) -- can be used in windows cnf files - - environment.homedir = homedir - + local homedir=osgetenv(ostype=="windows" and 'USERPROFILE' or 'HOME') or '' + if not homedir or homedir=="" then + homedir=char(127) + end + homedir=file.collapsepath(homedir) + ossetenv("HOME",homedir) + ossetenv("USERPROFILE",homedir) + environment.homedir=homedir end - --- The following code sets the name of the own binary and its --- path. This is fallback code as we have os.selfdir now. - do - - local args = environment.originalarguments or arg -- this needs a cleanup - - if not environment.ownmain then - environment.ownmain = status and string.match(string.lower(status.banner),"this is ([%a]+)") or "luatex" - end - - local ownbin = environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" - local ownpath = environment.ownpath or os.selfdir - - ownbin = file.collapsepath(ownbin) - ownpath = file.collapsepath(ownpath) - - if not ownpath or ownpath == "" or ownpath == "unset" then - ownpath = args[-1] or arg[-1] - ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) - if not ownpath or ownpath == "" then - ownpath = args[-0] or arg[-0] - ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) - end - local binary = ownbin - if not ownpath or ownpath == "" then - ownpath = ownpath and filedirname(binary) - end - if not ownpath or ownpath == "" then - if os.binsuffix ~= "" then - binary = file.replacesuffix(binary,os.binsuffix) - end - local path = osgetenv("PATH") - if path then - for p in gmatch(path,"[^"..io.pathseparator.."]+") do - local b = filejoin(p,binary) - if lfs.isfile(b) then - -- we assume that after changing to the path the currentdir function - -- resolves to the real location and use this side effect here; this - -- trick is needed because on the mac installations use symlinks in the - -- path instead of real locations - local olddir = lfs.currentdir() - if lfs.chdir(p) then - local pp = lfs.currentdir() - if trace_locating and p ~= pp then - report_initialization("following symlink '%s' to '%s'",p,pp) - end - ownpath = pp - lfs.chdir(olddir) - else - if trace_locating then - report_initialization("unable to check path '%s'",p) - end - ownpath = p - end - break - end - end + local args=environment.originalarguments or arg + if not environment.ownmain then + environment.ownmain=status and string.match(string.lower(status.banner),"this is ([%a]+)") or "luatex" + end + local ownbin=environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" + local ownpath=environment.ownpath or os.selfdir + ownbin=file.collapsepath(ownbin) + ownpath=file.collapsepath(ownpath) + if not ownpath or ownpath=="" or ownpath=="unset" then + ownpath=args[-1] or arg[-1] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath=="" then + ownpath=args[-0] or arg[-0] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + end + local binary=ownbin + if not ownpath or ownpath=="" then + ownpath=ownpath and filedirname(binary) + end + if not ownpath or ownpath=="" then + if os.binsuffix~="" then + binary=file.replacesuffix(binary,os.binsuffix) + end + local path=osgetenv("PATH") + if path then + for p in gmatch(path,"[^"..io.pathseparator.."]+") do + local b=filejoin(p,binary) + if lfs.isfile(b) then + local olddir=lfs.currentdir() + if lfs.chdir(p) then + local pp=lfs.currentdir() + if trace_locating and p~=pp then + report_initialization("following symlink '%s' to '%s'",p,pp) + end + ownpath=pp + lfs.chdir(olddir) + else + if trace_locating then + report_initialization("unable to check path '%s'",p) + end + ownpath=p end + break + end end - if not ownpath or ownpath == "" then - ownpath = "." - report_initialization("forcing fallback ownpath .") - elseif trace_locating then - report_initialization("using ownpath '%s'",ownpath) - end + end end - - environment.ownbin = ownbin - environment.ownpath = ownpath - + if not ownpath or ownpath=="" then + ownpath="." + report_initialization("forcing fallback ownpath .") + elseif trace_locating then + report_initialization("using ownpath '%s'",ownpath) + end + end + environment.ownbin=ownbin + environment.ownpath=ownpath end - -resolvers.ownpath = environment.ownpath - +resolvers.ownpath=environment.ownpath function resolvers.getownpath() - return environment.ownpath + return environment.ownpath end - --- The self variables permit us to use only a few (or even no) --- environment variables. - do - - local ownpath = environment.ownpath or dir.current() - - if ownpath then - ossetenv('SELFAUTOLOC', file.collapsepath(ownpath)) - ossetenv('SELFAUTODIR', file.collapsepath(ownpath .. "/..")) - ossetenv('SELFAUTOPARENT', file.collapsepath(ownpath .. "/../..")) - else - report_initialization("error: unable to locate ownpath") - os.exit() - end - -end - --- The running os: - --- todo: check is context sits here os.platform is more trustworthy --- that the bin check as mtx-update runs from another path - -local texos = environment.texos or osgetenv("TEXOS") -local texmfos = environment.texmfos or osgetenv('SELFAUTODIR') - -if not texos or texos == "" then - texos = file.basename(texmfos) -end - -ossetenv('TEXMFOS', texmfos) -- full bin path -ossetenv('TEXOS', texos) -- partial bin parent -ossetenv('SELFAUTOSYSTEM',os.platform) -- bonus - -environment.texos = texos -environment.texmfos = texmfos - --- The current root: - -local texroot = environment.texroot or osgetenv("TEXROOT") - -if not texroot or texroot == "" then - texroot = osgetenv('SELFAUTOPARENT') - ossetenv('TEXROOT',texroot) -end - -environment.texroot = file.collapsepath(texroot) - + local ownpath=environment.ownpath or dir.current() + if ownpath then + ossetenv('SELFAUTOLOC',file.collapsepath(ownpath)) + ossetenv('SELFAUTODIR',file.collapsepath(ownpath.."/..")) + ossetenv('SELFAUTOPARENT',file.collapsepath(ownpath.."/../..")) + else + report_initialization("error: unable to locate ownpath") + os.exit() + end +end +local texos=environment.texos or osgetenv("TEXOS") +local texmfos=environment.texmfos or osgetenv('SELFAUTODIR') +if not texos or texos=="" then + texos=file.basename(texmfos) +end +ossetenv('TEXMFOS',texmfos) +ossetenv('TEXOS',texos) +ossetenv('SELFAUTOSYSTEM',os.platform) +environment.texos=texos +environment.texmfos=texmfos +local texroot=environment.texroot or osgetenv("TEXROOT") +if not texroot or texroot=="" then + texroot=osgetenv('SELFAUTOPARENT') + ossetenv('TEXROOT',texroot) +end +environment.texroot=file.collapsepath(texroot) if profiler then - directives.register("system.profile",function() - profiler.start("luatex-profile.log") - end) + directives.register("system.profile",function() + profiler.start("luatex-profile.log") + end) end - --- a forward definition - if not resolvers.resolve then - function resolvers.resolve (s) return s end - function resolvers.unresolve(s) return s end - function resolvers.repath (s) return s end + function resolvers.resolve (s) return s end + function resolvers.unresolve(s) return s end + function resolvers.repath (s) return s end end @@ -13700,3216 +10469,2663 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-exp'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - -local format, find, gmatch, lower, char, sub = string.format, string.find, string.gmatch, string.lower, string.char, string.sub -local concat, sort = table.concat, table.sort -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns -local Ct, Cs, Cc, P, C, S = lpeg.Ct, lpeg.Cs, lpeg.Cc, lpeg.P, lpeg.C, lpeg.S -local type, next = type, next - -local ostype = os.type -local collapsepath = file.collapsepath - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_expansions = logs.reporter("resolvers","expansions") - -local resolvers = resolvers - --- As this bit of code is somewhat special it gets its own module. After --- all, when working on the main resolver code, I don't want to scroll --- past this every time. See data-obs.lua for the gsub variant. +-- original size: 14663, stripped down to: 9537 +if not modules then modules={} end modules ['data-exp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local format,find,gmatch,lower,char,sub=string.format,string.find,string.gmatch,string.lower,string.char,string.sub +local concat,sort=table.concat,table.sort +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local Ct,Cs,Cc,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.P,lpeg.C,lpeg.S +local type,next=type,next +local ostype=os.type +local collapsepath=file.collapsepath +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_expansions=logs.reporter("resolvers","expansions") +local resolvers=resolvers local function f_first(a,b) - local t, n = { }, 0 - for s in gmatch(b,"[^,]+") do - n = n + 1 ; t[n] = a .. s - end - return concat(t,",") + local t,n={},0 + for s in gmatch(b,"[^,]+") do + n=n+1;t[n]=a..s + end + return concat(t,",") end - local function f_second(a,b) - local t, n = { }, 0 - for s in gmatch(a,"[^,]+") do - n = n + 1 ; t[n] = s .. b - end - return concat(t,",") + local t,n={},0 + for s in gmatch(a,"[^,]+") do + n=n+1;t[n]=s..b + end + return concat(t,",") end - --- kpsewhich --expand-braces '{a,b}{c,d}' --- ac:bc:ad:bd - --- old {a,b}{c,d} => ac ad bc bd --- --- local function f_both(a,b) --- local t, n = { }, 0 --- for sa in gmatch(a,"[^,]+") do --- for sb in gmatch(b,"[^,]+") do --- n = n + 1 ; t[n] = sa .. sb --- end --- end --- return concat(t,",") --- end --- --- new {a,b}{c,d} => ac bc ad bd - local function f_both(a,b) - local t, n = { }, 0 - for sb in gmatch(b,"[^,]+") do -- and not sa - for sa in gmatch(a,"[^,]+") do -- sb - n = n + 1 ; t[n] = sa .. sb - end - end - return concat(t,",") -end - -local left = P("{") -local right = P("}") -local var = P((1 - S("{}" ))^0) -local set = P((1 - S("{},"))^0) -local other = P(1) - -local l_first = Cs( ( Cc("{") * (C(set) * left * C(var) * right / f_first) * Cc("}") + other )^0 ) -local l_second = Cs( ( Cc("{") * (left * C(var) * right * C(set) / f_second) * Cc("}") + other )^0 ) -local l_both = Cs( ( Cc("{") * (left * C(var) * right * left * C(var) * right / f_both) * Cc("}") + other )^0 ) -local l_rest = Cs( ( left * var * (left/"") * var * (right/"") * var * right + other )^0 ) - -local stripper_1 = lpeg.stripper ("{}@") -local replacer_1 = lpeg.replacer { { ",}", ",@}" }, { "{,", "{@," }, } - -local function splitpathexpr(str, newlist, validate) -- I couldn't resist lpegging it (nice exercise). - if trace_expansions then - report_expansions("expanding variable '%s'",str) - end - local t, ok, done = newlist or { }, false, false - local n = #t - str = lpegmatch(replacer_1,str) + local t,n={},0 + for sb in gmatch(b,"[^,]+") do + for sa in gmatch(a,"[^,]+") do + n=n+1;t[n]=sa..sb + end + end + return concat(t,",") +end +local left=P("{") +local right=P("}") +local var=P((1-S("{}" ))^0) +local set=P((1-S("{},"))^0) +local other=P(1) +local l_first=Cs((Cc("{")*(C(set)*left*C(var)*right/f_first)*Cc("}")+other )^0 ) +local l_second=Cs((Cc("{")*(left*C(var)*right*C(set)/f_second)*Cc("}")+other )^0 ) +local l_both=Cs((Cc("{")*(left*C(var)*right*left*C(var)*right/f_both)*Cc("}")+other )^0 ) +local l_rest=Cs((left*var*(left/"")*var*(right/"")*var*right+other )^0 ) +local stripper_1=lpeg.stripper ("{}@") +local replacer_1=lpeg.replacer { { ",}",",@}" },{ "{,","{@," },} +local function splitpathexpr(str,newlist,validate) + if trace_expansions then + report_expansions("expanding variable '%s'",str) + end + local t,ok,done=newlist or {},false,false + local n=#t + str=lpegmatch(replacer_1,str) + repeat + local old=str repeat - local old = str - repeat - local old = str - str = lpegmatch(l_first, str) - until old == str - repeat - local old = str - str = lpegmatch(l_second,str) - until old == str - repeat - local old = str - str = lpegmatch(l_both, str) - until old == str - repeat - local old = str - str = lpegmatch(l_rest, str) - until old == str - until old == str -- or not find(str,"{") - str = lpegmatch(stripper_1,str) - if validate then - for s in gmatch(str,"[^,]+") do - s = validate(s) - if s then - n = n + 1 - t[n] = s - end - end - else - for s in gmatch(str,"[^,]+") do - n = n + 1 - t[n] = s - end + local old=str + str=lpegmatch(l_first,str) + until old==str + repeat + local old=str + str=lpegmatch(l_second,str) + until old==str + repeat + local old=str + str=lpegmatch(l_both,str) + until old==str + repeat + local old=str + str=lpegmatch(l_rest,str) + until old==str + until old==str + str=lpegmatch(stripper_1,str) + if validate then + for s in gmatch(str,"[^,]+") do + s=validate(s) + if s then + n=n+1 + t[n]=s + end end - if trace_expansions then - for k=1,#t do - report_expansions("% 4i: %s",k,t[k]) - end + else + for s in gmatch(str,"[^,]+") do + n=n+1 + t[n]=s end - return t + end + if trace_expansions then + for k=1,#t do + report_expansions("% 4i: %s",k,t[k]) + end + end + return t end - --- We could make the previous one public. - local function validate(s) - s = collapsepath(s) -- already keeps the trailing / and // - return s ~= "" and not find(s,"^!*unset/*$") and s + s=collapsepath(s) + return s~="" and not find(s,"^!*unset/*$") and s end - -resolvers.validatedpath = validate -- keeps the trailing // - +resolvers.validatedpath=validate function resolvers.expandedpathfromlist(pathlist) - local newlist = { } - for k=1,#pathlist do - splitpathexpr(pathlist[k],newlist,validate) - end - return newlist -end - --- {a,b,c,d} --- a,b,c/{p,q,r},d --- a,b,c/{p,q,r}/d/{x,y,z}// --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a{b,c}{d,e}f --- {a,b,c,d} --- {a,b,c/{p,q,r},d} --- {a,b,c/{p,q,r}/d/{x,y,z}//} --- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} --- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} --- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} - -local cleanup = lpeg.replacer { - { "!" , "" }, - { "\\" , "/" }, + local newlist={} + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) + end + return newlist +end +local cleanup=lpeg.replacer { + { "!","" }, + { "\\","/" }, } - -function resolvers.cleanpath(str) -- tricky, maybe only simple paths - local doslashes = (P("\\")/"/" + 1)^0 - local donegation = (P("!") /"" )^0 - local homedir = lpegmatch(Cs(donegation * doslashes),environment.homedir or "") - if homedir == "~" or homedir == "" or not lfs.isdir(homedir) then - if trace_expansions then - report_expansions("no home dir set, ignoring dependent paths") - end - function resolvers.cleanpath(str) - if not str or find(str,"~") then - return "" -- special case - else - return lpegmatch(cleanup,str) - end - end - else - local dohome = ((P("~")+P("$HOME"))/homedir)^0 - local cleanup = Cs(donegation * dohome * doslashes) - function resolvers.cleanpath(str) - return str and lpegmatch(cleanup,str) or "" - end +function resolvers.cleanpath(str) + local doslashes=(P("\\")/"/"+1)^0 + local donegation=(P("!")/"" )^0 + local homedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") + if homedir=="~" or homedir=="" or not lfs.isdir(homedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent paths") end - return resolvers.cleanpath(str) -end - --- print(resolvers.cleanpath("")) --- print(resolvers.cleanpath("!")) --- print(resolvers.cleanpath("~")) --- print(resolvers.cleanpath("~/test")) --- print(resolvers.cleanpath("!~/test")) --- print(resolvers.cleanpath("~/test~test")) - --- This one strips quotes and funny tokens. - -local expandhome = P("~") / "$HOME" -- environment.homedir - -local dodouble = P('"')/"" * (expandhome + (1 - P('"')))^0 * P('"')/"" -local dosingle = P("'")/"" * (expandhome + (1 - P("'")))^0 * P("'")/"" -local dostring = (expandhome + 1 )^0 - -local stripper = Cs( - lpegpatterns.unspacer * (dosingle + dodouble + dostring) * lpegpatterns.unspacer + function resolvers.cleanpath(str) + if not str or find(str,"~") then + return "" + else + return lpegmatch(cleanup,str) + end + end + else + local dohome=((P("~")+P("$HOME"))/homedir)^0 + local cleanup=Cs(donegation*dohome*doslashes) + function resolvers.cleanpath(str) + return str and lpegmatch(cleanup,str) or "" + end + end + return resolvers.cleanpath(str) +end +local expandhome=P("~")/"$HOME" +local dodouble=P('"')/""*(expandhome+(1-P('"')))^0*P('"')/"" +local dosingle=P("'")/""*(expandhome+(1-P("'")))^0*P("'")/"" +local dostring=(expandhome+1 )^0 +local stripper=Cs( + lpegpatterns.unspacer*(dosingle+dodouble+dostring)*lpegpatterns.unspacer ) - -function resolvers.checkedvariable(str) -- assumes str is a string - return type(str) == "string" and lpegmatch(stripper,str) or str -end - --- The path splitter: - --- A config (optionally) has the paths split in tables. Internally --- we join them and split them after the expansion has taken place. This --- is more convenient. - -local cache = { } - ------ splitter = lpeg.tsplitat(S(ostype == "windows" and ";" or ":;")) -- maybe add , -local splitter = lpeg.tsplitat(";") -- as we move towards urls, prefixes and use tables we no longer do : - -local backslashswapper = lpeg.replacer("\\","/") - -local function splitconfigurationpath(str) -- beware, this can be either a path or a { specification } - if str then - local found = cache[str] - if not found then - if str == "" then - found = { } - else - local split = lpegmatch(splitter,lpegmatch(backslashswapper,str)) -- can be combined - found = { } - local noffound = 0 - for i=1,#split do - local s = split[i] - if not find(s,"^{*unset}*") then - noffound = noffound + 1 - found[noffound] = s - end - end - if trace_expansions then - report_expansions("splitting path specification '%s'",str) - for k=1,noffound do - report_expansions("% 4i: %s",k,found[k]) - end - end - cache[str] = found - end +function resolvers.checkedvariable(str) + return type(str)=="string" and lpegmatch(stripper,str) or str +end +local cache={} +local splitter=lpeg.tsplitat(";") +local backslashswapper=lpeg.replacer("\\","/") +local function splitconfigurationpath(str) + if str then + local found=cache[str] + if not found then + if str=="" then + found={} + else + local split=lpegmatch(splitter,lpegmatch(backslashswapper,str)) + found={} + local noffound=0 + for i=1,#split do + local s=split[i] + if not find(s,"^{*unset}*") then + noffound=noffound+1 + found[noffound]=s + end + end + if trace_expansions then + report_expansions("splitting path specification '%s'",str) + for k=1,noffound do + report_expansions("% 4i: %s",k,found[k]) + end end - return found + cache[str]=found + end end + return found + end end - -resolvers.splitconfigurationpath = splitconfigurationpath - +resolvers.splitconfigurationpath=splitconfigurationpath function resolvers.splitpath(str) - if type(str) == 'table' then - return str - else - return splitconfigurationpath(str) - end + if type(str)=='table' then + return str + else + return splitconfigurationpath(str) + end end - function resolvers.joinpath(str) - if type(str) == 'table' then - return file.joinpath(str) - else - return str - end -end - --- The next function scans directories and returns a hash where the --- entries are either strings or tables. - --- starting with . or .. etc or funny char - - - - --- a lot of this caching can be stripped away when we have ssd's everywhere --- --- we could cache all the (sub)paths here if needed - -local attributes, directory = lfs.attributes, lfs.dir - -local weird = P(".")^1 + lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) -local timer = { } -local scanned = { } -local nofscans = 0 -local scancache = { } - + if type(str)=='table' then + return file.joinpath(str) + else + return str + end +end +local attributes,directory=lfs.attributes,lfs.dir +local weird=P(".")^1+lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local timer={} +local scanned={} +local nofscans=0 +local scancache={} local function scan(files,spec,path,n,m,r) - local full = (path == "" and spec) or (spec .. path .. '/') - local dirs = { } - local nofdirs = 0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode = attributes(full..name,'mode') - if mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - else -- probably unique anyway - files[name] = path - local lower = lower(name) - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end - end - elseif mode == 'directory' then - m = m + 1 - nofdirs = nofdirs + 1 - if path ~= "" then - dirs[nofdirs] = path..'/'..name - else - dirs[nofdirs] = name - end - end - end - end - if nofdirs > 0 then - sort(dirs) - for i=1,nofdirs do - files, n, m, r = scan(files,spec,dirs[i],n,m,r) - end - end - scancache[sub(full,1,-2)] = files - return files, n, m, r -end - -local fullcache = { } - -function resolvers.scanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath = resolvers.resolve(path) -- no shortcut - if usecache then - local files = fullcache[realpath] - if files then - if trace_locating then - report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) - end - return files - end - end - if trace_locating then - report_expansions("scanning path '%s', branch '%s'",path,branch or path) - end - local files, n, m, r = scan({ },realpath .. '/',"",0,0,0) - files.__path__ = path -- can be selfautoparent:texmf-whatever - files.__files__ = n - files.__directories__ = m - files.__remappings__ = r - if trace_locating then - report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) - end - if usecache then - scanned[#scanned+1] = realpath - fullcache[realpath] = files - end - nofscans = nofscans + 1 - statistics.stoptiming(timer) - return files -end - -local function simplescan(files,spec,path) -- first match only, no map and such - local full = (path == "" and spec) or (spec .. path .. '/') - local dirs = { } - local nofdirs = 0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode = attributes(full..name,'mode') - if mode == 'file' then - if not files[name] then - -- only first match - files[name] = path - end - elseif mode == 'directory' then - nofdirs = nofdirs + 1 - if path ~= "" then - dirs[nofdirs] = path..'/'..name - else - dirs[nofdirs] = name - end - end - end - end - if nofdirs > 0 then - sort(dirs) - for i=1,nofdirs do - files = simplescan(files,spec,dirs[i]) - end - end - return files -end - -local simplecache = { } -local nofsharedscans = 0 - -function resolvers.simplescanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath = resolvers.resolve(path) -- no shortcut - if usecache then - local files = simplecache[realpath] - if not files then - files = scancache[realpath] - if files then - nofsharedscans = nofsharedscans + 1 - end + local full=(path=="" and spec) or (spec..path..'/') + local dirs={} + local nofdirs=0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode=attributes(full..name,'mode') + if mode=='file' then + n=n+1 + local f=files[name] + if f then + if type(f)=='string' then + files[name]={ f,path } + else + f[#f+1]=path + end + else + files[name]=path + local lower=lower(name) + if name~=lower then + files["remap:"..lower]=name + r=r+1 + end end - if files then - if trace_locating then - report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) - end - return files + elseif mode=='directory' then + m=m+1 + nofdirs=nofdirs+1 + if path~="" then + dirs[nofdirs]=path..'/'..name + else + dirs[nofdirs]=name end + end end - if trace_locating then - report_expansions("scanning path '%s', branch '%s'",path,branch or path) + end + if nofdirs>0 then + sort(dirs) + for i=1,nofdirs do + files,n,m,r=scan(files,spec,dirs[i],n,m,r) end - local files = simplescan({ },realpath .. '/',"") - if trace_locating then - report_expansions("%s files found",table.count(files)) + end + scancache[sub(full,1,-2)]=files + return files,n,m,r +end +local fullcache={} +function resolvers.scanfiles(path,branch,usecache) + statistics.starttiming(timer) + local realpath=resolvers.resolve(path) + if usecache then + local files=fullcache[realpath] + if files then + if trace_locating then + report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) + end + return files + end + end + if trace_locating then + report_expansions("scanning path '%s', branch '%s'",path,branch or path) + end + local files,n,m,r=scan({},realpath..'/',"",0,0,0) + files.__path__=path + files.__files__=n + files.__directories__=m + files.__remappings__=r + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) + end + if usecache then + scanned[#scanned+1]=realpath + fullcache[realpath]=files + end + nofscans=nofscans+1 + statistics.stoptiming(timer) + return files +end +local function simplescan(files,spec,path) + local full=(path=="" and spec) or (spec..path..'/') + local dirs={} + local nofdirs=0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode=attributes(full..name,'mode') + if mode=='file' then + if not files[name] then + files[name]=path + end + elseif mode=='directory' then + nofdirs=nofdirs+1 + if path~="" then + dirs[nofdirs]=path..'/'..name + else + dirs[nofdirs]=name + end + end end - if usecache then - scanned[#scanned+1] = realpath - simplecache[realpath] = files + end + if nofdirs>0 then + sort(dirs) + for i=1,nofdirs do + files=simplescan(files,spec,dirs[i]) end - nofscans = nofscans + 1 - statistics.stoptiming(timer) - return files + end + return files +end +local simplecache={} +local nofsharedscans=0 +function resolvers.simplescanfiles(path,branch,usecache) + statistics.starttiming(timer) + local realpath=resolvers.resolve(path) + if usecache then + local files=simplecache[realpath] + if not files then + files=scancache[realpath] + if files then + nofsharedscans=nofsharedscans+1 + end + end + if files then + if trace_locating then + report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) + end + return files + end + end + if trace_locating then + report_expansions("scanning path '%s', branch '%s'",path,branch or path) + end + local files=simplescan({},realpath..'/',"") + if trace_locating then + report_expansions("%s files found",table.count(files)) + end + if usecache then + scanned[#scanned+1]=realpath + simplecache[realpath]=files + end + nofscans=nofscans+1 + statistics.stoptiming(timer) + return files end - function resolvers.scandata() - table.sort(scanned) - return { - n = nofscans, - shared = nofsharedscans, - time = statistics.elapsedtime(timer), - paths = scanned, - } + table.sort(scanned) + return { + n=nofscans, + shared=nofsharedscans, + time=statistics.elapsedtime(timer), + paths=scanned, + } end - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-env'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} +-- original size: 8762, stripped down to: 6484 -local lower, gsub = string.lower, string.gsub - -local resolvers = resolvers - -local allocate = utilities.storage.allocate -local setmetatableindex = table.setmetatableindex -local suffixonly = file.suffixonly - -local formats = allocate() -local suffixes = allocate() -local dangerous = allocate() -local suffixmap = allocate() - -resolvers.formats = formats -resolvers.suffixes = suffixes -resolvers.dangerous = dangerous -resolvers.suffixmap = suffixmap - -local luasuffixes = utilities.lua.suffixes - -local relations = allocate { -- todo: handlers also here - core = { - ofm = { -- will become obsolete - names = { "ofm", "omega font metric", "omega font metrics" }, - variable = 'OFMFONTS', - suffixes = { 'ofm', 'tfm' }, - }, - ovf = { -- will become obsolete - names = { "ovf", "omega virtual font", "omega virtual fonts" }, - variable = 'OVFFONTS', - suffixes = { 'ovf', 'vf' }, - }, - tfm = { - names = { "tfm", "tex font metric", "tex font metrics" }, - variable = 'TFMFONTS', - suffixes = { 'tfm' }, - }, - vf = { - names = { "vf", "virtual font", "virtual fonts" }, - variable = 'VFFONTS', - suffixes = { 'vf' }, - }, - otf = { - names = { "otf", "opentype", "opentype font", "opentype fonts"}, - variable = 'OPENTYPEFONTS', - suffixes = { 'otf' }, - }, - ttf = { - names = { "ttf", "truetype", "truetype font", "truetype fonts", "truetype collection", "truetype collections", "truetype dictionary", "truetype dictionaries" }, - variable = 'TTFONTS', - suffixes = { 'ttf', 'ttc', 'dfont' }, - }, - afm = { - names = { "afm", "adobe font metric", "adobe font metrics" }, - variable = "AFMFONTS", - suffixes = { "afm" }, - }, - pfb = { - names = { "pfb", "type1", "type 1", "type1 font", "type 1 font", "type1 fonts", "type 1 fonts" }, - variable = 'T1FONTS', - suffixes = { 'pfb', 'pfa' }, - }, - fea = { - names = { "fea", "font feature", "font features", "font feature file", "font feature files" }, - variable = 'FONTFEATURES', - suffixes = { 'fea' }, - }, - cid = { - names = { "cid", "cid map", "cid maps", "cid file", "cid files" }, - variable = 'FONTCIDMAPS', - suffixes = { 'cid', 'cidmap' }, - }, - fmt = { - names = { "fmt", "format", "tex format" }, - variable = 'TEXFORMATS', - suffixes = { 'fmt' }, - }, - mem = { -- will become obsolete - names = { 'mem', "metapost format" }, - variable = 'MPMEMS', - suffixes = { 'mem' }, - }, - mp = { - names = { "mp" }, - variable = 'MPINPUTS', - suffixes = { 'mp', 'mpvi', 'mpiv', 'mpii' }, - }, - tex = { - names = { "tex" }, - variable = 'TEXINPUTS', - suffixes = { 'tex', "mkvi", "mkiv", "mkii" }, - }, - icc = { - names = { "icc", "icc profile", "icc profiles" }, - variable = 'ICCPROFILES', - suffixes = { 'icc' }, - }, - texmfscripts = { - names = { "texmfscript", "texmfscripts", "script", "scripts" }, - variable = 'TEXMFSCRIPTS', - suffixes = { 'rb', 'pl', 'py' }, - }, - lua = { - names = { "lua" }, - variable = 'LUAINPUTS', - suffixes = { luasuffixes.lua, luasuffixes.luc, luasuffixes.tma, luasuffixes.tmc }, - }, - lib = { - names = { "lib" }, - variable = 'CLUAINPUTS', - suffixes = os.libsuffix and { os.libsuffix } or { 'dll', 'so' }, - }, - bib = { - names = { 'bib' }, - suffixes = { 'bib' }, - }, - bst = { - names = { 'bst' }, - suffixes = { 'bst' }, - }, - fontconfig = { - names = { 'fontconfig', 'fontconfig file', 'fontconfig files' }, - variable = 'FONTCONFIG_PATH', - }, +if not modules then modules={} end modules ['data-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local lower,gsub=string.lower,string.gsub +local resolvers=resolvers +local allocate=utilities.storage.allocate +local setmetatableindex=table.setmetatableindex +local suffixonly=file.suffixonly +local formats=allocate() +local suffixes=allocate() +local dangerous=allocate() +local suffixmap=allocate() +resolvers.formats=formats +resolvers.suffixes=suffixes +resolvers.dangerous=dangerous +resolvers.suffixmap=suffixmap +local luasuffixes=utilities.lua.suffixes +local relations=allocate { + core={ + ofm={ + names={ "ofm","omega font metric","omega font metrics" }, + variable='OFMFONTS', + suffixes={ 'ofm','tfm' }, + }, + ovf={ + names={ "ovf","omega virtual font","omega virtual fonts" }, + variable='OVFFONTS', + suffixes={ 'ovf','vf' }, + }, + tfm={ + names={ "tfm","tex font metric","tex font metrics" }, + variable='TFMFONTS', + suffixes={ 'tfm' }, + }, + vf={ + names={ "vf","virtual font","virtual fonts" }, + variable='VFFONTS', + suffixes={ 'vf' }, + }, + otf={ + names={ "otf","opentype","opentype font","opentype fonts"}, + variable='OPENTYPEFONTS', + suffixes={ 'otf' }, + }, + ttf={ + names={ "ttf","truetype","truetype font","truetype fonts","truetype collection","truetype collections","truetype dictionary","truetype dictionaries" }, + variable='TTFONTS', + suffixes={ 'ttf','ttc','dfont' }, + }, + afm={ + names={ "afm","adobe font metric","adobe font metrics" }, + variable="AFMFONTS", + suffixes={ "afm" }, + }, + pfb={ + names={ "pfb","type1","type 1","type1 font","type 1 font","type1 fonts","type 1 fonts" }, + variable='T1FONTS', + suffixes={ 'pfb','pfa' }, + }, + fea={ + names={ "fea","font feature","font features","font feature file","font feature files" }, + variable='FONTFEATURES', + suffixes={ 'fea' }, }, - obsolete = { - enc = { - names = { "enc", "enc files", "enc file", "encoding files", "encoding file" }, - variable = 'ENCFONTS', - suffixes = { 'enc' }, - }, - map = { - names = { "map", "map files", "map file" }, - variable = 'TEXFONTMAPS', - suffixes = { 'map' }, - }, - lig = { - names = { "lig files", "lig file", "ligature file", "ligature files" }, - variable = 'LIGFONTS', - suffixes = { 'lig' }, - }, - opl = { - names = { "opl" }, - variable = 'OPLFONTS', - suffixes = { 'opl' }, - }, - ovp = { - names = { "ovp" }, - variable = 'OVPFONTS', - suffixes = { 'ovp' }, - }, + cid={ + names={ "cid","cid map","cid maps","cid file","cid files" }, + variable='FONTCIDMAPS', + suffixes={ 'cid','cidmap' }, }, - kpse = { -- subset - base = { - names = { 'base', "metafont format" }, - variable = 'MFBASES', - suffixes = { 'base', 'bas' }, - }, - cmap = { - names = { 'cmap', 'cmap files', 'cmap file' }, - variable = 'CMAPFONTS', - suffixes = { 'cmap' }, - }, - cnf = { - names = { 'cnf' }, - suffixes = { 'cnf' }, - }, - web = { - names = { 'web' }, - suffixes = { 'web', 'ch' } - }, - cweb = { - names = { 'cweb' }, - suffixes = { 'w', 'web', 'ch' }, - }, - gf = { - names = { 'gf' }, - suffixes = { 'gf' }, - }, - mf = { - names = { 'mf' }, - variable = 'MFINPUTS', - suffixes = { 'mf' }, - }, - mft = { - names = { 'mft' }, - suffixes = { 'mft' }, - }, - pk = { - names = { 'pk' }, - suffixes = { 'pk' }, - }, + fmt={ + names={ "fmt","format","tex format" }, + variable='TEXFORMATS', + suffixes={ 'fmt' }, }, + mem={ + names={ 'mem',"metapost format" }, + variable='MPMEMS', + suffixes={ 'mem' }, + }, + mp={ + names={ "mp" }, + variable='MPINPUTS', + suffixes={ 'mp','mpvi','mpiv','mpii' }, + }, + tex={ + names={ "tex" }, + variable='TEXINPUTS', + suffixes={ 'tex',"mkvi","mkiv","mkii" }, + }, + icc={ + names={ "icc","icc profile","icc profiles" }, + variable='ICCPROFILES', + suffixes={ 'icc' }, + }, + texmfscripts={ + names={ "texmfscript","texmfscripts","script","scripts" }, + variable='TEXMFSCRIPTS', + suffixes={ 'rb','pl','py' }, + }, + lua={ + names={ "lua" }, + variable='LUAINPUTS', + suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + }, + lib={ + names={ "lib" }, + variable='CLUAINPUTS', + suffixes=os.libsuffix and { os.libsuffix } or { 'dll','so' }, + }, + bib={ + names={ 'bib' }, + suffixes={ 'bib' }, + }, + bst={ + names={ 'bst' }, + suffixes={ 'bst' }, + }, + fontconfig={ + names={ 'fontconfig','fontconfig file','fontconfig files' }, + variable='FONTCONFIG_PATH', + }, + }, + obsolete={ + enc={ + names={ "enc","enc files","enc file","encoding files","encoding file" }, + variable='ENCFONTS', + suffixes={ 'enc' }, + }, + map={ + names={ "map","map files","map file" }, + variable='TEXFONTMAPS', + suffixes={ 'map' }, + }, + lig={ + names={ "lig files","lig file","ligature file","ligature files" }, + variable='LIGFONTS', + suffixes={ 'lig' }, + }, + opl={ + names={ "opl" }, + variable='OPLFONTS', + suffixes={ 'opl' }, + }, + ovp={ + names={ "ovp" }, + variable='OVPFONTS', + suffixes={ 'ovp' }, + }, + }, + kpse={ + base={ + names={ 'base',"metafont format" }, + variable='MFBASES', + suffixes={ 'base','bas' }, + }, + cmap={ + names={ 'cmap','cmap files','cmap file' }, + variable='CMAPFONTS', + suffixes={ 'cmap' }, + }, + cnf={ + names={ 'cnf' }, + suffixes={ 'cnf' }, + }, + web={ + names={ 'web' }, + suffixes={ 'web','ch' } + }, + cweb={ + names={ 'cweb' }, + suffixes={ 'w','web','ch' }, + }, + gf={ + names={ 'gf' }, + suffixes={ 'gf' }, + }, + mf={ + names={ 'mf' }, + variable='MFINPUTS', + suffixes={ 'mf' }, + }, + mft={ + names={ 'mft' }, + suffixes={ 'mft' }, + }, + pk={ + names={ 'pk' }, + suffixes={ 'pk' }, + }, + }, } - -resolvers.relations = relations - --- formats: maps a format onto a variable - +resolvers.relations=relations function resolvers.updaterelations() - for category, categories in next, relations do - for name, relation in next, categories do - local rn = relation.names - local rv = relation.variable - local rs = relation.suffixes - if rn and rv then - for i=1,#rn do - local rni = lower(gsub(rn[i]," ","")) - formats[rni] = rv - if rs then - suffixes[rni] = rs - for i=1,#rs do - local rsi = rs[i] - suffixmap[rsi] = rni - end - end - end - end - if rs then + for category,categories in next,relations do + for name,relation in next,categories do + local rn=relation.names + local rv=relation.variable + local rs=relation.suffixes + if rn and rv then + for i=1,#rn do + local rni=lower(gsub(rn[i]," ","")) + formats[rni]=rv + if rs then + suffixes[rni]=rs + for i=1,#rs do + local rsi=rs[i] + suffixmap[rsi]=rni end + end end + end + if rs then + end end + end end - -resolvers.updaterelations() -- push this in the metatable -> newindex - +resolvers.updaterelations() local function simplified(t,k) - return k and rawget(t,lower(gsub(k," ",""))) or nil + return k and rawget(t,lower(gsub(k," ",""))) or nil end - -setmetatableindex(formats, simplified) -setmetatableindex(suffixes, simplified) -setmetatableindex(suffixmap, simplified) - --- A few accessors, mostly for command line tool. - +setmetatableindex(formats,simplified) +setmetatableindex(suffixes,simplified) +setmetatableindex(suffixmap,simplified) function resolvers.suffixofformat(str) - local s = suffixes[str] - return s and s[1] or "" + local s=suffixes[str] + return s and s[1] or "" end - function resolvers.suffixofformat(str) - return suffixes[str] or { } + return suffixes[str] or {} end - -for name, format in next, formats do - dangerous[name] = true -- still needed ? +for name,format in next,formats do + dangerous[name]=true end - --- because vf searching is somewhat dangerous, we want to prevent --- too liberal searching esp because we do a lookup on the current --- path anyway; only tex (or any) is safe - -dangerous.tex = nil - - --- more helpers - +dangerous.tex=nil function resolvers.formatofvariable(str) - return formats[str] or '' + return formats[str] or '' end - -function resolvers.formatofsuffix(str) -- of file - return suffixmap[suffixonly(str)] or 'tex' -- so many map onto tex (like mkiv, cld etc) +function resolvers.formatofsuffix(str) + return suffixmap[suffixonly(str)] or 'tex' end - function resolvers.variableofformat(str) - return formats[str] or '' + return formats[str] or '' end - function resolvers.variableofformatorsuffix(str) - local v = formats[str] - if v then - return v - end - v = suffixmap[suffixonly(str)] - if v then - return formats[v] - end - return '' + local v=formats[str] + if v then + return v + end + v=suffixmap[suffixonly(str)] + if v then + return formats[v] + end + return '' end - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tmp'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - - - -local format, lower, gsub, concat = string.format, string.lower, string.gsub, table.concat -local serialize, serializetofile = table.serialize, table.tofile -local mkdirs, isdir = dir.mkdirs, lfs.isdir -local addsuffix, is_writable, is_readable = file.addsuffix, file.is_writable, file.is_readable - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) - -local report_caches = logs.reporter("resolvers","caches") -local report_resolvers = logs.reporter("resolvers","caching") - -local resolvers = resolvers - --- intermezzo - -local directive_cleanup = false directives.register("system.compile.cleanup", function(v) directive_cleanup = v end) -local directive_strip = false directives.register("system.compile.strip", function(v) directive_strip = v end) - -local compile = utilities.lua.compile +-- original size: 14075, stripped down to: 10764 +if not modules then modules={} end modules ['data-tmp']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat +local serialize,serializetofile=table.serialize,table.tofile +local mkdirs,isdir=dir.mkdirs,lfs.isdir +local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local report_caches=logs.reporter("resolvers","caches") +local report_resolvers=logs.reporter("resolvers","caching") +local resolvers=resolvers +local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) +local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) +local compile=utilities.lua.compile function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup == nil then cleanup = directive_cleanup end - if strip == nil then strip = directive_strip end - return compile(luafile,lucfile,cleanup,strip) -end - --- end of intermezzo - -caches = caches or { } -local caches = caches - -local luasuffixes = utilities.lua.suffixes - -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.tree = false -caches.force = true -caches.ask = false -caches.relocate = false -caches.defaults = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } - -local writable, readables, usedreadables = nil, { }, { } - --- we could use a metatable for writable and readable but not yet - + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compile(luafile,lucfile,cleanup,strip) +end +caches=caches or {} +local caches=caches +local luasuffixes=utilities.lua.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" +caches.direct=false +caches.tree=false +caches.force=true +caches.ask=false +caches.relocate=false +caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } +local writable,readables,usedreadables=nil,{},{} local function identify() - -- Combining the loops makes it messy. First we check the format cache path - -- and when the last component is not present we try to create it. - local texmfcaches = resolvers.cleanpathlist("TEXMFCACHE") - if texmfcaches then - for k=1,#texmfcaches do - local cachepath = texmfcaches[k] - if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) - cachepath = file.collapsepath(cachepath) - local valid = isdir(cachepath) - if valid then - if is_readable(cachepath) then - readables[#readables+1] = cachepath - if not writable and is_writable(cachepath) then - writable = cachepath - end - end - elseif not writable and caches.force then - local cacheparent = file.dirname(cachepath) - if is_writable(cacheparent) and true then -- we go on anyway (needed for mojca's kind of paths) - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("created: %s",cachepath) - writable = cachepath - readables[#readables+1] = cachepath - end - end - end - end + local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + cachepath=file.collapsepath(cachepath) + local valid=isdir(cachepath) + if valid then + if is_readable(cachepath) then + readables[#readables+1]=cachepath + if not writable and is_writable(cachepath) then + writable=cachepath end - end - end - -- As a last resort we check some temporary paths but this time we don't - -- create them. - local texmfcaches = caches.defaults - if texmfcaches then - for k=1,#texmfcaches do - local cachepath = texmfcaches[k] - cachepath = resolvers.expansion(cachepath) -- was getenv - if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) - local valid = isdir(cachepath) - if valid and is_readable(cachepath) then - if not writable and is_writable(cachepath) then - readables[#readables+1] = cachepath - writable = cachepath - break - end - end + end + elseif not writable and caches.force then + local cacheparent=file.dirname(cachepath) + if is_writable(cacheparent) and true then + if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("created: %s",cachepath) + writable=cachepath + readables[#readables+1]=cachepath + end end + end end + end end - -- Some extra checking. If we have no writable or readable path then we simply - -- quit. - if not writable then - report_caches("fatal error: there is no valid writable cache path defined") - os.exit() - elseif #readables == 0 then - report_caches("fatal error: there is no valid readable cache path defined") - os.exit() - end - -- why here - writable = dir.expandname(resolvers.cleanpath(writable)) -- just in case - -- moved here - local base, more, tree = caches.base, caches.more, caches.tree or caches.treehash() -- we have only one writable tree - if tree then - caches.tree = tree - writable = mkdirs(writable,base,more,tree) - for i=1,#readables do - readables[i] = file.join(readables[i],base,more,tree) - end - else - writable = mkdirs(writable,base,more) - for i=1,#readables do - readables[i] = file.join(readables[i],base,more) + end + local texmfcaches=caches.defaults + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + cachepath=resolvers.expansion(cachepath) + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + local valid=isdir(cachepath) + if valid and is_readable(cachepath) then + if not writable and is_writable(cachepath) then + readables[#readables+1]=cachepath + writable=cachepath + break + end end + end end - -- end - if trace_cache then - for i=1,#readables do - report_caches("using readable path '%s' (order %s)",readables[i],i) - end - report_caches("using writable path '%s'",writable) + end + if not writable then + report_caches("fatal error: there is no valid writable cache path defined") + os.exit() + elseif #readables==0 then + report_caches("fatal error: there is no valid readable cache path defined") + os.exit() + end + writable=dir.expandname(resolvers.cleanpath(writable)) + local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + if tree then + caches.tree=tree + writable=mkdirs(writable,base,more,tree) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more,tree) + end + else + writable=mkdirs(writable,base,more) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more) end - identify = function() - return writable, readables + end + if trace_cache then + for i=1,#readables do + report_caches("using readable path '%s' (order %s)",readables[i],i) end - return writable, readables + report_caches("using writable path '%s'",writable) + end + identify=function() + return writable,readables + end + return writable,readables end - function caches.usedpaths() - local writable, readables = identify() - if #readables > 1 then - local result = { } - for i=1,#readables do - local readable = readables[i] - if usedreadables[i] or readable == writable then - result[#result+1] = format("readable: '%s' (order %s)",readable,i) - end - end - result[#result+1] = format("writable: '%s'",writable) - return result - else - return writable + local writable,readables=identify() + if #readables>1 then + local result={} + for i=1,#readables do + local readable=readables[i] + if usedreadables[i] or readable==writable then + result[#result+1]=format("readable: '%s' (order %s)",readable,i) + end end + result[#result+1]=format("writable: '%s'",writable) + return result + else + return writable + end end - function caches.configfiles() - return concat(resolvers.instance.specification,";") + return concat(resolvers.instance.specification,";") end - function caches.hashed(tree) - tree = gsub(tree,"[\\/]+$","") - tree = lower(tree) - local hash = md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %s, hash %s",tree,hash) - end - return hash + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5.hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %s, hash %s",tree,hash) + end + return hash end - function caches.treehash() - local tree = caches.configfiles() - if not tree or tree == "" then - return false - else - return caches.hashed(tree) - end + local tree=caches.configfiles() + if not tree or tree=="" then + return false + else + return caches.hashed(tree) + end end - -local r_cache, w_cache = { }, { } -- normally w in in r but who cares - +local r_cache,w_cache={},{} local function getreadablepaths(...) - local tags = { ... } - local hash = concat(tags,"/") - local done = r_cache[hash] - if not done then - local writable, readables = identify() -- exit if not found - if #tags > 0 then - done = { } - for i=1,#readables do - done[i] = file.join(readables[i],...) - end - else - done = readables - end - r_cache[hash] = done + local tags={... } + local hash=concat(tags,"/") + local done=r_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done={} + for i=1,#readables do + done[i]=file.join(readables[i],...) + end + else + done=readables end - return done + r_cache[hash]=done + end + return done end - local function getwritablepath(...) - local tags = { ... } - local hash = concat(tags,"/") - local done = w_cache[hash] - if not done then - local writable, readables = identify() -- exit if not found - if #tags > 0 then - done = mkdirs(writable,...) - else - done = writable - end - w_cache[hash] = done - end - return done -end - -caches.getreadablepaths = getreadablepaths -caches.getwritablepath = getwritablepath - + local tags={... } + local hash=concat(tags,"/") + local done=w_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done=mkdirs(writable,...) + else + done=writable + end + w_cache[hash]=done + end + return done +end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath function caches.getfirstreadablefile(filename,...) - local rd = getreadablepaths(...) - for i=1,#rd do - local path = rd[i] - local fullname = file.join(path,filename) - if is_readable(fullname) then - usedreadables[i] = true - return fullname, path - end + local rd=getreadablepaths(...) + for i=1,#rd do + local path=rd[i] + local fullname=file.join(path,filename) + if is_readable(fullname) then + usedreadables[i]=true + return fullname,path end - return caches.setfirstwritablefile(filename,...) + end + return caches.setfirstwritablefile(filename,...) end - function caches.setfirstwritablefile(filename,...) - local wr = getwritablepath(...) - local fullname = file.join(wr,filename) - return fullname, wr + local wr=getwritablepath(...) + local fullname=file.join(wr,filename) + return fullname,wr end - -function caches.define(category,subcategory) -- for old times sake - return function() - return getwritablepath(category,subcategory) - end +function caches.define(category,subcategory) + return function() + return getwritablepath(category,subcategory) + end end - function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma), format("%s/%s.%s",path,name,luasuffixes.tmc) + return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) end - function caches.loaddata(readables,name) - if type(readables) == "string" then - readables = { readables } - end - for i=1,#readables do - local path = readables[i] - local tmaname, tmcname = caches.setluanames(path,name) - local loader = loadfile(tmcname) - if not loader then - -- in case we have a different engine - utilities.lua.compile(tmaname,tmcname) - -- - loader = loadfile(tmaname) - end - if loader then - loader = loader() - collectgarbage("step") - return loader - end - end - return false + if type(readables)=="string" then + readables={ readables } + end + for i=1,#readables do + local path=readables[i] + local tmaname,tmcname=caches.setluanames(path,name) + local loader=loadfile(tmcname) + if not loader then + utilities.lua.compile(tmaname,tmcname) + loader=loadfile(tmaname) + end + if loader then + loader=loader() + collectgarbage("step") + return loader + end + end + return false end - function caches.is_writable(filepath,filename) - local tmaname, tmcname = caches.setluanames(filepath,filename) - return is_writable(tmaname) + local tmaname,tmcname=caches.setluanames(filepath,filename) + return is_writable(tmaname) end - -local saveoptions = { compact = true } - --- add some point we will only use the internal bytecode compiler and --- then we can flag success in the tma so that it can trigger a compile --- if the other engine - +local saveoptions={ compact=true } function caches.savedata(filepath,filename,data,raw) - local tmaname, tmcname = caches.setluanames(filepath,filename) - local reduce, simplify = true, true - if raw then - reduce, simplify = false, false - end - data.cache_uuid = os.uuid() - if caches.direct then - file.savedata(tmaname,serialize(data,true,saveoptions)) - else - serializetofile(tmaname,data,true,saveoptions) - end - utilities.lua.compile(tmaname,tmcname) -end - --- moved from data-res: - -local content_state = { } - + local tmaname,tmcname=caches.setluanames(filepath,filename) + local reduce,simplify=true,true + if raw then + reduce,simplify=false,false + end + data.cache_uuid=os.uuid() + if caches.direct then + file.savedata(tmaname,serialize(data,true,saveoptions)) + else + serializetofile(tmaname,data,true,saveoptions) + end + utilities.lua.compile(tmaname,tmcname) +end +local content_state={} function caches.contentstate() - return content_state or { } + return content_state or {} end - function caches.loadcontent(cachename,dataname) - local name = caches.hashed(cachename) - local full, path = caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename = file.join(path,name) - local blob = loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) - if blob then - local data = blob() - if data and data.content then - if data.type == dataname then - if data.version == resolvers.cacheversion then - content_state[#content_state+1] = data.uuid - if trace_locating then - report_resolvers("loading '%s' for '%s' from '%s'",dataname,cachename,filename) - end - return data.content - else - report_resolvers("skipping '%s' for '%s' from '%s' (version mismatch)",dataname,cachename,filename) - end - else - report_resolvers("skipping '%s' for '%s' from '%s' (datatype mismatch)",dataname,cachename,filename) - end - elseif trace_locating then - report_resolvers("skipping '%s' for '%s' from '%s' (no content)",dataname,cachename,filename) - end - elseif trace_locating then - report_resolvers("skipping '%s' for '%s' from '%s' (invalid file)",dataname,cachename,filename) - end -end - -function caches.collapsecontent(content) - for k, v in next, content do - if type(v) == "table" and #v == 1 then - content[k] = v[1] - end - end -end - -function caches.savecontent(cachename,dataname,content) - local name = caches.hashed(cachename) - local full, path = caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename = file.join(path,name) -- is full - local luaname = addsuffix(filename,luasuffixes.lua) - local lucname = addsuffix(filename,luasuffixes.luc) - if trace_locating then - report_resolvers("preparing '%s' for '%s'",dataname,cachename) - end - local data = { - type = dataname, - root = cachename, - version = resolvers.cacheversion, - date = os.date("%Y-%m-%d"), - time = os.date("%H:%M:%S"), - content = content, - uuid = os.uuid(), - } - local ok = io.savedata(luaname,serialize(data,true)) - if ok then - if trace_locating then - report_resolvers("category '%s', cachename '%s' saved in '%s'",dataname,cachename,luaname) - end - if utilities.lua.compile(luaname,lucname) then - if trace_locating then - report_resolvers("'%s' compiled to '%s'",dataname,lucname) - end - return true + local name=caches.hashed(cachename) + local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + local filename=file.join(path,name) + local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) + if blob then + local data=blob() + if data and data.content then + if data.type==dataname then + if data.version==resolvers.cacheversion then + content_state[#content_state+1]=data.uuid + if trace_locating then + report_resolvers("loading '%s' for '%s' from '%s'",dataname,cachename,filename) + end + return data.content else - if trace_locating then - report_resolvers("compiling failed for '%s', deleting file '%s'",dataname,lucname) - end - os.remove(lucname) + report_resolvers("skipping '%s' for '%s' from '%s' (version mismatch)",dataname,cachename,filename) end + else + report_resolvers("skipping '%s' for '%s' from '%s' (datatype mismatch)",dataname,cachename,filename) + end elseif trace_locating then - report_resolvers("unable to save '%s' in '%s' (access error)",dataname,luaname) + report_resolvers("skipping '%s' for '%s' from '%s' (no content)",dataname,cachename,filename) end + elseif trace_locating then + report_resolvers("skipping '%s' for '%s' from '%s' (invalid file)",dataname,cachename,filename) + end end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-met'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local find, format = string.find, string.format -local sequenced = table.sequenced -local addurlscheme, urlhashed = url.addscheme, url.hashed - -local trace_locating = false - -trackers.register("resolvers.locating", function(v) trace_methods = v end) -trackers.register("resolvers.methods", function(v) trace_methods = v end) - - -local report_methods = logs.reporter("resolvers","methods") - -local allocate = utilities.storage.allocate - -local resolvers = resolvers - -local registered = { } - -local function splitmethod(filename) -- todo: filetype in specification - if not filename then - return { scheme = "unknown", original = filename } - end - if type(filename) == "table" then - return filename -- already split - end - filename = file.collapsepath(filename) - if not find(filename,"://") then - return { scheme = "file", path = filename, original = filename, filename = filename } - end - local specification = url.hashed(filename) - if not specification.scheme or specification.scheme == "" then - return { scheme = "file", path = filename, original = filename, filename = filename } - else - return specification +function caches.collapsecontent(content) + for k,v in next,content do + if type(v)=="table" and #v==1 then + content[k]=v[1] end + end end - -resolvers.splitmethod = splitmethod -- bad name but ok - --- the second argument is always analyzed (saves time later on) and the original --- gets passed as original but also as argument - -local function methodhandler(what,first,...) -- filename can be nil or false - local method = registered[what] - if method then - local how, namespace = method.how, method.namespace - if how == "uri" or how == "url" then - local specification = splitmethod(first) - local scheme = specification.scheme - local resolver = namespace and namespace[scheme] - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, scheme=%s, argument=%s",what,how,scheme,first) - end - return resolver(specification,...) - else - resolver = namespace.default or namespace.file - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, default, argument=%s",what,how,first) - end - return resolver(specification,...) - elseif trace_methods then - report_methods("resolver: method=%s, how=%s, no handler",what,how) - end - end - elseif how == "tag" then - local resolver = namespace and namespace[first] - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, tag=%s",what,how,first) - end - return resolver(...) - else - resolver = namespace.default or namespace.file - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, default",what,how) - end - return resolver(...) - elseif trace_methods then - report_methods("resolver: method=%s, how=%s, unknown",what,how) - end - end - end +function caches.savecontent(cachename,dataname,content) + local name=caches.hashed(cachename) + local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + local filename=file.join(path,name) + local luaname=addsuffix(filename,luasuffixes.lua) + local lucname=addsuffix(filename,luasuffixes.luc) + if trace_locating then + report_resolvers("preparing '%s' for '%s'",dataname,cachename) + end + local data={ + type=dataname, + root=cachename, + version=resolvers.cacheversion, + date=os.date("%Y-%m-%d"), + time=os.date("%H:%M:%S"), + content=content, + uuid=os.uuid(), + } + local ok=io.savedata(luaname,serialize(data,true)) + if ok then + if trace_locating then + report_resolvers("category '%s', cachename '%s' saved in '%s'",dataname,cachename,luaname) + end + if utilities.lua.compile(luaname,lucname) then + if trace_locating then + report_resolvers("'%s' compiled to '%s'",dataname,lucname) + end + return true else - report_methods("resolver: method=%s, unknown",what) + if trace_locating then + report_resolvers("compiling failed for '%s', deleting file '%s'",dataname,lucname) + end + os.remove(lucname) end + elseif trace_locating then + report_resolvers("unable to save '%s' in '%s' (access error)",dataname,luaname) + end end -resolvers.methodhandler = methodhandler - -function resolvers.registermethod(name,namespace,how) - registered[name] = { how = how or "tag", namespace = namespace } - namespace["byscheme"] = function(scheme,filename,...) - if scheme == "file" then - return methodhandler(name,filename,...) - else - return methodhandler(name,addurlscheme(filename,scheme),...) - end - end -end -local concatinators = allocate { notfound = file.join } -- concatinate paths -local locators = allocate { notfound = function() end } -- locate databases -local hashers = allocate { notfound = function() end } -- load databases -local generators = allocate { notfound = function() end } -- generate databases +end -- of closure -resolvers.concatinators = concatinators -resolvers.locators = locators -resolvers.hashers = hashers -resolvers.generators = generators +do -- create closure to overcome 200 locals limit -local registermethod = resolvers.registermethod +-- original size: 4863, stripped down to: 3890 +if not modules then modules={} end modules ['data-met']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,format=string.find,string.format +local sequenced=table.sequenced +local addurlscheme,urlhashed=url.addscheme,url.hashed +local trace_locating=false +trackers.register("resolvers.locating",function(v) trace_methods=v end) +trackers.register("resolvers.methods",function(v) trace_methods=v end) +local report_methods=logs.reporter("resolvers","methods") +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registered={} +local function splitmethod(filename) + if not filename then + return { scheme="unknown",original=filename } + end + if type(filename)=="table" then + return filename + end + filename=file.collapsepath(filename) + if not find(filename,"://") then + return { scheme="file",path=filename,original=filename,filename=filename } + end + local specification=url.hashed(filename) + if not specification.scheme or specification.scheme=="" then + return { scheme="file",path=filename,original=filename,filename=filename } + else + return specification + end +end +resolvers.splitmethod=splitmethod +local function methodhandler(what,first,...) + local method=registered[what] + if method then + local how,namespace=method.how,method.namespace + if how=="uri" or how=="url" then + local specification=splitmethod(first) + local scheme=specification.scheme + local resolver=namespace and namespace[scheme] + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, scheme=%s, argument=%s",what,how,scheme,first) + end + return resolver(specification,...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, default, argument=%s",what,how,first) + end + return resolver(specification,...) + elseif trace_methods then + report_methods("resolver: method=%s, how=%s, no handler",what,how) + end + end + elseif how=="tag" then + local resolver=namespace and namespace[first] + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, tag=%s",what,how,first) + end + return resolver(...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, default",what,how) + end + return resolver(...) + elseif trace_methods then + report_methods("resolver: method=%s, how=%s, unknown",what,how) + end + end + end + else + report_methods("resolver: method=%s, unknown",what) + end +end +resolvers.methodhandler=methodhandler +function resolvers.registermethod(name,namespace,how) + registered[name]={ how=how or "tag",namespace=namespace } + namespace["byscheme"]=function(scheme,filename,...) + if scheme=="file" then + return methodhandler(name,filename,...) + else + return methodhandler(name,addurlscheme(filename,scheme),...) + end + end +end +local concatinators=allocate { notfound=file.join } +local locators=allocate { notfound=function() end } +local hashers=allocate { notfound=function() end } +local generators=allocate { notfound=function() end } +resolvers.concatinators=concatinators +resolvers.locators=locators +resolvers.hashers=hashers +resolvers.generators=generators +local registermethod=resolvers.registermethod registermethod("concatinators",concatinators,"tag") -registermethod("locators", locators, "uri") -registermethod("hashers", hashers, "uri") -registermethod("generators", generators, "uri") +registermethod("locators",locators,"uri") +registermethod("hashers",hashers,"uri") +registermethod("generators",generators,"uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-res'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - --- In practice we will work within one tds tree, but i want to keep --- the option open to build tools that look at multiple trees, which is --- why we keep the tree specific data in a table. We used to pass the --- instance but for practical purposes we now avoid this and use a --- instance variable. We always have one instance active (sort of global). - --- todo: cache:/// home:/// selfautoparent:/// (sometime end 2012) - -local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch -local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys -local next, type, rawget = next, type, rawget -local os = os - -local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns - -local filedirname = file.dirname -local filebasename = file.basename -local suffixonly = file.suffixonly -local filejoin = file.join -local collapsepath = file.collapsepath -local joinpath = file.joinpath -local allocate = utilities.storage.allocate -local settings_to_array = utilities.parsers.settings_to_array -local setmetatableindex = table.setmetatableindex -local luasuffixes = utilities.lua.suffixes - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_resolving = logs.reporter("resolvers","resolving") - -local resolvers = resolvers - -local expandedpathfromlist = resolvers.expandedpathfromlist -local checkedvariable = resolvers.checkedvariable -local splitconfigurationpath = resolvers.splitconfigurationpath -local methodhandler = resolvers.methodhandler - -local initializesetter = utilities.setters.initialize - -local ostype, osname, osenv, ossetenv, osgetenv = os.type, os.name, os.env, os.setenv, os.getenv - -resolvers.cacheversion = '1.0.1' -resolvers.configbanner = '' -resolvers.homedir = environment.homedir -resolvers.criticalvars = allocate { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF", "TEXMF", "TEXOS" } -resolvers.luacnfname = "texmfcnf.lua" -resolvers.luacnfstate = "unknown" - --- The web2c tex binaries as well as kpse have built in paths for the configuration --- files and there can be a depressing truckload of them. This is actually the weak --- spot of a distribution. So we don't want: --- --- resolvers.luacnfspec = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}' --- --- but instead use: --- --- resolvers.luacnfspec = 'selfautoparent:{/texmf{-local,}{,/web2c}}' --- --- which does not make texlive happy as there is a texmf-local tree one level up --- (sigh), so we need this. We can assume web2c as mkiv does not run on older --- texlives anyway. --- --- texlive: --- --- selfautodir: --- selfautoparent: --- selfautodir:share/texmf-local/web2c --- selfautodir:share/texmf/web2c --- selfautodir:texmf-local/web2c --- selfautodir:texmf/web2c --- selfautoparent:share/texmf-local/web2c --- selfautoparent:share/texmf/web2c --- selfautoparent:texmf-local/web2c --- selfautoparent:texmf/web2c --- --- minimals: --- --- home:texmf/web2c --- selfautoparent:texmf-local/web2c --- selfautoparent:texmf-context/web2c --- selfautoparent:texmf/web2c +-- original size: 60360, stripped down to: 42573 +if not modules then modules={} end modules ['data-res']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local format,gsub,find,lower,upper,match,gmatch=string.format,string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch +local concat,insert,sortedkeys=table.concat,table.insert,table.sortedkeys +local next,type,rawget=next,type,rawget +local os=os +local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local filedirname=file.dirname +local filebasename=file.basename +local suffixonly=file.suffixonly +local filejoin=file.join +local collapsepath=file.collapsepath +local joinpath=file.joinpath +local allocate=utilities.storage.allocate +local settings_to_array=utilities.parsers.settings_to_array +local setmetatableindex=table.setmetatableindex +local luasuffixes=utilities.lua.suffixes +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_resolving=logs.reporter("resolvers","resolving") +local resolvers=resolvers +local expandedpathfromlist=resolvers.expandedpathfromlist +local checkedvariable=resolvers.checkedvariable +local splitconfigurationpath=resolvers.splitconfigurationpath +local methodhandler=resolvers.methodhandler +local initializesetter=utilities.setters.initialize +local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os.getenv +resolvers.cacheversion='1.0.1' +resolvers.configbanner='' +resolvers.homedir=environment.homedir +resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } +resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnfstate="unknown" if environment.default_texmfcnf then - -- unfortunately we now have quite some overkill in the spec (not so nice on a network) - resolvers.luacnfspec = environment.default_texmfcnf + resolvers.luacnfspec=environment.default_texmfcnf else - -- resolvers.luacnfspec = "selfautoparent:texmf{-local,-context,}/web2c" - resolvers.luacnfspec = "{selfautoloc:,selfautodir:,selfautoparent:}{,/texmf{-local,}/web2c}" -end - -resolvers.luacnfspec = 'home:texmf/web2c;' .. resolvers.luacnfspec - --- which (as we want users to use the web2c path) be can be simplified to this: --- --- if environment and environment.ownpath and string.find(environment.ownpath,"[\\/]texlive[\\/]") then --- resolvers.luacnfspec = 'selfautodir:/texmf-local/web2c,selfautoparent:/texmf-local/web2c,selfautoparent:/texmf/web2c' --- else --- resolvers.luacnfspec = 'selfautoparent:/texmf-local/web2c,selfautoparent:/texmf/web2c' --- end - - - -local unset_variable = "unset" - -local formats = resolvers.formats -local suffixes = resolvers.suffixes -local dangerous = resolvers.dangerous -local suffixmap = resolvers.suffixmap - -resolvers.defaultsuffixes = { "tex" } -- "mkiv", "cld" -- too tricky - -resolvers.instance = resolvers.instance or nil -- the current one (slow access) -local instance = resolvers.instance or nil -- the current one (fast access) - --- An instance has an environment (coming from the outside, kept raw), variables --- (coming from the configuration file), and expansions (variables with nested --- variables replaced). One can push something into the outer environment and --- its internal copy, but only the later one will be the raw unprefixed variant. - + resolvers.luacnfspec="{selfautoloc:,selfautodir:,selfautoparent:}{,/texmf{-local,}/web2c}" +end +resolvers.luacnfspec='home:texmf/web2c;'..resolvers.luacnfspec +local unset_variable="unset" +local formats=resolvers.formats +local suffixes=resolvers.suffixes +local dangerous=resolvers.dangerous +local suffixmap=resolvers.suffixmap +resolvers.defaultsuffixes={ "tex" } +resolvers.instance=resolvers.instance or nil +local instance=resolvers.instance or nil function resolvers.setenv(key,value,raw) - if instance then - -- this one will be consulted first when we stay inside - -- the current environment (prefixes are not resolved here) - instance.environment[key] = value - -- we feed back into the environment, and as this is used - -- by other applications (via os.execute) we need to make - -- sure that prefixes are resolve - ossetenv(key,raw and value or resolvers.resolve(value)) - end + if instance then + instance.environment[key]=value + ossetenv(key,raw and value or resolvers.resolve(value)) + end end - --- Beware we don't want empty here as this one can be called early on --- and therefore we use rawget. - local function getenv(key) - local value = rawget(instance.environment,key) - if value and value ~= "" then - return value - else - local e = osgetenv(key) - return e ~= nil and e ~= "" and checkedvariable(e) or "" - end -end - -resolvers.getenv = getenv -resolvers.env = getenv - --- We are going to use some metatable trickery where we backtrack from --- expansion to variable to environment. - + local value=rawget(instance.environment,key) + if value and value~="" then + return value + else + local e=osgetenv(key) + return e~=nil and e~="" and checkedvariable(e) or "" + end +end +resolvers.getenv=getenv +resolvers.env=getenv local function resolve(k) - return instance.expansions[k] -end - -local dollarstripper = lpeg.stripper("$") -local inhibitstripper = P("!")^0 * Cs(P(1)^0) -local backslashswapper = lpeg.replacer("\\","/") - -local somevariable = P("$") / "" -local somekey = C(R("az","AZ","09","__","--")^1) -local somethingelse = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "") - + P(";") * (P(";") / "") - + P(1) -local variableexpander = Cs( (somevariable * (somekey/resolve) + somethingelse)^1 ) - -local cleaner = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";" -local variablecleaner = Cs((cleaner + P(1))^0) - -local somevariable = R("az","AZ","09","__","--")^1 / resolve -local variable = (P("$")/"") * (somevariable + (P("{")/"") * somevariable * (P("}")/"")) -local variableresolver = Cs((variable + P(1))^0) - + return instance.expansions[k] +end +local dollarstripper=lpeg.stripper("$") +local inhibitstripper=P("!")^0*Cs(P(1)^0) +local backslashswapper=lpeg.replacer("\\","/") +local somevariable=P("$")/"" +local somekey=C(R("az","AZ","09","__","--")^1) +local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) +local variableexpander=Cs((somevariable*(somekey/resolve)+somethingelse)^1 ) +local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" +local variablecleaner=Cs((cleaner+P(1))^0) +local somevariable=R("az","AZ","09","__","--")^1/resolve +local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) +local variableresolver=Cs((variable+P(1))^0) local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end - -function resolvers.newinstance() -- todo: all vars will become lowercase and alphanum only - - if trace_locating then - report_resolving("creating instance") - end - - local environment, variables, expansions, order = allocate(), allocate(), allocate(), allocate() - - local newinstance = { - environment = environment, - variables = variables, - expansions = expansions, - order = order, - files = allocate(), - setups = allocate(), - found = allocate(), - foundintrees = allocate(), - hashes = allocate(), - hashed = allocate(), - specification = allocate(), - lists = allocate(), - data = allocate(), -- only for loading - fakepaths = allocate(), - remember = true, - diskcache = true, - renewcache = false, - renewtree = false, - loaderror = false, - savelists = true, - pattern = nil, -- lists - force_suffixes = true, - } - - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v = order[i][k] - if v ~= nil then - t[k] = v - return v - end - end - if v == nil then - v = "" - end - t[k] = v - return v - end) - - setmetatableindex(environment, function(t,k) - local v = osgetenv(k) - if v == nil then - v = variables[k] - end - if v ~= nil then - v = checkedvariable(v) or "" - end - v = resolvers.repath(v) -- for taco who has a : separated osfontdir - t[k] = v - return v - end) - - setmetatableindex(expansions, function(t,k) - local v = environment[k] - if type(v) == "string" then - v = lpegmatch(variableresolver,v) - v = lpegmatch(variablecleaner,v) - end - t[k] = v + return lpegmatch(variableexpander,var) or var +end +function resolvers.newinstance() + if trace_locating then + report_resolving("creating instance") + end + local environment,variables,expansions,order=allocate(),allocate(),allocate(),allocate() + local newinstance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files=allocate(), + setups=allocate(), + found=allocate(), + foundintrees=allocate(), + hashes=allocate(), + hashed=allocate(), + specification=allocate(), + lists=allocate(), + data=allocate(), + fakepaths=allocate(), + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v return v - end) - - return newinstance - + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end + if v~=nil then + v=checkedvariable(v) or "" + end + v=resolvers.repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + return newinstance end - -function resolvers.setinstance(someinstance) -- only one instance is active - instance = someinstance - resolvers.instance = someinstance - return someinstance +function resolvers.setinstance(someinstance) + instance=someinstance + resolvers.instance=someinstance + return someinstance end - function resolvers.reset() - return resolvers.setinstance(resolvers.newinstance()) + return resolvers.setinstance(resolvers.newinstance()) end - local function reset_hashes() - instance.lists = { } - instance.found = { } -end - -local slash = P("/") - -local pathexpressionpattern = Cs ( - Cc("^") * ( - Cc("%") * S(".-") - + slash^2 * P(-1) / "/.*" - + slash^2 / "/.-/" - + (1-slash) * P(-1) * Cc("/") - + P(1) - )^1 * Cc("$") -- yes or no $ + instance.lists={} + instance.found={} +end +local slash=P("/") +local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*"+slash^2/"/.-/"+(1-slash)*P(-1)*Cc("/")+P(1) + )^1*Cc("$") ) - -local cache = { } - +local cache={} local function makepathexpression(str) - if str == "." then - return "^%./$" - else - local c = cache[str] - if not c then - c = lpegmatch(pathexpressionpattern,str) - cache[str] = c - end - return c + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c end + return c + end end - local function reportcriticalvariables(cnfspec) - if trace_locating then - for i=1,#resolvers.criticalvars do - local k = resolvers.criticalvars[i] - local v = resolvers.getenv(k) or "unknown" -- this one will not resolve ! - report_resolving("variable '%s' set to '%s'",k,v) - end - report_resolving() - if cnfspec then - if type(cnfspec) == "table" then - report_resolving("using configuration specification '%s'",concat(cnfspec,",")) - else - report_resolving("using configuration specification '%s'",cnfspec) - end - end - report_resolving() + if trace_locating then + for i=1,#resolvers.criticalvars do + local k=resolvers.criticalvars[i] + local v=resolvers.getenv(k) or "unknown" + report_resolving("variable '%s' set to '%s'",k,v) + end + report_resolving() + if cnfspec then + if type(cnfspec)=="table" then + report_resolving("using configuration specification '%s'",concat(cnfspec,",")) + else + report_resolving("using configuration specification '%s'",cnfspec) + end end - reportcriticalvariables = function() end + report_resolving() + end + reportcriticalvariables=function() end end - local function identify_configuration_files() - local specification = instance.specification - if #specification == 0 then - local cnfspec = getenv("TEXMFCNF") - if cnfspec == "" then - cnfspec = resolvers.luacnfspec - resolvers.luacnfstate = "default" - else - resolvers.luacnfstate = "environment" - end - reportcriticalvariables(cnfspec) - local cnfpaths = expandedpathfromlist(resolvers.splitpath(cnfspec)) - local luacnfname = resolvers.luacnfname - for i=1,#cnfpaths do - local filename = collapsepath(filejoin(cnfpaths[i],luacnfname)) - local realname = resolvers.resolve(filename) - if lfs.isfile(realname) then - specification[#specification+1] = filename - if trace_locating then - report_resolving("found configuration file '%s'",realname) - end - elseif trace_locating then - report_resolving("unknown configuration file '%s'",realname) - end - end + local specification=instance.specification + if #specification==0 then + local cnfspec=getenv("TEXMFCNF") + if cnfspec=="" then + cnfspec=resolvers.luacnfspec + resolvers.luacnfstate="default" + else + resolvers.luacnfstate="environment" + end + reportcriticalvariables(cnfspec) + local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local luacnfname=resolvers.luacnfname + for i=1,#cnfpaths do + local filename=collapsepath(filejoin(cnfpaths[i],luacnfname)) + local realname=resolvers.resolve(filename) + if lfs.isfile(realname) then + specification[#specification+1]=filename if trace_locating then - report_resolving() + report_resolving("found configuration file '%s'",realname) end - elseif trace_locating then - report_resolving("configuration files already identified") + elseif trace_locating then + report_resolving("unknown configuration file '%s'",realname) + end end + if trace_locating then + report_resolving() + end + elseif trace_locating then + report_resolving("configuration files already identified") + end end - local function load_configuration_files() - local specification = instance.specification - if #specification > 0 then - local luacnfname = resolvers.luacnfname - for i=1,#specification do - local filename = specification[i] - local pathname = filedirname(filename) - local filename = filejoin(pathname,luacnfname) - local realname = resolvers.resolve(filename) -- no shortcut - local blob = loadfile(realname) - if blob then - local setups = instance.setups - local data = blob() - local parent = data and data.parent - if parent then - local filename = filejoin(pathname,parent) - local realname = resolvers.resolve(filename) -- no shortcut - local blob = loadfile(realname) - if blob then - local parentdata = blob() - if parentdata then - report_resolving("loading configuration file '%s'",filename) - data = table.merged(parentdata,data) - end - end - end - data = data and data.content - if data then - if trace_locating then - report_resolving("loading configuration file '%s'",filename) - report_resolving() - end - local variables = data.variables or { } - local warning = false - for k, v in next, data do - local variant = type(v) - if variant == "table" then - initializesetter(filename,k,v) - elseif variables[k] == nil then - if trace_locating and not warning then - report_resolving("variables like '%s' in configuration file '%s' should move to the 'variables' subtable", - k,resolvers.resolve(filename)) - warning = true - end - variables[k] = v - end - end - setups[pathname] = variables - if resolvers.luacnfstate == "default" then - -- the following code is not tested - local cnfspec = variables["TEXMFCNF"] - if cnfspec then - if trace_locating then - report_resolving("reloading configuration due to TEXMF redefinition") - end - -- we push the value into the main environment (osenv) so - -- that it takes precedence over the default one and therefore - -- also over following definitions - resolvers.setenv("TEXMFCNF",cnfspec) -- resolves prefixes - -- we now identify and load the specified configuration files - instance.specification = { } - identify_configuration_files() - load_configuration_files() - -- we prevent further overload of the configuration variable - resolvers.luacnfstate = "configuration" - -- we quit the outer loop - break - end - end - - else - if trace_locating then - report_resolving("skipping configuration file '%s' (no content)",filename) - end - setups[pathname] = { } - instance.loaderror = true - end - elseif trace_locating then - report_resolving("skipping configuration file '%s' (no valid format)",filename) + local specification=instance.specification + if #specification>0 then + local luacnfname=resolvers.luacnfname + for i=1,#specification do + local filename=specification[i] + local pathname=filedirname(filename) + local filename=filejoin(pathname,luacnfname) + local realname=resolvers.resolve(filename) + local blob=loadfile(realname) + if blob then + local setups=instance.setups + local data=blob() + local parent=data and data.parent + if parent then + local filename=filejoin(pathname,parent) + local realname=resolvers.resolve(filename) + local blob=loadfile(realname) + if blob then + local parentdata=blob() + if parentdata then + report_resolving("loading configuration file '%s'",filename) + data=table.merged(parentdata,data) + end + end + end + data=data and data.content + if data then + if trace_locating then + report_resolving("loading configuration file '%s'",filename) + report_resolving() + end + local variables=data.variables or {} + local warning=false + for k,v in next,data do + local variant=type(v) + if variant=="table" then + initializesetter(filename,k,v) + elseif variables[k]==nil then + if trace_locating and not warning then + report_resolving("variables like '%s' in configuration file '%s' should move to the 'variables' subtable", + k,resolvers.resolve(filename)) + warning=true + end + variables[k]=v end - instance.order[#instance.order+1] = instance.setups[pathname] - if instance.loaderror then - break + end + setups[pathname]=variables + if resolvers.luacnfstate=="default" then + local cnfspec=variables["TEXMFCNF"] + if cnfspec then + if trace_locating then + report_resolving("reloading configuration due to TEXMF redefinition") + end + resolvers.setenv("TEXMFCNF",cnfspec) + instance.specification={} + identify_configuration_files() + load_configuration_files() + resolvers.luacnfstate="configuration" + break end + end + else + if trace_locating then + report_resolving("skipping configuration file '%s' (no content)",filename) + end + setups[pathname]={} + instance.loaderror=true end - elseif trace_locating then - report_resolving("warning: no lua configuration files found") + elseif trace_locating then + report_resolving("skipping configuration file '%s' (no valid format)",filename) + end + instance.order[#instance.order+1]=instance.setups[pathname] + if instance.loaderror then + break + end end + elseif trace_locating then + report_resolving("warning: no lua configuration files found") + end end - --- scheme magic ... database loading - local function load_file_databases() - instance.loaderror, instance.files = false, allocate() - if not instance.renewcache then - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - resolvers.hashers.byscheme(hash.type,hash.name) - if instance.loaderror then break end - end + instance.loaderror,instance.files=false,allocate() + if not instance.renewcache then + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + resolvers.hashers.byscheme(hash.type,hash.name) + if instance.loaderror then break end end + end end - local function locate_file_databases() - -- todo: cache:// and tree:// (runtime) - local texmfpaths = resolvers.expandedpathlist("TEXMF") - if #texmfpaths > 0 then - for i=1,#texmfpaths do - local path = collapsepath(texmfpaths[i]) - path = gsub(path,"/+$","") -- in case $HOME expands to something with a trailing / - local stripped = lpegmatch(inhibitstripper,path) -- the !! thing - if stripped ~= "" then - local runtime = stripped == path - path = resolvers.cleanpath(path) - local spec = resolvers.splitmethod(stripped) - if runtime and (spec.noscheme or spec.scheme == "file") then - stripped = "tree:///" .. stripped - elseif spec.scheme == "cache" or spec.scheme == "file" then - stripped = spec.path - end - if trace_locating then - if runtime then - report_resolving("locating list of '%s' (runtime) (%s)",path,stripped) - else - report_resolving("locating list of '%s' (cached)",path) - end - end - methodhandler('locators',stripped) - end + local texmfpaths=resolvers.expandedpathlist("TEXMF") + if #texmfpaths>0 then + for i=1,#texmfpaths do + local path=collapsepath(texmfpaths[i]) + path=gsub(path,"/+$","") + local stripped=lpegmatch(inhibitstripper,path) + if stripped~="" then + local runtime=stripped==path + path=resolvers.cleanpath(path) + local spec=resolvers.splitmethod(stripped) + if runtime and (spec.noscheme or spec.scheme=="file") then + stripped="tree:///"..stripped + elseif spec.scheme=="cache" or spec.scheme=="file" then + stripped=spec.path end if trace_locating then - report_resolving() + if runtime then + report_resolving("locating list of '%s' (runtime) (%s)",path,stripped) + else + report_resolving("locating list of '%s' (cached)",path) + end end - elseif trace_locating then - report_resolving("no texmf paths are defined (using TEXMF)") - end -end - -local function generate_file_databases() - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - methodhandler('generators',hash.name) + methodhandler('locators',stripped) + end end if trace_locating then - report_resolving() + report_resolving() end + elseif trace_locating then + report_resolving("no texmf paths are defined (using TEXMF)") + end end - -local function save_file_databases() -- will become cachers - for i=1,#instance.hashes do - local hash = instance.hashes[i] - local cachename = hash.name - if hash.cache then - local content = instance.files[cachename] - caches.collapsecontent(content) - if trace_locating then - report_resolving("saving tree '%s'",cachename) - end - caches.savecontent(cachename,"files",content) - elseif trace_locating then - report_resolving("not saving runtime tree '%s'",cachename) - end +local function generate_file_databases() + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + methodhandler('generators',hash.name) + end + if trace_locating then + report_resolving() + end +end +local function save_file_databases() + for i=1,#instance.hashes do + local hash=instance.hashes[i] + local cachename=hash.name + if hash.cache then + local content=instance.files[cachename] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree '%s'",cachename) + end + caches.savecontent(cachename,"files",content) + elseif trace_locating then + report_resolving("not saving runtime tree '%s'",cachename) end + end end - function resolvers.renew(hashname) - if hashname and hashname ~= "" then - local expanded = resolvers.expansion(hashname) or "" - if expanded ~= "" then - if trace_locating then - report_resolving("identifying tree '%s' from '%s'",expanded,hashname) - end - hashname = expanded - else - if trace_locating then - report_resolving("identifying tree '%s'",hashname) - end - end - local realpath = resolvers.resolve(hashname) - if lfs.isdir(realpath) then - if trace_locating then - report_resolving("using path '%s'",realpath) - end - methodhandler('generators',hashname) - -- could be shared - local content = instance.files[hashname] - caches.collapsecontent(content) - if trace_locating then - report_resolving("saving tree '%s'",hashname) - end - caches.savecontent(hashname,"files",content) - -- till here - else - report_resolving("invalid path '%s'",realpath) - end + if hashname and hashname~="" then + local expanded=resolvers.expansion(hashname) or "" + if expanded~="" then + if trace_locating then + report_resolving("identifying tree '%s' from '%s'",expanded,hashname) + end + hashname=expanded + else + if trace_locating then + report_resolving("identifying tree '%s'",hashname) + end + end + local realpath=resolvers.resolve(hashname) + if lfs.isdir(realpath) then + if trace_locating then + report_resolving("using path '%s'",realpath) + end + methodhandler('generators',hashname) + local content=instance.files[hashname] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree '%s'",hashname) + end + caches.savecontent(hashname,"files",content) + else + report_resolving("invalid path '%s'",realpath) end + end end - local function load_databases() - locate_file_databases() - if instance.diskcache and not instance.renewcache then - load_file_databases() - if instance.loaderror then - generate_file_databases() - save_file_databases() - end - else - generate_file_databases() - if instance.renewcache then - save_file_databases() - end + locate_file_databases() + if instance.diskcache and not instance.renewcache then + load_file_databases() + if instance.loaderror then + generate_file_databases() + save_file_databases() end + else + generate_file_databases() + if instance.renewcache then + save_file_databases() + end + end end - function resolvers.appendhash(type,name,cache) - -- safeguard ... tricky as it's actually a bug when seen twice - if not instance.hashed[name] then - if trace_locating then - report_resolving("hash '%s' appended",name) - end - insert(instance.hashes, { type = type, name = name, cache = cache } ) - instance.hashed[name] = cache + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash '%s' appended",name) end + insert(instance.hashes,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end end - function resolvers.prependhash(type,name,cache) - -- safeguard ... tricky as it's actually a bug when seen twice - if not instance.hashed[name] then - if trace_locating then - report_resolving("hash '%s' prepended",name) - end - insert(instance.hashes, 1, { type = type, name = name, cache = cache } ) - instance.hashed[name] = cache - end -end - -function resolvers.extendtexmfvariable(specification) -- crap, we could better prepend the hash - local t = resolvers.splitpath(getenv("TEXMF")) -- okay? - insert(t,1,specification) - local newspec = concat(t,",") -- not ; - if instance.environment["TEXMF"] then - instance.environment["TEXMF"] = newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"] = newspec - else - -- weird - end - reset_hashes() + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash '%s' prepended",name) + end + insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end +end +function resolvers.extendtexmfvariable(specification) + local t=resolvers.splitpath(getenv("TEXMF")) + insert(t,1,specification) + local newspec=concat(t,",") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"]=newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"]=newspec + else + end + reset_hashes() end - function resolvers.splitexpansions() - local ie = instance.expansions - for k,v in next, ie do - local t, tn, h, p = { }, 0, { }, splitconfigurationpath(v) - for kk=1,#p do - local vv = p[kk] - if vv ~= "" and not h[vv] then - tn = tn + 1 - t[tn] = vv - h[vv] = true - end - end - if #t > 1 then - ie[k] = t - else - ie[k] = t[1] - end + local ie=instance.expansions + for k,v in next,ie do + local t,tn,h,p={},0,{},splitconfigurationpath(v) + for kk=1,#p do + local vv=p[kk] + if vv~="" and not h[vv] then + tn=tn+1 + t[tn]=vv + h[vv]=true + end + end + if #t>1 then + ie[k]=t + else + ie[k]=t[1] end + end end - --- end of split/join code - --- we used to have 'files' and 'configurations' so therefore the following --- shared function - function resolvers.datastate() - return caches.contentstate() + return caches.contentstate() end - function resolvers.variable(name) - local name = name and lpegmatch(dollarstripper,name) - local result = name and instance.variables[name] - return result ~= nil and result or "" + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.variables[name] + return result~=nil and result or "" end - function resolvers.expansion(name) - local name = name and lpegmatch(dollarstripper,name) - local result = name and instance.expansions[name] - return result ~= nil and result or "" + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.expansions[name] + return result~=nil and result or "" end - function resolvers.unexpandedpathlist(str) - local pth = resolvers.variable(str) - local lst = resolvers.splitpath(pth) - return expandedpathfromlist(lst) + local pth=resolvers.variable(str) + local lst=resolvers.splitpath(pth) + return expandedpathfromlist(lst) end - function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(resolvers.unexpandedpathlist(str)) end - -local done = { } - +local done={} function resolvers.resetextrapath() - local ep = instance.extra_paths - if not ep then - ep, done = { }, { } - instance.extra_paths = ep - elseif #ep > 0 then - instance.lists, done = { }, { } - end + local ep=instance.extra_paths + if not ep then + ep,done={},{} + instance.extra_paths=ep + elseif #ep>0 then + instance.lists,done={},{} + end end - function resolvers.registerextrapath(paths,subpaths) - paths = settings_to_array(paths) - subpaths = settings_to_array(subpaths) - local ep = instance.extra_paths or { } - local oldn = #ep - local newn = oldn - local nofpaths = #paths - local nofsubpaths = #subpaths - if nofpaths > 0 then - if nofsubpaths > 0 then - for i=1,nofpaths do - local p = paths[i] - for j=1,nofsubpaths do - local s = subpaths[j] - local ps = p .. "/" .. s - if not done[ps] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(ps) - done[ps] = true - end - end - end - else - for i=1,nofpaths do - local p = paths[i] - if not done[p] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(p) - done[p] = true - end - end + paths=settings_to_array(paths) + subpaths=settings_to_array(subpaths) + local ep=instance.extra_paths or {} + local oldn=#ep + local newn=oldn + local nofpaths=#paths + local nofsubpaths=#subpaths + if nofpaths>0 then + if nofsubpaths>0 then + for i=1,nofpaths do + local p=paths[i] + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=p.."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(ps) + done[ps]=true + end end - elseif nofsubpaths > 0 then - for i=1,oldn do - for j=1,nofsubpaths do - local s = subpaths[j] - local ps = ep[i] .. "/" .. s - if not done[ps] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(ps) - done[ps] = true - end - end + end + else + for i=1,nofpaths do + local p=paths[i] + if not done[p] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(p) + done[p]=true end + end end - if newn > 0 then - instance.extra_paths = ep -- register paths - end - if newn > oldn then - instance.lists = { } -- erase the cache + elseif nofsubpaths>0 then + for i=1,oldn do + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=ep[i].."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(ps) + done[ps]=true + end + end end + end + if newn>0 then + instance.extra_paths=ep + end + if newn>oldn then + instance.lists={} + end end - local function made_list(instance,list) - local ep = instance.extra_paths - if not ep or #ep == 0 then - return list - else - local done, new, newn = { }, { }, 0 - -- honour . .. ../.. but only when at the start - for k=1,#list do - local v = list[k] - if not done[v] then - if find(v,"^[%.%/]$") then - done[v] = true - newn = newn + 1 - new[newn] = v - else - break - end - end - end - -- first the extra paths - for k=1,#ep do - local v = ep[k] - if not done[v] then - done[v] = true - newn = newn + 1 - new[newn] = v - end - end - -- next the formal paths - for k=1,#list do - local v = list[k] - if not done[v] then - done[v] = true - newn = newn + 1 - new[newn] = v - end + local ep=instance.extra_paths + if not ep or #ep==0 then + return list + else + local done,new,newn={},{},0 + for k=1,#list do + local v=list[k] + if not done[v] then + if find(v,"^[%.%/]$") then + done[v]=true + newn=newn+1 + new[newn]=v + else + break end - return new + end + end + for k=1,#ep do + local v=ep[k] + if not done[v] then + done[v]=true + newn=newn+1 + new[newn]=v + end + end + for k=1,#list do + local v=list[k] + if not done[v] then + done[v]=true + newn=newn+1 + new[newn]=v + end end + return new + end end - function resolvers.cleanpathlist(str) - local t = resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i] = collapsepath(resolvers.cleanpath(t[i])) - end + local t=resolvers.expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(resolvers.cleanpath(t[i])) end - return t + end + return t end - function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) + return joinpath(resolvers.expandedpathlist(str)) end - function resolvers.expandedpathlist(str) - if not str then - return { } - elseif instance.savelists then - str = lpegmatch(dollarstripper,str) - local lists = instance.lists - local lst = lists[str] - if not lst then - local l = made_list(instance,resolvers.splitpath(resolvers.expansion(str))) - lst = expandedpathfromlist(l) - lists[str] = lst - end - return lst - else - local lst = resolvers.splitpath(resolvers.expansion(str)) - return made_list(instance,expandedpathfromlist(lst)) + if not str then + return {} + elseif instance.savelists then + str=lpegmatch(dollarstripper,str) + local lists=instance.lists + local lst=lists[str] + if not lst then + local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str))) + lst=expandedpathfromlist(l) + lists[str]=lst end + return lst + else + local lst=resolvers.splitpath(resolvers.expansion(str)) + return made_list(instance,expandedpathfromlist(lst)) + end end - -function resolvers.expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ // - str = lpegmatch(dollarstripper,str) - local tmp = resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp ~= "" and tmp or str) +function resolvers.expandedpathlistfromvariable(str) + str=lpegmatch(dollarstripper,str) + local tmp=resolvers.variableofformatorsuffix(str) + return resolvers.expandedpathlist(tmp~="" and tmp or str) end - function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(resolvers.expandedpathlistfromvariable(str)) end - -function resolvers.expandbraces(str) -- output variable and brace expansion of STRING --- local ori = resolvers.variable(str) --- if ori == "" then - local ori = str --- end - local pth = expandedpathfromlist(resolvers.splitpath(ori)) - return joinpath(pth) +function resolvers.expandbraces(str) + local ori=str + local pth=expandedpathfromlist(resolvers.splitpath(ori)) + return joinpath(pth) end - function resolvers.registerfilehash(name,content,someerror) - if content then - instance.files[name] = content - else - instance.files[name] = { } - if somerror == true then -- can be unset - instance.loaderror = someerror - end + if content then + instance.files[name]=content + else + instance.files[name]={} + if somerror==true then + instance.loaderror=someerror end + end end - local function isreadable(name) - local readable = lfs.isfile(name) -- not file.is_readable(name) asit can be a dir - if trace_detail then - if readable then - report_resolving("file '%s' is readable",name) - else - report_resolving("file '%s' is not readable", name) - end + local readable=lfs.isfile(name) + if trace_detail then + if readable then + report_resolving("file '%s' is readable",name) + else + report_resolving("file '%s' is not readable",name) end - return readable + end + return readable end - --- name --- name/name - local function collect_files(names) - local filelist, noffiles = { }, 0 - for k=1,#names do - local fname = names[k] + local filelist,noffiles={},0 + for k=1,#names do + local fname=names[k] + if trace_detail then + report_resolving("checking name '%s'",fname) + end + local bname=filebasename(fname) + local dname=filedirname(fname) + if dname=="" or find(dname,"^%.") then + dname=false + else + dname=gsub(dname,"%*",".*") + dname="/"..dname.."$" + end + local hashes=instance.hashes + for h=1,#hashes do + local hash=hashes[h] + local blobpath=hash.name + local files=blobpath and instance.files[blobpath] + if files then if trace_detail then - report_resolving("checking name '%s'",fname) - end - local bname = filebasename(fname) - local dname = filedirname(fname) - if dname == "" or find(dname,"^%.") then - dname = false - else - dname = gsub(dname,"*","%.*") - dname = "/" .. dname .. "$" + report_resolving("deep checking '%s' (%s)",blobpath,bname) + end + local blobfile=files[bname] + if not blobfile then + local rname="remap:"..bname + blobfile=files[rname] + if blobfile then + bname=files[rname] + blobfile=files[bname] + end end - local hashes = instance.hashes - for h=1,#hashes do - local hash = hashes[h] - local blobpath = hash.name - local files = blobpath and instance.files[blobpath] - if files then + if blobfile then + local blobroot=files.__path__ or blobpath + if type(blobfile)=='string' then + if not dname or find(blobfile,dname) then + local variant=hash.type + local search=filejoin(blobroot,blobfile,bname) + local result=methodhandler('concatinators',hash.type,blobroot,blobfile,bname) + if trace_detail then + report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + else + for kk=1,#blobfile do + local vv=blobfile[kk] + if not dname or find(vv,dname) then + local variant=hash.type + local search=filejoin(blobroot,vv,bname) + local result=methodhandler('concatinators',hash.type,blobroot,vv,bname) if trace_detail then - report_resolving("deep checking '%s' (%s)",blobpath,bname) - end - local blobfile = files[bname] - if not blobfile then - local rname = "remap:"..bname - blobfile = files[rname] - if blobfile then - bname = files[rname] - blobfile = files[bname] - end - end - if blobfile then - local blobroot = files.__path__ or blobpath - if type(blobfile) == 'string' then - if not dname or find(blobfile,dname) then - local variant = hash.type - -- local search = filejoin(blobpath,blobfile,bname) - local search = filejoin(blobroot,blobfile,bname) - local result = methodhandler('concatinators',hash.type,blobroot,blobfile,bname) - if trace_detail then - report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) - end - noffiles = noffiles + 1 - filelist[noffiles] = { variant, search, result } - end - else - for kk=1,#blobfile do - local vv = blobfile[kk] - if not dname or find(vv,dname) then - local variant = hash.type - -- local search = filejoin(blobpath,vv,bname) - local search = filejoin(blobroot,vv,bname) - local result = methodhandler('concatinators',hash.type,blobroot,vv,bname) - if trace_detail then - report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) - end - noffiles = noffiles + 1 - filelist[noffiles] = { variant, search, result } - end - end - end + report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) end - elseif trace_locating then - report_resolving("no match in '%s' (%s)",blobpath,bname) + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end end + end end + elseif trace_locating then + report_resolving("no match in '%s' (%s)",blobpath,bname) + end end - return noffiles > 0 and filelist or nil + end + return noffiles>0 and filelist or nil end - -local fit = { } - +local fit={} function resolvers.registerintrees(filename,format,filetype,usedmethod,foundname) - local foundintrees = instance.foundintrees - if usedmethod == "direct" and filename == foundname and fit[foundname] then - -- just an extra lookup after a test on presence - else - local t = { - filename = filename, - format = format ~= "" and format or nil, - filetype = filetype ~= "" and filetype or nil, - usedmethod = usedmethod, - foundname = foundname, - } - fit[foundname] = t - foundintrees[#foundintrees+1] = t - end + local foundintrees=instance.foundintrees + if usedmethod=="direct" and filename==foundname and fit[foundname] then + else + local t={ + filename=filename, + format=format~="" and format or nil, + filetype=filetype~="" and filetype or nil, + usedmethod=usedmethod, + foundname=foundname, + } + fit[foundname]=t + foundintrees[#foundintrees+1]=t + end end - --- split the next one up for readability (but this module needs a cleanup anyway) - -local function can_be_dir(name) -- can become local - local fakepaths = instance.fakepaths - if not fakepaths[name] then - if lfs.isdir(name) then - fakepaths[name] = 1 -- directory - else - fakepaths[name] = 2 -- no directory - end +local function can_be_dir(name) + local fakepaths=instance.fakepaths + if not fakepaths[name] then + if lfs.isdir(name) then + fakepaths[name]=1 + else + fakepaths[name]=2 end - return fakepaths[name] == 1 + end + return fakepaths[name]==1 end - -local preparetreepattern = Cs((P(".")/"%%." + P("-")/"%%-" + P(1))^0 * Cc("$")) - --- -- -- begin of main file search routing -- -- -- needs checking as previous has been patched - +local preparetreepattern=Cs((P(".")/"%%."+P("-")/"%%-"+P(1))^0*Cc("$")) local collect_instance_files - local function find_analyze(filename,askedformat,allresults) - local filetype, wantedfiles, ext = '', { }, suffixonly(filename) - -- too tricky as filename can be bla.1.2.3: - -- - -- if not suffixmap[ext] then - -- wantedfiles[#wantedfiles+1] = filename - -- end - wantedfiles[#wantedfiles+1] = filename - if askedformat == "" then - if ext == "" or not suffixmap[ext] then - local defaultsuffixes = resolvers.defaultsuffixes - for i=1,#defaultsuffixes do - local forcedname = filename .. '.' .. defaultsuffixes[i] - wantedfiles[#wantedfiles+1] = forcedname - filetype = resolvers.formatofsuffix(forcedname) - if trace_locating then - report_resolving("forcing filetype '%s'",filetype) - end - end - else - filetype = resolvers.formatofsuffix(filename) - if trace_locating then - report_resolving("using suffix based filetype '%s'",filetype) - end + local filetype,wantedfiles,ext='',{},suffixonly(filename) + wantedfiles[#wantedfiles+1]=filename + if askedformat=="" then + if ext=="" or not suffixmap[ext] then + local defaultsuffixes=resolvers.defaultsuffixes + for i=1,#defaultsuffixes do + local forcedname=filename..'.'..defaultsuffixes[i] + wantedfiles[#wantedfiles+1]=forcedname + filetype=resolvers.formatofsuffix(forcedname) + if trace_locating then + report_resolving("forcing filetype '%s'",filetype) end + end else - if ext == "" or not suffixmap[ext] then - local format_suffixes = suffixes[askedformat] - if format_suffixes then - for i=1,#format_suffixes do - wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i] - end - end - end - filetype = askedformat - if trace_locating then - report_resolving("using given filetype '%s'",filetype) + filetype=resolvers.formatofsuffix(filename) + if trace_locating then + report_resolving("using suffix based filetype '%s'",filetype) + end + end + else + if ext=="" or not suffixmap[ext] then + local format_suffixes=suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + wantedfiles[#wantedfiles+1]=filename.."."..format_suffixes[i] end + end end - return filetype, wantedfiles + filetype=askedformat + if trace_locating then + report_resolving("using given filetype '%s'",filetype) + end + end + return filetype,wantedfiles end - local function find_direct(filename,allresults) - if not dangerous[askedformat] and isreadable(filename) then - if trace_detail then - report_resolving("file '%s' found directly",filename) - end - return "direct", { filename } + if not dangerous[askedformat] and isreadable(filename) then + if trace_detail then + report_resolving("file '%s' found directly",filename) end + return "direct",{ filename } + end end - local function find_wildcard(filename,allresults) - if find(filename,'%*') then - if trace_locating then - report_resolving("checking wildcard '%s'", filename) - end - local method, result = resolvers.findwildcardfiles(filename) - if result then - return "wildcard", result - end - end -end - -local function find_qualified(filename,allresults) -- this one will be split too - if not file.is_qualified_path(filename) then - return - end + if find(filename,'%*') then if trace_locating then - report_resolving("checking qualified name '%s'", filename) + report_resolving("checking wildcard '%s'",filename) end - if isreadable(filename) then - if trace_detail then - report_resolving("qualified file '%s' found", filename) - end - return "qualified", { filename } + local method,result=resolvers.findwildcardfiles(filename) + if result then + return "wildcard",result end + end +end +local function find_qualified(filename,allresults) + if not file.is_qualified_path(filename) then + return + end + if trace_locating then + report_resolving("checking qualified name '%s'",filename) + end + if isreadable(filename) then if trace_detail then - report_resolving("locating qualified file '%s'", filename) - end - local forcedname, suffix = "", suffixonly(filename) - if suffix == "" then -- why - local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat] - if format_suffixes then - for i=1,#format_suffixes do - local s = format_suffixes[i] - forcedname = filename .. "." .. s - if isreadable(forcedname) then - if trace_locating then - report_resolving("no suffix, forcing format filetype '%s'", s) - end - return "qualified", { forcedname } - end - end + report_resolving("qualified file '%s' found",filename) + end + return "qualified",{ filename } + end + if trace_detail then + report_resolving("locating qualified file '%s'",filename) + end + local forcedname,suffix="",suffixonly(filename) + if suffix=="" then + local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + local s=format_suffixes[i] + forcedname=filename.."."..s + if isreadable(forcedname) then + if trace_locating then + report_resolving("no suffix, forcing format filetype '%s'",s) + end + return "qualified",{ forcedname } end + end end - if suffix and suffix ~= "" then - -- try to find in tree (no suffix manipulation), here we search for the - -- matching last part of the name - local basename = filebasename(filename) - local pattern = lpegmatch(preparetreepattern,filename) - -- messy .. to be sorted out - local savedformat = askedformat - local format = savedformat or "" - if format == "" then - askedformat = resolvers.formatofsuffix(suffix) + end + if suffix and suffix~="" then + local basename=filebasename(filename) + local pattern=lpegmatch(preparetreepattern,filename) + local savedformat=askedformat + local format=savedformat or "" + if format=="" then + askedformat=resolvers.formatofsuffix(suffix) + end + if not format then + askedformat="othertextfiles" + end + if basename~=filename then + local resolved=collect_instance_files(basename,askedformat,allresults) + if #resolved==0 then + local lowered=lower(basename) + if filename~=lowered then + resolved=collect_instance_files(lowered,askedformat,allresults) end - if not format then - askedformat = "othertextfiles" -- kind of everything, maybe all + end + resolvers.format=savedformat + if #resolved>0 then + local result={} + for r=1,#resolved do + local rr=resolved[r] + if find(rr,pattern) then + result[#result+1]=rr + end end - -- - if basename ~= filename then - local resolved = collect_instance_files(basename,askedformat,allresults) - if #resolved == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered,askedformat,allresults) - end - end - resolvers.format = savedformat - -- - if #resolved > 0 then - local result = { } - for r=1,#resolved do - local rr = resolved[r] - if find(rr,pattern) then - result[#result+1] = rr - end - end - if #result > 0 then - return "qualified", result - end - end + if #result>0 then + return "qualified",result end - -- a real wildcard: - -- - -- local filelist = collect_files({basename}) - -- result = { } - -- for f=1,#filelist do - -- local ff = filelist[f][3] or "" - -- if find(ff,pattern) then - -- result[#result+1], ok = ff, true - -- end - -- end - -- if #result > 0 then - -- return "qualified", result - -- end + end end + end end - local function check_subpath(fname) - if isreadable(fname) then - if trace_detail then - report_resolving("found '%s' by deep scanning",fname) - end - return fname + if isreadable(fname) then + if trace_detail then + report_resolving("found '%s' by deep scanning",fname) end + return fname + end end - local function find_intree(filename,filetype,wantedfiles,allresults) - local typespec = resolvers.variableofformat(filetype) - local pathlist = resolvers.expandedpathlist(typespec) - local method = "intree" - if pathlist and #pathlist > 0 then - -- list search - local filelist = collect_files(wantedfiles) - local dirlist = { } - if filelist then - for i=1,#filelist do - dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble - end - end + local typespec=resolvers.variableofformat(filetype) + local pathlist=resolvers.expandedpathlist(typespec) + local method="intree" + if pathlist and #pathlist>0 then + local filelist=collect_files(wantedfiles) + local dirlist={} + if filelist then + for i=1,#filelist do + dirlist[i]=filedirname(filelist[i][3]).."/" + end + end + if trace_detail then + report_resolving("checking filename '%s'",filename) + end + local result={} + for k=1,#pathlist do + local path=pathlist[k] + local pathname=lpegmatch(inhibitstripper,path) + local doscan=path==pathname + if not find (pathname,'//$') then + doscan=false + end + local done=false + if filelist then + local expression=makepathexpression(pathname) if trace_detail then - report_resolving("checking filename '%s'",filename) - end - local result = { } - for k=1,#pathlist do - local path = pathlist[k] - local pathname = lpegmatch(inhibitstripper,path) - local doscan = path == pathname -- no ^!! - if not find (pathname,'//$') then - doscan = false -- we check directly on the path + report_resolving("using pattern '%s' for path '%s'",expression,pathname) + end + for k=1,#filelist do + local fl=filelist[k] + local f=fl[2] + local d=dirlist[k] + if find(d,expression) then + result[#result+1]=resolvers.resolve(fl[3]) + done=true + if allresults then + if trace_detail then + report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d) + end + else + if trace_detail then + report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d) + end + break end - local done = false - -- using file list - if filelist then -- database - -- compare list entries with permitted pattern -- /xx /xx// - local expression = makepathexpression(pathname) - if trace_detail then - report_resolving("using pattern '%s' for path '%s'",expression,pathname) - end - for k=1,#filelist do - local fl = filelist[k] - local f = fl[2] - local d = dirlist[k] - if find(d,expression) then - -- todo, test for readable - result[#result+1] = resolvers.resolve(fl[3]) -- no shortcut - done = true - if allresults then - if trace_detail then - report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d) - end - else - if trace_detail then - report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d) - end + elseif trace_detail then + report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d) + end + end + end + if done then + method="database" + else + method="filesystem" + pathname=gsub(pathname,"/+$","") + pathname=resolvers.resolve(pathname) + local scheme=url.hasscheme(pathname) + if not scheme or scheme=="file" then + local pname=gsub(pathname,"%.%*$",'') + if not find(pname,"%*") then + if can_be_dir(pname) then + for k=1,#wantedfiles do + local w=wantedfiles[k] + local fname=check_subpath(filejoin(pname,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end + if not done and doscan then + local files=resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w=wantedfiles[k] + local subpath=files[w] + if not subpath or subpath=="" then + elseif type(subpath)=="string" then + local fname=check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp=subpath[i] + if sp=="" then + else + local fname=check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then break + end end - elseif trace_detail then - report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d) + end end - end - end - if done then - method = "database" - else - method = "filesystem" -- bonus, even when !! is specified - pathname = gsub(pathname,"/+$","") - pathname = resolvers.resolve(pathname) - local scheme = url.hasscheme(pathname) - if not scheme or scheme == "file" then - local pname = gsub(pathname,"%.%*$",'') - if not find(pname,"%*") then - if can_be_dir(pname) then - -- quick root scan first - for k=1,#wantedfiles do - local w = wantedfiles[k] - local fname = check_subpath(filejoin(pname,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - end - if not done and doscan then - -- collect files in path (and cache the result) - local files = resolvers.simplescanfiles(pname,false,true) - for k=1,#wantedfiles do - local w = wantedfiles[k] - local subpath = files[w] - if not subpath or subpath == "" then - -- rootscan already done - elseif type(subpath) == "string" then - local fname = check_subpath(filejoin(pname,subpath,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - else - for i=1,#subpath do - local sp = subpath[i] - if sp == "" then - -- roottest already done - else - local fname = check_subpath(filejoin(pname,sp,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - end - end - if done and not allresults then - break - end - end - end - end - end - else - -- no access needed for non existing path, speedup (esp in large tree with lots of fake) + if done and not allresults then + break end + end end + end end - -- todo recursive scanning - if done and not allresults then - break - end - end - if #result > 0 then - return method, result + else + end end + end + if done and not allresults then + break + end end + if #result>0 then + return method,result + end + end end - local function find_onpath(filename,filetype,wantedfiles,allresults) - if trace_detail then - report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) - end - local result = { } - for k=1,#wantedfiles do - local fname = wantedfiles[k] - if fname and isreadable(fname) then - filename = fname - result[#result+1] = filejoin('.',fname) - if not allresults then - break - end - end + if trace_detail then + report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename,filetype or '?',concat(wantedfiles," | ")) + end + local result={} + for k=1,#wantedfiles do + local fname=wantedfiles[k] + if fname and isreadable(fname) then + filename=fname + result[#result+1]=filejoin('.',fname) + if not allresults then + break + end end - if #result > 0 then - return "onpath", result - end -end - -local function find_otherwise(filename,filetype,wantedfiles,allresults) -- other text files | any | whatever - local filelist = collect_files(wantedfiles) - local fl = filelist and filelist[1] - if fl then - return "otherwise", { resolvers.resolve(fl[3]) } -- filename - end -end - --- we could have a loop over the 6 functions but then we'd have to --- always analyze - -collect_instance_files = function(filename,askedformat,allresults) -- uses nested - askedformat = askedformat or "" - filename = collapsepath(filename) - if allresults then - -- no need for caching, only used for tracing - local filetype, wantedfiles = find_analyze(filename,askedformat) - local results = { - { find_direct (filename,true) }, - { find_wildcard (filename,true) }, - { find_qualified(filename,true) }, - { find_intree (filename,filetype,wantedfiles,true) }, - { find_onpath (filename,filetype,wantedfiles,true) }, - { find_otherwise(filename,filetype,wantedfiles,true) }, - } - local result, status, done = { }, { }, { } - for k, r in next, results do - local method, list = r[1], r[2] - if method and list then - for i=1,#list do - local c = collapsepath(list[i]) - if not done[c] then - result[#result+1] = c - done[c] = true - end - status[#status+1] = format("%-10s: %s",method,c) - end - end - end - if trace_detail then - report_resolving("lookup status: %s",table.serialize(status,filename)) + end + if #result>0 then + return "onpath",result + end +end +local function find_otherwise(filename,filetype,wantedfiles,allresults) + local filelist=collect_files(wantedfiles) + local fl=filelist and filelist[1] + if fl then + return "otherwise",{ resolvers.resolve(fl[3]) } + end +end +collect_instance_files=function(filename,askedformat,allresults) + askedformat=askedformat or "" + filename=collapsepath(filename) + if allresults then + local filetype,wantedfiles=find_analyze(filename,askedformat) + local results={ + { find_direct (filename,true) }, + { find_wildcard (filename,true) }, + { find_qualified(filename,true) }, + { find_intree (filename,filetype,wantedfiles,true) }, + { find_onpath (filename,filetype,wantedfiles,true) }, + { find_otherwise(filename,filetype,wantedfiles,true) }, + } + local result,status,done={},{},{} + for k,r in next,results do + local method,list=r[1],r[2] + if method and list then + for i=1,#list do + local c=collapsepath(list[i]) + if not done[c] then + result[#result+1]=c + done[c]=true + end + status[#status+1]=format("%-10s: %s",method,c) end - return result, status - else - local method, result, stamp, filetype, wantedfiles - if instance.remember then - stamp = format("%s--%s", filename, askedformat) - result = stamp and instance.found[stamp] - if result then - if trace_locating then - report_resolving("remembered file '%s'",filename) - end - return result - end + end + end + if trace_detail then + report_resolving("lookup status: %s",table.serialize(status,filename)) + end + return result,status + else + local method,result,stamp,filetype,wantedfiles + if instance.remember then + stamp=format("%s--%s",filename,askedformat) + result=stamp and instance.found[stamp] + if result then + if trace_locating then + report_resolving("remembered file '%s'",filename) end - method, result = find_direct(filename) + return result + end + end + method,result=find_direct(filename) + if not result then + method,result=find_wildcard(filename) + if not result then + method,result=find_qualified(filename) if not result then - method, result = find_wildcard(filename) + filetype,wantedfiles=find_analyze(filename,askedformat) + method,result=find_intree(filename,filetype,wantedfiles) + if not result then + method,result=find_onpath(filename,filetype,wantedfiles) if not result then - method, result = find_qualified(filename) - if not result then - filetype, wantedfiles = find_analyze(filename,askedformat) - method, result = find_intree(filename,filetype,wantedfiles) - if not result then - method, result = find_onpath(filename,filetype,wantedfiles) - if not result then - method, result = find_otherwise(filename,filetype,wantedfiles) - end - end - end - end - end - if result and #result > 0 then - local foundname = collapsepath(result[1]) - resolvers.registerintrees(filename,askedformat,filetype,method,foundname) - result = { foundname } - else - result = { } -- maybe false - end - if stamp then - if trace_locating then - report_resolving("remembering file '%s'",filename) + method,result=find_otherwise(filename,filetype,wantedfiles) end - instance.found[stamp] = result + end end - return result + end + end + if result and #result>0 then + local foundname=collapsepath(result[1]) + resolvers.registerintrees(filename,askedformat,filetype,method,foundname) + result={ foundname } + else + result={} + end + if stamp then + if trace_locating then + report_resolving("remembering file '%s'",filename) + end + instance.found[stamp]=result end + return result + end end - --- -- -- end of main file search routing -- -- -- - - local function findfiles(filename,filetype,allresults) - local result, status = collect_instance_files(filename,filetype or "",allresults) - if not result or #result == 0 then - local lowered = lower(filename) - if filename ~= lowered then - result, status = collect_instance_files(lowered,filetype or "",allresults) - end + local result,status=collect_instance_files(filename,filetype or "",allresults) + if not result or #result==0 then + local lowered=lower(filename) + if filename~=lowered then + result,status=collect_instance_files(lowered,filetype or "",allresults) end - return result or { }, status + end + return result or {},status end - function resolvers.findfiles(filename,filetype) - return findfiles(filename,filetype,true) + return findfiles(filename,filetype,true) end - function resolvers.findfile(filename,filetype) - return findfiles(filename,filetype,false)[1] or "" + return findfiles(filename,filetype,false)[1] or "" end - function resolvers.findpath(filename,filetype) - return filedirname(findfiles(filename,filetype,false)[1] or "") + return filedirname(findfiles(filename,filetype,false)[1] or "") end - local function findgivenfiles(filename,allresults) - local bname, result = filebasename(filename), { } - local hashes = instance.hashes - local noffound = 0 - for k=1,#hashes do - local hash = hashes[k] - local files = instance.files[hash.name] or { } - local blist = files[bname] - if not blist then - local rname = "remap:"..bname - blist = files[rname] - if blist then - bname = files[rname] - blist = files[bname] - end - end - if blist then - if type(blist) == 'string' then - local found = methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" - if found ~= "" then - noffound = noffound + 1 - result[noffound] = resolvers.resolve(found) - if not allresults then break end - end - else - for kk=1,#blist do - local vv = blist[kk] - local found = methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" - if found ~= "" then - noffound = noffound + 1 - result[noffound] = resolvers.resolve(found) - if not allresults then break end - end - end - end + local bname,result=filebasename(filename),{} + local hashes=instance.hashes + local noffound=0 + for k=1,#hashes do + local hash=hashes[k] + local files=instance.files[hash.name] or {} + local blist=files[bname] + if not blist then + local rname="remap:"..bname + blist=files[rname] + if blist then + bname=files[rname] + blist=files[bname] + end + end + if blist then + if type(blist)=='string' then + local found=methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" + if found~="" then + noffound=noffound+1 + result[noffound]=resolvers.resolve(found) + if not allresults then break end + end + else + for kk=1,#blist do + local vv=blist[kk] + local found=methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" + if found~="" then + noffound=noffound+1 + result[noffound]=resolvers.resolve(found) + if not allresults then break end + end end + end end - return result + end + return result end - function resolvers.findgivenfiles(filename) - return findgivenfiles(filename,true) + return findgivenfiles(filename,true) end - function resolvers.findgivenfile(filename) - return findgivenfiles(filename,false)[1] or "" + return findgivenfiles(filename,false)[1] or "" end - local function doit(path,blist,bname,tag,variant,result,allresults) - local done = false - if blist and variant then - local resolve = resolvers.resolve -- added - if type(blist) == 'string' then - -- make function and share code - if find(lower(blist),path) then - local full = methodhandler('concatinators',variant,tag,blist,bname) or "" - result[#result+1] = resolve(full) - done = true - end - else - for kk=1,#blist do - local vv = blist[kk] - if find(lower(vv),path) then - local full = methodhandler('concatinators',variant,tag,vv,bname) or "" - result[#result+1] = resolve(full) - done = true - if not allresults then break end - end - end + local done=false + if blist and variant then + local resolve=resolvers.resolve + if type(blist)=='string' then + if find(lower(blist),path) then + local full=methodhandler('concatinators',variant,tag,blist,bname) or "" + result[#result+1]=resolve(full) + done=true + end + else + for kk=1,#blist do + local vv=blist[kk] + if find(lower(vv),path) then + local full=methodhandler('concatinators',variant,tag,vv,bname) or "" + result[#result+1]=resolve(full) + done=true + if not allresults then break end end + end end - return done + end + return done end - - -local makewildcard = Cs( - (P("^")^0 * P("/") * P(-1) + P(-1)) /".*" - + (P("^")^0 * P("/") / "")^0 * (P("*")/".*" + P("-")/"%%-" + P(".")/"%%." + P("?")/"."+ P("\\")/"/" + P(1))^0 +local makewildcard=Cs( + (P("^")^0*P("/")*P(-1)+P(-1))/".*"+(P("^")^0*P("/")/"")^0*(P("*")/".*"+P("-")/"%%-"+P(".")/"%%."+P("?")/"."+P("\\")/"/"+P(1))^0 ) - function resolvers.wildcardpattern(pattern) - return lpegmatch(makewildcard,pattern) or pattern -end - -local function findwildcardfiles(filename,allresults,result) -- todo: remap: and lpeg - result = result or { } - local base = filebasename(filename) - local dirn = filedirname(filename) - local path = lower(lpegmatch(makewildcard,dirn) or dirn) - local name = lower(lpegmatch(makewildcard,base) or base) - local files, done = instance.files, false - if find(name,"%*") then - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - local hashname, hashtype = hash.name, hash.type - for kk, hh in next, files[hashname] do - if not find(kk,"^remap:") then - if find(lower(kk),name) then - if doit(path,hh,kk,hashname,hashtype,result,allresults) then done = true end - if done and not allresults then break end - end - end - end - end - else - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - local hashname, hashtype = hash.name, hash.type - if doit(path,files[hashname][bname],bname,hashname,hashtype,result,allresults) then done = true end + return lpegmatch(makewildcard,pattern) or pattern +end +local function findwildcardfiles(filename,allresults,result) + result=result or {} + local base=filebasename(filename) + local dirn=filedirname(filename) + local path=lower(lpegmatch(makewildcard,dirn) or dirn) + local name=lower(lpegmatch(makewildcard,base) or base) + local files,done=instance.files,false + if find(name,"%*") then + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + local hashname,hashtype=hash.name,hash.type + for kk,hh in next,files[hashname] do + if not find(kk,"^remap:") then + if find(lower(kk),name) then + if doit(path,hh,kk,hashname,hashtype,result,allresults) then done=true end if done and not allresults then break end + end end + end end - -- we can consider also searching the paths not in the database, but then - -- we end up with a messy search (all // in all path specs) - return result + else + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + local hashname,hashtype=hash.name,hash.type + if doit(path,files[hashname][bname],bname,hashname,hashtype,result,allresults) then done=true end + if done and not allresults then break end + end + end + return result end - function resolvers.findwildcardfiles(filename,result) - return findwildcardfiles(filename,true,result) + return findwildcardfiles(filename,true,result) end - function resolvers.findwildcardfile(filename) - return findwildcardfiles(filename,false)[1] or "" + return findwildcardfiles(filename,false)[1] or "" end - --- main user functions - function resolvers.automount() - -- implemented later end - function resolvers.load(option) - statistics.starttiming(instance) - identify_configuration_files() - load_configuration_files() - if option ~= "nofiles" then - load_databases() - resolvers.automount() - end - statistics.stoptiming(instance) - local files = instance.files - return files and next(files) and true + statistics.starttiming(instance) + identify_configuration_files() + load_configuration_files() + if option~="nofiles" then + load_databases() + resolvers.automount() + end + statistics.stoptiming(instance) + local files=instance.files + return files and next(files) and true end - function resolvers.loadtime() - return statistics.elapsedtime(instance) + return statistics.elapsedtime(instance) end - local function report(str) + if trace_locating then + report_resolving(str) + else + print(str) + end +end +function resolvers.dowithfilesandreport(command,files,...) + if files and #files>0 then if trace_locating then - report_resolving(str) -- has already verbose - else - print(str) + report('') end -end - -function resolvers.dowithfilesandreport(command, files, ...) -- will move - if files and #files > 0 then - if trace_locating then - report('') -- ? - end - if type(files) == "string" then - files = { files } - end - for f=1,#files do - local file = files[f] - local result = command(file,...) - if type(result) == 'string' then - report(result) - else - for i=1,#result do - report(result[i]) -- could be unpack - end - end + if type(files)=="string" then + files={ files } + end + for f=1,#files do + local file=files[f] + local result=command(file,...) + if type(result)=='string' then + report(result) + else + for i=1,#result do + report(result[i]) end + end end + end end - --- obsolete - --- resolvers.varvalue = resolvers.variable -- output the value of variable $STRING. --- resolvers.expandvar = resolvers.expansion -- output variable expansion of STRING. - -function resolvers.showpath(str) -- output search path for file type NAME - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) +function resolvers.showpath(str) + return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) end - -function resolvers.registerfile(files, name, path) - if files[name] then - if type(files[name]) == 'string' then - files[name] = { files[name], path } - else - files[name] = path - end +function resolvers.registerfile(files,name,path) + if files[name] then + if type(files[name])=='string' then + files[name]={ files[name],path } else - files[name] = path + files[name]=path end + else + files[name]=path + end end - function resolvers.dowithpath(name,func) - local pathlist = resolvers.expandedpathlist(name) - for i=1,#pathlist do - func("^"..resolvers.cleanpath(pathlist[i])) - end + local pathlist=resolvers.expandedpathlist(name) + for i=1,#pathlist do + func("^"..resolvers.cleanpath(pathlist[i])) + end end - function resolvers.dowithvariable(name,func) - func(expandedvariable(name)) + func(expandedvariable(name)) end - function resolvers.locateformat(name) - local engine = environment.ownmain or "luatex" - local barename = file.removesuffix(name) - local fullname = file.addsuffix(barename,"fmt") - local fmtname = caches.getfirstreadablefile(fullname,"formats",engine) or "" - if fmtname == "" then - fmtname = resolvers.findfile(fullname) - fmtname = resolvers.cleanpath(fmtname) - end - if fmtname ~= "" then - local barename = file.removesuffix(fmtname) - local luaname = file.addsuffix(barename,luasuffixes.lua) - local lucname = file.addsuffix(barename,luasuffixes.luc) - local luiname = file.addsuffix(barename,luasuffixes.lui) - if lfs.isfile(luiname) then - return barename, luiname - elseif lfs.isfile(lucname) then - return barename, lucname - elseif lfs.isfile(luaname) then - return barename, luaname - end - end - return nil, nil + local engine=environment.ownmain or "luatex" + local barename=file.removesuffix(name) + local fullname=file.addsuffix(barename,"fmt") + local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" + if fmtname=="" then + fmtname=resolvers.findfile(fullname) + fmtname=resolvers.cleanpath(fmtname) + end + if fmtname~="" then + local barename=file.removesuffix(fmtname) + local luaname=file.addsuffix(barename,luasuffixes.lua) + local lucname=file.addsuffix(barename,luasuffixes.luc) + local luiname=file.addsuffix(barename,luasuffixes.lui) + if lfs.isfile(luiname) then + return barename,luiname + elseif lfs.isfile(lucname) then + return barename,lucname + elseif lfs.isfile(luaname) then + return barename,luaname + end + end + return nil,nil end - function resolvers.booleanvariable(str,default) - local b = resolvers.expansion(str) - if b == "" then - return default - else - b = toboolean(b) - return (b == nil and default) or b - end -end - -function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead - local instance = resolvers.instance - local hashes = instance.hashes - for i=1,#hashes do - local hash = hashes[i] - local blobtype = hash.type - local blobpath = hash.name - if blobpath then - if before then - before(blobtype,blobpath,pattern) - end - local files = instance.files[blobpath] - local total, checked, done = 0, 0, 0 - if files then - for k,v in next, files do - total = total + 1 - if find(k,"^remap:") then - k = files[k] - v = k -- files[k] -- chained - end - if find(k,pattern) then - if type(v) == "string" then - checked = checked + 1 - if handle(blobtype,blobpath,v,k) then - done = done + 1 - end - else - checked = checked + #v - for i=1,#v do - if handle(blobtype,blobpath,v[i],k) then - done = done + 1 - end - end - end - end + local b=resolvers.expansion(str) + if b=="" then + return default + else + b=toboolean(b) + return (b==nil and default) or b + end +end +function resolvers.dowithfilesintree(pattern,handle,before,after) + local instance=resolvers.instance + local hashes=instance.hashes + for i=1,#hashes do + local hash=hashes[i] + local blobtype=hash.type + local blobpath=hash.name + if blobpath then + if before then + before(blobtype,blobpath,pattern) + end + local files=instance.files[blobpath] + local total,checked,done=0,0,0 + if files then + for k,v in next,files do + total=total+1 + if find(k,"^remap:") then + k=files[k] + v=k + end + if find(k,pattern) then + if type(v)=="string" then + checked=checked+1 + if handle(blobtype,blobpath,v,k) then + done=done+1 + end + else + checked=checked+#v + for i=1,#v do + if handle(blobtype,blobpath,v[i],k) then + done=done+1 end + end end - if after then - after(blobtype,blobpath,pattern,total,checked,done) - end + end end + end + if after then + after(blobtype,blobpath,pattern,total,checked,done) + end end + end end - -resolvers.obsolete = resolvers.obsolete or { } -local obsolete = resolvers.obsolete - -resolvers.find_file = resolvers.findfile obsolete.find_file = resolvers.findfile -resolvers.find_files = resolvers.findfiles obsolete.find_files = resolvers.findfiles +resolvers.obsolete=resolvers.obsolete or {} +local obsolete=resolvers.obsolete +resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile +resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-pre'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- It could be interesting to hook the resolver in the file --- opener so that unresolved prefixes travel around and we --- get more abstraction. - --- As we use this beforehand we will move this up in the chain --- of loading. - - -local resolvers = resolvers -local prefixes = utilities.storage.allocate() -resolvers.prefixes = prefixes +-- original size: 6430, stripped down to: 4219 -local cleanpath, findgivenfile, expansion = resolvers.cleanpath, resolvers.findgivenfile, resolvers.expansion -local getenv = resolvers.getenv -- we can probably also use resolvers.expansion -local P, S, R, C, Cs, lpegmatch = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cs, lpeg.match -local joinpath, basename, dirname = file.join, file.basename, file.dirname -local getmetatable, rawset, type = getmetatable, rawset, type - --- getenv = function(...) return resolvers.getenv(...) end -- needs checking (definitions changes later on) - -prefixes.environment = function(str) - return cleanpath(expansion(str)) -end - -prefixes.relative = function(str,n) -- lfs.isfile - if io.exists(str) then - -- nothing - elseif io.exists("./" .. str) then - str = "./" .. str - else - local p = "../" - for i=1,n or 2 do - if io.exists(p .. str) then - str = p .. str - break - else - p = p .. "../" - end - end +if not modules then modules={} end modules ['data-pre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers=resolvers +local prefixes=utilities.storage.allocate() +resolvers.prefixes=prefixes +local cleanpath,findgivenfile,expansion=resolvers.cleanpath,resolvers.findgivenfile,resolvers.expansion +local getenv=resolvers.getenv +local P,S,R,C,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.match +local joinpath,basename,dirname=file.join,file.basename,file.dirname +local getmetatable,rawset,type=getmetatable,rawset,type +prefixes.environment=function(str) + return cleanpath(expansion(str)) +end +prefixes.relative=function(str,n) + if io.exists(str) then + elseif io.exists("./"..str) then + str="./"..str + else + local p="../" + for i=1,n or 2 do + if io.exists(p..str) then + str=p..str + break + else + p=p.."../" + end end - return cleanpath(str) + end + return cleanpath(str) end - -prefixes.auto = function(str) - local fullname = prefixes.relative(str) - if not lfs.isfile(fullname) then - fullname = prefixes.locate(str) - end - return fullname +prefixes.auto=function(str) + local fullname=prefixes.relative(str) + if not lfs.isfile(fullname) then + fullname=prefixes.locate(str) + end + return fullname end - -prefixes.locate = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath((fullname ~= "" and fullname) or str) +prefixes.locate=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath((fullname~="" and fullname) or str) end - -prefixes.filename = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath(basename((fullname ~= "" and fullname) or str)) -- no cleanpath needed here +prefixes.filename=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(basename((fullname~="" and fullname) or str)) end - -prefixes.pathname = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath(dirname((fullname ~= "" and fullname) or str)) +prefixes.pathname=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(dirname((fullname~="" and fullname) or str)) end - -prefixes.selfautoloc = function(str) - return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) +prefixes.selfautoloc=function(str) + return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) end - -prefixes.selfautoparent = function(str) - return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) +prefixes.selfautoparent=function(str) + return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) end - -prefixes.selfautodir = function(str) - return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) +prefixes.selfautodir=function(str) + return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) end - -prefixes.home = function(str) - return cleanpath(joinpath(getenv('HOME'),str)) +prefixes.home=function(str) + return cleanpath(joinpath(getenv('HOME'),str)) end - local function toppath() - local inputstack = resolvers.inputstack -- dependency, actually the code should move but it's - if not inputstack then -- more convenient to keep it here - return "." - end - local pathname = dirname(inputstack[#inputstack] or "") - if pathname == "" then - return "." - else - return pathname - end -end - -resolvers.toppath = toppath - -prefixes.toppath = function(str) - return cleanpath(joinpath(toppath(),str)) -end - -prefixes.env = prefixes.environment -prefixes.rel = prefixes.relative -prefixes.loc = prefixes.locate -prefixes.kpse = prefixes.locate -prefixes.full = prefixes.locate -prefixes.file = prefixes.filename -prefixes.path = prefixes.pathname - + local inputstack=resolvers.inputstack + if not inputstack then + return "." + end + local pathname=dirname(inputstack[#inputstack] or "") + if pathname=="" then + return "." + else + return pathname + end +end +resolvers.toppath=toppath +prefixes.toppath=function(str) + return cleanpath(joinpath(toppath(),str)) +end +prefixes.env=prefixes.environment +prefixes.rel=prefixes.relative +prefixes.loc=prefixes.locate +prefixes.kpse=prefixes.locate +prefixes.full=prefixes.locate +prefixes.file=prefixes.filename +prefixes.path=prefixes.pathname function resolvers.allprefixes(separator) - local all = table.sortedkeys(prefixes) - if separator then - for i=1,#all do - all[i] = all[i] .. ":" - end + local all=table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i]=all[i]..":" end - return all + end + return all end - local function _resolve_(method,target) - local action = prefixes[method] - if action then - return action(target) - else - return method .. ":" .. target - end -end - -local resolved, abstract = { }, { } - + local action=prefixes[method] + if action then + return action(target) + else + return method..":"..target + end +end +local resolved,abstract={},{} function resolvers.resetresolve(str) - resolved, abstract = { }, { } + resolved,abstract={},{} end - --- todo: use an lpeg (see data-lua for !! / stripper) - --- local function resolve(str) -- use schemes, this one is then for the commandline only --- if type(str) == "table" then --- local t = { } --- for i=1,#str do --- t[i] = resolve(str[i]) --- end --- return t --- else --- local res = resolved[str] --- if not res then --- res = gsub(str,"([a-z][a-z]+):([^ \"\';,]*)",_resolve_) -- home:xx;selfautoparent:xx; etc (comma added) --- resolved[str] = res --- abstract[res] = str --- end --- return res --- end --- end - --- home:xx;selfautoparent:xx; - -local pattern = Cs((C(R("az")^2) * P(":") * C((1-S(" \"\';,"))^1) / _resolve_ + P(1))^0) - -local function resolve(str) -- use schemes, this one is then for the commandline only - if type(str) == "table" then - local t = { } - for i=1,#str do - t[i] = resolve(str[i]) - end - return t - else - local res = resolved[str] - if not res then - res = lpegmatch(pattern,str) - resolved[str] = res - abstract[res] = str - end - return res +local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) +local function resolve(str) + if type(str)=="table" then + local t={} + for i=1,#str do + t[i]=resolve(str[i]) + end + return t + else + local res=resolved[str] + if not res then + res=lpegmatch(pattern,str) + resolved[str]=res + abstract[res]=str end + return res + end end - local function unresolve(str) - return abstract[str] or str + return abstract[str] or str end - -resolvers.resolve = resolve -resolvers.unresolve = unresolve - -if type(os.uname) == "function" then - - for k, v in next, os.uname() do - if not prefixes[k] then - prefixes[k] = function() return v end - end +resolvers.resolve=resolve +resolvers.unresolve=unresolve +if type(os.uname)=="function" then + for k,v in next,os.uname() do + if not prefixes[k] then + prefixes[k]=function() return v end end - + end end - -if os.type == "unix" then - - -- We need to distringuish between a prefix and something else : so we - -- have a special repath variant for linux. Also, when a new prefix is - -- defined, we need to remake the matcher. - - local pattern - - local function makepattern(t,k,v) - if t then - rawset(t,k,v) - end - local colon = P(":") - for k, v in table.sortedpairs(prefixes) do - if p then - p = P(k) + p - else - p = P(k) - end - end - pattern = Cs((p * colon + colon/";" + P(1))^0) - end - - makepattern() - - getmetatable(prefixes).__newindex = makepattern - - function resolvers.repath(str) - return lpegmatch(pattern,str) - end - -else -- already the default: - - function resolvers.repath(str) - return str +if os.type=="unix" then + local pattern + local function makepattern(t,k,v) + if t then + rawset(t,k,v) + end + local colon=P(":") + for k,v in table.sortedpairs(prefixes) do + if p then + p=P(k)+p + else + p=P(k) + end end - + pattern=Cs((p*colon+colon/";"+P(1))^0) + end + makepattern() + getmetatable(prefixes).__newindex=makepattern + function resolvers.repath(str) + return lpegmatch(pattern,str) + end +else + function resolvers.repath(str) + return str + end end @@ -16917,172 +13133,153 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-inp'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local allocate = utilities.storage.allocate -local resolvers = resolvers - -local methodhandler = resolvers.methodhandler -local registermethod = resolvers.registermethod - -local finders = allocate { helpers = { }, notfound = function() end } -local openers = allocate { helpers = { }, notfound = function() end } -local loaders = allocate { helpers = { }, notfound = function() return false, nil, 0 end } +-- original size: 910, stripped down to: 823 -registermethod("finders", finders, "uri") -registermethod("openers", openers, "uri") -registermethod("loaders", loaders, "uri") - -resolvers.finders = finders -resolvers.openers = openers -resolvers.loaders = loaders +if not modules then modules={} end modules ['data-inp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local methodhandler=resolvers.methodhandler +local registermethod=resolvers.registermethod +local finders=allocate { helpers={},notfound=function() end } +local openers=allocate { helpers={},notfound=function() end } +local loaders=allocate { helpers={},notfound=function() return false,nil,0 end } +registermethod("finders",finders,"uri") +registermethod("openers",openers,"uri") +registermethod("loaders",loaders,"uri") +resolvers.finders=finders +resolvers.openers=openers +resolvers.loaders=loaders end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-out'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local allocate = utilities.storage.allocate -local resolvers = resolvers +-- original size: 530, stripped down to: 475 -local registermethod = resolvers.registermethod - -local savers = allocate { helpers = { } } - -resolvers.savers = savers - -registermethod("savers", savers, "uri") +if not modules then modules={} end modules ['data-out']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registermethod=resolvers.registermethod +local savers=allocate { helpers={} } +resolvers.savers=savers +registermethod("savers",savers,"uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-fil'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_files = logs.reporter("resolvers","files") - -local resolvers = resolvers - -local finders, openers, loaders, savers = resolvers.finders, resolvers.openers, resolvers.loaders, resolvers.savers -local locators, hashers, generators, concatinators = resolvers.locators, resolvers.hashers, resolvers.generators, resolvers.concatinators - -local checkgarbage = utilities.garbagecollector and utilities.garbagecollector.check +-- original size: 3818, stripped down to: 3248 +if not modules then modules={} end modules ['data-fil']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_files=logs.reporter("resolvers","files") +local resolvers=resolvers +local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers +local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check function locators.file(specification) - local name = specification.filename - local realname = resolvers.resolve(name) -- no shortcut - if realname and realname ~= '' and lfs.isdir(realname) then - if trace_locating then - report_files("file locator '%s' found as '%s'",name,realname) - end - resolvers.appendhash('file',name,true) -- cache - elseif trace_locating then - report_files("file locator '%s' not found",name) + local name=specification.filename + local realname=resolvers.resolve(name) + if realname and realname~='' and lfs.isdir(realname) then + if trace_locating then + report_files("file locator '%s' found as '%s'",name,realname) end + resolvers.appendhash('file',name,true) + elseif trace_locating then + report_files("file locator '%s' not found",name) + end end - function hashers.file(specification) - local name = specification.filename - local content = caches.loadcontent(name,'files') - resolvers.registerfilehash(name,content,content==nil) + local name=specification.filename + local content=caches.loadcontent(name,'files') + resolvers.registerfilehash(name,content,content==nil) end - function generators.file(specification) - local path = specification.filename - local content = resolvers.scanfiles(path,false,true) -- scan once - resolvers.registerfilehash(path,content,true) + local path=specification.filename + local content=resolvers.scanfiles(path,false,true) + resolvers.registerfilehash(path,content,true) end - -concatinators.file = file.join - +concatinators.file=file.join function finders.file(specification,filetype) - local filename = specification.filename - local foundname = resolvers.findfile(filename,filetype) - if foundname and foundname ~= "" then - if trace_locating then - report_files("file finder: '%s' found",filename) - end - return foundname - else - if trace_locating then - report_files("file finder: %s' not found",filename) - end - return finders.notfound() + local filename=specification.filename + local foundname=resolvers.findfile(filename,filetype) + if foundname and foundname~="" then + if trace_locating then + report_files("file finder: '%s' found",filename) + end + return foundname + else + if trace_locating then + report_files("file finder: %s' not found",filename) end + return finders.notfound() + end end - --- The default textopener will be overloaded later on. - function openers.helpers.textopener(tag,filename,f) - return { - reader = function() return f:read () end, - close = function() logs.show_close(filename) return f:close() end, - } + return { + reader=function() return f:read () end, + close=function() logs.show_close(filename) return f:close() end, + } end - function openers.file(specification,filetype) - local filename = specification.filename - if filename and filename ~= "" then - local f = io.open(filename,"r") - if f then - if trace_locating then - report_files("file opener, '%s' opened",filename) - end - return openers.helpers.textopener("file",filename,f) - end - end - if trace_locating then - report_files("file opener, '%s' not found",filename) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"r") + if f then + if trace_locating then + report_files("file opener, '%s' opened",filename) + end + return openers.helpers.textopener("file",filename,f) end - return openers.notfound() + end + if trace_locating then + report_files("file opener, '%s' not found",filename) + end + return openers.notfound() end - function loaders.file(specification,filetype) - local filename = specification.filename - if filename and filename ~= "" then - local f = io.open(filename,"rb") - if f then - logs.show_load(filename) - if trace_locating then - report_files("file loader, '%s' loaded",filename) - end - local s = f:read("*a") -- io.readall(f) is faster but we never have large files here - if checkgarbage then - checkgarbage(#s) - end - f:close() - if s then - return true, s, #s - end - end - end - if trace_locating then - report_files("file loader, '%s' not found",filename) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"rb") + if f then + logs.show_load(filename) + if trace_locating then + report_files("file loader, '%s' loaded",filename) + end + local s=f:read("*a") + if checkgarbage then + checkgarbage(#s) + end + f:close() + if s then + return true,s,#s + end end - return loaders.notfound() + end + if trace_locating then + report_files("file loader, '%s' not found",filename) + end + return loaders.notfound() end @@ -17090,129 +13287,113 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-con'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local format, lower, gsub = string.format, string.lower, string.gsub - -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) -local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) - - - -containers = containers or { } -local containers = containers -containers.usecache = true - -local report_containers = logs.reporter("resolvers","containers") +-- original size: 4651, stripped down to: 3330 +if not modules then modules={} end modules ['data-con']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub=string.format,string.lower,string.gsub +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) +local trace_storage=false trackers.register("resolvers.storage",function(v) trace_storage=v end) +containers=containers or {} +local containers=containers +containers.usecache=true +local report_containers=logs.reporter("resolvers","containers") local function report(container,tag,name) - if trace_cache or trace_containers then - report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') - end -end - -local allocated = { } - -local mt = { - __index = function(t,k) - if k == "writable" then - local writable = caches.getwritablepath(t.category,t.subcategory) or { "." } - t.writable = writable - return writable - elseif k == "readables" then - local readables = caches.getreadablepaths(t.category,t.subcategory) or { "." } - t.readables = readables - return readables - end - end, - __storage__ = true + if trace_cache or trace_containers then + report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') + end +end +local allocated={} +local mt={ + __index=function(t,k) + if k=="writable" then + local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + t.writable=writable + return writable + elseif k=="readables" then + local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + t.readables=readables + return readables + end + end, + __storage__=true } - -function containers.define(category, subcategory, version, enabled) - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or math.pi, -- after all, this is TeX - trace = false, - -- writable = caches.getwritablepath and caches.getwritablepath (category,subcategory) or { "." }, - -- readables = caches.getreadablepaths and caches.getreadablepaths(category,subcategory) or { "." }, - } - setmetatable(s,mt) - c[subcategory] = s - end - return s +function containers.define(category,subcategory,version,enabled) + if category and subcategory then + local c=allocated[category] + if not c then + c={} + allocated[category]=c end -end - -function containers.is_usable(container, name) - return container.enabled and caches and caches.is_writable(container.writable, name) -end - -function containers.is_valid(container, name) - if name and name ~= "" then - local storage = container.storage[name] - return storage and storage.cache_version == container.version - else - return false + local s=c[subcategory] + if not s then + s={ + category=category, + subcategory=subcategory, + storage={}, + enabled=enabled, + version=version or math.pi, + trace=false, + } + setmetatable(s,mt) + c[subcategory]=s end + return s + end end - -function containers.read(container,name) - local storage = container.storage - local stored = storage[name] - if not stored and container.enabled and caches and containers.usecache then - stored = caches.loaddata(container.readables,name) - if stored and stored.cache_version == container.version then - report(container,"loaded",name) - else - stored = nil - end - storage[name] = stored - elseif stored then - report(container,"reusing",name) - end - return stored +function containers.is_usable(container,name) + return container.enabled and caches and caches.is_writable(container.writable,name) end - -function containers.write(container, name, data) - if data then - data.cache_version = container.version - if container.enabled and caches then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.writable, name, data) - report(container,"saved",name) - data.unique, data.shared = unique, shared - end - report(container,"stored",name) - container.storage[name] = data - end - return data +function containers.is_valid(container,name) + if name and name~="" then + local storage=container.storage[name] + return storage and storage.cache_version==container.version + else + return false + end +end +function containers.read(container,name) + local storage=container.storage + local stored=storage[name] + if not stored and container.enabled and caches and containers.usecache then + stored=caches.loaddata(container.readables,name) + if stored and stored.cache_version==container.version then + report(container,"loaded",name) + else + stored=nil + end + storage[name]=stored + elseif stored then + report(container,"reusing",name) + end + return stored +end +function containers.write(container,name,data) + if data then + data.cache_version=container.version + if container.enabled and caches then + local unique,shared=data.unique,data.shared + data.unique,data.shared=nil,nil + caches.savedata(container.writable,name,data) + report(container,"saved",name) + data.unique,data.shared=unique,shared + end + report(container,"stored",name) + container.storage[name]=data + end + return data end - function containers.content(container,name) - return container.storage[name] + return container.storage[name] end - function containers.cleanname(name) - return (gsub(lower(name),"[^%w%d]+","-")) + return (gsub(lower(name),"[^%w%d]+","-")) end @@ -17220,106 +13401,88 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-use'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_mounts = logs.reporter("resolvers","mounts") - -local resolvers = resolvers - --- we will make a better format, maybe something xml or just text or lua - -resolvers.automounted = resolvers.automounted or { } +-- original size: 3913, stripped down to: 2998 +if not modules then modules={} end modules ['data-use']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_mounts=logs.reporter("resolvers","mounts") +local resolvers=resolvers +resolvers.automounted=resolvers.automounted or {} function resolvers.automount(usecache) - local mountpaths = resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths == 0) and usecache then - mountpaths = caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths > 0 then - statistics.starttiming(resolvers.instance) - for k=1,#mountpaths do - local root = mountpaths[k] - local f = io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then -- or %W - -- skip - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %s",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() + local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) + if (not mountpaths or #mountpaths==0) and usecache then + mountpaths=caches.getreadablepaths("mount") + end + if mountpaths and #mountpaths>0 then + statistics.starttiming(resolvers.instance) + for k=1,#mountpaths do + local root=mountpaths[k] + local f=io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if find(line,"^[%%#%-]") then + elseif find(line,"^zip://") then + if trace_locating then + report_mounts("mounting %s",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) end + end end - statistics.stoptiming(resolvers.instance) - end -end - --- status info - -statistics.register("used config file", function() return caches.configfiles() end) -statistics.register("used cache path", function() return caches.usedpaths() end) - --- experiment (code will move) - -function statistics.savefmtstatus(texname,formatbanner,sourcefile) -- texname == formatname - local enginebanner = status.list().banner - if formatbanner and enginebanner and sourcefile then - local luvname = file.replacesuffix(texname,"luv") -- utilities.lua.suffixes.luv - local luvdata = { - enginebanner = enginebanner, - formatbanner = formatbanner, - sourcehash = md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), - sourcefile = sourcefile, - } - io.savedata(luvname,table.serialize(luvdata,true)) + f:close() + end end + statistics.stoptiming(resolvers.instance) + end +end +statistics.register("used config file",function() return caches.configfiles() end) +statistics.register("used cache path",function() return caches.usedpaths() end) +function statistics.savefmtstatus(texname,formatbanner,sourcefile) + local enginebanner=status.list().banner + if formatbanner and enginebanner and sourcefile then + local luvname=file.replacesuffix(texname,"luv") + local luvdata={ + enginebanner=enginebanner, + formatbanner=formatbanner, + sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcefile=sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) + end end - --- todo: check this at startup and return (say) 999 as signal that the run --- was aborted due to a wrong format in which case mtx-context can trigger --- a remake - function statistics.checkfmtstatus(texname) - local enginebanner = status.list().banner - if enginebanner and texname then - local luvname = file.replacesuffix(texname,"luv") -- utilities.lua.suffixes.luv - if lfs.isfile(luvname) then - local luv = dofile(luvname) - if luv and luv.sourcefile then - local sourcehash = md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") - local luvbanner = luv.enginebanner or "?" - if luvbanner ~= enginebanner then - return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) - end - local luvhash = luv.sourcehash or "?" - if luvhash ~= sourcehash then - return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) - end - else - return "invalid status file" - end - else - return "missing status file" - end + local enginebanner=status.list().banner + if enginebanner and texname then + local luvname=file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv=dofile(luvname) + if luv and luv.sourcefile then + local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local luvbanner=luv.enginebanner or "?" + if luvbanner~=enginebanner then + return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) + end + local luvhash=luv.sourcehash or "?" + if luvhash~=sourcehash then + return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) + end + else + return "invalid status file" + end + else + return "missing status file" end - return true + end + return true end @@ -17327,259 +13490,233 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-zip'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- partly redone .. needs testing - -local format, find, match = string.format, string.find, string.match +-- original size: 8537, stripped down to: 6805 -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_zip = logs.reporter("resolvers","zip") - - - -local resolvers = resolvers - -zip = zip or { } -local zip = zip - -zip.archives = zip.archives or { } -local archives = zip.archives - -zip.registeredfiles = zip.registeredfiles or { } -local registeredfiles = zip.registeredfiles - -local limited = false - -directives.register("system.inputmode", function(v) - if not limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - zip.open = i_limiter.protect(zip.open) - limited = true - end - end +if not modules then modules={} end modules ['data-zip']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,find,match=string.format,string.find,string.match +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_zip=logs.reporter("resolvers","zip") +local resolvers=resolvers +zip=zip or {} +local zip=zip +zip.archives=zip.archives or {} +local archives=zip.archives +zip.registeredfiles=zip.registeredfiles or {} +local registeredfiles=zip.registeredfiles +local limited=false +directives.register("system.inputmode",function(v) + if not limited then + local i_limiter=io.i_limiter(v) + if i_limiter then + zip.open=i_limiter.protect(zip.open) + limited=true + end + end end) - -local function validzip(str) -- todo: use url splitter - if not find(str,"^zip://") then - return "zip:///" .. str - else - return str - end +local function validzip(str) + if not find(str,"^zip://") then + return "zip:///"..str + else + return str + end end - function zip.openarchive(name) - if not name or name == "" then - return nil - else - local arch = archives[name] - if not arch then - local full = resolvers.findfile(name) or "" - arch = (full ~= "" and zip.open(full)) or false - archives[name] = arch - end - return arch + if not name or name=="" then + return nil + else + local arch=archives[name] + if not arch then + local full=resolvers.findfile(name) or "" + arch=(full~="" and zip.open(full)) or false + archives[name]=arch end + return arch + end end - function zip.closearchive(name) - if not name or (name == "" and archives[name]) then - zip.close(archives[name]) - archives[name] = nil - end + if not name or (name=="" and archives[name]) then + zip.close(archives[name]) + archives[name]=nil + end end - function resolvers.locators.zip(specification) - local archive = specification.filename - local zipfile = archive and archive ~= "" and zip.openarchive(archive) -- tricky, could be in to be initialized tree - if trace_locating then - if zipfile then - report_zip("locator, archive '%s' found",archive) - else - report_zip("locator, archive '%s' not found",archive) - end + local archive=specification.filename + local zipfile=archive and archive~="" and zip.openarchive(archive) + if trace_locating then + if zipfile then + report_zip("locator, archive '%s' found",archive) + else + report_zip("locator, archive '%s' not found",archive) end + end end - function resolvers.hashers.zip(specification) - local archive = specification.filename - if trace_locating then - report_zip("loading file '%s'",archive) - end - resolvers.usezipfile(specification.original) -end - -function resolvers.concatinators.zip(zipfile,path,name) -- ok ? - if not path or path == "" then - return format('%s?name=%s',zipfile,name) - else - return format('%s?name=%s/%s',zipfile,path,name) - end + local archive=specification.filename + if trace_locating then + report_zip("loading file '%s'",archive) + end + resolvers.usezipfile(specification.original) +end +function resolvers.concatinators.zip(zipfile,path,name) + if not path or path=="" then + return format('%s?name=%s',zipfile,name) + else + return format('%s?name=%s/%s',zipfile,path,name) + end end - function resolvers.finders.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("finder, archive '%s' found",archive) - end - local dfile = zfile:open(queryname) - if dfile then - dfile = zfile:close() - if trace_locating then - report_zip("finder, file '%s' found",queryname) - end - return specification.original - elseif trace_locating then - report_zip("finder, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("finder, unknown archive '%s'",archive) - end + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("finder, archive '%s' found",archive) end + local dfile=zfile:open(queryname) + if dfile then + dfile=zfile:close() + if trace_locating then + report_zip("finder, file '%s' found",queryname) + end + return specification.original + elseif trace_locating then + report_zip("finder, file '%s' not found",queryname) + end + elseif trace_locating then + report_zip("finder, unknown archive '%s'",archive) + end end - if trace_locating then - report_zip("finder, '%s' not found",original) - end - return resolvers.finders.notfound() + end + if trace_locating then + report_zip("finder, '%s' not found",original) + end + return resolvers.finders.notfound() end - function resolvers.openers.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("opener, archive '%s' opened",archive) - end - local dfile = zfile:open(queryname) - if dfile then - if trace_locating then - report_zip("opener, file '%s' found",queryname) - end - return resolvers.openers.helpers.textopener('zip',original,dfile) - elseif trace_locating then - report_zip("opener, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("opener, unknown archive '%s'",archive) - end + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("opener, archive '%s' opened",archive) end + local dfile=zfile:open(queryname) + if dfile then + if trace_locating then + report_zip("opener, file '%s' found",queryname) + end + return resolvers.openers.helpers.textopener('zip',original,dfile) + elseif trace_locating then + report_zip("opener, file '%s' not found",queryname) + end + elseif trace_locating then + report_zip("opener, unknown archive '%s'",archive) + end end - if trace_locating then - report_zip("opener, '%s' not found",original) - end - return resolvers.openers.notfound() + end + if trace_locating then + report_zip("opener, '%s' not found",original) + end + return resolvers.openers.notfound() end - function resolvers.loaders.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("loader, archive '%s' opened",archive) - end - local dfile = zfile:open(queryname) - if dfile then - logs.show_load(original) - if trace_locating then - report_zip("loader, file '%s' loaded",original) - end - local s = dfile:read("*all") - dfile:close() - return true, s, #s - elseif trace_locating then - report_zip("loader, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("loader, unknown archive '%s'",archive) - end + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("loader, archive '%s' opened",archive) end + local dfile=zfile:open(queryname) + if dfile then + logs.show_load(original) + if trace_locating then + report_zip("loader, file '%s' loaded",original) + end + local s=dfile:read("*all") + dfile:close() + return true,s,#s + elseif trace_locating then + report_zip("loader, file '%s' not found",queryname) + end + elseif trace_locating then + report_zip("loader, unknown archive '%s'",archive) + end end - if trace_locating then - report_zip("loader, '%s' not found",original) - end - return resolvers.openers.notfound() + end + if trace_locating then + report_zip("loader, '%s' not found",original) + end + return resolvers.openers.notfound() end - --- zip:///somefile.zip --- zip:///somefile.zip?tree=texmf-local -> mount - function resolvers.usezipfile(archive) - local specification = resolvers.splitmethod(archive) -- to be sure - local archive = specification.filename - if archive and not registeredfiles[archive] then - local z = zip.openarchive(archive) - if z then - local instance = resolvers.instance - local tree = url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering, registering archive '%s'",archive) - end - statistics.starttiming(instance) - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) -- resets hashes too - registeredfiles[archive] = z - instance.files[archive] = resolvers.registerzipfile(z,tree) - statistics.stoptiming(instance) - elseif trace_locating then - report_zip("registering, unknown archive '%s'",archive) - end + local specification=resolvers.splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=zip.openarchive(archive) + if z then + local instance=resolvers.instance + local tree=url.query(specification.query).tree or "" + if trace_locating then + report_zip("registering, registering archive '%s'",archive) + end + statistics.starttiming(instance) + resolvers.prependhash('zip',archive) + resolvers.extendtexmfvariable(archive) + registeredfiles[archive]=z + instance.files[archive]=resolvers.registerzipfile(z,tree) + statistics.stoptiming(instance) elseif trace_locating then - report_zip("registering, '%s' not found",archive) + report_zip("registering, unknown archive '%s'",archive) end + elseif trace_locating then + report_zip("registering, '%s' not found",archive) + end end - function resolvers.registerzipfile(z,tree) - local files, filter = { }, "" - if tree == "" then - filter = "^(.+)/(.-)$" + local files,filter={},"" + if tree=="" then + filter="^(.+)/(.-)$" + else + filter=format("^%s/(.+)/(.-)$",tree) + end + if trace_locating then + report_zip("registering, using filter '%s'",filter) + end + local register,n=resolvers.registerfile,0 + for i in z:files() do + local path,name=match(i.filename,filter) + if path then + if name and name~='' then + register(files,name,path) + n=n+1 + else + end else - filter = format("^%s/(.+)/(.-)$",tree) - end - if trace_locating then - report_zip("registering, using filter '%s'",filter) - end - local register, n = resolvers.registerfile, 0 - for i in z:files() do - local path, name = match(i.filename,filter) - if path then - if name and name ~= '' then - register(files, name, path) - n = n + 1 - else - -- directory - end - else - register(files, i.filename, '') - n = n + 1 - end + register(files,i.filename,'') + n=n+1 end - report_zip("registering, %s files registered",n) - return files + end + report_zip("registering, %s files registered",n) + return files end @@ -17587,286 +13724,244 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tre'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- \input tree://oeps1/**/oeps.tex - -local find, gsub, format = string.find, string.gsub, string.format - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_trees = logs.reporter("resolvers","trees") - -local resolvers = resolvers - -local done, found, notfound = { }, { }, resolvers.finders.notfound +-- original size: 2514, stripped down to: 2080 +if not modules then modules={} end modules ['data-tre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,gsub,format=string.find,string.gsub,string.format +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_trees=logs.reporter("resolvers","trees") +local resolvers=resolvers +local done,found,notfound={},{},resolvers.finders.notfound function resolvers.finders.tree(specification) - local spec = specification.filename - local fnd = found[spec] - if fnd == nil then - if spec ~= "" then - local path, name = file.dirname(spec), file.basename(spec) - if path == "" then path = "." end - local hash = done[path] - if not hash then - local pattern = path .. "/*" -- we will use the proper splitter - hash = dir.glob(pattern) - done[path] = hash - end - local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$" - for k=1,#hash do - local v = hash[k] - if find(v,pattern) then - found[spec] = v - return v - end - end + local spec=specification.filename + local fnd=found[spec] + if fnd==nil then + if spec~="" then + local path,name=file.dirname(spec),file.basename(spec) + if path=="" then path="." end + local hash=done[path] + if not hash then + local pattern=path.."/*" + hash=dir.glob(pattern) + done[path]=hash + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for k=1,#hash do + local v=hash[k] + if find(v,pattern) then + found[spec]=v + return v end - fnd = notfound() -- false - found[spec] = fnd + end end - return fnd + fnd=notfound() + found[spec]=fnd + end + return fnd end - function resolvers.locators.tree(specification) - local name = specification.filename - local realname = resolvers.resolve(name) -- no shortcut - if realname and realname ~= '' and lfs.isdir(realname) then - if trace_locating then - report_trees("locator '%s' found",realname) - end - resolvers.appendhash('tree',name,false) -- don't cache - elseif trace_locating then - report_trees("locator '%s' not found",name) + local name=specification.filename + local realname=resolvers.resolve(name) + if realname and realname~='' and lfs.isdir(realname) then + if trace_locating then + report_trees("locator '%s' found",realname) end + resolvers.appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator '%s' not found",name) + end end - function resolvers.hashers.tree(specification) - local name = specification.filename - if trace_locating then - report_trees("analysing '%s'",name) - end - resolvers.methodhandler("hashers",name) - - resolvers.generators.file(specification) + local name=specification.filename + if trace_locating then + report_trees("analysing '%s'",name) + end + resolvers.methodhandler("hashers",name) + resolvers.generators.file(specification) end - -resolvers.concatinators.tree = resolvers.concatinators.file -resolvers.generators.tree = resolvers.generators.file -resolvers.openers.tree = resolvers.openers.file -resolvers.loaders.tree = resolvers.loaders.file +resolvers.concatinators.tree=resolvers.concatinators.file +resolvers.generators.tree=resolvers.generators.file +resolvers.openers.tree=resolvers.openers.file +resolvers.loaders.tree=resolvers.loaders.file end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-sch'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local load = load -local gsub, concat, format = string.gsub, table.concat, string.format -local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders - -local trace_schemes = false trackers.register("resolvers.schemes",function(v) trace_schemes = v end) -local report_schemes = logs.reporter("resolvers","schemes") - -local http = require("socket.http") -local ltn12 = require("ltn12") - -local resolvers = resolvers -local schemes = resolvers.schemes or { } -resolvers.schemes = schemes - -local cleaners = { } -schemes.cleaners = cleaners - -local threshold = 24 * 60 * 60 - -directives.register("schemes.threshold", function(v) threshold = tonumber(v) or threshold end) +-- original size: 6218, stripped down to: 5165 +if not modules then modules={} end modules ['data-sch']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local load=load +local gsub,concat,format=string.gsub,table.concat,string.format +local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) +local report_schemes=logs.reporter("resolvers","schemes") +local http=require("socket.http") +local ltn12=require("ltn12") +local resolvers=resolvers +local schemes=resolvers.schemes or {} +resolvers.schemes=schemes +local cleaners={} +schemes.cleaners=cleaners +local threshold=24*60*60 +directives.register("schemes.threshold",function(v) threshold=tonumber(v) or threshold end) function cleaners.none(specification) - return specification.original + return specification.original end - function cleaners.strip(specification) - return (gsub(specification.original,"[^%a%d%.]+","-")) -- so we keep periods + return (gsub(specification.original,"[^%a%d%.]+","-")) end - function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) end - -local cleaner = cleaners.strip - -directives.register("schemes.cleanmethod", function(v) cleaner = cleaners[v] or cleaners.strip end) - +local cleaner=cleaners.strip +directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) function resolvers.schemes.cleanname(specification) - local hash = cleaner(specification) - if trace_schemes then - report_schemes("hashing %s to %s",specification.original,hash) - end - return hash + local hash=cleaner(specification) + if trace_schemes then + report_schemes("hashing %s to %s",specification.original,hash) + end + return hash end - -local cached, loaded, reused, thresholds, handlers = { }, { }, { }, { }, { } - -local function runcurl(name,cachename) -- we use sockets instead or the curl library when possible - local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name - os.spawn(command) +local cached,loaded,reused,thresholds,handlers={},{},{},{},{} +local function runcurl(name,cachename) + local command="curl --silent --create-dirs --output "..cachename.." "..name + os.spawn(command) end - local function fetch(specification) - local original = specification.original - local scheme = specification.scheme - local cleanname = schemes.cleanname(specification) - local cachename = caches.setfirstwritablefile(cleanname,"schemes") - if not cached[original] then - statistics.starttiming(schemes) - if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > (thresholds[protocol] or threshold)) then - cached[original] = cachename - local handler = handlers[scheme] - if handler then - if trace_schemes then - report_schemes("fetching '%s', protocol '%s', method 'built-in'",original,scheme) - end - logs.flush() - handler(specification,cachename) - else - if trace_schemes then - report_schemes("fetching '%s', protocol '%s', method 'curl'",original,scheme) - end - logs.flush() - runcurl(original,cachename) - end - end - if io.exists(cachename) then - cached[original] = cachename - if trace_schemes then - report_schemes("using cached '%s', protocol '%s', cachename '%s'",original,scheme,cachename) - end - else - cached[original] = "" - if trace_schemes then - report_schemes("using missing '%s', protocol '%s'",original,scheme) - end + local original=specification.original + local scheme=specification.scheme + local cleanname=schemes.cleanname(specification) + local cachename=caches.setfirstwritablefile(cleanname,"schemes") + if not cached[original] then + statistics.starttiming(schemes) + if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification)>(thresholds[protocol] or threshold)) then + cached[original]=cachename + local handler=handlers[scheme] + if handler then + if trace_schemes then + report_schemes("fetching '%s', protocol '%s', method 'built-in'",original,scheme) end - loaded[scheme] = loaded[scheme] + 1 - statistics.stoptiming(schemes) - else + logs.flush() + handler(specification,cachename) + else if trace_schemes then - report_schemes("reusing '%s', protocol '%s'",original,scheme) + report_schemes("fetching '%s', protocol '%s', method 'curl'",original,scheme) end - reused[scheme] = reused[scheme] + 1 + logs.flush() + runcurl(original,cachename) + end + end + if io.exists(cachename) then + cached[original]=cachename + if trace_schemes then + report_schemes("using cached '%s', protocol '%s', cachename '%s'",original,scheme,cachename) + end + else + cached[original]="" + if trace_schemes then + report_schemes("using missing '%s', protocol '%s'",original,scheme) + end + end + loaded[scheme]=loaded[scheme]+1 + statistics.stoptiming(schemes) + else + if trace_schemes then + report_schemes("reusing '%s', protocol '%s'",original,scheme) end - return cached[original] + reused[scheme]=reused[scheme]+1 + end + return cached[original] end - local function finder(specification,filetype) - return resolvers.methodhandler("finders",fetch(specification),filetype) + return resolvers.methodhandler("finders",fetch(specification),filetype) end - -local opener = openers.file -local loader = loaders.file - +local opener=openers.file +local loader=loaders.file local function install(scheme,handler,newthreshold) - handlers [scheme] = handler - loaded [scheme] = 0 - reused [scheme] = 0 - finders [scheme] = finder - openers [scheme] = opener - loaders [scheme] = loader - thresholds[scheme] = newthreshold or threshold -end - -schemes.install = install - + handlers [scheme]=handler + loaded [scheme]=0 + reused [scheme]=0 + finders [scheme]=finder + openers [scheme]=opener + loaders [scheme]=loader + thresholds[scheme]=newthreshold or threshold +end +schemes.install=install local function http_handler(specification,cachename) - local tempname = cachename .. ".tmp" - local f = io.open(tempname,"wb") - local status, message = http.request { - url = specification.original, - sink = ltn12.sink.file(f) - } - if not status then - os.remove(tempname) - else - os.remove(cachename) - os.rename(tempname,cachename) - end - return cachename + local tempname=cachename..".tmp" + local f=io.open(tempname,"wb") + local status,message=http.request { + url=specification.original, + sink=ltn12.sink.file(f) + } + if not status then + os.remove(tempname) + else + os.remove(cachename) + os.rename(tempname,cachename) + end + return cachename end - install('http',http_handler) -install('https') -- see pod +install('https') install('ftp') - -statistics.register("scheme handling time", function() - local l, r, nl, nr = { }, { }, 0, 0 - for k, v in table.sortedhash(loaded) do - if v > 0 then - nl = nl + 1 - l[nl] = k .. ":" .. v - end - end - for k, v in table.sortedhash(reused) do - if v > 0 then - nr = nr + 1 - r[nr] = k .. ":" .. v - end - end - local n = nl + nr - if n > 0 then - l = nl > 0 and concat(l) or "none" - r = nr > 0 and concat(r) or "none" - return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes), n, threshold, l, r) - else - return nil - end +statistics.register("scheme handling time",function() + local l,r,nl,nr={},{},0,0 + for k,v in table.sortedhash(loaded) do + if v>0 then + nl=nl+1 + l[nl]=k..":"..v + end + end + for k,v in table.sortedhash(reused) do + if v>0 then + nr=nr+1 + r[nr]=k..":"..v + end + end + local n=nl+nr + if n>0 then + l=nl>0 and concat(l) or "none" + r=nr>0 and concat(r) or "none" + return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", + statistics.elapsedtime(schemes),n,threshold,l,r) + else + return nil + end end) - --- We provide a few more helpers: - ------ http = require("socket.http") -local httprequest = http.request -local toquery = url.toquery - --- local function httprequest(url) --- return os.resultof(format("curl --silent %q", url)) --- end - +local httprequest=http.request +local toquery=url.toquery local function fetchstring(url,data) - local q = data and toquery(data) - if q then - url = url .. "?" .. q - end - local reply = httprequest(url) - return reply -- just one argument -end - -schemes.fetchstring = fetchstring - + local q=data and toquery(data) + if q then + url=url.."?"..q + end + local reply=httprequest(url) + return reply +end +schemes.fetchstring=fetchstring function schemes.fetchtable(url,data) - local reply = fetchstring(url,data) - if reply then - local s = load("return " .. reply) - if s then - return s() - end + local reply=fetchstring(url,data) + if reply then + local s=load("return "..reply) + if s then + return s() end + end end @@ -17874,278 +13969,241 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- We overload the regular loader. We do so because we operate mostly in --- tds and use our own loader code. Alternatively we could use a more --- extensive definition of package.path and package.cpath but even then --- we're not done. Also, we now have better tracing. --- --- -- local mylib = require("libtest") --- -- local mysql = require("luasql.mysql") - -local searchers = package.searchers or package.loaders - -local concat = table.concat - -local trace_libraries = false - -trackers.register("resolvers.libraries", function(v) trace_libraries = v end) -trackers.register("resolvers.locating", function(v) trace_libraries = v end) - -local report_libraries = logs.reporter("resolvers","libraries") - -local gsub, insert = string.gsub, table.insert -local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match -local unpack = unpack or table.unpack -local is_readable = file.is_readable - -local resolvers, package = resolvers, package - -local libsuffixes = { 'tex', 'lua' } -local clibsuffixes = { 'lib' } -local libformats = { 'TEXINPUTS', 'LUAINPUTS' } -local clibformats = { 'CLUAINPUTS' } - -local libpaths = nil -local clibpaths = nil -local libhash = { } -local clibhash = { } -local libextras = { } -local clibextras = { } +-- original size: 6387, stripped down to: 5108 -local pattern = Cs(P("!")^0 / "" * (P("/") * P(-1) / "/" + P("/")^1 / "/" + 1)^0) - -local function cleanpath(path) --hm, don't we have a helper for this? - return resolvers.resolve(lpegmatch(pattern,path)) +if not modules then modules={} end modules ['data-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local searchers=package.searchers or package.loaders +local concat=table.concat +local trace_libraries=false +trackers.register("resolvers.libraries",function(v) trace_libraries=v end) +trackers.register("resolvers.locating",function(v) trace_libraries=v end) +local report_libraries=logs.reporter("resolvers","libraries") +local gsub,insert=string.gsub,table.insert +local P,Cs,lpegmatch=lpeg.P,lpeg.Cs,lpeg.match +local unpack=unpack or table.unpack +local is_readable=file.is_readable +local resolvers,package=resolvers,package +local libsuffixes={ 'tex','lua' } +local clibsuffixes={ 'lib' } +local libformats={ 'TEXINPUTS','LUAINPUTS' } +local clibformats={ 'CLUAINPUTS' } +local libpaths=nil +local clibpaths=nil +local libhash={} +local clibhash={} +local libextras={} +local clibextras={} +local pattern=Cs(P("!")^0/""*(P("/")*P(-1)/"/"+P("/")^1/"/"+1)^0) +local function cleanpath(path) + return resolvers.resolve(lpegmatch(pattern,path)) end - local function getlibpaths() - if not libpaths then - libpaths = { } - for i=1,#libformats do - local paths = resolvers.expandedpathlistfromvariable(libformats[i]) - for i=1,#paths do - local path = cleanpath(paths[i]) - if not libhash[path] then - libpaths[#libpaths+1] = path - libhash[path] = true - end - end + if not libpaths then + libpaths={} + for i=1,#libformats do + local paths=resolvers.expandedpathlistfromvariable(libformats[i]) + for i=1,#paths do + local path=cleanpath(paths[i]) + if not libhash[path] then + libpaths[#libpaths+1]=path + libhash[path]=true end + end end - return libpaths + end + return libpaths end - local function getclibpaths() - if not clibpaths then - clibpaths = { } - for i=1,#clibformats do - local paths = resolvers.expandedpathlistfromvariable(clibformats[i]) - for i=1,#paths do - local path = cleanpath(paths[i]) - if not clibhash[path] then - clibpaths[#clibpaths+1] = path - clibhash[path] = true - end - end + if not clibpaths then + clibpaths={} + for i=1,#clibformats do + local paths=resolvers.expandedpathlistfromvariable(clibformats[i]) + for i=1,#paths do + local path=cleanpath(paths[i]) + if not clibhash[path] then + clibpaths[#clibpaths+1]=path + clibhash[path]=true end + end end - return clibpaths + end + return clibpaths end - -package.libpaths = getlibpaths -package.clibpaths = getclibpaths - +package.libpaths=getlibpaths +package.clibpaths=getclibpaths function package.extralibpath(...) - local libpaths = getlibpaths() - local paths = table.flattened { ... } - for i=1,#paths do - local path = cleanpath(paths[i]) - if not libhash[path] then - if trace_libraries then - report_libraries("! extra lua path '%s'",path) - end - libextras[#libextras+1] = path - libpaths[#libpaths +1] = path - end + local libpaths=getlibpaths() + local paths=table.flattened {... } + for i=1,#paths do + local path=cleanpath(paths[i]) + if not libhash[path] then + if trace_libraries then + report_libraries("! extra lua path '%s'",path) + end + libextras[#libextras+1]=path + libpaths[#libpaths+1]=path end + end end - function package.extraclibpath(...) - local clibpaths = getclibpaths() - local paths = table.flattened { ... } - for i=1,#paths do - local path = cleanpath(paths[i]) - if not clibhash[path] then - if trace_libraries then - report_libraries("! extra lib path '%s'",path) - end - clibextras[#clibextras+1] = path - clibpaths[#clibpaths +1] = path - end + local clibpaths=getclibpaths() + local paths=table.flattened {... } + for i=1,#paths do + local path=cleanpath(paths[i]) + if not clibhash[path] then + if trace_libraries then + report_libraries("! extra lib path '%s'",path) + end + clibextras[#clibextras+1]=path + clibpaths[#clibpaths+1]=path end + end end - if not searchers[-2] then - -- use package-path and package-cpath - searchers[-2] = searchers[2] + searchers[-2]=searchers[2] end - local function loadedaslib(resolved,rawname) - return package.loadlib(resolved,"luaopen_" .. gsub(rawname,"%.","_")) + return package.loadlib(resolved,"luaopen_"..gsub(rawname,"%.","_")) end - local function loadedbylua(name) - if trace_libraries then - report_libraries("! locating %q using normal loader",name) - end - local resolved = searchers[-2](name) + if trace_libraries then + report_libraries("! locating %q using normal loader",name) + end + local resolved=searchers[-2](name) end - local function loadedbyformat(name,rawname,suffixes,islib) + if trace_libraries then + report_libraries("! locating %q as %q using formats %q",rawname,name,concat(suffixes)) + end + for i=1,#suffixes do + local format=suffixes[i] + local resolved=resolvers.findfile(name,format) or "" if trace_libraries then - report_libraries("! locating %q as %q using formats %q",rawname,name,concat(suffixes)) + report_libraries("! checking for %q' using format %q",name,format) end - for i=1,#suffixes do -- so we use findfile and not a lookup loop - local format = suffixes[i] - local resolved = resolvers.findfile(name,format) or "" - if trace_libraries then - report_libraries("! checking for %q' using format %q",name,format) - end - if resolved ~= "" then - if trace_libraries then - report_libraries("! lib %q located on %q",name,resolved) - end - if islib then - return loadedaslib(resolved,rawname) - else - return loadfile(resolved) - end - end + if resolved~="" then + if trace_libraries then + report_libraries("! lib %q located on %q",name,resolved) + end + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) + end end + end end - local function loadedbypath(name,rawname,paths,islib,what) - if trace_libraries then - report_libraries("! locating %q as %q on %q paths",rawname,name,what) - end - for p=1,#paths do - local path = paths[p] - local resolved = file.join(path,name) - if trace_libraries then -- mode detail - report_libraries("! checking for %q using %q path %q",name,what,path) - end - if is_readable(resolved) then - if trace_libraries then - report_libraries("! lib %q located on %q",name,resolved) - end - if islib then - return loadedaslib(resolved,rawname) - else - return loadfile(resolved) - end - end + if trace_libraries then + report_libraries("! locating %q as %q on %q paths",rawname,name,what) + end + for p=1,#paths do + local path=paths[p] + local resolved=file.join(path,name) + if trace_libraries then + report_libraries("! checking for %q using %q path %q",name,what,path) + end + if is_readable(resolved) then + if trace_libraries then + report_libraries("! lib %q located on %q",name,resolved) + end + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) + end end + end end - local function notloaded(name) - if trace_libraries then - report_libraries("? unable to locate library %q",name) - end -end - -searchers[2] = function(name) - local thename = gsub(name,"%.","/") - local luaname = file.addsuffix(thename,"lua") - local libname = file.addsuffix(thename,os.libsuffix) - return - loadedbyformat(luaname,name,libsuffixes, false) - or loadedbyformat(libname,name,clibsuffixes, true) - or loadedbypath (luaname,name,getlibpaths (),false,"lua") - or loadedbypath (luaname,name,getclibpaths(),false,"lua") - or loadedbypath (libname,name,getclibpaths(),true, "lib") - or loadedbylua (name) - or notloaded (name) -end - --- searchers[3] = nil --- searchers[4] = nil - -resolvers.loadlualib = require + if trace_libraries then + report_libraries("? unable to locate library %q",name) + end +end +searchers[2]=function(name) + local thename=gsub(name,"%.","/") + local luaname=file.addsuffix(thename,"lua") + local libname=file.addsuffix(thename,os.libsuffix) + return + loadedbyformat(luaname,name,libsuffixes,false) + or loadedbyformat(libname,name,clibsuffixes,true) + or loadedbypath (luaname,name,getlibpaths (),false,"lua") + or loadedbypath (luaname,name,getclibpaths(),false,"lua") + or loadedbypath (libname,name,getclibpaths(),true,"lib") + or loadedbylua (name) + or notloaded (name) +end +resolvers.loadlualib=require end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-aux'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local find = string.find -local type, next = type, next - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local resolvers = resolvers - -local report_scripts = logs.reporter("resolvers","scripts") +-- original size: 2394, stripped down to: 2005 -function resolvers.updatescript(oldname,newname) -- oldname -> own.name, not per se a suffix - local scriptpath = "scripts/context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = resolvers.cleanpath(oldname) +if not modules then modules={} end modules ['data-aux']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find=string.find +local type,next=type,next +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local resolvers=resolvers +local report_scripts=logs.reporter("resolvers","scripts") +function resolvers.updatescript(oldname,newname) + local scriptpath="scripts/context/lua" + newname=file.addsuffix(newname,"lua") + local oldscript=resolvers.cleanpath(oldname) + if trace_locating then + report_scripts("to be replaced old script %s",oldscript) + end + local newscripts=resolvers.findfiles(newname) or {} + if #newscripts==0 then if trace_locating then - report_scripts("to be replaced old script %s", oldscript) + report_scripts("unable to locate new script") end - local newscripts = resolvers.findfiles(newname) or { } - if #newscripts == 0 then + else + for i=1,#newscripts do + local newscript=resolvers.cleanpath(newscripts[i]) + if trace_locating then + report_scripts("checking new script %s",newscript) + end + if oldscript==newscript then if trace_locating then - report_scripts("unable to locate new script") + report_scripts("old and new script are the same") end - else - for i=1,#newscripts do - local newscript = resolvers.cleanpath(newscripts[i]) - if trace_locating then - report_scripts("checking new script %s", newscript) - end - if oldscript == newscript then - if trace_locating then - report_scripts("old and new script are the same") - end - elseif not find(newscript,scriptpath) then - if trace_locating then - report_scripts("new script should come from %s",scriptpath) - end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_locating then - report_scripts("invalid new script name") - end - else - local newdata = io.loaddata(newscript) - if newdata then - if trace_locating then - report_scripts("old script content replaced by new content") - end - io.savedata(oldscript,newdata) - break - elseif trace_locating then - report_scripts("unable to load new script") - end - end + elseif not find(newscript,scriptpath) then + if trace_locating then + report_scripts("new script should come from %s",scriptpath) + end + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_locating then + report_scripts("invalid new script name") + end + else + local newdata=io.loaddata(newscript) + if newdata then + if trace_locating then + report_scripts("old script content replaced by new content") + end + io.savedata(oldscript,newdata) + break + elseif trace_locating then + report_scripts("unable to load new script") end + end end + end end @@ -18153,78 +14211,53 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tmf'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local resolvers = resolvers - -local report_tds = logs.reporter("resolvers","tds") - --- = << --- ? ?? --- < += --- > =+ +-- original size: 2610, stripped down to: 1637 +if not modules then modules={} end modules ['data-tmf']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers=resolvers +local report_tds=logs.reporter("resolvers","tds") function resolvers.load_tree(tree,resolve) - if type(tree) == "string" and tree ~= "" then - - local getenv, setenv = resolvers.getenv, resolvers.setenv - - -- later might listen to the raw osenv var as well - local texos = "texmf-" .. os.platform - - local oldroot = environment.texroot - local newroot = file.collapsepath(tree) - - local newtree = file.join(newroot,texos) - local newpath = file.join(newtree,"bin") - - if not lfs.isdir(newtree) then - report_tds("no '%s' under tree %s",texos,tree) - os.exit() - end - if not lfs.isdir(newpath) then - report_tds("no '%s/bin' under tree %s",texos,tree) - os.exit() - end - - local texmfos = newtree - - environment.texroot = newroot - environment.texos = texos - environment.texmfos = texmfos - - -- Beware, we need to obey the relocatable autoparent so we - -- set TEXMFCNF to its raw value. This is somewhat tricky when - -- we run a mkii job from within. Therefore, in mtxrun, there - -- is a resolve applied when we're in mkii/kpse mode or when - -- --resolve is passed to mtxrun. Maybe we should also set the - -- local AUTOPARENT etc. although these are alwasy set new. - - if resolve then - -- resolvers.luacnfspec = resolvers.joinpath(resolvers.resolve(resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)))) - resolvers.luacnfspec = resolvers.resolve(resolvers.luacnfspec) - end - - setenv('SELFAUTOPARENT', newroot) - setenv('SELFAUTODIR', newtree) - setenv('SELFAUTOLOC', newpath) - setenv('TEXROOT', newroot) - setenv('TEXOS', texos) - setenv('TEXMFOS', texmfos) - setenv('TEXMFCNF', resolvers.luacnfspec,true) -- already resolved - setenv('PATH', newpath .. io.pathseparator .. getenv('PATH')) - - report_tds("changing from root '%s' to '%s'",oldroot,newroot) - report_tds("prepending '%s' to PATH",newpath) - report_tds("setting TEXMFCNF to '%s'",resolvers.luacnfspec) - report_tds() + if type(tree)=="string" and tree~="" then + local getenv,setenv=resolvers.getenv,resolvers.setenv + local texos="texmf-"..os.platform + local oldroot=environment.texroot + local newroot=file.collapsepath(tree) + local newtree=file.join(newroot,texos) + local newpath=file.join(newtree,"bin") + if not lfs.isdir(newtree) then + report_tds("no '%s' under tree %s",texos,tree) + os.exit() + end + if not lfs.isdir(newpath) then + report_tds("no '%s/bin' under tree %s",texos,tree) + os.exit() + end + local texmfos=newtree + environment.texroot=newroot + environment.texos=texos + environment.texmfos=texmfos + if resolve then + resolvers.luacnfspec=resolvers.resolve(resolvers.luacnfspec) end + setenv('SELFAUTOPARENT',newroot) + setenv('SELFAUTODIR',newtree) + setenv('SELFAUTOLOC',newpath) + setenv('TEXROOT',newroot) + setenv('TEXOS',texos) + setenv('TEXMFOS',texmfos) + setenv('TEXMFCNF',resolvers.luacnfspec,true) + setenv('PATH',newpath..io.pathseparator..getenv('PATH')) + report_tds("changing from root '%s' to '%s'",oldroot,newroot) + report_tds("prepending '%s' to PATH",newpath) + report_tds("setting TEXMFCNF to '%s'",resolvers.luacnfspec) + report_tds() + end end @@ -18232,81 +14265,74 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-lst'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- used in mtxrun, can be loaded later .. todo - -local find, concat, upper, format = string.find, table.concat, string.upper, string.format -local fastcopy, sortedpairs = table.fastcopy, table.sortedpairs - -resolvers.listers = resolvers.listers or { } - -local resolvers = resolvers - -local report_lists = logs.reporter("resolvers","lists") +-- original size: 2632, stripped down to: 2278 +if not modules then modules={} end modules ['data-lst']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,concat,upper,format=string.find,table.concat,string.upper,string.format +local fastcopy,sortedpairs=table.fastcopy,table.sortedpairs +resolvers.listers=resolvers.listers or {} +local resolvers=resolvers +local report_lists=logs.reporter("resolvers","lists") local function tabstr(str) - if type(str) == 'table' then - return concat(str," | ") - else - return str - end + if type(str)=='table' then + return concat(str," | ") + else + return str + end end - function resolvers.listers.variables(pattern) - local instance = resolvers.instance - local environment = instance.environment - local variables = instance.variables - local expansions = instance.expansions - local pattern = upper(pattern or "") - local configured = { } - local order = instance.order - for i=1,#order do - for k, v in next, order[i] do - if v ~= nil and configured[k] == nil then - configured[k] = v - end - end - end - local env = fastcopy(environment) - local var = fastcopy(variables) - local exp = fastcopy(expansions) - for key, value in sortedpairs(configured) do - if key ~= "" and (pattern == "" or find(upper(key),pattern)) then - report_lists(key) - report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") - report_lists(" var: %s",tabstr(configured[key]) or "unset") - report_lists(" exp: %s",tabstr(expansions[key]) or "unset") - report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") - end + local instance=resolvers.instance + local environment=instance.environment + local variables=instance.variables + local expansions=instance.expansions + local pattern=upper(pattern or "") + local configured={} + local order=instance.order + for i=1,#order do + for k,v in next,order[i] do + if v~=nil and configured[k]==nil then + configured[k]=v + end end - instance.environment = fastcopy(env) - instance.variables = fastcopy(var) - instance.expansions = fastcopy(exp) + end + local env=fastcopy(environment) + local var=fastcopy(variables) + local exp=fastcopy(expansions) + for key,value in sortedpairs(configured) do + if key~="" and (pattern=="" or find(upper(key),pattern)) then + report_lists(key) + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") + end + end + instance.environment=fastcopy(env) + instance.variables=fastcopy(var) + instance.expansions=fastcopy(exp) end - function resolvers.listers.configurations(report) - local configurations = resolvers.instance.specification - local report = report or texio.write_nl - for i=1,#configurations do - report(format("file : %s",resolvers.resolve(configurations[i]))) - end - report("") - local list = resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) - for i=1,#list do - local li = resolvers.resolve(list[i]) - if lfs.isdir(li) then - report(format("path - %s",li)) - else - report(format("path + %s",li)) - end + local configurations=resolvers.instance.specification + local report=report or texio.write_nl + for i=1,#configurations do + report(format("file : %s",resolvers.resolve(configurations[i]))) + end + report("") + local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + for i=1,#list do + local li=resolvers.resolve(list[i]) + if lfs.isdir(li) then + report(format("path - %s",li)) + else + report(format("path + %s",li)) end + end end @@ -18314,264 +14340,234 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-sta'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this code is used in the updater - -local gmatch, match = string.gmatch, string.match -local type = type - -states = states or { } -local states = states - -states.data = states.data or { } -local data = states.data - -states.hash = states.hash or { } -local hash = states.hash - -states.tag = states.tag or "" -states.filename = states.filename or "" +-- original size: 5703, stripped down to: 2507 +if not modules then modules={} end modules ['luat-sta']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gmatch,match=string.gmatch,string.match +local type=type +states=states or {} +local states=states +states.data=states.data or {} +local data=states.data +states.hash=states.hash or {} +local hash=states.hash +states.tag=states.tag or "" +states.filename=states.filename or "" function states.save(filename,tag) - tag = tag or states.tag - filename = file.addsuffix(filename or states.filename,'lus') - io.savedata(filename, - "-- generator : luat-sta.lua\n" .. - "-- state tag : " .. tag .. "\n\n" .. - table.serialize(data[tag or states.tag] or {},true) - ) + tag=tag or states.tag + filename=file.addsuffix(filename or states.filename,'lus') + io.savedata(filename, + "-- generator : luat-sta.lua\n".."-- state tag : "..tag.."\n\n"..table.serialize(data[tag or states.tag] or {},true) + ) end - function states.load(filename,tag) - states.filename = filename - states.tag = tag or "whatever" - states.filename = file.addsuffix(states.filename,'lus') - data[states.tag], hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } + states.filename=filename + states.tag=tag or "whatever" + states.filename=file.addsuffix(states.filename,'lus') + data[states.tag],hash[states.tag]=(io.exists(filename) and dofile(filename)) or {},{} end - local function set_by_tag(tag,key,value,default,persistent) - local d, h = data[tag], hash[tag] - if d then - if type(d) == "table" then - local dkey, hkey = key, key - local pre, post = match(key,"(.+)%.([^%.]+)$") - if pre and post then - for k in gmatch(pre,"[^%.]+") do - local dk = d[k] - if not dk then - dk = { } - d[k] = dk - elseif type(dk) == "string" then - -- invalid table, unable to upgrade structure - -- hope for the best or delete the state file - break - end - d = dk - end - dkey, hkey = post, key - end - if value == nil then - value = default - elseif value == false then - -- special case - elseif persistent then - value = value or d[dkey] or default - else - value = value or default - end - d[dkey], h[hkey] = value, value - elseif type(d) == "string" then - -- weird - data[tag], hash[tag] = value, value + local d,h=data[tag],hash[tag] + if d then + if type(d)=="table" then + local dkey,hkey=key,key + local pre,post=match(key,"(.+)%.([^%.]+)$") + if pre and post then + for k in gmatch(pre,"[^%.]+") do + local dk=d[k] + if not dk then + dk={} + d[k]=dk + elseif type(dk)=="string" then + break + end + d=dk end + dkey,hkey=post,key + end + if value==nil then + value=default + elseif value==false then + elseif persistent then + value=value or d[dkey] or default + else + value=value or default + end + d[dkey],h[hkey]=value,value + elseif type(d)=="string" then + data[tag],hash[tag]=value,value end + end end - local function get_by_tag(tag,key,default) - local h = hash[tag] - if h and h[key] then - return h[key] - else - local d = data[tag] - if d then - for k in gmatch(key,"[^%.]+") do - local dk = d[k] - if dk ~= nil then - d = dk - else - return default - end - end - if d == false then - return false - else - return d or default - end + local h=hash[tag] + if h and h[key] then + return h[key] + else + local d=data[tag] + if d then + for k in gmatch(key,"[^%.]+") do + local dk=d[k] + if dk~=nil then + d=dk + else + return default end + end + if d==false then + return false + else + return d or default + end end + end end - -states.set_by_tag = set_by_tag -states.get_by_tag = get_by_tag - +states.set_by_tag=set_by_tag +states.get_by_tag=get_by_tag function states.set(key,value,default,persistent) - set_by_tag(states.tag,key,value,default,persistent) + set_by_tag(states.tag,key,value,default,persistent) end - function states.get(key,default) - return get_by_tag(states.tag,key,default) + return get_by_tag(states.tag,key,default) end - - - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-fmt'] = { - version = 1.001, - comment = "companion to mtxrun", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - - -local format = string.format -local quoted = string.quoted -local luasuffixes = utilities.lua.suffixes - -local report_format = logs.reporter("resolvers","formats") +-- original size: 5954, stripped down to: 4923 -local function primaryflags() -- not yet ok - local trackers = environment.argument("trackers") - local directives = environment.argument("directives") - local flags = "" - if trackers and trackers ~= "" then - flags = flags .. "--trackers=" .. quoted(trackers) - end - if directives and directives ~= "" then - flags = flags .. "--directives=" .. quoted(directives) - end - return flags +if not modules then modules={} end modules ['luat-fmt']={ + version=1.001, + comment="companion to mtxrun", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format=string.format +local concat=table.concat +local quoted=string.quoted +local luasuffixes=utilities.lua.suffixes +local report_format=logs.reporter("resolvers","formats") +local function primaryflags() + local trackers=environment.argument("trackers") + local directives=environment.argument("directives") + local flags={} + if trackers and trackers~="" then + flags={ "--trackers="..quoted(trackers) } + end + if directives and directives~="" then + flags={ "--directives="..quoted(directives) } + end + if environment.argument("jit") then + flags={ "--jiton" } + end + return concat(flags," ") end - function environment.make_format(name) - local engine = environment.ownmain or "luatex" - -- change to format path (early as we need expanded paths) - local olddir = dir.current() - local path = caches.getwritablepath("formats",engine) or "" -- maybe platform - if path ~= "" then - lfs.chdir(path) - end - report_format("format path: %s",dir.current()) - -- check source file - local texsourcename = file.addsuffix(name,"mkiv") - local fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" - if fulltexsourcename == "" then - texsourcename = file.addsuffix(name,"tex") - fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" - end - if fulltexsourcename == "" then - report_format("no tex source file with name: %s (mkiv or tex)",name) - lfs.chdir(olddir) - return - else - report_format("using tex source file: %s",fulltexsourcename) - end - local texsourcepath = dir.expandname(file.dirname(fulltexsourcename)) -- really needed - -- check specification - local specificationname = file.replacesuffix(fulltexsourcename,"lus") - local fullspecificationname = resolvers.findfile(specificationname,"tex") or "" - if fullspecificationname == "" then - specificationname = file.join(texsourcepath,"context.lus") - fullspecificationname = resolvers.findfile(specificationname,"tex") or "" - end - if fullspecificationname == "" then - report_format("unknown stub specification: %s",specificationname) - lfs.chdir(olddir) - return - end - local specificationpath = file.dirname(fullspecificationname) - -- load specification - local usedluastub = nil - local usedlualibs = dofile(fullspecificationname) - if type(usedlualibs) == "string" then - usedluastub = file.join(file.dirname(fullspecificationname),usedlualibs) - elseif type(usedlualibs) == "table" then - report_format("using stub specification: %s",fullspecificationname) - local texbasename = file.basename(name) - local luastubname = file.addsuffix(texbasename,luasuffixes.lua) - local lucstubname = file.addsuffix(texbasename,luasuffixes.luc) - -- pack libraries in stub - report_format("creating initialization file: %s",luastubname) - utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) - -- compile stub file (does not save that much as we don't use this stub at startup any more) - if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then - report_format("using compiled initialization file: %s",lucstubname) - usedluastub = lucstubname - else - report_format("using uncompiled initialization file: %s",luastubname) - usedluastub = luastubname - end - else - report_format("invalid stub specification: %s",fullspecificationname) - lfs.chdir(olddir) - return - end - -- generate format - local command = format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform == "unix" and "\\\\" or "\\") - report_format("running command: %s\n",command) - os.spawn(command) - -- remove related mem files - local pattern = file.removesuffix(file.basename(usedluastub)).."-*.mem" - -- report_format("removing related mplib format with pattern '%s'", pattern) - local mp = dir.glob(pattern) - if mp then - for i=1,#mp do - local name = mp[i] - report_format("removing related mplib format %s", file.basename(name)) - os.remove(name) - end - end + local engine=environment.ownmain or "luatex" + local olddir=dir.current() + local path=caches.getwritablepath("formats",engine) or "" + if path~="" then + lfs.chdir(path) + end + report_format("format path: %s",dir.current()) + local texsourcename=file.addsuffix(name,"mkiv") + local fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + if fulltexsourcename=="" then + texsourcename=file.addsuffix(name,"tex") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename=="" then + report_format("no tex source file with name: %s (mkiv or tex)",name) + lfs.chdir(olddir) + return + else + report_format("using tex source file: %s",fulltexsourcename) + end + local texsourcepath=dir.expandname(file.dirname(fulltexsourcename)) + local specificationname=file.replacesuffix(fulltexsourcename,"lus") + local fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + if fullspecificationname=="" then + specificationname=file.join(texsourcepath,"context.lus") + fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + end + if fullspecificationname=="" then + report_format("unknown stub specification: %s",specificationname) + lfs.chdir(olddir) + return + end + local specificationpath=file.dirname(fullspecificationname) + local usedluastub=nil + local usedlualibs=dofile(fullspecificationname) + if type(usedlualibs)=="string" then + usedluastub=file.join(file.dirname(fullspecificationname),usedlualibs) + elseif type(usedlualibs)=="table" then + report_format("using stub specification: %s",fullspecificationname) + local texbasename=file.basename(name) + local luastubname=file.addsuffix(texbasename,luasuffixes.lua) + local lucstubname=file.addsuffix(texbasename,luasuffixes.luc) + report_format("creating initialization file: %s",luastubname) + utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) + if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then + report_format("using compiled initialization file: %s",lucstubname) + usedluastub=lucstubname + else + report_format("using uncompiled initialization file: %s",luastubname) + usedluastub=luastubname + end + else + report_format("invalid stub specification: %s",fullspecificationname) lfs.chdir(olddir) + return + end + local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") + report_format("running command: %s\n",command) + os.spawn(command) + local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" + local mp=dir.glob(pattern) + if mp then + for i=1,#mp do + local name=mp[i] + report_format("removing related mplib format %s",file.basename(name)) + os.remove(name) + end + end + lfs.chdir(olddir) end - function environment.run_format(name,data,more) - if name and name ~= "" then - local engine = environment.ownmain or "luatex" - local barename = file.removesuffix(name) - local fmtname = caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) - if fmtname == "" then - fmtname = resolvers.findfile(file.addsuffix(barename,"fmt")) or "" - end - fmtname = resolvers.cleanpath(fmtname) - if fmtname == "" then - report_format("no format with name: %s",name) - else - local barename = file.removesuffix(name) -- expanded name - local luaname = file.addsuffix(barename,"luc") - if not lfs.isfile(luaname) then - luaname = file.addsuffix(barename,"lua") - end - if not lfs.isfile(luaname) then - report_format("using format name: %s",fmtname) - report_format("no luc/lua with name: %s",barename) - else - local command = format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more ~= "" and quoted(more) or "") - report_format("running command: %s",command) - os.spawn(command) - end - end + if name and name~="" then + local engine=environment.ownmain or "luatex" + local barename=file.removesuffix(name) + local fmtname=caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) + if fmtname=="" then + fmtname=resolvers.findfile(file.addsuffix(barename,"fmt")) or "" + end + fmtname=resolvers.cleanpath(fmtname) + if fmtname=="" then + report_format("no format with name: %s",name) + else + local barename=file.removesuffix(name) + local luaname=file.addsuffix(barename,"luc") + if not lfs.isfile(luaname) then + luaname=file.addsuffix(barename,"lua") + end + if not lfs.isfile(luaname) then + report_format("using format name: %s",fmtname) + report_format("no luc/lua with name: %s",barename) + else + local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") + report_format("running command: %s",command) + os.spawn(command) + end end + end end @@ -18579,128 +14575,106 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-tpl'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This is experimental code. Coming from dos and windows, I've always used %whatever% --- as template variables so let's stick to it. After all, it's easy to parse and stands --- out well. A double %% is turned into a regular %. - -utilities.templates = utilities.templates or { } -local templates = utilities.templates - -local trace_template = false trackers.register("templates.trace",function(v) trace_template = v end) -local report_template = logs.reporter("template") - -local format = string.format -local P, C, Cs, Carg, lpegmatch = lpeg.P, lpeg.C, lpeg.Cs, lpeg.Carg, lpeg.match - --- todo: make installable template.new +-- original size: 3570, stripped down to: 2441 +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local format=string.format +local P,C,Cs,Carg,lpegmatch=lpeg.P,lpeg.C,lpeg.Cs,lpeg.Carg,lpeg.match local replacer - local function replacekey(k,t,recursive) - local v = t[k] - if not v then - if trace_template then - report_template("unknown key %q",k) - end - return "" + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %q",k) + end + return "" + else + if trace_template then + report_template("setting key %q to value %q",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t) else - if trace_template then - report_template("setting key %q to value %q",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t) - else - return v - end + return v end + end end - -local sqlescape = lpeg.replacer { - { "'", "''" }, - { "\\", "\\\\" }, - { "\r\n", "\\n" }, - { "\r", "\\n" }, - -- { "\t", "\\t" }, +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, } - -local escapers = { - lua = function(s) - return format("%q",s) - end, - sql = function(s) - return lpegmatch(sqlescape,s) - end, +local escapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, } - -lpeg.patterns.sqlescape = sqlescape - -local function replacekeyunquoted(s,t,how,recurse) -- ".. \" " - local escaper = how and escapers[how] or escapers.lua - return escaper(replacekey(s,t,recurse)) -end - -local single = P("%") -- test %test% test : resolves test -local double = P("%%") -- test 10%% test : %% becomes % -local lquoted = P("%[") -- test %[test]" test : resolves test with escaped "'s -local rquoted = P("]%") -- - -local escape = double / '%%' -local nosingle = single / '' -local nodouble = double / '' -local nolquoted = lquoted / '' -local norquoted = rquoted / '' - -local key = nosingle * (C((1-nosingle)^1 * Carg(1) * Carg(2) * Carg(3))/replacekey) * nosingle -local unquoted = nolquoted * ((C((1 - norquoted)^1) * Carg(1) * Carg(2) * Carg(3))/replacekeyunquoted) * norquoted -local any = P(1) - - replacer = Cs((unquoted + escape + key + any)^0) - +lpeg.patterns.sqlescape=sqlescape +local function replacekeyunquoted(s,t,how,recurse) + local escaper=how and escapers[how] or escapers.lua + return escaper(replacekey(s,t,recurse)) +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local key=nosingle*(C((1-nosingle)^1*Carg(1)*Carg(2)*Carg(3))/replacekey)*nosingle +local unquoted=nolquoted*((C((1-norquoted)^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyunquoted)*norquoted +local any=P(1) + replacer=Cs((unquoted+escape+key+any)^0) local function replace(str,mapping,how,recurse) - if mapping then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end + if mapping then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end end - --- print(replace("test '%[x]%' test",{ x = [[a 'x'  a]] })) --- print(replace("test '%[x]%' test",{ x = [[a 'x'  a]] },'sql')) - -templates.replace = replace - +templates.replace=replace function templates.load(filename,mapping,how,recurse) - local data = io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end end - function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping = t - end - for k, v in next, t do - t[k] = replace(v,mapping,how,recurse) - end - return t + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t end --- inspect(utilities.templates.replace("test %one% test", { one = "%two%", two = "two" })) --- inspect(utilities.templates.resolve({ one = "%two%", two = "two", three = "%three%" })) - end -- of closure + +-- used libraries : l-lua.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-tab.lua util-sto.lua util-str.lua util-mrg.lua util-lua.lua util-prs.lua util-fmt.lua util-deb.lua trac-inf.lua trac-set.lua trac-log.lua trac-pro.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua luat-sta.lua luat-fmt.lua util-tpl.lua +-- skipped libraries : - +-- original bytes : 589196 +-- stripped bytes : 198197 + -- end library merge own = { } -- not local, might change diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua index 731f8ca21..4d5fd447d 100644 --- a/scripts/context/stubs/mswin/mtxrun.lua +++ b/scripts/context/stubs/mswin/mtxrun.lua @@ -54,112 +54,54 @@ if not modules then modules = { } end modules ['mtxrun'] = { do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +-- original size: 2319, stripped down to: 1038 + +if not modules then modules={} end modules ['l-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } - --- compatibility hacks ... try to avoid usage - -local major, minor = string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") - -_MAJORVERSION = tonumber(major) or 5 -_MINORVERSION = tonumber(minor) or 1 - --- basics: - +local major,minor=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") +_MAJORVERSION=tonumber(major) or 5 +_MINORVERSION=tonumber(minor) or 1 if loadstring then - - local loadnormal = load - - function load(first,...) - if type(first) == "string" then - return loadstring(first,...) - else - return loadnormal(first,...) - end + local loadnormal=load + function load(first,...) + if type(first)=="string" then + return loadstring(first,...) + else + return loadnormal(first,...) end - + end else - - loadstring = load - + loadstring=load end - --- table: - --- Starting with version 5.2 Lua no longer provide ipairs, which makes --- sense. As we already used the for loop and # in most places the --- impact on ConTeXt was not that large; the remaining ipairs already --- have been replaced. In a similar fashion we also hardly used pairs. --- --- Hm, actually ipairs was retained, but we no longer use it anyway. --- --- Just in case, we provide the fallbacks as discussed in Programming --- in Lua (http://www.lua.org/pil/7.3.html): - if not ipairs then - - -- for k, v in ipairs(t) do ... end - -- for k=1,#t do local v = t[k] ... end - - local function iterate(a,i) - i = i + 1 - local v = a[i] - if v ~= nil then - return i, v --, nil - end - end - - function ipairs(a) - return iterate, a, 0 + local function iterate(a,i) + i=i+1 + local v=a[i] + if v~=nil then + return i,v end - + end + function ipairs(a) + return iterate,a,0 + end end - if not pairs then - - -- for k, v in pairs(t) do ... end - -- for k, v in next, t do ... end - - function pairs(t) - return next, t -- , nil - end - + function pairs(t) + return next,t + end end - --- The unpack function has been moved to the table table, and for compatiility --- reasons we provide both now. - if not table.unpack then - - table.unpack = _G.unpack - + table.unpack=_G.unpack elseif not unpack then - - _G.unpack = table.unpack - + _G.unpack=table.unpack end - --- package: - --- if not package.seachers then --- --- package.searchers = package.loaders -- 5.2 --- --- elseif not package.loaders then --- --- package.loaders = package.searchers --- --- end - -if not package.loaders then -- brr, searchers is a special "loadlib function" userdata type - - package.loaders = package.searchers - +if not package.loaders then + package.loaders=package.searchers end @@ -167,7359 +109,3309 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-lpeg'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1) - --- move utf -> l-unicode --- move string -> l-string or keep it here - -local lpeg = require("lpeg") - --- tracing (only used when we encounter a problem in integration of lpeg in luatex) - --- some code will move to unicode and string - -local report = texio and texio.write_nl or print - --- local lpmatch = lpeg.match --- local lpprint = lpeg.print --- local lpp = lpeg.P --- local lpr = lpeg.R --- local lps = lpeg.S --- local lpc = lpeg.C --- local lpb = lpeg.B --- local lpv = lpeg.V --- local lpcf = lpeg.Cf --- local lpcb = lpeg.Cb --- local lpcg = lpeg.Cg --- local lpct = lpeg.Ct --- local lpcs = lpeg.Cs --- local lpcc = lpeg.Cc --- local lpcmt = lpeg.Cmt --- local lpcarg = lpeg.Carg - --- function lpeg.match(l,...) report("LPEG MATCH") lpprint(l) return lpmatch(l,...) end - --- function lpeg.P (l) local p = lpp (l) report("LPEG P =") lpprint(l) return p end --- function lpeg.R (l) local p = lpr (l) report("LPEG R =") lpprint(l) return p end --- function lpeg.S (l) local p = lps (l) report("LPEG S =") lpprint(l) return p end --- function lpeg.C (l) local p = lpc (l) report("LPEG C =") lpprint(l) return p end --- function lpeg.B (l) local p = lpb (l) report("LPEG B =") lpprint(l) return p end --- function lpeg.V (l) local p = lpv (l) report("LPEG V =") lpprint(l) return p end --- function lpeg.Cf (l) local p = lpcf (l) report("LPEG Cf =") lpprint(l) return p end --- function lpeg.Cb (l) local p = lpcb (l) report("LPEG Cb =") lpprint(l) return p end --- function lpeg.Cg (l) local p = lpcg (l) report("LPEG Cg =") lpprint(l) return p end --- function lpeg.Ct (l) local p = lpct (l) report("LPEG Ct =") lpprint(l) return p end --- function lpeg.Cs (l) local p = lpcs (l) report("LPEG Cs =") lpprint(l) return p end --- function lpeg.Cc (l) local p = lpcc (l) report("LPEG Cc =") lpprint(l) return p end --- function lpeg.Cmt (l) local p = lpcmt (l) report("LPEG Cmt =") lpprint(l) return p end --- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end - -local type, next = type, next -local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format - --- Beware, we predefine a bunch of patterns here and one reason for doing so --- is that we get consistent behaviour in some of the visualizers. - -lpeg.patterns = lpeg.patterns or { } -- so that we can share -local patterns = lpeg.patterns - -local P, R, S, V, Ct, C, Cs, Cc, Cp, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Cp, lpeg.Cmt -local lpegtype, lpegmatch = lpeg.type, lpeg.match - -local anything = P(1) -local endofstring = P(-1) -local alwaysmatched = P(true) - -patterns.anything = anything -patterns.endofstring = endofstring -patterns.beginofstring = alwaysmatched -patterns.alwaysmatched = alwaysmatched - -local digit, sign = R('09'), S('+-') -local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") -local newline = crlf + S("\r\n") -- cr + lf -local escaped = P("\\") * anything -local squote = P("'") -local dquote = P('"') -local space = P(" ") - -local utfbom_32_be = P('\000\000\254\255') -local utfbom_32_le = P('\255\254\000\000') -local utfbom_16_be = P('\255\254') -local utfbom_16_le = P('\254\255') -local utfbom_8 = P('\239\187\191') -local utfbom = utfbom_32_be + utfbom_32_le - + utfbom_16_be + utfbom_16_le - + utfbom_8 -local utftype = utfbom_32_be * Cc("utf-32-be") + utfbom_32_le * Cc("utf-32-le") - + utfbom_16_be * Cc("utf-16-be") + utfbom_16_le * Cc("utf-16-le") - + utfbom_8 * Cc("utf-8") + alwaysmatched * Cc("utf-8") -- assume utf8 -local utfoffset = utfbom_32_be * Cc(4) + utfbom_32_le * Cc(4) - + utfbom_16_be * Cc(2) + utfbom_16_le * Cc(2) - + utfbom_8 * Cc(3) + Cc(0) - -local utf8next = R("\128\191") - -patterns.utf8one = R("\000\127") -patterns.utf8two = R("\194\223") * utf8next -patterns.utf8three = R("\224\239") * utf8next * utf8next -patterns.utf8four = R("\240\244") * utf8next * utf8next * utf8next -patterns.utfbom = utfbom -patterns.utftype = utftype -patterns.utfoffset = utfoffset - -local utf8char = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four -local validutf8char = utf8char^0 * endofstring * Cc(true) + Cc(false) - -patterns.utf8 = utf8char -patterns.utf8char = utf8char -patterns.validutf8 = validutf8char -patterns.validutf8char = validutf8char - -local eol = S("\n\r") -local spacer = S(" \t\f\v") -- + char(0xc2, 0xa0) if we want utf (cf mail roberto) -local whitespace = eol + spacer -local nonspacer = 1 - spacer -local nonwhitespace = 1 - whitespace - -patterns.eol = eol -patterns.spacer = spacer -patterns.whitespace = whitespace -patterns.nonspacer = nonspacer -patterns.nonwhitespace = nonwhitespace - -local stripper = spacer^0 * C((spacer^0 * nonspacer^1)^0) -- from example by roberto - ------ collapser = Cs(spacer^0/"" * ((spacer^1 * P(-1) / "") + (spacer^1/" ") + P(1))^0) -local collapser = Cs(spacer^0/"" * nonspacer^0 * ((spacer^0/" " * nonspacer^1)^0)) - -patterns.stripper = stripper -patterns.collapser = collapser - -patterns.digit = digit -patterns.sign = sign -patterns.cardinal = sign^0 * digit^1 -patterns.integer = sign^0 * digit^1 -patterns.unsigned = digit^0 * P('.') * digit^1 -patterns.float = sign^0 * patterns.unsigned -patterns.cunsigned = digit^0 * P(',') * digit^1 -patterns.cfloat = sign^0 * patterns.cunsigned -patterns.number = patterns.float + patterns.integer -patterns.cnumber = patterns.cfloat + patterns.integer -patterns.oct = P("0") * R("07")^1 -patterns.octal = patterns.oct -patterns.HEX = P("0x") * R("09","AF")^1 -patterns.hex = P("0x") * R("09","af")^1 -patterns.hexadecimal = P("0x") * R("09","AF","af")^1 -patterns.lowercase = R("az") -patterns.uppercase = R("AZ") -patterns.letter = patterns.lowercase + patterns.uppercase -patterns.space = space -patterns.tab = P("\t") -patterns.spaceortab = patterns.space + patterns.tab -patterns.newline = newline -patterns.emptyline = newline^1 -patterns.equal = P("=") -patterns.comma = P(",") -patterns.commaspacer = P(",") * spacer^0 -patterns.period = P(".") -patterns.colon = P(":") -patterns.semicolon = P(";") -patterns.underscore = P("_") -patterns.escaped = escaped -patterns.squote = squote -patterns.dquote = dquote -patterns.nosquote = (escaped + (1-squote))^0 -patterns.nodquote = (escaped + (1-dquote))^0 -patterns.unsingle = (squote/"") * patterns.nosquote * (squote/"") -- will change to C in the middle -patterns.undouble = (dquote/"") * patterns.nodquote * (dquote/"") -- will change to C in the middle -patterns.unquoted = patterns.undouble + patterns.unsingle -- more often undouble -patterns.unspacer = ((patterns.spacer^1)/"")^0 - -patterns.singlequoted = squote * patterns.nosquote * squote -patterns.doublequoted = dquote * patterns.nodquote * dquote -patterns.quoted = patterns.doublequoted + patterns.singlequoted - -patterns.propername = R("AZ","az","__") * R("09","AZ","az", "__")^0 * P(-1) - -patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1 -patterns.beginline = #(1-newline) - -patterns.longtostring = Cs(whitespace^0/"" * nonwhitespace^0 * ((whitespace^0/" " * (patterns.quoted + nonwhitespace)^1)^0)) - -local function anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * V(1) } -end - -lpeg.anywhere = anywhere +-- original size: 25285, stripped down to: 13969 +if not modules then modules={} end modules ['l-lpeg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local lpeg=require("lpeg") +local report=texio and texio.write_nl or print +local type,next,tostring=type,next,tostring +local byte,char,gmatch,format=string.byte,string.char,string.gmatch,string.format +local floor=math.floor +lpeg.patterns=lpeg.patterns or {} +local patterns=lpeg.patterns +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt +local lpegtype,lpegmatch=lpeg.type,lpeg.match +local anything=P(1) +local endofstring=P(-1) +local alwaysmatched=P(true) +patterns.anything=anything +patterns.endofstring=endofstring +patterns.beginofstring=alwaysmatched +patterns.alwaysmatched=alwaysmatched +local digit,sign=R('09'),S('+-') +local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") +local newline=crlf+S("\r\n") +local escaped=P("\\")*anything +local squote=P("'") +local dquote=P('"') +local space=P(" ") +local utfbom_32_be=P('\000\000\254\255') +local utfbom_32_le=P('\255\254\000\000') +local utfbom_16_be=P('\255\254') +local utfbom_16_le=P('\254\255') +local utfbom_8=P('\239\187\191') +local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8 +local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8") +local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0) +local utf8next=R("\128\191") +patterns.utf8one=R("\000\127") +patterns.utf8two=R("\194\223")*utf8next +patterns.utf8three=R("\224\239")*utf8next*utf8next +patterns.utf8four=R("\240\244")*utf8next*utf8next*utf8next +patterns.utfbom=utfbom +patterns.utftype=utftype +patterns.utfoffset=utfoffset +local utf8char=patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four +local validutf8char=utf8char^0*endofstring*Cc(true)+Cc(false) +patterns.utf8=utf8char +patterns.utf8char=utf8char +patterns.validutf8=validutf8char +patterns.validutf8char=validutf8char +local eol=S("\n\r") +local spacer=S(" \t\f\v") +local whitespace=eol+spacer +local nonspacer=1-spacer +local nonwhitespace=1-whitespace +patterns.eol=eol +patterns.spacer=spacer +patterns.whitespace=whitespace +patterns.nonspacer=nonspacer +patterns.nonwhitespace=nonwhitespace +local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +patterns.stripper=stripper +patterns.collapser=collapser +patterns.digit=digit +patterns.sign=sign +patterns.cardinal=sign^0*digit^1 +patterns.integer=sign^0*digit^1 +patterns.unsigned=digit^0*P('.')*digit^1 +patterns.float=sign^0*patterns.unsigned +patterns.cunsigned=digit^0*P(',')*digit^1 +patterns.cfloat=sign^0*patterns.cunsigned +patterns.number=patterns.float+patterns.integer +patterns.cnumber=patterns.cfloat+patterns.integer +patterns.oct=P("0")*R("07")^1 +patterns.octal=patterns.oct +patterns.HEX=P("0x")*R("09","AF")^1 +patterns.hex=P("0x")*R("09","af")^1 +patterns.hexadecimal=P("0x")*R("09","AF","af")^1 +patterns.lowercase=R("az") +patterns.uppercase=R("AZ") +patterns.letter=patterns.lowercase+patterns.uppercase +patterns.space=space +patterns.tab=P("\t") +patterns.spaceortab=patterns.space+patterns.tab +patterns.newline=newline +patterns.emptyline=newline^1 +patterns.equal=P("=") +patterns.comma=P(",") +patterns.commaspacer=P(",")*spacer^0 +patterns.period=P(".") +patterns.colon=P(":") +patterns.semicolon=P(";") +patterns.underscore=P("_") +patterns.escaped=escaped +patterns.squote=squote +patterns.dquote=dquote +patterns.nosquote=(escaped+(1-squote))^0 +patterns.nodquote=(escaped+(1-dquote))^0 +patterns.unsingle=(squote/"")*patterns.nosquote*(squote/"") +patterns.undouble=(dquote/"")*patterns.nodquote*(dquote/"") +patterns.unquoted=patterns.undouble+patterns.unsingle +patterns.unspacer=((patterns.spacer^1)/"")^0 +patterns.singlequoted=squote*patterns.nosquote*squote +patterns.doublequoted=dquote*patterns.nodquote*dquote +patterns.quoted=patterns.doublequoted+patterns.singlequoted +patterns.propername=R("AZ","az","__")*R("09","AZ","az","__")^0*P(-1) +patterns.somecontent=(anything-newline-space)^1 +patterns.beginline=#(1-newline) +patterns.longtostring=Cs(whitespace^0/""*nonwhitespace^0*((whitespace^0/" "*(patterns.quoted+nonwhitespace)^1)^0)) +local function anywhere(pattern) + return P { P(pattern)+1*V(1) } +end +lpeg.anywhere=anywhere function lpeg.instringchecker(p) - p = anywhere(p) - return function(str) - return lpegmatch(p,str) and true or false - end + p=anywhere(p) + return function(str) + return lpegmatch(p,str) and true or false + end end - -function lpeg.splitter(pattern, action) - return (((1-P(pattern))^1)/action+1)^0 +function lpeg.splitter(pattern,action) + return (((1-P(pattern))^1)/action+1)^0 end - -function lpeg.tsplitter(pattern, action) - return Ct((((1-P(pattern))^1)/action+1)^0) +function lpeg.tsplitter(pattern,action) + return Ct((((1-P(pattern))^1)/action+1)^0) end - --- probleem: separator can be lpeg and that does not hash too well, but --- it's quite okay as the key is then not garbage collected - -local splitters_s, splitters_m, splitters_t = { }, { }, { } - +local splitters_s,splitters_m,splitters_t={},{},{} local function splitat(separator,single) - local splitter = (single and splitters_s[separator]) or splitters_m[separator] - if not splitter then - separator = P(separator) - local other = C((1 - separator)^0) - if single then - local any = anything - splitter = other * (separator * C(any^0) + "") -- ? - splitters_s[separator] = splitter - else - splitter = other * (separator * other)^0 - splitters_m[separator] = splitter - end + local splitter=(single and splitters_s[separator]) or splitters_m[separator] + if not splitter then + separator=P(separator) + local other=C((1-separator)^0) + if single then + local any=anything + splitter=other*(separator*C(any^0)+"") + splitters_s[separator]=splitter + else + splitter=other*(separator*other)^0 + splitters_m[separator]=splitter end - return splitter + end + return splitter end - local function tsplitat(separator) - local splitter = splitters_t[separator] - if not splitter then - splitter = Ct(splitat(separator)) - splitters_t[separator] = splitter - end - return splitter -end - -lpeg.splitat = splitat -lpeg.tsplitat = tsplitat - + local splitter=splitters_t[separator] + if not splitter then + splitter=Ct(splitat(separator)) + splitters_t[separator]=splitter + end + return splitter +end +lpeg.splitat=splitat +lpeg.tsplitat=tsplitat function string.splitup(str,separator) - if not separator then - separator = "," - end - return lpegmatch(splitters_m[separator] or splitat(separator),str) + if not separator then + separator="," + end + return lpegmatch(splitters_m[separator] or splitat(separator),str) end - --- local p = splitat("->",false) print(lpegmatch(p,"oeps->what->more")) -- oeps what more --- local p = splitat("->",true) print(lpegmatch(p,"oeps->what->more")) -- oeps what->more --- local p = splitat("->",false) print(lpegmatch(p,"oeps")) -- oeps --- local p = splitat("->",true) print(lpegmatch(p,"oeps")) -- oeps - -local cache = { } - +local cache={} function lpeg.split(separator,str) - local c = cache[separator] - if not c then - c = tsplitat(separator) - cache[separator] = c - end - return lpegmatch(c,str) + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c + end + return lpegmatch(c,str) end - function string.split(str,separator) - if separator then - local c = cache[separator] - if not c then - c = tsplitat(separator) - cache[separator] = c - end - return lpegmatch(c,str) - else - return { str } + if separator then + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c end -end - -local spacing = patterns.spacer^0 * newline -- sort of strip -local empty = spacing * Cc("") -local nonempty = Cs((1-spacing)^1) * spacing^-1 -local content = (empty + nonempty)^1 - -patterns.textline = content - -local linesplitter = tsplitat(newline) - -patterns.linesplitter = linesplitter - + return lpegmatch(c,str) + else + return { str } + end +end +local spacing=patterns.spacer^0*newline +local empty=spacing*Cc("") +local nonempty=Cs((1-spacing)^1)*spacing^-1 +local content=(empty+nonempty)^1 +patterns.textline=content +local linesplitter=tsplitat(newline) +patterns.linesplitter=linesplitter function string.splitlines(str) - return lpegmatch(linesplitter,str) + return lpegmatch(linesplitter,str) end - --- lpeg.splitters = cache -- no longer public - -local cache = { } - +local cache={} function lpeg.checkedsplit(separator,str) - local c = cache[separator] - if not c then - separator = P(separator) - local other = C((1 - separator)^1) - c = Ct(separator^0 * other * (separator^1 * other)^0) - cache[separator] = c - end - return lpegmatch(c,str) + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) end - function string.checkedsplit(str,separator) - local c = cache[separator] - if not c then - separator = P(separator) - local other = C((1 - separator)^1) - c = Ct(separator^0 * other * (separator^1 * other)^0) - cache[separator] = c - end - return lpegmatch(c,str) -end - --- from roberto's site: - -local function f2(s) local c1, c2 = byte(s,1,2) return c1 * 64 + c2 - 12416 end -local function f3(s) local c1, c2, c3 = byte(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end -local function f4(s) local c1, c2, c3, c4 = byte(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end - -local utf8byte = patterns.utf8one/byte + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 - -patterns.utf8byte = utf8byte - - - -local cache = { } - + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) +end +local function f2(s) local c1,c2=byte(s,1,2) return c1*64+c2-12416 end +local function f3(s) local c1,c2,c3=byte(s,1,3) return (c1*64+c2)*64+c3-925824 end +local function f4(s) local c1,c2,c3,c4=byte(s,1,4) return ((c1*64+c2)*64+c3)*64+c4-63447168 end +local utf8byte=patterns.utf8one/byte+patterns.utf8two/f2+patterns.utf8three/f3+patterns.utf8four/f4 +patterns.utf8byte=utf8byte +local cache={} function lpeg.stripper(str) - if type(str) == "string" then - local s = cache[str] - if not s then - s = Cs(((S(str)^1)/"" + 1)^0) - cache[str] = s - end - return s - else - return Cs(((str^1)/"" + 1)^0) + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs(((S(str)^1)/""+1)^0) + cache[str]=s end + return s + else + return Cs(((str^1)/""+1)^0) + end end - -local cache = { } - +local cache={} function lpeg.keeper(str) - if type(str) == "string" then - local s = cache[str] - if not s then - s = Cs((((1-S(str))^1)/"" + 1)^0) - cache[str] = s - end - return s + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs((((1-S(str))^1)/""+1)^0) + cache[str]=s + end + return s + else + return Cs((((1-str)^1)/""+1)^0) + end +end +function lpeg.frontstripper(str) + return (P(str)+P(true))*Cs(anything^0) +end +function lpeg.endstripper(str) + return Cs((1-P(str)*endofstring)^0) +end +function lpeg.replacer(one,two,makefunction,isutf) + local pattern + local u=isutf and utf8char or 1 + if type(one)=="table" then + local no=#one + local p=P(false) + if no==0 then + for k,v in next,one do + p=p+P(k)/v + end + pattern=Cs((p+u)^0) + elseif no==1 then + local o=one[1] + one,two=P(o[1]),o[2] + pattern=Cs((one/two+u)^0) + else + for i=1,no do + local o=one[i] + p=p+P(o[1])/o[2] + end + pattern=Cs((p+u)^0) + end + else + pattern=Cs((P(one)/(two or "")+u)^0) + end + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +function lpeg.finder(lst,makefunction) + local pattern + if type(lst)=="table" then + pattern=P(false) + if #lst==0 then + for k,v in next,lst do + pattern=pattern+P(k) + end else - return Cs((((1-str)^1)/"" + 1)^0) + for i=1,#lst do + pattern=pattern+P(lst[i]) + end end + else + pattern=P(lst) + end + pattern=(1-pattern)^0*pattern + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +local splitters_f,splitters_s={},{} +function lpeg.firstofsplit(separator) + local splitter=splitters_f[separator] + if not splitter then + separator=P(separator) + splitter=C((1-separator)^0) + splitters_f[separator]=splitter + end + return splitter +end +function lpeg.secondofsplit(separator) + local splitter=splitters_s[separator] + if not splitter then + separator=P(separator) + splitter=(1-separator)^0*separator*C(anything^0) + splitters_s[separator]=splitter + end + return splitter end - -function lpeg.frontstripper(str) -- or pattern (yet undocumented) - return (P(str) + P(true)) * Cs(anything^0) +function lpeg.balancer(left,right) + left,right=P(left),P(right) + return P { left*((1-left-right)+V(1))^0*right } end - -function lpeg.endstripper(str) -- or pattern (yet undocumented) - return Cs((1 - P(str) * endofstring)^0) +local nany=utf8char/"" +function lpeg.counter(pattern) + pattern=Cs((P(pattern)/" "+nany)^0) + return function(str) + return #lpegmatch(pattern,str) + end +end +local utfcharacters=utf and utf.characters or string.utfcharacters +local utfgmatch=unicode and unicode.utf8.gmatch +local utfchar=utf and utf.char or (unicode and unicode.utf8 and unicode.utf8.char) +lpeg.UP=lpeg.P +if utfcharacters then + function lpeg.US(str) + local p=P(false) + for uc in utfcharacters(str) do + p=p+P(uc) + end + return p + end +elseif utfgmatch then + function lpeg.US(str) + local p=P(false) + for uc in utfgmatch(str,".") do + p=p+P(uc) + end + return p + end +else + function lpeg.US(str) + local p=P(false) + local f=function(uc) + p=p+P(uc) + end + lpegmatch((utf8char/f)^0,str) + return p + end end - --- Just for fun I looked at the used bytecode and --- p = (p and p + pp) or pp gets one more (testset). - --- todo: cache when string - -function lpeg.replacer(one,two,makefunction,isutf) -- in principle we should sort the keys - local pattern - local u = isutf and utf8char or 1 - if type(one) == "table" then - local no = #one - local p = P(false) - if no == 0 then - for k, v in next, one do - p = p + P(k) / v - end - pattern = Cs((p + u)^0) - elseif no == 1 then - local o = one[1] - one, two = P(o[1]), o[2] - -- pattern = Cs(((1-one)^1 + one/two)^0) - pattern = Cs((one/two + u)^0) +local range=utf8byte*utf8byte+Cc(false) +function lpeg.UR(str,more) + local first,last + if type(str)=="number" then + first=str + last=more or first + else + first,last=lpegmatch(range,str) + if not last then + return P(str) + end + end + if first==last then + return P(str) + elseif utfchar and (last-first<8) then + local p=P(false) + for i=first,last do + p=p+P(utfchar(i)) + end + return p + else + local f=function(b) + return b>=first and b<=last + end + return utf8byte/f + end +end +function lpeg.is_lpeg(p) + return p and lpegtype(p)=="pattern" +end +function lpeg.oneof(list,...) + if type(list)~="table" then + list={ list,... } + end + local p=P(list[1]) + for l=2,#list do + p=p+P(list[l]) + end + return p +end +local sort=table.sort +local function copyindexed(old) + local new={} + for i=1,#old do + new[i]=old + end + return new +end +local function sortedkeys(tab) + local keys,s={},0 + for key,_ in next,tab do + s=s+1 + keys[s]=key + end + sort(keys) + return keys +end +function lpeg.append(list,pp,delayed,checked) + local p=pp + if #list>0 then + local keys=copyindexed(list) + sort(keys) + for i=#keys,1,-1 do + local k=keys[i] + if p then + p=P(k)+p + else + p=P(k) + end + end + elseif delayed then + local keys=sortedkeys(list) + if p then + for i=1,#keys,1 do + local k=keys[i] + local v=list[k] + p=P(k)/list+p + end + else + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)+p else - for i=1,no do - local o = one[i] - p = p + P(o[1]) / o[2] - end - pattern = Cs((p + u)^0) + p=P(k) end - else - pattern = Cs((P(one)/(two or "") + u)^0) + end + if p then + p=p/list + end end - if makefunction then - return function(str) - return lpegmatch(pattern,str) + elseif checked then + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + if k==v then + p=P(k)+p + else + p=P(k)/v+p end - else - return pattern - end -end - -function lpeg.finder(lst,makefunction) - local pattern - if type(lst) == "table" then - pattern = P(false) - if #lst == 0 then - for k, v in next, lst do - pattern = pattern + P(k) -- ignore key, so we can use a replacer table - end + else + if k==v then + p=P(k) else - for i=1,#lst do - pattern = pattern + P(lst[i]) - end + p=P(k)/v end - else - pattern = P(lst) + end end - pattern = (1-pattern)^0 * pattern - if makefunction then - return function(str) - return lpegmatch(pattern,str) - end - else - return pattern + else + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)/v+p + else + p=P(k)/v + end end + end + return p end - --- print(lpeg.match(lpeg.replacer("e","a"),"test test")) --- print(lpeg.match(lpeg.replacer{{"e","a"}},"test test")) --- print(lpeg.match(lpeg.replacer({ e = "a", t = "x" }),"test test")) - -local splitters_f, splitters_s = { }, { } - -function lpeg.firstofsplit(separator) -- always return value - local splitter = splitters_f[separator] - if not splitter then - separator = P(separator) - splitter = C((1 - separator)^0) - splitters_f[separator] = splitter +local function make(t) + local p + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + local v=t[k] + if not p then + if next(v) then + p=P(k)*make(v) + else + p=P(k) + end + else + if next(v) then + p=p+P(k)*make(v) + else + p=p+P(k) + end end - return splitter + end + return p end - -function lpeg.secondofsplit(separator) -- nil if not split - local splitter = splitters_s[separator] - if not splitter then - separator = P(separator) - splitter = (1 - separator)^0 * separator * C(anything^0) - splitters_s[separator] = splitter +function lpeg.utfchartabletopattern(list) + local tree={} + for i=1,#list do + local t=tree + for c in gmatch(list[i],".") do + if not t[c] then + t[c]={} + end + t=t[c] + end + end + return make(tree) +end +patterns.containseol=lpeg.finder(eol) +local function nextstep(n,step,result) + local m=n%step + local d=floor(n/step) + if d>0 then + local v=V(tostring(step)) + local s=result.start + for i=1,d do + if s then + s=v*s + else + s=v + end end - return splitter + result.start=s + end + if step>1 and result.start then + local v=V(tostring(step/2)) + result[tostring(step)]=v*v + end + if step>0 then + return nextstep(m,step/2,result) + else + return result + end end - -function lpeg.balancer(left,right) - left, right = P(left), P(right) - return P { left * ((1 - left - right) + V(1))^0 * right } +function lpeg.times(pattern,n) + return P(nextstep(n,2^16,{ "start",["1"]=pattern })) end --- print(1,lpegmatch(lpeg.firstofsplit(":"),"bc:de")) --- print(2,lpegmatch(lpeg.firstofsplit(":"),":de")) -- empty --- print(3,lpegmatch(lpeg.firstofsplit(":"),"bc")) --- print(4,lpegmatch(lpeg.secondofsplit(":"),"bc:de")) --- print(5,lpegmatch(lpeg.secondofsplit(":"),"bc:")) -- empty --- print(6,lpegmatch(lpeg.secondofsplit(":",""),"bc")) --- print(7,lpegmatch(lpeg.secondofsplit(":"),"bc")) --- print(9,lpegmatch(lpeg.secondofsplit(":","123"),"bc")) - --- -- slower: --- --- function lpeg.counter(pattern) --- local n, pattern = 0, (lpeg.P(pattern)/function() n = n + 1 end + lpeg.anything)^0 --- return function(str) n = 0 ; lpegmatch(pattern,str) ; return n end --- end - -local nany = utf8char/"" -function lpeg.counter(pattern) - pattern = Cs((P(pattern)/" " + nany)^0) - return function(str) - return #lpegmatch(pattern,str) - end -end +end -- of closure --- utf extensies +do -- create closure to overcome 200 locals limit -local utfcharacters = utf and utf.characters or string.utfcharacters -local utfgmatch = unicode and unicode.utf8.gmatch -local utfchar = utf and utf.char or (unicode and unicode.utf8 and unicode.utf8.char) +-- original size: 361, stripped down to: 322 -lpeg.UP = lpeg.P +if not modules then modules={} end modules ['l-functions']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +functions=functions or {} +function functions.dummy() end -if utfcharacters then - function lpeg.US(str) - local p = P(false) - for uc in utfcharacters(str) do - p = p + P(uc) - end - return p - end +end -- of closure +do -- create closure to overcome 200 locals limit -elseif utfgmatch then +-- original size: 5491, stripped down to: 2685 - function lpeg.US(str) - local p = P(false) - for uc in utfgmatch(str,".") do - p = p + P(uc) - end - return p - end +if not modules then modules={} end modules ['l-string']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local string=string +local sub,gmatch,format,char,byte,rep,lower=string.sub,string.gmatch,string.format,string.char,string.byte,string.rep,string.lower +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local P,S,C,Ct,Cc,Cs=lpeg.P,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cc,lpeg.Cs +local unquoted=patterns.squote*C(patterns.nosquote)*patterns.squote+patterns.dquote*C(patterns.nodquote)*patterns.dquote +function string.unquoted(str) + return lpegmatch(unquoted,str) or str +end +function string.quoted(str) + return format("%q",str) +end +function string.count(str,pattern) + local n=0 + for _ in gmatch(str,pattern) do + n=n+1 + end + return n +end +function string.limit(str,n,sentinel) + if #str>n then + sentinel=sentinel or "..." + return sub(str,1,(n-#sentinel))..sentinel + else + return str + end +end +local stripper=patterns.stripper +local collapser=patterns.collapser +local longtostring=patterns.longtostring +function string.strip(str) + return lpegmatch(stripper,str) or "" +end +function string.collapsespaces(str) + return lpegmatch(collapser,str) or "" +end +function string.longtostring(str) + return lpegmatch(longtostring,str) or "" +end +local pattern=P(" ")^0*P(-1) +function string.is_empty(str) + if str=="" then + return true + else + return lpegmatch(pattern,str) and true or false + end +end +local anything=patterns.anything +local allescapes=Cc("%")*S(".-+%?()[]*") +local someescapes=Cc("%")*S(".-+%()[]") +local matchescapes=Cc(".")*S("*?") +local pattern_a=Cs ((allescapes+anything )^0 ) +local pattern_b=Cs ((someescapes+matchescapes+anything )^0 ) +local pattern_c=Cs (Cc("^")*(someescapes+matchescapes+anything )^0*Cc("$") ) +function string.escapedpattern(str,simple) + return lpegmatch(simple and pattern_b or pattern_a,str) +end +function string.topattern(str,lowercase,strict) + if str=="" then + return ".*" + elseif strict then + str=lpegmatch(pattern_c,str) + else + str=lpegmatch(pattern_b,str) + end + if lowercase then + return lower(str) + else + return str + end +end +function string.valid(str,default) + return (type(str)=="string" and str~="" and str) or default or nil +end +string.itself=function(s) return s end +local pattern=Ct(C(1)^0) +function string.totable(str) + return lpegmatch(pattern,str) +end +local replacer=lpeg.replacer("@","%%") +function string.tformat(fmt,...) + return format(lpegmatch(replacer,fmt),...) +end +string.quote=string.quoted +string.unquote=string.unquoted -else - function lpeg.US(str) - local p = P(false) - local f = function(uc) - p = p + P(uc) - end - lpegmatch((utf8char/f)^0,str) - return p - end +end -- of closure -end +do -- create closure to overcome 200 locals limit -local range = utf8byte * utf8byte + Cc(false) -- utf8byte is already a capture +-- original size: 28973, stripped down to: 19400 -function lpeg.UR(str,more) - local first, last - if type(str) == "number" then - first = str - last = more or first +if not modules then modules={} end modules ['l-table']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring,tonumber,ipairs,select=type,next,tostring,tonumber,ipairs,select +local table,string=table,string +local concat,sort,insert,remove=table.concat,table.sort,table.insert,table.remove +local format,lower,dump=string.format,string.lower,string.dump +local getmetatable,setmetatable=getmetatable,setmetatable +local getinfo=debug.getinfo +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local floor=math.floor +local stripper=patterns.stripper +function table.strip(tab) + local lst,l={},0 + for i=1,#tab do + local s=lpegmatch(stripper,tab[i]) or "" + if s=="" then else - first, last = lpegmatch(range,str) - if not last then - return P(str) - end + l=l+1 + lst[l]=s end - if first == last then - return P(str) - elseif utfchar and (last - first < 8) then -- a somewhat arbitrary criterium - local p = P(false) - for i=first,last do - p = p + P(utfchar(i)) - end - return p -- nil when invalid range - else - local f = function(b) - return b >= first and b <= last - end - -- tricky, these nested captures - return utf8byte / f -- nil when invalid range + end + return lst +end +function table.keys(t) + if t then + local keys,k={},0 + for key,_ in next,t do + k=k+1 + keys[k]=key end + return keys + else + return {} + end end - --- print(lpeg.match(lpeg.Cs((C(lpeg.UR("αω"))/{ ["χ"] = "OEPS" })^0),"αωχαω")) - --- lpeg.print(lpeg.R("ab","cd","gh")) --- lpeg.print(lpeg.P("a","b","c")) --- lpeg.print(lpeg.S("a","b","c")) - --- print(lpeg.count("äáàa",lpeg.P("á") + lpeg.P("à"))) --- print(lpeg.count("äáàa",lpeg.UP("áà"))) --- print(lpeg.count("äáàa",lpeg.US("àá"))) --- print(lpeg.count("äáàa",lpeg.UR("aá"))) --- print(lpeg.count("äáàa",lpeg.UR("àá"))) --- print(lpeg.count("äáàa",lpeg.UR(0x0000,0xFFFF))) - -function lpeg.is_lpeg(p) - return p and lpegtype(p) == "pattern" -end - -function lpeg.oneof(list,...) -- lpeg.oneof("elseif","else","if","then") -- assume proper order - if type(list) ~= "table" then - list = { list, ... } - end - -- table.sort(list) -- longest match first - local p = P(list[1]) - for l=2,#list do - p = p + P(list[l]) - end - return p -end - --- For the moment here, but it might move to utilities. Beware, we need to --- have the longest keyword first, so 'aaa' comes beforte 'aa' which is why we --- loop back from the end cq. prepend. - -local sort = table.sort - -local function copyindexed(old) - local new = { } - for i=1,#old do - new[i] = old - end - return new +local function compare(a,b) + local ta,tb=type(a),type(b) + if ta==tb then + return a 0 then - local keys = copyindexed(list) - sort(keys) - for i=#keys,1,-1 do - local k = keys[i] - if p then - p = P(k) + p - else - p = P(k) - end - end - elseif delayed then -- hm, it looks like the lpeg parser resolves anyway - local keys = sortedkeys(list) - if p then - for i=1,#keys,1 do - local k = keys[i] - local v = list[k] - p = P(k)/list + p - end - else - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - p = P(k) + p - else - p = P(k) - end - end - if p then - p = p / list - end - end - elseif checked then - -- problem: substitution gives a capture - local keys = sortedkeys(list) - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - if k == v then - p = P(k) + p - else - p = P(k)/v + p - end - else - if k == v then - p = P(k) - else - p = P(k)/v - end - end + if tab then + local srt,category,s={},0,0 + for key,_ in next,tab do + s=s+1 + srt[s]=key + if category==3 then + else + local tkey=type(key) + if tkey=="string" then + category=(category==2 and 3) or 1 + elseif tkey=="number" then + category=(category==1 and 3) or 2 + else + category=3 end + end + end + if category==0 or category==3 then + sort(srt,compare) else - local keys = sortedkeys(list) - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - p = P(k)/v + p - else - p = P(k)/v - end - end + sort(srt) end - return p + return srt + else + return {} + end end - --- inspect(lpeg.append({ a = "1", aa = "1", aaa = "1" } ,nil,true)) --- inspect(lpeg.append({ ["degree celsius"] = "1", celsius = "1", degree = "1" } ,nil,true)) - --- function lpeg.exact_match(words,case_insensitive) --- local pattern = concat(words) --- if case_insensitive then --- local pattern = S(upper(characters)) + S(lower(characters)) --- local list = { } --- for i=1,#words do --- list[lower(words[i])] = true --- end --- return Cmt(pattern^1, function(_,i,s) --- return list[lower(s)] and i --- end) --- else --- local pattern = S(concat(words)) --- local list = { } --- for i=1,#words do --- list[words[i]] = true --- end --- return Cmt(pattern^1, function(_,i,s) --- return list[s] and i --- end) --- end --- end - --- experiment: - -local function make(t) - local p - local keys = sortedkeys(t) - for i=1,#keys do - local k = keys[i] - local v = t[k] - if not p then - if next(v) then - p = P(k) * make(v) - else - p = P(k) - end - else - if next(v) then - p = p + P(k) * make(v) - else - p = p + P(k) - end - end +local function sortedhashkeys(tab) + if tab then + local srt,s={},0 + for key,_ in next,tab do + if key then + s=s+1 + srt[s]=key + end end - return p + sort(srt) + return srt + else + return {} + end end - -function lpeg.utfchartabletopattern(list) -- goes to util-lpg - local tree = { } - for i=1,#list do - local t = tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c] = { } - end - t = t[c] - end +function table.allkeys(t) + local keys={} + for i=1,#t do + for k,v in next,t[i] do + keys[k]=true end - return make(tree) -end - --- inspect ( lpeg.utfchartabletopattern { --- utfchar(0x00A0), -- nbsp --- utfchar(0x2000), -- enquad --- utfchar(0x2001), -- emquad --- utfchar(0x2002), -- enspace --- utfchar(0x2003), -- emspace --- utfchar(0x2004), -- threeperemspace --- utfchar(0x2005), -- fourperemspace --- utfchar(0x2006), -- sixperemspace --- utfchar(0x2007), -- figurespace --- utfchar(0x2008), -- punctuationspace --- utfchar(0x2009), -- breakablethinspace --- utfchar(0x200A), -- hairspace --- utfchar(0x200B), -- zerowidthspace --- utfchar(0x202F), -- narrownobreakspace --- utfchar(0x205F), -- math thinspace --- } ) - --- a few handy ones: --- --- faster than find(str,"[\n\r]") when match and # > 7 and always faster when # > 3 - -patterns.containseol = lpeg.finder(eol) -- (1-eol)^0 * eol - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-functions'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -functions = functions or { } - -function functions.dummy() end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-string'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local string = string -local sub, gmatch, format, char, byte, rep, lower = string.sub, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower -local lpegmatch, patterns = lpeg.match, lpeg.patterns -local P, S, C, Ct, Cc, Cs = lpeg.P, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cs - --- Some functions are already defined in l-lpeg and maybe some from here will --- move there (unless we also expose caches). - --- if not string.split then --- --- function string.split(str,pattern) --- local t = { } --- if #str > 0 then --- local n = 1 --- for s in gmatch(str..pattern,"(.-)"..pattern) do --- t[n] = s --- n = n + 1 --- end --- end --- return t --- end --- --- end - --- function string.unquoted(str) --- return (gsub(str,"^([\"\'])(.*)%1$","%2")) -- interesting pattern --- end - -local unquoted = patterns.squote * C(patterns.nosquote) * patterns.squote - + patterns.dquote * C(patterns.nodquote) * patterns.dquote - -function string.unquoted(str) - return lpegmatch(unquoted,str) or str + end + return sortedkeys(keys) end - --- print(string.unquoted("test")) --- print(string.unquoted([["t\"est"]])) --- print(string.unquoted([["t\"est"x]])) --- print(string.unquoted("\'test\'")) --- print(string.unquoted('"test"')) --- print(string.unquoted('"test"')) - -function string.quoted(str) - return format("%q",str) -- always " +table.sortedkeys=sortedkeys +table.sortedhashkeys=sortedhashkeys +local function nothing() end +local function sortedhash(t) + if t then + local n,s=0,sortedkeys(t) + local function kv(s) + n=n+1 + local k=s[n] + return k,t[k] + end + return kv,s + else + return nothing + end +end +table.sortedhash=sortedhash +table.sortedpairs=sortedhash +function table.append(t,list) + local n=#t + for i=1,#list do + n=n+1 + t[n]=list[i] + end + return t +end +function table.prepend(t,list) + local nl=#list + local nt=nl+#t + for i=#t,1,-1 do + t[nt]=t[i] + nt=nt-1 + end + for i=1,#list do + t[i]=list[i] + end + return t +end +function table.merge(t,...) + t=t or {} + for i=1,select("#",...) do + for k,v in next,(select(i,...)) do + t[k]=v + end + end + return t end - -function string.count(str,pattern) -- variant 3 - local n = 0 - for _ in gmatch(str,pattern) do -- not for utf - n = n + 1 +function table.merged(...) + local t={} + for i=1,select("#",...) do + for k,v in next,(select(i,...)) do + t[k]=v end - return n + end + return t end - -function string.limit(str,n,sentinel) -- not utf proof - if #str > n then - sentinel = sentinel or "..." - return sub(str,1,(n-#sentinel)) .. sentinel - else - return str +function table.imerge(t,...) + local nt=#t + for i=1,select("#",...) do + local nst=select(i,...) + for j=1,#nst do + nt=nt+1 + t[nt]=nst[j] end + end + return t end - -local stripper = patterns.stripper -local collapser = patterns.collapser -local longtostring = patterns.longtostring - -function string.strip(str) - return lpegmatch(stripper,str) or "" -end - -function string.collapsespaces(str) - return lpegmatch(collapser,str) or "" -end - -function string.longtostring(str) - return lpegmatch(longtostring,str) or "" -end - --- function string.is_empty(str) --- return not find(str,"%S") --- end - -local pattern = P(" ")^0 * P(-1) - -function string.is_empty(str) - if str == "" then - return true - else - return lpegmatch(pattern,str) and true or false +function table.imerged(...) + local tmp,ntmp={},0 + for i=1,select("#",...) do + local nst=select(i,...) + for j=1,#nst do + ntmp=ntmp+1 + tmp[ntmp]=nst[j] + end + end + return tmp +end +local function fastcopy(old,metatabletoo) + if old then + local new={} + for k,v in next,old do + if type(v)=="table" then + new[k]=fastcopy(v,metatabletoo) + else + new[k]=v + end end -end - - --- if not string.escapedpattern then --- --- local patterns_escapes = { --- ["%"] = "%%", --- ["."] = "%.", --- ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", --- ["["] = "%[", ["]"] = "%]", --- ["("] = "%(", [")"] = "%)", --- -- ["{"] = "%{", ["}"] = "%}" --- -- ["^"] = "%^", ["$"] = "%$", --- } --- --- local simple_escapes = { --- ["-"] = "%-", --- ["."] = "%.", --- ["?"] = ".", --- ["*"] = ".*", --- } --- --- function string.escapedpattern(str,simple) --- return (gsub(str,".",simple and simple_escapes or patterns_escapes)) --- end --- --- function string.topattern(str,lowercase,strict) --- if str == "" then --- return ".*" --- else --- str = gsub(str,".",simple_escapes) --- if lowercase then --- str = lower(str) --- end --- if strict then --- return "^" .. str .. "$" --- else --- return str --- end --- end --- end --- --- end - ---- needs checking - -local anything = patterns.anything -local allescapes = Cc("%") * S(".-+%?()[]*") -- also {} and ^$ ? -local someescapes = Cc("%") * S(".-+%()[]") -- also {} and ^$ ? -local matchescapes = Cc(".") * S("*?") -- wildcard and single match - -local pattern_a = Cs ( ( allescapes + anything )^0 ) -local pattern_b = Cs ( ( someescapes + matchescapes + anything )^0 ) -local pattern_c = Cs ( Cc("^") * ( someescapes + matchescapes + anything )^0 * Cc("$") ) - -function string.escapedpattern(str,simple) - return lpegmatch(simple and pattern_b or pattern_a,str) -end - -function string.topattern(str,lowercase,strict) - if str == "" then - return ".*" - elseif strict then - str = lpegmatch(pattern_c,str) - else - str = lpegmatch(pattern_b,str) + if metatabletoo then + local mt=getmetatable(old) + if mt then + setmetatable(new,mt) + end end - if lowercase then - return lower(str) - else - return str + return new + else + return {} + end +end +local function copy(t,tables) + tables=tables or {} + local tcopy={} + if not tables[t] then + tables[t]=tcopy + end + for i,v in next,t do + if type(i)=="table" then + if tables[i] then + i=tables[i] + else + i=copy(i,tables) + end end + if type(v)~="table" then + tcopy[i]=v + elseif tables[v] then + tcopy[i]=tables[v] + else + tcopy[i]=copy(v,tables) + end + end + local mt=getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy +end +table.fastcopy=fastcopy +table.copy=copy +function table.derive(parent) + local child={} + if parent then + setmetatable(child,{ __index=parent }) + end + return child end - --- print(string.escapedpattern("12+34*.tex",false)) --- print(string.escapedpattern("12+34*.tex",true)) --- print(string.topattern ("12+34*.tex",false,false)) --- print(string.topattern ("12+34*.tex",false,true)) - -function string.valid(str,default) - return (type(str) == "string" and str ~= "" and str) or default or nil -end - --- handy fallback - -string.itself = function(s) return s end - --- also handy (see utf variant) - -local pattern = Ct(C(1)^0) -- string and not utf ! - -function string.totable(str) - return lpegmatch(pattern,str) +function table.tohash(t,value) + local h={} + if t then + if value==nil then value=true end + for _,v in next,t do + h[v]=value + end + end + return h end - --- handy from within tex: - -local replacer = lpeg.replacer("@","%%") -- Watch the escaped % in lpeg! - -function string.tformat(fmt,...) - return format(lpegmatch(replacer,fmt),...) +function table.fromhash(t) + local hsh,h={},0 + for k,v in next,t do + if v then + h=h+1 + hsh[h]=k + end + end + return hsh end - --- obsolete names: - -string.quote = string.quoted -string.unquote = string.unquoted - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-table'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +local noquotes,hexify,handle,reduce,compact,inline,functions +local reserved=table.tohash { + 'and','break','do','else','elseif','end','false','for','function','if', + 'in','local','nil','not','or','repeat','return','then','true','until','while', } - -local type, next, tostring, tonumber, ipairs, select = type, next, tostring, tonumber, ipairs, select -local table, string = table, string -local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, lower, dump = string.format, string.lower, string.dump -local getmetatable, setmetatable = getmetatable, setmetatable -local getinfo = debug.getinfo -local lpegmatch, patterns = lpeg.match, lpeg.patterns -local floor = math.floor - --- extra functions, some might go (when not used) - -local stripper = patterns.stripper - -function table.strip(tab) - local lst, l = { }, 0 - for i=1,#tab do - local s = lpegmatch(stripper,tab[i]) or "" - if s == "" then - -- skip this one +local function simple_table(t) + if #t>0 then + local n=0 + for _,v in next,t do + n=n+1 + end + if n==#t then + local tt,nt={},0 + for i=1,#t do + local v=t[i] + local tv=type(v) + if tv=="number" then + nt=nt+1 + if hexify then + tt[nt]=format("0x%04X",v) + else + tt[nt]=tostring(v) + end + elseif tv=="boolean" then + nt=nt+1 + tt[nt]=tostring(v) + elseif tv=="string" then + nt=nt+1 + tt[nt]=format("%q",v) else - l = l + 1 - lst[l] = s + tt=nil + break end + end + return tt end - return lst + end + return nil end - -function table.keys(t) - if t then - local keys, k = { }, 0 - for key, _ in next, t do - k = k + 1 - keys[k] = key - end - return keys +local propername=patterns.propername +local function dummy() end +local function do_serialize(root,name,depth,level,indexed) + if level>0 then + depth=depth.." " + if indexed then + handle(format("%s{",depth)) else - return { } + local tn=type(name) + if tn=="number" then + if hexify then + handle(format("%s[0x%04X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) + end + elseif tn=="string" then + if noquotes and not reserved[name] and lpegmatch(propername,name) then + handle(format("%s%s={",depth,name)) + else + handle(format("%s[%q]={",depth,name)) + end + elseif tn=="boolean" then + handle(format("%s[%s]={",depth,tostring(name))) + else + handle(format("%s{",depth)) + end end -end - -local function compare(a,b) - local ta, tb = type(a), type(b) -- needed, else 11 < 2 - if ta == tb then - return a < b - else - return tostring(a) < tostring(b) + end + if root and next(root) then + local first,last=nil,0 + if compact then + last=#root + for k=1,last do + if root[k]==nil then + last=k-1 + break + end + end + if last>0 then + first=1 + end end -end - -local function sortedkeys(tab) - if tab then - local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed - for key,_ in next, tab do - s = s + 1 - srt[s] = key - if category == 3 then - -- no further check + local sk=sortedkeys(root) + for i=1,#sk do + local k=sk[i] + local v=root[k] + local t,tk=type(v),type(k) + if compact and first and tk=="number" and k>=first and k<=last then + if t=="number" then + if hexify then + handle(format("%s 0x%04X,",depth,v)) + else + handle(format("%s %s,",depth,v)) + end + elseif t=="string" then + if reduce and tonumber(v) then + handle(format("%s %s,",depth,v)) + else + handle(format("%s %q,",depth,v)) + end + elseif t=="table" then + if not next(v) then + handle(format("%s {},",depth)) + elseif inline then + local st=simple_table(v) + if st then + handle(format("%s { %s },",depth,concat(st,", "))) else - local tkey = type(key) - if tkey == "string" then - category = (category == 2 and 3) or 1 - elseif tkey == "number" then - category = (category == 1 and 3) or 2 - else - category = 3 - end + do_serialize(v,k,depth,level+1,true) end + else + do_serialize(v,k,depth,level+1,true) + end + elseif t=="boolean" then + handle(format("%s %s,",depth,tostring(v))) + elseif t=="function" then + if functions then + handle(format('%s load(%q),',depth,dump(v))) + else + handle(format('%s "function",',depth)) + end + else + handle(format("%s %q,",depth,tostring(v))) + end + elseif k=="__p__" then + if false then + handle(format("%s __p__=nil,",depth)) end - if category == 0 or category == 3 then - sort(srt,compare) + elseif t=="number" then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif tk=="boolean" then + if hexify then + handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) + else + handle(format("%s [%s]=%s,",depth,tostring(k),v)) + end + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + if hexify then + handle(format("%s %s=0x%04X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) + end else - sort(srt) + if hexify then + handle(format("%s [%q]=0x%04X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end end - return srt - else - return { } - end -end - -local function sortedhashkeys(tab) -- fast one - if tab then - local srt, s = { }, 0 - for key,_ in next, tab do - if key then - s= s + 1 - srt[s] = key + elseif t=="string" then + if reduce and tonumber(v) then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) end - end - sort(srt) - return srt - else - return { } - end -end - -function table.allkeys(t) - local keys = { } - for i=1,#t do - for k, v in next, t[i] do - keys[k] = true - end - end - return sortedkeys(keys) -end - -table.sortedkeys = sortedkeys -table.sortedhashkeys = sortedhashkeys - -local function nothing() end - -local function sortedhash(t) - if t then - local n, s = 0, sortedkeys(t) -- the robust one - local function kv(s) - n = n + 1 - local k = s[n] - return k, t[k] - end - return kv, s - else - return nothing - end -end - -table.sortedhash = sortedhash -table.sortedpairs = sortedhash -- obsolete - -function table.append(t,list) - local n = #t - for i=1,#list do - n = n + 1 - t[n] = list[i] - end - return t -end - -function table.prepend(t, list) - local nl = #list - local nt = nl + #t - for i=#t,1,-1 do - t[nt] = t[i] - nt = nt - 1 - end - for i=1,#list do - t[i] = list[i] - end - return t -end - --- function table.merge(t, ...) -- first one is target --- t = t or { } --- local lst = { ... } --- for i=1,#lst do --- for k, v in next, lst[i] do --- t[k] = v --- end --- end --- return t --- end - -function table.merge(t, ...) -- first one is target - t = t or { } - for i=1,select("#",...) do - for k, v in next, (select(i,...)) do - t[k] = v - end - end - return t -end - --- function table.merged(...) --- local tmp, lst = { }, { ... } --- for i=1,#lst do --- for k, v in next, lst[i] do --- tmp[k] = v --- end --- end --- return tmp --- end - -function table.merged(...) - local t = { } - for i=1,select("#",...) do - for k, v in next, (select(i,...)) do - t[k] = v - end - end - return t -end - --- function table.imerge(t, ...) --- local lst, nt = { ... }, #t --- for i=1,#lst do --- local nst = lst[i] --- for j=1,#nst do --- nt = nt + 1 --- t[nt] = nst[j] --- end --- end --- return t --- end - -function table.imerge(t, ...) - local nt = #t - for i=1,select("#",...) do - local nst = select(i,...) - for j=1,#nst do - nt = nt + 1 - t[nt] = nst[j] - end - end - return t -end - --- function table.imerged(...) --- local tmp, ntmp, lst = { }, 0, {...} --- for i=1,#lst do --- local nst = lst[i] --- for j=1,#nst do --- ntmp = ntmp + 1 --- tmp[ntmp] = nst[j] --- end --- end --- return tmp --- end - -function table.imerged(...) - local tmp, ntmp = { }, 0 - for i=1,select("#",...) do - local nst = select(i,...) - for j=1,#nst do - ntmp = ntmp + 1 - tmp[ntmp] = nst[j] - end - end - return tmp -end - -local function fastcopy(old,metatabletoo) -- fast one - if old then - local new = { } - for k, v in next, old do - if type(v) == "table" then - new[k] = fastcopy(v,metatabletoo) -- was just table.copy + elseif tk=="boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%s,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end + else + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,v)) else - new[k] = v + handle(format("%s [%s]=%q,",depth,k,v)) end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,tostring(k),v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end end - if metatabletoo then - -- optional second arg - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) + elseif t=="table" then + if not next(v) then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]={},",depth,k)) + else + handle(format("%s [%s]={},",depth,k)) end - end - return new - else - return { } - end -end - --- todo : copy without metatable - -local function copy(t, tables) -- taken from lua wiki, slightly adapted - tables = tables or { } - local tcopy = {} - if not tables[t] then - tables[t] = tcopy - end - for i,v in next, t do -- brrr, what happens with sparse indexed - if type(i) == "table" then - if tables[i] then - i = tables[i] + elseif tk=="boolean" then + handle(format("%s [%s]={},",depth,tostring(k))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={},",depth,k)) + else + handle(format("%s [%q]={},",depth,k)) + end + elseif inline then + local st=simple_table(v) + if st then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif tk=="boolean" then + handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else - i = copy(i, tables) + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) end - end - if type(v) ~= "table" then - tcopy[i] = v - elseif tables[v] then - tcopy[i] = tables[v] + else + do_serialize(v,k,depth,level+1) + end else - tcopy[i] = copy(v, tables) - end - end - local mt = getmetatable(t) - if mt then - setmetatable(tcopy,mt) - end - return tcopy -end - -table.fastcopy = fastcopy -table.copy = copy - -function table.derive(parent) -- for the moment not public - local child = { } - if parent then - setmetatable(child,{ __index = parent }) - end - return child -end - -function table.tohash(t,value) - local h = { } - if t then - if value == nil then value = true end - for _, v in next, t do -- no ipairs here - h[v] = value - end - end - return h -end - -function table.fromhash(t) - local hsh, h = { }, 0 - for k, v in next, t do -- no ipairs here - if v then - h = h + 1 - hsh[h] = k - end - end - return hsh -end - -local noquotes, hexify, handle, reduce, compact, inline, functions - -local reserved = table.tohash { -- intercept a language inconvenience: no reserved words as key - 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', - 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', -} - -local function simple_table(t) - if #t > 0 then - local n = 0 - for _,v in next, t do - n = n + 1 - end - if n == #t then - local tt, nt = { }, 0 - for i=1,#t do - local v = t[i] - local tv = type(v) - if tv == "number" then - nt = nt + 1 - if hexify then - tt[nt] = format("0x%04X",v) - else - tt[nt] = tostring(v) -- tostring not needed - end - elseif tv == "boolean" then - nt = nt + 1 - tt[nt] = tostring(v) - elseif tv == "string" then - nt = nt + 1 - tt[nt] = format("%q",v) - else - tt = nil - break - end - end - return tt + do_serialize(v,k,depth,level+1) end - end - return nil -end - --- Because this is a core function of mkiv I moved some function calls --- inline. --- --- twice as fast in a test: --- --- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) - --- problem: there no good number_to_string converter with the best resolution - --- probably using .. is faster than format --- maybe split in a few cases (yes/no hexify) - --- todo: %g faster on numbers than %s - -local propername = patterns.propername -- was find(name,"^%a[%w%_]*$") - -local function dummy() end - -local function do_serialize(root,name,depth,level,indexed) - if level > 0 then - depth = depth .. " " - if indexed then - handle(format("%s{",depth)) + elseif t=="boolean" then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%s,",depth,k,tostring(v))) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%s,",depth,k,tostring(v))) else - local tn = type(name) - if tn == "number" then - if hexify then - handle(format("%s[0x%04X]={",depth,name)) - else - handle(format("%s[%s]={",depth,name)) - end - elseif tn == "string" then - if noquotes and not reserved[name] and lpegmatch(propername,name) then - handle(format("%s%s={",depth,name)) - else - handle(format("%s[%q]={",depth,name)) - end - elseif tn == "boolean" then - handle(format("%s[%s]={",depth,tostring(name))) - else - handle(format("%s{",depth)) - end - end - end - -- we could check for k (index) being number (cardinal) - if root and next(root) then - -- local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) - -- if compact then - -- -- NOT: for k=1,#root do (we need to quit at nil) - -- for k,v in ipairs(root) do -- can we use next? - -- if not first then first = k end - -- last = last + 1 - -- end - -- end - local first, last = nil, 0 - if compact then - last = #root - for k=1,last do - if root[k] == nil then - last = k - 1 - break - end - end - if last > 0 then - first = 1 - end + handle(format("%s [%q]=%s,",depth,k,tostring(v))) end - local sk = sortedkeys(root) - for i=1,#sk do - local k = sk[i] - local v = root[k] - -- circular - local t, tk = type(v), type(k) - if compact and first and tk == "number" and k >= first and k <= last then - if t == "number" then - if hexify then - handle(format("%s 0x%04X,",depth,v)) - else - handle(format("%s %s,",depth,v)) -- %.99g - end - elseif t == "string" then - if reduce and tonumber(v) then - handle(format("%s %s,",depth,v)) - else - handle(format("%s %q,",depth,v)) - end - elseif t == "table" then - if not next(v) then - handle(format("%s {},",depth)) - elseif inline then -- and #t > 0 - local st = simple_table(v) - if st then - handle(format("%s { %s },",depth,concat(st,", "))) - else - do_serialize(v,k,depth,level+1,true) - end - else - do_serialize(v,k,depth,level+1,true) - end - elseif t == "boolean" then - handle(format("%s %s,",depth,tostring(v))) - elseif t == "function" then - if functions then - handle(format('%s load(%q),',depth,dump(v))) - else - handle(format('%s "function",',depth)) - end - else - handle(format("%s %q,",depth,tostring(v))) - end - elseif k == "__p__" then -- parent - if false then - handle(format("%s __p__=nil,",depth)) - end - elseif t == "number" then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g - end - elseif tk == "boolean" then - if hexify then - handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) - else - handle(format("%s [%s]=%s,",depth,tostring(k),v)) -- %.99g - end - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - if hexify then - handle(format("%s %s=0x%04X,",depth,k,v)) - else - handle(format("%s %s=%s,",depth,k,v)) -- %.99g - end - else - if hexify then - handle(format("%s [%q]=0x%04X,",depth,k,v)) - else - handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g - end - end - elseif t == "string" then - if reduce and tonumber(v) then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%s,",depth,tostring(k),v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%s,",depth,k,v)) - else - handle(format("%s [%q]=%s,",depth,k,v)) - end - else - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,v)) - else - handle(format("%s [%s]=%q,",depth,k,v)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%q,",depth,tostring(k),v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%q,",depth,k,v)) - else - handle(format("%s [%q]=%q,",depth,k,v)) - end - end - elseif t == "table" then - if not next(v) then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]={},",depth,k)) - else - handle(format("%s [%s]={},",depth,k)) - end - elseif tk == "boolean" then - handle(format("%s [%s]={},",depth,tostring(k))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s={},",depth,k)) - else - handle(format("%s [%q]={},",depth,k)) - end - elseif inline then - local st = simple_table(v) - if st then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) - else - handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) - end - elseif tk == "boolean" then - handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s={ %s },",depth,k,concat(st,", "))) - else - handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) - end - else - do_serialize(v,k,depth,level+1) - end - else - do_serialize(v,k,depth,level+1) - end - elseif t == "boolean" then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) - else - handle(format("%s [%s]=%s,",depth,k,tostring(v))) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%s,",depth,k,tostring(v))) - else - handle(format("%s [%q]=%s,",depth,k,tostring(v))) - end - elseif t == "function" then - if functions then - local f = getinfo(v).what == "C" and dump(dummy) or dump(v) - -- local f = getinfo(v).what == "C" and dump(function(...) return v(...) end) or dump(v) - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=load(%q),",depth,k,f)) - else - handle(format("%s [%s]=load(%q),",depth,k,f)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=load(%q),",depth,tostring(k),f)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=load(%q),",depth,k,f)) - else - handle(format("%s [%q]=load(%q),",depth,k,f)) - end - end + elseif t=="function" then + if functions then + local f=getinfo(v).what=="C" and dump(dummy) or dump(v) + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=load(%q),",depth,k,f)) else - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) - else - handle(format("%s [%s]=%q,",depth,k,tostring(v))) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%q,",depth,k,tostring(v))) - else - handle(format("%s [%q]=%q,",depth,k,tostring(v))) - end + handle(format("%s [%s]=load(%q),",depth,k,f)) end - end - end - if level > 0 then - handle(format("%s},",depth)) - end -end - --- replacing handle by a direct t[#t+1] = ... (plus test) is not much --- faster (0.03 on 1.00 for zapfino.tma) - -local function serialize(_handle,root,name,specification) -- handle wins - local tname = type(name) - if type(specification) == "table" then - noquotes = specification.noquotes - hexify = specification.hexify - handle = _handle or specification.handle or print - reduce = specification.reduce or false - functions = specification.functions - compact = specification.compact - inline = specification.inline and compact - if functions == nil then - functions = true - end - if compact == nil then - compact = true - end - if inline == nil then - inline = compact - end - else - noquotes = false - hexify = false - handle = _handle or print - reduce = false - compact = true - inline = true - functions = true - end - if tname == "string" then - if name == "return" then - handle("return {") - else - handle(name .. "={") - end - elseif tname == "number" then - if hexify then - handle(format("[0x%04X]={",name)) - else - handle("[" .. name .. "]={") + elseif tk=="boolean" then + handle(format("%s [%s]=load(%q),",depth,tostring(k),f)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=load(%q),",depth,k,f)) + else + handle(format("%s [%q]=load(%q),",depth,k,f)) + end end - elseif tname == "boolean" then - if name then - handle("return {") + else + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%q,",depth,k,tostring(v))) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,tostring(v))) else - handle("{") + handle(format("%s [%q]=%q,",depth,k,tostring(v))) end + end + end + end + if level>0 then + handle(format("%s},",depth)) + end +end +local function serialize(_handle,root,name,specification) + local tname=type(name) + if type(specification)=="table" then + noquotes=specification.noquotes + hexify=specification.hexify + handle=_handle or specification.handle or print + reduce=specification.reduce or false + functions=specification.functions + compact=specification.compact + inline=specification.inline and compact + if functions==nil then + functions=true + end + if compact==nil then + compact=true + end + if inline==nil then + inline=compact + end + else + noquotes=false + hexify=false + handle=_handle or print + reduce=false + compact=true + inline=true + functions=true + end + if tname=="string" then + if name=="return" then + handle("return {") + else + handle(name.."={") + end + elseif tname=="number" then + if hexify then + handle(format("[0x%04X]={",name)) + else + handle("["..name.."]={") + end + elseif tname=="boolean" then + if name then + handle("return {") else - handle("t={") - end - if root then - -- The dummy access will initialize a table that has a delayed initialization - -- using a metatable. (maybe explicitly test for metatable) - if getmetatable(root) then -- todo: make this an option, maybe even per subtable - local dummy = root._w_h_a_t_e_v_e_r_ - root._w_h_a_t_e_v_e_r_ = nil - end - -- Let's forget about empty tables. - if next(root) then - do_serialize(root,name,"",0) - end + handle("{") end - handle("}") -end - --- name: --- --- true : return { } --- false : { } --- nil : t = { } --- string : string = { } --- "return" : return { } --- number : [number] = { } - -function table.serialize(root,name,specification) - local t, n = { }, 0 - local function flush(s) - n = n + 1 - t[n] = s + else + handle("t={") + end + if root then + if getmetatable(root) then + local dummy=root._w_h_a_t_e_v_e_r_ + root._w_h_a_t_e_v_e_r_=nil end - serialize(flush,root,name,specification) - return concat(t,"\n") -end - -table.tohandle = serialize - --- sometimes tables are real use (zapfino extra pro is some 85M) in which --- case a stepwise serialization is nice; actually, we could consider: --- --- for line in table.serializer(root,name,reduce,noquotes) do --- ...(line) --- end --- --- so this is on the todo list - -local maxtab = 2*1024 - -function table.tofile(filename,root,name,specification) - local f = io.open(filename,'w') - if f then - if maxtab > 1 then - local t, n = { }, 0 - local function flush(s) - n = n + 1 - t[n] = s - if n > maxtab then - f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice - t, n = { }, 0 -- we could recycle t if needed - end - end - serialize(flush,root,name,specification) - f:write(concat(t,"\n"),"\n") - else - local function flush(s) - f:write(s,"\n") - end - serialize(flush,root,name,specification) - end - f:close() - io.flush() + if next(root) then + do_serialize(root,name,"",0) end + end + handle("}") end - -local function flattened(t,f,depth) - if f == nil then - f = { } - depth = 0xFFFF - elseif tonumber(f) then - -- assume that only two arguments are given - depth = f - f = { } - elseif not depth then - depth = 0xFFFF - end - for k, v in next, t do - if type(k) ~= "number" then - if depth > 0 and type(v) == "table" then - flattened(v,f,depth-1) - else - f[k] = v - end - end - end - local n = #f - for k=1,#t do - local v = t[k] - if depth > 0 and type(v) == "table" then - flattened(v,f,depth-1) - n = #f - else - n = n + 1 - f[n] = v - end - end - return f -end - -table.flattened = flattened - -local function unnest(t,f) -- only used in mk, for old times sake - if not f then -- and only relevant for token lists - f = { } -- this one can become obsolete - end - for i=1,#t do - local v = t[i] - if type(v) == "table" then - if type(v[1]) == "table" then - unnest(v,f) - else - f[#f+1] = v - end - else - f[#f+1] = v - end - end - return f -end - -function table.unnest(t) -- bad name - return unnest(t) -end - -local function are_equal(a,b,n,m) -- indexed - if a and b and #a == #b then - n = n or 1 - m = m or #a - for i=n,m do - local ai, bi = a[i], b[i] - if ai==bi then - -- same - elseif type(ai) == "table" and type(bi) == "table" then - if not are_equal(ai,bi) then - return false - end - else - return false - end - end - return true - else - return false - end -end - -local function identical(a,b) -- assumes same structure - for ka, va in next, a do - local vb = b[ka] - if va == vb then - -- same - elseif type(va) == "table" and type(vb) == "table" then - if not identical(va,vb) then - return false - end - else - return false - end - end - return true -end - -table.identical = identical -table.are_equal = are_equal - --- maybe also make a combined one - -function table.compact(t) -- remove empty tables, assumes subtables - if t then - for k, v in next, t do - if not next(v) then -- no type checking - t[k] = nil - end - end - end -end - -function table.contains(t, v) - if t then - for i=1, #t do - if t[i] == v then - return i - end - end - end - return false -end - -function table.count(t) - local n = 0 - for k, v in next, t do - n = n + 1 - end - return n -end - -function table.swapped(t,s) -- hash - local n = { } - if s then - for k, v in next, s do - n[k] = v - end - end - for k, v in next, t do - n[v] = k - end - return n -end - -function table.mirrored(t) -- hash - local n = { } - for k, v in next, t do - n[v] = k - n[k] = v - end - return n -end - -function table.reversed(t) - if t then - local tt, tn = { }, #t - if tn > 0 then - local ttn = 0 - for i=tn,1,-1 do - ttn = ttn + 1 - tt[ttn] = t[i] - end - end - return tt - end -end - -function table.reverse(t) - if t then - local n = #t - for i=1,floor(n/2) do - local j = n - i + 1 - t[i], t[j] = t[j], t[i] - end - return t - end -end - -function table.sequenced(t,sep) -- hash only - if t then - local s, n = { }, 0 - for k, v in sortedhash(t) do - if simple then - if v == true then - n = n + 1 - s[n] = k - elseif v and v~= "" then - n = n + 1 - s[n] = k .. "=" .. tostring(v) - end - else - n = n + 1 - s[n] = k .. "=" .. tostring(v) - end - end - return concat(s, sep or " | ") - else - return "" - end -end - -function table.print(t,...) - if type(t) ~= "table" then - print(tostring(t)) - else - table.tohandle(print,t,...) - end -end - --- -- -- obsolete but we keep them for a while and might comment them later -- -- -- - --- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) - -function table.sub(t,i,j) - return { unpack(t,i,j) } -end - --- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) - -function table.is_empty(t) - return not t or not next(t) -end - -function table.has_one_entry(t) - return t and not next(t,next(t)) -end - --- new - -function table.loweredkeys(t) -- maybe utf - local l = { } - for k, v in next, t do - l[lower(k)] = v - end - return l -end - --- new, might move (maybe duplicate) - -function table.unique(old) - local hash = { } - local new = { } - local n = 0 - for i=1,#old do - local oi = old[i] - if not hash[oi] then - n = n + 1 - new[n] = oi - hash[oi] = true - end - end - return new -end - -function table.sorted(t,...) - sort(t,...) - return t -- still sorts in-place -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-io'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local io = io -local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format -local concat = table.concat -local floor = math.floor -local type = type - -if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator = "\\", ";" -else - io.fileseparator, io.pathseparator = "/" , ":" -end - -local function readall(f) - return f:read("*all") -end - --- The next one is upto 50% faster on large files and less memory consumption due --- to less intermediate large allocations. This phenomena was discussed on the --- luatex dev list. - -local function readall(f) - local size = f:seek("end") - if size == 0 then - return "" - elseif size < 1024*1024 then - f:seek("set",0) - return f:read('*all') - else - local done = f:seek("set",0) - if size < 1024*1024 then - step = 1024 * 1024 - elseif size > 16*1024*1024 then - step = 16*1024*1024 - else - step = floor(size/(1024*1024)) * 1024 * 1024 / 8 - end - local data = { } - while true do - local r = f:read(step) - if not r then - return concat(data) - else - data[#data+1] = r - end - end - end -end - -io.readall = readall - -function io.loaddata(filename,textmode) -- return nil if empty - local f = io.open(filename,(textmode and 'r') or 'rb') - if f then --- local data = f:read('*all') - local data = readall(f) - f:close() - if #data > 0 then - return data - end - end -end - -function io.savedata(filename,data,joiner) - local f = io.open(filename,"wb") - if f then - if type(data) == "table" then - f:write(concat(data,joiner or "")) - elseif type(data) == "function" then - data(f) - else - f:write(data or "") - end - f:close() - io.flush() - return true - else - return false - end -end - -function io.loadlines(filename,n) -- return nil if empty - local f = io.open(filename,'r') - if not f then - -- no file - elseif n then - local lines = { } - for i=1,n do - local line = f:read("*lines") - if line then - lines[#lines+1] = line - else - break - end - end - f:close() - lines = concat(lines,"\n") - if #lines > 0 then - return lines - end - else - local line = f:read("*line") or "" - f:close() - if #line > 0 then - return line - end - end -end - -function io.loadchunk(filename,n) - local f = io.open(filename,'rb') - if f then - local data = f:read(n or 1024) - f:close() - if #data > 0 then - return data - end - end -end - -function io.exists(filename) - local f = io.open(filename) - if f == nil then - return false - else - f:close() - return true - end -end - -function io.size(filename) - local f = io.open(filename) - if f == nil then - return 0 - else - local s = f:seek("end") - f:close() - return s - end -end - -function io.noflines(f) - if type(f) == "string" then - local f = io.open(filename) - if f then - local n = f and io.noflines(f) or 0 - f:close() - return n - else - return 0 - end - else - local n = 0 - for _ in f:lines() do - n = n + 1 - end - f:seek('set',0) - return n - end -end - -local nextchar = { - [ 4] = function(f) - return f:read(1,1,1,1) - end, - [ 2] = function(f) - return f:read(1,1) - end, - [ 1] = function(f) - return f:read(1) - end, - [-2] = function(f) - local a, b = f:read(1,1) - return b, a - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - return d, c, b, a - end -} - -function io.characters(f,n) - if f then - return nextchar[n or 1], f - end -end - -local nextbyte = { - [4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return byte(a), byte(b), byte(c), byte(d) - end - end, - [3] = function(f) - local a, b, c = f:read(1,1,1) - if b then - return byte(a), byte(b), byte(c) - end - end, - [2] = function(f) - local a, b = f:read(1,1) - if b then - return byte(a), byte(b) - end - end, - [1] = function (f) - local a = f:read(1) - if a then - return byte(a) - end - end, - [-2] = function (f) - local a, b = f:read(1,1) - if b then - return byte(b), byte(a) - end - end, - [-3] = function(f) - local a, b, c = f:read(1,1,1) - if b then - return byte(c), byte(b), byte(a) - end - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return byte(d), byte(c), byte(b), byte(a) - end - end -} - -function io.bytes(f,n) - if f then - return nextbyte[n or 1], f - else - return nil, nil - end -end - -function io.ask(question,default,options) - while true do - io.write(question) - if options then - io.write(format(" [%s]",concat(options,"|"))) - end - if default then - io.write(format(" [%s]",default)) - end - io.write(format(" ")) - io.flush() - local answer = io.read() - answer = gsub(answer,"^%s*(.*)%s*$","%1") - if answer == "" and default then - return default - elseif not options then - return answer - else - for k=1,#options do - if options[k] == answer then - return answer - end - end - local pattern = "^" .. answer - for k=1,#options do - local v = options[k] - if find(v,pattern) then - return v - end - end - end - end -end - -local function readnumber(f,n,m) - if m then - f:seek("set",n) - n = m - end - if n == 1 then - return byte(f:read(1)) - elseif n == 2 then - local a, b = byte(f:read(2),1,2) - return 256 * a + b - elseif n == 3 then - local a, b, c = byte(f:read(3),1,3) - return 256*256 * a + 256 * b + c - elseif n == 4 then - local a, b, c, d = byte(f:read(4),1,4) - return 256*256*256 * a + 256*256 * b + 256 * c + d - elseif n == 8 then - local a, b = readnumber(f,4), readnumber(f,4) - return 256 * a + b - elseif n == 12 then - local a, b, c = readnumber(f,4), readnumber(f,4), readnumber(f,4) - return 256*256 * a + 256 * b + c - elseif n == -2 then - local b, a = byte(f:read(2),1,2) - return 256*a + b - elseif n == -3 then - local c, b, a = byte(f:read(3),1,3) - return 256*256 * a + 256 * b + c - elseif n == -4 then - local d, c, b, a = byte(f:read(4),1,4) - return 256*256*256 * a + 256*256 * b + 256*c + d - elseif n == -8 then - local h, g, f, e, d, c, b, a = byte(f:read(8),1,8) - return 256*256*256*256*256*256*256 * a + - 256*256*256*256*256*256 * b + - 256*256*256*256*256 * c + - 256*256*256*256 * d + - 256*256*256 * e + - 256*256 * f + - 256 * g + - h - else - return 0 - end -end - -io.readnumber = readnumber - -function io.readstring(f,n,m) - if m then - f:seek("set",n) - n = m - end - local str = gsub(f:read(n),"\000","") - return str -end - --- - -if not io.i_limiter then function io.i_limiter() end end -- dummy so we can test safely -if not io.o_limiter then function io.o_limiter() end end -- dummy so we can test safely - --- This works quite ok: --- --- function io.piped(command,writer) --- local pipe = io.popen(command) --- -- for line in pipe:lines() do --- -- print(line) --- -- end --- while true do --- local line = pipe:read(1) --- if not line then --- break --- elseif line ~= "\n" then --- writer(line) --- end --- end --- return pipe:close() -- ok, status, (error)code --- end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-number'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module will be replaced when we have the bit library - -local tostring, tonumber = tostring, tonumber -local format, floor, match, rep = string.format, math.floor, string.match, string.rep -local concat, insert = table.concat, table.insert -local lpegmatch = lpeg.match - -number = number or { } -local number = number - -if bit32 then - - local btest, bor = bit32.btest, bit32.bor - - function number.bit(p) - return 2 ^ (p - 1) -- 1-based indexing - end - - number.hasbit = btest - number.setbit = bor - - function number.setbit(x,p) - return btest(x,p) and x or x + p - end - - function number.clearbit(x,p) - return btest(x,p) and x - p or x - end - -else - - -- http://ricilake.blogspot.com/2007/10/iterating-bits-in-lua.html - - function number.bit(p) - return 2 ^ (p - 1) -- 1-based indexing - end - - function number.hasbit(x, p) -- typical call: if hasbit(x, bit(3)) then ... - return x % (p + p) >= p - end - - function number.setbit(x, p) - return (x % (p + p) >= p) and x or x + p - end - - function number.clearbit(x, p) - return (x % (p + p) >= p) and x - p or x - end - -end - --- print(number.tobitstring(8)) --- print(number.tobitstring(14)) --- print(number.tobitstring(66)) --- print(number.tobitstring(0x00)) --- print(number.tobitstring(0xFF)) --- print(number.tobitstring(46260767936,4)) - -if bit32 then - - local bextract = bit32.extract - - local t = { - "0", "0", "0", "0", "0", "0", "0", "0", - "0", "0", "0", "0", "0", "0", "0", "0", - "0", "0", "0", "0", "0", "0", "0", "0", - "0", "0", "0", "0", "0", "0", "0", "0", - } - - function number.tobitstring(b,m) - -- if really needed we can speed this one up - -- because small numbers need less extraction - local n = 32 - for i=0,31 do - local v = bextract(b,i) - local k = 32 - i - if v == 1 then - n = k - t[k] = "1" - else - t[k] = "0" - end - end - if m then - m = 33 - m * 8 - if m < 1 then - m = 1 - end - return concat(t,"",m) - elseif n < 8 then - return concat(t) - elseif n < 16 then - return concat(t,"",9) - elseif n < 24 then - return concat(t,"",17) - else - return concat(t,"",25) - end - end - -else - - function number.tobitstring(n,m) - if n > 0 then - local t = { } - while n > 0 do - insert(t,1,n % 2 > 0 and 1 or 0) - n = floor(n/2) - end - local nn = 8 - #t % 8 - if nn > 0 and nn < 8 then - for i=1,nn do - insert(t,1,0) - end - end - if m then - m = m * 8 - #t - if m > 0 then - insert(t,1,rep("0",m)) - end - end - return concat(t) - elseif m then - rep("00000000",m) - else - return "00000000" - end - end - -end - -function number.valid(str,default) - return tonumber(str) or default or nil -end - -function number.toevenhex(n) - local s = format("%X",n) - if #s % 2 == 0 then - return s - else - return "0" .. s - end -end - --- a,b,c,d,e,f = number.toset(100101) --- --- function number.toset(n) --- return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") --- end --- --- -- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5% --- -- on --- --- for i=1,1000000 do --- local a,b,c,d,e,f,g,h = number.toset(12345678) --- local a,b,c,d = number.toset(1234) --- local a,b,c = number.toset(123) --- local a,b,c = number.toset("123") --- end - -local one = lpeg.C(1-lpeg.S('')/tonumber)^1 - -function number.toset(n) - return lpegmatch(one,tostring(n)) -end - --- function number.bits(n,zero) --- local t, i = { }, (zero and 0) or 1 --- while n > 0 do --- local m = n % 2 --- if m > 0 then --- insert(t,1,i) --- end --- n = floor(n/2) --- i = i + 1 --- end --- return t --- end --- --- -- a bit faster - -local function bits(n,i,...) - if n > 0 then - local m = n % 2 - local n = floor(n/2) - if m > 0 then - return bits(n, i+1, i, ...) - else - return bits(n, i+1, ...) - end - else - return ... - end -end - -function number.bits(n) - return { bits(n,1) } -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-set'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This will become obsolete when we have the bitset library embedded. - -set = set or { } - -local nums = { } -local tabs = { } -local concat = table.concat -local next, type = next, type - -set.create = table.tohash - -function set.tonumber(t) - if next(t) then - local s = "" - -- we could save mem by sorting, but it slows down - for k, v in next, t do - if v then - -- why bother about the leading space - s = s .. " " .. k - end - end - local n = nums[s] - if not n then - n = #tabs + 1 - tabs[n] = t - nums[s] = n - end - return n - else - return 0 - end -end - -function set.totable(n) - if n == 0 then - return { } - else - return tabs[n] or { } - end -end - -function set.tolist(n) - if n == 0 or not tabs[n] then - return "" - else - local t, n = { }, 0 - for k, v in next, tabs[n] do - if v then - n = n + 1 - t[n] = k - end - end - return concat(t," ") - end -end - -function set.contains(n,s) - if type(n) == "table" then - return n[s] - elseif n == 0 then - return false - else - local t = tabs[n] - return t and t[s] - end -end - - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-os'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This file deals with some operating system issues. Please don't bother me --- with the pros and cons of operating systems as they all have their flaws --- and benefits. Bashing one of them won't help solving problems and fixing --- bugs faster and is a waste of time and energy. --- --- path separators: / or \ ... we can use / everywhere --- suffixes : dll so exe ... no big deal --- quotes : we can use "" in most cases --- expansion : unless "" are used * might give side effects --- piping/threads : somewhat different for each os --- locations : specific user file locations and settings can change over time --- --- os.type : windows | unix (new, we already guessed os.platform) --- os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) --- os.platform : extended os.name with architecture - --- os.sleep() => socket.sleep() --- math.randomseed(tonumber(string.sub(string.reverse(tostring(math.floor(socket.gettime()*10000))),1,6))) - --- maybe build io.flush in os.execute - -local os = os -local date, time = os.date, os.time -local find, format, gsub, upper, gmatch = string.find, string.format, string.gsub, string.upper, string.gmatch -local concat = table.concat -local random, ceil, randomseed = math.random, math.ceil, math.randomseed -local rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring = rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring - --- The following code permits traversing the environment table, at least --- in luatex. Internally all environment names are uppercase. - --- The randomseed in Lua is not that random, although this depends on the operating system as well --- as the binary (Luatex is normally okay). But to be sure we set the seed anyway. - -math.initialseed = tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6)) - -randomseed(math.initialseed) - -if not os.__getenv__ then - - os.__getenv__ = os.getenv - os.__setenv__ = os.setenv - - if os.env then - - local osgetenv = os.getenv - local ossetenv = os.setenv - local osenv = os.env local _ = osenv.PATH -- initialize the table - - function os.setenv(k,v) - if v == nil then - v = "" - end - local K = upper(k) - osenv[K] = v - if type(v) == "table" then - v = concat(v,";") -- path - end - ossetenv(K,v) - end - - function os.getenv(k) - local K = upper(k) - local v = osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) - if v == "" then - return nil - else - return v - end - end - - else - - local ossetenv = os.setenv - local osgetenv = os.getenv - local osenv = { } - - function os.setenv(k,v) - if v == nil then - v = "" - end - local K = upper(k) - osenv[K] = v - end - - function os.getenv(k) - local K = upper(k) - local v = osenv[K] or osgetenv(K) or osgetenv(k) - if v == "" then - return nil - else - return v - end - end - - local function __index(t,k) - return os.getenv(k) - end - local function __newindex(t,k,v) - os.setenv(k,v) - end - - os.env = { } - - setmetatable(os.env, { __index = __index, __newindex = __newindex } ) - - end - -end - --- end of environment hack - -local execute, spawn, exec, iopopen, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.popen, io.flush - -function os.execute(...) ioflush() return execute(...) end -function os.spawn (...) ioflush() return spawn (...) end -function os.exec (...) ioflush() return exec (...) end -function io.popen (...) ioflush() return iopopen(...) end - -function os.resultof(command) - local handle = io.popen(command,"r") - return handle and handle:read("*all") or "" -end - -if not io.fileseparator then - if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" - else - io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" - end -end - -os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" -os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" - -if os.type == "windows" then - os.libsuffix, os.binsuffix, os.binsuffixes = 'dll', 'exe', { 'exe', 'cmd', 'bat' } -else - os.libsuffix, os.binsuffix, os.binsuffixes = 'so', '', { '' } -end - -local launchers = { - windows = "start %s", - macosx = "open %s", - unix = "$BROWSER %s &> /dev/null &", -} - -function os.launch(str) - os.execute(format(launchers[os.name] or launchers.unix,str)) -end - -if not os.times then - -- utime = user time - -- stime = system time - -- cutime = children user time - -- cstime = children system time - function os.times() - return { - utime = os.gettimeofday(), -- user - stime = 0, -- system - cutime = 0, -- children user - cstime = 0, -- children system - } - end -end - -os.gettimeofday = os.gettimeofday or os.clock - -local startuptime = os.gettimeofday() - -function os.runtime() - return os.gettimeofday() - startuptime -end - - --- no need for function anymore as we have more clever code and helpers now --- this metatable trickery might as well disappear - -os.resolvers = os.resolvers or { } -- will become private - -local resolvers = os.resolvers - -local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil -local osix = osmt.__index - -osmt.__index = function(t,k) - return (resolvers[k] or osix)(t,k) -end - -setmetatable(os,osmt) - --- we can use HOSTTYPE on some platforms - -local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" - -local function guess() - local architecture = os.resultof("uname -m") or "" - if architecture ~= "" then - return architecture - end - architecture = os.getenv("HOSTTYPE") or "" - if architecture ~= "" then - return architecture - end - return os.resultof("echo $HOSTTYPE") or "" -end - -if platform ~= "" then - - os.platform = platform - -elseif os.type == "windows" then - - -- we could set the variable directly, no function needed here - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" - if find(architecture,"AMD64") then - platform = "mswin-64" - else - platform = "mswin" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "linux" then - - function os.resolvers.platform(t,k) - -- we sometimes have HOSTTYPE set so let's check that first - local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "macosx" then - - --[[ - Identifying the architecture of OSX is quite a mess and this - is the best we can come up with. For some reason $HOSTTYPE is - a kind of pseudo environment variable, not known to the current - environment. And yes, uname cannot be trusted either, so there - is a change that you end up with a 32 bit run on a 64 bit system. - Also, some proper 64 bit intel macs are too cheap (low-end) and - therefore not permitted to run the 64 bit kernel. - ]]-- - - function os.resolvers.platform(t,k) - -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" - -- if architecture == "" then - -- architecture = os.resultof("echo $HOSTTYPE") or "" - -- end - local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" - if architecture == "" then - -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") - platform = "osx-intel" - elseif find(architecture,"i386") then - platform = "osx-intel" - elseif find(architecture,"x86_64") then - platform = "osx-64" - else - platform = "osx-ppc" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "sunos" then - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.resultof("uname -m") or "" - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "freebsd" then - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.resultof("uname -m") or "" - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "kfreebsd" then - - function os.resolvers.platform(t,k) - -- we sometimes have HOSTTYPE set so let's check that first - local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then - platform = "kfreebsd-amd64" - else - platform = "kfreebsd-i386" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -else - - -- platform = "linux" - -- os.setenv("MTX_PLATFORM",platform) - -- os.platform = platform - - function os.resolvers.platform(t,k) - local platform = "linux" - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -end - --- beware, we set the randomseed - --- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the --- version number as well as two reserved bits. All other bits are set using a random or pseudorandom --- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal --- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. --- --- as we don't call this function too often there is not so much risk on repetition - -local t = { 8, 9, "a", "b" } - -function os.uuid() - return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", - random(0xFFFF),random(0xFFFF), - random(0x0FFF), - t[ceil(random(4))] or 8,random(0x0FFF), - random(0xFFFF), - random(0xFFFF),random(0xFFFF),random(0xFFFF) - ) -end - -local d - -function os.timezone(delta) - d = d or tonumber(tonumber(date("%H")-date("!%H"))) - if delta then - if d > 0 then - return format("+%02i:00",d) - else - return format("-%02i:00",-d) - end - else - return 1 - end -end - -local timeformat = format("%%s%s",os.timezone(true)) -local dateformat = "!%Y-%m-%d %H:%M:%S" - -function os.fulltime(t,default) - t = tonumber(t) or 0 - if t > 0 then - -- valid time - elseif default then - return default - else - t = nil - end - return format(timeformat,date(dateformat,t)) -end - -local dateformat = "%Y-%m-%d %H:%M:%S" - -function os.localtime(t,default) - t = tonumber(t) or 0 - if t > 0 then - -- valid time - elseif default then - return default - else - t = nil - end - return date(dateformat,t) -end - -function os.converttime(t,default) - local t = tonumber(t) - if t and t > 0 then - return date(dateformat,t) - else - return default or "-" - end -end - -local memory = { } - -local function which(filename) - local fullname = memory[filename] - if fullname == nil then - local suffix = file.suffix(filename) - local suffixes = suffix == "" and os.binsuffixes or { suffix } - for directory in gmatch(os.getenv("PATH"),"[^" .. io.pathseparator .."]+") do - local df = file.join(directory,filename) - for i=1,#suffixes do - local dfs = file.addsuffix(df,suffixes[i]) - if io.exists(dfs) then - fullname = dfs - break - end - end - end - if not fullname then - fullname = false - end - memory[filename] = fullname - end - return fullname -end - -os.which = which -os.where = which - -function os.today() - return date("!*t") -- table with values -end - -function os.now() - return date("!%Y-%m-%d %H:%M:%S") -- 2011-12-04 14:59:12 -end - - --- print(os.which("inkscape.exe")) --- print(os.which("inkscape")) --- print(os.which("gs.exe")) --- print(os.which("ps2pdf")) - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-file'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- needs a cleanup - -file = file or { } -local file = file - -local insert, concat = table.insert, table.concat -local match = string.match -local lpegmatch = lpeg.match -local getcurrentdir, attributes = lfs.currentdir, lfs.attributes -local checkedsplit = string.checkedsplit - --- local patterns = file.patterns or { } --- file.patterns = patterns - -local P, R, S, C, Cs, Cp, Cc, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpeg.Cp, lpeg.Cc, lpeg.Ct - -local colon = P(":") -local period = P(".") -local periods = P("..") -local fwslash = P("/") -local bwslash = P("\\") -local slashes = S("\\/") -local noperiod = 1-period -local noslashes = 1-slashes -local name = noperiod^1 -local suffix = period/"" * (1-period-slashes)^1 * -1 - -local pattern = C((noslashes^0 * slashes^1)^1) - -local function pathpart(name,default) - return name and lpegmatch(pattern,name) or default or "" -end - -local pattern = (noslashes^0 * slashes)^1 * C(noslashes^1) * -1 - -local function basename(name) - return name and lpegmatch(pattern,name) or name -end - -local pattern = (noslashes^0 * slashes^1)^0 * Cs((1-suffix)^1) * suffix^0 - -local function nameonly(name) - return name and lpegmatch(pattern,name) or name -end - -local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * C(noperiod^1) * -1 - -local function suffixonly(name) - return name and lpegmatch(pattern,name) or "" -end - -file.pathpart = pathpart -file.basename = basename -file.nameonly = nameonly -file.suffixonly = suffixonly -file.suffix = suffixonly - -file.dirname = pathpart -- obsolete -file.extname = suffixonly -- obsolete - --- actually these are schemes - -local drive = C(R("az","AZ")) * colon -local path = C(((1-slashes)^0 * slashes)^0) -local suffix = period * C(P(1-period)^0 * P(-1)) -local base = C((1-suffix)^0) -local rest = C(P(1)^0) - -drive = drive + Cc("") -path = path + Cc("") -base = base + Cc("") -suffix = suffix + Cc("") - -local pattern_a = drive * path * base * suffix -local pattern_b = path * base * suffix -local pattern_c = C(drive * path) * C(base * suffix) -- trick: two extra captures -local pattern_d = path * rest - -function file.splitname(str,splitdrive) - if not str then - -- error - elseif splitdrive then - return lpegmatch(pattern_a,str) -- returns drive, path, base, suffix - else - return lpegmatch(pattern_b,str) -- returns path, base, suffix - end -end - -function file.splitbase(str) - return str and lpegmatch(pattern_d,str) -- returns path, base+suffix -end - -function file.nametotable(str,splitdrive) -- returns table - if str then - local path, drive, subpath, name, base, suffix = lpegmatch(pattern_c,str) - if splitdrive then - return { - path = path, - drive = drive, - subpath = subpath, - name = name, - base = base, - suffix = suffix, - } - else - return { - path = path, - name = name, - base = base, - suffix = suffix, - } - end - end -end - -local pattern = Cs(((period * noperiod^1 * -1)/"" + 1)^1) - -function file.removesuffix(name) - return name and lpegmatch(pattern,name) -end - --- local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * Cp() * noperiod^1 * -1 --- --- function file.addsuffix(name, suffix) --- local p = lpegmatch(pattern,name) --- if p then --- return name --- else --- return name .. "." .. suffix --- end --- end - -local suffix = period/"" * (1-period-slashes)^1 * -1 -local pattern = Cs((noslashes^0 * slashes^1)^0 * ((1-suffix)^1)) * Cs(suffix) - -function file.addsuffix(filename,suffix,criterium) - if not filename or not suffix or suffix == "" then - return filename - elseif criterium == true then - return filename .. "." .. suffix - elseif not criterium then - local n, s = lpegmatch(pattern,filename) - if not s or s == "" then - return filename .. "." .. suffix - else - return filename - end - else - local n, s = lpegmatch(pattern,filename) - if s and s ~= "" then - local t = type(criterium) - if t == "table" then - -- keep if in criterium - for i=1,#criterium do - if s == criterium[i] then - return filename - end - end - elseif t == "string" then - -- keep if criterium - if s == criterium then - return filename - end - end - end - return (n or filename) .. "." .. suffix - end -end - --- print("1 " .. file.addsuffix("name","new") .. " -> name.new") --- print("2 " .. file.addsuffix("name.old","new") .. " -> name.old") --- print("3 " .. file.addsuffix("name.old","new",true) .. " -> name.old.new") --- print("4 " .. file.addsuffix("name.old","new","new") .. " -> name.new") --- print("5 " .. file.addsuffix("name.old","new","old") .. " -> name.old") --- print("6 " .. file.addsuffix("name.old","new","foo") .. " -> name.new") --- print("7 " .. file.addsuffix("name.old","new",{"foo","bar"}) .. " -> name.new") --- print("8 " .. file.addsuffix("name.old","new",{"old","bar"}) .. " -> name.old") - -local suffix = period * (1-period-slashes)^1 * -1 -local pattern = Cs((1-suffix)^0) - -function file.replacesuffix(name,suffix) - if name and suffix and suffix ~= "" then - return lpegmatch(pattern,name) .. "." .. suffix - else - return name - end -end - --- - -local reslasher = lpeg.replacer(P("\\"),"/") - -function file.reslash(str) - return str and lpegmatch(reslasher,str) -end - --- We should be able to use: --- --- local writable = P(1) * P("w") * Cc(true) --- --- function file.is_writable(name) --- local a = attributes(name) or attributes(pathpart(name,".")) --- return a and lpegmatch(writable,a.permissions) or false --- end --- --- But after some testing Taco and I came up with the more robust --- variant: - -function file.is_writable(name) - if not name then - -- error - elseif lfs.isdir(name) then - name = name .. "/m_t_x_t_e_s_t.tmp" - local f = io.open(name,"wb") - if f then - f:close() - os.remove(name) - return true - end - elseif lfs.isfile(name) then - local f = io.open(name,"ab") - if f then - f:close() - return true - end - else - local f = io.open(name,"ab") - if f then - f:close() - os.remove(name) - return true - end - end - return false -end - -local readable = P("r") * Cc(true) - -function file.is_readable(name) - if name then - local a = attributes(name) - return a and lpegmatch(readable,a.permissions) or false - else - return false - end -end - -file.isreadable = file.is_readable -- depricated -file.iswritable = file.is_writable -- depricated - -function file.size(name) - if name then - local a = attributes(name) - return a and a.size or 0 - else - return 0 - end -end - -function file.splitpath(str,separator) -- string .. reslash is a bonus (we could do a direct split) - return str and checkedsplit(lpegmatch(reslasher,str),separator or io.pathseparator) -end - -function file.joinpath(tab,separator) -- table - return tab and concat(tab,separator or io.pathseparator) -- can have trailing // -end - -local stripper = Cs(P(fwslash)^0/"" * reslasher) -local isnetwork = fwslash * fwslash * (1-fwslash) + (1-fwslash-colon)^1 * colon -local isroot = fwslash^1 * -1 -local hasroot = fwslash^1 - -local deslasher = lpeg.replacer(S("\\/")^1,"/") - --- If we have a network or prefix then there is a change that we end up with two --- // in the middle ... we could prevent this if we (1) expand prefixes: and (2) --- split and rebuild as url. Of course we could assume no network paths (which --- makes sense) adn assume either mapped drives (windows) or mounts (unix) but --- then we still have to deal with urls ... anyhow, multiple // are never a real --- problem but just ugly. - -function file.join(...) - local lst = { ... } - local one = lst[1] - if lpegmatch(isnetwork,one) then - local two = lpegmatch(deslasher,concat(lst,"/",2)) - return one .. "/" .. two - elseif lpegmatch(isroot,one) then - local two = lpegmatch(deslasher,concat(lst,"/",2)) - if lpegmatch(hasroot,two) then - return two - else - return "/" .. two - end - elseif one == "" then - return lpegmatch(stripper,concat(lst,"/",2)) - else - return lpegmatch(deslasher,concat(lst,"/")) - end -end - --- print(file.join("c:/whatever","name")) --- print(file.join("//","/y")) --- print(file.join("/","/y")) --- print(file.join("","/y")) --- print(file.join("/x/","/y")) --- print(file.join("x/","/y")) --- print(file.join("http://","/y")) --- print(file.join("http://a","/y")) --- print(file.join("http:///a","/y")) --- print(file.join("//nas-1","/y")) - --- The previous one fails on "a.b/c" so Taco came up with a split based --- variant. After some skyping we got it sort of compatible with the old --- one. After that the anchoring to currentdir was added in a better way. --- Of course there are some optimizations too. Finally we had to deal with --- windows drive prefixes and things like sys://. Eventually gsubs and --- finds were replaced by lpegs. - -local drivespec = R("az","AZ")^1 * colon -local anchors = fwslash + drivespec -local untouched = periods + (1-period)^1 * P(-1) -local splitstarter = (Cs(drivespec * (bwslash/"/" + fwslash)^0) + Cc(false)) * Ct(lpeg.splitat(S("/\\")^1)) -local absolute = fwslash - -function file.collapsepath(str,anchor) - if not str then - return - end - if anchor and not lpegmatch(anchors,str) then - str = getcurrentdir() .. "/" .. str - end - if str == "" or str =="." then - return "." - elseif lpegmatch(untouched,str) then - return lpegmatch(reslasher,str) - end - local starter, oldelements = lpegmatch(splitstarter,str) - local newelements = { } - local i = #oldelements - while i > 0 do - local element = oldelements[i] - if element == '.' then - -- do nothing - elseif element == '..' then - local n = i - 1 - while n > 0 do - local element = oldelements[n] - if element ~= '..' and element ~= '.' then - oldelements[n] = '.' - break - else - n = n - 1 - end - end - if n < 1 then - insert(newelements,1,'..') - end - elseif element ~= "" then - insert(newelements,1,element) - end - i = i - 1 - end - if #newelements == 0 then - return starter or "." - elseif starter then - return starter .. concat(newelements, '/') - elseif lpegmatch(absolute,str) then - return "/" .. concat(newelements,'/') - else - return concat(newelements, '/') - end -end - --- local function test(str) --- print(string.format("%-20s %-15s %-15s",str,file.collapsepath(str),file.collapsepath(str,true))) --- end --- test("a/b.c/d") test("b.c/d") test("b.c/..") --- test("/") test("c:/..") test("sys://..") --- test("") test("./") test(".") test("..") test("./..") test("../..") --- test("a") test("./a") test("/a") test("a/../..") --- test("a/./b/..") test("a/aa/../b/bb") test("a/.././././b/..") test("a/./././b/..") --- test("a/b/c/../..") test("./a/b/c/../..") test("a/b/c/../..") - -local validchars = R("az","09","AZ","--","..") -local pattern_a = lpeg.replacer(1-validchars) -local pattern_a = Cs((validchars + P(1)/"-")^1) -local whatever = P("-")^0 / "" -local pattern_b = Cs(whatever * (1 - whatever * -1)^1) - -function file.robustname(str,strict) - if str then - str = lpegmatch(pattern_a,str) or str - if strict then - return lpegmatch(pattern_b,str) or str -- two step is cleaner (less backtracking) - else - return str - end - end -end - -file.readdata = io.loaddata -file.savedata = io.savedata - -function file.copy(oldname,newname) - if oldname and newname then - file.savedata(newname,io.loaddata(oldname)) - end -end - --- also rewrite previous - -local letter = R("az","AZ") + S("_-+") -local separator = P("://") - -local qualified = period^0 * fwslash - + letter * colon - + letter^1 * separator - + letter^1 * fwslash -local rootbased = fwslash - + letter * colon - -lpeg.patterns.qualified = qualified -lpeg.patterns.rootbased = rootbased - --- ./name ../name /name c: :// name/name - -function file.is_qualified_path(filename) - return filename and lpegmatch(qualified,filename) ~= nil -end - -function file.is_rootbased_path(filename) - return filename and lpegmatch(rootbased,filename) ~= nil -end - --- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end --- --- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } --- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } --- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } --- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } - --- -- maybe: --- --- if os.type == "windows" then --- local currentdir = getcurrentdir --- function getcurrentdir() --- return lpegmatch(reslasher,currentdir()) --- end --- end - --- for myself: - -function file.strip(name,dir) - if name then - local b, a = match(name,"^(.-)" .. dir .. "(.*)$") - return a ~= "" and a or name - end -end - --- local debuglist = { --- "pathpart", "basename", "nameonly", "suffixonly", "suffix", "dirname", "extname", --- "addsuffix", "removesuffix", "replacesuffix", "join", --- "strip","collapsepath", "joinpath", "splitpath", --- } - --- for i=1,#debuglist do --- local name = debuglist[i] --- local f = file[name] --- file[name] = function(...) --- print(name,f(...)) --- return f(...) --- end --- end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-md5'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This also provides file checksums and checkers. - -local md5, file = md5, file -local gsub, format, byte = string.gsub, string.format, string.byte - -local function convert(str,fmt) - return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) -end - -if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end -if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end -if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end - --- local P, Cs, lpegmatch = lpeg.P, lpeg.Cs,lpeg.match --- --- if not md5.HEX then --- local function remap(chr) return format("%02X",byte(chr)) end --- function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end --- end --- --- if not md5.hex then --- local function remap(chr) return format("%02x",byte(chr)) end --- function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end --- end --- --- if not md5.dec then --- local function remap(chr) return format("%03i",byte(chr)) end --- function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end --- end - --- if not md5.HEX then --- local pattern_HEX = Cs( ( P(1) / function(chr) return format("%02X",byte(chr)) end)^0 ) --- function md5.HEX(str) return lpegmatch(pattern_HEX,md5.sum(str)) end --- end --- --- if not md5.hex then --- local pattern_hex = Cs( ( P(1) / function(chr) return format("%02x",byte(chr)) end)^0 ) --- function md5.hex(str) return lpegmatch(pattern_hex,md5.sum(str)) end --- end --- --- if not md5.dec then --- local pattern_dec = Cs( ( P(1) / function(chr) return format("%02i",byte(chr)) end)^0 ) --- function md5.dec(str) return lpegmatch(pattern_dec,md5.sum(str)) end --- end - -function file.needsupdating(oldname,newname,threshold) -- size modification access change - local oldtime = lfs.attributes(oldname,"modification") - if oldtime then - local newtime = lfs.attributes(newname,"modification") - if not newtime then - return true -- no new file, so no updating needed - elseif newtime >= oldtime then - return false -- new file definitely needs updating - elseif oldtime - newtime < (threshold or 1) then - return false -- new file is probably still okay - else - return true -- new file has to be updated - end - else - return false -- no old file, so no updating needed - end -end - -file.needs_updating = file.needsupdating - -function file.syncmtimes(oldname,newname) - local oldtime = lfs.attributes(oldname,"modification") - if oldtime and lfs.isfile(newname) then - lfs.touch(newname,oldtime,oldtime) - end -end - -function file.checksum(name) - if md5 then - local data = io.loaddata(name) - if data then - return md5.HEX(data) - end - end - return nil -end - -function file.loadchecksum(name) - if md5 then - local data = io.loaddata(name .. ".md5") - return data and (gsub(data,"%s","")) - end - return nil -end - -function file.savechecksum(name,checksum) - if not checksum then checksum = file.checksum(name) end - if checksum then - io.savedata(name .. ".md5",checksum) - return checksum - end - return nil -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-url'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local char, format, byte = string.char, string.format, string.byte -local concat = table.concat -local tonumber, type = tonumber, type -local P, C, R, S, Cs, Cc, Ct, Cf, Cg, V = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.Cf, lpeg.Cg, lpeg.V -local lpegmatch, lpegpatterns, replacer = lpeg.match, lpeg.patterns, lpeg.replacer - --- from wikipedia: --- --- foo://username:password@example.com:8042/over/there/index.dtb?type=animal;name=narwhal#nose --- \_/ \_______________/ \_________/ \__/ \___/ \_/ \______________________/ \__/ --- | | | | | | | | --- | userinfo hostname port | | query fragment --- | \________________________________/\_____________|____|/ --- scheme | | | | --- | authority path | | --- | | | --- | path interpretable as filename --- | ___________|____________ | --- / \ / \ | --- urn:example:animal:ferret:nose interpretable as extension - -url = url or { } -local url = url - -local tochar = function(s) return char(tonumber(s,16)) end - -local colon = P(":") -local qmark = P("?") -local hash = P("#") -local slash = P("/") -local percent = P("%") -local endofstring = P(-1) - -local hexdigit = R("09","AF","af") -local plus = P("+") -local nothing = Cc("") -local escapedchar = (percent * C(hexdigit * hexdigit)) / tochar -local escaped = (plus / " ") + escapedchar - -local noslash = P("/") / "" - --- we assume schemes with more than 1 character (in order to avoid problems with windows disks) --- we also assume that when we have a scheme, we also have an authority --- --- maybe we should already split the query (better for unescaping as = & can be part of a value - -local schemestr = Cs((escaped+(1-colon-slash-qmark-hash))^2) -local authoritystr = Cs((escaped+(1- slash-qmark-hash))^0) -local pathstr = Cs((escaped+(1- qmark-hash))^0) ------ querystr = Cs((escaped+(1- hash))^0) -local querystr = Cs(( (1- hash))^0) -local fragmentstr = Cs((escaped+(1- endofstring))^0) - -local scheme = schemestr * colon + nothing -local authority = slash * slash * authoritystr + nothing -local path = slash * pathstr + nothing -local query = qmark * querystr + nothing -local fragment = hash * fragmentstr + nothing - -local validurl = scheme * authority * path * query * fragment -local parser = Ct(validurl) - -lpegpatterns.url = validurl -lpegpatterns.urlsplitter = parser - -local escapes = { } - -setmetatable(escapes, { __index = function(t,k) - local v = format("%%%02X",byte(k)) - t[k] = v - return v -end }) - -local escaper = Cs((R("09","AZ","az")^1 + P(" ")/"%%20" + S("-./_")^1 + P(1) / escapes)^0) -- space happens most -local unescaper = Cs((escapedchar + 1)^0) - -lpegpatterns.urlunescaped = escapedchar -lpegpatterns.urlescaper = escaper -lpegpatterns.urlunescaper = unescaper - --- todo: reconsider Ct as we can as well have five return values (saves a table) --- so we can have two parsers, one with and one without - -local function split(str) - return (type(str) == "string" and lpegmatch(parser,str)) or str -end - -local isscheme = schemestr * colon * slash * slash -- this test also assumes authority - -local function hasscheme(str) - if str then - local scheme = lpegmatch(isscheme,str) -- at least one character - return scheme ~= "" and scheme or false - else - return false - end -end - - --- todo: cache them - -local rootletter = R("az","AZ") - + S("_-+") -local separator = P("://") -local qualified = P(".")^0 * P("/") - + rootletter * P(":") - + rootletter^1 * separator - + rootletter^1 * P("/") -local rootbased = P("/") - + rootletter * P(":") - -local barswapper = replacer("|",":") -local backslashswapper = replacer("\\","/") - --- queries: - -local equal = P("=") -local amp = P("&") -local key = Cs(((escapedchar+1)-equal )^0) -local value = Cs(((escapedchar+1)-amp -endofstring)^0) - -local splitquery = Cf ( Ct("") * P { "sequence", - sequence = V("pair") * (amp * V("pair"))^0, - pair = Cg(key * equal * value), -}, rawset) - --- hasher - -local function hashed(str) -- not yet ok (/test?test) - if str == "" then - return { - scheme = "invalid", - original = str, - } - end - local s = split(str) - local rawscheme = s[1] - local rawquery = s[4] - local somescheme = rawscheme ~= "" - local somequery = rawquery ~= "" - if not somescheme and not somequery then - s = { - scheme = "file", - authority = "", - path = str, - query = "", - fragment = "", - original = str, - noscheme = true, - filename = str, - } - else -- not always a filename but handy anyway - local authority, path, filename = s[2], s[3] - if authority == "" then - filename = path - elseif path == "" then - filename = "" - else - filename = authority .. "/" .. path - end - s = { - scheme = rawscheme, - authority = authority, - path = path, - query = lpegmatch(unescaper,rawquery), -- unescaped, but possible conflict with & and = - queries = lpegmatch(splitquery,rawquery), -- split first and then unescaped - fragment = s[5], - original = str, - noscheme = false, - filename = filename, - } - end - return s -end - --- inspect(hashed("template://test")) - --- Here we assume: --- --- files: /// = relative --- files: //// = absolute (!) - - - -url.split = split -url.hasscheme = hasscheme -url.hashed = hashed - -function url.addscheme(str,scheme) -- no authority - if hasscheme(str) then - return str - elseif not scheme then - return "file:///" .. str - else - return scheme .. ":///" .. str - end -end - -function url.construct(hash) -- dodo: we need to escape ! - local fullurl, f = { }, 0 - local scheme, authority, path, query, fragment = hash.scheme, hash.authority, hash.path, hash.query, hash.fragment - if scheme and scheme ~= "" then - f = f + 1 ; fullurl[f] = scheme .. "://" - end - if authority and authority ~= "" then - f = f + 1 ; fullurl[f] = authority - end - if path and path ~= "" then - f = f + 1 ; fullurl[f] = "/" .. path - end - if query and query ~= "" then - f = f + 1 ; fullurl[f] = "?".. query - end - if fragment and fragment ~= "" then - f = f + 1 ; fullurl[f] = "#".. fragment - end - return lpegmatch(escaper,concat(fullurl)) -end - -local pattern = Cs(noslash * R("az","AZ") * (S(":|")/":") * noslash * P(1)^0) - -function url.filename(filename) - local spec = hashed(filename) - local path = spec.path - return (spec.scheme == "file" and path and lpegmatch(pattern,path)) or filename -end - --- print(url.filename("/c|/test")) --- print(url.filename("/c/test")) - -local function escapestring(str) - return lpegmatch(escaper,str) -end - -url.escape = escapestring - -function url.query(str) - if type(str) == "string" then - return lpegmatch(splitquery,str) or "" - else - return str - end -end - -function url.toquery(data) - local td = type(data) - if td == "string" then - return #str and escape(data) or nil -- beware of double escaping - elseif td == "table" then - if next(data) then - local t = { } - for k, v in next, data do - t[#t+1] = format("%s=%s",k,escapestring(v)) - end - return concat(t,"&") - end - else - -- nil is a signal that no query - end -end - --- /test/ | /test | test/ | test => test - -local pattern = Cs(noslash^0 * (1 - noslash * P(-1))^0) - -function url.barepath(path) - if not path or path == "" then - return "" - else - return lpegmatch(pattern,path) - end -end - --- print(url.barepath("/test"),url.barepath("test/"),url.barepath("/test/"),url.barepath("test")) --- print(url.barepath("/x/yz"),url.barepath("x/yz/"),url.barepath("/x/yz/"),url.barepath("x/yz")) - - - - - - - - - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-dir'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- dir.expandname will be merged with cleanpath and collapsepath - -local type, select = type, select -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub -local concat, insert, remove = table.concat, table.insert, table.remove -local lpegmatch = lpeg.match - -local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V - -dir = dir or { } -local dir = dir -local lfs = lfs - -local attributes = lfs.attributes -local walkdir = lfs.dir -local isdir = lfs.isdir -local isfile = lfs.isfile -local currentdir = lfs.currentdir - --- in case we load outside luatex - -if not isdir then - function isdir(name) - local a = attributes(name) - return a and a.mode == "directory" - end - lfs.isdir = isdir -end - -if not isfile then - function isfile(name) - local a = attributes(name) - return a and a.mode == "file" - end - lfs.isfile = isfile -end - --- handy - -function dir.current() - return (gsub(currentdir(),"\\","/")) -end - --- optimizing for no find (*) does not save time - - -local lfsisdir = isdir - -local function isdir(path) - path = gsub(path,"[/\\]+$","") - return lfsisdir(path) -end - -lfs.isdir = isdir - -local function globpattern(path,patt,recurse,action) - if path == "/" then - path = path .. "." - elseif not find(path,"/$") then - path = path .. '/' - end - if isdir(path) then -- lfs.isdir does not like trailing / - for name in walkdir(path) do -- lfs.dir accepts trailing / - local full = path .. name - local mode = attributes(full,'mode') - if mode == 'file' then - if find(full,patt) then - action(full) - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - globpattern(full,patt,recurse,action) - end - end - end -end - -dir.globpattern = globpattern - -local function collectpattern(path,patt,recurse,result) - local ok, scanner - result = result or { } - if path == "/" then - ok, scanner, first = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe - else - ok, scanner, first = xpcall(function() return walkdir(path) end, function() end) -- kepler safe - end - if ok and type(scanner) == "function" then - if not find(path,"/$") then path = path .. '/' end - for name in scanner, first do - local full = path .. name - local attr = attributes(full) - local mode = attr.mode - if mode == 'file' then - if find(full,patt) then - result[name] = attr - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - attr.list = collectpattern(full,patt,recurse) - result[name] = attr - end - end - end - return result -end - -dir.collectpattern = collectpattern - -local pattern = Ct { - [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), - [2] = C(((1-S("*?/"))^0 * P("/"))^0), - [3] = C(P(1)^0) -} - -local filter = Cs ( ( - P("**") / ".*" + - P("*") / "[^/]*" + - P("?") / "[^/]" + - P(".") / "%%." + - P("+") / "%%+" + - P("-") / "%%-" + - P(1) -)^0 ) - -local function glob(str,t) - if type(t) == "function" then - if type(str) == "table" then - for s=1,#str do - glob(str[s],t) - end - elseif isfile(str) then - t(str) - else - local split = lpegmatch(pattern,str) -- we could use the file splitter - if split then - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = lpegmatch(filter,start .. base) - globpattern(start,result,recurse,t) - end - end - else - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) - end - return t - elseif isfile(str) then - if t then - t[#t+1] = str - return t - else - return { str } - end - else - local split = lpegmatch(pattern,str) -- we could use the file splitter - if split then - local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = lpegmatch(filter,start .. base) - globpattern(start,result,recurse,action) - return t - else - return { } - end - end - end -end - -dir.glob = glob - - -local function globfiles(path,recurse,func,files) -- func == pattern or function - if type(func) == "string" then - local s = func - func = function(name) return find(name,s) end - end - files = files or { } - local noffiles = #files - for name in walkdir(path) do - if find(name,"^%.") then - --- skip - else - local mode = attributes(name,'mode') - if mode == "directory" then - if recurse then - globfiles(path .. "/" .. name,recurse,func,files) - end - elseif mode == "file" then - if not func or func(name) then - noffiles = noffiles + 1 - files[noffiles] = path .. "/" .. name - end - end - end - end - return files -end - -dir.globfiles = globfiles - --- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") --- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") --- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") --- t = dir.glob("f:/minimal/tex/**/*") --- print(dir.ls("f:/minimal/tex/**/*")) --- print(dir.ls("*.tex")) - -function dir.ls(pattern) - return concat(glob(pattern),"\n") -end - - -local make_indeed = true -- false - -local onwindows = os.type == "windows" or find(os.getenv("PATH"),";") - -if onwindows then - - function dir.mkdirs(...) - local str, pth = "", "" - for i=1,select("#",...) do - local s = select(i,...) - if s == "" then - -- skip - elseif str == "" then - str = s - else - str = str .. "/" .. s - end - end - local first, middle, last - local drive = false - first, middle, last = match(str,"^(//)(//*)(.*)$") - if first then - -- empty network path == local path - else - first, last = match(str,"^(//)/*(.-)$") - if first then - middle, last = match(str,"([^/]+)/+(.-)$") - if middle then - pth = "//" .. middle - else - pth = "//" .. last - last = "" - end - else - first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") - if first then - pth, drive = first .. middle, true - else - middle, last = match(str,"^(/*)(.-)$") - if not middle then - last = str - end - end - end - end - for s in gmatch(last,"[^/]+") do - if pth == "" then - pth = s - elseif drive then - pth, drive = pth .. s, false - else - pth = pth .. "/" .. s - end - if make_indeed and not isdir(pth) then - lfs.mkdir(pth) - end - end - return pth, (isdir(pth) == true) - end - - -else - - function dir.mkdirs(...) - local str, pth = "", "" - for i=1,select("#",...) do - local s = select(i,...) - if s and s ~= "" then -- we catch nil and false - if str ~= "" then - str = str .. "/" .. s - else - str = s - end - end - end - str = gsub(str,"/+","/") - if find(str,"^/") then - pth = "/" - for s in gmatch(str,"[^/]+") do - local first = (pth == "/") - if first then - pth = pth .. s - else - pth = pth .. "/" .. s - end - if make_indeed and not first and not isdir(pth) then - lfs.mkdir(pth) - end - end - else - pth = "." - for s in gmatch(str,"[^/]+") do - pth = pth .. "/" .. s - if make_indeed and not isdir(pth) then - lfs.mkdir(pth) - end - end - end - return pth, (isdir(pth) == true) - end - - -end - -dir.makedirs = dir.mkdirs - --- we can only define it here as it uses dir.current - -if onwindows then - - function dir.expandname(str) -- will be merged with cleanpath and collapsepath - local first, nothing, last = match(str,"^(//)(//*)(.*)$") - if first then - first = dir.current() .. "/" - end - if not first then - first, last = match(str,"^(//)/*(.*)$") - end - if not first then - first, last = match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d = currentdir() - if lfs.chdir(first) then - first = dir.current() - end - lfs.chdir(d) - end - end - if not first then - first, last = dir.current(), str - end - last = gsub(last,"//","/") - last = gsub(last,"/%./","/") - last = gsub(last,"^/*","") - first = gsub(first,"/*$","") - if last == "" or last == "." then - return first - else - return first .. "/" .. last - end - end - -else - - function dir.expandname(str) -- will be merged with cleanpath and collapsepath - if not find(str,"^/") then - str = currentdir() .. "/" .. str - end - str = gsub(str,"//","/") - str = gsub(str,"/%./","/") - str = gsub(str,"(.)/%.$","%1") - return str - end - -end - -file.expandname = dir.expandname -- for convenience - -local stack = { } - -function dir.push(newdir) - insert(stack,lfs.currentdir()) -end - -function dir.pop() - local d = remove(stack) - if d then - lfs.chdir(d) - end - return d -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-boolean'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local type, tonumber = type, tonumber - -boolean = boolean or { } -local boolean = boolean - -function boolean.tonumber(b) - if b then return 1 else return 0 end -- test and return or return -end - -function toboolean(str,tolerant) - if str == nil then - return false - elseif str == false then - return false - elseif str == true then - return true - elseif str == "true" then - return true - elseif str == "false" then - return false - elseif not tolerant then - return false - elseif str == 0 then - return false - elseif (tonumber(str) or 0) > 0 then - return true - else - return str == "yes" or str == "on" or str == "t" - end -end - -string.toboolean = toboolean - -function string.booleanstring(str) - if str == nil then - return false - elseif str == false then - return false - elseif str == true then - return true - elseif str == "true" then - return true - elseif str == "false" then - return false - elseif str == 0 then - return false - elseif (tonumber(str) or 0) > 0 then - return true - else - return str == "yes" or str == "on" or str == "t" - end -end - -function string.is_boolean(str,default) - if type(str) == "string" then - if str == "true" or str == "yes" or str == "on" or str == "t" then - return true - elseif str == "false" or str == "no" or str == "off" or str == "f" then - return false - end - end - return default -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-unicode'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module will be reorganized - --- todo: utf.sub replacement (used in syst-aux) - --- we put these in the utf namespace: - -utf = utf or (unicode and unicode.utf8) or { } - -utf.characters = utf.characters or string.utfcharacters -utf.values = utf.values or string.utfvalues - --- string.utfvalues --- string.utfcharacters --- string.characters --- string.characterpairs --- string.bytes --- string.bytepairs - -local type = type -local char, byte, format, sub = string.char, string.byte, string.format, string.sub -local concat = table.concat -local P, C, R, Cs, Ct, Cmt, Cc, Carg = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Carg -local lpegmatch, patterns = lpeg.match, lpeg.patterns - -local bytepairs = string.bytepairs - -local finder = lpeg.finder -local replacer = lpeg.replacer - -local utfvalues = utf.values -local utfgmatch = utf.gmatch -- not always present - -local p_utftype = patterns.utftype -local p_utfoffset = patterns.utfoffset -local p_utf8char = patterns.utf8char -local p_utf8byte = patterns.utf8byte -local p_utfbom = patterns.utfbom -local p_newline = patterns.newline -local p_whitespace = patterns.whitespace - -if not unicode then - - unicode = { utf = utf } -- for a while - -end - -if not utf.char then - - local floor, char = math.floor, string.char - - function utf.char(n) - if n < 0x80 then - -- 0aaaaaaa : 0x80 - return char(n) - elseif n < 0x800 then - -- 110bbbaa : 0xC0 : n >> 6 - -- 10aaaaaa : 0x80 : n & 0x3F - return char( - 0xC0 + floor(n/0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x10000 then - -- 1110bbbb : 0xE0 : n >> 12 - -- 10bbbbaa : 0x80 : (n >> 6) & 0x3F - -- 10aaaaaa : 0x80 : n & 0x3F - return char( - 0xE0 + floor(n/0x1000), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x200000 then - -- 11110ccc : 0xF0 : n >> 18 - -- 10ccbbbb : 0x80 : (n >> 12) & 0x3F - -- 10bbbbaa : 0x80 : (n >> 6) & 0x3F - -- 10aaaaaa : 0x80 : n & 0x3F - -- dddd : ccccc - 1 - return char( - 0xF0 + floor(n/0x40000), - 0x80 + (floor(n/0x1000) % 0x40), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - else - return "" - end - end - -end - -if not utf.byte then - - local utf8byte = patterns.utf8byte - - function utf.byte(c) - return lpegmatch(utf8byte,c) - end - -end - -local utfchar, utfbyte = utf.char, utf.byte - --- As we want to get rid of the (unmaintained) utf library we implement our own --- variants (in due time an independent module): - -function utf.filetype(data) - return data and lpegmatch(p_utftype,data) or "unknown" -end - -local toentities = Cs ( - ( - patterns.utf8one - + ( - patterns.utf8two - + patterns.utf8three - + patterns.utf8four - ) / function(s) local b = utfbyte(s) if b < 127 then return s else return format("&#%X;",b) end end - )^0 -) - -patterns.toentities = toentities - -function utf.toentities(str) - return lpegmatch(toentities,str) -end - - - - -local one = P(1) -local two = C(1) * C(1) -local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1) - --- actually one of them is already utf ... sort of useless this one - --- function utf.char(n) --- if n < 0x80 then --- return char(n) --- elseif n < 0x800 then --- return char( --- 0xC0 + floor(n/0x40), --- 0x80 + (n % 0x40) --- ) --- elseif n < 0x10000 then --- return char( --- 0xE0 + floor(n/0x1000), --- 0x80 + (floor(n/0x40) % 0x40), --- 0x80 + (n % 0x40) --- ) --- elseif n < 0x40000 then --- return char( --- 0xF0 + floor(n/0x40000), --- 0x80 + floor(n/0x1000), --- 0x80 + (floor(n/0x40) % 0x40), --- 0x80 + (n % 0x40) --- ) --- else --- -- return char( --- -- 0xF1 + floor(n/0x1000000), --- -- 0x80 + floor(n/0x40000), --- -- 0x80 + floor(n/0x1000), --- -- 0x80 + (floor(n/0x40) % 0x40), --- -- 0x80 + (n % 0x40) --- -- ) --- return "?" --- end --- end --- --- merge into: - -local pattern = P("\254\255") * Cs( ( - four / function(a,b,c,d) - local ab = 0xFF * byte(a) + byte(b) - local cd = 0xFF * byte(c) + byte(d) - return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) - end - + two / function(a,b) - return utfchar(byte(a)*256 + byte(b)) - end - + one - )^1 ) - + P("\255\254") * Cs( ( - four / function(b,a,d,c) - local ab = 0xFF * byte(a) + byte(b) - local cd = 0xFF * byte(c) + byte(d) - return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) - end - + two / function(b,a) - return utfchar(byte(a)*256 + byte(b)) - end - + one - )^1 ) - -function string.toutf(s) -- in string namespace - return lpegmatch(pattern,s) or s -- todo: utf32 -end - -local validatedutf = Cs ( - ( - patterns.utf8one - + patterns.utf8two - + patterns.utf8three - + patterns.utf8four - + P(1) / "�" - )^0 -) - -patterns.validatedutf = validatedutf - -function utf.is_valid(str) - return type(str) == "string" and lpegmatch(validatedutf,str) or false -end - -if not utf.len then - - -- -- alternative 1: 0.77 - -- - -- local utfcharcounter = utfbom^-1 * Cs((p_utf8char/'!')^0) - -- - -- function utf.len(str) - -- return #lpegmatch(utfcharcounter,str or "") - -- end - -- - -- -- alternative 2: 1.70 - -- - -- local n = 0 - -- - -- local utfcharcounter = utfbom^-1 * (p_utf8char/function() n = n + 1 end)^0 -- slow - -- - -- function utf.length(str) - -- n = 0 - -- lpegmatch(utfcharcounter,str or "") - -- return n - -- end - -- - -- -- alternative 3: 0.24 (native unicode.utf8.len: 0.047) - - -- local n = 0 - -- - -- -- local utfcharcounter = lpeg.patterns.utfbom^-1 * P ( ( Cp() * ( - -- -- patterns.utf8one ^1 * Cc(1) - -- -- + patterns.utf8two ^1 * Cc(2) - -- -- + patterns.utf8three^1 * Cc(3) - -- -- + patterns.utf8four ^1 * Cc(4) ) * Cp() / function(f,d,t) n = n + (t - f)/d end - -- -- )^0 ) -- just as many captures as below - -- - -- -- local utfcharcounter = lpeg.patterns.utfbom^-1 * P ( ( - -- -- (Cmt(patterns.utf8one ^1,function(_,_,s) n = n + #s return true end)) - -- -- + (Cmt(patterns.utf8two ^1,function(_,_,s) n = n + #s/2 return true end)) - -- -- + (Cmt(patterns.utf8three^1,function(_,_,s) n = n + #s/3 return true end)) - -- -- + (Cmt(patterns.utf8four ^1,function(_,_,s) n = n + #s/4 return true end)) - -- -- )^0 ) -- not interesting as it creates strings but sometimes faster - -- - -- -- The best so far: - -- - -- local utfcharcounter = utfbom^-1 * P ( ( - -- Cp() * (patterns.utf8one )^1 * Cp() / function(f,t) n = n + t - f end - -- + Cp() * (patterns.utf8two )^1 * Cp() / function(f,t) n = n + (t - f)/2 end - -- + Cp() * (patterns.utf8three)^1 * Cp() / function(f,t) n = n + (t - f)/3 end - -- + Cp() * (patterns.utf8four )^1 * Cp() / function(f,t) n = n + (t - f)/4 end - -- )^0 ) - - -- function utf.len(str) - -- n = 0 - -- lpegmatch(utfcharcounter,str or "") - -- return n - -- end - - local n, f = 0, 1 - - local utfcharcounter = patterns.utfbom^-1 * Cmt ( - Cc(1) * patterns.utf8one ^1 - + Cc(2) * patterns.utf8two ^1 - + Cc(3) * patterns.utf8three^1 - + Cc(4) * patterns.utf8four ^1, - function(_,t,d) -- due to Cc no string captures, so faster - n = n + (t - f)/d - f = t - return true - end - )^0 - - function utf.len(str) - n, f = 0, 1 - lpegmatch(utfcharcounter,str or "") - return n - end - -end - -utf.length = utf.len - -if not utf.sub then - - -- inefficient as lpeg just copies ^n - - -- local function sub(str,start,stop) - -- local pattern = p_utf8char^-(start-1) * C(p_utf8char^-(stop-start+1)) - -- inspect(pattern) - -- return lpegmatch(pattern,str) or "" - -- end - - -- local b, e, n, first, last = 0, 0, 0, 0, 0 - -- - -- local function slide(s,p) - -- n = n + 1 - -- if n == first then - -- b = p - -- if not last then - -- return nil - -- end - -- end - -- if n == last then - -- e = p - -- return nil - -- else - -- return p - -- end - -- end - -- - -- local pattern = Cmt(p_utf8char,slide)^0 - -- - -- function utf.sub(str,start,stop) -- todo: from the end - -- if not start then - -- return str - -- end - -- b, e, n, first, last = 0, 0, 0, start, stop - -- lpegmatch(pattern,str) - -- if not stop then - -- return sub(str,b) - -- else - -- return sub(str,b,e-1) - -- end - -- end - - -- print(utf.sub("Hans Hagen is my name")) - -- print(utf.sub("Hans Hagen is my name",5)) - -- print(utf.sub("Hans Hagen is my name",5,10)) - - local utflength = utf.length - - -- also negative indices, upto 10 times slower than a c variant - - local b, e, n, first, last = 0, 0, 0, 0, 0 - - local function slide_zero(s,p) - n = n + 1 - if n >= last then - e = p - 1 - else - return p - end - end - - local function slide_one(s,p) - n = n + 1 - if n == first then - b = p - end - if n >= last then - e = p - 1 - else - return p - end - end - - local function slide_two(s,p) - n = n + 1 - if n == first then - b = p - else - return true - end - end - - local pattern_zero = Cmt(p_utf8char,slide_zero)^0 - local pattern_one = Cmt(p_utf8char,slide_one )^0 - local pattern_two = Cmt(p_utf8char,slide_two )^0 - - function utf.sub(str,start,stop) - if not start then - return str - end - if start == 0 then - start = 1 - end - if not stop then - if start < 0 then - local l = utflength(str) -- we can inline this function if needed - start = l + start - else - start = start - 1 - end - b, n, first = 0, 0, start - lpegmatch(pattern_two,str) - if n >= first then - return sub(str,b) - else - return "" - end - end - if start < 0 or stop < 0 then - local l = utf.length(str) - if start < 0 then - start = l + start - if start <= 0 then - start = 1 - else - start = start + 1 - end - end - if stop < 0 then - stop = l + stop - if stop == 0 then - stop = 1 - else - stop = stop + 1 - end - end - end - if start > stop then - return "" - elseif start > 1 then - b, e, n, first, last = 0, 0, 0, start - 1, stop - lpegmatch(pattern_one,str) - if n >= first and e == 0 then - e = #str - end - return sub(str,b,e) - else - b, e, n, last = 1, 0, 0, stop - lpegmatch(pattern_zero,str) - if e == 0 then - e = #str - end - return sub(str,b,e) - end - end - - -- local n = 100000 - -- local str = string.rep("123456àáâãäå",100) - -- - -- for i=-15,15,1 do - -- for j=-15,15,1 do - -- if utf.xsub(str,i,j) ~= utf.sub(str,i,j) then - -- print("error",i,j,"l>"..utf.xsub(str,i,j),"s>"..utf.sub(str,i,j)) - -- end - -- end - -- if utf.xsub(str,i) ~= utf.sub(str,i) then - -- print("error",i,"l>"..utf.xsub(str,i),"s>"..utf.sub(str,i)) - -- end - -- end - - -- print(" 1, 7",utf.xsub(str, 1, 7),utf.sub(str, 1, 7)) - -- print(" 0, 7",utf.xsub(str, 0, 7),utf.sub(str, 0, 7)) - -- print(" 0, 9",utf.xsub(str, 0, 9),utf.sub(str, 0, 9)) - -- print(" 4 ",utf.xsub(str, 4 ),utf.sub(str, 4 )) - -- print(" 0 ",utf.xsub(str, 0 ),utf.sub(str, 0 )) - -- print(" 0, 0",utf.xsub(str, 0, 0),utf.sub(str, 0, 0)) - -- print(" 4, 4",utf.xsub(str, 4, 4),utf.sub(str, 4, 4)) - -- print(" 4, 0",utf.xsub(str, 4, 0),utf.sub(str, 4, 0)) - -- print("-3, 0",utf.xsub(str,-3, 0),utf.sub(str,-3, 0)) - -- print(" 0,-3",utf.xsub(str, 0,-3),utf.sub(str, 0,-3)) - -- print(" 5,-3",utf.xsub(str,-5,-3),utf.sub(str,-5,-3)) - -- print("-3 ",utf.xsub(str,-3 ),utf.sub(str,-3 )) - -end - --- a replacement for simple gsubs: - -function utf.remapper(mapping) - local pattern = Cs((p_utf8char/mapping)^0) - return function(str) - if not str or str == "" then - return "" - else - return lpegmatch(pattern,str) - end - end, pattern -end - --- local remap = utf.remapper { a = 'd', b = "c", c = "b", d = "a" } --- print(remap("abcd 1234 abcd")) - --- - -function utf.replacer(t) -- no precheck, always string builder - local r = replacer(t,false,false,true) - return function(str) - return lpegmatch(r,str) - end -end - -function utf.subtituter(t) -- with precheck and no building if no match - local f = finder (t) - local r = replacer(t,false,false,true) - return function(str) - local i = lpegmatch(f,str) - if not i then - return str - elseif i > #str then - return str - else - -- return sub(str,1,i-2) .. lpegmatch(r,str,i-1) -- slower - return lpegmatch(r,str) - end - end -end - --- inspect(utf.split("a b c d")) --- inspect(utf.split("a b c d",true)) - -local utflinesplitter = p_utfbom^-1 * lpeg.tsplitat(p_newline) -local utfcharsplitter_ows = p_utfbom^-1 * Ct(C(p_utf8char)^0) -local utfcharsplitter_iws = p_utfbom^-1 * Ct((p_whitespace^1 + C(p_utf8char))^0) -local utfcharsplitter_raw = Ct(C(p_utf8char)^0) - -patterns.utflinesplitter = utflinesplitter - -function utf.splitlines(str) - return lpegmatch(utflinesplitter,str or "") -end - -function utf.split(str,ignorewhitespace) -- new - if ignorewhitespace then - return lpegmatch(utfcharsplitter_iws,str or "") - else - return lpegmatch(utfcharsplitter_ows,str or "") - end -end - -function utf.totable(str) -- keeps bom - return lpegmatch(utfcharsplitter_raw,str) -end - --- 0 EF BB BF UTF-8 --- 1 FF FE UTF-16-little-endian --- 2 FE FF UTF-16-big-endian --- 3 FF FE 00 00 UTF-32-little-endian --- 4 00 00 FE FF UTF-32-big-endian --- --- \000 fails in <= 5.0 but is valid in >=5.1 where %z is depricated - --- utf.name = { --- [0] = 'utf-8', --- [1] = 'utf-16-le', --- [2] = 'utf-16-be', --- [3] = 'utf-32-le', --- [4] = 'utf-32-be' --- } --- --- function utf.magic(f) --- local str = f:read(4) --- if not str then --- f:seek('set') --- return 0 --- -- elseif find(str,"^%z%z\254\255") then -- depricated --- -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged --- elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) --- return 4 --- -- elseif find(str,"^\255\254%z%z") then -- depricated --- -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged --- elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) --- return 3 --- elseif find(str,"^\254\255") then --- f:seek('set',2) --- return 2 --- elseif find(str,"^\255\254") then --- f:seek('set',2) --- return 1 --- elseif find(str,"^\239\187\191") then --- f:seek('set',3) --- return 0 --- else --- f:seek('set') --- return 0 --- end --- end - -function utf.magic(f) -- not used - local str = f:read(4) or "" - local off = lpegmatch(p_utfoffset,str) - if off < 4 then - f:seek('set',off) - end - return lpegmatch(p_utftype,str) -end - -local function utf16_to_utf8_be(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, 0 - for left, right in bytepairs(t[i]) do - if right then - local now = 256*left + right - if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong - more = 0 - r = r + 1 - result[r] = utfchar(now) - elseif now >= 0xD800 and now <= 0xDBFF then - more = now - else - r = r + 1 - result[r] = utfchar(now) - end - end - end - t[i] = concat(result,"",1,r) -- we reused tmp, hence t - end - return t -end - -local function utf16_to_utf8_le(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, 0 - for left, right in bytepairs(t[i]) do - if right then - local now = 256*right + left - if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong - more = 0 - r = r + 1 - result[r] = utfchar(now) - elseif now >= 0xD800 and now <= 0xDBFF then - more = now - else - r = r + 1 - result[r] = utfchar(now) - end - end - end - t[i] = concat(result,"",1,r) -- we reused tmp, hence t - end - return t -end - -local function utf32_to_utf8_be(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, -1 - for a,b in bytepairs(t[i]) do - if a and b then - if more < 0 then - more = 256*256*256*a + 256*256*b - else - r = r + 1 - result[t] = utfchar(more + 256*a + b) - more = -1 - end - else - break - end - end - t[i] = concat(result,"",1,r) - end - return t -end - -local function utf32_to_utf8_le(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, -1 - for a,b in bytepairs(t[i]) do - if a and b then - if more < 0 then - more = 256*b + a - else - r = r + 1 - result[t] = utfchar(more + 256*256*256*b + 256*256*a) - more = -1 - end - else - break - end - end - t[i] = concat(result,"",1,r) - end - return t -end - -utf.utf32_to_utf8_be = utf32_to_utf8_be -utf.utf32_to_utf8_le = utf32_to_utf8_le -utf.utf16_to_utf8_be = utf16_to_utf8_be -utf.utf16_to_utf8_le = utf16_to_utf8_le - -function utf.utf8_to_utf8(t) - return type(t) == "string" and lpegmatch(utflinesplitter,t) or t -end - -function utf.utf16_to_utf8(t,endian) - return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t -end - -function utf.utf32_to_utf8(t,endian) - return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t -end - -local function little(c) - local b = byte(c) - if b < 0x10000 then - return char(b%256,b/256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1%256,b1/256,b2%256,b2/256) - end -end - -local function big(c) - local b = byte(c) - if b < 0x10000 then - return char(b/256,b%256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1/256,b1%256,b2/256,b2%256) - end -end - --- function utf.utf8_to_utf16(str,littleendian) --- if littleendian then --- return char(255,254) .. utfgsub(str,".",little) --- else --- return char(254,255) .. utfgsub(str,".",big) --- end --- end - -local _, l_remap = utf.remapper(little) -local _, b_remap = utf.remapper(big) - -function utf.utf8_to_utf16(str,littleendian) - if littleendian then - return char(255,254) .. lpegmatch(l_remap,str) - else - return char(254,255) .. lpegmatch(b_remap,str) - end -end - --- function utf.tocodes(str,separator) -- can be sped up with an lpeg --- local t, n = { }, 0 --- for u in utfvalues(str) do --- n = n + 1 --- t[n] = format("0x%04X",u) --- end --- return concat(t,separator or " ") --- end - -local pattern = Cs ( - (p_utf8byte / function(unicode ) return format( "0x%04X", unicode) end) * - (p_utf8byte * Carg(1) / function(unicode,separator) return format("%s0x%04X",separator,unicode) end)^0 -) - -function utf.tocodes(str,separator) - return lpegmatch(pattern,str,1,separator or " ") -end - -function utf.ustring(s) - return format("U+%05X",type(s) == "number" and s or utfbyte(s)) -end - -function utf.xstring(s) - return format("0x%05X",type(s) == "number" and s or utfbyte(s)) -end - --- - -local p_nany = p_utf8char / "" - -if utfgmatch then - - function utf.count(str,what) - if type(what) == "string" then - local n = 0 - for _ in utfgmatch(str,what) do - n = n + 1 - end - return n - else -- 4 times slower but still faster than / function - return #lpegmatch(Cs((P(what)/" " + p_nany)^0),str) - end - end - -else - - local cache = { } - - function utf.count(str,what) - if type(what) == "string" then - local p = cache[what] - if not p then - p = Cs((P(what)/" " + p_nany)^0) - cache[p] = p - end - return #lpegmatch(p,str) - else -- 4 times slower but still faster than / function - return #lpegmatch(Cs((P(what)/" " + p_nany)^0),str) - end - end - -end - --- maybe also register as string.utf* - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-math'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan - -if not math.round then - function math.round(x) return floor(x + 0.5) end -end - -if not math.div then - function math.div(n,m) return floor(n/m) end -end - -if not math.mod then - function math.mod(n,m) return n % m end -end - -local pipi = 2*math.pi/360 - -if not math.sind then - function math.sind(d) return sin(d*pipi) end - function math.cosd(d) return cos(d*pipi) end - function math.tand(d) return tan(d*pipi) end -end - -if not math.odd then - function math.odd (n) return n % 2 ~= 0 end - function math.even(n) return n % 2 == 0 end -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-tab'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or {} -utilities.tables = utilities.tables or { } -local tables = utilities.tables - -local format, gmatch, rep, gsub = string.format, string.gmatch, string.rep, string.gsub -local concat, insert, remove = table.concat, table.insert, table.remove -local setmetatable, getmetatable, tonumber, tostring = setmetatable, getmetatable, tonumber, tostring -local type, next, rawset, tonumber, load, select = type, next, rawset, tonumber, load, select -local lpegmatch, P, Cs = lpeg.match, lpeg.P, lpeg.Cs -local serialize = table.serialize - -local splitter = lpeg.tsplitat(".") - -function tables.definetable(target,nofirst,nolast) -- defines undefined tables - local composed, shortcut, t = nil, nil, { } - local snippets = lpegmatch(splitter,target) - for i=1,#snippets - (nolast and 1 or 0) do - local name = snippets[i] - if composed then - composed = shortcut .. "." .. name - shortcut = shortcut .. "_" .. name - t[#t+1] = format("local %s = %s if not %s then %s = { } %s = %s end",shortcut,composed,shortcut,shortcut,composed,shortcut) - else - composed = name - shortcut = name - if not nofirst then - t[#t+1] = format("%s = %s or { }",composed,composed) - end - end - end - if nolast then - composed = shortcut .. "." .. snippets[#snippets] - end - return concat(t,"\n"), composed -end - --- local t = tables.definedtable("a","b","c","d") - -function tables.definedtable(...) - local t = _G - for i=1,select("#",...) do - local li = select(i,...) - local tl = t[li] - if not tl then - tl = { } - t[li] = tl - end - t = tl - end - return t -end - -function tables.accesstable(target,root) - local t = root or _G - for name in gmatch(target,"([^%.]+)") do - t = t[name] - if not t then - return - end - end - return t -end - -function tables.migratetable(target,v,root) - local t = root or _G - local names = string.split(target,".") - for i=1,#names-1 do - local name = names[i] - t[name] = t[name] or { } - t = t[name] - if not t then - return - end - end - t[names[#names]] = v -end - -function tables.removevalue(t,value) -- todo: n - if value then - for i=1,#t do - if t[i] == value then - remove(t,i) - -- remove all, so no: return - end - end - end -end - -function tables.insertbeforevalue(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i,extra) - return - end - end - insert(t,1,extra) -end - -function tables.insertaftervalue(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i+1,extra) - return - end - end - insert(t,#t+1,extra) -end - --- experimental - -local function toxml(t,d,result,step) - for k, v in table.sortedpairs(t) do - if type(v) == "table" then - if type(k) == "number" then - result[#result+1] = format("%s",d,k) - toxml(v,d..step,result,step) - result[#result+1] = format("%s",d,k) - else - result[#result+1] = format("%s<%s>",d,k) - toxml(v,d..step,result,step) - result[#result+1] = format("%s",d,k) - end - elseif type(k) == "number" then - result[#result+1] = format("%s%s",d,k,v,k) - else - result[#result+1] = format("%s<%s>%s",d,k,tostring(v),k) - end - end -end - -function table.toxml(t,name,nobanner,indent,spaces) - local noroot = name == false - local result = (nobanner or noroot) and { } or { "" } - local indent = rep(" ",indent or 0) - local spaces = rep(" ",spaces or 1) - if noroot then - toxml( t, inndent, result, spaces) - else - toxml( { [name or "root"] = t }, indent, result, spaces) - end - return concat(result,"\n") -end - --- also experimental - --- encapsulate(table,utilities.tables) --- encapsulate(table,utilities.tables,true) --- encapsulate(table,true) - -function tables.encapsulate(core,capsule,protect) - if type(capsule) ~= "table" then - protect = true - capsule = { } - end - for key, value in next, core do - if capsule[key] then - print(format("\ninvalid inheritance '%s' in '%s': %s",key,tostring(core))) - os.exit() - else - capsule[key] = value - end - end - if protect then - for key, value in next, core do - core[key] = nil - end - setmetatable(core, { - __index = capsule, - __newindex = function(t,key,value) - if capsule[key] then - print(format("\ninvalid overload '%s' in '%s'",key,tostring(core))) - os.exit() - else - rawset(t,key,value) - end - end - } ) - end -end - -local function fastserialize(t,r,outer) -- no mixes - r[#r+1] = "{" - local n = #t - if n > 0 then - for i=1,n do - local v = t[i] - local tv = type(v) - if tv == "string" then - r[#r+1] = format("%q,",v) - elseif tv == "number" then - r[#r+1] = format("%s,",v) - elseif tv == "table" then - fastserialize(v,r) - elseif tv == "boolean" then - r[#r+1] = format("%s,",tostring(v)) - end - end - else - for k, v in next, t do - local tv = type(v) - if tv == "string" then - r[#r+1] = format("[%q]=%q,",k,v) - elseif tv == "number" then - r[#r+1] = format("[%q]=%s,",k,v) - elseif tv == "table" then - r[#r+1] = format("[%q]=",k) - fastserialize(v,r) - elseif tv == "boolean" then - r[#r+1] = format("[%q]=%s,",k,tostring(v)) - end - end - end - if outer then - r[#r+1] = "}" - else - r[#r+1] = "}," - end - return r -end - --- local f_hashed_string = formatters["[%q]=%q,"] --- local f_hashed_number = formatters["[%q]=%s,"] --- local f_hashed_table = formatters["[%q]="] --- local f_hashed_true = formatters["[%q]=true,"] --- local f_hashed_false = formatters["[%q]=false,"] --- --- local f_indexed_string = formatters["%q,"] --- local f_indexed_number = formatters["%s,"] --- ----- f_indexed_true = formatters["true,"] --- ----- f_indexed_false = formatters["false,"] --- --- local function fastserialize(t,r,outer) -- no mixes --- r[#r+1] = "{" --- local n = #t --- if n > 0 then --- for i=1,n do --- local v = t[i] --- local tv = type(v) --- if tv == "string" then --- r[#r+1] = f_indexed_string(v) --- elseif tv == "number" then --- r[#r+1] = f_indexed_number(v) --- elseif tv == "table" then --- fastserialize(v,r) --- elseif tv == "boolean" then --- -- r[#r+1] = v and f_indexed_true(k) or f_indexed_false(k) --- r[#r+1] = v and "true," or "false," --- end --- end --- else --- for k, v in next, t do --- local tv = type(v) --- if tv == "string" then --- r[#r+1] = f_hashed_string(k,v) --- elseif tv == "number" then --- r[#r+1] = f_hashed_number(k,v) --- elseif tv == "table" then --- r[#r+1] = f_hashed_table(k) --- fastserialize(v,r) --- elseif tv == "boolean" then --- r[#r+1] = v and f_hashed_true(k) or f_hashed_false(k) --- end --- end --- end --- if outer then --- r[#r+1] = "}" --- else --- r[#r+1] = "}," --- end --- return r --- end - -function table.fastserialize(t,prefix) -- so prefix should contain the = - return concat(fastserialize(t,{ prefix or "return" },true)) -end - -function table.deserialize(str) - if not str or str == "" then - return - end - local code = load(str) - if not code then - return - end - code = code() - if not code then - return - end - return code -end - --- inspect(table.fastserialize { a = 1, b = { 4, { 5, 6 } }, c = { d = 7, e = 'f"g\nh' } }) - -function table.load(filename) - if filename then - local t = io.loaddata(filename) - if t and t ~= "" then - t = load(t) - if type(t) == "function" then - t = t() - if type(t) == "table" then - return t - end - end - end - end -end - -function table.save(filename,t,n,...) - io.savedata(filename,serialize(t,n == nil and true or n,...)) -end - -local function slowdrop(t) - local r = { } - local l = { } - for i=1,#t do - local ti = t[i] - local j = 0 - for k, v in next, ti do - j = j + 1 - l[j] = format("%s=%q",k,v) - end - r[i] = format(" {%s},\n",concat(l)) - end - return format("return {\n%s}",concat(r)) -end - -local function fastdrop(t) - local r = { "return {\n" } - for i=1,#t do - local ti = t[i] - r[#r+1] = " {" - for k, v in next, ti do - r[#r+1] = format("%s=%q",k,v) - end - r[#r+1] = "},\n" - end - r[#r+1] = "}" - return concat(r) -end - -function table.drop(t,slow) - if #t == 0 then - return "return { }" - elseif slow == true then - return slowdrop(t) -- less memory - else - return fastdrop(t) -- some 15% faster - end -end - -function table.autokey(t,k) - local v = { } - t[k] = v - return v -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-sto'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local setmetatable, getmetatable = setmetatable, getmetatable - -utilities = utilities or { } -utilities.storage = utilities.storage or { } -local storage = utilities.storage - -local report = texio and texio.write_nl or print - -function storage.mark(t) - if not t then - report("fatal error: storage cannot be marked") - return -- os.exit() - end - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) - end - m.__storage__ = true - return t -end - -function storage.allocate(t) - t = t or { } - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) - end - m.__storage__ = true - return t -end - -function storage.marked(t) - local m = getmetatable(t) - return m and m.__storage__ -end - -function storage.checked(t) - if not t then - report("fatal error: storage has not been allocated") - return -- os.exit() - end - return t -end - --- function utilities.storage.delay(parent,name,filename) --- local m = getmetatable(parent) --- m.__list[name] = filename --- end --- --- function utilities.storage.predefine(parent) --- local list = { } --- local m = getmetatable(parent) or { --- __list = list, --- __index = function(t,k) --- local l = require(list[k]) --- t[k] = l --- return l --- end --- } --- setmetatable(parent,m) --- end --- --- bla = { } --- utilities.storage.predefine(bla) --- utilities.storage.delay(bla,"test","oepsoeps") --- local t = bla.test --- table.print(t) --- print(t.a) - -function storage.setinitializer(data,initialize) - local m = getmetatable(data) or { } - m.__index = function(data,k) - m.__index = nil -- so that we can access the entries during initializing - initialize() - return data[k] - end - setmetatable(data, m) -end - -local keyisvalue = { __index = function(t,k) - t[k] = k - return k -end } - -function storage.sparse(t) - t = t or { } - setmetatable(t,keyisvalue) - return t -end - --- table namespace ? - -local function f_empty () return "" end -- t,k -local function f_self (t,k) t[k] = k return k end -local function f_table (t,k) local v = { } t[k] = v return v end -local function f_ignore() end -- t,k,v - -local t_empty = { __index = f_empty } -local t_self = { __index = f_self } -local t_table = { __index = f_table } -local t_ignore = { __newindex = f_ignore } - -function table.setmetatableindex(t,f) - local m = getmetatable(t) - if m then - if f == "empty" then - m.__index = f_empty - elseif f == "key" then - m.__index = f_self - elseif f == "table" then - m.__index = f_table - else - m.__index = f - end - else - if f == "empty" then - setmetatable(t, t_empty) - elseif f == "key" then - setmetatable(t, t_self) - elseif f == "table" then - setmetatable(t, t_table) - else - setmetatable(t,{ __index = f }) - end - end - return t -end - -function table.setmetatablenewindex(t,f) - local m = getmetatable(t) - if m then - if f == "ignore" then - m.__newindex = f_ignore - else - m.__newindex = f - end - else - if f == "ignore" then - setmetatable(t, t_ignore) - else - setmetatable(t,{ __newindex = f }) - end - end - return t -end - -function table.setmetatablecall(t,f) - local m = getmetatable(t) - if m then - m.__call = f - else - setmetatable(t,{ __call = f }) - end - return t -end - -function table.setmetatablekey(t,key,value) - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) - end - m[key] = value - return t -end - -function table.getmetatablekey(t,key,value) - local m = getmetatable(t) - return m and m[key] -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-str'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or {} -utilities.strings = utilities.strings or { } -local strings = utilities.strings - -local load = load -local format, gsub, rep, sub = string.format, string.gsub, string.rep, string.sub -local concat = table.concat -local P, V, C, S, R, Ct, Cs, Cp, Carg = lpeg.P, lpeg.V, lpeg.C, lpeg.S, lpeg.R, lpeg.Ct, lpeg.Cs, lpeg.Cp, lpeg.Carg -local patterns, lpegmatch = lpeg.patterns, lpeg.match -local utfchar, utfbyte = utf.char, utf.byte -local setmetatableindex = table.setmetatableindex --- - -local stripper = patterns.stripzeros - -local function points(n) - return (not n or n == 0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) -end - -local function basepoints(n) - return (not n or n == 0) and "0bp" or lpegmatch(stripper,format("%.5fbp", n*(7200/7227)/65536)) -end - -number.points = points -number.basepoints = basepoints - --- str = " \n \ntest \n test\ntest " --- print("["..string.gsub(string.collapsecrlf(str),"\n","+").."]") - -local rubish = patterns.spaceortab^0 * patterns.newline -local anyrubish = patterns.spaceortab + patterns.newline -local anything = patterns.anything -local stripped = (patterns.spaceortab^1 / "") * patterns.newline -local leading = rubish^0 / "" -local trailing = (anyrubish^1 * patterns.endofstring) / "" -local redundant = rubish^3 / "\n" - -local pattern = Cs(leading * (trailing + redundant + stripped + anything)^0) - -function strings.collapsecrlf(str) - return lpegmatch(pattern,str) -end - --- The following functions might end up in another namespace. - -local repeaters = { } -- watch how we also moved the -1 in depth-1 to the creator - -function strings.newrepeater(str,offset) - offset = offset or 0 - local s = repeaters[str] - if not s then - s = { } - repeaters[str] = s - end - local t = s[offset] - if t then - return t - end - t = { } - setmetatableindex(t, function(t,k) - if not k then - return "" - end - local n = k + offset - local s = n > 0 and rep(str,n) or "" - t[k] = s - return s - end) - s[offset] = t - return t -end - --- local dashes = strings.newrepeater("--",-1) --- print(dashes[2],dashes[3],dashes[1]) - -local extra, tab, start = 0, 0, 4, 0 - -local nspaces = strings.newrepeater(" ") - -local pattern = - Carg(1) / function(t) - extra, tab, start = 0, t or 7, 1 - end - * Cs(( - Cp() * patterns.tab / function(position) - local current = (position - start + 1) + extra - local spaces = tab-(current-1) % tab - if spaces > 0 then - extra = extra + spaces - 1 - return nspaces[spaces] -- rep(" ",spaces) - else - return "" - end - end - + patterns.newline * Cp() / function(position) - extra, start = 0, position - end - + patterns.anything - )^1) - -function strings.tabtospace(str,tab) - return lpegmatch(pattern,str,1,tab or 7) -end - --- local t = { --- "1234567123456712345671234567", --- "\tb\tc", --- "a\tb\tc", --- "aa\tbb\tcc", --- "aaa\tbbb\tccc", --- "aaaa\tbbbb\tcccc", --- "aaaaa\tbbbbb\tccccc", --- "aaaaaa\tbbbbbb\tcccccc\n aaaaaa\tbbbbbb\tcccccc", --- "one\n two\nxxx three\nxx four\nx five\nsix", --- } --- for k=1,#t do --- print(strings.tabtospace(t[k])) --- end - -function strings.striplong(str) -- strips all leading spaces - str = gsub(str,"^%s*","") - str = gsub(str,"[\n\r]+ *","\n") - return str -end - --- local template = string.striplong([[ --- aaaa --- bb --- cccccc --- ]]) - -function strings.nice(str) - str = gsub(str,"[:%-+_]+"," ") -- maybe more - return str -end - --- Work in progress. Interesting is that compared to the built-in this --- is faster in luatex than in luajittex where we have a comparable speed. - -local n = 0 - --- we are somewhat sloppy in parsing prefixes as it's not that critical --- --- this does not work out ok: --- --- function fnc(...) -- 1,2,3 --- print(...,...,...) -- 1,1,1,2,3 --- end - -local prefix_any = C((S("+- .") + R("09"))^0) -local prefix_tab = C((1-R("az","AZ","09","%%"))^0) - --- we've split all cases as then we can optimize them (let's omit the fuzzy u) - -local format_s = function(f) - n = n + 1 - if f and f ~= "" then - return format("format('%%%ss',(select(%s,...)))",f,n) - else - return format("(select(%s,...))",n) - end -end - -local format_q = function() - n = n + 1 - return format("format('%%q',(select(%s,...)))",n) -- maybe an own lpeg -end - -local format_i = function(f) - n = n + 1 - if f and f ~= "" then - return format("format('%%%si',(select(%s,...)))",f,n) - else - return format("(select(%s,...))",n) - end -end - -local format_d = format_i - -local format_f = function(f) - n = n + 1 - return format("format('%%%sf',(select(%s,...)))",f,n) -end - -local format_g = function(f) - n = n + 1 - return format("format('%%%sg',(select(%s,...)))",f,n) -end - -local format_G = function(f) - n = n + 1 - return format("format('%%%sG',(select(%s,...)))",f,n) -end - -local format_e = function(f) - n = n + 1 - return format("format('%%%se',(select(%s,...)))",f,n) -end - -local format_E = function(f) - n = n + 1 - return format("format('%%%sE',(select(%s,...)))",f,n) -end - -local format_x = function(f) - n = n + 1 - return format("format('%%%sx',(select(%s,...)))",f,n) -end - -local format_X = function(f) - n = n + 1 - return format("format('%%%sX',(select(%s,...)))",f,n) -end - -local format_o = function(f) - n = n + 1 - return format("format('%%%so',(select(%s,...)))",f,n) -end - -local format_c = function() - n = n + 1 - return format("utfchar((select(%s,...)))",n) -end - -local format_r = function(f) - n = n + 1 - return format("format('%%%s.0f',(select(%s,...)))",f,n) -end - -local format_v = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +function table.serialize(root,name,specification) + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + end + serialize(flush,root,name,specification) + return concat(t,"\n") +end +table.tohandle=serialize +local maxtab=2*1024 +function table.tofile(filename,root,name,specification) + local f=io.open(filename,'w') + if f then + if maxtab>1 then + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + if n>maxtab then + f:write(concat(t,"\n"),"\n") + t,n={},0 + end + end + serialize(flush,root,name,specification) + f:write(concat(t,"\n"),"\n") else - return format("format('0x%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) + local function flush(s) + f:write(s,"\n") + end + serialize(flush,root,name,specification) end + f:close() + io.flush() + end end - -local format_V = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +local function flattened(t,f,depth) + if f==nil then + f={} + depth=0xFFFF + elseif tonumber(f) then + depth=f + f={} + elseif not depth then + depth=0xFFFF + end + for k,v in next,t do + if type(k)~="number" then + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + else + f[k]=v + end + end + end + local n=#f + for k=1,#t do + local v=t[k] + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + n=#f + else + n=n+1 + f[n]=v + end + end + return f +end +table.flattened=flattened +local function unnest(t,f) + if not f then + f={} + end + for i=1,#t do + local v=t[i] + if type(v)=="table" then + if type(v[1])=="table" then + unnest(v,f) + else + f[#f+1]=v + end else - return format("format('0x%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) + f[#f+1]=v end + end + return f end - -local format_u = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +function table.unnest(t) + return unnest(t) +end +local function are_equal(a,b,n,m) + if a and b and #a==#b then + n=n or 1 + m=m or #a + for i=n,m do + local ai,bi=a[i],b[i] + if ai==bi then + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end + else + return false + end + end + return true + else + return false + end +end +local function identical(a,b) + for ka,va in next,a do + local vb=b[ka] + if va==vb then + elseif type(va)=="table" and type(vb)=="table" then + if not identical(va,vb) then + return false + end else - return format("format('u+%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) + return false end + end + return true end - -local format_U = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) - else - return format("format('U+%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +table.identical=identical +table.are_equal=are_equal +function table.compact(t) + if t then + for k,v in next,t do + if not next(v) then + t[k]=nil + end end + end end - -local format_p = function() - n = n + 1 - return format("points((select(%s,...)))",n) +function table.contains(t,v) + if t then + for i=1,#t do + if t[i]==v then + return i + end + end + end + return false end - -local format_b = function() - n = n + 1 - return format("basepoints((select(%s,...)))",n) +function table.count(t) + local n=0 + for k,v in next,t do + n=n+1 + end + return n +end +function table.swapped(t,s) + local n={} + if s then + for k,v in next,s do + n[k]=v + end + end + for k,v in next,t do + n[v]=k + end + return n +end +function table.mirrored(t) + local n={} + for k,v in next,t do + n[v]=k + n[k]=v + end + return n end - -local format_t = function(f) - n = n + 1 - if f and f ~= "" then - return format("concat((select(%s,...)),%q)",n,f) - else - return format("concat((select(%s,...)))",n) +function table.reversed(t) + if t then + local tt,tn={},#t + if tn>0 then + local ttn=0 + for i=tn,1,-1 do + ttn=ttn+1 + tt[ttn]=t[i] + end end + return tt + end end - -local format_l = function() - n = n + 1 - return format("(select(%s,...) and 'true' or 'false')",n) +function table.reverse(t) + if t then + local n=#t + for i=1,floor(n/2) do + local j=n-i+1 + t[i],t[j]=t[j],t[i] + end + return t + end +end +function table.sequenced(t,sep) + if t then + local s,n={},0 + for k,v in sortedhash(t) do + if simple then + if v==true then + n=n+1 + s[n]=k + elseif v and v~="" then + n=n+1 + s[n]=k.."="..tostring(v) + end + else + n=n+1 + s[n]=k.."="..tostring(v) + end + end + return concat(s,sep or " | ") + else + return "" + end end - -local format_a = function(s) - return format("%q",s) +function table.print(t,...) + if type(t)~="table" then + print(tostring(t)) + else + table.tohandle(print,t,...) + end end - -local builder = Ct { "start", - start = (P("%") * ( - V("s") + V("q") - + V("i") + V("d") - + V("f") + V("g") + V("G") + V("e") + V("E") - + V("x") + V("X") + V("o") - -- - + V("c") - -- - + V("r") - + V("v") + V("V") + V("u") + V("U") - + V("p") + V("b") - + V("t") - + V("l") - ) - + V("a") - )^0, - -- - ["s"] = (prefix_any * P("s")) / format_s, -- %s => regular %s (string) - ["q"] = (prefix_any * P("q")) / format_q, -- %q => regular %q (quoted string) - ["i"] = (prefix_any * P("i")) / format_i, -- %i => regular %i (integer) - ["d"] = (prefix_any * P("d")) / format_d, -- %d => regular %d (integer) - ["f"] = (prefix_any * P("f")) / format_f, -- %f => regular %f (float) - ["g"] = (prefix_any * P("g")) / format_g, -- %g => regular %g (float) - ["G"] = (prefix_any * P("G")) / format_G, -- %G => regular %G (float) - ["e"] = (prefix_any * P("e")) / format_e, -- %e => regular %e (float) - ["E"] = (prefix_any * P("E")) / format_E, -- %E => regular %E (float) - ["x"] = (prefix_any * P("x")) / format_x, -- %x => regular %x (hexadecimal) - ["X"] = (prefix_any * P("X")) / format_X, -- %X => regular %X (HEXADECIMAL) - ["o"] = (prefix_any * P("o")) / format_o, -- %o => regular %o (octal) - -- - ["c"] = (prefix_any * P("c")) / format_c, -- %c => utf character (extension to regular) - -- - ["r"] = (prefix_any * P("r")) / format_r, -- %r => round - ["v"] = (prefix_any * P("v")) / format_v, -- %v => 0x0a1b2 (when - no 0x) - ["V"] = (prefix_any * P("V")) / format_V, -- %V => 0x0A1B2 (when - no 0x) - ["u"] = (prefix_any * P("u")) / format_u, -- %u => u+0a1b2 (when - no u+) - ["U"] = (prefix_any * P("U")) / format_U, -- %U => U+0A1B2 (when - no U+) - ["p"] = (prefix_any * P("p")) / format_p, -- %p => 12.345pt / maybe: P (and more units) - ["b"] = (prefix_any * P("b")) / format_b, -- %b => 12.342bp / maybe: B (and more units) - ["t"] = (prefix_tab * P("t")) / format_t, -- %t => concat - ["l"] = (prefix_tab * P("l")) / format_l, -- %l => boolean - -- - ["a"] = Cs(((1-P("%"))^1 + P("%%")/"%%")^1) / format_a, -- %a => text (including %%) -} - --- we can be clever and only alias what is needed - -local template = [[ -local format = string.format -local concat = table.concat -local points = number.points -local basepoints = number.basepoints -local utfchar = utf.char -local utfbyte = utf.byte -return function(...) - return %s +function table.sub(t,i,j) + return { unpack(t,i,j) } end -]] - -local function make(t,str) - n = 0 - local p = lpegmatch(builder,str) --- inspect(p) - local c = format(template,concat(p,"..")) --- inspect(c) - formatter = load(c)() - t[str] = formatter - return formatter +function table.is_empty(t) + return not t or not next(t) end - -local formatters = string.formatters or { } -string.formatters = formatters - -setmetatableindex(formatters,make) - -function string.makeformatter(str) - return formatters[str] +function table.has_one_entry(t) + return t and not next(t,next(t)) end - -function string.formatter(str,...) - return formatters[str](...) +function table.loweredkeys(t) + local l={} + for k,v in next,t do + l[lower(k)]=v + end + return l +end +function table.unique(old) + local hash={} + local new={} + local n=0 + for i=1,#old do + local oi=old[i] + if not hash[oi] then + n=n+1 + new[n]=oi + hash[oi]=true + end + end + return new +end +function table.sorted(t,...) + sort(t,...) + return t end - --- local p1 = "%s test %f done %p and %c and %V or %+t or %%" --- local p2 = "%s test %f done %s and %s and 0x%05X or %s or %%" --- --- local t = { 1,2,3,4 } --- local r = "" --- --- local format, formatter, formatters = string.format, string.formatter, string.formatters --- local utfchar, utfbyte, concat, points = utf.char, utf.byte, table.concat, number.points --- --- local c = os.clock() --- local f = formatters[p1] --- for i=1,500000 do --- -- r = formatters[p1]("hans",123.45,123.45,123,"a",t) --- r = formatter(p1,"hans",123.45,123.45,123,"a",t) --- -- r = f("hans",123.45,123.45,123,"a",t) --- end --- print(os.clock()-c,r) --- --- local c = os.clock() --- for i=1,500000 do --- r = format(p2,"hans",123.45,points(123.45),utfchar(123),utfbyte("a"),concat(t,"+")) --- end --- print(os.clock()-c,r) - --- local f = format --- function string.format(fmt,...) --- print(fmt,...) --- return f(fmt,...) --- end end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-mrg'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- hm, quite unreadable - -local gsub, format = string.gsub, string.format -local concat = table.concat -local type, next = type, next +-- original size: 8723, stripped down to: 6325 -utilities = utilities or {} -local merger = utilities.merger or { } -utilities.merger = merger -utilities.report = logs and logs.reporter("system") or print - -merger.strip_comment = true - -local m_begin_merge = "begin library merge" -local m_end_merge = "end library merge" -local m_begin_closure = "do -- create closure to overcome 200 locals limit" -local m_end_closure = "end -- of closure" - -local m_pattern = - "%c+" .. - "%-%-%s+" .. m_begin_merge .. - "%c+(.-)%c+" .. - "%-%-%s+" .. m_end_merge .. - "%c+" - -local m_format = - "\n\n-- " .. m_begin_merge .. - "\n%s\n" .. - "-- " .. m_end_merge .. "\n\n" - -local m_faked = - "-- " .. "created merged file" .. "\n\n" .. - "-- " .. m_begin_merge .. "\n\n" .. - "-- " .. m_end_merge .. "\n\n" - -local function self_fake() - return m_faked +if not modules then modules={} end modules ['l-io']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local io=io +local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format +local concat=table.concat +local floor=math.floor +local type=type +if string.find(os.getenv("PATH"),";") then + io.fileseparator,io.pathseparator="\\",";" +else + io.fileseparator,io.pathseparator="/",":" end - -local function self_nothing() +local function readall(f) + return f:read("*all") +end +local function readall(f) + local size=f:seek("end") + if size==0 then return "" + elseif size<1024*1024 then + f:seek("set",0) + return f:read('*all') + else + local done=f:seek("set",0) + if size<1024*1024 then + step=1024*1024 + elseif size>16*1024*1024 then + step=16*1024*1024 + else + step=floor(size/(1024*1024))*1024*1024/8 + end + local data={} + while true do + local r=f:read(step) + if not r then + return concat(data) + else + data[#data+1]=r + end + end + end end - -local function self_load(name) - local data = io.loaddata(name) or "" - if data == "" then - utilities.report("merge: unknown file %s",name) +io.readall=readall +function io.loaddata(filename,textmode) + local f=io.open(filename,(textmode and 'r') or 'rb') + if f then + local data=readall(f) + f:close() + if #data>0 then + return data + end + end +end +function io.savedata(filename,data,joiner) + local f=io.open(filename,"wb") + if f then + if type(data)=="table" then + f:write(concat(data,joiner or "")) + elseif type(data)=="function" then + data(f) else - utilities.report("merge: inserting %s",name) + f:write(data or "") + end + f:close() + io.flush() + return true + else + return false + end +end +function io.loadlines(filename,n) + local f=io.open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[#lines+1]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines end - return data or "" + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end + end end - -local function self_save(name, data) - if data ~= "" then - if merger.strip_comment then - local n = #data - -- saves some 20K .. scite comments - data = gsub(data,"%-%-~[^\n\r]*[\r\n]","") - -- saves some 20K .. ldx comments - data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-ldx%]%]%-%-","") - utilities.report("merge: %s bytes of comment stripped, %s bytes of code left",n-#data,#data) - end - io.savedata(name,data) - utilities.report("merge: saving %s",name) +function io.loadchunk(filename,n) + local f=io.open(filename,'rb') + if f then + local data=f:read(n or 1024) + f:close() + if #data>0 then + return data end + end end - -local function self_swap(data,code) - return data ~= "" and (gsub(data,m_pattern, function() return format(m_format,code) end, 1)) or "" +function io.exists(filename) + local f=io.open(filename) + if f==nil then + return false + else + f:close() + return true + end end - -local function self_libs(libs,list) - local result, f, frozen, foundpath = { }, nil, false, nil - result[#result+1] = "\n" - if type(libs) == 'string' then libs = { libs } end - if type(list) == 'string' then list = { list } end - for i=1,#libs do - local lib = libs[i] - for j=1,#list do - local pth = gsub(list[j],"\\","/") -- file.clean_path - utilities.report("merge: checking library path %s",pth) - local name = pth .. "/" .. lib - if lfs.isfile(name) then - foundpath = pth - end - end - if foundpath then break end - end - if foundpath then - utilities.report("merge: using library path %s",foundpath) - local right, wrong = { }, { } - for i=1,#libs do - local lib = libs[i] - local fullname = foundpath .. "/" .. lib - if lfs.isfile(fullname) then - utilities.report("merge: using library %s",fullname) - right[#right+1] = lib - result[#result+1] = m_begin_closure - result[#result+1] = io.loaddata(fullname,true) - result[#result+1] = m_end_closure - else - utilities.report("merge: skipping library %s",fullname) - wrong[#wrong+1] = lib - end - end - if #right > 0 then - utilities.report("merge: used libraries: %s",concat(right," ")) - end - if #wrong > 0 then - utilities.report("merge: skipped libraries: %s",concat(wrong," ")) - end +function io.size(filename) + local f=io.open(filename) + if f==nil then + return 0 + else + local s=f:seek("end") + f:close() + return s + end +end +function io.noflines(f) + if type(f)=="string" then + local f=io.open(filename) + if f then + local n=f and io.noflines(f) or 0 + f:close() + return n else - utilities.report("merge: no valid library path found") + return 0 + end + else + local n=0 + for _ in f:lines() do + n=n+1 end - return concat(result, "\n\n") + f:seek('set',0) + return n + end +end +local nextchar={ + [ 4]=function(f) + return f:read(1,1,1,1) + end, + [ 2]=function(f) + return f:read(1,1) + end, + [ 1]=function(f) + return f:read(1) + end, + [-2]=function(f) + local a,b=f:read(1,1) + return b,a + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + return d,c,b,a + end +} +function io.characters(f,n) + if f then + return nextchar[n or 1],f + end end - -function merger.selfcreate(libs,list,target) - if target then - self_save(target,self_swap(self_fake(),self_libs(libs,list))) +local nextbyte={ + [4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(a),byte(b),byte(c),byte(d) + end + end, + [3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(a),byte(b),byte(c) + end + end, + [2]=function(f) + local a,b=f:read(1,1) + if b then + return byte(a),byte(b) + end + end, + [1]=function (f) + local a=f:read(1) + if a then + return byte(a) + end + end, + [-2]=function (f) + local a,b=f:read(1,1) + if b then + return byte(b),byte(a) + end + end, + [-3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(c),byte(b),byte(a) + end + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(d),byte(c),byte(b),byte(a) end + end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1],f + else + return nil,nil + end end - -function merger.selfmerge(name,libs,list,target) - self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) +function io.ask(question,default,options) + while true do + io.write(question) + if options then + io.write(format(" [%s]",concat(options,"|"))) + end + if default then + io.write(format(" [%s]",default)) + end + io.write(format(" ")) + io.flush() + local answer=io.read() + answer=gsub(answer,"^%s*(.*)%s*$","%1") + if answer=="" and default then + return default + elseif not options then + return answer + else + for k=1,#options do + if options[k]==answer then + return answer + end + end + local pattern="^"..answer + for k=1,#options do + local v=options[k] + if find(v,pattern) then + return v + end + end + end + end end - -function merger.selfclean(name) - self_save(name,self_swap(self_load(name),self_nothing())) +local function readnumber(f,n,m) + if m then + f:seek("set",n) + n=m + end + if n==1 then + return byte(f:read(1)) + elseif n==2 then + local a,b=byte(f:read(2),1,2) + return 256*a+b + elseif n==3 then + local a,b,c=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==4 then + local a,b,c,d=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==8 then + local a,b=readnumber(f,4),readnumber(f,4) + return 256*a+b + elseif n==12 then + local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) + return 256*256*a+256*b+c + elseif n==-2 then + local b,a=byte(f:read(2),1,2) + return 256*a+b + elseif n==-3 then + local c,b,a=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==-4 then + local d,c,b,a=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==-8 then + local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) + return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + else + return 0 + end end +io.readnumber=readnumber +function io.readstring(f,n,m) + if m then + f:seek("set",n) + n=m + end + local str=gsub(f:read(n),"\000","") + return str +end +if not io.i_limiter then function io.i_limiter() end end +if not io.o_limiter then function io.o_limiter() end end end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - comment = "the strip code is written by Peter Cawley", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +-- original size: 4851, stripped down to: 2828 -local rep, sub, byte, dump, format = string.rep, string.sub, string.byte, string.dump, string.format -local load, loadfile, type = load, loadfile, type - -utilities = utilities or {} -utilities.lua = utilities.lua or { } -local luautilities = utilities.lua - -utilities.report = logs and logs.reporter("system") or print -- can be overloaded later - -local tracestripping = false -local forcestupidcompile = true -- use internal bytecode compiler -luautilities.stripcode = true -- support stripping when asked for -luautilities.alwaysstripcode = false -- saves 1 meg on 7 meg compressed format file (2012.08.12) -luautilities.nofstrippedchunks = 0 -luautilities.nofstrippedbytes = 0 -local strippedchunks = { } -- allocate() -luautilities.strippedchunks = strippedchunks - -luautilities.suffixes = { - tma = "tma", - tmc = jit and "tmb" or "tmc", - lua = "lua", - luc = jit and "lub" or "luc", - lui = "lui", - luv = "luv", - luj = "luj", - tua = "tua", - tuc = "tuc", +if not modules then modules={} end modules ['l-number']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } - -local function fatalerror(name) - utilities.report(format("fatal error in %q",name or "unknown")) +local tostring,tonumber=tostring,tonumber +local format,floor,match,rep=string.format,math.floor,string.match,string.rep +local concat,insert=table.concat,table.insert +local lpegmatch=lpeg.match +number=number or {} +local number=number +if bit32 then + local btest,bor=bit32.btest,bit32.bor + function number.bit(p) + return 2^(p-1) + end + number.hasbit=btest + number.setbit=bor + function number.setbit(x,p) + return btest(x,p) and x or x+p + end + function number.clearbit(x,p) + return btest(x,p) and x-p or x + end +else + function number.bit(p) + return 2^(p-1) + end + function number.hasbit(x,p) + return x%(p+p)>=p + end + function number.setbit(x,p) + return (x%(p+p)>=p) and x or x+p + end + function number.clearbit(x,p) + return (x%(p+p)>=p) and x-p or x + end end - -if jit or status.luatex_version >= 74 then - - local function register(name) - if tracestripping then - utilities.report("stripped bytecode: %s",name or "unknown") - end - strippedchunks[#strippedchunks+1] = name - luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1 - end - - local function stupidcompile(luafile,lucfile,strip) - local code = io.loaddata(luafile) - if code and code ~= "" then - code = load(code) - if code then - code = dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) - if code and code ~= "" then - register(name) - io.savedata(lucfile,code) - return true, 0 - end - else - fatalerror() - end - else - fatalerror() - end - return false, 0 - end - - -- quite subtle ... doing this wrong incidentally can give more bytes - - function luautilities.loadedluacode(fullname,forcestrip,name) - -- quite subtle ... doing this wrong incidentally can give more bytes - name = name or fullname - local code = loadfile(fullname) - if code then - code() - end - if forcestrip and luautilities.stripcode then - if type(forcestrip) == "function" then - forcestrip = forcestrip(fullname) - end - if forcestrip or luautilities.alwaysstripcode then - register(name) - return load(dump(code,true)), 0 - else - return code, 0 - end - elseif luautilities.alwaysstripcode then - register(name) - return load(dump(code,true)), 0 - else - return code, 0 - end +if bit32 then + local bextract=bit32.extract + local t={ + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + } + function number.tobitstring(b,m) + local n=32 + for i=0,31 do + local v=bextract(b,i) + local k=32-i + if v==1 then + n=k + t[k]="1" + else + t[k]="0" + end end - - function luautilities.strippedloadstring(code,forcestrip,name) -- not executed - if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then - code = load(code) - if not code then - fatalerror(name) - end - register(name) - code = dump(code,true) - end - return load(code), 0 - end - - function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true - utilities.report("lua: compiling %s into %s",luafile,lucfile) - os.remove(lucfile) - local done = stupidcompile(luafile,lucfile,strip ~= false) - if done then - utilities.report("lua: %s dumped into %s (stripped)",luafile,lucfile) - if cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - utilities.report("lua: removing %s",luafile) - os.remove(luafile) - end - end - return done + if m then + m=33-m*8 + if m<1 then + m=1 + end + return concat(t,"",m) + elseif n<8 then + return concat(t) + elseif n<16 then + return concat(t,"",9) + elseif n<24 then + return concat(t,"",17) + else + return concat(t,"",25) end - + end else - - -- The next function was posted by Peter Cawley on the lua list and strips line - -- number information etc. from the bytecode data blob. We only apply this trick - -- when we store data tables. Stripping makes the compressed format file about - -- 1MB smaller (and uncompressed we save at least 6MB). - -- - -- You can consider this feature an experiment, so it might disappear. There is - -- no noticeable gain in runtime although the memory footprint should be somewhat - -- smaller (and the file system has a bit less to deal with). - -- - -- Begin of borrowed code ... works for Lua 5.1 which LuaTeX currently uses ... - - local function register(name,before,after) - local delta = before - after - if tracestripping then - utilities.report("stripped bytecode: %s, before %s, after %s, delta %s",name or "unknown",before,after,delta) + function number.tobitstring(n,m) + if n>0 then + local t={} + while n>0 do + insert(t,1,n%2>0 and 1 or 0) + n=floor(n/2) + end + local nn=8-#t%8 + if nn>0 and nn<8 then + for i=1,nn do + insert(t,1,0) end - strippedchunks[#strippedchunks+1] = name - luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1 - luautilities.nofstrippedbytes = luautilities.nofstrippedbytes + delta - return delta - end - - local strip_code_pc - - if _MAJORVERSION == 5 and _MINORVERSION == 1 then - - strip_code_pc = function(dump,name) - local before = #dump - local version, format, endian, int, size, ins, num = byte(dump,5,11) - local subint - if endian == 1 then - subint = function(dump, i, l) - local val = 0 - for n = l, 1, -1 do - val = val * 256 + byte(dump,i + n - 1) - end - return val, i + l - end - else - subint = function(dump, i, l) - local val = 0 - for n = 1, l, 1 do - val = val * 256 + byte(dump,i + n - 1) - end - return val, i + l - end - end - local strip_function - strip_function = function(dump) - local count, offset = subint(dump, 1, size) - local stripped, dirty = rep("\0", size), offset + count - offset = offset + count + int * 2 + 4 - offset = offset + int + subint(dump, offset, int) * ins - count, offset = subint(dump, offset, int) - for n = 1, count do - local t - t, offset = subint(dump, offset, 1) - if t == 1 then - offset = offset + 1 - elseif t == 4 then - offset = offset + size + subint(dump, offset, size) - elseif t == 3 then - offset = offset + num - end - end - count, offset = subint(dump, offset, int) - stripped = stripped .. sub(dump,dirty, offset - 1) - for n = 1, count do - local proto, off = strip_function(sub(dump,offset, -1)) - stripped, offset = stripped .. proto, offset + off - 1 - end - offset = offset + subint(dump, offset, int) * int + int - count, offset = subint(dump, offset, int) - for n = 1, count do - offset = offset + subint(dump, offset, size) + size + int * 2 - end - count, offset = subint(dump, offset, int) - for n = 1, count do - offset = offset + subint(dump, offset, size) + size - end - stripped = stripped .. rep("\0", int * 3) - return stripped, offset - end - dump = sub(dump,1,12) .. strip_function(sub(dump,13,-1)) - local after = #dump - local delta = register(name,before,after) - return dump, delta + end + if m then + m=m*8-#t + if m>0 then + insert(t,1,rep("0",m)) end - + end + return concat(t) + elseif m then + rep("00000000",m) else - - strip_code_pc = function(dump,name) - return dump, 0 - end - - end - - -- ... end of borrowed code. - - -- quite subtle ... doing this wrong incidentally can give more bytes - - function luautilities.loadedluacode(fullname,forcestrip,name) - -- quite subtle ... doing this wrong incidentally can give more bytes - name = name or fullname - local code = loadfile(fullname) - if code then - code() - end - if forcestrip and luautilities.stripcode then - if type(forcestrip) == "function" then - forcestrip = forcestrip(fullname) - end - if forcestrip then - local code, n = strip_code_pc(dump(code),name) - return load(code), n - elseif luautilities.alwaysstripcode then - return load(strip_code_pc(dump(code),name)) - else - return code, 0 - end - elseif luautilities.alwaysstripcode then - return load(strip_code_pc(dump(code),name)) - else - return code, 0 - end - end - - function luautilities.strippedloadstring(code,forcestrip,name) -- not executed - local n = 0 - if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then - code = load(code) - if not code then - fatalerror(name) - end - code, n = strip_code_pc(dump(code),name) - end - return load(code), n - end - - local function stupidcompile(luafile,lucfile,strip) - local code = io.loaddata(luafile) - local n = 0 - if code and code ~= "" then - code = load(code) - if not code then - fatalerror() - end - code = dump(code) - if strip then - code, n = strip_code_pc(code,luautilities.stripcode or luautilities.alwaysstripcode,luafile) -- last one is reported - end - if code and code ~= "" then - io.savedata(lucfile,code) - end - end - return n + return "00000000" end - - local luac_normal = "texluac -o %q %q" - local luac_strip = "texluac -s -o %q %q" - - function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true - utilities.report("lua: compiling %s into %s",luafile,lucfile) - os.remove(lucfile) - local done = false - if strip ~= false then - strip = true - end - if forcestupidcompile then - fallback = true - elseif strip then - done = os.spawn(format(luac_strip, lucfile,luafile)) == 0 - else - done = os.spawn(format(luac_normal,lucfile,luafile)) == 0 - end - if not done and fallback then - local n = stupidcompile(luafile,lucfile,strip) - if n > 0 then - utilities.report("lua: %s dumped into %s (%i bytes stripped)",luafile,lucfile,n) - else - utilities.report("lua: %s dumped into %s (unstripped)",luafile,lucfile) - end - cleanup = false -- better see how bad it is - done = true -- hm - end - if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - utilities.report("lua: removing %s",luafile) - os.remove(luafile) - end - return done + end +end +function number.valid(str,default) + return tonumber(str) or default or nil +end +function number.toevenhex(n) + local s=format("%X",n) + if #s%2==0 then + return s + else + return "0"..s + end +end +local one=lpeg.C(1-lpeg.S('')/tonumber)^1 +function number.toset(n) + return lpegmatch(one,tostring(n)) +end +local function bits(n,i,...) + if n>0 then + local m=n%2 + local n=floor(n/2) + if m>0 then + return bits(n,i+1,i,...) + else + return bits(n,i+1,...) end - + else + return... + end +end +function number.bits(n) + return { bits(n,1) } end - --- local getmetatable, type = getmetatable, type --- --- local types = { } --- --- function luautilities.registerdatatype(d,name) --- types[getmetatable(d)] = name --- end --- --- function luautilities.datatype(d) --- local t = type(d) --- if t == "userdata" then --- local m = getmetatable(d) --- return m and types[m] or "userdata" --- else --- return t --- end --- end --- --- luautilities.registerdatatype(lpeg.P("!"),"lpeg") --- --- print(luautilities.datatype(lpeg.P("oeps"))) end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-prs'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +-- original size: 1923, stripped down to: 1133 -local lpeg, table, string = lpeg, table, string -local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf, Cp = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.Cp -local lpegmatch, patterns = lpeg.match, lpeg.patterns -local concat, format, gmatch, find = table.concat, string.format, string.gmatch, string.find -local tostring, type, next, rawset = tostring, type, next, rawset - -utilities = utilities or {} -utilities.parsers = utilities.parsers or { } -local parsers = utilities.parsers -parsers.patterns = parsers.patterns or { } - -local setmetatableindex = table.setmetatableindex -local sortedhash = table.sortedhash - --- we share some patterns - -local digit = R("09") -local space = P(' ') -local equal = P("=") -local comma = P(",") -local lbrace = P("{") -local rbrace = P("}") -local lparent = P("(") -local rparent = P(")") -local period = S(".") -local punctuation = S(".,:;") -local spacer = patterns.spacer -local whitespace = patterns.whitespace -local newline = patterns.newline -local anything = patterns.anything -local endofstring = patterns.endofstring - -local nobrace = 1 - ( lbrace + rbrace ) -local noparent = 1 - ( lparent + rparent) - --- we could use a Cf Cg construct - -local escape, left, right = P("\\"), P('{'), P('}') - -patterns.balanced = P { - [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, - [2] = left * V(1) * right +if not modules then modules={} end modules ['l-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } +set=set or {} +local nums={} +local tabs={} +local concat=table.concat +local next,type=next,type +set.create=table.tohash +function set.tonumber(t) + if next(t) then + local s="" + for k,v in next,t do + if v then + s=s.." "..k + end + end + local n=nums[s] + if not n then + n=#tabs+1 + tabs[n]=t + nums[s]=n + end + return n + else + return 0 + end +end +function set.totable(n) + if n==0 then + return {} + else + return tabs[n] or {} + end +end +function set.tolist(n) + if n==0 or not tabs[n] then + return "" + else + local t,n={},0 + for k,v in next,tabs[n] do + if v then + n=n+1 + t[n]=k + end + end + return concat(t," ") + end +end +function set.contains(n,s) + if type(n)=="table" then + return n[s] + elseif n==0 then + return false + else + local t=tabs[n] + return t and t[s] + end +end -local nestedbraces = P { lbrace * (nobrace + V(1))^0 * rbrace } -local nestedparents = P { lparent * (noparent + V(1))^0 * rparent } -local spaces = space^0 -local argument = Cs((lbrace/"") * ((nobrace + nestedbraces)^0) * (rbrace/"")) -local content = (1-endofstring)^0 - -patterns.nestedbraces = nestedbraces -- no capture -patterns.nestedparents = nestedparents -- no capture -patterns.nested = nestedbraces -- no capture -patterns.argument = argument -- argument after e.g. = -patterns.content = content -- rest after e.g = - -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) + C((nestedbraces + (1-comma))^0) - -local key = C((1-equal-comma)^1) -local pattern_a = (space+comma)^0 * (key * equal * value + key * C("")) -local pattern_c = (space+comma)^0 * (key * equal * value) - -local key = C((1-space-equal-comma)^1) -local pattern_b = spaces * comma^0 * spaces * (key * ((spaces * equal * spaces * value) + C(""))) - --- "a=1, b=2, c=3, d={a{b,c}d}, e=12345, f=xx{a{b,c}d}xx, g={}" : outer {} removes, leading spaces ignored - --- todo: rewrite to fold etc --- --- parse = lpeg.Cf(lpeg.Carg(1) * lpeg.Cg(key * equal * value) * separator^0,rawset)^0 -- lpeg.match(parse,"...",1,hash) - -local hash = { } -local function set(key,value) - hash[key] = value -end +end -- of closure -local pattern_a_s = (pattern_a/set)^1 -local pattern_b_s = (pattern_b/set)^1 -local pattern_c_s = (pattern_c/set)^1 +do -- create closure to overcome 200 locals limit -parsers.patterns.settings_to_hash_a = pattern_a_s -parsers.patterns.settings_to_hash_b = pattern_b_s -parsers.patterns.settings_to_hash_c = pattern_c_s +-- original size: 13731, stripped down to: 8450 -function parsers.make_settings_to_hash_pattern(set,how) - if how == "strict" then - return (pattern_c/set)^1 - elseif how == "tolerant" then - return (pattern_b/set)^1 - else - return (pattern_a/set)^1 +if not modules then modules={} end modules ['l-os']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local os=os +local date,time=os.date,os.time +local find,format,gsub,upper,gmatch=string.find,string.format,string.gsub,string.upper,string.gmatch +local concat=table.concat +local random,ceil,randomseed=math.random,math.ceil,math.randomseed +local rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring=rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring +math.initialseed=tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6)) +randomseed(math.initialseed) +if not os.__getenv__ then + os.__getenv__=os.getenv + os.__setenv__=os.setenv + if os.env then + local osgetenv=os.getenv + local ossetenv=os.setenv + local osenv=os.env local _=osenv.PATH + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + if type(v)=="table" then + v=concat(v,";") + end + ossetenv(K,v) end -end - -function parsers.settings_to_hash(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_a_s,str) - return hash - else - return { } + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + else + local ossetenv=os.setenv + local osgetenv=os.getenv + local osenv={} + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + end + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + local function __index(t,k) + return os.getenv(k) + end + local function __newindex(t,k,v) + os.setenv(k,v) end + os.env={} + setmetatable(os.env,{ __index=__index,__newindex=__newindex } ) + end end - -function parsers.settings_to_hash_tolerant(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_b_s,str) - return hash +local execute,spawn,exec,iopopen,ioflush=os.execute,os.spawn or os.execute,os.exec or os.execute,io.popen,io.flush +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end +function io.popen (...) ioflush() return iopopen(...) end +function os.resultof(command) + local handle=io.popen(command,"r") + return handle and handle:read("*all") or "" +end +if not io.fileseparator then + if find(os.getenv("PATH"),";") then + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + else + io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" + end +end +os.type=os.type or (io.pathseparator==";" and "windows") or "unix" +os.name=os.name or (os.type=="windows" and "mswin" ) or "linux" +if os.type=="windows" then + os.libsuffix,os.binsuffix,os.binsuffixes='dll','exe',{ 'exe','cmd','bat' } +else + os.libsuffix,os.binsuffix,os.binsuffixes='so','',{ '' } +end +local launchers={ + windows="start %s", + macosx="open %s", + unix="$BROWSER %s &> /dev/null &", +} +function os.launch(str) + os.execute(format(launchers[os.name] or launchers.unix,str)) +end +if not os.times then + function os.times() + return { + utime=os.gettimeofday(), + stime=0, + cutime=0, + cstime=0, + } + end +end +os.gettimeofday=os.gettimeofday or os.clock +local startuptime=os.gettimeofday() +function os.runtime() + return os.gettimeofday()-startuptime +end +os.resolvers=os.resolvers or {} +local resolvers=os.resolvers +local osmt=getmetatable(os) or { __index=function(t,k) t[k]="unset" return "unset" end } +local osix=osmt.__index +osmt.__index=function(t,k) + return (resolvers[k] or osix)(t,k) +end +setmetatable(os,osmt) +local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" +local function guess() + local architecture=os.resultof("uname -m") or "" + if architecture~="" then + return architecture + end + architecture=os.getenv("HOSTTYPE") or "" + if architecture~="" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end +if platform~="" then + os.platform=platform +elseif os.type=="windows" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform="mswin-64" + else + platform="mswin" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="linux" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform="linux-64" + elseif find(architecture,"ppc") then + platform="linux-ppc" + else + platform="linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="macosx" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + if architecture=="" then + platform="osx-intel" + elseif find(architecture,"i386") then + platform="osx-intel" + elseif find(architecture,"x86_64") then + platform="osx-64" + else + platform="osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="sunos" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform="solaris-sparc" + else + platform="solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="freebsd" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform="freebsd-amd64" + else + platform="freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="kfreebsd" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform="kfreebsd-amd64" + else + platform="kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +else + function os.resolvers.platform(t,k) + local platform="linux" + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +end +local t={ 8,9,"a","b" } +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end +local d +function os.timezone(delta) + d=d or tonumber(tonumber(date("%H")-date("!%H"))) + if delta then + if d>0 then + return format("+%02i:00",d) else - return { } + return format("-%02i:00",-d) end + else + return 1 + end end - -function parsers.settings_to_hash_strict(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_c_s,str) - return next(hash) and hash - else - return nil +local timeformat=format("%%s%s",os.timezone(true)) +local dateformat="!%Y-%m-%d %H:%M:%S" +function os.fulltime(t,default) + t=tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=nil + end + return format(timeformat,date(dateformat,t)) +end +local dateformat="%Y-%m-%d %H:%M:%S" +function os.localtime(t,default) + t=tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=nil + end + return date(dateformat,t) +end +function os.converttime(t,default) + local t=tonumber(t) + if t and t>0 then + return date(dateformat,t) + else + return default or "-" + end +end +local memory={} +local function which(filename) + local fullname=memory[filename] + if fullname==nil then + local suffix=file.suffix(filename) + local suffixes=suffix=="" and os.binsuffixes or { suffix } + for directory in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local df=file.join(directory,filename) + for i=1,#suffixes do + local dfs=file.addsuffix(df,suffixes[i]) + if io.exists(dfs) then + fullname=dfs + break + end + end + end + if not fullname then + fullname=false end + memory[filename]=fullname + end + return fullname +end +os.which=which +os.where=which +function os.today() + return date("!*t") +end +function os.now() + return date("!%Y-%m-%d %H:%M:%S") end -local separator = comma * space^0 -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) - + C((nestedbraces + (1-comma))^0) -local pattern = spaces * Ct(value*(separator*value)^0) --- "aap, {noot}, mies" : outer {} removes, leading spaces ignored +end -- of closure -parsers.patterns.settings_to_array = pattern +do -- create closure to overcome 200 locals limit --- we could use a weak table as cache +-- original size: 15501, stripped down to: 8354 -function parsers.settings_to_array(str,strict) - if not str or str == "" then - return { } - elseif strict then - if find(str,"{") then - return lpegmatch(pattern,str) - else - return { str } +if not modules then modules={} end modules ['l-file']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +file=file or {} +local file=file +local insert,concat=table.insert,table.concat +local match=string.match +local lpegmatch=lpeg.match +local getcurrentdir,attributes=lfs.currentdir,lfs.attributes +local checkedsplit=string.checkedsplit +local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local colon=P(":") +local period=P(".") +local periods=P("..") +local fwslash=P("/") +local bwslash=P("\\") +local slashes=S("\\/") +local noperiod=1-period +local noslashes=1-slashes +local name=noperiod^1 +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=C((1-(slashes^1*noslashes^1*-1))^1)*P(1) +local function pathpart(name,default) + return name and lpegmatch(pattern,name) or default or "" +end +local pattern=(noslashes^0*slashes)^1*C(noslashes^1)*-1 +local function basename(name) + return name and lpegmatch(pattern,name) or name +end +local pattern=(noslashes^0*slashes^1)^0*Cs((1-suffix)^1)*suffix^0 +local function nameonly(name) + return name and lpegmatch(pattern,name) or name +end +local pattern=(noslashes^0*slashes)^0*(noperiod^1*period)^1*C(noperiod^1)*-1 +local function suffixonly(name) + return name and lpegmatch(pattern,name) or "" +end +file.pathpart=pathpart +file.basename=basename +file.nameonly=nameonly +file.suffixonly=suffixonly +file.suffix=suffixonly +file.dirname=pathpart +file.extname=suffixonly +local drive=C(R("az","AZ"))*colon +local path=C((noslashes^0*slashes)^0) +local suffix=period*C(P(1-period)^0*P(-1)) +local base=C((1-suffix)^0) +local rest=C(P(1)^0) +drive=drive+Cc("") +path=path+Cc("") +base=base+Cc("") +suffix=suffix+Cc("") +local pattern_a=drive*path*base*suffix +local pattern_b=path*base*suffix +local pattern_c=C(drive*path)*C(base*suffix) +local pattern_d=path*rest +function file.splitname(str,splitdrive) + if not str then + elseif splitdrive then + return lpegmatch(pattern_a,str) + else + return lpegmatch(pattern_b,str) + end +end +function file.splitbase(str) + return str and lpegmatch(pattern_d,str) +end +function file.nametotable(str,splitdrive) + if str then + local path,drive,subpath,name,base,suffix=lpegmatch(pattern_c,str) + if splitdrive then + return { + path=path, + drive=drive, + subpath=subpath, + name=name, + base=base, + suffix=suffix, + } + else + return { + path=path, + name=name, + base=base, + suffix=suffix, + } + end + end +end +local pattern=Cs(((period*noperiod^1*-1)/""+1)^1) +function file.removesuffix(name) + return name and lpegmatch(pattern,name) +end +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=Cs((noslashes^0*slashes^1)^0*((1-suffix)^1))*Cs(suffix) +function file.addsuffix(filename,suffix,criterium) + if not filename or not suffix or suffix=="" then + return filename + elseif criterium==true then + return filename.."."..suffix + elseif not criterium then + local n,s=lpegmatch(pattern,filename) + if not s or s=="" then + return filename.."."..suffix + else + return filename + end + else + local n,s=lpegmatch(pattern,filename) + if s and s~="" then + local t=type(criterium) + if t=="table" then + for i=1,#criterium do + if s==criterium[i] then + return filename + end end - else - return lpegmatch(pattern,str) + elseif t=="string" then + if s==criterium then + return filename + end + end end + return (n or filename).."."..suffix + end end - -local function set(t,v) - t[#t+1] = v +local suffix=period*(1-period-slashes)^1*-1 +local pattern=Cs((1-suffix)^0) +function file.replacesuffix(name,suffix) + if name and suffix and suffix~="" then + return lpegmatch(pattern,name).."."..suffix + else + return name + end end - -local value = P(Carg(1)*value) / set -local pattern = value*(separator*value)^0 * Carg(1) - -function parsers.add_settings_to_array(t,str) - return lpegmatch(pattern,str,nil,t) +local reslasher=lpeg.replacer(P("\\"),"/") +function file.reslash(str) + return str and lpegmatch(reslasher,str) end - -function parsers.hash_to_string(h,separator,yes,no,strict,omit) - if h then - local t, tn, s = { }, 0, table.sortedkeys(h) - omit = omit and table.tohash(omit) - for i=1,#s do - local key = s[i] - if not omit or not omit[key] then - local value = h[key] - if type(value) == "boolean" then - if yes and no then - if value then - tn = tn + 1 - t[tn] = key .. '=' .. yes - elseif not strict then - tn = tn + 1 - t[tn] = key .. '=' .. no - end - elseif value or not strict then - tn = tn + 1 - t[tn] = key .. '=' .. tostring(value) - end - else - tn = tn + 1 - t[tn] = key .. '=' .. value - end - end - end - return concat(t,separator or ",") - else - return "" +function file.is_writable(name) + if not name then + elseif lfs.isdir(name) then + name=name.."/m_t_x_t_e_s_t.tmp" + local f=io.open(name,"wb") + if f then + f:close() + os.remove(name) + return true + end + elseif lfs.isfile(name) then + local f=io.open(name,"ab") + if f then + f:close() + return true + end + else + local f=io.open(name,"ab") + if f then + f:close() + os.remove(name) + return true end + end + return false end - -function parsers.array_to_string(a,separator) - if a then - return concat(a,separator or ",") +local readable=P("r")*Cc(true) +function file.is_readable(name) + if name then + local a=attributes(name) + return a and lpegmatch(readable,a.permissions) or false + else + return false + end +end +file.isreadable=file.is_readable +file.iswritable=file.is_writable +function file.size(name) + if name then + local a=attributes(name) + return a and a.size or 0 + else + return 0 + end +end +function file.splitpath(str,separator) + return str and checkedsplit(lpegmatch(reslasher,str),separator or io.pathseparator) +end +function file.joinpath(tab,separator) + return tab and concat(tab,separator or io.pathseparator) +end +local stripper=Cs(P(fwslash)^0/""*reslasher) +local isnetwork=fwslash*fwslash*(1-fwslash)+(1-fwslash-colon)^1*colon +local isroot=fwslash^1*-1 +local hasroot=fwslash^1 +local deslasher=lpeg.replacer(S("\\/")^1,"/") +function file.join(...) + local lst={... } + local one=lst[1] + if lpegmatch(isnetwork,one) then + local two=lpegmatch(deslasher,concat(lst,"/",2)) + return one.."/"..two + elseif lpegmatch(isroot,one) then + local two=lpegmatch(deslasher,concat(lst,"/",2)) + if lpegmatch(hasroot,two) then + return two + else + return "/"..two + end + elseif one=="" then + return lpegmatch(stripper,concat(lst,"/",2)) + else + return lpegmatch(deslasher,concat(lst,"/")) + end +end +local drivespec=R("az","AZ")^1*colon +local anchors=fwslash+drivespec +local untouched=periods+(1-period)^1*P(-1) +local splitstarter=(Cs(drivespec*(bwslash/"/"+fwslash)^0)+Cc(false))*Ct(lpeg.splitat(S("/\\")^1)) +local absolute=fwslash +function file.collapsepath(str,anchor) + if not str then + return + end + if anchor and not lpegmatch(anchors,str) then + str=getcurrentdir().."/"..str + end + if str=="" or str=="." then + return "." + elseif lpegmatch(untouched,str) then + return lpegmatch(reslasher,str) + end + local starter,oldelements=lpegmatch(splitstarter,str) + local newelements={} + local i=#oldelements + while i>0 do + local element=oldelements[i] + if element=='.' then + elseif element=='..' then + local n=i-1 + while n>0 do + local element=oldelements[n] + if element~='..' and element~='.' then + oldelements[n]='.' + break + else + n=n-1 + end + end + if n<1 then + insert(newelements,1,'..') + end + elseif element~="" then + insert(newelements,1,element) + end + i=i-1 + end + if #newelements==0 then + return starter or "." + elseif starter then + return starter..concat(newelements,'/') + elseif lpegmatch(absolute,str) then + return "/"..concat(newelements,'/') + else + return concat(newelements,'/') + end +end +local validchars=R("az","09","AZ","--","..") +local pattern_a=lpeg.replacer(1-validchars) +local pattern_a=Cs((validchars+P(1)/"-")^1) +local whatever=P("-")^0/"" +local pattern_b=Cs(whatever*(1-whatever*-1)^1) +function file.robustname(str,strict) + if str then + str=lpegmatch(pattern_a,str) or str + if strict then + return lpegmatch(pattern_b,str) or str else - return "" + return str end + end end - -function parsers.settings_to_set(str,t) -- tohash? -- todo: lpeg -- duplicate anyway - t = t or { } --- for s in gmatch(str,"%s*([^, ]+)") do -- space added - for s in gmatch(str,"[^, ]+") do -- space added - t[s] = true - end - return t +file.readdata=io.loaddata +file.savedata=io.savedata +function file.copy(oldname,newname) + if oldname and newname then + file.savedata(newname,io.loaddata(oldname)) + end +end +local letter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=period^0*fwslash+letter*colon+letter^1*separator+letter^1*fwslash +local rootbased=fwslash+letter*colon +lpeg.patterns.qualified=qualified +lpeg.patterns.rootbased=rootbased +function file.is_qualified_path(filename) + return filename and lpegmatch(qualified,filename)~=nil end - -function parsers.simple_hash_to_string(h, separator) - local t, tn = { }, 0 - for k, v in sortedhash(h) do - if v then - tn = tn + 1 - t[tn] = k - end - end - return concat(t,separator or ",") +function file.is_rootbased_path(filename) + return filename and lpegmatch(rootbased,filename)~=nil +end +function file.strip(name,dir) + if name then + local b,a=match(name,"^(.-)"..dir.."(.*)$") + return a~="" and a or name + end end --- for chem (currently one level) -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) - + C(digit^1 * lparent * (noparent + nestedparents)^1 * rparent) - + C((nestedbraces + (1-comma))^1) -local pattern_a = spaces * Ct(value*(separator*value)^0) +end -- of closure -local function repeater(n,str) - if not n then - return str - else - local s = lpegmatch(pattern_a,str) - if n == 1 then - return unpack(s) - else - local t, tn = { }, 0 - for i=1,n do - for j=1,#s do - tn = tn + 1 - t[tn] = s[j] - end - end - return unpack(t) - end +do -- create closure to overcome 200 locals limit + +-- original size: 3445, stripped down to: 1803 + +if not modules then modules={} end modules ['l-md5']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local md5,file=md5,file +local gsub,format,byte=string.gsub,string.format,string.byte +local function convert(str,fmt) + return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +end +if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end +if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end +if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end +function file.needsupdating(oldname,newname,threshold) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime then + local newtime=lfs.attributes(newname,"modification") + if not newtime then + return true + elseif newtime>=oldtime then + return false + elseif oldtime-newtime<(threshold or 1) then + return false + else + return true + end + else + return false + end +end +file.needs_updating=file.needsupdating +function file.syncmtimes(oldname,newname) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime and lfs.isfile(newname) then + lfs.touch(newname,oldtime,oldtime) + end +end +function file.checksum(name) + if md5 then + local data=io.loaddata(name) + if data then + return md5.HEX(data) end + end + return nil +end +function file.loadchecksum(name) + if md5 then + local data=io.loaddata(name..".md5") + return data and (gsub(data,"%s","")) + end + return nil +end +function file.savechecksum(name,checksum) + if not checksum then checksum=file.checksum(name) end + if checksum then + io.savedata(name..".md5",checksum) + return checksum + end + return nil end -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) - + (C(digit^1)/tonumber * lparent * Cs((noparent + nestedparents)^1) * rparent) / repeater - + C((nestedbraces + (1-comma))^1) -local pattern_b = spaces * Ct(value*(separator*value)^0) -function parsers.settings_to_array_with_repeat(str,expand) -- beware: "" => { } - if expand then - return lpegmatch(pattern_b,str) or { } - else - return lpegmatch(pattern_a,str) or { } +end -- of closure + +do -- create closure to overcome 200 locals limit + +-- original size: 11806, stripped down to: 5417 + +if not modules then modules={} end modules ['l-url']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local char,format,byte=string.char,string.format,string.byte +local concat=table.concat +local tonumber,type=tonumber,type +local P,C,R,S,Cs,Cc,Ct,Cf,Cg,V=lpeg.P,lpeg.C,lpeg.R,lpeg.S,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.Cf,lpeg.Cg,lpeg.V +local lpegmatch,lpegpatterns,replacer=lpeg.match,lpeg.patterns,lpeg.replacer +url=url or {} +local url=url +local tochar=function(s) return char(tonumber(s,16)) end +local colon=P(":") +local qmark=P("?") +local hash=P("#") +local slash=P("/") +local percent=P("%") +local endofstring=P(-1) +local hexdigit=R("09","AF","af") +local plus=P("+") +local nothing=Cc("") +local escapedchar=(percent*C(hexdigit*hexdigit))/tochar +local escaped=(plus/" ")+escapedchar +local noslash=P("/")/"" +local schemestr=Cs((escaped+(1-colon-slash-qmark-hash))^2) +local authoritystr=Cs((escaped+(1- slash-qmark-hash))^0) +local pathstr=Cs((escaped+(1- qmark-hash))^0) +local querystr=Cs(((1- hash))^0) +local fragmentstr=Cs((escaped+(1- endofstring))^0) +local scheme=schemestr*colon+nothing +local authority=slash*slash*authoritystr+nothing +local path=slash*pathstr+nothing +local query=qmark*querystr+nothing +local fragment=hash*fragmentstr+nothing +local validurl=scheme*authority*path*query*fragment +local parser=Ct(validurl) +lpegpatterns.url=validurl +lpegpatterns.urlsplitter=parser +local escapes={} +setmetatable(escapes,{ __index=function(t,k) + local v=format("%%%02X",byte(k)) + t[k]=v + return v +end }) +local escaper=Cs((R("09","AZ","az")^1+P(" ")/"%%20"+S("-./_")^1+P(1)/escapes)^0) +local unescaper=Cs((escapedchar+1)^0) +lpegpatterns.urlunescaped=escapedchar +lpegpatterns.urlescaper=escaper +lpegpatterns.urlunescaper=unescaper +local function split(str) + return (type(str)=="string" and lpegmatch(parser,str)) or str +end +local isscheme=schemestr*colon*slash*slash +local function hasscheme(str) + if str then + local scheme=lpegmatch(isscheme,str) + return scheme~="" and scheme or false + else + return false + end +end +local rootletter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=P(".")^0*P("/")+rootletter*P(":")+rootletter^1*separator+rootletter^1*P("/") +local rootbased=P("/")+rootletter*P(":") +local barswapper=replacer("|",":") +local backslashswapper=replacer("\\","/") +local equal=P("=") +local amp=P("&") +local key=Cs(((escapedchar+1)-equal )^0) +local value=Cs(((escapedchar+1)-amp -endofstring)^0) +local splitquery=Cf (Ct("")*P { "sequence", + sequence=V("pair")*(amp*V("pair"))^0, + pair=Cg(key*equal*value), +},rawset) +local function hashed(str) + if str=="" then + return { + scheme="invalid", + original=str, + } + end + local s=split(str) + local rawscheme=s[1] + local rawquery=s[4] + local somescheme=rawscheme~="" + local somequery=rawquery~="" + if not somescheme and not somequery then + s={ + scheme="file", + authority="", + path=str, + query="", + fragment="", + original=str, + noscheme=true, + filename=str, + } + else + local authority,path,filename=s[2],s[3] + if authority=="" then + filename=path + elseif path=="" then + filename="" + else + filename=authority.."/"..path + end + s={ + scheme=rawscheme, + authority=authority, + path=path, + query=lpegmatch(unescaper,rawquery), + queries=lpegmatch(splitquery,rawquery), + fragment=s[5], + original=str, + noscheme=false, + filename=filename, + } + end + return s +end +url.split=split +url.hasscheme=hasscheme +url.hashed=hashed +function url.addscheme(str,scheme) + if hasscheme(str) then + return str + elseif not scheme then + return "file:///"..str + else + return scheme..":///"..str + end +end +function url.construct(hash) + local fullurl,f={},0 + local scheme,authority,path,query,fragment=hash.scheme,hash.authority,hash.path,hash.query,hash.fragment + if scheme and scheme~="" then + f=f+1;fullurl[f]=scheme.."://" + end + if authority and authority~="" then + f=f+1;fullurl[f]=authority + end + if path and path~="" then + f=f+1;fullurl[f]="/"..path + end + if query and query~="" then + f=f+1;fullurl[f]="?"..query + end + if fragment and fragment~="" then + f=f+1;fullurl[f]="#"..fragment + end + return lpegmatch(escaper,concat(fullurl)) +end +local pattern=Cs(noslash*R("az","AZ")*(S(":|")/":")*noslash*P(1)^0) +function url.filename(filename) + local spec=hashed(filename) + local path=spec.path + return (spec.scheme=="file" and path and lpegmatch(pattern,path)) or filename +end +local function escapestring(str) + return lpegmatch(escaper,str) +end +url.escape=escapestring +function url.query(str) + if type(str)=="string" then + return lpegmatch(splitquery,str) or "" + else + return str + end +end +function url.toquery(data) + local td=type(data) + if td=="string" then + return #str and escape(data) or nil + elseif td=="table" then + if next(data) then + local t={} + for k,v in next,data do + t[#t+1]=format("%s=%s",k,escapestring(v)) + end + return concat(t,"&") end + else + end +end +local pattern=Cs(noslash^0*(1-noslash*P(-1))^0) +function url.barepath(path) + if not path or path=="" then + return "" + else + return lpegmatch(pattern,path) + end end --- -local value = lbrace * C((nobrace + nestedbraces)^0) * rbrace -local pattern = Ct((space + value)^0) +end -- of closure -function parsers.arguments_to_table(str) - return lpegmatch(pattern,str) -end +do -- create closure to overcome 200 locals limit --- temporary here (unoptimized) +-- original size: 13035, stripped down to: 8133 -function parsers.getparameters(self,class,parentclass,settings) - local sc = self[class] - if not sc then - sc = { } - self[class] = sc - if parentclass then - local sp = self[parentclass] - if not sp then - sp = { } - self[parentclass] = sp - end - setmetatableindex(sc,sp) - end +if not modules then modules={} end modules ['l-dir']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,select=type,select +local find,gmatch,match,gsub=string.find,string.gmatch,string.match,string.gsub +local concat,insert,remove=table.concat,table.insert,table.remove +local lpegmatch=lpeg.match +local P,S,R,C,Cc,Cs,Ct,Cv,V=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Cv,lpeg.V +dir=dir or {} +local dir=dir +local lfs=lfs +local attributes=lfs.attributes +local walkdir=lfs.dir +local isdir=lfs.isdir +local isfile=lfs.isfile +local currentdir=lfs.currentdir +if not isdir then + function isdir(name) + local a=attributes(name) + return a and a.mode=="directory" + end + lfs.isdir=isdir +end +if not isfile then + function isfile(name) + local a=attributes(name) + return a and a.mode=="file" + end + lfs.isfile=isfile +end +function dir.current() + return (gsub(currentdir(),"\\","/")) +end +local lfsisdir=isdir +local function isdir(path) + path=gsub(path,"[/\\]+$","") + return lfsisdir(path) +end +lfs.isdir=isdir +local function globpattern(path,patt,recurse,action) + if path=="/" then + path=path.."." + elseif not find(path,"/$") then + path=path..'/' + end + if isdir(path) then + for name in walkdir(path) do + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if find(full,patt) then + action(full) + end + elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then + globpattern(full,patt,recurse,action) + end end - parsers.settings_to_hash(settings,sc) + end end - -function parsers.listitem(str) - return gmatch(str,"[^, ]+") +dir.globpattern=globpattern +local function collectpattern(path,patt,recurse,result) + local ok,scanner + result=result or {} + if path=="/" then + ok,scanner,first=xpcall(function() return walkdir(path..".") end,function() end) + else + ok,scanner,first=xpcall(function() return walkdir(path) end,function() end) + end + if ok and type(scanner)=="function" then + if not find(path,"/$") then path=path..'/' end + for name in scanner,first do + local full=path..name + local attr=attributes(full) + local mode=attr.mode + if mode=='file' then + if find(full,patt) then + result[name]=attr + end + elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then + attr.list=collectpattern(full,patt,recurse) + result[name]=attr + end + end + end + return result end - --- - -local pattern = Cs { "start", - start = V("one") + V("two") + V("three"), - rest = (Cc(",") * V("thousand"))^0 * (P(".") + endofstring) * anything^0, - thousand = digit * digit * digit, - one = digit * V("rest"), - two = digit * digit * V("rest"), - three = V("thousand") * V("rest"), +dir.collectpattern=collectpattern +local pattern=Ct { + [1]=(C(P(".")+P("/")^1)+C(R("az","AZ")*P(":")*P("/")^0)+Cc("./"))*V(2)*V(3), + [2]=C(((1-S("*?/"))^0*P("/"))^0), + [3]=C(P(1)^0) } - -patterns.splitthousands = pattern -- maybe better in the parsers namespace ? - -function parsers.splitthousands(str) - return lpegmatch(pattern,str) or str -end - --- print(parsers.splitthousands("11111111111.11")) - -local optionalwhitespace = whitespace^0 - -patterns.words = Ct((Cs((1-punctuation-whitespace)^1) + anything)^1) -patterns.sentences = Ct((optionalwhitespace * Cs((1-period)^0 * period))^1) -patterns.paragraphs = Ct((optionalwhitespace * Cs((whitespace^1*endofstring/"" + 1 - (spacer^0*newline*newline))^1))^1) - --- local str = " Word1 word2. \n Word3 word4. \n\n Word5 word6.\n " --- inspect(lpegmatch(patterns.paragraphs,str)) --- inspect(lpegmatch(patterns.sentences,str)) --- inspect(lpegmatch(patterns.words,str)) - --- handy for k="v" [, ] k="v" - -local dquote = P('"') -local equal = P('=') -local escape = P('\\') -local separator = S(' ,') - -local key = C((1-equal)^1) -local value = dquote * C((1-dquote-escape*dquote)^0) * dquote - -local pattern = Cf(Ct("") * Cg(key * equal * value) * separator^0,rawset)^0 - -parsers.patterns.keq_to_hash_c = pattern - -function parsers.keq_to_hash(str) - if str and str ~= "" then - return lpegmatch(pattern,str) +local filter=Cs (( + P("**")/".*"+P("*")/"[^/]*"+P("?")/"[^/]"+P(".")/"%%."+P("+")/"%%+"+P("-")/"%%-"+P(1) +)^0 ) +local function glob(str,t) + if type(t)=="function" then + if type(str)=="table" then + for s=1,#str do + glob(str[s],t) + end + elseif isfile(str) then + t(str) + else + local split=lpegmatch(pattern,str) + if split then + local root,path,base=split[1],split[2],split[3] + local recurse=find(base,"%*%*") + local start=root..path + local result=lpegmatch(filter,start..base) + globpattern(start,result,recurse,t) + end + end + else + if type(str)=="table" then + local t=t or {} + for s=1,#str do + glob(str[s],t) + end + return t + elseif isfile(str) then + if t then + t[#t+1]=str + return t + else + return { str } + end else - return { } + local split=lpegmatch(pattern,str) + if split then + local t=t or {} + local action=action or function(name) t[#t+1]=name end + local root,path,base=split[1],split[2],split[3] + local recurse=find(base,"%*%*") + local start=root..path + local result=lpegmatch(filter,start..base) + globpattern(start,result,recurse,action) + return t + else + return {} + end + end + end +end +dir.glob=glob +local function globfiles(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if recurse then + globfiles(path.."/"..name,recurse,func,files) + end + elseif mode=="file" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name + end + end end + end + return files end - --- inspect(lpeg.match(pattern,[[key="value"]])) - -local defaultspecification = { separator = ",", quote = '"' } - --- this version accepts multiple separators and quotes as used in the --- database module - -function parsers.csvsplitter(specification) - specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification - local separator = specification.separator - local quotechar = specification.quote - local separator = S(separator ~= "" and separator or ",") - local whatever = C((1 - separator - newline)^0) - if quotechar and quotechar ~= "" then - local quotedata = nil - for chr in gmatch(quotechar,".") do - local quotechar = P(chr) - local quoteword = quotechar * C((1 - quotechar)^0) * quotechar - if quotedata then - quotedata = quotedata + quoteword - else - quotedata = quoteword - end +dir.globfiles=globfiles +function dir.ls(pattern) + return concat(glob(pattern),"\n") +end +local make_indeed=true +local onwindows=os.type=="windows" or find(os.getenv("PATH"),";") +if onwindows then + function dir.mkdirs(...) + local str,pth="","" + for i=1,select("#",...) do + local s=select(i,...) + if s=="" then + elseif str=="" then + str=s + else + str=str.."/"..s + end + end + local first,middle,last + local drive=false + first,middle,last=match(str,"^(//)(//*)(.*)$") + if first then + else + first,last=match(str,"^(//)/*(.-)$") + if first then + middle,last=match(str,"([^/]+)/+(.-)$") + if middle then + pth="//"..middle + else + pth="//"..last + last="" end - whatever = quotedata + whatever + else + first,middle,last=match(str,"^([a-zA-Z]:)(/*)(.-)$") + if first then + pth,drive=first..middle,true + else + middle,last=match(str,"^(/*)(.-)$") + if not middle then + last=str + end + end + end end - local parser = Ct((Ct(whatever * (separator * whatever)^0) * S("\n\r"))^0 ) - return function(data) - return lpegmatch(parser,data) + for s in gmatch(last,"[^/]+") do + if pth=="" then + pth=s + elseif drive then + pth,drive=pth..s,false + else + pth=pth.."/"..s + end + if make_indeed and not isdir(pth) then + lfs.mkdir(pth) + end end -end - --- and this is a slightly patched version of a version posted by Philipp Gesang - --- local mycsvsplitter = utilities.parsers.rfc4180splitter() --- --- local crap = [[ --- first,second,third,fourth --- "1","2","3","4" --- "a","b","c","d" --- "foo","bar""baz","boogie","xyzzy" --- ]] --- --- local list, names = mycsvsplitter(crap,true) inspect(list) inspect(names) --- local list, names = mycsvsplitter(crap) inspect(list) inspect(names) - -function parsers.rfc4180splitter(specification) - specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification - local separator = specification.separator --> rfc: COMMA - local quotechar = P(specification.quote) --> DQUOTE - local dquotechar = quotechar * quotechar --> 2DQUOTE - / specification.quote - local separator = S(separator ~= "" and separator or ",") - local escaped = quotechar - * Cs((dquotechar + (1 - quotechar))^0) - * quotechar - local non_escaped = C((1 - quotechar - newline - separator)^1) - local field = escaped + non_escaped - local record = Ct((field * separator^-1)^1) - local headerline = record * Cp() - local wholeblob = Ct((newline^-1 * record)^0) - return function(data,getheader) - if getheader then - local header, position = lpegmatch(headerline,data) - local data = lpegmatch(wholeblob,data,position) - return data, header + return pth,(isdir(pth)==true) + end +else + function dir.mkdirs(...) + local str,pth="","" + for i=1,select("#",...) do + local s=select(i,...) + if s and s~="" then + if str~="" then + str=str.."/"..s else - return lpegmatch(wholeblob,data) + str=s end + end end -end - --- utilities.parsers.stepper("1,7-",9,function(i) print(">>>",i) end) --- utilities.parsers.stepper("1-3,7,8,9") --- utilities.parsers.stepper("1-3,6,7",function(i) print(">>>",i) end) --- utilities.parsers.stepper(" 1 : 3, ,7 ") --- utilities.parsers.stepper("1:4,9:13,24:*",30) - -local function ranger(first,last,n,action) - if not first then - -- forget about it - elseif last == true then - for i=first,n or first do - action(i) + str=gsub(str,"/+","/") + if find(str,"^/") then + pth="/" + for s in gmatch(str,"[^/]+") do + local first=(pth=="/") + if first then + pth=pth..s + else + pth=pth.."/"..s end - elseif last then - for i=first,last do - action(i) + if make_indeed and not first and not isdir(pth) then + lfs.mkdir(pth) end + end else - action(first) + pth="." + for s in gmatch(str,"[^/]+") do + pth=pth.."/"..s + if make_indeed and not isdir(pth) then + lfs.mkdir(pth) + end + end end + return pth,(isdir(pth)==true) + end end - -local cardinal = patterns.cardinal / tonumber -local spacers = patterns.spacer^0 -local endofstring = patterns.endofstring - -local stepper = spacers * ( C(cardinal) * ( spacers * S(":-") * spacers * ( C(cardinal) + Cc(true) ) + Cc(false) ) - * Carg(1) * Carg(2) / ranger * S(", ")^0 )^1 - -local stepper = spacers * ( C(cardinal) * ( spacers * S(":-") * spacers * ( C(cardinal) + (P("*") + endofstring) * Cc(true) ) + Cc(false) ) - * Carg(1) * Carg(2) / ranger * S(", ")^0 )^1 * endofstring -- we're sort of strict (could do without endofstring) - -function utilities.parsers.stepper(str,n,action) - if type(n) == "function" then - lpegmatch(stepper,str,1,false,n or print) +dir.makedirs=dir.mkdirs +if onwindows then + function dir.expandname(str) + local first,nothing,last=match(str,"^(//)(//*)(.*)$") + if first then + first=dir.current().."/" + end + if not first then + first,last=match(str,"^(//)/*(.*)$") + end + if not first then + first,last=match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d=currentdir() + if lfs.chdir(first) then + first=dir.current() + end + lfs.chdir(d) + end + end + if not first then + first,last=dir.current(),str + end + last=gsub(last,"//","/") + last=gsub(last,"/%./","/") + last=gsub(last,"^/*","") + first=gsub(first,"/*$","") + if last=="" or last=="." then + return first else - lpegmatch(stepper,str,1,n,action or print) + return first.."/"..last + end + end +else + function dir.expandname(str) + if not find(str,"^/") then + str=currentdir().."/"..str end + str=gsub(str,"//","/") + str=gsub(str,"/%./","/") + str=gsub(str,"(.)/%.$","%1") + return str + end +end +file.expandname=dir.expandname +local stack={} +function dir.push(newdir) + insert(stack,lfs.currentdir()) +end +function dir.pop() + local d=remove(stack) + if d then + lfs.chdir(d) + end + return d end @@ -7527,95 +3419,71 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-fmt'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or { } -utilities.formatters = utilities.formatters or { } -local formatters = utilities.formatters +-- original size: 1822, stripped down to: 1544 -local concat, format = table.concat, string.format -local tostring, type = tostring, type -local strip = string.strip - -local P, R, Cs = lpeg.P, lpeg.R, lpeg.Cs -local lpegmatch = lpeg.match - --- temporary here - -local digit = R("09") -local period = P(".") -local zero = P("0") -local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") -local number = digit^1 * (case_1 + case_2) -local stripper = Cs((number + 1)^0) - - -lpeg.patterns.stripzeros = stripper - -function formatters.stripzeros(str) - return lpegmatch(stripper,str) +if not modules then modules={} end modules ['l-boolean']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,tonumber=type,tonumber +boolean=boolean or {} +local boolean=boolean +function boolean.tonumber(b) + if b then return 1 else return 0 end end - -function formatters.formatcolumns(result,between) - if result and #result > 0 then - between = between or " " - local widths, numbers = { }, { } - local first = result[1] - local n = #first - for i=1,n do - widths[i] = 0 - end - for i=1,#result do - local r = result[i] - for j=1,n do - local rj = r[j] - local tj = type(rj) - if tj == "number" then - numbers[j] = true - end - if tj ~= "string" then - rj = tostring(rj) - r[j] = rj - end - local w = #rj - if w > widths[j] then - widths[j] = w - end - end - end - for i=1,n do - local w = widths[i] - if numbers[i] then - if w > 80 then - widths[i] = "%s" .. between - else - widths[i] = "%0" .. w .. "i" .. between - end - else - if w > 80 then - widths[i] = "%s" .. between - elseif w > 0 then - widths[i] = "%-" .. w .. "s" .. between - else - widths[i] = "%s" - end - end - end - local template = strip(concat(widths)) - for i=1,#result do - local str = format(template,unpack(result[i])) - result[i] = strip(str) - end +function toboolean(str,tolerant) + if str==nil then + return false + elseif str==false then + return false + elseif str==true then + return true + elseif str=="true" then + return true + elseif str=="false" then + return false + elseif not tolerant then + return false + elseif str==0 then + return false + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +string.toboolean=toboolean +function string.booleanstring(str) + if str==nil then + return false + elseif str==false then + return false + elseif str==true then + return true + elseif str=="true" then + return true + elseif str=="false" then + return false + elseif str==0 then + return false + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +function string.is_boolean(str,default) + if type(str)=="string" then + if str=="true" or str=="yes" or str=="on" or str=="t" then + return true + elseif str=="false" or str=="no" or str=="off" or str=="f" then + return false end - return result + end + return default end @@ -7623,136 +3491,468 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-deb'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- the tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) - -local debug = require "debug" - -local getinfo = debug.getinfo -local type, next, tostring = type, next, tostring -local format, find = string.format, string.find -local is_boolean = string.is_boolean - -utilities = utilities or { } -utilities.debugger = utilities.debugger or { } -local debugger = utilities.debugger +-- original size: 24092, stripped down to: 11311 -local counters = { } -local names = { } - --- one - -local function hook() - local f = getinfo(2) -- "nS" - if f then - local n = "unknown" - if f.what == "C" then - n = f.name or '' - if not names[n] then - names[n] = format("%42s",n) - end +if not modules then modules={} end modules ['l-unicode']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utf=utf or (unicode and unicode.utf8) or {} +utf.characters=utf.characters or string.utfcharacters +utf.values=utf.values or string.utfvalues +local type=type +local char,byte,format,sub=string.char,string.byte,string.format,string.sub +local concat=table.concat +local P,C,R,Cs,Ct,Cmt,Cc,Carg=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Ct,lpeg.Cmt,lpeg.Cc,lpeg.Carg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local bytepairs=string.bytepairs +local finder=lpeg.finder +local replacer=lpeg.replacer +local utfvalues=utf.values +local utfgmatch=utf.gmatch +local p_utftype=patterns.utftype +local p_utfoffset=patterns.utfoffset +local p_utf8char=patterns.utf8char +local p_utf8byte=patterns.utf8byte +local p_utfbom=patterns.utfbom +local p_newline=patterns.newline +local p_whitespace=patterns.whitespace +if not unicode then + unicode={ utf=utf } +end +if not utf.char then + local floor,char=math.floor,string.char + function utf.char(n) + if n<0x80 then + return char(n) + elseif n<0x800 then + return char( + 0xC0+floor(n/0x40), + 0x80+(n%0x40) + ) + elseif n<0x10000 then + return char( + 0xE0+floor(n/0x1000), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) + elseif n<0x200000 then + return char( + 0xF0+floor(n/0x40000), + 0x80+(floor(n/0x1000)%0x40), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) + else + return "" + end + end +end +if not utf.byte then + local utf8byte=patterns.utf8byte + function utf.byte(c) + return lpegmatch(utf8byte,c) + end +end +local utfchar,utfbyte=utf.char,utf.byte +function utf.filetype(data) + return data and lpegmatch(p_utftype,data) or "unknown" +end +local toentities=Cs ( + ( + patterns.utf8one+( + patterns.utf8two+patterns.utf8three+patterns.utf8four + )/function(s) local b=utfbyte(s) if b<127 then return s else return format("&#%X;",b) end end + )^0 +) +patterns.toentities=toentities +function utf.toentities(str) + return lpegmatch(toentities,str) +end +local one=P(1) +local two=C(1)*C(1) +local four=C(R(utfchar(0xD8),utfchar(0xFF)))*C(1)*C(1)*C(1) +local pattern=P("\254\255")*Cs(( + four/function(a,b,c,d) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(a,b) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 )+P("\255\254")*Cs(( + four/function(b,a,d,c) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(b,a) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 ) +function string.toutf(s) + return lpegmatch(pattern,s) or s +end +local validatedutf=Cs ( + ( + patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four+P(1)/"�" + )^0 +) +patterns.validatedutf=validatedutf +function utf.is_valid(str) + return type(str)=="string" and lpegmatch(validatedutf,str) or false +end +if not utf.len then + local n,f=0,1 + local utfcharcounter=patterns.utfbom^-1*Cmt ( + Cc(1)*patterns.utf8one^1+Cc(2)*patterns.utf8two^1+Cc(3)*patterns.utf8three^1+Cc(4)*patterns.utf8four^1, + function(_,t,d) + n=n+(t-f)/d + f=t + return true + end + )^0 + function utf.len(str) + n,f=0,1 + lpegmatch(utfcharcounter,str or "") + return n + end +end +utf.length=utf.len +if not utf.sub then + local utflength=utf.length + local b,e,n,first,last=0,0,0,0,0 + local function slide_zero(s,p) + n=n+1 + if n>=last then + e=p-1 + else + return p + end + end + local function slide_one(s,p) + n=n+1 + if n==first then + b=p + end + if n>=last then + e=p-1 + else + return p + end + end + local function slide_two(s,p) + n=n+1 + if n==first then + b=p + else + return true + end + end + local pattern_zero=Cmt(p_utf8char,slide_zero)^0 + local pattern_one=Cmt(p_utf8char,slide_one )^0 + local pattern_two=Cmt(p_utf8char,slide_two )^0 + function utf.sub(str,start,stop) + if not start then + return str + end + if start==0 then + start=1 + end + if not stop then + if start<0 then + local l=utflength(str) + start=l+start + else + start=start-1 + end + b,n,first=0,0,start + lpegmatch(pattern_two,str) + if n>=first then + return sub(str,b) + else + return "" + end + end + if start<0 or stop<0 then + local l=utf.length(str) + if start<0 then + start=l+start + if start<=0 then + start=1 else - -- source short_src linedefined what name namewhat nups func - n = f.name or f.namewhat or f.what - if not n or n == "" then - n = "?" - end - if not names[n] then - names[n] = format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") - end + start=start+1 end - counters[n] = (counters[n] or 0) + 1 - end -end - -function debugger.showstats(printer,threshold) -- hm, something has changed, rubish now - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - local dataset = { } - for name, count in next, counters do - dataset[#dataset+1] = { name, count } - end - table.sort(dataset,function(a,b) return a[2] == b[2] and b[1] > a[1] or a[2] > b[2] end) - for i=1,#dataset do - local d = dataset[i] - local name = d[1] - local count = d[2] - if count > threshold and not find(name,"for generator") then -- move up - printer(format("%8i %s\n", count, names[name])) - total = total + count + end + if stop<0 then + stop=l+stop + if stop==0 then + stop=1 + else + stop=stop+1 end - grandtotal = grandtotal + count - functions = functions + 1 + end + end + if start>stop then + return "" + elseif start>1 then + b,e,n,first,last=0,0,0,start-1,stop + lpegmatch(pattern_one,str) + if n>=first and e==0 then + e=#str + end + return sub(str,b,e) + else + b,e,n,last=1,0,0,stop + lpegmatch(pattern_zero,str) + if e==0 then + e=#str + end + return sub(str,b,e) end - printer("\n") - printer(format("functions : % 10i\n", functions)) - printer(format("total : % 10i\n", total)) - printer(format("grand total: % 10i\n", grandtotal)) - printer(format("threshold : % 10i\n", threshold)) + end end - -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() +function utf.remapper(mapping) + local pattern=Cs((p_utf8char/mapping)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern +end +function utf.replacer(t) + local r=replacer(t,false,false,true) + return function(str) + return lpegmatch(r,str) + end +end +function utf.subtituter(t) + local f=finder (t) + local r=replacer(t,false,false,true) + return function(str) + local i=lpegmatch(f,str) + if not i then + return str + elseif i>#str then + return str + else + return lpegmatch(r,str) + end + end +end +local utflinesplitter=p_utfbom^-1*lpeg.tsplitat(p_newline) +local utfcharsplitter_ows=p_utfbom^-1*Ct(C(p_utf8char)^0) +local utfcharsplitter_iws=p_utfbom^-1*Ct((p_whitespace^1+C(p_utf8char))^0) +local utfcharsplitter_raw=Ct(C(p_utf8char)^0) +patterns.utflinesplitter=utflinesplitter +function utf.splitlines(str) + return lpegmatch(utflinesplitter,str or "") +end +function utf.split(str,ignorewhitespace) + if ignorewhitespace then + return lpegmatch(utfcharsplitter_iws,str or "") + else + return lpegmatch(utfcharsplitter_ows,str or "") + end +end +function utf.totable(str) + return lpegmatch(utfcharsplitter_raw,str) +end +function utf.magic(f) + local str=f:read(4) or "" + local off=lpegmatch(p_utfoffset,str) + if off<4 then + f:seek('set',off) + end + return lpegmatch(p_utftype,str) +end +local function utf16_to_utf8_be(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,0 + for left,right in bytepairs(t[i]) do + if right then + local now=256*left+right + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + r=r+1 + result[r]=utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + else + r=r+1 + result[r]=utfchar(now) + end + end end + t[i]=concat(result,"",1,r) + end + return t end - -function debugger.enable() - debug.sethook(hook,"c") +local function utf16_to_utf8_le(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,0 + for left,right in bytepairs(t[i]) do + if right then + local now=256*right+left + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + r=r+1 + result[r]=utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + else + r=r+1 + result[r]=utfchar(now) + end + end + end + t[i]=concat(result,"",1,r) + end + return t +end +local function utf32_to_utf8_be(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,-1 + for a,b in bytepairs(t[i]) do + if a and b then + if more<0 then + more=256*256*256*a+256*256*b + else + r=r+1 + result[t]=utfchar(more+256*a+b) + more=-1 + end + else + break + end + end + t[i]=concat(result,"",1,r) + end + return t +end +local function utf32_to_utf8_le(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,-1 + for a,b in bytepairs(t[i]) do + if a and b then + if more<0 then + more=256*b+a + else + r=r+1 + result[t]=utfchar(more+256*256*256*b+256*256*a) + more=-1 + end + else + break + end + end + t[i]=concat(result,"",1,r) + end + return t +end +utf.utf32_to_utf8_be=utf32_to_utf8_be +utf.utf32_to_utf8_le=utf32_to_utf8_le +utf.utf16_to_utf8_be=utf16_to_utf8_be +utf.utf16_to_utf8_le=utf16_to_utf8_le +function utf.utf8_to_utf8(t) + return type(t)=="string" and lpegmatch(utflinesplitter,t) or t +end +function utf.utf16_to_utf8(t,endian) + return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +end +function utf.utf32_to_utf8(t,endian) + return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +end +local function little(c) + local b=byte(c) + if b<0x10000 then + return char(b%256,b/256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end +end +local function big(c) + local b=byte(c) + if b<0x10000 then + return char(b/256,b%256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end +end +local _,l_remap=utf.remapper(little) +local _,b_remap=utf.remapper(big) +function utf.utf8_to_utf16(str,littleendian) + if littleendian then + return char(255,254)..lpegmatch(l_remap,str) + else + return char(254,255)..lpegmatch(b_remap,str) + end +end +local pattern=Cs ( + (p_utf8byte/function(unicode ) return format("0x%04X",unicode) end)*(p_utf8byte*Carg(1)/function(unicode,separator) return format("%s0x%04X",separator,unicode) end)^0 +) +function utf.tocodes(str,separator) + return lpegmatch(pattern,str,1,separator or " ") end - -function debugger.disable() - debug.sethook() +function utf.ustring(s) + return format("U+%05X",type(s)=="number" and s or utfbyte(s)) end - - - - - -local is_node = node and node.is_node -local is_lpeg = lpeg and lpeg.type - -function inspect(i) -- global function - local ti = type(i) - if ti == "table" then - table.print(i,"table") - elseif is_node and is_node(i) then - table.print(nodes.astable(i),tostring(i)) - elseif is_lpeg and is_lpeg(i) then - lpeg.print(i) - else - print(tostring(i)) - end - return i -- so that we can inline the inspect +function utf.xstring(s) + return format("0x%05X",type(s)=="number" and s or utfbyte(s)) end - --- from the lua book: - -function traceback() - local level = 1 - while true do - local info = debug.getinfo(level, "Sl") - if not info then - break - elseif info.what == "C" then - print(format("%3i : C function",level)) - else - print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) - end - level = level + 1 +local p_nany=p_utf8char/"" +if utfgmatch then + function utf.count(str,what) + if type(what)=="string" then + local n=0 + for _ in utfgmatch(str,what) do + n=n+1 + end + return n + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) + end + end +else + local cache={} + function utf.count(str,what) + if type(what)=="string" then + local p=cache[what] + if not p then + p=Cs((P(what)/" "+p_nany)^0) + cache[p]=p + end + return #lpegmatch(p,str) + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) end + end end @@ -7760,198 +3960,473 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-inf'] = { - version = 1.001, - comment = "companion to trac-inf.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- As we want to protect the global tables, we no longer store the timing --- in the tables themselves but in a hidden timers table so that we don't --- get warnings about assignments. This is more efficient than using rawset --- and rawget. +-- original size: 915, stripped down to: 836 -local format, lower = string.format, string.lower -local clock = os.gettimeofday or os.clock -- should go in environment -local write_nl = texio and texio.write_nl or print +if not modules then modules={} end modules ['l-math']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local floor,sin,cos,tan=math.floor,math.sin,math.cos,math.tan +if not math.round then + function math.round(x) return floor(x+0.5) end +end +if not math.div then + function math.div(n,m) return floor(n/m) end +end +if not math.mod then + function math.mod(n,m) return n%m end +end +local pipi=2*math.pi/360 +if not math.sind then + function math.sind(d) return sin(d*pipi) end + function math.cosd(d) return cos(d*pipi) end + function math.tand(d) return tan(d*pipi) end +end +if not math.odd then + function math.odd (n) return n%2~=0 end + function math.even(n) return n%2==0 end +end -statistics = statistics or { } -local statistics = statistics -statistics.enable = true -statistics.threshold = 0.01 +end -- of closure -local statusinfo, n, registered, timers = { }, 0, { }, { } +do -- create closure to overcome 200 locals limit -table.setmetatableindex(timers,function(t,k) - local v = { timing = 0, loadtime = 0 } - t[k] = v - return v -end) +-- original size: 10334, stripped down to: 6756 -local function hastiming(instance) - return instance and timers[instance] +if not modules then modules={} end modules ['util-tab']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.tables=utilities.tables or {} +local tables=utilities.tables +local format,gmatch,rep,gsub=string.format,string.gmatch,string.rep,string.gsub +local concat,insert,remove=table.concat,table.insert,table.remove +local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local type,next,rawset,tonumber,load,select=type,next,rawset,tonumber,load,select +local lpegmatch,P,Cs=lpeg.match,lpeg.P,lpeg.Cs +local serialize=table.serialize +local splitter=lpeg.tsplitat(".") +function tables.definetable(target,nofirst,nolast) + local composed,shortcut,t=nil,nil,{} + local snippets=lpegmatch(splitter,target) + for i=1,#snippets-(nolast and 1 or 0) do + local name=snippets[i] + if composed then + composed=shortcut.."."..name + shortcut=shortcut.."_"..name + t[#t+1]=format("local %s = %s if not %s then %s = { } %s = %s end",shortcut,composed,shortcut,shortcut,composed,shortcut) + else + composed=name + shortcut=name + if not nofirst then + t[#t+1]=format("%s = %s or { }",composed,composed) + end + end + end + if nolast then + composed=shortcut.."."..snippets[#snippets] + end + return concat(t,"\n"),composed end - -local function resettiming(instance) - timers[instance or "notimer"] = { timing = 0, loadtime = 0 } +function tables.definedtable(...) + local t=_G + for i=1,select("#",...) do + local li=select(i,...) + local tl=t[li] + if not tl then + tl={} + t[li]=tl + end + t=tl + end + return t end - -local function starttiming(instance) - local timer = timers[instance or "notimer"] - local it = timer.timing or 0 - if it == 0 then - timer.starttime = clock() - if not timer.loadtime then - timer.loadtime = 0 - end +function tables.accesstable(target,root) + local t=root or _G + for name in gmatch(target,"([^%.]+)") do + t=t[name] + if not t then + return end - timer.timing = it + 1 + end + return t end - -local function stoptiming(instance, report) - local timer = timers[instance or "notimer"] - local it = timer.timing - if it > 1 then - timer.timing = it - 1 - else - local starttime = timer.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - timer.stoptime = stoptime - timer.loadtime = timer.loadtime + loadtime - if report then - statistics.report("load time %0.3f",loadtime) - end - timer.timing = 0 - return loadtime - end +function tables.migratetable(target,v,root) + local t=root or _G + local names=string.split(target,".") + for i=1,#names-1 do + local name=names[i] + t[name]=t[name] or {} + t=t[name] + if not t then + return end - return 0 + end + t[names[#names]]=v end - -local function elapsedtime(instance) - local timer = timers[instance or "notimer"] - return format("%0.3f",timer and timer.loadtime or 0) +function tables.removevalue(t,value) + if value then + for i=1,#t do + if t[i]==value then + remove(t,i) + end + end + end end - -local function elapsedindeed(instance) - local timer = timers[instance or "notimer"] - return (timer and timer.loadtime or 0) > statistics.threshold +function tables.insertbeforevalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) end - -local function elapsedseconds(instance,rest) -- returns nil if 0 seconds - if elapsedindeed(instance) then - return format("%s seconds %s", elapsedtime(instance),rest or "") +function tables.insertaftervalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i+1,extra) + return end + end + insert(t,#t+1,extra) end - -statistics.hastiming = hastiming -statistics.resettiming = resettiming -statistics.starttiming = starttiming -statistics.stoptiming = stoptiming -statistics.elapsedtime = elapsedtime -statistics.elapsedindeed = elapsedindeed -statistics.elapsedseconds = elapsedseconds - --- general function .. we might split this module - -function statistics.register(tag,fnc) - if statistics.enable and type(fnc) == "function" then - local rt = registered[tag] or (#statusinfo + 1) - statusinfo[rt] = { tag, fnc } - registered[tag] = rt - if #tag > n then n = #tag end +local function toxml(t,d,result,step) + for k,v in table.sortedpairs(t) do + if type(v)=="table" then + if type(k)=="number" then + result[#result+1]=format("%s",d,k) + toxml(v,d..step,result,step) + result[#result+1]=format("%s",d,k) + else + result[#result+1]=format("%s<%s>",d,k) + toxml(v,d..step,result,step) + result[#result+1]=format("%s",d,k) + end + elseif type(k)=="number" then + result[#result+1]=format("%s%s",d,k,v,k) + else + result[#result+1]=format("%s<%s>%s",d,k,tostring(v),k) end + end end - -function statistics.show(reporter) - if statistics.enable then - if not reporter then reporter = function(tag,data,n) write_nl(tag .. " " .. data) end end - -- this code will move - local register = statistics.register - register("luatex banner", function() - return lower(status.banner) - end) - register("control sequences", function() - return format("%s of %s + %s", status.cs_count, status.hash_size,status.hash_extra) - end) - register("callbacks", function() - local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 - return format("%s direct, %s indirect, %s total", total-indirect, indirect, total) - end) - collectgarbage("collect") - register("current memory usage", statistics.memused) - register("runtime",statistics.runtime) - for i=1,#statusinfo do - local s = statusinfo[i] - local r = s[2]() - if r then - reporter(s[1],r,n) - end +function table.toxml(t,name,nobanner,indent,spaces) + local noroot=name==false + local result=(nobanner or noroot) and {} or { "" } + local indent=rep(" ",indent or 0) + local spaces=rep(" ",spaces or 1) + if noroot then + toxml(t,inndent,result,spaces) + else + toxml({ [name or "root"]=t },indent,result,spaces) + end + return concat(result,"\n") +end +function tables.encapsulate(core,capsule,protect) + if type(capsule)~="table" then + protect=true + capsule={} + end + for key,value in next,core do + if capsule[key] then + print(format("\ninvalid inheritance '%s' in '%s': %s",key,tostring(core))) + os.exit() + else + capsule[key]=value + end + end + if protect then + for key,value in next,core do + core[key]=nil + end + setmetatable(core,{ + __index=capsule, + __newindex=function(t,key,value) + if capsule[key] then + print(format("\ninvalid overload '%s' in '%s'",key,tostring(core))) + os.exit() + else + rawset(t,key,value) end - write_nl("") -- final newline - statistics.enable = false + end + } ) + end +end +local function fastserialize(t,r,outer) + r[#r+1]="{" + local n=#t + if n>0 then + for i=1,n do + local v=t[i] + local tv=type(v) + if tv=="string" then + r[#r+1]=format("%q,",v) + elseif tv=="number" then + r[#r+1]=format("%s,",v) + elseif tv=="table" then + fastserialize(v,r) + elseif tv=="boolean" then + r[#r+1]=format("%s,",tostring(v)) + end + end + else + for k,v in next,t do + local tv=type(v) + if tv=="string" then + r[#r+1]=format("[%q]=%q,",k,v) + elseif tv=="number" then + r[#r+1]=format("[%q]=%s,",k,v) + elseif tv=="table" then + r[#r+1]=format("[%q]=",k) + fastserialize(v,r) + elseif tv=="boolean" then + r[#r+1]=format("[%q]=%s,",k,tostring(v)) + end end + end + if outer then + r[#r+1]="}" + else + r[#r+1]="}," + end + return r end - -local template, report_statistics, nn = nil, nil, 0 -- we only calcute it once - -function statistics.showjobstat(tag,data,n) - if not logs then - -- sorry - elseif type(data) == "table" then - for i=1,#data do - statistics.showjobstat(tag,data[i],n) - end - else - if not template or n > nn then - template, n = format("%%-%ss - %%s",n), nn - report_statistics = logs.reporter("mkiv lua stats") +function table.fastserialize(t,prefix) + return concat(fastserialize(t,{ prefix or "return" },true)) +end +function table.deserialize(str) + if not str or str=="" then + return + end + local code=load(str) + if not code then + return + end + code=code() + if not code then + return + end + return code +end +function table.load(filename) + if filename then + local t=io.loaddata(filename) + if t and t~="" then + t=load(t) + if type(t)=="function" then + t=t() + if type(t)=="table" then + return t end - report_statistics(format(template,tag,data)) + end end + end end - -function statistics.memused() -- no math.round yet -) - local round = math.round or math.floor - return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) +function table.save(filename,t,n,...) + io.savedata(filename,serialize(t,n==nil and true or n,...)) end - -starttiming(statistics) - -function statistics.formatruntime(runtime) -- indirect so it can be overloaded and - return format("%s seconds", runtime) -- indeed that happens in cure-uti.lua +local function slowdrop(t) + local r={} + local l={} + for i=1,#t do + local ti=t[i] + local j=0 + for k,v in next,ti do + j=j+1 + l[j]=format("%s=%q",k,v) + end + r[i]=format(" {%s},\n",concat(l)) + end + return format("return {\n%s}",concat(r)) end - -function statistics.runtime() - stoptiming(statistics) - return statistics.formatruntime(elapsedtime(statistics)) +local function fastdrop(t) + local r={ "return {\n" } + for i=1,#t do + local ti=t[i] + r[#r+1]=" {" + for k,v in next,ti do + r[#r+1]=format("%s=%q",k,v) + end + r[#r+1]="},\n" + end + r[#r+1]="}" + return concat(r) end - -function statistics.timed(action,report) - report = report or logs.reporter("system") - starttiming("run") - action() - stoptiming("run") - report("total runtime: %s",elapsedtime("run")) +function table.drop(t,slow) + if #t==0 then + return "return { }" + elseif slow==true then + return slowdrop(t) + else + return fastdrop(t) + end +end +function table.autokey(t,k) + local v={} + t[k]=v + return v end --- where, not really the best spot for this: -commands = commands or { } +end -- of closure + +do -- create closure to overcome 200 locals limit -function commands.resettimer(name) - resettiming(name or "whatever") - starttiming(name or "whatever") -end +-- original size: 4270, stripped down to: 2989 -function commands.elapsedtime(name) - stoptiming(name or "whatever") - context(elapsedtime(name or "whatever")) +if not modules then modules={} end modules ['util-sto']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local setmetatable,getmetatable=setmetatable,getmetatable +utilities=utilities or {} +utilities.storage=utilities.storage or {} +local storage=utilities.storage +local report=texio and texio.write_nl or print +function storage.mark(t) + if not t then + report("fatal error: storage cannot be marked") + return + end + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.allocate(t) + t=t or {} + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.marked(t) + local m=getmetatable(t) + return m and m.__storage__ +end +function storage.checked(t) + if not t then + report("fatal error: storage has not been allocated") + return + end + return t +end +function storage.setinitializer(data,initialize) + local m=getmetatable(data) or {} + m.__index=function(data,k) + m.__index=nil + initialize() + return data[k] + end + setmetatable(data,m) +end +local keyisvalue={ __index=function(t,k) + t[k]=k + return k +end } +function storage.sparse(t) + t=t or {} + setmetatable(t,keyisvalue) + return t +end +local function f_empty () return "" end +local function f_self (t,k) t[k]=k return k end +local function f_table (t,k) local v={} t[k]=v return v end +local function f_ignore() end +local t_empty={ __index=f_empty } +local t_self={ __index=f_self } +local t_table={ __index=f_table } +local t_ignore={ __newindex=f_ignore } +function table.setmetatableindex(t,f) + local m=getmetatable(t) + if m then + if f=="empty" then + m.__index=f_empty + elseif f=="key" then + m.__index=f_self + elseif f=="table" then + m.__index=f_table + else + m.__index=f + end + else + if f=="empty" then + setmetatable(t,t_empty) + elseif f=="key" then + setmetatable(t,t_self) + elseif f=="table" then + setmetatable(t,t_table) + else + setmetatable(t,{ __index=f }) + end + end + return t +end +function table.setmetatablenewindex(t,f) + local m=getmetatable(t) + if m then + if f=="ignore" then + m.__newindex=f_ignore + else + m.__newindex=f + end + else + if f=="ignore" then + setmetatable(t,t_ignore) + else + setmetatable(t,{ __newindex=f }) + end + end + return t +end +function table.setmetatablecall(t,f) + local m=getmetatable(t) + if m then + m.__call=f + else + setmetatable(t,{ __call=f }) + end + return t +end +function table.setmetatablekey(t,key,value) + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m[key]=value + return t +end +function table.getmetatablekey(t,key,value) + local m=getmetatable(t) + return m and m[key] end @@ -7959,382 +4434,1165 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-set'] = { -- might become util-set.lua - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +-- original size: 11610, stripped down to: 7440 + +if not modules then modules={} end modules ['util-str']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.strings=utilities.strings or {} +local strings=utilities.strings +local load=load +local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local concat=table.concat +local P,V,C,S,R,Ct,Cs,Cp,Carg=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg +local patterns,lpegmatch=lpeg.patterns,lpeg.match +local utfchar,utfbyte=utf.char,utf.byte +local setmetatableindex=table.setmetatableindex +local stripper=patterns.stripzeros +local function points(n) + return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) +end +local function basepoints(n) + return (not n or n==0) and "0bp" or lpegmatch(stripper,format("%.5fbp",n*(7200/7227)/65536)) +end +number.points=points +number.basepoints=basepoints +local rubish=patterns.spaceortab^0*patterns.newline +local anyrubish=patterns.spaceortab+patterns.newline +local anything=patterns.anything +local stripped=(patterns.spaceortab^1/"")*patterns.newline +local leading=rubish^0/"" +local trailing=(anyrubish^1*patterns.endofstring)/"" +local redundant=rubish^3/"\n" +local pattern=Cs(leading*(trailing+redundant+stripped+anything)^0) +function strings.collapsecrlf(str) + return lpegmatch(pattern,str) +end +local repeaters={} +function strings.newrepeater(str,offset) + offset=offset or 0 + local s=repeaters[str] + if not s then + s={} + repeaters[str]=s + end + local t=s[offset] + if t then + return t + end + t={} + setmetatableindex(t,function(t,k) + if not k then + return "" + end + local n=k+offset + local s=n>0 and rep(str,n) or "" + t[k]=s + return s + end) + s[offset]=t + return t +end +local extra,tab,start=0,0,4,0 +local nspaces=strings.newrepeater(" ") +local pattern=Carg(1)/function(t) + extra,tab,start=0,t or 7,1 + end*Cs(( + Cp()*patterns.tab/function(position) + local current=(position-start+1)+extra + local spaces=tab-(current-1)%tab + if spaces>0 then + extra=extra+spaces-1 + return nspaces[spaces] + else + return "" + end + end+patterns.newline*Cp()/function(position) + extra,start=0,position + end+patterns.anything + )^1) +function strings.tabtospace(str,tab) + return lpegmatch(pattern,str,1,tab or 7) +end +function strings.striplong(str) + str=gsub(str,"^%s*","") + str=gsub(str,"[\n\r]+ *","\n") + return str +end +function strings.nice(str) + str=gsub(str,"[:%-+_]+"," ") + return str +end +local n=0 +local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_tab=C((1-R("az","AZ","09","%%"))^0) +local format_s=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%ss',(select(%s,...)))",f,n) + else + return format("(select(%s,...))",n) + end +end +local format_q=function() + n=n+1 + return format("format('%%q',(select(%s,...)))",n) +end +local format_i=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%si',(select(%s,...)))",f,n) + else + return format("(select(%s,...))",n) + end +end +local format_d=format_i +local format_f=function(f) + n=n+1 + return format("format('%%%sf',(select(%s,...)))",f,n) +end +local format_g=function(f) + n=n+1 + return format("format('%%%sg',(select(%s,...)))",f,n) +end +local format_G=function(f) + n=n+1 + return format("format('%%%sG',(select(%s,...)))",f,n) +end +local format_e=function(f) + n=n+1 + return format("format('%%%se',(select(%s,...)))",f,n) +end +local format_E=function(f) + n=n+1 + return format("format('%%%sE',(select(%s,...)))",f,n) +end +local format_x=function(f) + n=n+1 + return format("format('%%%sx',(select(%s,...)))",f,n) +end +local format_X=function(f) + n=n+1 + return format("format('%%%sX',(select(%s,...)))",f,n) +end +local format_o=function(f) + n=n+1 + return format("format('%%%so',(select(%s,...)))",f,n) +end +local format_c=function() + n=n+1 + return format("utfchar((select(%s,...)))",n) +end +local format_r=function(f) + n=n+1 + return format("format('%%%s.0f',(select(%s,...)))",f,n) +end +local format_v=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('0x%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_V=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('0x%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_u=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('u+%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_U=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('U+%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_p=function() + n=n+1 + return format("points((select(%s,...)))",n) +end +local format_b=function() + n=n+1 + return format("basepoints((select(%s,...)))",n) +end +local format_t=function(f) + n=n+1 + if f and f~="" then + return format("concat((select(%s,...)),%q)",n,f) + else + return format("concat((select(%s,...)))",n) + end +end +local format_l=function() + n=n+1 + return format("(select(%s,...) and 'true' or 'false')",n) +end +local format_a=function(s) + return format("%q",s) +end +local builder=Ct { "start", + start=(P("%")*( + V("s")+V("q")+V("i")+V("d")+V("f")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") ++V("c") ++V("r")+V("v")+V("V")+V("u")+V("U")+V("p")+V("b")+V("t")+V("l") + )+V("a") + )^0, + ["s"]=(prefix_any*P("s"))/format_s, + ["q"]=(prefix_any*P("q"))/format_q, + ["i"]=(prefix_any*P("i"))/format_i, + ["d"]=(prefix_any*P("d"))/format_d, + ["f"]=(prefix_any*P("f"))/format_f, + ["g"]=(prefix_any*P("g"))/format_g, + ["G"]=(prefix_any*P("G"))/format_G, + ["e"]=(prefix_any*P("e"))/format_e, + ["E"]=(prefix_any*P("E"))/format_E, + ["x"]=(prefix_any*P("x"))/format_x, + ["X"]=(prefix_any*P("X"))/format_X, + ["o"]=(prefix_any*P("o"))/format_o, + ["c"]=(prefix_any*P("c"))/format_c, + ["r"]=(prefix_any*P("r"))/format_r, + ["v"]=(prefix_any*P("v"))/format_v, + ["V"]=(prefix_any*P("V"))/format_V, + ["u"]=(prefix_any*P("u"))/format_u, + ["U"]=(prefix_any*P("U"))/format_U, + ["p"]=(prefix_any*P("p"))/format_p, + ["b"]=(prefix_any*P("b"))/format_b, + ["t"]=(prefix_tab*P("t"))/format_t, + ["l"]=(prefix_tab*P("l"))/format_l, + ["a"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_a, } +local template=[[ +local format = string.format +local concat = table.concat +local points = number.points +local basepoints = number.basepoints +local utfchar = utf.char +local utfbyte = utf.byte +return function(...) + return %s +end +]] +local function make(t,str) + n=0 + local p=lpegmatch(builder,str) + local c=format(template,concat(p,"..")) + formatter=load(c)() + t[str]=formatter + return formatter +end +local formatters=string.formatters or {} +string.formatters=formatters +setmetatableindex(formatters,make) +function string.makeformatter(str) + return formatters[str] +end +function string.formatter(str,...) + return formatters[str](...) +end --- maybe this should be util-set.lua -local type, next, tostring = type, next, tostring -local concat = table.concat -local format, find, lower, gsub, topattern = string.format, string.find, string.lower, string.gsub, string.topattern -local is_boolean = string.is_boolean -local settings_to_hash = utilities.parsers.settings_to_hash -local allocate = utilities.storage.allocate - -utilities = utilities or { } -local utilities = utilities -utilities.setters = utilities.setters or { } -local setters = utilities.setters - -local data = { } -- maybe just local - --- We can initialize from the cnf file. This is sort of tricky as --- later defined setters also need to be initialized then. If set --- this way, we need to ensure that they are not reset later on. - -local trace_initialize = false -- only for testing during development - -function setters.initialize(filename,name,values) -- filename only for diagnostics - local setter = data[name] - if setter then - frozen = true -- don't permitoverload --- trace_initialize = true - local data = setter.data - if data then - for key, newvalue in next, values do - local newvalue = is_boolean(newvalue,newvalue) - local functions = data[key] - if functions then - local oldvalue = functions.value - if functions.frozen then - if trace_initialize then - setter.report("%s: %q is frozen to %q",filename,key,tostring(oldvalue)) - end - elseif #functions > 0 and not oldvalue then --- elseif #functions > 0 and oldvalue == nil then - if trace_initialize then - setter.report("%s: %q is set to %q",filename,key,tostring(newvalue)) - end - for i=1,#functions do - functions[i](newvalue) - end - functions.value = newvalue - functions.frozen = functions.frozen or frozen - else - if trace_initialize then - setter.report("%s: %q is kept as %q",filename,key,tostring(oldvalue)) - end - end - else - -- we do a simple preregistration i.e. not in the - -- list as it might be an obsolete entry - functions = { default = newvalue, frozen = frozen } - data[key] = functions - if trace_initialize then - setter.report("%s: %q default to %q",filename,key,tostring(newvalue)) - end - end - end - return true - end +end -- of closure + +do -- create closure to overcome 200 locals limit + +-- original size: 7050, stripped down to: 5641 + +if not modules then modules={} end modules ['util-mrg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gsub,format=string.gsub,string.format +local concat=table.concat +local type,next=type,next +utilities=utilities or {} +local merger=utilities.merger or {} +utilities.merger=merger +utilities.report=logs and logs.reporter("system") or print +merger.strip_comment=true +local m_begin_merge="begin library merge" +local m_end_merge="end library merge" +local m_begin_closure="do -- create closure to overcome 200 locals limit" +local m_end_closure="end -- of closure" +local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" +local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" +local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" +local m_report=[[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] +local function self_fake() + return m_faked +end +local function self_nothing() + return "" +end +local function self_load(name) + local data=io.loaddata(name) or "" + if data=="" then + utilities.report("merge: unknown file %s",name) + else + utilities.report("merge: inserting %s",name) + end + return data or "" +end +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local equals=P("=")^0 +local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 +local close=P("]")*C(equals)*P("]") +local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) +local longstring=open*(1-closeeq)^0*close +local space=patterns.space +local eol=patterns.newline +local quoted=patterns.quoted +local emptyline=space^0*eol +local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") +local operator2=S("*+/") +local operator3=S("-") +local separator=S(",;") +local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" +local strings=quoted +local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" +local longstr=longstring +local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" +local pack=((eol+space)^0/"")*operator1*((eol+space)^0/"")+((eol+space)^0/"")*operator2*((space)^0/"")+((eol+space)^1/"")*operator3*((space)^1/"")+((space)^0/"")*separator*((space)^0/"") +local lines=emptyline^2/"\n" +local spaces=(space*space)/" " +local compact=Cs (( + ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 +)^1 ) +local strip=Cs((emptyline^2/"\n"+1)^0) +local function self_compact(data) + if merger.strip_comment then + local before=#data + data=lpeg.match(compact,data) + data=lpeg.match(strip,data) + local after=#data + local delta=before-after + utilities.report("merge: %s bytes compacted to %s (%s bytes stripped)",before,after,delta) + data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + return data,delta + else + return data,0 + end +end +local function self_save(name,data) + if data~="" then + io.savedata(name,data) + utilities.report("merge: saving %s bytes in %s",#data,name) + end +end +local function self_swap(data,code) + return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +end +local function self_libs(libs,list) + local result,f,frozen,foundpath={},nil,false,nil + result[#result+1]="\n" + if type(libs)=='string' then libs={ libs } end + if type(list)=='string' then list={ list } end + for i=1,#libs do + local lib=libs[i] + for j=1,#list do + local pth=gsub(list[j],"\\","/") + utilities.report("merge: checking library path %s",pth) + local name=pth.."/"..lib + if lfs.isfile(name) then + foundpath=pth + end end + if foundpath then break end + end + if foundpath then + utilities.report("merge: using library path %s",foundpath) + local right,wrong,original,stripped={},{},0,0 + for i=1,#libs do + local lib=libs[i] + local fullname=foundpath.."/"..lib + if lfs.isfile(fullname) then + utilities.report("merge: using library %s",fullname) + local data=io.loaddata(fullname,true) + original=original+#data + local data,delta=self_compact(data) + right[#right+1]=lib + result[#result+1]=m_begin_closure + result[#result+1]=data + result[#result+1]=m_end_closure + stripped=stripped+delta + else + utilities.report("merge: skipping library %s",fullname) + wrong[#wrong+1]=lib + end + end + right=#right>0 and concat(right," ") or "-" + wrong=#wrong>0 and concat(wrong," ") or "-" + utilities.report("merge: used libraries: %s",right) + utilities.report("merge: skipped libraries: %s",wrong) + utilities.report("merge: original bytes: %s",original) + utilities.report("merge: stripped bytes: %s",stripped) + result[#result+1]=format(m_report,right,wrong,original,stripped) + else + utilities.report("merge: no valid library path found") + end + return concat(result,"\n\n") +end +function merger.selfcreate(libs,list,target) + if target then + self_save(target,self_swap(self_fake(),self_libs(libs,list))) + end +end +function merger.selfmerge(name,libs,list,target) + self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) +end +function merger.selfclean(name) + self_save(name,self_swap(self_load(name),self_nothing())) end --- user interface code -local function set(t,what,newvalue) - local data = t.data - if not data.frozen then - local done = t.done - if type(what) == "string" then - what = settings_to_hash(what) -- inefficient but ok +end -- of closure + +do -- create closure to overcome 200 locals limit + +-- original size: 12211, stripped down to: 8441 + +if not modules then modules={} end modules ['util-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + comment="the strip code is written by Peter Cawley", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local rep,sub,byte,dump,format=string.rep,string.sub,string.byte,string.dump,string.format +local load,loadfile,type=load,loadfile,type +utilities=utilities or {} +utilities.lua=utilities.lua or {} +local luautilities=utilities.lua +utilities.report=logs and logs.reporter("system") or print +local tracestripping=false +local forcestupidcompile=true +luautilities.stripcode=true +luautilities.alwaysstripcode=false +luautilities.nofstrippedchunks=0 +luautilities.nofstrippedbytes=0 +local strippedchunks={} +luautilities.strippedchunks=strippedchunks +luautilities.suffixes={ + tma="tma", + tmc=jit and "tmb" or "tmc", + lua="lua", + luc=jit and "lub" or "luc", + lui="lui", + luv="luv", + luj="luj", + tua="tua", + tuc="tuc", +} +local function fatalerror(name) + utilities.report(format("fatal error in %q",name or "unknown")) +end +if jit or status.luatex_version>=74 then + local function register(name) + if tracestripping then + utilities.report("stripped bytecode: %s",name or "unknown") + end + strippedchunks[#strippedchunks+1]=name + luautilities.nofstrippedchunks=luautilities.nofstrippedchunks+1 + end + local function stupidcompile(luafile,lucfile,strip) + local code=io.loaddata(luafile) + if code and code~="" then + code=load(code) + if code then + code=dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) + if code and code~="" then + register(name) + io.savedata(lucfile,code) + return true,0 + end + else + fatalerror() + end + else + fatalerror() + end + return false,0 + end + function luautilities.loadedluacode(fullname,forcestrip,name) + name=name or fullname + local code=loadfile(fullname) + if code then + code() + end + if forcestrip and luautilities.stripcode then + if type(forcestrip)=="function" then + forcestrip=forcestrip(fullname) + end + if forcestrip or luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end + elseif luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end + end + function luautilities.strippedloadstring(code,forcestrip,name) + if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then + code=load(code) + if not code then + fatalerror(name) + end + register(name) + code=dump(code,true) + end + return load(code),0 + end + function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) + utilities.report("lua: compiling %s into %s",luafile,lucfile) + os.remove(lucfile) + local done=stupidcompile(luafile,lucfile,strip~=false) + if done then + utilities.report("lua: %s dumped into %s (stripped)",luafile,lucfile) + if cleanup==true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + utilities.report("lua: removing %s",luafile) + os.remove(luafile) + end + end + return done + end +else + local function register(name,before,after) + local delta=before-after + if tracestripping then + utilities.report("stripped bytecode: %s, before %s, after %s, delta %s",name or "unknown",before,after,delta) + end + strippedchunks[#strippedchunks+1]=name + luautilities.nofstrippedchunks=luautilities.nofstrippedchunks+1 + luautilities.nofstrippedbytes=luautilities.nofstrippedbytes+delta + return delta + end + local strip_code_pc + if _MAJORVERSION==5 and _MINORVERSION==1 then + strip_code_pc=function(dump,name) + local before=#dump + local version,format,endian,int,size,ins,num=byte(dump,5,11) + local subint + if endian==1 then + subint=function(dump,i,l) + local val=0 + for n=l,1,-1 do + val=val*256+byte(dump,i+n-1) + end + return val,i+l + end + else + subint=function(dump,i,l) + local val=0 + for n=1,l,1 do + val=val*256+byte(dump,i+n-1) + end + return val,i+l end - if type(what) ~= "table" then - return + end + local strip_function + strip_function=function(dump) + local count,offset=subint(dump,1,size) + local stripped,dirty=rep("\0",size),offset+count + offset=offset+count+int*2+4 + offset=offset+int+subint(dump,offset,int)*ins + count,offset=subint(dump,offset,int) + for n=1,count do + local t + t,offset=subint(dump,offset,1) + if t==1 then + offset=offset+1 + elseif t==4 then + offset=offset+size+subint(dump,offset,size) + elseif t==3 then + offset=offset+num + end end - if not done then -- catch ... why not set? - done = { } - t.done = done + count,offset=subint(dump,offset,int) + stripped=stripped..sub(dump,dirty,offset-1) + for n=1,count do + local proto,off=strip_function(sub(dump,offset,-1)) + stripped,offset=stripped..proto,offset+off-1 end - for w, value in next, what do - if value == "" then - value = newvalue - elseif not value then - value = false -- catch nil - else - value = is_boolean(value,value) - end - w = topattern(w,true,true) - for name, functions in next, data do - if done[name] then - -- prevent recursion due to wildcards - elseif find(name,w) then - done[name] = true - for i=1,#functions do - functions[i](value) - end - functions.value = value - end - end + offset=offset+subint(dump,offset,int)*int+int + count,offset=subint(dump,offset,int) + for n=1,count do + offset=offset+subint(dump,offset,size)+size+int*2 end - end -end - -local function reset(t) - local data = t.data - if not data.frozen then - for name, functions in next, data do - for i=1,#functions do - functions[i](false) - end - functions.value = false + count,offset=subint(dump,offset,int) + for n=1,count do + offset=offset+subint(dump,offset,size)+size end + stripped=stripped..rep("\0",int*3) + return stripped,offset + end + dump=sub(dump,1,12)..strip_function(sub(dump,13,-1)) + local after=#dump + local delta=register(name,before,after) + return dump,delta + end + else + strip_code_pc=function(dump,name) + return dump,0 + end + end + function luautilities.loadedluacode(fullname,forcestrip,name) + name=name or fullname + local code=loadfile(fullname) + if code then + code() + end + if forcestrip and luautilities.stripcode then + if type(forcestrip)=="function" then + forcestrip=forcestrip(fullname) + end + if forcestrip then + local code,n=strip_code_pc(dump(code),name) + return load(code),n + elseif luautilities.alwaysstripcode then + return load(strip_code_pc(dump(code),name)) + else + return code,0 + end + elseif luautilities.alwaysstripcode then + return load(strip_code_pc(dump(code),name)) + else + return code,0 + end + end + function luautilities.strippedloadstring(code,forcestrip,name) + local n=0 + if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then + code=load(code) + if not code then + fatalerror(name) + end + code,n=strip_code_pc(dump(code),name) + end + return load(code),n + end + local function stupidcompile(luafile,lucfile,strip) + local code=io.loaddata(luafile) + local n=0 + if code and code~="" then + code=load(code) + if not code then + fatalerror() + end + code=dump(code) + if strip then + code,n=strip_code_pc(code,luautilities.stripcode or luautilities.alwaysstripcode,luafile) + end + if code and code~="" then + io.savedata(lucfile,code) + end + end + return n + end + local luac_normal="texluac -o %q %q" + local luac_strip="texluac -s -o %q %q" + function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) + utilities.report("lua: compiling %s into %s",luafile,lucfile) + os.remove(lucfile) + local done=false + if strip~=false then + strip=true + end + if forcestupidcompile then + fallback=true + elseif strip then + done=os.spawn(format(luac_strip,lucfile,luafile))==0 + else + done=os.spawn(format(luac_normal,lucfile,luafile))==0 + end + if not done and fallback then + local n=stupidcompile(luafile,lucfile,strip) + if n>0 then + utilities.report("lua: %s dumped into %s (%i bytes stripped)",luafile,lucfile,n) + else + utilities.report("lua: %s dumped into %s (unstripped)",luafile,lucfile) + end + cleanup=false + done=true end + if done and cleanup==true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + utilities.report("lua: removing %s",luafile) + os.remove(luafile) + end + return done + end end -local function enable(t,what) - set(t,what,true) -end -local function disable(t,what) - local data = t.data - if not what or what == "" then - t.done = { } - reset(t) - else - set(t,what,false) - end -end +end -- of closure -function setters.register(t,what,...) - local data = t.data - what = lower(what) - local functions = data[what] - if not functions then - functions = { } - data[what] = functions - if trace_initialize then - t.report("defining %s",what) - end - end - local default = functions.default -- can be set from cnf file - for i=1,select("#",...) do - local fnc = select(i,...) - local typ = type(fnc) - if typ == "string" then - if trace_initialize then - t.report("coupling %s to %s",what,fnc) - end - local s = fnc -- else wrong reference - fnc = function(value) set(t,s,value) end - elseif typ ~= "function" then - fnc = nil - end - if fnc then - functions[#functions+1] = fnc - -- default: set at command line or in cnf file - -- value : set in tex run (needed when loading runtime) - local value = functions.value or default - if value ~= nil then - fnc(value) - functions.value = value - end - end - end - return false -- so we can use it in an assignment -end +do -- create closure to overcome 200 locals limit -function setters.enable(t,what) - local e = t.enable - t.enable, t.done = enable, { } - enable(t,what) - t.enable, t.done = e, { } -end +-- original size: 14514, stripped down to: 10374 -function setters.disable(t,what) - local e = t.disable - t.disable, t.done = disable, { } - disable(t,what) - t.disable, t.done = e, { } +if not modules then modules={} end modules ['util-prs']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local lpeg,table,string=lpeg,table,string +local P,R,V,S,C,Ct,Cs,Carg,Cc,Cg,Cf,Cp=lpeg.P,lpeg.R,lpeg.V,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cs,lpeg.Carg,lpeg.Cc,lpeg.Cg,lpeg.Cf,lpeg.Cp +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local concat,format,gmatch,find=table.concat,string.format,string.gmatch,string.find +local tostring,type,next,rawset=tostring,type,next,rawset +utilities=utilities or {} +utilities.parsers=utilities.parsers or {} +local parsers=utilities.parsers +parsers.patterns=parsers.patterns or {} +local setmetatableindex=table.setmetatableindex +local sortedhash=table.sortedhash +local digit=R("09") +local space=P(' ') +local equal=P("=") +local comma=P(",") +local lbrace=P("{") +local rbrace=P("}") +local lparent=P("(") +local rparent=P(")") +local period=S(".") +local punctuation=S(".,:;") +local spacer=patterns.spacer +local whitespace=patterns.whitespace +local newline=patterns.newline +local anything=patterns.anything +local endofstring=patterns.endofstring +local nobrace=1-(lbrace+rbrace ) +local noparent=1-(lparent+rparent) +local escape,left,right=P("\\"),P('{'),P('}') +patterns.balanced=P { + [1]=((escape*(left+right))+(1-(left+right))+V(2))^0, + [2]=left*V(1)*right +} +local nestedbraces=P { lbrace*(nobrace+V(1))^0*rbrace } +local nestedparents=P { lparent*(noparent+V(1))^0*rparent } +local spaces=space^0 +local argument=Cs((lbrace/"")*((nobrace+nestedbraces)^0)*(rbrace/"")) +local content=(1-endofstring)^0 +patterns.nestedbraces=nestedbraces +patterns.nestedparents=nestedparents +patterns.nested=nestedbraces +patterns.argument=argument +patterns.content=content +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local key=C((1-equal-comma)^1) +local pattern_a=(space+comma)^0*(key*equal*value+key*C("")) +local pattern_c=(space+comma)^0*(key*equal*value) +local key=C((1-space-equal-comma)^1) +local pattern_b=spaces*comma^0*spaces*(key*((spaces*equal*spaces*value)+C(""))) +local hash={} +local function set(key,value) + hash[key]=value +end +local pattern_a_s=(pattern_a/set)^1 +local pattern_b_s=(pattern_b/set)^1 +local pattern_c_s=(pattern_c/set)^1 +parsers.patterns.settings_to_hash_a=pattern_a_s +parsers.patterns.settings_to_hash_b=pattern_b_s +parsers.patterns.settings_to_hash_c=pattern_c_s +function parsers.make_settings_to_hash_pattern(set,how) + if how=="strict" then + return (pattern_c/set)^1 + elseif how=="tolerant" then + return (pattern_b/set)^1 + else + return (pattern_a/set)^1 + end end - -function setters.reset(t) - t.done = { } - reset(t) +function parsers.settings_to_hash(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_a_s,str) + return hash + else + return {} + end end - -function setters.list(t) -- pattern - local list = table.sortedkeys(t.data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what +function parsers.settings_to_hash_tolerant(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_b_s,str) + return hash + else + return {} + end +end +function parsers.settings_to_hash_strict(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_c_s,str) + return next(hash) and hash + else + return nil + end +end +local separator=comma*space^0 +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local pattern=spaces*Ct(value*(separator*value)^0) +parsers.patterns.settings_to_array=pattern +function parsers.settings_to_array(str,strict) + if not str or str=="" then + return {} + elseif strict then + if find(str,"{") then + return lpegmatch(pattern,str) + else + return { str } + end + else + return lpegmatch(pattern,str) + end +end +local function set(t,v) + t[#t+1]=v +end +local value=P(Carg(1)*value)/set +local pattern=value*(separator*value)^0*Carg(1) +function parsers.add_settings_to_array(t,str) + return lpegmatch(pattern,str,nil,t) +end +function parsers.hash_to_string(h,separator,yes,no,strict,omit) + if h then + local t,tn,s={},0,table.sortedkeys(h) + omit=omit and table.tohash(omit) + for i=1,#s do + local key=s[i] + if not omit or not omit[key] then + local value=h[key] + if type(value)=="boolean" then + if yes and no then + if value then + tn=tn+1 + t[tn]=key..'='..yes + elseif not strict then + tn=tn+1 + t[tn]=key..'='..no + end + elseif value or not strict then + tn=tn+1 + t[tn]=key..'='..tostring(value) + end else - user[#user+1] = what + tn=tn+1 + t[tn]=key..'='..value end + end end - return user, system + return concat(t,separator or ",") + else + return "" + end end - -function setters.show(t) - local category = t.name - local list = setters.list(t) - t.report() - for k=1,#list do - local name = list[k] - local functions = t.data[name] - if functions then - local value, default, modules = functions.value, functions.default, #functions - value = value == nil and "unset" or tostring(value) - default = default == nil and "unset" or tostring(default) - t.report("%-50s modules: %2i default: %-12s value: %-12s",name,modules,default,value) - end +function parsers.array_to_string(a,separator) + if a then + return concat(a,separator or ",") + else + return "" + end +end +function parsers.settings_to_set(str,t) + t=t or {} + for s in gmatch(str,"[^, ]+") do + t[s]=true + end + return t +end +function parsers.simple_hash_to_string(h,separator) + local t,tn={},0 + for k,v in sortedhash(h) do + if v then + tn=tn+1 + t[tn]=k end - t.report() + end + return concat(t,separator or ",") end - --- we could have used a bit of oo and the trackers:enable syntax but --- there is already a lot of code around using the singular tracker - --- we could make this into a module but we also want the rest avaliable - -local enable, disable, register, list, show = setters.enable, setters.disable, setters.register, setters.list, setters.show - -local write_nl = texio and texio.write_nl or print - -local function report(setter,...) - local report = logs and logs.report - if report then - report(setter.name,...) - else -- fallback, as this module is loaded before the logger - write_nl(format("%-15s : %s\n",setter.name,format(...))) +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) +local pattern_a=spaces*Ct(value*(separator*value)^0) +local function repeater(n,str) + if not n then + return str + else + local s=lpegmatch(pattern_a,str) + if n==1 then + return unpack(s) + else + local t,tn={},0 + for i=1,n do + for j=1,#s do + tn=tn+1 + t[tn]=s[j] + end + end + return unpack(t) end + end end - -local function default(setter,name) - local d = setter.data[name] - return d and d.default +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+(C(digit^1)/tonumber*lparent*Cs((noparent+nestedparents)^1)*rparent)/repeater+C((nestedbraces+(1-comma))^1) +local pattern_b=spaces*Ct(value*(separator*value)^0) +function parsers.settings_to_array_with_repeat(str,expand) + if expand then + return lpegmatch(pattern_b,str) or {} + else + return lpegmatch(pattern_a,str) or {} + end end - -local function value(setter,name) - local d = setter.data[name] - return d and (d.value or d.default) -end - -function setters.new(name) -- we could use foo:bar syntax (but not used that often) - local setter -- we need to access it in setter itself - setter = { - data = allocate(), -- indexed, but also default and value fields - name = name, - report = function(...) report (setter,...) end, - enable = function(...) enable (setter,...) end, - disable = function(...) disable (setter,...) end, - register = function(...) register(setter,...) end, - list = function(...) list (setter,...) end, - show = function(...) show (setter,...) end, - default = function(...) return default (setter,...) end, - value = function(...) return value (setter,...) end, - } - data[name] = setter - return setter +local value=lbrace*C((nobrace+nestedbraces)^0)*rbrace +local pattern=Ct((space+value)^0) +function parsers.arguments_to_table(str) + return lpegmatch(pattern,str) end - -trackers = setters.new("trackers") -directives = setters.new("directives") -experiments = setters.new("experiments") - -local t_enable, t_disable, t_report = trackers .enable, trackers .disable, trackers .report -local d_enable, d_disable, d_report = directives .enable, directives .disable, directives .report -local e_enable, e_disable, e_report = experiments.enable, experiments.disable, experiments.report - --- nice trick: we overload two of the directives related functions with variants that --- do tracing (itself using a tracker) .. proof of concept - -local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) -local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) - -function directives.enable(...) - if trace_directives then - d_report("enabling: %s",concat({...}," ")) +function parsers.getparameters(self,class,parentclass,settings) + local sc=self[class] + if not sc then + sc={} + self[class]=sc + if parentclass then + local sp=self[parentclass] + if not sp then + sp={} + self[parentclass]=sp + end + setmetatableindex(sc,sp) end - d_enable(...) + end + parsers.settings_to_hash(settings,sc) end - -function directives.disable(...) - if trace_directives then - d_report("disabling: %s",concat({...}," ")) - end - d_disable(...) +function parsers.listitem(str) + return gmatch(str,"[^, ]+") +end +local pattern=Cs { "start", + start=V("one")+V("two")+V("three"), + rest=(Cc(",")*V("thousand"))^0*(P(".")+endofstring)*anything^0, + thousand=digit*digit*digit, + one=digit*V("rest"), + two=digit*digit*V("rest"), + three=V("thousand")*V("rest"), +} +patterns.splitthousands=pattern +function parsers.splitthousands(str) + return lpegmatch(pattern,str) or str +end +local optionalwhitespace=whitespace^0 +patterns.words=Ct((Cs((1-punctuation-whitespace)^1)+anything)^1) +patterns.sentences=Ct((optionalwhitespace*Cs((1-period)^0*period))^1) +patterns.paragraphs=Ct((optionalwhitespace*Cs((whitespace^1*endofstring/""+1-(spacer^0*newline*newline))^1))^1) +local dquote=P('"') +local equal=P('=') +local escape=P('\\') +local separator=S(' ,') +local key=C((1-equal)^1) +local value=dquote*C((1-dquote-escape*dquote)^0)*dquote +local pattern=Cf(Ct("")*Cg(key*equal*value)*separator^0,rawset)^0 +parsers.patterns.keq_to_hash_c=pattern +function parsers.keq_to_hash(str) + if str and str~="" then + return lpegmatch(pattern,str) + else + return {} + end end - -function experiments.enable(...) - if trace_experiments then - e_report("enabling: %s",concat({...}," ")) +local defaultspecification={ separator=",",quote='"' } +function parsers.csvsplitter(specification) + specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=specification.quote + local separator=S(separator~="" and separator or ",") + local whatever=C((1-separator-newline)^0) + if quotechar and quotechar~="" then + local quotedata=nil + for chr in gmatch(quotechar,".") do + local quotechar=P(chr) + local quoteword=quotechar*C((1-quotechar)^0)*quotechar + if quotedata then + quotedata=quotedata+quoteword + else + quotedata=quoteword + end end - e_enable(...) + whatever=quotedata+whatever + end + local parser=Ct((Ct(whatever*(separator*whatever)^0)*S("\n\r"))^0 ) + return function(data) + return lpegmatch(parser,data) + end end - -function experiments.disable(...) - if trace_experiments then - e_report("disabling: %s",concat({...}," ")) - end - e_disable(...) +function parsers.rfc4180splitter(specification) + specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=P(specification.quote) + local dquotechar=quotechar*quotechar +/specification.quote + local separator=S(separator~="" and separator or ",") + local escaped=quotechar*Cs((dquotechar+(1-quotechar))^0)*quotechar + local non_escaped=C((1-quotechar-newline-separator)^1) + local field=escaped+non_escaped + local record=Ct((field*separator^-1)^1) + local headerline=record*Cp() + local wholeblob=Ct((newline^-1*record)^0) + return function(data,getheader) + if getheader then + local header,position=lpegmatch(headerline,data) + local data=lpegmatch(wholeblob,data,position) + return data,header + else + return lpegmatch(wholeblob,data) + end + end +end +local function ranger(first,last,n,action) + if not first then + elseif last==true then + for i=first,n or first do + action(i) + end + elseif last then + for i=first,last do + action(i) + end + else + action(first) + end +end +local cardinal=patterns.cardinal/tonumber +local spacers=patterns.spacer^0 +local endofstring=patterns.endofstring +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1 +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+(P("*")+endofstring)*Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1*endofstring +function utilities.parsers.stepper(str,n,action) + if type(n)=="function" then + lpegmatch(stepper,str,1,false,n or print) + else + lpegmatch(stepper,str,1,n,action or print) + end end --- a useful example - -directives.register("system.nostatistics", function(v) - statistics.enable = not v -end) - -directives.register("system.nolibraries", function(v) - libraries = nil -- we discard this tracing for security -end) - --- experiment -if environment then +end -- of closure - -- The engineflags are known earlier than environment.arguments but maybe we - -- need to handle them both as the later are parsed differently. The c: prefix - -- is used by mtx-context to isolate the flags from those that concern luatex. +do -- create closure to overcome 200 locals limit - local engineflags = environment.engineflags +-- original size: 3006, stripped down to: 2072 - if engineflags then - local list = engineflags["c:trackers"] or engineflags["trackers"] - if type(list) == "string" then - setters.initialize("commandline flags","trackers",settings_to_hash(list)) - -- t_enable(list) - end - local list = engineflags["c:directives"] or engineflags["directives"] - if type(list) == "string" then - setters.initialize("commandline flags","directives", settings_to_hash(list)) - -- d_enable(list) +if not modules then modules={} end modules ['util-fmt']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.formatters=utilities.formatters or {} +local formatters=utilities.formatters +local concat,format=table.concat,string.format +local tostring,type=tostring,type +local strip=string.strip +local P,R,Cs=lpeg.P,lpeg.R,lpeg.Cs +local lpegmatch=lpeg.match +local digit=R("09") +local period=P(".") +local zero=P("0") +local trailingzeros=zero^0*-digit +local case_1=period*trailingzeros/"" +local case_2=period*(digit-trailingzeros)^1*(trailingzeros/"") +local number=digit^1*(case_1+case_2) +local stripper=Cs((number+1)^0) +lpeg.patterns.stripzeros=stripper +function formatters.stripzeros(str) + return lpegmatch(stripper,str) +end +function formatters.formatcolumns(result,between) + if result and #result>0 then + between=between or " " + local widths,numbers={},{} + local first=result[1] + local n=#first + for i=1,n do + widths[i]=0 + end + for i=1,#result do + local r=result[i] + for j=1,n do + local rj=r[j] + local tj=type(rj) + if tj=="number" then + numbers[j]=true + end + if tj~="string" then + rj=tostring(rj) + r[j]=rj + end + local w=#rj + if w>widths[j] then + widths[j]=w end + end end - -end - --- here - -if texconfig then - - -- this happens too late in ini mode but that is no problem - - local function set(k,v) - v = tonumber(v) - if v then - texconfig[k] = v + for i=1,n do + local w=widths[i] + if numbers[i] then + if w>80 then + widths[i]="%s"..between + else + widths[i]="%0"..w.."i"..between + end + else + if w>80 then + widths[i]="%s"..between + elseif w>0 then + widths[i]="%-"..w.."s"..between + else + widths[i]="%s" end + end end - - directives.register("luatex.expanddepth", function(v) set("expand_depth",v) end) - directives.register("luatex.hashextra", function(v) set("hash_extra",v) end) - directives.register("luatex.nestsize", function(v) set("nest_size",v) end) - directives.register("luatex.maxinopen", function(v) set("max_in_open",v) end) - directives.register("luatex.maxprintline", function(v) set("max_print_line",v) end) - directives.register("luatex.maxstrings", function(v) set("max_strings",v) end) - directives.register("luatex.paramsize", function(v) set("param_size",v) end) - directives.register("luatex.savesize", function(v) set("save_size",v) end) - directives.register("luatex.stacksize", function(v) set("stack_size",v) end) - + local template=strip(concat(widths)) + for i=1,#result do + local str=format(template,unpack(result[i])) + result[i]=strip(str) + end + end + return result end @@ -8342,1234 +5600,1558 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-log'] = { - version = 1.001, - comment = "companion to trac-log.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- todo: less categories, more subcategories (e.g. nodes) - - -local write_nl, write = texio and texio.write_nl or print, texio and texio.write or io.write -local format, gmatch, find = string.format, string.gmatch, string.find -local concat, insert, remove = table.concat, table.insert, table.remove -local topattern = string.topattern -local texcount = tex and tex.count -local next, type, select = next, type, select - -local setmetatableindex = table.setmetatableindex -local formatters = string.formatters - - - -logs = logs or { } -local logs = logs - -local moreinfo = [[ -More information about ConTeXt and the tools that come with it can be found at: - -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] - --- basic loggers - -local function ignore() end - -setmetatableindex(logs, function(t,k) t[k] = ignore ; return ignore end) - -local report, subreport, status, settarget, setformats, settranslations +-- original size: 4118, stripped down to: 2901 -local direct, subdirect, writer, pushtarget, poptarget - -if tex and (tex.jobname or tex.formatname) then +if not modules then modules={} end modules ['util-deb']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local debug=require "debug" +local getinfo=debug.getinfo +local type,next,tostring=type,next,tostring +local format,find=string.format,string.find +local is_boolean=string.is_boolean +utilities=utilities or {} +utilities.debugger=utilities.debugger or {} +local debugger=utilities.debugger +local counters={} +local names={} +local function hook() + local f=getinfo(2) + if f then + local n="unknown" + if f.what=="C" then + n=f.name or '' + if not names[n] then + names[n]=format("%42s",n) + end + else + n=f.name or f.namewhat or f.what + if not n or n=="" then + n="?" + end + if not names[n] then + names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + end + counters[n]=(counters[n] or 0)+1 + end +end +function debugger.showstats(printer,threshold) + printer=printer or texio.write or print + threshold=threshold or 0 + local total,grandtotal,functions=0,0,0 + local dataset={} + for name,count in next,counters do + dataset[#dataset+1]={ name,count } + end + table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + for i=1,#dataset do + local d=dataset[i] + local name=d[1] + local count=d[2] + if count>threshold and not find(name,"for generator") then + printer(format("%8i %s\n",count,names[name])) + total=total+count + end + grandtotal=grandtotal+count + functions=functions+1 + end + printer("\n") + printer(format("functions : % 10i\n",functions)) + printer(format("total : % 10i\n",total)) + printer(format("grand total: % 10i\n",grandtotal)) + printer(format("threshold : % 10i\n",threshold)) +end +function debugger.savestats(filename,threshold) + local f=io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end +function debugger.enable() + debug.sethook(hook,"c") +end +function debugger.disable() + debug.sethook() +end +local is_node=node and node.is_node +local is_lpeg=lpeg and lpeg.type +function inspect(i) + local ti=type(i) + if ti=="table" then + table.print(i,"table") + elseif is_node and is_node(i) then + table.print(nodes.astable(i),tostring(i)) + elseif is_lpeg and is_lpeg(i) then + lpeg.print(i) + else + print(tostring(i)) + end + return i +end +function traceback() + local level=1 + while true do + local info=debug.getinfo(level,"Sl") + if not info then + break + elseif info.what=="C" then + print(format("%3i : C function",level)) + else + print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) + end + level=level+1 + end +end - local valueiskey = { __index = function(t,k) t[k] = k return k end } -- will be helper - local target = "term and log" +end -- of closure - logs.flush = io.flush +do -- create closure to overcome 200 locals limit - local formats = { } setmetatable(formats, valueiskey) - local translations = { } setmetatable(translations,valueiskey) +-- original size: 6209, stripped down to: 4958 - writer = function(...) - write_nl(target,...) +if not modules then modules={} end modules ['trac-inf']={ + version=1.001, + comment="companion to trac-inf.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower=string.format,string.lower +local concat=table.concat +local clock=os.gettimeofday or os.clock +local write_nl=texio and texio.write_nl or print +statistics=statistics or {} +local statistics=statistics +statistics.enable=true +statistics.threshold=0.01 +local statusinfo,n,registered,timers={},0,{},{} +table.setmetatableindex(timers,function(t,k) + local v={ timing=0,loadtime=0 } + t[k]=v + return v +end) +local function hastiming(instance) + return instance and timers[instance] +end +local function resettiming(instance) + timers[instance or "notimer"]={ timing=0,loadtime=0 } +end +local function starttiming(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing or 0 + if it==0 then + timer.starttime=clock() + if not timer.loadtime then + timer.loadtime=0 + end + end + timer.timing=it+1 +end +local function stoptiming(instance,report) + local timer=timers[instance or "notimer"] + local it=timer.timing + if it>1 then + timer.timing=it-1 + else + local starttime=timer.starttime + if starttime then + local stoptime=clock() + local loadtime=stoptime-starttime + timer.stoptime=stoptime + timer.loadtime=timer.loadtime+loadtime + if report then + statistics.report("load time %0.3f",loadtime) + end + timer.timing=0 + return loadtime end - - newline = function() - write_nl(target,"\n") + end + return 0 +end +local function elapsedtime(instance) + local timer=timers[instance or "notimer"] + return format("%0.3f",timer and timer.loadtime or 0) +end +local function elapsedindeed(instance) + local timer=timers[instance or "notimer"] + return (timer and timer.loadtime or 0)>statistics.threshold +end +local function elapsedseconds(instance,rest) + if elapsedindeed(instance) then + return format("%s seconds %s",elapsedtime(instance),rest or "") + end +end +statistics.hastiming=hastiming +statistics.resettiming=resettiming +statistics.starttiming=starttiming +statistics.stoptiming=stoptiming +statistics.elapsedtime=elapsedtime +statistics.elapsedindeed=elapsedindeed +statistics.elapsedseconds=elapsedseconds +function statistics.register(tag,fnc) + if statistics.enable and type(fnc)=="function" then + local rt=registered[tag] or (#statusinfo+1) + statusinfo[rt]={ tag,fnc } + registered[tag]=rt + if #tag>n then n=#tag end + end +end +function statistics.show(reporter) + if statistics.enable then + if not reporter then reporter=function(tag,data,n) write_nl(tag.." "..data) end end + local register=statistics.register + register("luatex banner",function() + return lower(status.banner) + end) + register("control sequences",function() + return format("%s of %s + %s",status.cs_count,status.hash_size,status.hash_extra) + end) + register("callbacks",function() + local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 + return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) + end) + if jit then + local status={ jit.status() } + if status[1] then + register("luajit status",function() + return concat(status," ",2) + end) + end end - - local f_one = formatters["%-15s > %s\n"] - local f_two = formatters["%-15s >\n"] - - report = function(a,b,c,...) - if c then - write_nl(target,f_one(translations[a],format(formats[b],c,...))) - elseif b then - write_nl(target,f_one(translations[a],formats[b])) - elseif a then - write_nl(target,f_two(translations[a])) - else - write_nl(target,"\n") - end + collectgarbage("collect") + register("current memory usage",statistics.memused) + register("runtime",statistics.runtime) + for i=1,#statusinfo do + local s=statusinfo[i] + local r=s[2]() + if r then + reporter(s[1],r,n) + end end - - local f_one = formatters["%-15s > %s"] - local f_two = formatters["%-15s >"] - - direct = function(a,b,c,...) - if c then - return f_one(translations[a],format(formats[b],c,...)) - elseif b then - return f_one(translations[a],formats[b]) - elseif a then - return f_two(translations[a]) - else - return "" - end + write_nl("") + statistics.enable=false + end +end +local template,report_statistics,nn=nil,nil,0 +function statistics.showjobstat(tag,data,n) + if not logs then + elseif type(data)=="table" then + for i=1,#data do + statistics.showjobstat(tag,data[i],n) end - - local f_one = formatters["%-15s > %s > %s\n"] - local f_two = formatters["%-15s > %s >\n"] - - subreport = function(a,s,b,c,...) - if c then - write_nl(target,f_one(translations[a],translations[s],format(formats[b],c,...))) - elseif b then - write_nl(target,f_one(translations[a],translations[s],formats[b])) - elseif a then - write_nl(target,f_two(translations[a],translations[s])) - else - write_nl(target,"\n") - end + else + if not template or n>nn then + template,n=format("%%-%ss - %%s",n),nn + report_statistics=logs.reporter("mkiv lua stats") end + report_statistics(format(template,tag,data)) + end +end +function statistics.memused() + local round=math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000),round(status.luastate_bytes/1000000)) +end +starttiming(statistics) +function statistics.formatruntime(runtime) + return format("%s seconds",runtime) +end +function statistics.runtime() + stoptiming(statistics) + return statistics.formatruntime(elapsedtime(statistics)) +end +function statistics.timed(action,report) + report=report or logs.reporter("system") + starttiming("run") + action() + stoptiming("run") + report("total runtime: %s",elapsedtime("run")) +end +commands=commands or {} +function commands.resettimer(name) + resettiming(name or "whatever") + starttiming(name or "whatever") +end +function commands.elapsedtime(name) + stoptiming(name or "whatever") + context(elapsedtime(name or "whatever")) +end - local f_one = formatters["%-15s > %s > %s"] - local f_two = formatters["%-15s > %s >"] - - subdirect = function(a,s,b,c,...) - if c then - return f_one(translations[a],translations[s],format(formats[b],c,...)) - elseif b then - return f_one(translations[a],translations[s],formats[b]) - elseif a then - return f_two(translations[a],translations[s]) - else - return "" - end - end - local f_one = formatters["%-15s : %s\n"] - local f_two = formatters["%-15s :\n"] +end -- of closure - status = function(a,b,c,...) - if c then - write_nl(target,f_one(translations[a],format(formats[b],c,...))) - elseif b then - write_nl(target,f_one(translations[a],formats[b])) - elseif a then - write_nl(target,f_two(translations[a])) - else - write_nl(target,"\n") - end - end +do -- create closure to overcome 200 locals limit - local targets = { - logfile = "log", - log = "log", - file = "log", - console = "term", - terminal = "term", - both = "term and log", - } +-- original size: 12560, stripped down to: 8979 - settarget = function(whereto) - target = targets[whereto or "both"] or targets.both - if target == "term" or target == "term and log" then - logs.flush = io.flush +if not modules then modules={} end modules ['trac-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring=type,next,tostring +local concat=table.concat +local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local is_boolean=string.is_boolean +local settings_to_hash=utilities.parsers.settings_to_hash +local allocate=utilities.storage.allocate +utilities=utilities or {} +local utilities=utilities +utilities.setters=utilities.setters or {} +local setters=utilities.setters +local data={} +local trace_initialize=false +function setters.initialize(filename,name,values) + local setter=data[name] + if setter then + frozen=true + local data=setter.data + if data then + for key,newvalue in next,values do + local newvalue=is_boolean(newvalue,newvalue) + local functions=data[key] + if functions then + local oldvalue=functions.value + if functions.frozen then + if trace_initialize then + setter.report("%s: %q is frozen to %q",filename,key,tostring(oldvalue)) + end + elseif #functions>0 and not oldvalue then + if trace_initialize then + setter.report("%s: %q is set to %q",filename,key,tostring(newvalue)) + end + for i=1,#functions do + functions[i](newvalue) + end + functions.value=newvalue + functions.frozen=functions.frozen or frozen + else + if trace_initialize then + setter.report("%s: %q is kept as %q",filename,key,tostring(oldvalue)) + end + end else - logs.flush = ignore + functions={ default=newvalue,frozen=frozen } + data[key]=functions + if trace_initialize then + setter.report("%s: %q default to %q",filename,key,tostring(newvalue)) + end end + end + return true end - - local stack = { } - - pushtarget = function(newtarget) - insert(stack,target) - settarget(newtarget) - end - - poptarget = function() - if #stack > 0 then - settarget(remove(stack)) + end +end +local function set(t,what,newvalue) + local data=t.data + if not data.frozen then + local done=t.done + if type(what)=="string" then + what=settings_to_hash(what) + end + if type(what)~="table" then + return + end + if not done then + done={} + t.done=done + end + for w,value in next,what do + if value=="" then + value=newvalue + elseif not value then + value=false + else + value=is_boolean(value,value) + end + w=topattern(w,true,true) + for name,functions in next,data do + if done[name] then + elseif find(name,w) then + done[name]=true + for i=1,#functions do + functions[i](value) + end + functions.value=value end + end end - - setformats = function(f) - formats = f + end +end +local function reset(t) + local data=t.data + if not data.frozen then + for name,functions in next,data do + for i=1,#functions do + functions[i](false) + end + functions.value=false end - - settranslations = function(t) - translations = t + end +end +local function enable(t,what) + set(t,what,true) +end +local function disable(t,what) + local data=t.data + if not what or what=="" then + t.done={} + reset(t) + else + set(t,what,false) + end +end +function setters.register(t,what,...) + local data=t.data + what=lower(what) + local functions=data[what] + if not functions then + functions={} + data[what]=functions + if trace_initialize then + t.report("defining %s",what) + end + end + local default=functions.default + for i=1,select("#",...) do + local fnc=select(i,...) + local typ=type(fnc) + if typ=="string" then + if trace_initialize then + t.report("coupling %s to %s",what,fnc) + end + local s=fnc + fnc=function(value) set(t,s,value) end + elseif typ~="function" then + fnc=nil + end + if fnc then + functions[#functions+1]=fnc + local value=functions.value or default + if value~=nil then + fnc(value) + functions.value=value + end end - -else - - logs.flush = ignore - - writer = write_nl - - newline = function() - write_nl("\n") + end + return false +end +function setters.enable(t,what) + local e=t.enable + t.enable,t.done=enable,{} + enable(t,what) + t.enable,t.done=e,{} +end +function setters.disable(t,what) + local e=t.disable + t.disable,t.done=disable,{} + disable(t,what) + t.disable,t.done=e,{} +end +function setters.reset(t) + t.done={} + reset(t) +end +function setters.list(t) + local list=table.sortedkeys(t.data) + local user,system={},{} + for l=1,#list do + local what=list[l] + if find(what,"^%*") then + system[#system+1]=what + else + user[#user+1]=what end - - local f_one = formatters["%-15s | %s"] - local f_two = formatters["%-15s |"] - - report = function(a,b,c,...) - if c then - write_nl(f_one(a,format(b,c,...))) - elseif b then - write_nl(f_one(a,b)) - elseif a then - write_nl(f_two(a)) - else - write_nl("") - end + end + return user,system +end +function setters.show(t) + local category=t.name + local list=setters.list(t) + t.report() + for k=1,#list do + local name=list[k] + local functions=t.data[name] + if functions then + local value,default,modules=functions.value,functions.default,#functions + value=value==nil and "unset" or tostring(value) + default=default==nil and "unset" or tostring(default) + t.report("%-50s modules: %2i default: %-12s value: %-12s",name,modules,default,value) + end + end + t.report() +end +local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show +local write_nl=texio and texio.write_nl or print +local function report(setter,...) + local report=logs and logs.report + if report then + report(setter.name,...) + else + write_nl(format("%-15s : %s\n",setter.name,format(...))) + end +end +local function default(setter,name) + local d=setter.data[name] + return d and d.default +end +local function value(setter,name) + local d=setter.data[name] + return d and (d.value or d.default) +end +function setters.new(name) + local setter + setter={ + data=allocate(), + name=name, + report=function(...) report (setter,...) end, + enable=function(...) enable (setter,...) end, + disable=function(...) disable (setter,...) end, + register=function(...) register(setter,...) end, + list=function(...) list (setter,...) end, + show=function(...) show (setter,...) end, + default=function(...) return default (setter,...) end, + value=function(...) return value (setter,...) end, + } + data[name]=setter + return setter +end +trackers=setters.new("trackers") +directives=setters.new("directives") +experiments=setters.new("experiments") +local t_enable,t_disable,t_report=trackers .enable,trackers .disable,trackers .report +local d_enable,d_disable,d_report=directives .enable,directives .disable,directives .report +local e_enable,e_disable,e_report=experiments.enable,experiments.disable,experiments.report +local trace_directives=false local trace_directives=false trackers.register("system.directives",function(v) trace_directives=v end) +local trace_experiments=false local trace_experiments=false trackers.register("system.experiments",function(v) trace_experiments=v end) +function directives.enable(...) + if trace_directives then + d_report("enabling: %s",concat({...}," ")) + end + d_enable(...) +end +function directives.disable(...) + if trace_directives then + d_report("disabling: %s",concat({...}," ")) + end + d_disable(...) +end +function experiments.enable(...) + if trace_experiments then + e_report("enabling: %s",concat({...}," ")) + end + e_enable(...) +end +function experiments.disable(...) + if trace_experiments then + e_report("disabling: %s",concat({...}," ")) + end + e_disable(...) +end +directives.register("system.nostatistics",function(v) + statistics.enable=not v +end) +directives.register("system.nolibraries",function(v) + libraries=nil +end) +if environment then + local engineflags=environment.engineflags + if engineflags then + local list=engineflags["c:trackers"] or engineflags["trackers"] + if type(list)=="string" then + setters.initialize("commandline flags","trackers",settings_to_hash(list)) end - - local f_one = formatters["%-15s | %s | %s"] - local f_two = formatters["%-15s | %s |"] - - subreport = function(a,sub,b,c,...) - if c then - write_nl(f_one(a,sub,format(b,c,...))) - elseif b then - write_nl(f_one(a,sub,b)) - elseif a then - write_nl(f_two(a,sub)) - else - write_nl("") - end + local list=engineflags["c:directives"] or engineflags["directives"] + if type(list)=="string" then + setters.initialize("commandline flags","directives",settings_to_hash(list)) end - - local f_one = formatters["%-15s : %s\n"] - local f_two = formatters["%-15s :\n"] - - status = function(a,b,c,...) -- not to be used in lua anyway - if c then - write_nl(f_one(a,format(b,c,...))) - elseif b then - write_nl(f_one(a,b)) -- b can have %'s - elseif a then - write_nl(f_two(a)) - else - write_nl("\n") - end + end +end +if texconfig then + local function set(k,v) + v=tonumber(v) + if v then + texconfig[k]=v end - - direct = ignore - subdirect = ignore - - settarget = ignore - pushtarget = ignore - poptarget = ignore - setformats = ignore - settranslations = ignore - + end + directives.register("luatex.expanddepth",function(v) set("expand_depth",v) end) + directives.register("luatex.hashextra",function(v) set("hash_extra",v) end) + directives.register("luatex.nestsize",function(v) set("nest_size",v) end) + directives.register("luatex.maxinopen",function(v) set("max_in_open",v) end) + directives.register("luatex.maxprintline",function(v) set("max_print_line",v) end) + directives.register("luatex.maxstrings",function(v) set("max_strings",v) end) + directives.register("luatex.paramsize",function(v) set("param_size",v) end) + directives.register("luatex.savesize",function(v) set("save_size",v) end) + directives.register("luatex.stacksize",function(v) set("stack_size",v) end) end -logs.report = report -logs.subreport = subreport -logs.status = status -logs.settarget = settarget -logs.pushtarget = pushtarget -logs.poptarget = poptarget -logs.setformats = setformats -logs.settranslations = settranslations -logs.direct = direct -logs.subdirect = subdirect -logs.writer = writer -logs.newline = newline - --- installer +end -- of closure --- todo: renew (un) locks when a new one is added and wildcard +do -- create closure to overcome 200 locals limit -local data, states = { }, nil +-- original size: 17885, stripped down to: 13242 +if not modules then modules={} end modules ['trac-log']={ + version=1.001, + comment="companion to trac-log.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local write_nl,write=texio and texio.write_nl or print,texio and texio.write or io.write +local format,gmatch,find=string.format,string.gmatch,string.find +local concat,insert,remove=table.concat,table.insert,table.remove +local topattern=string.topattern +local texcount=tex and tex.count +local next,type,select=next,type,select +local setmetatableindex=table.setmetatableindex +local formatters=string.formatters +logs=logs or {} +local logs=logs +local moreinfo=[[ +More information about ConTeXt and the tools that come with it can be found at: +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] +local function ignore() end +setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) +local report,subreport,status,settarget,setformats,settranslations +local direct,subdirect,writer,pushtarget,poptarget +if tex and (tex.jobname or tex.formatname) then + local valueiskey={ __index=function(t,k) t[k]=k return k end } + local target="term and log" + logs.flush=io.flush + local formats={} setmetatable(formats,valueiskey) + local translations={} setmetatable(translations,valueiskey) + writer=function(...) + write_nl(target,...) + end + newline=function() + write_nl(target,"\n") + end + local f_one=formatters["%-15s > %s\n"] + local f_two=formatters["%-15s >\n"] + report=function(a,b,c,...) + if c then + write_nl(target,f_one(translations[a],format(formats[b],c,...))) + elseif b then + write_nl(target,f_one(translations[a],formats[b])) + elseif a then + write_nl(target,f_two(translations[a])) + else + write_nl(target,"\n") + end + end + local f_one=formatters["%-15s > %s"] + local f_two=formatters["%-15s >"] + direct=function(a,b,c,...) + if c then + return f_one(translations[a],format(formats[b],c,...)) + elseif b then + return f_one(translations[a],formats[b]) + elseif a then + return f_two(translations[a]) + else + return "" + end + end + local f_one=formatters["%-15s > %s > %s\n"] + local f_two=formatters["%-15s > %s >\n"] + subreport=function(a,s,b,c,...) + if c then + write_nl(target,f_one(translations[a],translations[s],format(formats[b],c,...))) + elseif b then + write_nl(target,f_one(translations[a],translations[s],formats[b])) + elseif a then + write_nl(target,f_two(translations[a],translations[s])) + else + write_nl(target,"\n") + end + end + local f_one=formatters["%-15s > %s > %s"] + local f_two=formatters["%-15s > %s >"] + subdirect=function(a,s,b,c,...) + if c then + return f_one(translations[a],translations[s],format(formats[b],c,...)) + elseif b then + return f_one(translations[a],translations[s],formats[b]) + elseif a then + return f_two(translations[a],translations[s]) + else + return "" + end + end + local f_one=formatters["%-15s : %s\n"] + local f_two=formatters["%-15s :\n"] + status=function(a,b,c,...) + if c then + write_nl(target,f_one(translations[a],format(formats[b],c,...))) + elseif b then + write_nl(target,f_one(translations[a],formats[b])) + elseif a then + write_nl(target,f_two(translations[a])) + else + write_nl(target,"\n") + end + end + local targets={ + logfile="log", + log="log", + file="log", + console="term", + terminal="term", + both="term and log", + } + settarget=function(whereto) + target=targets[whereto or "both"] or targets.both + if target=="term" or target=="term and log" then + logs.flush=io.flush + else + logs.flush=ignore + end + end + local stack={} + pushtarget=function(newtarget) + insert(stack,target) + settarget(newtarget) + end + poptarget=function() + if #stack>0 then + settarget(remove(stack)) + end + end + setformats=function(f) + formats=f + end + settranslations=function(t) + translations=t + end +else + logs.flush=ignore + writer=write_nl + newline=function() + write_nl("\n") + end + local f_one=formatters["%-15s | %s"] + local f_two=formatters["%-15s |"] + report=function(a,b,c,...) + if c then + write_nl(f_one(a,format(b,c,...))) + elseif b then + write_nl(f_one(a,b)) + elseif a then + write_nl(f_two(a)) + else + write_nl("") + end + end + local f_one=formatters["%-15s | %s | %s"] + local f_two=formatters["%-15s | %s |"] + subreport=function(a,sub,b,c,...) + if c then + write_nl(f_one(a,sub,format(b,c,...))) + elseif b then + write_nl(f_one(a,sub,b)) + elseif a then + write_nl(f_two(a,sub)) + else + write_nl("") + end + end + local f_one=formatters["%-15s : %s\n"] + local f_two=formatters["%-15s :\n"] + status=function(a,b,c,...) + if c then + write_nl(f_one(a,format(b,c,...))) + elseif b then + write_nl(f_one(a,b)) + elseif a then + write_nl(f_two(a)) + else + write_nl("\n") + end + end + direct=ignore + subdirect=ignore + settarget=ignore + pushtarget=ignore + poptarget=ignore + setformats=ignore + settranslations=ignore +end +logs.report=report +logs.subreport=subreport +logs.status=status +logs.settarget=settarget +logs.pushtarget=pushtarget +logs.poptarget=poptarget +logs.setformats=setformats +logs.settranslations=settranslations +logs.direct=direct +logs.subdirect=subdirect +logs.writer=writer +logs.newline=newline +local data,states={},nil function logs.reporter(category,subcategory) - local logger = data[category] - if not logger then - local state = false - if states == true then - state = true - elseif type(states) == "table" then - for c, _ in next, states do - if find(category,c) then - state = true - break - end - end + local logger=data[category] + if not logger then + local state=false + if states==true then + state=true + elseif type(states)=="table" then + for c,_ in next,states do + if find(category,c) then + state=true + break end - logger = { - reporters = { }, - state = state, - } - data[category] = logger - end - local reporter = logger.reporters[subcategory or "default"] - if not reporter then - if subcategory then - reporter = function(...) - if not logger.state then - subreport(category,subcategory,...) - end - end - logger.reporters[subcategory] = reporter - else - local tag = category - reporter = function(...) - if not logger.state then - report(category,...) - end - end - logger.reporters.default = reporter + end + end + logger={ + reporters={}, + state=state, + } + data[category]=logger + end + local reporter=logger.reporters[subcategory or "default"] + if not reporter then + if subcategory then + reporter=function(...) + if not logger.state then + subreport(category,subcategory,...) end + end + logger.reporters[subcategory]=reporter + else + local tag=category + reporter=function(...) + if not logger.state then + report(category,...) + end + end + logger.reporters.default=reporter end - return reporter + end + return reporter end - -logs.new = logs.reporter -- for old times sake - --- context specicific: this ends up in the macro stream - -local ctxreport = logs.writer - +logs.new=logs.reporter +local ctxreport=logs.writer function logs.setmessenger(m) - ctxreport = m + ctxreport=m end - function logs.messenger(category,subcategory) - -- we need to avoid catcode mess (todo: fast context) - if subcategory then - return function(...) - ctxreport(subdirect(category,subcategory,...)) - end - else - return function(...) - ctxreport(direct(category,...)) - end + if subcategory then + return function(...) + ctxreport(subdirect(category,subcategory,...)) + end + else + return function(...) + ctxreport(direct(category,...)) end + end end - --- so far - local function setblocked(category,value) - if category == true then - -- lock all - category, value = "*", true - elseif category == false then - -- unlock all - category, value = "*", false - elseif value == nil then - -- lock selective - value = true - end - if category == "*" then - states = value - for k, v in next, data do - v.state = value - end - else - states = utilities.parsers.settings_to_hash(category) - for c, _ in next, states do - if data[c] then - v.state = value - else - c = topattern(c,true,true) - for k, v in next, data do - if find(k,c) then - v.state = value - end - end - end + if category==true then + category,value="*",true + elseif category==false then + category,value="*",false + elseif value==nil then + value=true + end + if category=="*" then + states=value + for k,v in next,data do + v.state=value + end + else + states=utilities.parsers.settings_to_hash(category) + for c,_ in next,states do + if data[c] then + v.state=value + else + c=topattern(c,true,true) + for k,v in next,data do + if find(k,c) then + v.state=value + end end + end end + end end - function logs.disable(category,value) - setblocked(category,value == nil and true or value) + setblocked(category,value==nil and true or value) end - function logs.enable(category) - setblocked(category,false) + setblocked(category,false) end - function logs.categories() - return table.sortedkeys(data) + return table.sortedkeys(data) end - function logs.show() - local n, c, s, max = 0, 0, 0, 0 - for category, v in table.sortedpairs(data) do - n = n + 1 - local state = v.state - local reporters = v.reporters - local nc = #category - if nc > c then - c = nc - end - for subcategory, _ in next, reporters do - local ns = #subcategory - if ns > c then - s = ns - end - local m = nc + ns - if m > max then - max = m - end - end - local subcategories = concat(table.sortedkeys(reporters),", ") - if state == true then - state = "disabled" - elseif state == false then - state = "enabled" - else - state = "unknown" - end - -- no new here - report("logging","category: '%s', subcategories: '%s', state: '%s'",category,subcategories,state) + local n,c,s,max=0,0,0,0 + for category,v in table.sortedpairs(data) do + n=n+1 + local state=v.state + local reporters=v.reporters + local nc=#category + if nc>c then + c=nc + end + for subcategory,_ in next,reporters do + local ns=#subcategory + if ns>c then + s=ns + end + local m=nc+ns + if m>max then + max=m + end end - report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) + local subcategories=concat(table.sortedkeys(reporters),", ") + if state==true then + state="disabled" + elseif state==false then + state="enabled" + else + state="unknown" + end + report("logging","category: '%s', subcategories: '%s', state: '%s'",category,subcategories,state) + end + report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) end - -directives.register("logs.blocked", function(v) - setblocked(v,true) +directives.register("logs.blocked",function(v) + setblocked(v,true) end) - -directives.register("logs.target", function(v) - settarget(v) +directives.register("logs.target",function(v) + settarget(v) end) - --- tex specific loggers (might move elsewhere) - -local report_pages = logs.reporter("pages") -- not needed but saves checking when we grep for it - -local real, user, sub - +local report_pages=logs.reporter("pages") +local real,user,sub function logs.start_page_number() - real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno --- real, user, sub = 0, 0, 0 -end - -local timing = false -local starttime = nil -local lasttime = nil - -trackers.register("pages.timing", function(v) -- only for myself (diagnostics) - starttime = os.clock() - timing = true + real,user,sub=texcount.realpageno,texcount.userpageno,texcount.subpageno +end +local timing=false +local starttime=nil +local lasttime=nil +trackers.register("pages.timing",function(v) + starttime=os.clock() + timing=true end) - -function logs.stop_page_number() -- the first page can includes the initialization so we omit this in average - if timing then - local elapsed, average - local stoptime = os.clock() - if not lasttime or real < 2 then - elapsed = stoptime - average = stoptime - starttime = stoptime - else - elapsed = stoptime - lasttime - average = (stoptime - starttime) / (real - 1) - end - lasttime = stoptime - if real > 0 then - if user > 0 then - if sub > 0 then - report_pages("flushing realpage %s, userpage %s, subpage %s, time %0.04f / %0.04f",real,user,sub,elapsed,average) - else - report_pages("flushing realpage %s, userpage %s, time %0.04f / %0.04f",real,user,elapsed,average) - end - else - report_pages("flushing realpage %s, time %0.04f / %0.04f",real,elapsed,average) - end - else - report_pages("flushing page, time %0.04f / %0.04f",elapsed,average) - end +function logs.stop_page_number() + if timing then + local elapsed,average + local stoptime=os.clock() + if not lasttime or real<2 then + elapsed=stoptime + average=stoptime + starttime=stoptime + else + elapsed=stoptime-lasttime + average=(stoptime-starttime)/(real-1) + end + lasttime=stoptime + if real>0 then + if user>0 then + if sub>0 then + report_pages("flushing realpage %s, userpage %s, subpage %s, time %0.04f / %0.04f",real,user,sub,elapsed,average) + else + report_pages("flushing realpage %s, userpage %s, time %0.04f / %0.04f",real,user,elapsed,average) + end + else + report_pages("flushing realpage %s, time %0.04f / %0.04f",real,elapsed,average) + end else - if real > 0 then - if user > 0 then - if sub > 0 then - report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) - else - report_pages("flushing realpage %s, userpage %s",real,user) - end - else - report_pages("flushing realpage %s",real) - end + report_pages("flushing page, time %0.04f / %0.04f",elapsed,average) + end + else + if real>0 then + if user>0 then + if sub>0 then + report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - report_pages("flushing page") + report_pages("flushing realpage %s, userpage %s",real,user) end + else + report_pages("flushing realpage %s",real) + end + else + report_pages("flushing page") end - logs.flush() + end + logs.flush() end - -logs.report_job_stat = statistics and statistics.showjobstat - -local report_files = logs.reporter("files") - -local nesting = 0 -local verbose = false -local hasscheme = url.hasscheme - --- we don't have show_open and show_close callbacks yet - +logs.report_job_stat=statistics and statistics.showjobstat +local report_files=logs.reporter("files") +local nesting=0 +local verbose=false +local hasscheme=url.hasscheme function logs.show_open(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- nesting = nesting + 1 - -- report_files("level %s, opening %s",nesting,name) - -- else - -- write(format("(%s",name)) -- tex adds a space - -- end - -- end end - function logs.show_close(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, closing %s",nesting,name) - -- nesting = nesting - 1 - -- else - -- write(")") -- tex adds a space - -- end - -- end end - function logs.show_load(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, loading %s",nesting+1,name) - -- else - -- write(format("(%s)",name)) - -- end - -- end end - --- there may be scripts out there using this: - -local simple = logs.reporter("comment") - -logs.simple = simple -logs.simpleline = simple - --- obsolete - -function logs.setprogram () end -- obsolete -function logs.extendbanner() end -- obsolete -function logs.reportlines () end -- obsolete -function logs.reportbanner() end -- obsolete -function logs.reportline () end -- obsolete -function logs.simplelines () end -- obsolete -function logs.help () end -- obsolete - --- applications - +local simple=logs.reporter("comment") +logs.simple=simple +logs.simpleline=simple +function logs.setprogram () end +function logs.extendbanner() end +function logs.reportlines () end +function logs.reportbanner() end +function logs.reportline () end +function logs.simplelines () end +function logs.help () end local function reportlines(t,str) - if str then - for line in gmatch(str,"(.-)[\n\r]") do - t.report(line) - end + if str then + for line in gmatch(str,"(.-)[\n\r]") do + t.report(line) end + end end - local function reportbanner(t) - local banner = t.banner - if banner then - t.report(banner) - t.report() - end + local banner=t.banner + if banner then + t.report(banner) + t.report() + end end - local function reportversion(t) - local banner = t.banner - if banner then - t.report(banner) - end + local banner=t.banner + if banner then + t.report(banner) + end end - local function reporthelp(t,...) - local helpinfo = t.helpinfo - if type(helpinfo) == "string" then - reportlines(t,helpinfo) - elseif type(helpinfo) == "table" then - local n = select("#",...) - for i=1,n do - reportlines(t,t.helpinfo[select(i,...)]) - if i < n then - t.report() - end - end + local helpinfo=t.helpinfo + if type(helpinfo)=="string" then + reportlines(t,helpinfo) + elseif type(helpinfo)=="table" then + local n=select("#",...) + for i=1,n do + reportlines(t,t.helpinfo[select(i,...)]) + if i %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) - for i=1,10 do - local f = io.open(whereto,"a") -- we can consider keepint the file open - if f then - f:write(message) - f:close() - break - else - sleep(0.1) - end + local message=format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) + for i=1,10 do + local f=io.open(whereto,"a") + if f then + f:write(message) + f:close() + break + else + sleep(0.1) end + end end - -local report_system = logs.reporter("system","logs") - +local report_system=logs.reporter("system","logs") function logs.obsolete(old,new) - local o = loadstring("return " .. new)() - if type(o) == "function" then - return function(...) - report_system("function %s is obsolete, use %s",old,new) - loadstring(old .. "=" .. new .. " return ".. old)()(...) - end - elseif type(o) == "table" then - local t, m = { }, { } - m.__index = function(t,k) - report_system("table %s is obsolete, use %s",old,new) - m.__index, m.__newindex = o, o - return o[k] - end - m.__newindex = function(t,k,v) - report_system("table %s is obsolete, use %s",old,new) - m.__index, m.__newindex = o, o - o[k] = v - end - if libraries then - libraries.obsolete[old] = t -- true - end - setmetatable(t,m) - return t - end + local o=loadstring("return "..new)() + if type(o)=="function" then + return function(...) + report_system("function %s is obsolete, use %s",old,new) + loadstring(old.."="..new.." return "..old)()(...) + end + elseif type(o)=="table" then + local t,m={},{} + m.__index=function(t,k) + report_system("table %s is obsolete, use %s",old,new) + m.__index,m.__newindex=o,o + return o[k] + end + m.__newindex=function(t,k,v) + report_system("table %s is obsolete, use %s",old,new) + m.__index,m.__newindex=o,o + o[k]=v + end + if libraries then + libraries.obsolete[old]=t + end + setmetatable(t,m) + return t + end end - if utilities then - utilities.report = report_system + utilities.report=report_system end - if tex and tex.error then - function logs.texerrormessage(...) -- for the moment we put this function here - tex.error(format(...), { }) - end + function logs.texerrormessage(...) + tex.error(format(...),{}) + end else - function logs.texerrormessage(...) - print(format(...)) - end + function logs.texerrormessage(...) + print(format(...)) + end end +io.stdout:setvbuf('no') +io.stderr:setvbuf('no') --- do we still need io.flush then? - -io.stdout:setvbuf('no') -io.stderr:setvbuf('no') - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-pro'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local getmetatable, setmetatable, rawset, type = getmetatable, setmetatable, rawset, type - --- The protection implemented here is probably not that tight but good enough to catch --- problems due to naive usage. --- --- There's a more extensive version (trac-xxx.lua) that supports nesting. --- --- This will change when we have _ENV in lua 5.2+ - -local trace_namespaces = false trackers.register("system.namespaces", function(v) trace_namespaces = v end) -local report_system = logs.reporter("system","protection") +end -- of closure -namespaces = namespaces or { } -local namespaces = namespaces +do -- create closure to overcome 200 locals limit -local registered = { } +-- original size: 5789, stripped down to: 3469 +if not modules then modules={} end modules ['trac-pro']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local getmetatable,setmetatable,rawset,type=getmetatable,setmetatable,rawset,type +local trace_namespaces=false trackers.register("system.namespaces",function(v) trace_namespaces=v end) +local report_system=logs.reporter("system","protection") +namespaces=namespaces or {} +local namespaces=namespaces +local registered={} local function report_index(k,name) - if trace_namespaces then - report_system("reference to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) - else - report_system("reference to '%s' in protected namespace '%s'",k,name) - end + if trace_namespaces then + report_system("reference to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) + else + report_system("reference to '%s' in protected namespace '%s'",k,name) + end end - local function report_newindex(k,name) - if trace_namespaces then - report_system("assignment to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) - else - report_system("assignment to '%s' in protected namespace '%s'",k,name) - end + if trace_namespaces then + report_system("assignment to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) + else + report_system("assignment to '%s' in protected namespace '%s'",k,name) + end end - local function register(name) - local data = name == "global" and _G or _G[name] - if not data then - return -- error - end - registered[name] = data - local m = getmetatable(data) - if not m then - m = { } - setmetatable(data,m) - end - local index, newindex = { }, { } - m.__saved__index = m.__index - m.__no__index = function(t,k) - if not index[k] then - index[k] = true - report_index(k,name) - end - return nil - end - m.__saved__newindex = m.__newindex - m.__no__newindex = function(t,k,v) - if not newindex[k] then - newindex[k] = true - report_newindex(k,name) - end - rawset(t,k,v) + local data=name=="global" and _G or _G[name] + if not data then + return + end + registered[name]=data + local m=getmetatable(data) + if not m then + m={} + setmetatable(data,m) + end + local index,newindex={},{} + m.__saved__index=m.__index + m.__no__index=function(t,k) + if not index[k] then + index[k]=true + report_index(k,name) end - m.__protection__depth = 0 -end - -local function private(name) -- maybe save name - local data = registered[name] + return nil + end + m.__saved__newindex=m.__newindex + m.__no__newindex=function(t,k,v) + if not newindex[k] then + newindex[k]=true + report_newindex(k,name) + end + rawset(t,k,v) + end + m.__protection__depth=0 +end +local function private(name) + local data=registered[name] + if not data then + data=_G[name] if not data then - data = _G[name] - if not data then - data = { } - _G[name] = data - end - register(name) + data={} + _G[name]=data end - return data + register(name) + end + return data end - local function protect(name) - local data = registered[name] - if not data then - return - end - local m = getmetatable(data) - local pd = m.__protection__depth - if pd > 0 then - m.__protection__depth = pd + 1 - else - m.__save_d_index, m.__saved__newindex = m.__index, m.__newindex - m.__index, m.__newindex = m.__no__index, m.__no__newindex - m.__protection__depth = 1 - end + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>0 then + m.__protection__depth=pd+1 + else + m.__save_d_index,m.__saved__newindex=m.__index,m.__newindex + m.__index,m.__newindex=m.__no__index,m.__no__newindex + m.__protection__depth=1 + end end - local function unprotect(name) - local data = registered[name] - if not data then - return - end - local m = getmetatable(data) - local pd = m.__protection__depth - if pd > 1 then - m.__protection__depth = pd - 1 - else - m.__index, m.__newindex = m.__saved__index, m.__saved__newindex - m.__protection__depth = 0 - end + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>1 then + m.__protection__depth=pd-1 + else + m.__index,m.__newindex=m.__saved__index,m.__saved__newindex + m.__protection__depth=0 + end end - local function protectall() - for name, _ in next, registered do - if name ~= "global" then - protect(name) - end + for name,_ in next,registered do + if name~="global" then + protect(name) end + end end - local function unprotectall() - for name, _ in next, registered do - if name ~= "global" then - unprotect(name) - end - end -end - -namespaces.register = register -- register when defined -namespaces.private = private -- allocate and register if needed -namespaces.protect = protect -namespaces.unprotect = unprotect -namespaces.protectall = protectall -namespaces.unprotectall = unprotectall - -namespaces.private("namespaces") registered = { } register("global") -- unreachable - -directives.register("system.protect", function(v) - if v then - protectall() - else - unprotectall() - end + for name,_ in next,registered do + if name~="global" then + unprotect(name) + end + end +end +namespaces.register=register +namespaces.private=private +namespaces.protect=protect +namespaces.unprotect=unprotect +namespaces.protectall=protectall +namespaces.unprotectall=unprotectall +namespaces.private("namespaces") registered={} register("global") +directives.register("system.protect",function(v) + if v then + protectall() + else + unprotectall() + end end) - -directives.register("system.checkglobals", function(v) - if v then - report_system("enabling global namespace guard") - protect("global") - else - report_system("disabling global namespace guard") - unprotect("global") - end +directives.register("system.checkglobals",function(v) + if v then + report_system("enabling global namespace guard") + protect("global") + else + report_system("disabling global namespace guard") + unprotect("global") + end end) --- dummy section (will go to luat-dum.lua) - - - - - - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-env'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- A former version provided functionality for non embeded core --- scripts i.e. runtime library loading. Given the amount of --- Lua code we use now, this no longer makes sense. Much of this --- evolved before bytecode arrays were available and so a lot of --- code has disappeared already. - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_lua = logs.reporter("resolvers","lua") - -local allocate, mark = utilities.storage.allocate, utilities.storage.mark - -local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find -local unquoted, quoted = string.unquoted, string.quoted -local concat, insert, remove = table.concat, table.insert, table.remove -local loadedluacode = utilities.lua.loadedluacode -local luasuffixes = utilities.lua.suffixes - -environment = environment or { } -local environment = environment - --- precautions - -os.setlocale(nil,nil) -- useless feature and even dangerous in luatex +-- original size: 12260, stripped down to: 8100 +if not modules then modules={} end modules ['luat-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_lua=logs.reporter("resolvers","lua") +local allocate,mark=utilities.storage.allocate,utilities.storage.mark +local format,sub,match,gsub,find=string.format,string.sub,string.match,string.gsub,string.find +local unquoted,quoted=string.unquoted,string.quoted +local concat,insert,remove=table.concat,table.insert,table.remove +local loadedluacode=utilities.lua.loadedluacode +local luasuffixes=utilities.lua.suffixes +environment=environment or {} +local environment=environment +os.setlocale(nil,nil) function os.setlocale() - -- no way you can mess with it end - --- dirty tricks (we will replace the texlua call by luatex --luaonly) - -local validengines = allocate { - ["luatex"] = true, - ["luajittex"] = true, - -- ["luatex.exe"] = true, - -- ["luajittex.exe"] = true, +local validengines=allocate { + ["luatex"]=true, + ["luajittex"]=true, } - -local basicengines = allocate { - ["luatex"] = "luatex", - ["texlua"] = "luatex", - ["texluac"] = "luatex", - ["luajittex"] = "luajittex", - ["texluajit"] = "luajittex", - -- ["texlua.exe"] = "luatex", - -- ["texluajit.exe"] = "luajittex", +local basicengines=allocate { + ["luatex"]="luatex", + ["texlua"]="luatex", + ["texluac"]="luatex", + ["luajittex"]="luajittex", + ["texluajit"]="luajittex", } - -environment.validengines = validengines -environment.basicengines = basicengines - -if arg and validengines[file.removesuffix(arg[0])] and arg[1] == "--luaonly" then - arg[-1] = arg[0] - arg[ 0] = arg[2] - for k=3,#arg do - arg[k-2] = arg[k] - end - remove(arg) -- last - remove(arg) -- pre-last +environment.validengines=validengines +environment.basicengines=basicengines +if arg and validengines[file.removesuffix(arg[0])] and arg[1]=="--luaonly" then + arg[-1]=arg[0] + arg[ 0]=arg[2] + for k=3,#arg do + arg[k-2]=arg[k] + end + remove(arg) + remove(arg) end - --- This is an ugly hack but it permits symlinking a script (say 'context') to 'mtxrun' as in: --- --- ln -s /opt/minimals/tex/texmf-linux-64/bin/mtxrun context --- --- The special mapping hack is needed because 'luatools' boils down to 'mtxrun --script base' --- but it's unlikely that there will be more of this - do - - local originalzero = file.basename(arg[0]) - local specialmapping = { luatools == "base" } - - if originalzero ~= "mtxrun" and originalzero ~= "mtxrun.lua" then - arg[0] = specialmapping[originalzero] or originalzero - insert(arg,0,"--script") - insert(arg,0,"mtxrun") - end - -end - --- environment - -environment.arguments = allocate() -environment.files = allocate() -environment.sortedflags = nil - -local mt = { - __index = function(_,k) - if k == "version" then - local version = tex.toks and tex.toks.contextversiontoks - if version and version ~= "" then - rawset(environment,"version",version) - return version - else - return "unknown" - end - elseif k == "jobname" or k == "formatname" then - local name = tex and tex[k] - if name or name== "" then - rawset(environment,k,name) - return name - else - return "unknown" - end - elseif k == "outputfilename" then - local name = environment.jobname - rawset(environment,k,name) - return name - end + local originalzero=file.basename(arg[0]) + local specialmapping={ luatools=="base" } + if originalzero~="mtxrun" and originalzero~="mtxrun.lua" then + arg[0]=specialmapping[originalzero] or originalzero + insert(arg,0,"--script") + insert(arg,0,"mtxrun") + end +end +environment.arguments=allocate() +environment.files=allocate() +environment.sortedflags=nil +local mt={ + __index=function(_,k) + if k=="version" then + local version=tex.toks and tex.toks.contextversiontoks + if version and version~="" then + rawset(environment,"version",version) + return version + else + return "unknown" + end + elseif k=="jobname" or k=="formatname" then + local name=tex and tex[k] + if name or name=="" then + rawset(environment,k,name) + return name + else + return "unknown" + end + elseif k=="outputfilename" then + local name=environment.jobname + rawset(environment,k,name) + return name end + end } - setmetatable(environment,mt) - --- context specific arguments (in order not to confuse the engine) - function environment.initializearguments(arg) - local arguments, files = { }, { } - environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index=1,#arg do - local argument = arg[index] - if index > 0 then - local flag, value = match(argument,"^%-+(.-)=(.-)$") - if flag then - flag = gsub(flag,"^c:","") - arguments[flag] = unquoted(value or "") - else - flag = match(argument,"^%-+(.+)") - if flag then - flag = gsub(flag,"^c:","") - arguments[flag] = true - else - files[#files+1] = argument - end - end + local arguments,files={},{} + environment.arguments,environment.files,environment.sortedflags=arguments,files,nil + for index=1,#arg do + local argument=arg[index] + if index>0 then + local flag,value=match(argument,"^%-+(.-)=(.-)$") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=unquoted(value or "") + else + flag=match(argument,"^%-+(.+)") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=true + else + files[#files+1]=argument end + end end - environment.ownname = file.reslash(environment.ownname or arg[0] or 'unknown.lua') + end + environment.ownname=file.reslash(environment.ownname or arg[0] or 'unknown.lua') end - function environment.setargument(name,value) - environment.arguments[name] = value + environment.arguments[name]=value end - --- todo: defaults, better checks e.g on type (boolean versus string) --- --- tricky: too many hits when we support partials unless we add --- a registration of arguments so from now on we have 'partial' - function environment.getargument(name,partial) - local arguments, sortedflags = environment.arguments, environment.sortedflags - if arguments[name] then - return arguments[name] - elseif partial then - if not sortedflags then - sortedflags = allocate(table.sortedkeys(arguments)) - for k=1,#sortedflags do - sortedflags[k] = "^" .. sortedflags[k] - end - environment.sortedflags = sortedflags - end - -- example of potential clash: ^mode ^modefile - for k=1,#sortedflags do - local v = sortedflags[k] - if find(name,v) then - return arguments[sub(v,2,#v)] - end - end + local arguments,sortedflags=environment.arguments,environment.sortedflags + if arguments[name] then + return arguments[name] + elseif partial then + if not sortedflags then + sortedflags=allocate(table.sortedkeys(arguments)) + for k=1,#sortedflags do + sortedflags[k]="^"..sortedflags[k] + end + environment.sortedflags=sortedflags end - return nil + for k=1,#sortedflags do + local v=sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] + end + end + end + return nil end - -environment.argument = environment.getargument - -function environment.splitarguments(separator) -- rather special, cut-off before separator - local done, before, after = false, { }, { } - local originalarguments = environment.originalarguments - for k=1,#originalarguments do - local v = originalarguments[k] - if not done and v == separator then - done = true - elseif done then - after[#after+1] = v - else - before[#before+1] = v - end +environment.argument=environment.getargument +function environment.splitarguments(separator) + local done,before,after=false,{},{} + local originalarguments=environment.originalarguments + for k=1,#originalarguments do + local v=originalarguments[k] + if not done and v==separator then + done=true + elseif done then + after[#after+1]=v + else + before[#before+1]=v end - return before, after + end + return before,after end - function environment.reconstructcommandline(arg,noquote) - arg = arg or environment.originalarguments - if noquote and #arg == 1 then - -- we could just do: return unquoted(resolvers.resolve(arg[i])) - local a = arg[1] - a = resolvers.resolve(a) - a = unquoted(a) - return a - elseif #arg > 0 then - local result = { } - for i=1,#arg do - -- we could just do: result[#result+1] = format("%q",unquoted(resolvers.resolve(arg[i]))) - local a = arg[i] - a = resolvers.resolve(a) - a = unquoted(a) - a = gsub(a,'"','\\"') -- tricky - if find(a," ") then - result[#result+1] = quoted(a) - else - result[#result+1] = a - end - end - return concat(result," ") - else - return "" + arg=arg or environment.originalarguments + if noquote and #arg==1 then + local a=arg[1] + a=resolvers.resolve(a) + a=unquoted(a) + return a + elseif #arg>0 then + local result={} + for i=1,#arg do + local a=arg[i] + a=resolvers.resolve(a) + a=unquoted(a) + a=gsub(a,'"','\\"') + if find(a," ") then + result[#result+1]=quoted(a) + else + result[#result+1]=a + end end + return concat(result," ") + else + return "" + end end - --- -- to be tested: --- --- function environment.reconstructcommandline(arg,noquote) --- arg = arg or environment.originalarguments --- if noquote and #arg == 1 then --- return unquoted(resolvers.resolve(arg[1])) --- elseif #arg > 0 then --- local result = { } --- for i=1,#arg do --- result[#result+1] = format("%q",unquoted(resolvers.resolve(arg[i]))) -- always quote --- end --- return concat(result," ") --- else --- return "" --- end --- end - if arg then - - -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) - local newarg, instring = { }, false - - for index=1,#arg do - local argument = arg[index] - if find(argument,"^\"") then - newarg[#newarg+1] = gsub(argument,"^\"","") - if not find(argument,"\"$") then - instring = true - end - elseif find(argument,"\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") - instring = false - elseif instring then - newarg[#newarg] = newarg[#newarg] .. " " .. argument - else - newarg[#newarg+1] = argument - end - end - for i=1,-5,-1 do - newarg[i] = arg[i] - end - - environment.initializearguments(newarg) - - environment.originalarguments = mark(newarg) - environment.rawarguments = mark(arg) - - arg = { } -- prevent duplicate handling - + local newarg,instring={},false + for index=1,#arg do + local argument=arg[index] + if find(argument,"^\"") then + newarg[#newarg+1]=gsub(argument,"^\"","") + if not find(argument,"\"$") then + instring=true + end + elseif find(argument,"\"$") then + newarg[#newarg]=newarg[#newarg].." "..gsub(argument,"\"$","") + instring=false + elseif instring then + newarg[#newarg]=newarg[#newarg].." "..argument + else + newarg[#newarg+1]=argument + end + end + for i=1,-5,-1 do + newarg[i]=arg[i] + end + environment.initializearguments(newarg) + environment.originalarguments=mark(newarg) + environment.rawarguments=mark(arg) + arg={} end - --- weird place ... depends on a not yet loaded module - function environment.texfile(filename) - return resolvers.findfile(filename,'tex') + return resolvers.findfile(filename,'tex') +end +function environment.luafile(filename) + local resolved=resolvers.findfile(filename,'tex') or "" + if resolved~="" then + return resolved + end + resolved=resolvers.findfile(filename,'texmfscripts') or "" + if resolved~="" then + return resolved + end + return resolvers.findfile(filename,'luatexlibs') or "" +end +local stripindeed=false directives.register("system.compile.strip",function(v) stripindeed=v end) +local function strippable(filename) + if stripindeed then + local modu=modules[file.nameonly(filename)] + return modu and modu.dataonly + else + return false + end end - -function environment.luafile(filename) -- needs checking - local resolved = resolvers.findfile(filename,'tex') or "" - if resolved ~= "" then - return resolved - end - resolved = resolvers.findfile(filename,'texmfscripts') or "" - if resolved ~= "" then - return resolved +function environment.luafilechunk(filename,silent) + filename=file.replacesuffix(filename,"lua") + local fullname=environment.luafile(filename) + if fullname and fullname~="" then + local data=loadedluacode(fullname,strippable,filename) + if trace_locating then + report_lua("loading file %s%s",fullname,not data and " failed" or "") + elseif not silent then + texio.write("<",data and "+ " or "- ",fullname,">") end - return resolvers.findfile(filename,'luatexlibs') or "" -end - --- local function checkstrip(filename) --- local modu = modules[file.nameonly(filename)] --- return modu and modu.dataonly --- end - -local stripindeed = false directives.register("system.compile.strip", function(v) stripindeed = v end) - -local function strippable(filename) - if stripindeed then - local modu = modules[file.nameonly(filename)] - return modu and modu.dataonly - else - return false + return data + else + if trace_locating then + report_lua("unknown file %s",filename) end -end - -function environment.luafilechunk(filename,silent) -- used for loading lua bytecode in the format - filename = file.replacesuffix(filename, "lua") - local fullname = environment.luafile(filename) - if fullname and fullname ~= "" then - local data = loadedluacode(fullname,strippable,filename) - if trace_locating then - report_lua("loading file %s%s", fullname, not data and " failed" or "") - elseif not silent then - texio.write("<",data and "+ " or "- ",fullname,">") - end - return data - else + return nil + end +end +function environment.loadluafile(filename,version) + local lucname,luaname,chunk + local basename=file.removesuffix(filename) + if basename==filename then + luaname=file.addsuffix(basename,luasuffixes.lua) + lucname=file.addsuffix(basename,luasuffixes.luc) + else + luaname=basename + lucname=nil + end + local fullname=(lucname and environment.luafile(lucname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %s",fullname) + end + chunk=loadfile(fullname) + end + if chunk then + assert(chunk)() + if version then + local v=version + if modules and modules[filename] then + v=modules[filename].version + elseif versions and versions[filename] then + v=versions[filename] + end + if v==version then + return true + else if trace_locating then - report_lua("unknown file %s", filename) + report_lua("version mismatch for %s: lua=%s, luc=%s",filename,v,version) end - return nil - end -end - --- the next ones can use the previous ones / combine - -function environment.loadluafile(filename, version) - local lucname, luaname, chunk - local basename = file.removesuffix(filename) - if basename == filename then - luaname = fiule.addsuffix(basename,luasuffixes.lua) - lucname = fiule.addsuffix(basename,luasuffixes.luc) + environment.loadluafile(filename) + end else - luaname = basename -- forced suffix - lucname = nil + return true end - -- when not overloaded by explicit suffix we look for a luc file first - local fullname = (lucname and environment.luafile(lucname)) or "" - if fullname ~= "" then - if trace_locating then - report_lua("loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - end - if chunk then - assert(chunk)() - if version then - -- we check of the version number of this chunk matches - local v = version -- can be nil - if modules and modules[filename] then - v = modules[filename].version -- new method - elseif versions and versions[filename] then - v = versions[filename] -- old method - end - if v == version then - return true - else - if trace_locating then - report_lua("version mismatch for %s: lua=%s, luc=%s", filename, v, version) - end - environment.loadluafile(filename) - end - else - return true - end + end + fullname=(luaname and environment.luafile(luaname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %s",fullname) end - fullname = (luaname and environment.luafile(luaname)) or "" - if fullname ~= "" then - if trace_locating then - report_lua("loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - if not chunk then - if trace_locating then - report_lua("unknown file %s", filename) - end - else - assert(chunk)() - return true - end + chunk=loadfile(fullname) + if not chunk then + if trace_locating then + report_lua("unknown file %s",filename) + end + else + assert(chunk)() + return true end - return false + end + return false end @@ -9577,1226 +7159,978 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-tab'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc --- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the --- trouble - --- todo: when serializing optionally remap named entities to hex (if known in char-ent.lua) --- maybe when letter -> utf, else name .. then we need an option to the serializer .. a bit --- of work so we delay this till we cleanup - -local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) - -local report_xml = logs and logs.reporter("xml","core") or function(...) print(format(...)) end - - - -xml = xml or { } -local xml = xml - - -local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber -local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub -local utfchar = utf.char -local lpegmatch = lpeg.match -local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs - - - -xml.xmlns = xml.xmlns or { } - -local check = P(false) -local parse = check - +-- original size: 42438, stripped down to: 26556 - -function xml.registerns(namespace, pattern) -- pattern can be an lpeg - check = check + C(P(lower(pattern))) / namespace - parse = P { P(check) + 1 * V(1) } +if not modules then modules={} end modules ['lxml-tab']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_entities=false trackers.register("xml.entities",function(v) trace_entities=v end) +local report_xml=logs and logs.reporter("xml","core") or function(...) print(format(...)) end +xml=xml or {} +local xml=xml +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,setmetatable,getmetatable,tonumber=type,next,setmetatable,getmetatable,tonumber +local format,lower,find,match,gsub=string.format,string.lower,string.find,string.match,string.gsub +local utfchar=utf.char +local lpegmatch=lpeg.match +local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs +xml.xmlns=xml.xmlns or {} +local check=P(false) +local parse=check +function xml.registerns(namespace,pattern) + check=check+C(P(lower(pattern)))/namespace + parse=P { P(check)+1*V(1) } end - - - function xml.checkns(namespace,url) - local ns = lpegmatch(parse,lower(url)) - if ns and namespace ~= ns then - xml.xmlns[namespace] = ns - end + local ns=lpegmatch(parse,lower(url)) + if ns and namespace~=ns then + xml.xmlns[namespace]=ns + end end - - - function xml.resolvens(url) - return lpegmatch(parse,lower(url)) or "" -end - - - - - --- not just one big nested table capture (lpeg overflow) - -local nsremap, resolvens = xml.xmlns, xml.resolvens - -local stack = { } -local top = { } -local dt = { } -local at = { } -local xmlns = { } -local errorstr = nil -local entities = { } -local strip = false -local cleanup = false -local utfize = false -local resolve_predefined = false -local unify_predefined = false - -local dcache = { } -local hcache = { } -local acache = { } - -local mt = { } - -local function initialize_mt(root) - mt = { __index = root } -- will be redefined later -end - -function xml.setproperty(root,k,v) - getmetatable(root).__index[k] = v -end - -function xml.checkerror(top,toclose) - return "" -- can be set -end - -local function add_attribute(namespace,tag,value) - if cleanup and #value > 0 then - value = cleanup(value) -- new - end - if tag == "xmlns" then - xmlns[#xmlns+1] = resolvens(value) - at[tag] = value - elseif namespace == "" then - at[tag] = value - elseif namespace == "xmlns" then - xml.checkns(tag,value) - at["xmlns:" .. tag] = value - else - -- for the moment this way: - at[namespace .. ":" .. tag] = value - end + return lpegmatch(parse,lower(url)) or "" +end +local nsremap,resolvens=xml.xmlns,xml.resolvens +local stack={} +local top={} +local dt={} +local at={} +local xmlns={} +local errorstr=nil +local entities={} +local strip=false +local cleanup=false +local utfize=false +local resolve_predefined=false +local unify_predefined=false +local dcache={} +local hcache={} +local acache={} +local mt={} +local function initialize_mt(root) + mt={ __index=root } end - -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } +function xml.setproperty(root,k,v) + getmetatable(root).__index[k]=v end - -local function add_begin(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } - setmetatable(top, mt) - dt = top.dt - stack[#stack+1] = top - at = { } +function xml.checkerror(top,toclose) + return "" end - -local function add_end(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local toclose = remove(stack) - top = stack[#stack] - if #stack < 1 then - errorstr = format("nothing to close with %s %s", tag, xml.checkerror(top,toclose) or "") - elseif toclose.tg ~= tag then -- no namespace check - errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.checkerror(top,toclose) or "") - end - dt = top.dt - dt[#dt+1] = toclose - -- dt[0] = top -- nasty circular reference when serializing table - if toclose.at.xmlns then - remove(xmlns) - end +local function add_attribute(namespace,tag,value) + if cleanup and #value>0 then + value=cleanup(value) + end + if tag=="xmlns" then + xmlns[#xmlns+1]=resolvens(value) + at[tag]=value + elseif namespace=="" then + at[tag]=value + elseif namespace=="xmlns" then + xml.checkns(tag,value) + at["xmlns:"..tag]=value + else + at[namespace..":"..tag]=value + end +end +local function add_empty(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=(namespace=="" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top=stack[#stack] + dt=top.dt + local t={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=top } + dt[#dt+1]=t + setmetatable(t,mt) + if at.xmlns then + remove(xmlns) + end + at={} +end +local function add_begin(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=(namespace=="" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=stack[#stack] } + setmetatable(top,mt) + dt=top.dt + stack[#stack+1]=top + at={} +end +local function add_end(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local toclose=remove(stack) + top=stack[#stack] + if #stack<1 then + errorstr=format("nothing to close with %s %s",tag,xml.checkerror(top,toclose) or "") + elseif toclose.tg~=tag then + errorstr=format("unable to close %s with %s %s",toclose.tg,tag,xml.checkerror(top,toclose) or "") + end + dt=top.dt + dt[#dt+1]=toclose + if toclose.at.xmlns then + remove(xmlns) + end end - local function add_text(text) - if cleanup and #text > 0 then - dt[#dt+1] = cleanup(text) - else - dt[#dt+1] = text - end + if cleanup and #text>0 then + dt[#dt+1]=cleanup(text) + else + dt[#dt+1]=text + end +end +local function add_special(what,spacing,text) + if #spacing>0 then + dt[#dt+1]=spacing + end + if strip and (what=="@cm@" or what=="@dt@") then + else + dt[#dt+1]={ special=true,ns="",tg=what,dt={ text } } + end end - -local function add_special(what, spacing, text) - if #spacing > 0 then - dt[#dt+1] = spacing - end - if strip and (what == "@cm@" or what == "@dt@") then - -- forget it - else - dt[#dt+1] = { special=true, ns="", tg=what, dt={ text } } - end -end - local function set_message(txt) - errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") + errorstr="garbage at the end of the file: "..gsub(txt,"([ \n\r\t]*)","") end - -local reported_attribute_errors = { } - +local reported_attribute_errors={} local function attribute_value_error(str) - if not reported_attribute_errors[str] then - report_xml("invalid attribute value: %q",str) - reported_attribute_errors[str] = true - at._error_ = str - end - return str + if not reported_attribute_errors[str] then + report_xml("invalid attribute value: %q",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str end - local function attribute_specification_error(str) - if not reported_attribute_errors[str] then - report_xml("invalid attribute specification: %q",str) - reported_attribute_errors[str] = true - at._error_ = str - end - return str -end - -xml.placeholders = { - unknown_dec_entity = function(str) return (str == "" and "&error;") or format("&%s;",str) end, - unknown_hex_entity = function(str) return format("&#x%s;",str) end, - unknown_any_entity = function(str) return format("&#x%s;",str) end, + if not reported_attribute_errors[str] then + report_xml("invalid attribute specification: %q",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str +end +xml.placeholders={ + unknown_dec_entity=function(str) return (str=="" and "&error;") or format("&%s;",str) end, + unknown_hex_entity=function(str) return format("&#x%s;",str) end, + unknown_any_entity=function(str) return format("&#x%s;",str) end, } - -local placeholders = xml.placeholders - +local placeholders=xml.placeholders local function fromhex(s) - local n = tonumber(s,16) - if n then - return utfchar(n) - else - return format("h:%s",s), true - end + local n=tonumber(s,16) + if n then + return utfchar(n) + else + return format("h:%s",s),true + end end - local function fromdec(s) - local n = tonumber(s) - if n then - return utfchar(n) - else - return format("d:%s",s), true - end -end - --- one level expansion (simple case), no checking done - -local rest = (1-P(";"))^0 -local many = P(1)^0 - -local parsedentity = - P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + - (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) - --- parsing in the xml file - -local predefined_unified = { - [38] = "&", - [42] = """, - [47] = "'", - [74] = "<", - [76] = ">", + local n=tonumber(s) + if n then + return utfchar(n) + else + return format("d:%s",s),true + end +end +local rest=(1-P(";"))^0 +local many=P(1)^0 +local parsedentity=P("&")*(P("#x")*(rest/fromhex)+P("#")*(rest/fromdec))*P(";")*P(-1)+(P("#x")*(many/fromhex)+P("#")*(many/fromdec)) +local predefined_unified={ + [38]="&", + [42]=""", + [47]="'", + [74]="<", + [76]=">", } - -local predefined_simplified = { - [38] = "&", amp = "&", - [42] = '"', quot = '"', - [47] = "'", apos = "'", - [74] = "<", lt = "<", - [76] = ">", gt = ">", -} - -local nofprivates = 0xF0000 -- shared but seldom used - -local privates_u = { -- unescaped - [ [[&]] ] = "&", - [ [["]] ] = """, - [ [[']] ] = "'", - [ [[<]] ] = "<", - [ [[>]] ] = ">", +local predefined_simplified={ + [38]="&",amp="&", + [42]='"',quot='"', + [47]="'",apos="'", + [74]="<",lt="<", + [76]=">",gt=">", } - -local privates_p = { +local nofprivates=0xF0000 +local privates_u={ + [ [[&]] ]="&", + [ [["]] ]=""", + [ [[']] ]="'", + [ [[<]] ]="<", + [ [[>]] ]=">", } - -local privates_n = { - -- keeps track of defined ones +local privates_p={} +local privates_n={ } - -local escaped = utf.remapper(privates_u) - +local escaped=utf.remapper(privates_u) local function unescaped(s) - local p = privates_n[s] - if not p then - nofprivates = nofprivates + 1 - p = utfchar(nofprivates) - privates_n[s] = p - s = "&" .. s .. ";" -- todo: use char-ent to map to hex - privates_u[p] = s - privates_p[p] = s - end - return p -end - -local unprivatized = utf.remapper(privates_p) - -xml.privatetoken = unescaped -xml.unprivatized = unprivatized -xml.privatecodes = privates_n - + local p=privates_n[s] + if not p then + nofprivates=nofprivates+1 + p=utfchar(nofprivates) + privates_n[s]=p + s="&"..s..";" + privates_u[p]=s + privates_p[p]=s + end + return p +end +local unprivatized=utf.remapper(privates_p) +xml.privatetoken=unescaped +xml.unprivatized=unprivatized +xml.privatecodes=privates_n local function handle_hex_entity(str) - local h = hcache[str] - if not h then - local n = tonumber(str,16) - h = unify_predefined and predefined_unified[n] - if h then - if trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %s",str,h) - end - elseif utfize then - h = (n and utfchar(n)) or xml.unknown_hex_entity(str) or "" - if not n then - report_xml("utfize, ignoring hex entity &#x%s;",str) - elseif trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %s",str,h) - end - else - if trace_entities then - report_xml("found entity &#x%s;",str) - end - h = "&#x" .. str .. ";" - end - hcache[str] = h + local h=hcache[str] + if not h then + local n=tonumber(str,16) + h=unify_predefined and predefined_unified[n] + if h then + if trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %s",str,h) + end + elseif utfize then + h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" + if not n then + report_xml("utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %s",str,h) + end + else + if trace_entities then + report_xml("found entity &#x%s;",str) + end + h="&#x"..str..";" end - return h + hcache[str]=h + end + return h end - local function handle_dec_entity(str) - local d = dcache[str] - if not d then - local n = tonumber(str) - d = unify_predefined and predefined_unified[n] - if d then + local d=dcache[str] + if not d then + local n=tonumber(str) + d=unify_predefined and predefined_unified[n] + if d then + if trace_entities then + report_xml("utfize, converting dec entity &#%s; into %s",str,d) + end + elseif utfize then + d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" + if not n then + report_xml("utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + report_xml("utfize, converting dec entity &#%s; into %s",str,d) + end + else + if trace_entities then + report_xml("found entity &#%s;",str) + end + d="&#"..str..";" + end + dcache[str]=d + end + return d +end +xml.parsedentitylpeg=parsedentity +local function handle_any_entity(str) + if resolve then + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + if trace_entities then + report_xml("resolved entity &%s; -> %s (predefined)",str,a) + end + else + if type(resolve)=="function" then + a=resolve(str) or entities[str] + else + a=entities[str] + end + if a then + if type(a)=="function" then if trace_entities then - report_xml("utfize, converting dec entity &#%s; into %s",str,d) - end - elseif utfize then - d = (n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" - if not n then - report_xml("utfize, ignoring dec entity &#%s;",str) - elseif trace_entities then - report_xml("utfize, converting dec entity &#%s; into %s",str,d) + report_xml("expanding entity &%s; (function)",str) end + a=a(str) or "" + end + a=lpegmatch(parsedentity,a) or a + if trace_entities then + report_xml("resolved entity &%s; -> %s (internal)",str,a) + end else + local unknown_any_entity=placeholders.unknown_any_entity + if unknown_any_entity then + a=unknown_any_entity(str) or "" + end + if a then if trace_entities then - report_xml("found entity &#%s;",str) - end - d = "&#" .. str .. ";" - end - dcache[str] = d - end - return d -end - -xml.parsedentitylpeg = parsedentity - -local function handle_any_entity(str) - if resolve then - local a = acache[str] -- per instance ! todo - if not a then - a = resolve_predefined and predefined_simplified[str] - if a then - if trace_entities then - report_xml("resolved entity &%s; -> %s (predefined)",str,a) - end - else - if type(resolve) == "function" then - a = resolve(str) or entities[str] - else - a = entities[str] - end - if a then - if type(a) == "function" then - if trace_entities then - report_xml("expanding entity &%s; (function)",str) - end - a = a(str) or "" - end - a = lpegmatch(parsedentity,a) or a -- for nested - if trace_entities then - report_xml("resolved entity &%s; -> %s (internal)",str,a) - end - else - local unknown_any_entity = placeholders.unknown_any_entity - if unknown_any_entity then - a = unknown_any_entity(str) or "" - end - if a then - if trace_entities then - report_xml("resolved entity &%s; -> %s (external)",str,a) - end - else - if trace_entities then - report_xml("keeping entity &%s;",str) - end - if str == "" then - a = "&error;" - else - a = "&" .. str .. ";" - end - end - end + report_xml("resolved entity &%s; -> %s (external)",str,a) end - acache[str] = a - elseif trace_entities then - if not acache[str] then - report_xml("converting entity &%s; into %s",str,a) - acache[str] = a + else + if trace_entities then + report_xml("keeping entity &%s;",str) end - end - return a - else - local a = acache[str] - if not a then - a = resolve_predefined and predefined_simplified[str] - if a then - -- one of the predefined - acache[str] = a - if trace_entities then - report_xml("entity &%s; becomes %s",str,tostring(a)) - end - elseif str == "" then - if trace_entities then - report_xml("invalid entity &%s;",str) - end - a = "&error;" - acache[str] = a + if str=="" then + a="&error;" else - if trace_entities then - report_xml("entity &%s; is made private",str) - end - -- a = "&" .. str .. ";" - a = unescaped(str) - acache[str] = a + a="&"..str..";" end + end end - return a + end + acache[str]=a + elseif trace_entities then + if not acache[str] then + report_xml("converting entity &%s; into %s",str,a) + acache[str]=a + end + end + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %s",str,tostring(a)) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a="&error;" + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a + end end + return a + end end - local function handle_end_entity(chr) - report_xml("error in entity, %q found instead of ';'",chr) -end - -local space = S(' \r\n\t') -local open = P('<') -local close = P('>') -local squote = S("'") -local dquote = S('"') -local equal = P('=') -local slash = P('/') -local colon = P(':') -local semicolon = P(';') -local ampersand = P('&') -local valid = R('az', 'AZ', '09') + S('_-.') -local name_yes = C(valid^1) * colon * C(valid^1) -local name_nop = C(P(true)) * C(valid^1) -local name = name_yes + name_nop -local utfbom = lpeg.patterns.utfbom -- no capture -local spacing = C(space^0) - ------ entitycontent = (1-open-semicolon)^0 -local anyentitycontent = (1-open-semicolon-space-close)^0 -local hexentitycontent = R("AF","af","09")^0 -local decentitycontent = R("09")^0 -local parsedentity = P("#")/"" * ( - P("x")/"" * (hexentitycontent/handle_hex_entity) + - (decentitycontent/handle_dec_entity) - ) + (anyentitycontent/handle_any_entity) -local entity = ampersand/"" * parsedentity * ( (semicolon/"") + #(P(1)/handle_end_entity)) - -local text_unparsed = C((1-open)^1) -local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) - -local somespace = space^1 -local optionalspace = space^0 - ------ value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value -local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value - -local endofattributes = slash * close + close -- recovery of flacky html -local whatever = space * name * optionalspace * equal ------ wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error ------ wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error ------ wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error -local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error - -local attributevalue = value + wrongvalue - -local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute ------ attributes = (attribute)^0 - -local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 - -local parsedtext = text_parsed / add_text -local unparsedtext = text_unparsed / add_text -local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example - -local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty -local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin -local endelement = (spacing * open * slash * name * optionalspace * close) / add_end - -local begincomment = open * P("!--") -local endcomment = P("--") * close -local begininstruction = open * P("?") -local endinstruction = P("?") * close -local begincdata = open * P("![CDATA[") -local endcdata = P("]]") * close - -local someinstruction = C((1 - endinstruction)^0) -local somecomment = C((1 - endcomment )^0) -local somecdata = C((1 - endcdata )^0) - -local function normalentity(k,v ) entities[k] = v end -local function systementity(k,v,n) entities[k] = v end -local function publicentity(k,v,n) entities[k] = v end - --- todo: separate dtd parser - -local begindoctype = open * P("!DOCTYPE") -local enddoctype = close -local beginset = P("[") -local endset = P("]") -local doctypename = C((1-somespace-close)^0) -local elementdoctype = optionalspace * P("') +local squote=S("'") +local dquote=S('"') +local equal=P('=') +local slash=P('/') +local colon=P(':') +local semicolon=P(';') +local ampersand=P('&') +local valid=R('az','AZ','09')+S('_-.') +local name_yes=C(valid^1)*colon*C(valid^1) +local name_nop=C(P(true))*C(valid^1) +local name=name_yes+name_nop +local utfbom=lpeg.patterns.utfbom +local spacing=C(space^0) +local anyentitycontent=(1-open-semicolon-space-close)^0 +local hexentitycontent=R("AF","af","09")^0 +local decentitycontent=R("09")^0 +local parsedentity=P("#")/""*( + P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) + )+(anyentitycontent/handle_any_entity) +local entity=ampersand/""*parsedentity*((semicolon/"")+#(P(1)/handle_end_entity)) +local text_unparsed=C((1-open)^1) +local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) +local somespace=space^1 +local optionalspace=space^0 +local value=(squote*Cs((entity+(1-squote))^0)*squote)+(dquote*Cs((entity+(1-dquote))^0)*dquote) +local endofattributes=slash*close+close +local whatever=space*name*optionalspace*equal +local wrongvalue=Cs(P(entity+(1-space-endofattributes))^1)/attribute_value_error +local attributevalue=value+wrongvalue +local attribute=(somespace*name*optionalspace*equal*optionalspace*attributevalue)/add_attribute +local attributes=(attribute+somespace^-1*(((1-endofattributes)^1)/attribute_specification_error))^0 +local parsedtext=text_parsed/add_text +local unparsedtext=text_unparsed/add_text +local balanced=P { "["*((1-S"[]")+V(1))^0*"]" } +local emptyelement=(spacing*open*name*attributes*optionalspace*slash*close)/add_empty +local beginelement=(spacing*open*name*attributes*optionalspace*close)/add_begin +local endelement=(spacing*open*slash*name*optionalspace*close)/add_end +local begincomment=open*P("!--") +local endcomment=P("--")*close +local begininstruction=open*P("?") +local endinstruction=P("?")*close +local begincdata=open*P("![CDATA[") +local endcdata=P("]]")*close +local someinstruction=C((1-endinstruction)^0) +local somecomment=C((1-endcomment )^0) +local somecdata=C((1-endcdata )^0) +local function normalentity(k,v ) entities[k]=v end +local function systementity(k,v,n) entities[k]=v end +local function publicentity(k,v,n) entities[k]=v end +local begindoctype=open*P("!DOCTYPE") +local enddoctype=close +local beginset=P("[") +local endset=P("]") +local doctypename=C((1-somespace-close)^0) +local elementdoctype=optionalspace*P(" & - cleanup = settings.text_cleanup - entities = settings.entities or { } - -- - if utfize == nil then - settings.utfize_entities = true - utfize = true - end - if resolve_predefined == nil then - settings.resolve_predefined_entities = true - resolve_predefined = true - end - -- - -- - stack, top, at, xmlns, errorstr = { }, { }, { }, { }, nil - acache, hcache, dcache = { }, { }, { } -- not stored - reported_attribute_errors = { } - if settings.parent_root then - mt = getmetatable(settings.parent_root) - else - initialize_mt(top) - end - stack[#stack+1] = top - top.dt = { } - dt = top.dt - if not data or data == "" then - errorstr = "empty xml file" - elseif utfize or resolve then - if lpegmatch(grammar_parsed_text,data) then - errorstr = "" - else - errorstr = "invalid xml file - parsed text" - end - elseif type(data) == "string" then - if lpegmatch(grammar_unparsed_text,data) then - errorstr = "" - else - errorstr = "invalid xml file - unparsed text" - end - else - errorstr = "invalid xml file - no text at all" - end - local result - if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={ }, er = true } } } - setmetatable(stack, mt) - local errorhandler = settings.error_handler - if errorhandler == false then - -- no error message - else - errorhandler = errorhandler or xml.errorhandler - if errorhandler then - local currentresource = settings.currentresource - if currentresource and currentresource ~= "" then - xml.errorhandler(format("load error in [%s]: %s",currentresource,errorstr)) - else - xml.errorhandler(format("load error: %s",errorstr)) - end - end - end - else - result = stack[1] - end - if not settings.no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={ }, entities = entities, settings = settings } - setmetatable(result, mt) - local rdt = result.dt - for k=1,#rdt do - local v = rdt[k] - if type(v) == "table" and not v.special then -- always table -) - result.ri = k -- rootindex - v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! - break - end +local function _xmlconvert_(data,settings) + settings=settings or {} + strip=settings.strip_cm_and_dt + utfize=settings.utfize_entities + resolve=settings.resolve_entities + resolve_predefined=settings.resolve_predefined_entities + unify_predefined=settings.unify_predefined_entities + cleanup=settings.text_cleanup + entities=settings.entities or {} + if utfize==nil then + settings.utfize_entities=true + utfize=true + end + if resolve_predefined==nil then + settings.resolve_predefined_entities=true + resolve_predefined=true + end + stack,top,at,xmlns,errorstr={},{},{},{},nil + acache,hcache,dcache={},{},{} + reported_attribute_errors={} + if settings.parent_root then + mt=getmetatable(settings.parent_root) + else + initialize_mt(top) + end + stack[#stack+1]=top + top.dt={} + dt=top.dt + if not data or data=="" then + errorstr="empty xml file" + elseif utfize or resolve then + if lpegmatch(grammar_parsed_text,data) then + errorstr="" + else + errorstr="invalid xml file - parsed text" + end + elseif type(data)=="string" then + if lpegmatch(grammar_unparsed_text,data) then + errorstr="" + else + errorstr="invalid xml file - unparsed text" + end + else + errorstr="invalid xml file - no text at all" + end + local result + if errorstr and errorstr~="" then + result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } + setmetatable(stack,mt) + local errorhandler=settings.error_handler + if errorhandler==false then + else + errorhandler=errorhandler or xml.errorhandler + if errorhandler then + local currentresource=settings.currentresource + if currentresource and currentresource~="" then + xml.errorhandler(format("load error in [%s]: %s",currentresource,errorstr)) + else + xml.errorhandler(format("load error: %s",errorstr)) end + end end - if errorstr and errorstr ~= "" then - result.error = true + else + result=stack[1] + end + if not settings.no_root then + result={ special=true,ns="",tg='@rt@',dt=result.dt,at={},entities=entities,settings=settings } + setmetatable(result,mt) + local rdt=result.dt + for k=1,#rdt do + local v=rdt[k] + if type(v)=="table" and not v.special then + result.ri=k + v.__p__=result + break + end end - result.statistics = { - entities = { - decimals = dcache, - hexadecimals = hcache, - names = acache, - } + end + if errorstr and errorstr~="" then + result.error=true + end + result.statistics={ + entities={ + decimals=dcache, + hexadecimals=hcache, + names=acache, } - strip, utfize, resolve, resolve_predefined = nil, nil, nil, nil - unify_predefined, cleanup, entities = nil, nil, nil - stack, top, at, xmlns, errorstr = nil, nil, nil, nil, nil - acache, hcache, dcache = nil, nil, nil - reported_attribute_errors, mt, errorhandler = nil, nil, nil - return result + } + strip,utfize,resolve,resolve_predefined=nil,nil,nil,nil + unify_predefined,cleanup,entities=nil,nil,nil + stack,top,at,xmlns,errorstr=nil,nil,nil,nil,nil + acache,hcache,dcache=nil,nil,nil + reported_attribute_errors,mt,errorhandler=nil,nil,nil + return result end - --- Because we can have a crash (stack issues) with faulty xml, we wrap this one --- in a protector: - function xmlconvert(data,settings) - local ok, result = pcall(function() return _xmlconvert_(data,settings) end) - if ok then - return result - else - return _xmlconvert_("",settings) - end -end - -xml.convert = xmlconvert - -function xml.inheritedconvert(data,xmldata) -- xmldata is parent - local settings = xmldata.settings - if settings then - settings.parent_root = xmldata -- to be tested - end - -- settings.no_root = true - local xc = xmlconvert(data,settings) -- hm, we might need to locate settings - -- xc.settings = nil - -- xc.entities = nil - -- xc.special = nil - -- xc.ri = nil - -- print(xc.tg) - return xc + local ok,result=pcall(function() return _xmlconvert_(data,settings) end) + if ok then + return result + else + return _xmlconvert_("",settings) + end +end +xml.convert=xmlconvert +function xml.inheritedconvert(data,xmldata) + local settings=xmldata.settings + if settings then + settings.parent_root=xmldata + end + local xc=xmlconvert(data,settings) + return xc end - - - function xml.is_valid(root) - return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er + return root and root.dt and root.dt[1] and type(root.dt[1])=="table" and not root.dt[1].er end - function xml.package(tag,attributes,data) - local ns, tg = match(tag,"^(.-):?([^:]+)$") - local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } - setmetatable(t, mt) - return t + local ns,tg=match(tag,"^(.-):?([^:]+)$") + local t={ ns=ns,tg=tg,dt=data or "",at=attributes or {} } + setmetatable(t,mt) + return t end - function xml.is_valid(root) - return root and not root.error + return root and not root.error end - -xml.errorhandler = report_xml - - - +xml.errorhandler=report_xml function xml.load(filename,settings) - local data = "" - if type(filename) == "string" then - -- local data = io.loaddata(filename) - -todo: check type in io.loaddata - local f = io.open(filename,'r') -- why not 'rb' - if f then - data = f:read("*all") -- io.readall(f) ... only makes sense for large files - f:close() - end - elseif filename then -- filehandle - data = filename:read("*all") -- io.readall(f) ... only makes sense for large files - end - if settings then - settings.currentresource = filename - local result = xmlconvert(data,settings) - settings.currentresource = nil - return result - else - return xmlconvert(data,{ currentresource = filename }) - end + local data="" + if type(filename)=="string" then + local f=io.open(filename,'r') + if f then + data=f:read("*all") + f:close() + end + elseif filename then + data=filename:read("*all") + end + if settings then + settings.currentresource=filename + local result=xmlconvert(data,settings) + settings.currentresource=nil + return result + else + return xmlconvert(data,{ currentresource=filename }) + end end - - - -local no_root = { no_root = true } - +local no_root={ no_root=true } function xml.toxml(data) - if type(data) == "string" then - local root = { xmlconvert(data,no_root) } - return (#root > 1 and root) or root[1] - else - return data - end + if type(data)=="string" then + local root={ xmlconvert(data,no_root) } + return (#root>1 and root) or root[1] + else + return data + end end - - - local function copy(old,tables) - if old then - tables = tables or { } - local new = { } - if not tables[old] then - tables[old] = new - end - for k,v in next, old do - new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) - end - return new - else - return { } + if old then + tables=tables or {} + local new={} + if not tables[old] then + tables[old]=new end -end - -xml.copy = copy - - - --- todo: add when not present - -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt = root.dt - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then - return - end - end - insert(dt, 1, { special = true, ns = "", tg = "@pi@", dt = { "xml version='1.0' standalone='yes'" } } ) - insert(dt, 2, "\n" ) + for k,v in next,old do + new[k]=(type(v)=="table" and (tables[v] or copy(v,tables))) or v end -end - - - --- new experimental reorganized serialize - -local function verbose_element(e,handlers) -- options - local handle = handlers.handle - local serialize = handlers.serialize - local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } - if ats then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,escaped(v)) - end + local mt=getmetatable(old) + if mt then + setmetatable(new,mt) end - if ern and trace_entities and ern ~= ens then - ens = ern + return new + else + return {} + end +end +xml.copy=copy +function xml.checkbom(root) + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" and find(v.dt[1],"xml.*version=") then + return + end end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - handle("<",ens,":",etg," ",concat(ats," "),">") - else - handle("<",ens,":",etg,">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - handle(escaped(e)) - else - serialize(e,handlers) - end - end - handle("") + insert(dt,1,{ special=true,ns="",tg="@pi@",dt={ "xml version='1.0' standalone='yes'" } } ) + insert(dt,2,"\n" ) + end +end +local function verbose_element(e,handlers) + local handle=handlers.handle + local serialize=handlers.serialize + local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn + local ats=eat and next(eat) and {} + if ats then + for k,v in next,eat do + ats[#ats+1]=format('%s=%q',k,escaped(v)) + end + end + if ern and trace_entities and ern~=ens then + ens=ern + end + if ens~="" then + if edt and #edt>0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) else - if ats then - handle("<",ens,":",etg," ",concat(ats," "),"/>") - else - handle("<",ens,":",etg,"/>") - end + serialize(e,handlers) end + end + handle("") else - if edt and #edt > 0 then - if ats then - handle("<",etg," ",concat(ats," "),">") - else - handle("<",etg,">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - handle(escaped(e)) -- option: hexify escaped entities - else - serialize(e,handlers) - end - end - handle("") + if ats then + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") + end + end + else + if edt and #edt>0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) else - if ats then - handle("<",etg," ",concat(ats," "),"/>") - else - handle("<",etg,"/>") - end + serialize(e,handlers) end + end + handle("") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") + else + handle("<",etg,"/>") + end end + end end - local function verbose_pi(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_comment(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_cdata(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_doctype(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_root(e,handlers) - handlers.serialize(e.dt,handlers) + handlers.serialize(e.dt,handlers) end - local function verbose_text(e,handlers) - handlers.handle(escaped(e)) + handlers.handle(escaped(e)) end - local function verbose_document(e,handlers) - local serialize = handlers.serialize - local functions = handlers.functions - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - functions["@tx@"](ei,handlers) - else - serialize(ei,handlers) - end + local serialize=handlers.serialize + local functions=handlers.functions + for i=1,#e do + local ei=e[i] + if type(ei)=="string" then + functions["@tx@"](ei,handlers) + else + serialize(ei,handlers) end + end end - local function serialize(e,handlers,...) - local initialize = handlers.initialize - local finalize = handlers.finalize - local functions = handlers.functions - if initialize then - local state = initialize(...) - if not state == true then - return state - end - end - local etg = e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - -- elseif type(e) == "string" then - -- functions["@tx@"](e,handlers) - else - functions["@dc@"](e,handlers) -- dc ? - end - if finalize then - return finalize() - end + local initialize=handlers.initialize + local finalize=handlers.finalize + local functions=handlers.functions + if initialize then + local state=initialize(...) + if not state==true then + return state + end + end + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end - local function xserialize(e,handlers) - local functions = handlers.functions - local etg = e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - -- elseif type(e) == "string" then - -- functions["@tx@"](e,handlers) - else - functions["@dc@"](e,handlers) - end -end - -local handlers = { } - + local functions=handlers.functions + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end +end +local handlers={} local function newhandlers(settings) - local t = table.copy(handlers.verbose or { }) -- merge - if settings then - for k,v in next, settings do - if type(v) == "table" then - local tk = t[k] if not tk then tk = { } t[k] = tk end - for kk,vv in next, v do - tk[kk] = vv - end - else - t[k] = v - end - end - if settings.name then - handlers[settings.name] = t - end + local t=table.copy(handlers.verbose or {}) + if settings then + for k,v in next,settings do + if type(v)=="table" then + local tk=t[k] if not tk then tk={} t[k]=tk end + for kk,vv in next,v do + tk[kk]=vv + end + else + t[k]=v + end end - utilities.storage.mark(t) - return t + if settings.name then + handlers[settings.name]=t + end + end + utilities.storage.mark(t) + return t end - -local nofunction = function() end - +local nofunction=function() end function xml.sethandlersfunction(handler,name,fnc) - handler.functions[name] = fnc or nofunction + handler.functions[name]=fnc or nofunction end - function xml.gethandlersfunction(handler,name) - return handler.functions[name] + return handler.functions[name] end - function xml.gethandlers(name) - return handlers[name] + return handlers[name] end - newhandlers { - name = "verbose", - initialize = false, -- faster than nil and mt lookup - finalize = false, -- faster than nil and mt lookup - serialize = xserialize, - handle = print, - functions = { - ["@dc@"] = verbose_document, - ["@dt@"] = verbose_doctype, - ["@rt@"] = verbose_root, - ["@el@"] = verbose_element, - ["@pi@"] = verbose_pi, - ["@cm@"] = verbose_comment, - ["@cd@"] = verbose_cdata, - ["@tx@"] = verbose_text, - } + name="verbose", + initialize=false, + finalize=false, + serialize=xserialize, + handle=print, + functions={ + ["@dc@"]=verbose_document, + ["@dt@"]=verbose_doctype, + ["@rt@"]=verbose_root, + ["@el@"]=verbose_element, + ["@pi@"]=verbose_pi, + ["@cm@"]=verbose_comment, + ["@cd@"]=verbose_cdata, + ["@tx@"]=verbose_text, + } } - - - --- maybe this will move to lxml-xml - local result - -local xmlfilehandler = newhandlers { - name = "file", - initialize = function(name) - result = io.open(name,"wb") - return result - end, - finalize = function() - result:close() - return true - end, - handle = function(...) - result:write(...) - end, +local xmlfilehandler=newhandlers { + name="file", + initialize=function(name) + result=io.open(name,"wb") + return result + end, + finalize=function() + result:close() + return true + end, + handle=function(...) + result:write(...) + end, } - --- no checking on writeability here but not faster either --- --- local xmlfilehandler = newhandlers { --- initialize = function(name) --- io.output(name,"wb") --- return true --- end, --- finalize = function() --- io.close() --- return true --- end, --- handle = io.write, --- } - function xml.save(root,name) - serialize(root,xmlfilehandler,name) + serialize(root,xmlfilehandler,name) end - local result - -local xmlstringhandler = newhandlers { - name = "string", - initialize = function() - result = { } - return result - end, - finalize = function() - return concat(result) - end, - handle = function(...) - result[#result+1] = concat { ... } - end, +local xmlstringhandler=newhandlers { + name="string", + initialize=function() + result={} + return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, } - -local function xmltostring(root) -- 25% overhead due to collecting - if not root then - return "" - elseif type(root) == 'string' then - return root - else -- if next(root) then -- next is faster than type (and >0 test) - return serialize(root,xmlstringhandler) or "" - end +local function xmltostring(root) + if not root then + return "" + elseif type(root)=='string' then + return root + else + return serialize(root,xmlstringhandler) or "" + end end - -local function __tostring(root) -- inline - return (root and xmltostring(root)) or "" +local function __tostring(root) + return (root and xmltostring(root)) or "" end - -initialize_mt = function(root) -- redefinition - mt = { __tostring = __tostring, __index = root } +initialize_mt=function(root) + mt={ __tostring=__tostring,__index=root } end - -xml.defaulthandlers = handlers -xml.newhandlers = newhandlers -xml.serialize = serialize -xml.tostring = xmltostring - - - +xml.defaulthandlers=handlers +xml.newhandlers=newhandlers +xml.serialize=serialize +xml.tostring=xmltostring local function xmlstring(e,handle) - if not handle or (e.special and e.tg ~= "@rt@") then - -- nothing - elseif e.tg then - local edt = e.dt - if edt then - for i=1,#edt do - xmlstring(edt[i],handle) - end - end - else - handle(e) + if not handle or (e.special and e.tg~="@rt@") then + elseif e.tg then + local edt=e.dt + if edt then + for i=1,#edt do + xmlstring(edt[i],handle) + end end + else + handle(e) + end end - -xml.string = xmlstring - - - - +xml.string=xmlstring function xml.settings(e) - while e do - local s = e.settings - if s then - return s - else - e = e.__p__ - end + while e do + local s=e.settings + if s then + return s + else + e=e.__p__ end - return nil + end + return nil end - function xml.root(e) - local r = e - while e do - e = e.__p__ - if e then - r = e - end + local r=e + while e do + e=e.__p__ + if e then + r=e end - return r + end + return r end - function xml.parent(root) - return root.__p__ + return root.__p__ end - function xml.body(root) - return root.ri and root.dt[root.ri] or root -- not ok yet + return root.ri and root.dt[root.ri] or root end - function xml.name(root) - if not root then - return "" - end - local ns = root.ns - local tg = root.tg - if ns == "" then - return tg - else - return ns .. ":" .. tg - end + if not root then + return "" + end + local ns=root.ns + local tg=root.tg + if ns=="" then + return tg + else + return ns..":"..tg + end end - - - function xml.erase(dt,k) - if dt then - if k then - dt[k] = "" - else for k=1,#dt do - dt[1] = { "" } - end end - end + if dt then + if k then + dt[k]="" + else for k=1,#dt do + dt[1]={ "" } + end end + end end - - - function xml.assign(dt,k,root) - if dt and k then - dt[k] = type(root) == "table" and xml.body(root) or root - return dt[k] - else - return xml.body(root) - end + if dt and k then + dt[k]=type(root)=="table" and xml.body(root) or root + return dt[k] + else + return xml.body(root) + end +end +function xml.tocdata(e,wrapper) + local whatever=type(e)=="table" and xmltostring(e.dt) or e or "" + if wrapper then + whatever=format("<%s>%s",wrapper,whatever,wrapper) + end + local t={ special=true,ns="",tg="@cd@",at={},rn="",dt={ whatever },__p__=e } + setmetatable(t,getmetatable(e)) + e.dt={ t } end - --- the following helpers may move - - - -function xml.tocdata(e,wrapper) -- a few more in the aux module - local whatever = type(e) == "table" and xmltostring(e.dt) or e or "" - if wrapper then - whatever = format("<%s>%s",wrapper,whatever,wrapper) - end - local t = { special = true, ns = "", tg = "@cd@", at = { }, rn = "", dt = { whatever }, __p__ = e } - setmetatable(t,getmetatable(e)) - e.dt = { t } -end - function xml.makestandalone(root) - if root.ri then - local dt = root.dt - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi@" then - local txt = v.dt[1] - if find(txt,"xml.*version=") then - v.dt[1] = txt .. " standalone='yes'" - break - end - end + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" then + local txt=v.dt[1] + if find(txt,"xml.*version=") then + v.dt[1]=txt.." standalone='yes'" + break end + end end - return root + end + return root end - function xml.kind(e) - local dt = e and e.dt - if dt then - local n = #dt - if n == 1 then - local d = dt[1] - if d.special then - local tg = d.tg - if tg == "@cd@" then - return "cdata" - elseif tg == "@cm" then - return "comment" - elseif tg == "@pi@" then - return "instruction" - elseif tg == "@dt@" then - return "declaration" - end - elseif type(d) == "string" then - return "text" - end - return "element" - elseif n > 0 then - return "mixed" - end + local dt=e and e.dt + if dt then + local n=#dt + if n==1 then + local d=dt[1] + if d.special then + local tg=d.tg + if tg=="@cd@" then + return "cdata" + elseif tg=="@cm" then + return "comment" + elseif tg=="@pi@" then + return "instruction" + elseif tg=="@dt@" then + return "declaration" + end + elseif type(d)=="string" then + return "text" + end + return "element" + elseif n>0 then + return "mixed" end - return "empty" + end + return "empty" end @@ -10804,1308 +8138,1036 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-lpt'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- e.ni is only valid after a filter run --- todo: B/C/[get first match] - -local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, tonumber, tostring, setmetatable, load, select = type, next, tonumber, tostring, setmetatable, load, select -local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns - -local setmetatableindex = table.setmetatableindex - --- beware, this is not xpath ... e.g. position is different (currently) and --- we have reverse-sibling as reversed preceding sibling - - - - - -local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end -local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end -local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end - -local report_lpath = logs.reporter("xml","lpath") - - - -local xml = xml - -local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end -local lpathcached = 0 function xml.lpathcached() return lpathcached end - -xml.functions = xml.functions or { } -- internal -local functions = xml.functions - -xml.expressions = xml.expressions or { } -- in expressions -local expressions = xml.expressions - -xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) -local finalizers = xml.finalizers - -xml.specialhandler = xml.specialhandler or { } -local specialhandler = xml.specialhandler - -lpegpatterns.xml = lpegpatterns.xml or { } -local xmlpatterns = lpegpatterns.xml - -finalizers.xml = finalizers.xml or { } -finalizers.tex = finalizers.tex or { } - -local function fallback (t, name) - local fn = finalizers[name] - if fn then - t[name] = fn - else - report_lpath("unknown sub finalizer '%s'",tostring(name)) - fn = function() end - end - return fn -end - -setmetatableindex(finalizers.xml, fallback) -setmetatableindex(finalizers.tex, fallback) - -xml.defaultprotocol = "xml" - --- as xsl does not follow xpath completely here we will also --- be more liberal especially with regards to the use of | and --- the rootpath: --- --- test : all 'test' under current --- /test : 'test' relative to current --- a|b|c : set of names --- (a|b|c) : idem --- ! : not --- --- after all, we're not doing transformations but filtering. in --- addition we provide filter functions (last bit) --- --- todo: optimizer --- --- .. : parent --- * : all kids --- / : anchor here --- // : /**/ --- ** : all in between --- --- so far we had (more practical as we don't transform) --- --- {/test} : kids 'test' under current node --- {test} : any kid with tag 'test' --- {//test} : same as above - --- evaluator (needs to be redone, for the moment copied) +-- original size: 47510, stripped down to: 30425 --- todo: apply_axis(list,notable) and collection vs single - -local apply_axis = { } - -apply_axis['root'] = function(list) - local collected = { } - for l=1,#list do - local ll = list[l] - local rt = ll - while ll do - ll = ll.__p__ - if ll then - rt = ll - end - end - collected[l] = rt +if not modules then modules={} end modules ['lxml-lpt']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,tonumber,tostring,setmetatable,load,select=type,next,tonumber,tostring,setmetatable,load,select +local format,upper,lower,gmatch,gsub,find,rep=string.format,string.upper,string.lower,string.gmatch,string.gsub,string.find,string.rep +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local setmetatableindex=table.setmetatableindex +local trace_lpath=false if trackers then trackers.register("xml.path",function(v) trace_lpath=v end) end +local trace_lparse=false if trackers then trackers.register("xml.parse",function(v) trace_lparse=v end) end +local trace_lprofile=false if trackers then trackers.register("xml.profile",function(v) trace_lpath=v trace_lparse=v trace_lprofile=v end) end +local report_lpath=logs.reporter("xml","lpath") +local xml=xml +local lpathcalls=0 function xml.lpathcalls () return lpathcalls end +local lpathcached=0 function xml.lpathcached() return lpathcached end +xml.functions=xml.functions or {} +local functions=xml.functions +xml.expressions=xml.expressions or {} +local expressions=xml.expressions +xml.finalizers=xml.finalizers or {} +local finalizers=xml.finalizers +xml.specialhandler=xml.specialhandler or {} +local specialhandler=xml.specialhandler +lpegpatterns.xml=lpegpatterns.xml or {} +local xmlpatterns=lpegpatterns.xml +finalizers.xml=finalizers.xml or {} +finalizers.tex=finalizers.tex or {} +local function fallback (t,name) + local fn=finalizers[name] + if fn then + t[name]=fn + else + report_lpath("unknown sub finalizer '%s'",tostring(name)) + fn=function() end + end + return fn +end +setmetatableindex(finalizers.xml,fallback) +setmetatableindex(finalizers.tex,fallback) +xml.defaultprotocol="xml" +local apply_axis={} +apply_axis['root']=function(list) + local collected={} + for l=1,#list do + local ll=list[l] + local rt=ll + while ll do + ll=ll.__p__ + if ll then + rt=ll + end end - return collected -end - -apply_axis['self'] = function(list) - return list -end - -apply_axis['child'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local dt = ll.dt - if dt then -- weird that this is needed - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - end - end - ll.en = en + collected[l]=rt + end + return collected +end +apply_axis['self']=function(list) + return list +end +apply_axis['child']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local dt=ll.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en end + end + ll.en=en end - return collected + end + return collected end - local function collect(list,collected,c) - local dt = list.dt - if dt then - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - c = collect(dk,collected,c) - end - end - list.en = en + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end end - return c + list.en=en + end + return c end - -apply_axis['descendant'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - c = collect(list[l],collected,c) - end - return collected +apply_axis['descendant']=function(list) + local collected,c={},0 + for l=1,#list do + c=collect(list[l],collected,c) + end + return collected end - local function collect(list,collected,c) - local dt = list.dt - if dt then - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - c = collect(dk,collected,c) - end - end - list.en = en - end - return c -end -apply_axis['descendant-or-self'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - if ll.special ~= true then -- catch double root - c = c + 1 - collected[c] = ll - end - c = collect(ll,collected,c) - end - return collected -end - -apply_axis['ancestor'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - while ll do - ll = ll.__p__ - if ll then - c = c + 1 - collected[c] = ll - end - end + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end end - return collected -end - -apply_axis['ancestor-or-self'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - c = c + 1 - collected[c] = ll - while ll do - ll = ll.__p__ - if ll then - c = c + 1 - collected[c] = ll - end - end + list.en=en + end + return c +end +apply_axis['descendant-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + if ll.special~=true then + c=c+1 + collected[c]=ll + end + c=collect(ll,collected,c) + end + return collected +end +apply_axis['ancestor']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end end - return collected -end - -apply_axis['parent'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local pl = list[l].__p__ - if pl then - c = c + 1 - collected[c] = pl - end + end + return collected +end +apply_axis['ancestor-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + c=c+1 + collected[c]=ll + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end end - return collected -end - -apply_axis['attribute'] = function(list) - return { } -end - -apply_axis['namespace'] = function(list) - return { } -end - -apply_axis['following'] = function(list) -- incomplete - return { } -end - -apply_axis['preceding'] = function(list) -- incomplete - return { } -end - -apply_axis['following-sibling'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=ll.ni+1,#d do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end + end + return collected +end +apply_axis['parent']=function(list) + local collected,c={},0 + for l=1,#list do + local pl=list[l].__p__ + if pl then + c=c+1 + collected[c]=pl + end + end + return collected +end +apply_axis['attribute']=function(list) + return {} +end +apply_axis['namespace']=function(list) + return {} +end +apply_axis['following']=function(list) + return {} +end +apply_axis['preceding']=function(list) + return {} +end +apply_axis['following-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni+1,#d do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end end - return collected -end - -apply_axis['preceding-sibling'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=1,ll.ni-1 do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end + end + return collected +end +apply_axis['preceding-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=1,ll.ni-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end end - return collected -end - -apply_axis['reverse-sibling'] = function(list) -- reverse preceding - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=ll.ni-1,1,-1 do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end + end + return collected +end +apply_axis['reverse-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni-1,1,-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end end - return collected + end + return collected end - -apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] -apply_axis['auto-descendant'] = apply_axis['descendant'] -apply_axis['auto-child'] = apply_axis['child'] -apply_axis['auto-self'] = apply_axis['self'] -apply_axis['initial-child'] = apply_axis['child'] - +apply_axis['auto-descendant-or-self']=apply_axis['descendant-or-self'] +apply_axis['auto-descendant']=apply_axis['descendant'] +apply_axis['auto-child']=apply_axis['child'] +apply_axis['auto-self']=apply_axis['self'] +apply_axis['initial-child']=apply_axis['child'] local function apply_nodes(list,directive,nodes) - -- todo: nodes[1] etc ... negated node name in set ... when needed - -- ... currently ignored - local maxn = #nodes - if maxn == 3 then --optimized loop - local nns, ntg = nodes[2], nodes[3] - if not nns and not ntg then -- wildcard + local maxn=#nodes + if maxn==3 then + local nns,ntg=nodes[2],nodes[3] + if not nns and not ntg then + if directive then + return list + else + return {} + end + else + local collected,c,m,p={},0,0,nil + if not nns then + for l=1,#list do + local ll=list[l] + local ltg=ll.tg + if ltg then if directive then - return list - else - return { } + if ntg==ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif ntg~=ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end - else - local collected, c, m, p = { }, 0, 0, nil - if not nns then -- only check tag - for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - if directive then - if ntg == ltg then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif ntg ~= ltg then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - elseif not ntg then -- only check namespace - for l=1,#list do - local ll = list[l] - local lns = ll.rn or ll.ns - if lns then - if directive then - if lns == nns then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif lns ~= nns then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - else -- check both - for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - local lns = ll.rn or ll.ns - local ok = ltg == ntg and lns == nns - if directive then - if ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif not ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end + end + end + elseif not ntg then + for l=1,#list do + local ll=list[l] + local lns=ll.rn or ll.ns + if lns then + if directive then + if lns==nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif lns~=nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end - return collected + end end - else - local collected, c, m, p = { }, 0, 0, nil + else for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - local lns = ll.rn or ll.ns - local ok = false - for n=1,maxn,3 do - local nns, ntg = nodes[n+1], nodes[n+2] - ok = (not ntg or ltg == ntg) and (not nns or lns == nns) - if ok then - break - end - end - if directive then - if ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif not ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=ltg==ntg and lns==nns + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end + end end - return collected + end + return collected end -end - -local quit_expression = false - -local function apply_expression(list,expression,order) - local collected, c = { }, 0 - quit_expression = false + else + local collected,c,m,p={},0,0,nil for l=1,#list do - local ll = list[l] - if expression(list,ll,l,order) then -- nasty, order alleen valid als n=1 - c = c + 1 - collected[c] = ll - end - if quit_expression then + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=false + for n=1,maxn,3 do + local nns,ntg=nodes[n+1],nodes[n+2] + ok=(not ntg or ltg==ntg) and (not nns or lns==nns) + if ok then break + end + end + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end + end end return collected + end end - -local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb - -local spaces = S(" \n\r\t\f")^0 -local lp_space = S(" \n\r\t\f") -local lp_any = P(1) -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_or = P("|") / " or " -local lp_and = P("&") / " and " - -local lp_builtin = P ( - P("text") / "(ll.dt[1] or '')" + -- fragile - P("content") / "ll.dt" + - -- P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + - P("name") / "((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)" + - P("tag") / "ll.tg" + - P("position") / "l" + -- is element in finalizer - P("firstindex") / "1" + - P("lastindex") / "(#ll.__p__.dt or 1)" + - P("firstelement") / "1" + - P("lastelement") / "(ll.__p__.en or 1)" + - P("first") / "1" + - P("last") / "#list" + - P("rootposition") / "order" + - P("order") / "order" + - P("element") / "(ll.ei or 1)" + - P("index") / "(ll.ni or 1)" + - P("match") / "(ll.mi or 1)" + - -- P("namespace") / "ll.ns" + - P("ns") / "ll.ns" - ) * ((spaces * P("(") * spaces * P(")"))/"") - --- for the moment we keep namespaces with attributes - -local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * ((R("az","AZ") + S("-_:"))^1) * Cc("'])") - --- lp_fastpos_p = (P("+")^0 * R("09")^1 * P(-1)) / function(s) return "l==" .. s end --- lp_fastpos_n = (P("-") * R("09")^1 * P(-1)) / function(s) return "(" .. s .. "<0 and (#list+".. s .. "==l))" end - -lp_fastpos_p = P("+")^0 * R("09")^1 * P(-1) / "l==%0" -lp_fastpos_n = P("-") * R("09")^1 * P(-1) / "(%0<0 and (#list+%0==l))" - -local lp_fastpos = lp_fastpos_n + lp_fastpos_p - -local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") - --- local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling --- return t .. "(" --- end - --- local lp_lua_function = (R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / "%0(" -local lp_lua_function = Cs((R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(")) / "%0" - -local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling - if expressions[t] then - return "expr." .. t .. "(" - else - return "expr.error(" - end -end - -local lparent = P("(") -local rparent = P(")") -local noparent = 1 - (lparent+rparent) -local nested = P{lparent * (noparent + V(1))^0 * rparent} -local value = P(lparent * C((noparent + nested)^0) * rparent) -- P{"("*C(((1-S("()"))+V(1))^0)*")"} - -local lp_child = Cc("expr.child(ll,'") * R("az","AZ","--","__")^1 * Cc("')") -local lp_number = S("+-") * R("09")^1 -local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") -local lp_content = (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) - +local quit_expression=false +local function apply_expression(list,expression,order) + local collected,c={},0 + quit_expression=false + for l=1,#list do + local ll=list[l] + if expression(list,ll,l,order) then + c=c+1 + collected[c]=ll + end + if quit_expression then + break + end + end + return collected +end +local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb +local spaces=S(" \n\r\t\f")^0 +local lp_space=S(" \n\r\t\f") +local lp_any=P(1) +local lp_noequal=P("!=")/"~="+P("<=")+P(">=")+P("==") +local lp_doequal=P("=")/"==" +local lp_or=P("|")/" or " +local lp_and=P("&")/" and " +local lp_builtin=P ( + P("text")/"(ll.dt[1] or '')"+ + P("content")/"ll.dt"+ + P("name")/"((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)"+P("tag")/"ll.tg"+P("position")/"l"+ + P("firstindex")/"1"+P("lastindex")/"(#ll.__p__.dt or 1)"+P("firstelement")/"1"+P("lastelement")/"(ll.__p__.en or 1)"+P("first")/"1"+P("last")/"#list"+P("rootposition")/"order"+P("order")/"order"+P("element")/"(ll.ei or 1)"+P("index")/"(ll.ni or 1)"+P("match")/"(ll.mi or 1)"+ + P("ns")/"ll.ns" + )*((spaces*P("(")*spaces*P(")"))/"") +local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") +lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" +lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" +local lp_fastpos=lp_fastpos_n+lp_fastpos_p +local lp_reserved=C("and")+C("or")+C("not")+C("div")+C("mod")+C("true")+C("false") +local lp_lua_function=Cs((R("az","AZ","__")^1*(P(".")*R("az","AZ","__")^1)^1)*("("))/"%0" +local lp_function=C(R("az","AZ","__")^1)*P("(")/function(t) + if expressions[t] then + return "expr."..t.."(" + else + return "expr.error(" + end +end +local lparent=P("(") +local rparent=P(")") +local noparent=1-(lparent+rparent) +local nested=P{lparent*(noparent+V(1))^0*rparent} +local value=P(lparent*C((noparent+nested)^0)*rparent) +local lp_child=Cc("expr.child(ll,'")*R("az","AZ","--","__")^1*Cc("')") +local lp_number=S("+-")*R("09")^1 +local lp_string=Cc("'")*R("az","AZ","--","__")^1*Cc("'") +local lp_content=(P("'")*(1-P("'"))^0*P("'")+P('"')*(1-P('"'))^0*P('"')) local cleaner - -local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) - if expressions[t] then - s = s and s ~= "" and lpegmatch(cleaner,s) - if s and s ~= "" then - return "expr." .. t .. "(ll," .. s ..")" - else - return "expr." .. t .. "(ll)" - end - else - return "expr.error(" .. t .. ")" - end -end - -local content = - lp_builtin + - lp_attribute + - lp_special + - lp_noequal + lp_doequal + - lp_or + lp_and + - lp_reserved + - lp_lua_function + lp_function + - lp_content + -- too fragile - lp_child + - lp_any - -local converter = Cs ( - lp_fastpos + (P { lparent * (V(1))^0 * rparent + content } )^0 +local lp_special=(C(P("name")+P("text")+P("tag")+P("count")+P("child")))*value/function(t,s) + if expressions[t] then + s=s and s~="" and lpegmatch(cleaner,s) + if s and s~="" then + return "expr."..t.."(ll,"..s..")" + else + return "expr."..t.."(ll)" + end + else + return "expr.error("..t..")" + end +end +local content=lp_builtin+lp_attribute+lp_special+lp_noequal+lp_doequal+lp_or+lp_and+lp_reserved+lp_lua_function+lp_function+lp_content+ + lp_child+lp_any +local converter=Cs ( + lp_fastpos+(P { lparent*(V(1))^0*rparent+content } )^0 ) - -cleaner = Cs ( ( - lp_reserved + - lp_number + - lp_string + -1 )^1 ) - - - -local template_e = [[ +cleaner=Cs (( + lp_reserved+lp_number+lp_string+1 )^1 ) +local template_e=[[ local expr = xml.expressions return function(list,ll,l,order) return %s end ]] - -local template_f_y = [[ +local template_f_y=[[ local finalizer = xml.finalizers['%s']['%s'] return function(collection) return finalizer(collection,%s) end ]] - -local template_f_n = [[ +local template_f_n=[[ return xml.finalizers['%s']['%s'] ]] - --- - -local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } -local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } -local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } -local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } -local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } -local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } -local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } -local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } -local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } -local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } -local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } -local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } -local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } -local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } -local register_reverse_sibling = { kind = "axis", axis = "reverse-sibling" } -- , apply = apply_axis["reverse-sibling"] } - -local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } -local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } -local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } -local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } - -local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } - -local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } - -local skip = { } - +local register_self={ kind="axis",axis="self" } +local register_parent={ kind="axis",axis="parent" } +local register_descendant={ kind="axis",axis="descendant" } +local register_child={ kind="axis",axis="child" } +local register_descendant_or_self={ kind="axis",axis="descendant-or-self" } +local register_root={ kind="axis",axis="root" } +local register_ancestor={ kind="axis",axis="ancestor" } +local register_ancestor_or_self={ kind="axis",axis="ancestor-or-self" } +local register_attribute={ kind="axis",axis="attribute" } +local register_namespace={ kind="axis",axis="namespace" } +local register_following={ kind="axis",axis="following" } +local register_following_sibling={ kind="axis",axis="following-sibling" } +local register_preceding={ kind="axis",axis="preceding" } +local register_preceding_sibling={ kind="axis",axis="preceding-sibling" } +local register_reverse_sibling={ kind="axis",axis="reverse-sibling" } +local register_auto_descendant_or_self={ kind="axis",axis="auto-descendant-or-self" } +local register_auto_descendant={ kind="axis",axis="auto-descendant" } +local register_auto_self={ kind="axis",axis="auto-self" } +local register_auto_child={ kind="axis",axis="auto-child" } +local register_initial_child={ kind="axis",axis="initial-child" } +local register_all_nodes={ kind="nodes",nodetest=true,nodes={ true,false,false } } +local skip={} local function errorrunner_e(str,cnv) - if not skip[str] then - report_lpath("error in expression: %s => %s",str,cnv) - skip[str] = cnv or str - end - return false + if not skip[str] then + report_lpath("error in expression: %s => %s",str,cnv) + skip[str]=cnv or str + end + return false end local function errorrunner_f(str,arg) - report_lpath("error in finalizer: %s(%s)",str,arg or "") - return false + report_lpath("error in finalizer: %s(%s)",str,arg or "") + return false end - local function register_nodes(nodetest,nodes) - return { kind = "nodes", nodetest = nodetest, nodes = nodes } + return { kind="nodes",nodetest=nodetest,nodes=nodes } end - local function register_expression(expression) - local converted = lpegmatch(converter,expression) - local runner = load(format(template_e,converted)) - runner = (runner and runner()) or function() errorrunner_e(expression,converted) end - return { kind = "expression", expression = expression, converted = converted, evaluator = runner } + local converted=lpegmatch(converter,expression) + local runner=load(format(template_e,converted)) + runner=(runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind="expression",expression=expression,converted=converted,evaluator=runner } end - local function register_finalizer(protocol,name,arguments) - local runner - if arguments and arguments ~= "" then - runner = load(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) - else - runner = load(format(template_f_n,protocol or xml.defaultprotocol,name)) - end - runner = (runner and runner()) or function() errorrunner_f(name,arguments) end - return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } -end - -local expression = P { "ex", - ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", - sq = "'" * (1 - S("'"))^0 * "'", - dq = '"' * (1 - S('"'))^0 * '"', + local runner + if arguments and arguments~="" then + runner=load(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) + else + runner=load(format(template_f_n,protocol or xml.defaultprotocol,name)) + end + runner=(runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind="finalizer",name=name,arguments=arguments,finalizer=runner } +end +local expression=P { "ex", + ex="["*C((V("sq")+V("dq")+(1-S("[]"))+V("ex"))^0)*"]", + sq="'"*(1-S("'"))^0*"'", + dq='"'*(1-S('"'))^0*'"', } - -local arguments = P { "ar", - ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", - nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, - sq = P("'") * (1 - P("'"))^0 * P("'"), - dq = P('"') * (1 - P('"'))^0 * P('"'), +local arguments=P { "ar", + ar="("*Cs((V("sq")+V("dq")+V("nq")+P(1-P(")")))^0)*")", + nq=((1-S("),'\""))^1)/function(s) return format("%q",s) end, + sq=P("'")*(1-P("'"))^0*P("'"), + dq=P('"')*(1-P('"'))^0*P('"'), } - --- todo: better arg parser - local function register_error(str) - return { kind = "error", error = format("unparsed: %s",str) } -end - --- there is a difference in * and /*/ and so we need to catch a few special cases - -local special_1 = P("*") * Cc(register_auto_descendant) * Cc(register_all_nodes) -- last one not needed -local special_2 = P("/") * Cc(register_auto_self) -local special_3 = P("") * Cc(register_auto_self) - -local no_nextcolon = P(-1) + #(1-P(":")) -- newer lpeg needs the P(-1) -local no_nextlparent = P(-1) + #(1-P("(")) -- newer lpeg needs the P(-1) - -local pathparser = Ct { "patterns", -- can be made a bit faster by moving some patterns outside - - patterns = spaces * V("protocol") * spaces * ( - ( V("special") * spaces * P(-1) ) + - ( V("initial") * spaces * V("step") * spaces * (P("/") * spaces * V("step") * spaces)^0 ) - ), - - protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), - - -- the / is needed for // as descendant or self is somewhat special - -- step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, - step = ((V("shortcuts") + P("/") + V("axis")) * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, - - axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + - V("descendant_or_self") + V("following_sibling") + V("following") + - V("reverse_sibling") + V("preceding_sibling") + V("preceding") + V("ancestor_or_self") + - #(1-P(-1)) * Cc(register_auto_child), - - special = special_1 + special_2 + special_3, - - initial = (P("/") * spaces * Cc(register_initial_child))^-1, - - error = (P(1)^1) / register_error, - - shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), - - shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, - - s_descendant_or_self = (P("***/") + P("/")) * Cc(register_descendant_or_self), --- *** is a bonus - s_descendant = P("**") * Cc(register_descendant), - s_child = P("*") * no_nextcolon * Cc(register_child ), - s_parent = P("..") * Cc(register_parent ), - s_self = P("." ) * Cc(register_self ), - s_root = P("^^") * Cc(register_root ), - s_ancestor = P("^") * Cc(register_ancestor ), - - descendant = P("descendant::") * Cc(register_descendant ), - child = P("child::") * Cc(register_child ), - parent = P("parent::") * Cc(register_parent ), - self = P("self::") * Cc(register_self ), - root = P('root::') * Cc(register_root ), - ancestor = P('ancestor::') * Cc(register_ancestor ), - descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), - ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), - -- attribute = P('attribute::') * Cc(register_attribute ), - -- namespace = P('namespace::') * Cc(register_namespace ), - following = P('following::') * Cc(register_following ), - following_sibling = P('following-sibling::') * Cc(register_following_sibling ), - preceding = P('preceding::') * Cc(register_preceding ), - preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), - reverse_sibling = P('reverse-sibling::') * Cc(register_reverse_sibling ), - - nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, - - expressions = expression / register_expression, - - letters = R("az")^1, - name = (1-S("/[]()|:*!"))^1, -- make inline - negate = P("!") * Cc(false), - - nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), - nodetest = V("negate") + Cc(true), - nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), - wildnodename = (C(V("name")) + P("*") * Cc(false)) * no_nextlparent, - nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, - - finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, - + return { kind="error",error=format("unparsed: %s",str) } +end +local special_1=P("*")*Cc(register_auto_descendant)*Cc(register_all_nodes) +local special_2=P("/")*Cc(register_auto_self) +local special_3=P("")*Cc(register_auto_self) +local no_nextcolon=P(-1)+#(1-P(":")) +local no_nextlparent=P(-1)+#(1-P("(")) +local pathparser=Ct { "patterns", + patterns=spaces*V("protocol")*spaces*( + (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) + ), + protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), + step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + axis=V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), + special=special_1+special_2+special_3, + initial=(P("/")*spaces*Cc(register_initial_child))^-1, + error=(P(1)^1)/register_error, + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, + s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), + s_descendant=P("**")*Cc(register_descendant), + s_child=P("*")*no_nextcolon*Cc(register_child ), + s_parent=P("..")*Cc(register_parent ), + s_self=P("." )*Cc(register_self ), + s_root=P("^^")*Cc(register_root ), + s_ancestor=P("^")*Cc(register_ancestor ), + descendant=P("descendant::")*Cc(register_descendant ), + child=P("child::")*Cc(register_child ), + parent=P("parent::")*Cc(register_parent ), + self=P("self::")*Cc(register_self ), + root=P('root::')*Cc(register_root ), + ancestor=P('ancestor::')*Cc(register_ancestor ), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), + following=P('following::')*Cc(register_following ), + following_sibling=P('following-sibling::')*Cc(register_following_sibling ), + preceding=P('preceding::')*Cc(register_preceding ), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), + nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, + expressions=expression/register_expression, + letters=R("az")^1, + name=(1-S("/[]()|:*!"))^1, + negate=P("!")*Cc(false), + nodefunction=V("negate")+P("not")*Cc(false)+Cc(true), + nodetest=V("negate")+Cc(true), + nodename=(V("negate")+Cc(true))*spaces*((V("wildnodename")*P(":")*V("wildnodename"))+(Cc(false)*V("wildnodename"))), + wildnodename=(C(V("name"))+P("*")*Cc(false))*no_nextlparent, + nodeset=spaces*Ct(V("nodename")*(spaces*P("|")*spaces*V("nodename"))^0)*spaces, + finalizer=(Cb("protocol")*P("/")^-1*C(V("name"))*arguments*P(-1))/register_finalizer, } - -xmlpatterns.pathparser = pathparser - -local cache = { } - +xmlpatterns.pathparser=pathparser +local cache={} local function nodesettostring(set,nodetest) - local t = { } - for i=1,#set,3 do - local directive, ns, tg = set[i], set[i+1], set[i+2] - if not ns or ns == "" then ns = "*" end - if not tg or tg == "" then tg = "*" end - tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) - t[i] = (directive and tg) or format("not(%s)",tg) - end - if nodetest == false then - return format("not(%s)",concat(t,"|")) - else - return concat(t,"|") - end + local t={} + for i=1,#set,3 do + local directive,ns,tg=set[i],set[i+1],set[i+2] + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + tg=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + t[i]=(directive and tg) or format("not(%s)",tg) + end + if nodetest==false then + return format("not(%s)",concat(t,"|")) + else + return concat(t,"|") + end end - local function tagstostring(list) - if #list == 0 then - return "no elements" - else - local t = { } - for i=1, #list do - local li = list[i] - local ns, tg = li.ns, li.tg - if not ns or ns == "" then ns = "*" end - if not tg or tg == "" then tg = "*" end - t[i] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) - end - return concat(t," ") - end -end - -xml.nodesettostring = nodesettostring - -local lpath -- we have a harmless kind of circular reference - -local lshowoptions = { functions = false } - + if #list==0 then + return "no elements" + else + local t={} + for i=1,#list do + local li=list[i] + local ns,tg=li.ns,li.tg + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + t[i]=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end +end +xml.nodesettostring=nodesettostring +local lpath +local lshowoptions={ functions=false } local function lshow(parsed) - if type(parsed) == "string" then - parsed = lpath(parsed) - end - report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, - table.serialize(parsed,false,lshowoptions)) + if type(parsed)=="string" then + parsed=lpath(parsed) + end + report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, + table.serialize(parsed,false,lshowoptions)) end - -xml.lshow = lshow - +xml.lshow=lshow local function add_comment(p,str) - local pc = p.comment - if not pc then - p.comment = { str } - else - pc[#pc+1] = str - end -end - -lpath = function (pattern) -- the gain of caching is rather minimal - lpathcalls = lpathcalls + 1 - if type(pattern) == "table" then - return pattern - else - local parsed = cache[pattern] - if parsed then - lpathcached = lpathcached + 1 - else - parsed = lpegmatch(pathparser,pattern) - if parsed then - parsed.pattern = pattern - local np = #parsed - if np == 0 then - parsed = { pattern = pattern, register_self, state = "parsing error" } - report_lpath("parsing error in '%s'",pattern) - lshow(parsed) - else - -- we could have done this with a more complex parser but this - -- is cleaner - local pi = parsed[1] - if pi.axis == "auto-child" then - if false then - add_comment(parsed, "auto-child replaced by auto-descendant-or-self") - parsed[1] = register_auto_descendant_or_self - else - add_comment(parsed, "auto-child replaced by auto-descendant") - parsed[1] = register_auto_descendant - end - elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then - add_comment(parsed, "initial-child removed") -- we could also make it a auto-self - remove(parsed,1) - end - local np = #parsed -- can have changed - if np > 1 then - local pnp = parsed[np] - if pnp.kind == "nodes" and pnp.nodetest == true then - local nodes = pnp.nodes - if nodes[1] == true and nodes[2] == false and nodes[3] == false then - add_comment(parsed, "redundant final wildcard filter removed") - remove(parsed,np) - end - end - end - end + local pc=p.comment + if not pc then + p.comment={ str } + else + pc[#pc+1]=str + end +end +lpath=function (pattern) + lpathcalls=lpathcalls+1 + if type(pattern)=="table" then + return pattern + else + local parsed=cache[pattern] + if parsed then + lpathcached=lpathcached+1 + else + parsed=lpegmatch(pathparser,pattern) + if parsed then + parsed.pattern=pattern + local np=#parsed + if np==0 then + parsed={ pattern=pattern,register_self,state="parsing error" } + report_lpath("parsing error in '%s'",pattern) + lshow(parsed) + else + local pi=parsed[1] + if pi.axis=="auto-child" then + if false then + add_comment(parsed,"auto-child replaced by auto-descendant-or-self") + parsed[1]=register_auto_descendant_or_self else - parsed = { pattern = pattern } + add_comment(parsed,"auto-child replaced by auto-descendant") + parsed[1]=register_auto_descendant end - cache[pattern] = parsed - if trace_lparse and not trace_lprofile then - lshow(parsed) + elseif pi.axis=="initial-child" and np>1 and parsed[2].axis then + add_comment(parsed,"initial-child removed") + remove(parsed,1) + end + local np=#parsed + if np>1 then + local pnp=parsed[np] + if pnp.kind=="nodes" and pnp.nodetest==true then + local nodes=pnp.nodes + if nodes[1]==true and nodes[2]==false and nodes[3]==false then + add_comment(parsed,"redundant final wildcard filter removed") + remove(parsed,np) + end end + end end - return parsed + else + parsed={ pattern=pattern } + end + cache[pattern]=parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) + end end + return parsed + end end - -xml.lpath = lpath - --- we can move all calls inline and then merge the trace back --- technically we can combine axis and the next nodes which is --- what we did before but this a bit cleaner (but slower too) --- but interesting is that it's not that much faster when we --- go inline --- --- beware: we need to return a collection even when we filter --- else the (simple) cache gets messed up - --- caching found lookups saves not that much (max .1 sec on a 8 sec run) --- and it also messes up finalizers - --- watch out: when there is a finalizer, it's always called as there --- can be cases that a finalizer returns (or does) something in case --- there is no match; an example of this is count() - -local profiled = { } xml.profiled = profiled - +xml.lpath=lpath +local profiled={} xml.profiled=profiled local function profiled_apply(list,parsed,nofparsed,order) - local p = profiled[parsed.pattern] - if p then - p.tested = p.tested + 1 - else - p = { tested = 1, matched = 0, finalized = 0 } - profiled[parsed.pattern] = p - end - local collected = list - for i=1,nofparsed do - local pi = parsed[i] - local kind = pi.kind - if kind == "axis" then - collected = apply_axis[pi.axis](collected) - elseif kind == "nodes" then - collected = apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind == "expression" then - collected = apply_expression(collected,pi.evaluator,order) - elseif kind == "finalizer" then - collected = pi.finalizer(collected) -- no check on # here - p.matched = p.matched + 1 - p.finalized = p.finalized + 1 - return collected - end - if not collected or #collected == 0 then - local pn = i < nofparsed and parsed[nofparsed] - if pn and pn.kind == "finalizer" then - collected = pn.finalizer(collected) - p.finalized = p.finalized + 1 - return collected - end - return nil - end - end - if collected then - p.matched = p.matched + 1 + local p=profiled[parsed.pattern] + if p then + p.tested=p.tested+1 + else + p={ tested=1,matched=0,finalized=0 } + profiled[parsed.pattern]=p + end + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + p.matched=p.matched+1 + p.finalized=p.finalized+1 + return collected + end + if not collected or #collected==0 then + local pn=i %s",(collected and #collected) or 0,pi.expression,pi.converted) - elseif kind == "finalizer" then - collected = pi.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") - return collected - end - if not collected or #collected == 0 then - local pn = i < nofparsed and parsed[nofparsed] - if pn and pn.kind == "finalizer" then - collected = pn.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") - return collected - end - return nil - end + if trace_lparse then + lshow(parsed) + end + report_lpath("collecting: %s",parsed.pattern) + report_lpath("root tags : %s",tagstostring(list)) + report_lpath("order : %s",order or "unset") + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected==0 then + local pn=i oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps oeps oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -xmlpatterns.escaped = escaped -xmlpatterns.unescaped = unescaped -xmlpatterns.cleansed = cleansed - -function xml.escaped (str) return lpegmatch(escaped,str) end + end +end +function xml.stripleadingspaces(dk,d,k) + if d and k then + local dkm=d[k-1] + if dkm and type(dkm)=="string" then + local s=match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") + end + end +end +local normal=(1-S("<&>"))^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local escaped=Cs(normal*(special*normal)^0) +local normal=(1-S"&")^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local unescaped=Cs(normal*(special*normal)^0) +local cleansed=Cs(((P("<")*(1-P(">"))^0*P(">"))/""+1)^0) +xmlpatterns.escaped=escaped +xmlpatterns.unescaped=unescaped +xmlpatterns.cleansed=cleansed +function xml.escaped (str) return lpegmatch(escaped,str) end function xml.unescaped(str) return lpegmatch(unescaped,str) end -function xml.cleansed (str) return lpegmatch(cleansed,str) end - --- this might move - +function xml.cleansed (str) return lpegmatch(cleansed,str) end function xml.fillin(root,pattern,str,check) - local e = xml.first(root,pattern) - if e then - local n = #e.dt - if not check or n == 0 or (n == 1 and e.dt[1] == "") then - e.dt = { str } - end + local e=xml.first(root,pattern) + if e then + local n=#e.dt + if not check or n==0 or (n==1 and e.dt[1]=="") then + e.dt={ str } end + end end @@ -12210,800 +9242,690 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-aux'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- not all functions here make sense anymore vbut we keep them for --- compatibility reasons - -local trace_manipulations = false trackers.register("lxml.manipulations", function(v) trace_manipulations = v end) - -local report_xml = logs.reporter("xml") - -local xml = xml - -local xmlconvert, xmlcopy, xmlname = xml.convert, xml.copy, xml.name -local xmlinheritedconvert = xml.inheritedconvert -local xmlapplylpath = xml.applylpath -local xmlfilter = xml.filter - -local type, setmetatable, getmetatable = type, setmetatable, getmetatable -local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat -local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip -local utfbyte = utf.byte +-- original size: 23813, stripped down to: 16826 +if not modules then modules={} end modules ['lxml-aux']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_manipulations=false trackers.register("lxml.manipulations",function(v) trace_manipulations=v end) +local report_xml=logs.reporter("xml") +local xml=xml +local xmlconvert,xmlcopy,xmlname=xml.convert,xml.copy,xml.name +local xmlinheritedconvert=xml.inheritedconvert +local xmlapplylpath=xml.applylpath +local xmlfilter=xml.filter +local type,setmetatable,getmetatable=type,setmetatable,getmetatable +local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat +local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip +local utfbyte=utf.byte local function report(what,pattern,c,e) - report_xml("%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) + report_xml("%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end - local function withelements(e,handle,depth) - if e and handle then - local edt = e.dt - if edt then - depth = depth or 0 - for i=1,#edt do - local e = edt[i] - if type(e) == "table" then - handle(e,depth) - withelements(e,handle,depth+1) - end - end + if e and handle then + local edt=e.dt + if edt then + depth=depth or 0 + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + handle(e,depth) + withelements(e,handle,depth+1) end + end end + end end - -xml.withelements = withelements - -function xml.withelement(e,n,handle) -- slow - if e and n ~= 0 and handle then - local edt = e.dt - if edt then - if n > 0 then - for i=1,#edt do - local ei = edt[i] - if type(ei) == "table" then - if n == 1 then - handle(ei) - return - else - n = n - 1 - end - end - end - elseif n < 0 then - for i=#edt,1,-1 do - local ei = edt[i] - if type(ei) == "table" then - if n == -1 then - handle(ei) - return - else - n = n + 1 - end - end - end +xml.withelements=withelements +function xml.withelement(e,n,handle) + if e and n~=0 and handle then + local edt=e.dt + if edt then + if n>0 then + for i=1,#edt do + local ei=edt[i] + if type(ei)=="table" then + if n==1 then + handle(ei) + return + else + n=n-1 + end + end + end + elseif n<0 then + for i=#edt,1,-1 do + local ei=edt[i] + if type(ei)=="table" then + if n==-1 then + handle(ei) + return + else + n=n+1 end + end end + end end + end end - function xml.each(root,pattern,handle,reverse) - local collected = xmlapplylpath(root,pattern) - if collected then - if reverse then - for c=#collected,1,-1 do - handle(collected[c]) - end - else - for c=1,#collected do - handle(collected[c]) - end - end - return collected + local collected=xmlapplylpath(root,pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end end + return collected + end end - function xml.processattributes(root,pattern,handle) - local collected = xmlapplylpath(root,pattern) - if collected and handle then - for c=1,#collected do - handle(collected[c].at) - end + local collected=xmlapplylpath(root,pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) end - return collected + end + return collected end - - - --- are these still needed -> lxml-cmp.lua - -function xml.collect(root, pattern) - return xmlapplylpath(root,pattern) +function xml.collect(root,pattern) + return xmlapplylpath(root,pattern) end - -function xml.collecttexts(root, pattern, flatten) -- todo: variant with handle - local collected = xmlapplylpath(root,pattern) - if collected and flatten then - local xmltostring = xml.tostring - for c=1,#collected do - collected[c] = xmltostring(collected[c].dt) - end +function xml.collecttexts(root,pattern,flatten) + local collected=xmlapplylpath(root,pattern) + if collected and flatten then + local xmltostring=xml.tostring + for c=1,#collected do + collected[c]=xmltostring(collected[c].dt) end - return collected or { } + end + return collected or {} end - -function xml.collect_tags(root, pattern, nonamespace) - local collected = xmlapplylpath(root,pattern) - if collected then - local t, n = { }, 0 - for c=1,#collected do - local e = collected[c] - local ns, tg = e.ns, e.tg - n = n + 1 - if nonamespace then - t[n] = tg - elseif ns == "" then - t[n] = tg - else - t[n] = ns .. ":" .. tg - end - end - return t +function xml.collect_tags(root,pattern,nonamespace) + local collected=xmlapplylpath(root,pattern) + if collected then + local t,n={},0 + for c=1,#collected do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace then + t[n]=tg + elseif ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg + end end + return t + end end - - - -local no_root = { no_root = true } - +local no_root={ no_root=true } local function redo_ni(d) - for k=1,#d do - local dk = d[k] - if type(dk) == "table" then - dk.ni = k - end + for k=1,#d do + local dk=d[k] + if type(dk)=="table" then + dk.ni=k end + end end - local function xmltoelement(whatever,root) - if not whatever then - return nil - end - local element - if type(whatever) == "string" then - element = xmlinheritedconvert(whatever,root) -- beware, not really a root - else - element = whatever -- we assume a table - end - if element.error then - return whatever -- string - end - if element then - end - return element -end - -xml.toelement = xmltoelement - + if not whatever then + return nil + end + local element + if type(whatever)=="string" then + element=xmlinheritedconvert(whatever,root) + else + element=whatever + end + if element.error then + return whatever + end + if element then + end + return element +end +xml.toelement=xmltoelement local function copiedelement(element,newparent) - if type(element) == "string" then - return element - else - element = xmlcopy(element).dt - if newparent and type(element) == "table" then - element.__p__ = newparent - end - return element + if type(element)=="string" then + return element + else + element=xmlcopy(element).dt + if newparent and type(element)=="table" then + element.__p__=newparent end + return element + end end - function xml.delete(root,pattern) - if not pattern or pattern == "" then - local p = root.__p__ + if not pattern or pattern=="" then + local p=root.__p__ + if p then + if trace_manipulations then + report('deleting',"--",c,root) + end + local d=p.dt + remove(d,root.ni) + redo_ni(d) + end + else + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ if p then - if trace_manipulations then - report('deleting',"--",c,root) - end - local d = p.dt - remove(d,root.ni) - redo_ni(d) -- can be made faster and inlined - end - else - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local p = e.__p__ - if p then - if trace_manipulations then - report('deleting',pattern,c,e) - end - local d = p.dt - remove(d,e.ni) - redo_ni(d) -- can be made faster and inlined - end - end + if trace_manipulations then + report('deleting',pattern,c,e) + end + local d=p.dt + remove(d,e.ni) + redo_ni(d) end + end end + end end - function xml.replace(root,pattern,whatever) - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local p = e.__p__ - if p then - if trace_manipulations then - report('replacing',pattern,c,e) - end - local d = p.dt - d[e.ni] = copiedelement(element,p) - redo_ni(d) -- probably not needed - end - end + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ + if p then + if trace_manipulations then + report('replacing',pattern,c,e) + end + local d=p.dt + d[e.ni]=copiedelement(element,p) + redo_ni(d) + end end + end end - local function wrap(e,wrapper) - local t = { - rn = e.rn, - tg = e.tg, - ns = e.ns, - at = e.at, - dt = e.dt, - __p__ = e, - } - setmetatable(t,getmetatable(e)) - e.rn = wrapper.rn or e.rn or "" - e.tg = wrapper.tg or e.tg or "" - e.ns = wrapper.ns or e.ns or "" - e.at = fastcopy(wrapper.at) - e.dt = { t } + local t={ + rn=e.rn, + tg=e.tg, + ns=e.ns, + at=e.at, + dt=e.dt, + __p__=e, + } + setmetatable(t,getmetatable(e)) + e.rn=wrapper.rn or e.rn or "" + e.tg=wrapper.tg or e.tg or "" + e.ns=wrapper.ns or e.ns or "" + e.at=fastcopy(wrapper.at) + e.dt={ t } end - function xml.wrap(root,pattern,whatever) - if whatever then - local wrapper = xmltoelement(whatever,root) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - if trace_manipulations then - report('wrapping',pattern,c,e) - end - wrap(e,wrapper) - end + if whatever then + local wrapper=xmltoelement(whatever,root) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if trace_manipulations then + report('wrapping',pattern,c,e) end - else - wrap(root,xmltoelement(pattern)) + wrap(e,wrapper) + end end + else + wrap(root,xmltoelement(pattern)) + end end - local function inject_element(root,pattern,whatever,prepend) - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - local function inject_e(e) - local r = e.__p__ - local d, k, rri = r.dt, e.ni, r.ri - local edt = (rri and d[rri].dt) or (d and d[k] and d[k].dt) - if edt then - local be, af - local cp = copiedelement(element,e) - if prepend then - be, af = cp, edt - else - be, af = edt, cp - end - local bn = #be - for i=1,#af do - bn = bn + 1 - be[bn] = af[i] - end - if rri then - r.dt[rri].dt = be - else - d[k].dt = be - end - redo_ni(d) - end - end - if not collected then - -- nothing - elseif collected.tg then - -- first or so - inject_e(collected) - else - for c=1,#collected do - inject_e(collected[c]) - end + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function inject_e(e) + local r=e.__p__ + local d,k,rri=r.dt,e.ni,r.ri + local edt=(rri and d[rri].dt) or (d and d[k] and d[k].dt) + if edt then + local be,af + local cp=copiedelement(element,e) + if prepend then + be,af=cp,edt + else + be,af=edt,cp + end + local bn=#be + for i=1,#af do + bn=bn+1 + be[bn]=af[i] + end + if rri then + r.dt[rri].dt=be + else + d[k].dt=be + end + redo_ni(d) end -end - -local function insert_element(root,pattern,whatever,before) -- todo: element als functie - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - local function insert_e(e) - local r = e.__p__ - local d, k = r.dt, e.ni - if not before then - k = k + 1 - end - insert(d,k,copiedelement(element,r)) - redo_ni(d) - end - if not collected then - -- nothing - elseif collected.tg then - -- first or so - insert_e(collected) - else - for c=1,#collected do - insert_e(collected[c]) - end + end + if not collected then + elseif collected.tg then + inject_e(collected) + else + for c=1,#collected do + inject_e(collected[c]) + end + end +end +local function insert_element(root,pattern,whatever,before) + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function insert_e(e) + local r=e.__p__ + local d,k=r.dt,e.ni + if not before then + k=k+1 + end + insert(d,k,copiedelement(element,r)) + redo_ni(d) + end + if not collected then + elseif collected.tg then + insert_e(collected) + else + for c=1,#collected do + insert_e(collected[c]) end + end end - -xml.insert_element = insert_element -xml.insertafter = insert_element -xml.insertbefore = function(r,p,e) insert_element(r,p,e,true) end -xml.injectafter = inject_element -xml.injectbefore = function(r,p,e) inject_element(r,p,e,true) end - +xml.insert_element=insert_element +xml.insertafter=insert_element +xml.insertbefore=function(r,p,e) insert_element(r,p,e,true) end +xml.injectafter=inject_element +xml.injectbefore=function(r,p,e) inject_element(r,p,e,true) end local function include(xmldata,pattern,attribute,recursive,loaddata) - -- parse="text" (default: xml), encoding="" (todo) - -- attribute = attribute or 'href' - pattern = pattern or 'include' - loaddata = loaddata or io.loaddata - local collected = xmlapplylpath(xmldata,pattern) - if collected then - for c=1,#collected do - local ek = collected[c] - local name = nil - local ekdt = ek.dt - local ekat = ek.at - local epdt = ek.__p__.dt - if not attribute or attribute == "" then - name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- check, probably always tab or str - end - if not name then - for a in gmatch(attribute or "href","([^|]+)") do - name = ekat[a] - if name then break end - end - end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - epdt[ek.ni] = "" -- xml.empty(d,k) - elseif ekat["parse"] == "text" then - -- for the moment hard coded - epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) - else - local xi = xmlinheritedconvert(data,xmldata) - if not xi then - epdt[ek.ni] = "" -- xml.empty(d,k) - else - if recursive then - include(xi,pattern,attribute,recursive,loaddata) - end - epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) - end - end + pattern=pattern or 'include' + loaddata=loaddata or io.loaddata + local collected=xmlapplylpath(xmldata,pattern) + if collected then + for c=1,#collected do + local ek=collected[c] + local name=nil + local ekdt=ek.dt + local ekat=ek.at + local epdt=ek.__p__.dt + if not attribute or attribute=="" then + name=(type(ekdt)=="table" and ekdt[1]) or ekdt + end + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name=ekat[a] + if name then break end end + end + local data=(name and name~="" and loaddata(name)) or "" + if data=="" then + epdt[ek.ni]="" + elseif ekat["parse"]=="text" then + epdt[ek.ni]=xml.escaped(data) + else + local xi=xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni]="" + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni]=xml.body(xi) + end + end end + end end - -xml.include = include - +xml.include=include local function stripelement(e,nolines,anywhere) - local edt = e.dt - if edt then - if anywhere then - local t, n = { }, 0 - for e=1,#edt do - local str = edt[e] - if type(str) ~= "string" then - n = n + 1 - t[n] = str - elseif str ~= "" then - -- todo: lpeg for each case - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s*(.-)%s*$","%1") - if str ~= "" then - n = n + 1 - t[n] = str - end - end - end - e.dt = t + local edt=e.dt + if edt then + if anywhere then + local t,n={},0 + for e=1,#edt do + local str=edt[e] + if type(str)~="string" then + n=n+1 + t[n]=str + elseif str~="" then + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"^%s*(.-)%s*$","%1") + if str~="" then + n=n+1 + t[n]=str + end + end + end + e.dt=t + else + if #edt>0 then + local str=edt[1] + if type(str)~="string" then + elseif str=="" then + remove(edt,1) else - -- we can assume a regular sparse xml table with no successive strings - -- otherwise we should use a while loop - if #edt > 0 then - -- strip front - local str = edt[1] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt,1) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s+","") - if str == "" then - remove(edt,1) - else - edt[1] = str - end - end - end - local nedt = #edt - if nedt > 0 then - -- strip end - local str = edt[nedt] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"%s+$","") - if str == "" then - remove(edt) - else - edt[nedt] = str - end - end - end + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"^%s+","") + if str=="" then + remove(edt,1) + else + edt[1]=str + end + end + end + local nedt=#edt + if nedt>0 then + local str=edt[nedt] + if type(str)~="string" then + elseif str=="" then + remove(edt) + else + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"%s+$","") + if str=="" then + remove(edt) + else + edt[nedt]=str + end end + end end - return e -- convenient + end + return e end - -xml.stripelement = stripelement - -function xml.strip(root,pattern,nolines,anywhere) -- strips all leading and trailing spacing - local collected = xmlapplylpath(root,pattern) -- beware, indices no longer are valid now - if collected then - for i=1,#collected do - stripelement(collected[i],nolines,anywhere) - end +xml.stripelement=stripelement +function xml.strip(root,pattern,nolines,anywhere) + local collected=xmlapplylpath(root,pattern) + if collected then + for i=1,#collected do + stripelement(collected[i],nolines,anywhere) end + end end - -local function renamespace(root, oldspace, newspace) -- fast variant - local ndt = #root.dt - for i=1,ndt or 0 do - local e = root[i] - if type(e) == "table" then - if e.ns == oldspace then - e.ns = newspace - if e.rn then - e.rn = newspace - end - end - local edt = e.dt - if edt then - renamespace(edt, oldspace, newspace) - end +local function renamespace(root,oldspace,newspace) + local ndt=#root.dt + for i=1,ndt or 0 do + local e=root[i] + if type(e)=="table" then + if e.ns==oldspace then + e.ns=newspace + if e.rn then + e.rn=newspace end + end + local edt=e.dt + if edt then + renamespace(edt,oldspace,newspace) + end end + end end - -xml.renamespace = renamespace - -function xml.remaptag(root, pattern, newtg) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - collected[c].tg = newtg - end +xml.renamespace=renamespace +function xml.remaptag(root,pattern,newtg) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].tg=newtg end + end end - -function xml.remapnamespace(root, pattern, newns) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - collected[c].ns = newns - end +function xml.remapnamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].ns=newns end + end end - -function xml.checknamespace(root, pattern, newns) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - if (not e.rn or e.rn == "") and e.ns == "" then - e.rn = newns - end - end +function xml.checknamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if (not e.rn or e.rn=="") and e.ns=="" then + e.rn=newns + end end + end end - -function xml.remapname(root, pattern, newtg, newns, newrn) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - e.tg, e.ns, e.rn = newtg, newns, newrn - end +function xml.remapname(root,pattern,newtg,newns,newrn) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + e.tg,e.ns,e.rn=newtg,newns,newrn end + end end - - - function xml.cdatatotext(e) - local dt = e.dt - if #dt == 1 then - local first = dt[1] - if first.tg == "@cd@" then - e.dt = first.dt - end - else - -- maybe option - end -end - --- local x = xml.convert("123") --- xml.texttocdata(xml.first(x,"a")) --- print(x) -- 23]]> - -function xml.texttocdata(e) -- could be a finalizer - local dt = e.dt - local s = xml.tostring(dt) -- no shortcut? - e.tg = "@cd@" - e.special = true - e.ns = "" - e.rn = "" - e.dt = { s } - e.at = nil -end - --- local x = xml.convert("123") --- xml.tocdata(xml.first(x,"a")) --- print(x) -- 123]]> - -function xml.elementtocdata(e) -- could be a finalizer - local dt = e.dt - local s = xml.tostring(e) -- no shortcut? - e.tg = "@cd@" - e.special = true - e.ns = "" - e.rn = "" - e.dt = { s } - e.at = nil -end - -xml.builtinentities = table.tohash { "amp", "quot", "apos", "lt", "gt" } -- used often so share - -local entities = characters and characters.entities or nil -local builtinentities = xml.builtinentities - -function xml.addentitiesdoctype(root,option) -- we could also have a 'resolve' i.e. inline hex - if not entities then - require("char-ent") - entities = characters.entities - end - if entities and root and root.tg == "@rt@" and root.statistics then - local list = { } - local hexify = option == "hexadecimal" - for k, v in table.sortedhash(root.statistics.entities.names) do - if not builtinentities[k] then - local e = entities[k] - if not e then - e = format("[%s]",k) - elseif hexify then - e = format("&#%05X;",utfbyte(k)) - end - list[#list+1] = format(" ",k,e) - end - end - local dt = root.dt - local n = dt[1].tg == "@pi@" and 2 or 1 - if #list > 0 then - insert(dt, n, { "\n" }) - insert(dt, n, { - tg = "@dt@", -- beware, doctype is unparsed - dt = { format("Something [\n%s\n] ",concat(list)) }, - ns = "", - special = true, - }) - insert(dt, n, { "\n\n" }) - else - -- insert(dt, n, { table.serialize(root.statistics) }) - end + local dt=e.dt + if #dt==1 then + local first=dt[1] + if first.tg=="@cd@" then + e.dt=first.dt + end + else + end +end +function xml.texttocdata(e) + local dt=e.dt + local s=xml.tostring(dt) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +function xml.elementtocdata(e) + local dt=e.dt + local s=xml.tostring(e) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +xml.builtinentities=table.tohash { "amp","quot","apos","lt","gt" } +local entities=characters and characters.entities or nil +local builtinentities=xml.builtinentities +function xml.addentitiesdoctype(root,option) + if not entities then + require("char-ent") + entities=characters.entities + end + if entities and root and root.tg=="@rt@" and root.statistics then + local list={} + local hexify=option=="hexadecimal" + for k,v in table.sortedhash(root.statistics.entities.names) do + if not builtinentities[k] then + local e=entities[k] + if not e then + e=format("[%s]",k) + elseif hexify then + e=format("&#%05X;",utfbyte(k)) + end + list[#list+1]=format(" ",k,e) + end end -end - --- local str = [==[ --- --- --- test   test { test --- --- --- ]==] --- --- local x = xml.convert(str) --- xml.addentitiesdoctype(x,"hexadecimal") --- print(x) - - - -xml.all = xml.each -xml.insert = xml.insertafter -xml.inject = xml.injectafter -xml.after = xml.insertafter -xml.before = xml.insertbefore -xml.process = xml.each - --- obsolete - -xml.obsolete = xml.obsolete or { } -local obsolete = xml.obsolete - -xml.strip_whitespace = xml.strip obsolete.strip_whitespace = xml.strip -xml.collect_elements = xml.collect obsolete.collect_elements = xml.collect -xml.delete_element = xml.delete obsolete.delete_element = xml.delete -xml.replace_element = xml.replace obsolete.replace_element = xml.replacet -xml.each_element = xml.each obsolete.each_element = xml.each -xml.process_elements = xml.process obsolete.process_elements = xml.process -xml.insert_element_after = xml.insertafter obsolete.insert_element_after = xml.insertafter -xml.insert_element_before = xml.insertbefore obsolete.insert_element_before = xml.insertbefore -xml.inject_element_after = xml.injectafter obsolete.inject_element_after = xml.injectafter -xml.inject_element_before = xml.injectbefore obsolete.inject_element_before = xml.injectbefore -xml.process_attributes = xml.processattributes obsolete.process_attributes = xml.processattributes -xml.collect_texts = xml.collecttexts obsolete.collect_texts = xml.collecttexts -xml.inject_element = xml.inject obsolete.inject_element = xml.inject -xml.remap_tag = xml.remaptag obsolete.remap_tag = xml.remaptag -xml.remap_name = xml.remapname obsolete.remap_name = xml.remapname -xml.remap_namespace = xml.remapnamespace obsolete.remap_namespace = xml.remapnamespace - --- new (probably ok) - + local dt=root.dt + local n=dt[1].tg=="@pi@" and 2 or 1 + if #list>0 then + insert(dt,n,{ "\n" }) + insert(dt,n,{ + tg="@dt@", + dt={ format("Something [\n%s\n] ",concat(list)) }, + ns="", + special=true, + }) + insert(dt,n,{ "\n\n" }) + else + end + end +end +xml.all=xml.each +xml.insert=xml.insertafter +xml.inject=xml.injectafter +xml.after=xml.insertafter +xml.before=xml.insertbefore +xml.process=xml.each +xml.obsolete=xml.obsolete or {} +local obsolete=xml.obsolete +xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip +xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect +xml.delete_element=xml.delete obsolete.delete_element=xml.delete +xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.each_element=xml.each obsolete.each_element=xml.each +xml.process_elements=xml.process obsolete.process_elements=xml.process +xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter +xml.insert_element_before=xml.insertbefore obsolete.insert_element_before=xml.insertbefore +xml.inject_element_after=xml.injectafter obsolete.inject_element_after=xml.injectafter +xml.inject_element_before=xml.injectbefore obsolete.inject_element_before=xml.injectbefore +xml.process_attributes=xml.processattributes obsolete.process_attributes=xml.processattributes +xml.collect_texts=xml.collecttexts obsolete.collect_texts=xml.collecttexts +xml.inject_element=xml.inject obsolete.inject_element=xml.inject +xml.remap_tag=xml.remaptag obsolete.remap_tag=xml.remaptag +xml.remap_name=xml.remapname obsolete.remap_name=xml.remapname +xml.remap_namespace=xml.remapnamespace obsolete.remap_namespace=xml.remapnamespace function xml.cdata(e) - if e then - local dt = e.dt - if dt and #dt == 1 then - local first = dt[1] - return first.tg == "@cd@" and first.dt[1] or "" - end + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" end - return "" + end + return "" end - function xml.finalizers.xml.cdata(collected) - if collected then - local e = collected[1] - if e then - local dt = e.dt - if dt and #dt == 1 then - local first = dt[1] - return first.tg == "@cd@" and first.dt[1] or "" - end - end + if collected then + local e=collected[1] + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" + end end - return "" -end - -function xml.insertcomment(e,str,n) -- also insertcdata - table.insert(e.dt,n or 1,{ - tg = "@cm@", - ns = "", - special = true, - at = { }, - dt = { str }, - }) -end - -function xml.setcdata(e,str) -- also setcomment - e.dt = { { - tg = "@cd@", - ns = "", - special = true, - at = { }, - dt = { str }, - } } + end + return "" +end +function xml.insertcomment(e,str,n) + table.insert(e.dt,n or 1,{ + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.setcdata(e,str) + e.dt={ { + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + } } end - --- maybe helpers like this will move to an autoloader - function xml.separate(x,pattern) - local collected = xmlapplylpath(x,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local d = e.dt - if d == x then - report_xml("warning: xml.separate changes root") - x = d - end - local t, n = { "\n" }, 1 - local i, nd = 1, #d - while i <= nd do - while i <= nd do - local di = d[i] - if type(di) == "string" then - if di == "\n" or find(di,"^%s+$") then -- first test is speedup - i = i + 1 - else - d[i] = strip(di) - break - end - else - break - end - end - if i > nd then - break - end - t[n+1] = "\n" - t[n+2] = d[i] - t[n+3] = "\n" - n = n + 3 - i = i + 1 + local collected=xmlapplylpath(x,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local d=e.dt + if d==x then + report_xml("warning: xml.separate changes root") + x=d + end + local t,n={ "\n" },1 + local i,nd=1,#d + while i<=nd do + while i<=nd do + local di=d[i] + if type(di)=="string" then + if di=="\n" or find(di,"^%s+$") then + i=i+1 + else + d[i]=strip(di) + break end - t[n+1] = "\n" - setmetatable(t,getmetatable(d)) - e.dt = t + else + break + end + end + if i>nd then + break end + t[n+1]="\n" + t[n+2]=d[i] + t[n+3]="\n" + n=n+3 + i=i+1 + end + t[n+1]="\n" + setmetatable(t,getmetatable(d)) + e.dt=t end - return x + end + return x end - --- - -local helpers = xml.helpers or { } -xml.helpers = helpers - +local helpers=xml.helpers or {} +xml.helpers=helpers local function normal(e,action) - local edt = e.dt - if edt then - for i=1,#edt do - local str = edt[i] - if type(str) == "string" and str ~= "" then - edt[i] = action(str) - end - end + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)=="string" and str~="" then + edt[i]=action(str) + end end + end end - local function recurse(e,action) - local edt = e.dt - if edt then - for i=1,#edt do - local str = edt[i] - if type(str) ~= "string" then - recurse(str,action,recursive) - elseif str ~= "" then - edt[i] = action(str) - end - end + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)~="string" then + recurse(str,action,recursive) + elseif str~="" then + edt[i]=action(str) + end end + end end - function helpers.recursetext(collected,action,recursive) - if recursive then - for i=1,#collected do - recurse(collected[i],action) - end - else - for i=1,#collected do - normal(collected[i],action) - end + if recursive then + for i=1,#collected do + recurse(collected[i],action) + end + else + for i=1,#collected do + normal(collected[i],action) end + end end @@ -13011,450 +9933,375 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-xml'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local concat = table.concat -local find, lower, upper = string.find, string.lower, string.upper - -local xml = xml +-- original size: 10274, stripped down to: 7538 -local finalizers = xml.finalizers.xml -local xmlfilter = xml.filter -- we could inline this one for speed -local xmltostring = xml.tostring -local xmlserialize = xml.serialize -local xmlcollected = xml.collected -local xmlnewhandlers = xml.newhandlers - -local function first(collected) -- wrong ? - return collected and collected[1] +if not modules then modules={} end modules ['lxml-xml']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat=table.concat +local find,lower,upper=string.find,string.lower,string.upper +local xml=xml +local finalizers=xml.finalizers.xml +local xmlfilter=xml.filter +local xmltostring=xml.tostring +local xmlserialize=xml.serialize +local xmlcollected=xml.collected +local xmlnewhandlers=xml.newhandlers +local function first(collected) + return collected and collected[1] end - local function last(collected) - return collected and collected[#collected] + return collected and collected[#collected] end - local function all(collected) - return collected + return collected end - --- local function reverse(collected) --- if collected then --- local nc = #collected --- if nc > 0 then --- local reversed, r = { }, 0 --- for c=nc,1,-1 do --- r = r + 1 --- reversed[r] = collected[c] --- end --- return reversed --- else --- return collected --- end --- end --- end - -local reverse = table.reversed - +local reverse=table.reversed local function attribute(collected,name) - if collected and #collected > 0 then - local at = collected[1].at - return at and at[name] - end + if collected and #collected>0 then + local at=collected[1].at + return at and at[name] + end end - local function att(id,name) - local at = id.at - return at and at[name] + local at=id.at + return at and at[name] end - local function count(collected) - return collected and #collected or 0 + return collected and #collected or 0 end - local function position(collected,n) - if not collected then - return 0 - end - local nc = #collected - if nc == 0 then - return 0 - end - n = tonumber(n) or 0 - if n < 0 then - return collected[nc + n + 1] - elseif n > 0 then - return collected[n] - else - return collected[1].mi or 0 - end + if not collected then + return 0 + end + local nc=#collected + if nc==0 then + return 0 + end + n=tonumber(n) or 0 + if n<0 then + return collected[nc+n+1] + elseif n>0 then + return collected[n] + else + return collected[1].mi or 0 + end end - local function match(collected) - return collected and #collected > 0 and collected[1].mi or 0 -- match + return collected and #collected>0 and collected[1].mi or 0 end - local function index(collected) - return collected and #collected > 0 and collected[1].ni or 0 -- 0 is new + return collected and #collected>0 and collected[1].ni or 0 end - local function attributes(collected,arguments) - if collected and #collected > 0 then - local at = collected[1].at - if arguments then - return at[arguments] - elseif next(at) then - return at -- all of them + if collected and #collected>0 then + local at=collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at + end + end +end +local function chainattribute(collected,arguments) + if collected and #collected>0 then + local e=collected[1] + while e do + local at=e.at + if at then + local a=at[arguments] + if a then + return a end + else + break + end + e=e.__p__ end + end + return "" end - -local function chainattribute(collected,arguments) -- todo: optional levels - if collected and #collected > 0 then - local e = collected[1] - while e do - local at = e.at - if at then - local a = at[arguments] - if a then - return a - end - else - break -- error - end - e = e.__p__ - end - end +local function raw(collected) + if collected and #collected>0 then + local e=collected[1] or collected + return e and xmltostring(e) or "" + else return "" + end end - -local function raw(collected) -- hybrid (not much different from text so it might go) - if collected and #collected > 0 then - local e = collected[1] or collected - return e and xmltostring(e) or "" -- only first as we cannot concat function - else - return "" - end -end - --- - -local xmltexthandler = xmlnewhandlers { - name = "string", - initialize = function() - result = { } - return result - end, - finalize = function() - return concat(result) - end, - handle = function(...) - result[#result+1] = concat { ... } - end, - escape = false, +local xmltexthandler=xmlnewhandlers { + name="string", + initialize=function() + result={} + return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, + escape=false, } - local function xmltotext(root) - local dt = root.dt - if not dt then - return "" - end - local nt = #dt -- string or table - if nt == 0 then - return "" - elseif nt == 1 and type(dt[1]) == "string" then - return dt[1] -- no escaping of " ' < > & - else - return xmlserialize(root,xmltexthandler) or "" - end -end - --- - -local function text(collected) -- hybrid - if collected then -- no # test here ! - local e = collected[1] or collected -- why fallback to element, how about cdata - return e and xmltotext(e) or "" - else - return "" - end + local dt=root.dt + if not dt then + return "" + end + local nt=#dt + if nt==0 then + return "" + elseif nt==1 and type(dt[1])=="string" then + return dt[1] + else + return xmlserialize(root,xmltexthandler) or "" + end +end +local function text(collected) + if collected then + local e=collected[1] or collected + return e and xmltotext(e) or "" + else + return "" + end end - local function texts(collected) - if not collected then - return { } -- why no nil - end - local nc = #collected - if nc == 0 then - return { } -- why no nil - end - local t, n = { }, 0 - for c=1,nc do - local e = collected[c] - if e and e.dt then - n = n + 1 - t[n] = e.dt - end - end - return t + if not collected then + return {} + end + local nc=#collected + if nc==0 then + return {} + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + if e and e.dt then + n=n+1 + t[n]=e.dt + end + end + return t end - local function tag(collected,n) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local c - if n == 0 or not n then - c = collected[1] - elseif n > 1 then - c = collected[n] - else - c = collected[nc-n+1] - end - return c and c.tg + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + return c and c.tg end - local function name(collected,n) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local c - if n == 0 or not n then - c = collected[1] - elseif n > 1 then - c = collected[n] - else - c = collected[nc-n+1] - end - if not c then - -- sorry - elseif c.ns == "" then - return c.tg - else - return c.ns .. ":" .. c.tg - end + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + if not c then + elseif c.ns=="" then + return c.tg + else + return c.ns..":"..c.tg + end end - local function tags(collected,nonamespace) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local t, n = { }, 0 - for c=1,nc do - local e = collected[c] - local ns, tg = e.ns, e.tg - n = n + 1 - if nonamespace or ns == "" then - t[n] = tg - else - t[n] = ns .. ":" .. tg - end + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace or ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg end - return t + end + return t end - local function empty(collected,spacesonly) - if not collected then - return true - end - local nc = #collected - if nc == 0 then - return true - end - for c=1,nc do - local e = collected[c] - if e then - local edt = e.dt - if edt then - local n = #edt - if n == 1 then - local edk = edt[1] - local typ = type(edk) - if typ == "table" then - return false - elseif edk ~= "" then - return false - elseif spacesonly and not find(edk,"%S") then - return false - end - elseif n > 1 then - return false - end - end + if not collected then + return true + end + local nc=#collected + if nc==0 then + return true + end + for c=1,nc do + local e=collected[c] + if e then + local edt=e.dt + if edt then + local n=#edt + if n==1 then + local edk=edt[1] + local typ=type(edk) + if typ=="table" then + return false + elseif edk~="" then + return false + elseif spacesonly and not find(edk,"%S") then + return false + end + elseif n>1 then + return false end + end end - return true -end - -finalizers.first = first -finalizers.last = last -finalizers.all = all -finalizers.reverse = reverse -finalizers.elements = all -finalizers.default = all -finalizers.attribute = attribute -finalizers.att = att -finalizers.count = count -finalizers.position = position -finalizers.match = match -finalizers.index = index -finalizers.attributes = attributes -finalizers.chainattribute = chainattribute -finalizers.text = text -finalizers.texts = texts -finalizers.tag = tag -finalizers.name = name -finalizers.tags = tags -finalizers.empty = empty - --- shortcuts -- we could support xmlfilter(id,pattern,first) - + end + return true +end +finalizers.first=first +finalizers.last=last +finalizers.all=all +finalizers.reverse=reverse +finalizers.elements=all +finalizers.default=all +finalizers.attribute=attribute +finalizers.att=att +finalizers.count=count +finalizers.position=position +finalizers.match=match +finalizers.index=index +finalizers.attributes=attributes +finalizers.chainattribute=chainattribute +finalizers.text=text +finalizers.texts=texts +finalizers.tag=tag +finalizers.name=name +finalizers.tags=tags +finalizers.empty=empty function xml.first(id,pattern) - return first(xmlfilter(id,pattern)) + return first(xmlfilter(id,pattern)) end - function xml.last(id,pattern) - return last(xmlfilter(id,pattern)) + return last(xmlfilter(id,pattern)) end - function xml.count(id,pattern) - return count(xmlfilter(id,pattern)) + return count(xmlfilter(id,pattern)) end - function xml.attribute(id,pattern,a,default) - return attribute(xmlfilter(id,pattern),a,default) + return attribute(xmlfilter(id,pattern),a,default) end - function xml.raw(id,pattern) - if pattern then - return raw(xmlfilter(id,pattern)) - else - return raw(id) - end -end - -function xml.text(id,pattern) -- brrr either content or element (when cdata) - if pattern then - -- return text(xmlfilter(id,pattern)) - local collected = xmlfilter(id,pattern) - return collected and #collected > 0 and xmltotext(collected[1]) or "" - elseif id then - -- return text(id) - return xmltotext(id) or "" - else - return "" - end + if pattern then + return raw(xmlfilter(id,pattern)) + else + return raw(id) + end +end +function xml.text(id,pattern) + if pattern then + local collected=xmlfilter(id,pattern) + return collected and #collected>0 and xmltotext(collected[1]) or "" + elseif id then + return xmltotext(id) or "" + else + return "" + end end - -xml.content = text - --- - -function xml.position(id,pattern,n) -- element - return position(xmlfilter(id,pattern),n) +xml.content=text +function xml.position(id,pattern,n) + return position(xmlfilter(id,pattern),n) end - -function xml.match(id,pattern) -- number - return match(xmlfilter(id,pattern)) +function xml.match(id,pattern) + return match(xmlfilter(id,pattern)) end - function xml.empty(id,pattern,spacesonly) - return empty(xmlfilter(id,pattern),spacesonly) + return empty(xmlfilter(id,pattern),spacesonly) end - -xml.all = xml.filter -xml.index = xml.position -xml.found = xml.filter - --- a nice one: - +xml.all=xml.filter +xml.index=xml.position +xml.found=xml.filter local function totable(x) - local t = { } - for e in xmlcollected(x[1] or x,"/*") do - t[e.tg] = xmltostring(e.dt) or "" - end - return next(t) and t or nil -end - -xml.table = totable -finalizers.table = totable - + local t={} + for e in xmlcollected(x[1] or x,"/*") do + t[e.tg]=xmltostring(e.dt) or "" + end + return next(t) and t or nil +end +xml.table=totable +finalizers.table=totable local function textonly(e,t) - if e then - local edt = e.dt - if edt then - for i=1,#edt do - local e = edt[i] - if type(e) == "table" then - textonly(e,t) - else - t[#t+1] = e - end - end + if e then + local edt=e.dt + if edt then + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + textonly(e,t) + else + t[#t+1]=e end + end end - return t + end + return t end - -function xml.textonly(e) -- no pattern - return concat(textonly(e,{})) +function xml.textonly(e) + return concat(textonly(e,{})) end - --- - --- local x = xml.convert("123") --- xml.filter(x,"**/lowerall()") print(x) --- xml.filter(x,"**/upperall()") print(x) - function finalizers.lowerall(collected) - for c=1,#collected do - local e = collected[c] - if not e.special then - e.tg = lower(e.tg) - local eat = e.at - if eat then - local t = { } - for k,v in next, eat do - t[lower(k)] = v - end - e.at = t - end - end + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=lower(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[lower(k)]=v + end + e.at=t + end end + end end - function finalizers.upperall(collected) - for c=1,#collected do - local e = collected[c] - if not e.special then - e.tg = upper(e.tg) - local eat = e.at - if eat then - local t = { } - for k,v in next, eat do - t[upper(k)] = v - end - e.at = t - end - end + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=upper(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[upper(k)]=v + end + e.at=t + end end + end end @@ -13462,237 +10309,159 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-ini'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - -local gsub, find, gmatch, char = string.gsub, string.find, string.gmatch, string.char -local next, type = next, type - -local filedirname, filebasename, filejoin = file.dirname, file.basename, file.join - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_initialization = logs.reporter("resolvers","initialization") +-- original size: 7894, stripped down to: 5497 -local ostype, osname, ossetenv, osgetenv = os.type, os.name, os.setenv, os.getenv - --- The code here used to be part of a data-res but for convenience --- we now split it over multiple files. As this file is now the --- starting point we introduce resolvers here. - -resolvers = resolvers or { } -local resolvers = resolvers - --- We don't want the kpse library to kick in. Also, we want to be able to --- execute programs. Control over execution is implemented later. - -texconfig.kpse_init = false -texconfig.shell_escape = 't' - -if kpse and kpse.default_texmfcnf then - local default_texmfcnf = kpse.default_texmfcnf() - -- looks more like context: - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTODIR","selfautodir:") - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTOPARENT","selfautoparent:") - default_texmfcnf = gsub(default_texmfcnf,"$HOME","home:") - -- - environment.default_texmfcnf = default_texmfcnf -end - -kpse = { original = kpse } - -setmetatable(kpse, { - __index = function(kp,name) - report_initialization("fatal error: kpse library is accessed (key: %s)",name) - os.exit() - end +if not modules then modules={} end modules ['data-ini']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char +local next,type=next,type +local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_initialization=logs.reporter("resolvers","initialization") +local ostype,osname,ossetenv,osgetenv=os.type,os.name,os.setenv,os.getenv +resolvers=resolvers or {} +local resolvers=resolvers +texconfig.kpse_init=false +texconfig.shell_escape='t' +if not (environment and environment.default_texmfcnf) and kpse and kpse.default_texmfcnf then + local default_texmfcnf=kpse.default_texmfcnf() + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTODIR","selfautodir:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOPARENT","selfautoparent:") + default_texmfcnf=gsub(default_texmfcnf,"$HOME","home:") + environment.default_texmfcnf=default_texmfcnf +end +kpse={ original=kpse } +setmetatable(kpse,{ + __index=function(kp,name) + report_initialization("fatal error: kpse library is accessed (key: %s)",name) + os.exit() + end } ) - --- First we check a couple of environment variables. Some might be --- set already but we need then later on. We start with the system --- font path. - do - - local osfontdir = osgetenv("OSFONTDIR") - - if osfontdir and osfontdir ~= "" then - -- ok - elseif osname == "windows" then - ossetenv("OSFONTDIR","c:/windows/fonts//") - elseif osname == "macosx" then - ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") - end - + local osfontdir=osgetenv("OSFONTDIR") + if osfontdir and osfontdir~="" then + elseif osname=="windows" then + ossetenv("OSFONTDIR","c:/windows/fonts//") + elseif osname=="macosx" then + ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + end end - --- Next comes the user's home path. We need this as later on we have --- to replace ~ with its value. - do - - local homedir = osgetenv(ostype == "windows" and 'USERPROFILE' or 'HOME') or '' - - if not homedir or homedir == "" then - homedir = char(127) -- we need a value, later we wil trigger on it - end - - homedir = file.collapsepath(homedir) - - ossetenv("HOME", homedir) -- can be used in unix cnf files - ossetenv("USERPROFILE",homedir) -- can be used in windows cnf files - - environment.homedir = homedir - + local homedir=osgetenv(ostype=="windows" and 'USERPROFILE' or 'HOME') or '' + if not homedir or homedir=="" then + homedir=char(127) + end + homedir=file.collapsepath(homedir) + ossetenv("HOME",homedir) + ossetenv("USERPROFILE",homedir) + environment.homedir=homedir end - --- The following code sets the name of the own binary and its --- path. This is fallback code as we have os.selfdir now. - do - - local args = environment.originalarguments or arg -- this needs a cleanup - - if not environment.ownmain then - environment.ownmain = status and string.match(string.lower(status.banner),"this is ([%a]+)") or "luatex" - end - - local ownbin = environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" - local ownpath = environment.ownpath or os.selfdir - - ownbin = file.collapsepath(ownbin) - ownpath = file.collapsepath(ownpath) - - if not ownpath or ownpath == "" or ownpath == "unset" then - ownpath = args[-1] or arg[-1] - ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) - if not ownpath or ownpath == "" then - ownpath = args[-0] or arg[-0] - ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) - end - local binary = ownbin - if not ownpath or ownpath == "" then - ownpath = ownpath and filedirname(binary) - end - if not ownpath or ownpath == "" then - if os.binsuffix ~= "" then - binary = file.replacesuffix(binary,os.binsuffix) - end - local path = osgetenv("PATH") - if path then - for p in gmatch(path,"[^"..io.pathseparator.."]+") do - local b = filejoin(p,binary) - if lfs.isfile(b) then - -- we assume that after changing to the path the currentdir function - -- resolves to the real location and use this side effect here; this - -- trick is needed because on the mac installations use symlinks in the - -- path instead of real locations - local olddir = lfs.currentdir() - if lfs.chdir(p) then - local pp = lfs.currentdir() - if trace_locating and p ~= pp then - report_initialization("following symlink '%s' to '%s'",p,pp) - end - ownpath = pp - lfs.chdir(olddir) - else - if trace_locating then - report_initialization("unable to check path '%s'",p) - end - ownpath = p - end - break - end - end + local args=environment.originalarguments or arg + if not environment.ownmain then + environment.ownmain=status and string.match(string.lower(status.banner),"this is ([%a]+)") or "luatex" + end + local ownbin=environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" + local ownpath=environment.ownpath or os.selfdir + ownbin=file.collapsepath(ownbin) + ownpath=file.collapsepath(ownpath) + if not ownpath or ownpath=="" or ownpath=="unset" then + ownpath=args[-1] or arg[-1] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath=="" then + ownpath=args[-0] or arg[-0] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + end + local binary=ownbin + if not ownpath or ownpath=="" then + ownpath=ownpath and filedirname(binary) + end + if not ownpath or ownpath=="" then + if os.binsuffix~="" then + binary=file.replacesuffix(binary,os.binsuffix) + end + local path=osgetenv("PATH") + if path then + for p in gmatch(path,"[^"..io.pathseparator.."]+") do + local b=filejoin(p,binary) + if lfs.isfile(b) then + local olddir=lfs.currentdir() + if lfs.chdir(p) then + local pp=lfs.currentdir() + if trace_locating and p~=pp then + report_initialization("following symlink '%s' to '%s'",p,pp) + end + ownpath=pp + lfs.chdir(olddir) + else + if trace_locating then + report_initialization("unable to check path '%s'",p) + end + ownpath=p end + break + end end - if not ownpath or ownpath == "" then - ownpath = "." - report_initialization("forcing fallback ownpath .") - elseif trace_locating then - report_initialization("using ownpath '%s'",ownpath) - end + end end - - environment.ownbin = ownbin - environment.ownpath = ownpath - + if not ownpath or ownpath=="" then + ownpath="." + report_initialization("forcing fallback ownpath .") + elseif trace_locating then + report_initialization("using ownpath '%s'",ownpath) + end + end + environment.ownbin=ownbin + environment.ownpath=ownpath end - -resolvers.ownpath = environment.ownpath - +resolvers.ownpath=environment.ownpath function resolvers.getownpath() - return environment.ownpath + return environment.ownpath end - --- The self variables permit us to use only a few (or even no) --- environment variables. - do - - local ownpath = environment.ownpath or dir.current() - - if ownpath then - ossetenv('SELFAUTOLOC', file.collapsepath(ownpath)) - ossetenv('SELFAUTODIR', file.collapsepath(ownpath .. "/..")) - ossetenv('SELFAUTOPARENT', file.collapsepath(ownpath .. "/../..")) - else - report_initialization("error: unable to locate ownpath") - os.exit() - end - -end - --- The running os: - --- todo: check is context sits here os.platform is more trustworthy --- that the bin check as mtx-update runs from another path - -local texos = environment.texos or osgetenv("TEXOS") -local texmfos = environment.texmfos or osgetenv('SELFAUTODIR') - -if not texos or texos == "" then - texos = file.basename(texmfos) -end - -ossetenv('TEXMFOS', texmfos) -- full bin path -ossetenv('TEXOS', texos) -- partial bin parent -ossetenv('SELFAUTOSYSTEM',os.platform) -- bonus - -environment.texos = texos -environment.texmfos = texmfos - --- The current root: - -local texroot = environment.texroot or osgetenv("TEXROOT") - -if not texroot or texroot == "" then - texroot = osgetenv('SELFAUTOPARENT') - ossetenv('TEXROOT',texroot) -end - -environment.texroot = file.collapsepath(texroot) - + local ownpath=environment.ownpath or dir.current() + if ownpath then + ossetenv('SELFAUTOLOC',file.collapsepath(ownpath)) + ossetenv('SELFAUTODIR',file.collapsepath(ownpath.."/..")) + ossetenv('SELFAUTOPARENT',file.collapsepath(ownpath.."/../..")) + else + report_initialization("error: unable to locate ownpath") + os.exit() + end +end +local texos=environment.texos or osgetenv("TEXOS") +local texmfos=environment.texmfos or osgetenv('SELFAUTODIR') +if not texos or texos=="" then + texos=file.basename(texmfos) +end +ossetenv('TEXMFOS',texmfos) +ossetenv('TEXOS',texos) +ossetenv('SELFAUTOSYSTEM',os.platform) +environment.texos=texos +environment.texmfos=texmfos +local texroot=environment.texroot or osgetenv("TEXROOT") +if not texroot or texroot=="" then + texroot=osgetenv('SELFAUTOPARENT') + ossetenv('TEXROOT',texroot) +end +environment.texroot=file.collapsepath(texroot) if profiler then - directives.register("system.profile",function() - profiler.start("luatex-profile.log") - end) + directives.register("system.profile",function() + profiler.start("luatex-profile.log") + end) end - --- a forward definition - if not resolvers.resolve then - function resolvers.resolve (s) return s end - function resolvers.unresolve(s) return s end - function resolvers.repath (s) return s end + function resolvers.resolve (s) return s end + function resolvers.unresolve(s) return s end + function resolvers.repath (s) return s end end @@ -13700,3216 +10469,2663 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-exp'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - -local format, find, gmatch, lower, char, sub = string.format, string.find, string.gmatch, string.lower, string.char, string.sub -local concat, sort = table.concat, table.sort -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns -local Ct, Cs, Cc, P, C, S = lpeg.Ct, lpeg.Cs, lpeg.Cc, lpeg.P, lpeg.C, lpeg.S -local type, next = type, next - -local ostype = os.type -local collapsepath = file.collapsepath - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_expansions = logs.reporter("resolvers","expansions") - -local resolvers = resolvers - --- As this bit of code is somewhat special it gets its own module. After --- all, when working on the main resolver code, I don't want to scroll --- past this every time. See data-obs.lua for the gsub variant. +-- original size: 14663, stripped down to: 9537 +if not modules then modules={} end modules ['data-exp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local format,find,gmatch,lower,char,sub=string.format,string.find,string.gmatch,string.lower,string.char,string.sub +local concat,sort=table.concat,table.sort +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local Ct,Cs,Cc,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.P,lpeg.C,lpeg.S +local type,next=type,next +local ostype=os.type +local collapsepath=file.collapsepath +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_expansions=logs.reporter("resolvers","expansions") +local resolvers=resolvers local function f_first(a,b) - local t, n = { }, 0 - for s in gmatch(b,"[^,]+") do - n = n + 1 ; t[n] = a .. s - end - return concat(t,",") + local t,n={},0 + for s in gmatch(b,"[^,]+") do + n=n+1;t[n]=a..s + end + return concat(t,",") end - local function f_second(a,b) - local t, n = { }, 0 - for s in gmatch(a,"[^,]+") do - n = n + 1 ; t[n] = s .. b - end - return concat(t,",") + local t,n={},0 + for s in gmatch(a,"[^,]+") do + n=n+1;t[n]=s..b + end + return concat(t,",") end - --- kpsewhich --expand-braces '{a,b}{c,d}' --- ac:bc:ad:bd - --- old {a,b}{c,d} => ac ad bc bd --- --- local function f_both(a,b) --- local t, n = { }, 0 --- for sa in gmatch(a,"[^,]+") do --- for sb in gmatch(b,"[^,]+") do --- n = n + 1 ; t[n] = sa .. sb --- end --- end --- return concat(t,",") --- end --- --- new {a,b}{c,d} => ac bc ad bd - local function f_both(a,b) - local t, n = { }, 0 - for sb in gmatch(b,"[^,]+") do -- and not sa - for sa in gmatch(a,"[^,]+") do -- sb - n = n + 1 ; t[n] = sa .. sb - end - end - return concat(t,",") -end - -local left = P("{") -local right = P("}") -local var = P((1 - S("{}" ))^0) -local set = P((1 - S("{},"))^0) -local other = P(1) - -local l_first = Cs( ( Cc("{") * (C(set) * left * C(var) * right / f_first) * Cc("}") + other )^0 ) -local l_second = Cs( ( Cc("{") * (left * C(var) * right * C(set) / f_second) * Cc("}") + other )^0 ) -local l_both = Cs( ( Cc("{") * (left * C(var) * right * left * C(var) * right / f_both) * Cc("}") + other )^0 ) -local l_rest = Cs( ( left * var * (left/"") * var * (right/"") * var * right + other )^0 ) - -local stripper_1 = lpeg.stripper ("{}@") -local replacer_1 = lpeg.replacer { { ",}", ",@}" }, { "{,", "{@," }, } - -local function splitpathexpr(str, newlist, validate) -- I couldn't resist lpegging it (nice exercise). - if trace_expansions then - report_expansions("expanding variable '%s'",str) - end - local t, ok, done = newlist or { }, false, false - local n = #t - str = lpegmatch(replacer_1,str) + local t,n={},0 + for sb in gmatch(b,"[^,]+") do + for sa in gmatch(a,"[^,]+") do + n=n+1;t[n]=sa..sb + end + end + return concat(t,",") +end +local left=P("{") +local right=P("}") +local var=P((1-S("{}" ))^0) +local set=P((1-S("{},"))^0) +local other=P(1) +local l_first=Cs((Cc("{")*(C(set)*left*C(var)*right/f_first)*Cc("}")+other )^0 ) +local l_second=Cs((Cc("{")*(left*C(var)*right*C(set)/f_second)*Cc("}")+other )^0 ) +local l_both=Cs((Cc("{")*(left*C(var)*right*left*C(var)*right/f_both)*Cc("}")+other )^0 ) +local l_rest=Cs((left*var*(left/"")*var*(right/"")*var*right+other )^0 ) +local stripper_1=lpeg.stripper ("{}@") +local replacer_1=lpeg.replacer { { ",}",",@}" },{ "{,","{@," },} +local function splitpathexpr(str,newlist,validate) + if trace_expansions then + report_expansions("expanding variable '%s'",str) + end + local t,ok,done=newlist or {},false,false + local n=#t + str=lpegmatch(replacer_1,str) + repeat + local old=str repeat - local old = str - repeat - local old = str - str = lpegmatch(l_first, str) - until old == str - repeat - local old = str - str = lpegmatch(l_second,str) - until old == str - repeat - local old = str - str = lpegmatch(l_both, str) - until old == str - repeat - local old = str - str = lpegmatch(l_rest, str) - until old == str - until old == str -- or not find(str,"{") - str = lpegmatch(stripper_1,str) - if validate then - for s in gmatch(str,"[^,]+") do - s = validate(s) - if s then - n = n + 1 - t[n] = s - end - end - else - for s in gmatch(str,"[^,]+") do - n = n + 1 - t[n] = s - end + local old=str + str=lpegmatch(l_first,str) + until old==str + repeat + local old=str + str=lpegmatch(l_second,str) + until old==str + repeat + local old=str + str=lpegmatch(l_both,str) + until old==str + repeat + local old=str + str=lpegmatch(l_rest,str) + until old==str + until old==str + str=lpegmatch(stripper_1,str) + if validate then + for s in gmatch(str,"[^,]+") do + s=validate(s) + if s then + n=n+1 + t[n]=s + end end - if trace_expansions then - for k=1,#t do - report_expansions("% 4i: %s",k,t[k]) - end + else + for s in gmatch(str,"[^,]+") do + n=n+1 + t[n]=s end - return t + end + if trace_expansions then + for k=1,#t do + report_expansions("% 4i: %s",k,t[k]) + end + end + return t end - --- We could make the previous one public. - local function validate(s) - s = collapsepath(s) -- already keeps the trailing / and // - return s ~= "" and not find(s,"^!*unset/*$") and s + s=collapsepath(s) + return s~="" and not find(s,"^!*unset/*$") and s end - -resolvers.validatedpath = validate -- keeps the trailing // - +resolvers.validatedpath=validate function resolvers.expandedpathfromlist(pathlist) - local newlist = { } - for k=1,#pathlist do - splitpathexpr(pathlist[k],newlist,validate) - end - return newlist -end - --- {a,b,c,d} --- a,b,c/{p,q,r},d --- a,b,c/{p,q,r}/d/{x,y,z}// --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a{b,c}{d,e}f --- {a,b,c,d} --- {a,b,c/{p,q,r},d} --- {a,b,c/{p,q,r}/d/{x,y,z}//} --- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} --- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} --- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} - -local cleanup = lpeg.replacer { - { "!" , "" }, - { "\\" , "/" }, + local newlist={} + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) + end + return newlist +end +local cleanup=lpeg.replacer { + { "!","" }, + { "\\","/" }, } - -function resolvers.cleanpath(str) -- tricky, maybe only simple paths - local doslashes = (P("\\")/"/" + 1)^0 - local donegation = (P("!") /"" )^0 - local homedir = lpegmatch(Cs(donegation * doslashes),environment.homedir or "") - if homedir == "~" or homedir == "" or not lfs.isdir(homedir) then - if trace_expansions then - report_expansions("no home dir set, ignoring dependent paths") - end - function resolvers.cleanpath(str) - if not str or find(str,"~") then - return "" -- special case - else - return lpegmatch(cleanup,str) - end - end - else - local dohome = ((P("~")+P("$HOME"))/homedir)^0 - local cleanup = Cs(donegation * dohome * doslashes) - function resolvers.cleanpath(str) - return str and lpegmatch(cleanup,str) or "" - end +function resolvers.cleanpath(str) + local doslashes=(P("\\")/"/"+1)^0 + local donegation=(P("!")/"" )^0 + local homedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") + if homedir=="~" or homedir=="" or not lfs.isdir(homedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent paths") end - return resolvers.cleanpath(str) -end - --- print(resolvers.cleanpath("")) --- print(resolvers.cleanpath("!")) --- print(resolvers.cleanpath("~")) --- print(resolvers.cleanpath("~/test")) --- print(resolvers.cleanpath("!~/test")) --- print(resolvers.cleanpath("~/test~test")) - --- This one strips quotes and funny tokens. - -local expandhome = P("~") / "$HOME" -- environment.homedir - -local dodouble = P('"')/"" * (expandhome + (1 - P('"')))^0 * P('"')/"" -local dosingle = P("'")/"" * (expandhome + (1 - P("'")))^0 * P("'")/"" -local dostring = (expandhome + 1 )^0 - -local stripper = Cs( - lpegpatterns.unspacer * (dosingle + dodouble + dostring) * lpegpatterns.unspacer + function resolvers.cleanpath(str) + if not str or find(str,"~") then + return "" + else + return lpegmatch(cleanup,str) + end + end + else + local dohome=((P("~")+P("$HOME"))/homedir)^0 + local cleanup=Cs(donegation*dohome*doslashes) + function resolvers.cleanpath(str) + return str and lpegmatch(cleanup,str) or "" + end + end + return resolvers.cleanpath(str) +end +local expandhome=P("~")/"$HOME" +local dodouble=P('"')/""*(expandhome+(1-P('"')))^0*P('"')/"" +local dosingle=P("'")/""*(expandhome+(1-P("'")))^0*P("'")/"" +local dostring=(expandhome+1 )^0 +local stripper=Cs( + lpegpatterns.unspacer*(dosingle+dodouble+dostring)*lpegpatterns.unspacer ) - -function resolvers.checkedvariable(str) -- assumes str is a string - return type(str) == "string" and lpegmatch(stripper,str) or str -end - --- The path splitter: - --- A config (optionally) has the paths split in tables. Internally --- we join them and split them after the expansion has taken place. This --- is more convenient. - -local cache = { } - ------ splitter = lpeg.tsplitat(S(ostype == "windows" and ";" or ":;")) -- maybe add , -local splitter = lpeg.tsplitat(";") -- as we move towards urls, prefixes and use tables we no longer do : - -local backslashswapper = lpeg.replacer("\\","/") - -local function splitconfigurationpath(str) -- beware, this can be either a path or a { specification } - if str then - local found = cache[str] - if not found then - if str == "" then - found = { } - else - local split = lpegmatch(splitter,lpegmatch(backslashswapper,str)) -- can be combined - found = { } - local noffound = 0 - for i=1,#split do - local s = split[i] - if not find(s,"^{*unset}*") then - noffound = noffound + 1 - found[noffound] = s - end - end - if trace_expansions then - report_expansions("splitting path specification '%s'",str) - for k=1,noffound do - report_expansions("% 4i: %s",k,found[k]) - end - end - cache[str] = found - end +function resolvers.checkedvariable(str) + return type(str)=="string" and lpegmatch(stripper,str) or str +end +local cache={} +local splitter=lpeg.tsplitat(";") +local backslashswapper=lpeg.replacer("\\","/") +local function splitconfigurationpath(str) + if str then + local found=cache[str] + if not found then + if str=="" then + found={} + else + local split=lpegmatch(splitter,lpegmatch(backslashswapper,str)) + found={} + local noffound=0 + for i=1,#split do + local s=split[i] + if not find(s,"^{*unset}*") then + noffound=noffound+1 + found[noffound]=s + end + end + if trace_expansions then + report_expansions("splitting path specification '%s'",str) + for k=1,noffound do + report_expansions("% 4i: %s",k,found[k]) + end end - return found + cache[str]=found + end end + return found + end end - -resolvers.splitconfigurationpath = splitconfigurationpath - +resolvers.splitconfigurationpath=splitconfigurationpath function resolvers.splitpath(str) - if type(str) == 'table' then - return str - else - return splitconfigurationpath(str) - end + if type(str)=='table' then + return str + else + return splitconfigurationpath(str) + end end - function resolvers.joinpath(str) - if type(str) == 'table' then - return file.joinpath(str) - else - return str - end -end - --- The next function scans directories and returns a hash where the --- entries are either strings or tables. - --- starting with . or .. etc or funny char - - - - --- a lot of this caching can be stripped away when we have ssd's everywhere --- --- we could cache all the (sub)paths here if needed - -local attributes, directory = lfs.attributes, lfs.dir - -local weird = P(".")^1 + lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) -local timer = { } -local scanned = { } -local nofscans = 0 -local scancache = { } - + if type(str)=='table' then + return file.joinpath(str) + else + return str + end +end +local attributes,directory=lfs.attributes,lfs.dir +local weird=P(".")^1+lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local timer={} +local scanned={} +local nofscans=0 +local scancache={} local function scan(files,spec,path,n,m,r) - local full = (path == "" and spec) or (spec .. path .. '/') - local dirs = { } - local nofdirs = 0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode = attributes(full..name,'mode') - if mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - else -- probably unique anyway - files[name] = path - local lower = lower(name) - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end - end - elseif mode == 'directory' then - m = m + 1 - nofdirs = nofdirs + 1 - if path ~= "" then - dirs[nofdirs] = path..'/'..name - else - dirs[nofdirs] = name - end - end - end - end - if nofdirs > 0 then - sort(dirs) - for i=1,nofdirs do - files, n, m, r = scan(files,spec,dirs[i],n,m,r) - end - end - scancache[sub(full,1,-2)] = files - return files, n, m, r -end - -local fullcache = { } - -function resolvers.scanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath = resolvers.resolve(path) -- no shortcut - if usecache then - local files = fullcache[realpath] - if files then - if trace_locating then - report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) - end - return files - end - end - if trace_locating then - report_expansions("scanning path '%s', branch '%s'",path,branch or path) - end - local files, n, m, r = scan({ },realpath .. '/',"",0,0,0) - files.__path__ = path -- can be selfautoparent:texmf-whatever - files.__files__ = n - files.__directories__ = m - files.__remappings__ = r - if trace_locating then - report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) - end - if usecache then - scanned[#scanned+1] = realpath - fullcache[realpath] = files - end - nofscans = nofscans + 1 - statistics.stoptiming(timer) - return files -end - -local function simplescan(files,spec,path) -- first match only, no map and such - local full = (path == "" and spec) or (spec .. path .. '/') - local dirs = { } - local nofdirs = 0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode = attributes(full..name,'mode') - if mode == 'file' then - if not files[name] then - -- only first match - files[name] = path - end - elseif mode == 'directory' then - nofdirs = nofdirs + 1 - if path ~= "" then - dirs[nofdirs] = path..'/'..name - else - dirs[nofdirs] = name - end - end - end - end - if nofdirs > 0 then - sort(dirs) - for i=1,nofdirs do - files = simplescan(files,spec,dirs[i]) - end - end - return files -end - -local simplecache = { } -local nofsharedscans = 0 - -function resolvers.simplescanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath = resolvers.resolve(path) -- no shortcut - if usecache then - local files = simplecache[realpath] - if not files then - files = scancache[realpath] - if files then - nofsharedscans = nofsharedscans + 1 - end + local full=(path=="" and spec) or (spec..path..'/') + local dirs={} + local nofdirs=0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode=attributes(full..name,'mode') + if mode=='file' then + n=n+1 + local f=files[name] + if f then + if type(f)=='string' then + files[name]={ f,path } + else + f[#f+1]=path + end + else + files[name]=path + local lower=lower(name) + if name~=lower then + files["remap:"..lower]=name + r=r+1 + end end - if files then - if trace_locating then - report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) - end - return files + elseif mode=='directory' then + m=m+1 + nofdirs=nofdirs+1 + if path~="" then + dirs[nofdirs]=path..'/'..name + else + dirs[nofdirs]=name end + end end - if trace_locating then - report_expansions("scanning path '%s', branch '%s'",path,branch or path) + end + if nofdirs>0 then + sort(dirs) + for i=1,nofdirs do + files,n,m,r=scan(files,spec,dirs[i],n,m,r) end - local files = simplescan({ },realpath .. '/',"") - if trace_locating then - report_expansions("%s files found",table.count(files)) + end + scancache[sub(full,1,-2)]=files + return files,n,m,r +end +local fullcache={} +function resolvers.scanfiles(path,branch,usecache) + statistics.starttiming(timer) + local realpath=resolvers.resolve(path) + if usecache then + local files=fullcache[realpath] + if files then + if trace_locating then + report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) + end + return files + end + end + if trace_locating then + report_expansions("scanning path '%s', branch '%s'",path,branch or path) + end + local files,n,m,r=scan({},realpath..'/',"",0,0,0) + files.__path__=path + files.__files__=n + files.__directories__=m + files.__remappings__=r + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) + end + if usecache then + scanned[#scanned+1]=realpath + fullcache[realpath]=files + end + nofscans=nofscans+1 + statistics.stoptiming(timer) + return files +end +local function simplescan(files,spec,path) + local full=(path=="" and spec) or (spec..path..'/') + local dirs={} + local nofdirs=0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode=attributes(full..name,'mode') + if mode=='file' then + if not files[name] then + files[name]=path + end + elseif mode=='directory' then + nofdirs=nofdirs+1 + if path~="" then + dirs[nofdirs]=path..'/'..name + else + dirs[nofdirs]=name + end + end end - if usecache then - scanned[#scanned+1] = realpath - simplecache[realpath] = files + end + if nofdirs>0 then + sort(dirs) + for i=1,nofdirs do + files=simplescan(files,spec,dirs[i]) end - nofscans = nofscans + 1 - statistics.stoptiming(timer) - return files + end + return files +end +local simplecache={} +local nofsharedscans=0 +function resolvers.simplescanfiles(path,branch,usecache) + statistics.starttiming(timer) + local realpath=resolvers.resolve(path) + if usecache then + local files=simplecache[realpath] + if not files then + files=scancache[realpath] + if files then + nofsharedscans=nofsharedscans+1 + end + end + if files then + if trace_locating then + report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) + end + return files + end + end + if trace_locating then + report_expansions("scanning path '%s', branch '%s'",path,branch or path) + end + local files=simplescan({},realpath..'/',"") + if trace_locating then + report_expansions("%s files found",table.count(files)) + end + if usecache then + scanned[#scanned+1]=realpath + simplecache[realpath]=files + end + nofscans=nofscans+1 + statistics.stoptiming(timer) + return files end - function resolvers.scandata() - table.sort(scanned) - return { - n = nofscans, - shared = nofsharedscans, - time = statistics.elapsedtime(timer), - paths = scanned, - } + table.sort(scanned) + return { + n=nofscans, + shared=nofsharedscans, + time=statistics.elapsedtime(timer), + paths=scanned, + } end - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-env'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} +-- original size: 8762, stripped down to: 6484 -local lower, gsub = string.lower, string.gsub - -local resolvers = resolvers - -local allocate = utilities.storage.allocate -local setmetatableindex = table.setmetatableindex -local suffixonly = file.suffixonly - -local formats = allocate() -local suffixes = allocate() -local dangerous = allocate() -local suffixmap = allocate() - -resolvers.formats = formats -resolvers.suffixes = suffixes -resolvers.dangerous = dangerous -resolvers.suffixmap = suffixmap - -local luasuffixes = utilities.lua.suffixes - -local relations = allocate { -- todo: handlers also here - core = { - ofm = { -- will become obsolete - names = { "ofm", "omega font metric", "omega font metrics" }, - variable = 'OFMFONTS', - suffixes = { 'ofm', 'tfm' }, - }, - ovf = { -- will become obsolete - names = { "ovf", "omega virtual font", "omega virtual fonts" }, - variable = 'OVFFONTS', - suffixes = { 'ovf', 'vf' }, - }, - tfm = { - names = { "tfm", "tex font metric", "tex font metrics" }, - variable = 'TFMFONTS', - suffixes = { 'tfm' }, - }, - vf = { - names = { "vf", "virtual font", "virtual fonts" }, - variable = 'VFFONTS', - suffixes = { 'vf' }, - }, - otf = { - names = { "otf", "opentype", "opentype font", "opentype fonts"}, - variable = 'OPENTYPEFONTS', - suffixes = { 'otf' }, - }, - ttf = { - names = { "ttf", "truetype", "truetype font", "truetype fonts", "truetype collection", "truetype collections", "truetype dictionary", "truetype dictionaries" }, - variable = 'TTFONTS', - suffixes = { 'ttf', 'ttc', 'dfont' }, - }, - afm = { - names = { "afm", "adobe font metric", "adobe font metrics" }, - variable = "AFMFONTS", - suffixes = { "afm" }, - }, - pfb = { - names = { "pfb", "type1", "type 1", "type1 font", "type 1 font", "type1 fonts", "type 1 fonts" }, - variable = 'T1FONTS', - suffixes = { 'pfb', 'pfa' }, - }, - fea = { - names = { "fea", "font feature", "font features", "font feature file", "font feature files" }, - variable = 'FONTFEATURES', - suffixes = { 'fea' }, - }, - cid = { - names = { "cid", "cid map", "cid maps", "cid file", "cid files" }, - variable = 'FONTCIDMAPS', - suffixes = { 'cid', 'cidmap' }, - }, - fmt = { - names = { "fmt", "format", "tex format" }, - variable = 'TEXFORMATS', - suffixes = { 'fmt' }, - }, - mem = { -- will become obsolete - names = { 'mem', "metapost format" }, - variable = 'MPMEMS', - suffixes = { 'mem' }, - }, - mp = { - names = { "mp" }, - variable = 'MPINPUTS', - suffixes = { 'mp', 'mpvi', 'mpiv', 'mpii' }, - }, - tex = { - names = { "tex" }, - variable = 'TEXINPUTS', - suffixes = { 'tex', "mkvi", "mkiv", "mkii" }, - }, - icc = { - names = { "icc", "icc profile", "icc profiles" }, - variable = 'ICCPROFILES', - suffixes = { 'icc' }, - }, - texmfscripts = { - names = { "texmfscript", "texmfscripts", "script", "scripts" }, - variable = 'TEXMFSCRIPTS', - suffixes = { 'rb', 'pl', 'py' }, - }, - lua = { - names = { "lua" }, - variable = 'LUAINPUTS', - suffixes = { luasuffixes.lua, luasuffixes.luc, luasuffixes.tma, luasuffixes.tmc }, - }, - lib = { - names = { "lib" }, - variable = 'CLUAINPUTS', - suffixes = os.libsuffix and { os.libsuffix } or { 'dll', 'so' }, - }, - bib = { - names = { 'bib' }, - suffixes = { 'bib' }, - }, - bst = { - names = { 'bst' }, - suffixes = { 'bst' }, - }, - fontconfig = { - names = { 'fontconfig', 'fontconfig file', 'fontconfig files' }, - variable = 'FONTCONFIG_PATH', - }, +if not modules then modules={} end modules ['data-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local lower,gsub=string.lower,string.gsub +local resolvers=resolvers +local allocate=utilities.storage.allocate +local setmetatableindex=table.setmetatableindex +local suffixonly=file.suffixonly +local formats=allocate() +local suffixes=allocate() +local dangerous=allocate() +local suffixmap=allocate() +resolvers.formats=formats +resolvers.suffixes=suffixes +resolvers.dangerous=dangerous +resolvers.suffixmap=suffixmap +local luasuffixes=utilities.lua.suffixes +local relations=allocate { + core={ + ofm={ + names={ "ofm","omega font metric","omega font metrics" }, + variable='OFMFONTS', + suffixes={ 'ofm','tfm' }, + }, + ovf={ + names={ "ovf","omega virtual font","omega virtual fonts" }, + variable='OVFFONTS', + suffixes={ 'ovf','vf' }, + }, + tfm={ + names={ "tfm","tex font metric","tex font metrics" }, + variable='TFMFONTS', + suffixes={ 'tfm' }, + }, + vf={ + names={ "vf","virtual font","virtual fonts" }, + variable='VFFONTS', + suffixes={ 'vf' }, + }, + otf={ + names={ "otf","opentype","opentype font","opentype fonts"}, + variable='OPENTYPEFONTS', + suffixes={ 'otf' }, + }, + ttf={ + names={ "ttf","truetype","truetype font","truetype fonts","truetype collection","truetype collections","truetype dictionary","truetype dictionaries" }, + variable='TTFONTS', + suffixes={ 'ttf','ttc','dfont' }, + }, + afm={ + names={ "afm","adobe font metric","adobe font metrics" }, + variable="AFMFONTS", + suffixes={ "afm" }, + }, + pfb={ + names={ "pfb","type1","type 1","type1 font","type 1 font","type1 fonts","type 1 fonts" }, + variable='T1FONTS', + suffixes={ 'pfb','pfa' }, + }, + fea={ + names={ "fea","font feature","font features","font feature file","font feature files" }, + variable='FONTFEATURES', + suffixes={ 'fea' }, }, - obsolete = { - enc = { - names = { "enc", "enc files", "enc file", "encoding files", "encoding file" }, - variable = 'ENCFONTS', - suffixes = { 'enc' }, - }, - map = { - names = { "map", "map files", "map file" }, - variable = 'TEXFONTMAPS', - suffixes = { 'map' }, - }, - lig = { - names = { "lig files", "lig file", "ligature file", "ligature files" }, - variable = 'LIGFONTS', - suffixes = { 'lig' }, - }, - opl = { - names = { "opl" }, - variable = 'OPLFONTS', - suffixes = { 'opl' }, - }, - ovp = { - names = { "ovp" }, - variable = 'OVPFONTS', - suffixes = { 'ovp' }, - }, + cid={ + names={ "cid","cid map","cid maps","cid file","cid files" }, + variable='FONTCIDMAPS', + suffixes={ 'cid','cidmap' }, }, - kpse = { -- subset - base = { - names = { 'base', "metafont format" }, - variable = 'MFBASES', - suffixes = { 'base', 'bas' }, - }, - cmap = { - names = { 'cmap', 'cmap files', 'cmap file' }, - variable = 'CMAPFONTS', - suffixes = { 'cmap' }, - }, - cnf = { - names = { 'cnf' }, - suffixes = { 'cnf' }, - }, - web = { - names = { 'web' }, - suffixes = { 'web', 'ch' } - }, - cweb = { - names = { 'cweb' }, - suffixes = { 'w', 'web', 'ch' }, - }, - gf = { - names = { 'gf' }, - suffixes = { 'gf' }, - }, - mf = { - names = { 'mf' }, - variable = 'MFINPUTS', - suffixes = { 'mf' }, - }, - mft = { - names = { 'mft' }, - suffixes = { 'mft' }, - }, - pk = { - names = { 'pk' }, - suffixes = { 'pk' }, - }, + fmt={ + names={ "fmt","format","tex format" }, + variable='TEXFORMATS', + suffixes={ 'fmt' }, }, + mem={ + names={ 'mem',"metapost format" }, + variable='MPMEMS', + suffixes={ 'mem' }, + }, + mp={ + names={ "mp" }, + variable='MPINPUTS', + suffixes={ 'mp','mpvi','mpiv','mpii' }, + }, + tex={ + names={ "tex" }, + variable='TEXINPUTS', + suffixes={ 'tex',"mkvi","mkiv","mkii" }, + }, + icc={ + names={ "icc","icc profile","icc profiles" }, + variable='ICCPROFILES', + suffixes={ 'icc' }, + }, + texmfscripts={ + names={ "texmfscript","texmfscripts","script","scripts" }, + variable='TEXMFSCRIPTS', + suffixes={ 'rb','pl','py' }, + }, + lua={ + names={ "lua" }, + variable='LUAINPUTS', + suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + }, + lib={ + names={ "lib" }, + variable='CLUAINPUTS', + suffixes=os.libsuffix and { os.libsuffix } or { 'dll','so' }, + }, + bib={ + names={ 'bib' }, + suffixes={ 'bib' }, + }, + bst={ + names={ 'bst' }, + suffixes={ 'bst' }, + }, + fontconfig={ + names={ 'fontconfig','fontconfig file','fontconfig files' }, + variable='FONTCONFIG_PATH', + }, + }, + obsolete={ + enc={ + names={ "enc","enc files","enc file","encoding files","encoding file" }, + variable='ENCFONTS', + suffixes={ 'enc' }, + }, + map={ + names={ "map","map files","map file" }, + variable='TEXFONTMAPS', + suffixes={ 'map' }, + }, + lig={ + names={ "lig files","lig file","ligature file","ligature files" }, + variable='LIGFONTS', + suffixes={ 'lig' }, + }, + opl={ + names={ "opl" }, + variable='OPLFONTS', + suffixes={ 'opl' }, + }, + ovp={ + names={ "ovp" }, + variable='OVPFONTS', + suffixes={ 'ovp' }, + }, + }, + kpse={ + base={ + names={ 'base',"metafont format" }, + variable='MFBASES', + suffixes={ 'base','bas' }, + }, + cmap={ + names={ 'cmap','cmap files','cmap file' }, + variable='CMAPFONTS', + suffixes={ 'cmap' }, + }, + cnf={ + names={ 'cnf' }, + suffixes={ 'cnf' }, + }, + web={ + names={ 'web' }, + suffixes={ 'web','ch' } + }, + cweb={ + names={ 'cweb' }, + suffixes={ 'w','web','ch' }, + }, + gf={ + names={ 'gf' }, + suffixes={ 'gf' }, + }, + mf={ + names={ 'mf' }, + variable='MFINPUTS', + suffixes={ 'mf' }, + }, + mft={ + names={ 'mft' }, + suffixes={ 'mft' }, + }, + pk={ + names={ 'pk' }, + suffixes={ 'pk' }, + }, + }, } - -resolvers.relations = relations - --- formats: maps a format onto a variable - +resolvers.relations=relations function resolvers.updaterelations() - for category, categories in next, relations do - for name, relation in next, categories do - local rn = relation.names - local rv = relation.variable - local rs = relation.suffixes - if rn and rv then - for i=1,#rn do - local rni = lower(gsub(rn[i]," ","")) - formats[rni] = rv - if rs then - suffixes[rni] = rs - for i=1,#rs do - local rsi = rs[i] - suffixmap[rsi] = rni - end - end - end - end - if rs then + for category,categories in next,relations do + for name,relation in next,categories do + local rn=relation.names + local rv=relation.variable + local rs=relation.suffixes + if rn and rv then + for i=1,#rn do + local rni=lower(gsub(rn[i]," ","")) + formats[rni]=rv + if rs then + suffixes[rni]=rs + for i=1,#rs do + local rsi=rs[i] + suffixmap[rsi]=rni end + end end + end + if rs then + end end + end end - -resolvers.updaterelations() -- push this in the metatable -> newindex - +resolvers.updaterelations() local function simplified(t,k) - return k and rawget(t,lower(gsub(k," ",""))) or nil + return k and rawget(t,lower(gsub(k," ",""))) or nil end - -setmetatableindex(formats, simplified) -setmetatableindex(suffixes, simplified) -setmetatableindex(suffixmap, simplified) - --- A few accessors, mostly for command line tool. - +setmetatableindex(formats,simplified) +setmetatableindex(suffixes,simplified) +setmetatableindex(suffixmap,simplified) function resolvers.suffixofformat(str) - local s = suffixes[str] - return s and s[1] or "" + local s=suffixes[str] + return s and s[1] or "" end - function resolvers.suffixofformat(str) - return suffixes[str] or { } + return suffixes[str] or {} end - -for name, format in next, formats do - dangerous[name] = true -- still needed ? +for name,format in next,formats do + dangerous[name]=true end - --- because vf searching is somewhat dangerous, we want to prevent --- too liberal searching esp because we do a lookup on the current --- path anyway; only tex (or any) is safe - -dangerous.tex = nil - - --- more helpers - +dangerous.tex=nil function resolvers.formatofvariable(str) - return formats[str] or '' + return formats[str] or '' end - -function resolvers.formatofsuffix(str) -- of file - return suffixmap[suffixonly(str)] or 'tex' -- so many map onto tex (like mkiv, cld etc) +function resolvers.formatofsuffix(str) + return suffixmap[suffixonly(str)] or 'tex' end - function resolvers.variableofformat(str) - return formats[str] or '' + return formats[str] or '' end - function resolvers.variableofformatorsuffix(str) - local v = formats[str] - if v then - return v - end - v = suffixmap[suffixonly(str)] - if v then - return formats[v] - end - return '' + local v=formats[str] + if v then + return v + end + v=suffixmap[suffixonly(str)] + if v then + return formats[v] + end + return '' end - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tmp'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - - - -local format, lower, gsub, concat = string.format, string.lower, string.gsub, table.concat -local serialize, serializetofile = table.serialize, table.tofile -local mkdirs, isdir = dir.mkdirs, lfs.isdir -local addsuffix, is_writable, is_readable = file.addsuffix, file.is_writable, file.is_readable - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) - -local report_caches = logs.reporter("resolvers","caches") -local report_resolvers = logs.reporter("resolvers","caching") - -local resolvers = resolvers - --- intermezzo - -local directive_cleanup = false directives.register("system.compile.cleanup", function(v) directive_cleanup = v end) -local directive_strip = false directives.register("system.compile.strip", function(v) directive_strip = v end) - -local compile = utilities.lua.compile +-- original size: 14075, stripped down to: 10764 +if not modules then modules={} end modules ['data-tmp']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat +local serialize,serializetofile=table.serialize,table.tofile +local mkdirs,isdir=dir.mkdirs,lfs.isdir +local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local report_caches=logs.reporter("resolvers","caches") +local report_resolvers=logs.reporter("resolvers","caching") +local resolvers=resolvers +local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) +local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) +local compile=utilities.lua.compile function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup == nil then cleanup = directive_cleanup end - if strip == nil then strip = directive_strip end - return compile(luafile,lucfile,cleanup,strip) -end - --- end of intermezzo - -caches = caches or { } -local caches = caches - -local luasuffixes = utilities.lua.suffixes - -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.tree = false -caches.force = true -caches.ask = false -caches.relocate = false -caches.defaults = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } - -local writable, readables, usedreadables = nil, { }, { } - --- we could use a metatable for writable and readable but not yet - + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compile(luafile,lucfile,cleanup,strip) +end +caches=caches or {} +local caches=caches +local luasuffixes=utilities.lua.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" +caches.direct=false +caches.tree=false +caches.force=true +caches.ask=false +caches.relocate=false +caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } +local writable,readables,usedreadables=nil,{},{} local function identify() - -- Combining the loops makes it messy. First we check the format cache path - -- and when the last component is not present we try to create it. - local texmfcaches = resolvers.cleanpathlist("TEXMFCACHE") - if texmfcaches then - for k=1,#texmfcaches do - local cachepath = texmfcaches[k] - if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) - cachepath = file.collapsepath(cachepath) - local valid = isdir(cachepath) - if valid then - if is_readable(cachepath) then - readables[#readables+1] = cachepath - if not writable and is_writable(cachepath) then - writable = cachepath - end - end - elseif not writable and caches.force then - local cacheparent = file.dirname(cachepath) - if is_writable(cacheparent) and true then -- we go on anyway (needed for mojca's kind of paths) - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("created: %s",cachepath) - writable = cachepath - readables[#readables+1] = cachepath - end - end - end - end + local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + cachepath=file.collapsepath(cachepath) + local valid=isdir(cachepath) + if valid then + if is_readable(cachepath) then + readables[#readables+1]=cachepath + if not writable and is_writable(cachepath) then + writable=cachepath end - end - end - -- As a last resort we check some temporary paths but this time we don't - -- create them. - local texmfcaches = caches.defaults - if texmfcaches then - for k=1,#texmfcaches do - local cachepath = texmfcaches[k] - cachepath = resolvers.expansion(cachepath) -- was getenv - if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) - local valid = isdir(cachepath) - if valid and is_readable(cachepath) then - if not writable and is_writable(cachepath) then - readables[#readables+1] = cachepath - writable = cachepath - break - end - end + end + elseif not writable and caches.force then + local cacheparent=file.dirname(cachepath) + if is_writable(cacheparent) and true then + if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("created: %s",cachepath) + writable=cachepath + readables[#readables+1]=cachepath + end end + end end + end end - -- Some extra checking. If we have no writable or readable path then we simply - -- quit. - if not writable then - report_caches("fatal error: there is no valid writable cache path defined") - os.exit() - elseif #readables == 0 then - report_caches("fatal error: there is no valid readable cache path defined") - os.exit() - end - -- why here - writable = dir.expandname(resolvers.cleanpath(writable)) -- just in case - -- moved here - local base, more, tree = caches.base, caches.more, caches.tree or caches.treehash() -- we have only one writable tree - if tree then - caches.tree = tree - writable = mkdirs(writable,base,more,tree) - for i=1,#readables do - readables[i] = file.join(readables[i],base,more,tree) - end - else - writable = mkdirs(writable,base,more) - for i=1,#readables do - readables[i] = file.join(readables[i],base,more) + end + local texmfcaches=caches.defaults + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + cachepath=resolvers.expansion(cachepath) + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + local valid=isdir(cachepath) + if valid and is_readable(cachepath) then + if not writable and is_writable(cachepath) then + readables[#readables+1]=cachepath + writable=cachepath + break + end end + end end - -- end - if trace_cache then - for i=1,#readables do - report_caches("using readable path '%s' (order %s)",readables[i],i) - end - report_caches("using writable path '%s'",writable) + end + if not writable then + report_caches("fatal error: there is no valid writable cache path defined") + os.exit() + elseif #readables==0 then + report_caches("fatal error: there is no valid readable cache path defined") + os.exit() + end + writable=dir.expandname(resolvers.cleanpath(writable)) + local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + if tree then + caches.tree=tree + writable=mkdirs(writable,base,more,tree) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more,tree) + end + else + writable=mkdirs(writable,base,more) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more) end - identify = function() - return writable, readables + end + if trace_cache then + for i=1,#readables do + report_caches("using readable path '%s' (order %s)",readables[i],i) end - return writable, readables + report_caches("using writable path '%s'",writable) + end + identify=function() + return writable,readables + end + return writable,readables end - function caches.usedpaths() - local writable, readables = identify() - if #readables > 1 then - local result = { } - for i=1,#readables do - local readable = readables[i] - if usedreadables[i] or readable == writable then - result[#result+1] = format("readable: '%s' (order %s)",readable,i) - end - end - result[#result+1] = format("writable: '%s'",writable) - return result - else - return writable + local writable,readables=identify() + if #readables>1 then + local result={} + for i=1,#readables do + local readable=readables[i] + if usedreadables[i] or readable==writable then + result[#result+1]=format("readable: '%s' (order %s)",readable,i) + end end + result[#result+1]=format("writable: '%s'",writable) + return result + else + return writable + end end - function caches.configfiles() - return concat(resolvers.instance.specification,";") + return concat(resolvers.instance.specification,";") end - function caches.hashed(tree) - tree = gsub(tree,"[\\/]+$","") - tree = lower(tree) - local hash = md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %s, hash %s",tree,hash) - end - return hash + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5.hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %s, hash %s",tree,hash) + end + return hash end - function caches.treehash() - local tree = caches.configfiles() - if not tree or tree == "" then - return false - else - return caches.hashed(tree) - end + local tree=caches.configfiles() + if not tree or tree=="" then + return false + else + return caches.hashed(tree) + end end - -local r_cache, w_cache = { }, { } -- normally w in in r but who cares - +local r_cache,w_cache={},{} local function getreadablepaths(...) - local tags = { ... } - local hash = concat(tags,"/") - local done = r_cache[hash] - if not done then - local writable, readables = identify() -- exit if not found - if #tags > 0 then - done = { } - for i=1,#readables do - done[i] = file.join(readables[i],...) - end - else - done = readables - end - r_cache[hash] = done + local tags={... } + local hash=concat(tags,"/") + local done=r_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done={} + for i=1,#readables do + done[i]=file.join(readables[i],...) + end + else + done=readables end - return done + r_cache[hash]=done + end + return done end - local function getwritablepath(...) - local tags = { ... } - local hash = concat(tags,"/") - local done = w_cache[hash] - if not done then - local writable, readables = identify() -- exit if not found - if #tags > 0 then - done = mkdirs(writable,...) - else - done = writable - end - w_cache[hash] = done - end - return done -end - -caches.getreadablepaths = getreadablepaths -caches.getwritablepath = getwritablepath - + local tags={... } + local hash=concat(tags,"/") + local done=w_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done=mkdirs(writable,...) + else + done=writable + end + w_cache[hash]=done + end + return done +end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath function caches.getfirstreadablefile(filename,...) - local rd = getreadablepaths(...) - for i=1,#rd do - local path = rd[i] - local fullname = file.join(path,filename) - if is_readable(fullname) then - usedreadables[i] = true - return fullname, path - end + local rd=getreadablepaths(...) + for i=1,#rd do + local path=rd[i] + local fullname=file.join(path,filename) + if is_readable(fullname) then + usedreadables[i]=true + return fullname,path end - return caches.setfirstwritablefile(filename,...) + end + return caches.setfirstwritablefile(filename,...) end - function caches.setfirstwritablefile(filename,...) - local wr = getwritablepath(...) - local fullname = file.join(wr,filename) - return fullname, wr + local wr=getwritablepath(...) + local fullname=file.join(wr,filename) + return fullname,wr end - -function caches.define(category,subcategory) -- for old times sake - return function() - return getwritablepath(category,subcategory) - end +function caches.define(category,subcategory) + return function() + return getwritablepath(category,subcategory) + end end - function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma), format("%s/%s.%s",path,name,luasuffixes.tmc) + return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) end - function caches.loaddata(readables,name) - if type(readables) == "string" then - readables = { readables } - end - for i=1,#readables do - local path = readables[i] - local tmaname, tmcname = caches.setluanames(path,name) - local loader = loadfile(tmcname) - if not loader then - -- in case we have a different engine - utilities.lua.compile(tmaname,tmcname) - -- - loader = loadfile(tmaname) - end - if loader then - loader = loader() - collectgarbage("step") - return loader - end - end - return false + if type(readables)=="string" then + readables={ readables } + end + for i=1,#readables do + local path=readables[i] + local tmaname,tmcname=caches.setluanames(path,name) + local loader=loadfile(tmcname) + if not loader then + utilities.lua.compile(tmaname,tmcname) + loader=loadfile(tmaname) + end + if loader then + loader=loader() + collectgarbage("step") + return loader + end + end + return false end - function caches.is_writable(filepath,filename) - local tmaname, tmcname = caches.setluanames(filepath,filename) - return is_writable(tmaname) + local tmaname,tmcname=caches.setluanames(filepath,filename) + return is_writable(tmaname) end - -local saveoptions = { compact = true } - --- add some point we will only use the internal bytecode compiler and --- then we can flag success in the tma so that it can trigger a compile --- if the other engine - +local saveoptions={ compact=true } function caches.savedata(filepath,filename,data,raw) - local tmaname, tmcname = caches.setluanames(filepath,filename) - local reduce, simplify = true, true - if raw then - reduce, simplify = false, false - end - data.cache_uuid = os.uuid() - if caches.direct then - file.savedata(tmaname,serialize(data,true,saveoptions)) - else - serializetofile(tmaname,data,true,saveoptions) - end - utilities.lua.compile(tmaname,tmcname) -end - --- moved from data-res: - -local content_state = { } - + local tmaname,tmcname=caches.setluanames(filepath,filename) + local reduce,simplify=true,true + if raw then + reduce,simplify=false,false + end + data.cache_uuid=os.uuid() + if caches.direct then + file.savedata(tmaname,serialize(data,true,saveoptions)) + else + serializetofile(tmaname,data,true,saveoptions) + end + utilities.lua.compile(tmaname,tmcname) +end +local content_state={} function caches.contentstate() - return content_state or { } + return content_state or {} end - function caches.loadcontent(cachename,dataname) - local name = caches.hashed(cachename) - local full, path = caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename = file.join(path,name) - local blob = loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) - if blob then - local data = blob() - if data and data.content then - if data.type == dataname then - if data.version == resolvers.cacheversion then - content_state[#content_state+1] = data.uuid - if trace_locating then - report_resolvers("loading '%s' for '%s' from '%s'",dataname,cachename,filename) - end - return data.content - else - report_resolvers("skipping '%s' for '%s' from '%s' (version mismatch)",dataname,cachename,filename) - end - else - report_resolvers("skipping '%s' for '%s' from '%s' (datatype mismatch)",dataname,cachename,filename) - end - elseif trace_locating then - report_resolvers("skipping '%s' for '%s' from '%s' (no content)",dataname,cachename,filename) - end - elseif trace_locating then - report_resolvers("skipping '%s' for '%s' from '%s' (invalid file)",dataname,cachename,filename) - end -end - -function caches.collapsecontent(content) - for k, v in next, content do - if type(v) == "table" and #v == 1 then - content[k] = v[1] - end - end -end - -function caches.savecontent(cachename,dataname,content) - local name = caches.hashed(cachename) - local full, path = caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename = file.join(path,name) -- is full - local luaname = addsuffix(filename,luasuffixes.lua) - local lucname = addsuffix(filename,luasuffixes.luc) - if trace_locating then - report_resolvers("preparing '%s' for '%s'",dataname,cachename) - end - local data = { - type = dataname, - root = cachename, - version = resolvers.cacheversion, - date = os.date("%Y-%m-%d"), - time = os.date("%H:%M:%S"), - content = content, - uuid = os.uuid(), - } - local ok = io.savedata(luaname,serialize(data,true)) - if ok then - if trace_locating then - report_resolvers("category '%s', cachename '%s' saved in '%s'",dataname,cachename,luaname) - end - if utilities.lua.compile(luaname,lucname) then - if trace_locating then - report_resolvers("'%s' compiled to '%s'",dataname,lucname) - end - return true + local name=caches.hashed(cachename) + local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + local filename=file.join(path,name) + local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) + if blob then + local data=blob() + if data and data.content then + if data.type==dataname then + if data.version==resolvers.cacheversion then + content_state[#content_state+1]=data.uuid + if trace_locating then + report_resolvers("loading '%s' for '%s' from '%s'",dataname,cachename,filename) + end + return data.content else - if trace_locating then - report_resolvers("compiling failed for '%s', deleting file '%s'",dataname,lucname) - end - os.remove(lucname) + report_resolvers("skipping '%s' for '%s' from '%s' (version mismatch)",dataname,cachename,filename) end + else + report_resolvers("skipping '%s' for '%s' from '%s' (datatype mismatch)",dataname,cachename,filename) + end elseif trace_locating then - report_resolvers("unable to save '%s' in '%s' (access error)",dataname,luaname) + report_resolvers("skipping '%s' for '%s' from '%s' (no content)",dataname,cachename,filename) end + elseif trace_locating then + report_resolvers("skipping '%s' for '%s' from '%s' (invalid file)",dataname,cachename,filename) + end end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-met'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local find, format = string.find, string.format -local sequenced = table.sequenced -local addurlscheme, urlhashed = url.addscheme, url.hashed - -local trace_locating = false - -trackers.register("resolvers.locating", function(v) trace_methods = v end) -trackers.register("resolvers.methods", function(v) trace_methods = v end) - - -local report_methods = logs.reporter("resolvers","methods") - -local allocate = utilities.storage.allocate - -local resolvers = resolvers - -local registered = { } - -local function splitmethod(filename) -- todo: filetype in specification - if not filename then - return { scheme = "unknown", original = filename } - end - if type(filename) == "table" then - return filename -- already split - end - filename = file.collapsepath(filename) - if not find(filename,"://") then - return { scheme = "file", path = filename, original = filename, filename = filename } - end - local specification = url.hashed(filename) - if not specification.scheme or specification.scheme == "" then - return { scheme = "file", path = filename, original = filename, filename = filename } - else - return specification +function caches.collapsecontent(content) + for k,v in next,content do + if type(v)=="table" and #v==1 then + content[k]=v[1] end + end end - -resolvers.splitmethod = splitmethod -- bad name but ok - --- the second argument is always analyzed (saves time later on) and the original --- gets passed as original but also as argument - -local function methodhandler(what,first,...) -- filename can be nil or false - local method = registered[what] - if method then - local how, namespace = method.how, method.namespace - if how == "uri" or how == "url" then - local specification = splitmethod(first) - local scheme = specification.scheme - local resolver = namespace and namespace[scheme] - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, scheme=%s, argument=%s",what,how,scheme,first) - end - return resolver(specification,...) - else - resolver = namespace.default or namespace.file - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, default, argument=%s",what,how,first) - end - return resolver(specification,...) - elseif trace_methods then - report_methods("resolver: method=%s, how=%s, no handler",what,how) - end - end - elseif how == "tag" then - local resolver = namespace and namespace[first] - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, tag=%s",what,how,first) - end - return resolver(...) - else - resolver = namespace.default or namespace.file - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, default",what,how) - end - return resolver(...) - elseif trace_methods then - report_methods("resolver: method=%s, how=%s, unknown",what,how) - end - end - end +function caches.savecontent(cachename,dataname,content) + local name=caches.hashed(cachename) + local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + local filename=file.join(path,name) + local luaname=addsuffix(filename,luasuffixes.lua) + local lucname=addsuffix(filename,luasuffixes.luc) + if trace_locating then + report_resolvers("preparing '%s' for '%s'",dataname,cachename) + end + local data={ + type=dataname, + root=cachename, + version=resolvers.cacheversion, + date=os.date("%Y-%m-%d"), + time=os.date("%H:%M:%S"), + content=content, + uuid=os.uuid(), + } + local ok=io.savedata(luaname,serialize(data,true)) + if ok then + if trace_locating then + report_resolvers("category '%s', cachename '%s' saved in '%s'",dataname,cachename,luaname) + end + if utilities.lua.compile(luaname,lucname) then + if trace_locating then + report_resolvers("'%s' compiled to '%s'",dataname,lucname) + end + return true else - report_methods("resolver: method=%s, unknown",what) + if trace_locating then + report_resolvers("compiling failed for '%s', deleting file '%s'",dataname,lucname) + end + os.remove(lucname) end + elseif trace_locating then + report_resolvers("unable to save '%s' in '%s' (access error)",dataname,luaname) + end end -resolvers.methodhandler = methodhandler - -function resolvers.registermethod(name,namespace,how) - registered[name] = { how = how or "tag", namespace = namespace } - namespace["byscheme"] = function(scheme,filename,...) - if scheme == "file" then - return methodhandler(name,filename,...) - else - return methodhandler(name,addurlscheme(filename,scheme),...) - end - end -end -local concatinators = allocate { notfound = file.join } -- concatinate paths -local locators = allocate { notfound = function() end } -- locate databases -local hashers = allocate { notfound = function() end } -- load databases -local generators = allocate { notfound = function() end } -- generate databases +end -- of closure -resolvers.concatinators = concatinators -resolvers.locators = locators -resolvers.hashers = hashers -resolvers.generators = generators +do -- create closure to overcome 200 locals limit -local registermethod = resolvers.registermethod +-- original size: 4863, stripped down to: 3890 +if not modules then modules={} end modules ['data-met']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,format=string.find,string.format +local sequenced=table.sequenced +local addurlscheme,urlhashed=url.addscheme,url.hashed +local trace_locating=false +trackers.register("resolvers.locating",function(v) trace_methods=v end) +trackers.register("resolvers.methods",function(v) trace_methods=v end) +local report_methods=logs.reporter("resolvers","methods") +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registered={} +local function splitmethod(filename) + if not filename then + return { scheme="unknown",original=filename } + end + if type(filename)=="table" then + return filename + end + filename=file.collapsepath(filename) + if not find(filename,"://") then + return { scheme="file",path=filename,original=filename,filename=filename } + end + local specification=url.hashed(filename) + if not specification.scheme or specification.scheme=="" then + return { scheme="file",path=filename,original=filename,filename=filename } + else + return specification + end +end +resolvers.splitmethod=splitmethod +local function methodhandler(what,first,...) + local method=registered[what] + if method then + local how,namespace=method.how,method.namespace + if how=="uri" or how=="url" then + local specification=splitmethod(first) + local scheme=specification.scheme + local resolver=namespace and namespace[scheme] + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, scheme=%s, argument=%s",what,how,scheme,first) + end + return resolver(specification,...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, default, argument=%s",what,how,first) + end + return resolver(specification,...) + elseif trace_methods then + report_methods("resolver: method=%s, how=%s, no handler",what,how) + end + end + elseif how=="tag" then + local resolver=namespace and namespace[first] + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, tag=%s",what,how,first) + end + return resolver(...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, default",what,how) + end + return resolver(...) + elseif trace_methods then + report_methods("resolver: method=%s, how=%s, unknown",what,how) + end + end + end + else + report_methods("resolver: method=%s, unknown",what) + end +end +resolvers.methodhandler=methodhandler +function resolvers.registermethod(name,namespace,how) + registered[name]={ how=how or "tag",namespace=namespace } + namespace["byscheme"]=function(scheme,filename,...) + if scheme=="file" then + return methodhandler(name,filename,...) + else + return methodhandler(name,addurlscheme(filename,scheme),...) + end + end +end +local concatinators=allocate { notfound=file.join } +local locators=allocate { notfound=function() end } +local hashers=allocate { notfound=function() end } +local generators=allocate { notfound=function() end } +resolvers.concatinators=concatinators +resolvers.locators=locators +resolvers.hashers=hashers +resolvers.generators=generators +local registermethod=resolvers.registermethod registermethod("concatinators",concatinators,"tag") -registermethod("locators", locators, "uri") -registermethod("hashers", hashers, "uri") -registermethod("generators", generators, "uri") +registermethod("locators",locators,"uri") +registermethod("hashers",hashers,"uri") +registermethod("generators",generators,"uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-res'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - --- In practice we will work within one tds tree, but i want to keep --- the option open to build tools that look at multiple trees, which is --- why we keep the tree specific data in a table. We used to pass the --- instance but for practical purposes we now avoid this and use a --- instance variable. We always have one instance active (sort of global). - --- todo: cache:/// home:/// selfautoparent:/// (sometime end 2012) - -local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch -local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys -local next, type, rawget = next, type, rawget -local os = os - -local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns - -local filedirname = file.dirname -local filebasename = file.basename -local suffixonly = file.suffixonly -local filejoin = file.join -local collapsepath = file.collapsepath -local joinpath = file.joinpath -local allocate = utilities.storage.allocate -local settings_to_array = utilities.parsers.settings_to_array -local setmetatableindex = table.setmetatableindex -local luasuffixes = utilities.lua.suffixes - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_resolving = logs.reporter("resolvers","resolving") - -local resolvers = resolvers - -local expandedpathfromlist = resolvers.expandedpathfromlist -local checkedvariable = resolvers.checkedvariable -local splitconfigurationpath = resolvers.splitconfigurationpath -local methodhandler = resolvers.methodhandler - -local initializesetter = utilities.setters.initialize - -local ostype, osname, osenv, ossetenv, osgetenv = os.type, os.name, os.env, os.setenv, os.getenv - -resolvers.cacheversion = '1.0.1' -resolvers.configbanner = '' -resolvers.homedir = environment.homedir -resolvers.criticalvars = allocate { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF", "TEXMF", "TEXOS" } -resolvers.luacnfname = "texmfcnf.lua" -resolvers.luacnfstate = "unknown" - --- The web2c tex binaries as well as kpse have built in paths for the configuration --- files and there can be a depressing truckload of them. This is actually the weak --- spot of a distribution. So we don't want: --- --- resolvers.luacnfspec = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}' --- --- but instead use: --- --- resolvers.luacnfspec = 'selfautoparent:{/texmf{-local,}{,/web2c}}' --- --- which does not make texlive happy as there is a texmf-local tree one level up --- (sigh), so we need this. We can assume web2c as mkiv does not run on older --- texlives anyway. --- --- texlive: --- --- selfautodir: --- selfautoparent: --- selfautodir:share/texmf-local/web2c --- selfautodir:share/texmf/web2c --- selfautodir:texmf-local/web2c --- selfautodir:texmf/web2c --- selfautoparent:share/texmf-local/web2c --- selfautoparent:share/texmf/web2c --- selfautoparent:texmf-local/web2c --- selfautoparent:texmf/web2c --- --- minimals: --- --- home:texmf/web2c --- selfautoparent:texmf-local/web2c --- selfautoparent:texmf-context/web2c --- selfautoparent:texmf/web2c +-- original size: 60360, stripped down to: 42573 +if not modules then modules={} end modules ['data-res']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local format,gsub,find,lower,upper,match,gmatch=string.format,string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch +local concat,insert,sortedkeys=table.concat,table.insert,table.sortedkeys +local next,type,rawget=next,type,rawget +local os=os +local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local filedirname=file.dirname +local filebasename=file.basename +local suffixonly=file.suffixonly +local filejoin=file.join +local collapsepath=file.collapsepath +local joinpath=file.joinpath +local allocate=utilities.storage.allocate +local settings_to_array=utilities.parsers.settings_to_array +local setmetatableindex=table.setmetatableindex +local luasuffixes=utilities.lua.suffixes +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_resolving=logs.reporter("resolvers","resolving") +local resolvers=resolvers +local expandedpathfromlist=resolvers.expandedpathfromlist +local checkedvariable=resolvers.checkedvariable +local splitconfigurationpath=resolvers.splitconfigurationpath +local methodhandler=resolvers.methodhandler +local initializesetter=utilities.setters.initialize +local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os.getenv +resolvers.cacheversion='1.0.1' +resolvers.configbanner='' +resolvers.homedir=environment.homedir +resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } +resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnfstate="unknown" if environment.default_texmfcnf then - -- unfortunately we now have quite some overkill in the spec (not so nice on a network) - resolvers.luacnfspec = environment.default_texmfcnf + resolvers.luacnfspec=environment.default_texmfcnf else - -- resolvers.luacnfspec = "selfautoparent:texmf{-local,-context,}/web2c" - resolvers.luacnfspec = "{selfautoloc:,selfautodir:,selfautoparent:}{,/texmf{-local,}/web2c}" -end - -resolvers.luacnfspec = 'home:texmf/web2c;' .. resolvers.luacnfspec - --- which (as we want users to use the web2c path) be can be simplified to this: --- --- if environment and environment.ownpath and string.find(environment.ownpath,"[\\/]texlive[\\/]") then --- resolvers.luacnfspec = 'selfautodir:/texmf-local/web2c,selfautoparent:/texmf-local/web2c,selfautoparent:/texmf/web2c' --- else --- resolvers.luacnfspec = 'selfautoparent:/texmf-local/web2c,selfautoparent:/texmf/web2c' --- end - - - -local unset_variable = "unset" - -local formats = resolvers.formats -local suffixes = resolvers.suffixes -local dangerous = resolvers.dangerous -local suffixmap = resolvers.suffixmap - -resolvers.defaultsuffixes = { "tex" } -- "mkiv", "cld" -- too tricky - -resolvers.instance = resolvers.instance or nil -- the current one (slow access) -local instance = resolvers.instance or nil -- the current one (fast access) - --- An instance has an environment (coming from the outside, kept raw), variables --- (coming from the configuration file), and expansions (variables with nested --- variables replaced). One can push something into the outer environment and --- its internal copy, but only the later one will be the raw unprefixed variant. - + resolvers.luacnfspec="{selfautoloc:,selfautodir:,selfautoparent:}{,/texmf{-local,}/web2c}" +end +resolvers.luacnfspec='home:texmf/web2c;'..resolvers.luacnfspec +local unset_variable="unset" +local formats=resolvers.formats +local suffixes=resolvers.suffixes +local dangerous=resolvers.dangerous +local suffixmap=resolvers.suffixmap +resolvers.defaultsuffixes={ "tex" } +resolvers.instance=resolvers.instance or nil +local instance=resolvers.instance or nil function resolvers.setenv(key,value,raw) - if instance then - -- this one will be consulted first when we stay inside - -- the current environment (prefixes are not resolved here) - instance.environment[key] = value - -- we feed back into the environment, and as this is used - -- by other applications (via os.execute) we need to make - -- sure that prefixes are resolve - ossetenv(key,raw and value or resolvers.resolve(value)) - end + if instance then + instance.environment[key]=value + ossetenv(key,raw and value or resolvers.resolve(value)) + end end - --- Beware we don't want empty here as this one can be called early on --- and therefore we use rawget. - local function getenv(key) - local value = rawget(instance.environment,key) - if value and value ~= "" then - return value - else - local e = osgetenv(key) - return e ~= nil and e ~= "" and checkedvariable(e) or "" - end -end - -resolvers.getenv = getenv -resolvers.env = getenv - --- We are going to use some metatable trickery where we backtrack from --- expansion to variable to environment. - + local value=rawget(instance.environment,key) + if value and value~="" then + return value + else + local e=osgetenv(key) + return e~=nil and e~="" and checkedvariable(e) or "" + end +end +resolvers.getenv=getenv +resolvers.env=getenv local function resolve(k) - return instance.expansions[k] -end - -local dollarstripper = lpeg.stripper("$") -local inhibitstripper = P("!")^0 * Cs(P(1)^0) -local backslashswapper = lpeg.replacer("\\","/") - -local somevariable = P("$") / "" -local somekey = C(R("az","AZ","09","__","--")^1) -local somethingelse = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "") - + P(";") * (P(";") / "") - + P(1) -local variableexpander = Cs( (somevariable * (somekey/resolve) + somethingelse)^1 ) - -local cleaner = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";" -local variablecleaner = Cs((cleaner + P(1))^0) - -local somevariable = R("az","AZ","09","__","--")^1 / resolve -local variable = (P("$")/"") * (somevariable + (P("{")/"") * somevariable * (P("}")/"")) -local variableresolver = Cs((variable + P(1))^0) - + return instance.expansions[k] +end +local dollarstripper=lpeg.stripper("$") +local inhibitstripper=P("!")^0*Cs(P(1)^0) +local backslashswapper=lpeg.replacer("\\","/") +local somevariable=P("$")/"" +local somekey=C(R("az","AZ","09","__","--")^1) +local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) +local variableexpander=Cs((somevariable*(somekey/resolve)+somethingelse)^1 ) +local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" +local variablecleaner=Cs((cleaner+P(1))^0) +local somevariable=R("az","AZ","09","__","--")^1/resolve +local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) +local variableresolver=Cs((variable+P(1))^0) local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end - -function resolvers.newinstance() -- todo: all vars will become lowercase and alphanum only - - if trace_locating then - report_resolving("creating instance") - end - - local environment, variables, expansions, order = allocate(), allocate(), allocate(), allocate() - - local newinstance = { - environment = environment, - variables = variables, - expansions = expansions, - order = order, - files = allocate(), - setups = allocate(), - found = allocate(), - foundintrees = allocate(), - hashes = allocate(), - hashed = allocate(), - specification = allocate(), - lists = allocate(), - data = allocate(), -- only for loading - fakepaths = allocate(), - remember = true, - diskcache = true, - renewcache = false, - renewtree = false, - loaderror = false, - savelists = true, - pattern = nil, -- lists - force_suffixes = true, - } - - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v = order[i][k] - if v ~= nil then - t[k] = v - return v - end - end - if v == nil then - v = "" - end - t[k] = v - return v - end) - - setmetatableindex(environment, function(t,k) - local v = osgetenv(k) - if v == nil then - v = variables[k] - end - if v ~= nil then - v = checkedvariable(v) or "" - end - v = resolvers.repath(v) -- for taco who has a : separated osfontdir - t[k] = v - return v - end) - - setmetatableindex(expansions, function(t,k) - local v = environment[k] - if type(v) == "string" then - v = lpegmatch(variableresolver,v) - v = lpegmatch(variablecleaner,v) - end - t[k] = v + return lpegmatch(variableexpander,var) or var +end +function resolvers.newinstance() + if trace_locating then + report_resolving("creating instance") + end + local environment,variables,expansions,order=allocate(),allocate(),allocate(),allocate() + local newinstance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files=allocate(), + setups=allocate(), + found=allocate(), + foundintrees=allocate(), + hashes=allocate(), + hashed=allocate(), + specification=allocate(), + lists=allocate(), + data=allocate(), + fakepaths=allocate(), + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v return v - end) - - return newinstance - + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end + if v~=nil then + v=checkedvariable(v) or "" + end + v=resolvers.repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + return newinstance end - -function resolvers.setinstance(someinstance) -- only one instance is active - instance = someinstance - resolvers.instance = someinstance - return someinstance +function resolvers.setinstance(someinstance) + instance=someinstance + resolvers.instance=someinstance + return someinstance end - function resolvers.reset() - return resolvers.setinstance(resolvers.newinstance()) + return resolvers.setinstance(resolvers.newinstance()) end - local function reset_hashes() - instance.lists = { } - instance.found = { } -end - -local slash = P("/") - -local pathexpressionpattern = Cs ( - Cc("^") * ( - Cc("%") * S(".-") - + slash^2 * P(-1) / "/.*" - + slash^2 / "/.-/" - + (1-slash) * P(-1) * Cc("/") - + P(1) - )^1 * Cc("$") -- yes or no $ + instance.lists={} + instance.found={} +end +local slash=P("/") +local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*"+slash^2/"/.-/"+(1-slash)*P(-1)*Cc("/")+P(1) + )^1*Cc("$") ) - -local cache = { } - +local cache={} local function makepathexpression(str) - if str == "." then - return "^%./$" - else - local c = cache[str] - if not c then - c = lpegmatch(pathexpressionpattern,str) - cache[str] = c - end - return c + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c end + return c + end end - local function reportcriticalvariables(cnfspec) - if trace_locating then - for i=1,#resolvers.criticalvars do - local k = resolvers.criticalvars[i] - local v = resolvers.getenv(k) or "unknown" -- this one will not resolve ! - report_resolving("variable '%s' set to '%s'",k,v) - end - report_resolving() - if cnfspec then - if type(cnfspec) == "table" then - report_resolving("using configuration specification '%s'",concat(cnfspec,",")) - else - report_resolving("using configuration specification '%s'",cnfspec) - end - end - report_resolving() + if trace_locating then + for i=1,#resolvers.criticalvars do + local k=resolvers.criticalvars[i] + local v=resolvers.getenv(k) or "unknown" + report_resolving("variable '%s' set to '%s'",k,v) + end + report_resolving() + if cnfspec then + if type(cnfspec)=="table" then + report_resolving("using configuration specification '%s'",concat(cnfspec,",")) + else + report_resolving("using configuration specification '%s'",cnfspec) + end end - reportcriticalvariables = function() end + report_resolving() + end + reportcriticalvariables=function() end end - local function identify_configuration_files() - local specification = instance.specification - if #specification == 0 then - local cnfspec = getenv("TEXMFCNF") - if cnfspec == "" then - cnfspec = resolvers.luacnfspec - resolvers.luacnfstate = "default" - else - resolvers.luacnfstate = "environment" - end - reportcriticalvariables(cnfspec) - local cnfpaths = expandedpathfromlist(resolvers.splitpath(cnfspec)) - local luacnfname = resolvers.luacnfname - for i=1,#cnfpaths do - local filename = collapsepath(filejoin(cnfpaths[i],luacnfname)) - local realname = resolvers.resolve(filename) - if lfs.isfile(realname) then - specification[#specification+1] = filename - if trace_locating then - report_resolving("found configuration file '%s'",realname) - end - elseif trace_locating then - report_resolving("unknown configuration file '%s'",realname) - end - end + local specification=instance.specification + if #specification==0 then + local cnfspec=getenv("TEXMFCNF") + if cnfspec=="" then + cnfspec=resolvers.luacnfspec + resolvers.luacnfstate="default" + else + resolvers.luacnfstate="environment" + end + reportcriticalvariables(cnfspec) + local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local luacnfname=resolvers.luacnfname + for i=1,#cnfpaths do + local filename=collapsepath(filejoin(cnfpaths[i],luacnfname)) + local realname=resolvers.resolve(filename) + if lfs.isfile(realname) then + specification[#specification+1]=filename if trace_locating then - report_resolving() + report_resolving("found configuration file '%s'",realname) end - elseif trace_locating then - report_resolving("configuration files already identified") + elseif trace_locating then + report_resolving("unknown configuration file '%s'",realname) + end end + if trace_locating then + report_resolving() + end + elseif trace_locating then + report_resolving("configuration files already identified") + end end - local function load_configuration_files() - local specification = instance.specification - if #specification > 0 then - local luacnfname = resolvers.luacnfname - for i=1,#specification do - local filename = specification[i] - local pathname = filedirname(filename) - local filename = filejoin(pathname,luacnfname) - local realname = resolvers.resolve(filename) -- no shortcut - local blob = loadfile(realname) - if blob then - local setups = instance.setups - local data = blob() - local parent = data and data.parent - if parent then - local filename = filejoin(pathname,parent) - local realname = resolvers.resolve(filename) -- no shortcut - local blob = loadfile(realname) - if blob then - local parentdata = blob() - if parentdata then - report_resolving("loading configuration file '%s'",filename) - data = table.merged(parentdata,data) - end - end - end - data = data and data.content - if data then - if trace_locating then - report_resolving("loading configuration file '%s'",filename) - report_resolving() - end - local variables = data.variables or { } - local warning = false - for k, v in next, data do - local variant = type(v) - if variant == "table" then - initializesetter(filename,k,v) - elseif variables[k] == nil then - if trace_locating and not warning then - report_resolving("variables like '%s' in configuration file '%s' should move to the 'variables' subtable", - k,resolvers.resolve(filename)) - warning = true - end - variables[k] = v - end - end - setups[pathname] = variables - if resolvers.luacnfstate == "default" then - -- the following code is not tested - local cnfspec = variables["TEXMFCNF"] - if cnfspec then - if trace_locating then - report_resolving("reloading configuration due to TEXMF redefinition") - end - -- we push the value into the main environment (osenv) so - -- that it takes precedence over the default one and therefore - -- also over following definitions - resolvers.setenv("TEXMFCNF",cnfspec) -- resolves prefixes - -- we now identify and load the specified configuration files - instance.specification = { } - identify_configuration_files() - load_configuration_files() - -- we prevent further overload of the configuration variable - resolvers.luacnfstate = "configuration" - -- we quit the outer loop - break - end - end - - else - if trace_locating then - report_resolving("skipping configuration file '%s' (no content)",filename) - end - setups[pathname] = { } - instance.loaderror = true - end - elseif trace_locating then - report_resolving("skipping configuration file '%s' (no valid format)",filename) + local specification=instance.specification + if #specification>0 then + local luacnfname=resolvers.luacnfname + for i=1,#specification do + local filename=specification[i] + local pathname=filedirname(filename) + local filename=filejoin(pathname,luacnfname) + local realname=resolvers.resolve(filename) + local blob=loadfile(realname) + if blob then + local setups=instance.setups + local data=blob() + local parent=data and data.parent + if parent then + local filename=filejoin(pathname,parent) + local realname=resolvers.resolve(filename) + local blob=loadfile(realname) + if blob then + local parentdata=blob() + if parentdata then + report_resolving("loading configuration file '%s'",filename) + data=table.merged(parentdata,data) + end + end + end + data=data and data.content + if data then + if trace_locating then + report_resolving("loading configuration file '%s'",filename) + report_resolving() + end + local variables=data.variables or {} + local warning=false + for k,v in next,data do + local variant=type(v) + if variant=="table" then + initializesetter(filename,k,v) + elseif variables[k]==nil then + if trace_locating and not warning then + report_resolving("variables like '%s' in configuration file '%s' should move to the 'variables' subtable", + k,resolvers.resolve(filename)) + warning=true + end + variables[k]=v end - instance.order[#instance.order+1] = instance.setups[pathname] - if instance.loaderror then - break + end + setups[pathname]=variables + if resolvers.luacnfstate=="default" then + local cnfspec=variables["TEXMFCNF"] + if cnfspec then + if trace_locating then + report_resolving("reloading configuration due to TEXMF redefinition") + end + resolvers.setenv("TEXMFCNF",cnfspec) + instance.specification={} + identify_configuration_files() + load_configuration_files() + resolvers.luacnfstate="configuration" + break end + end + else + if trace_locating then + report_resolving("skipping configuration file '%s' (no content)",filename) + end + setups[pathname]={} + instance.loaderror=true end - elseif trace_locating then - report_resolving("warning: no lua configuration files found") + elseif trace_locating then + report_resolving("skipping configuration file '%s' (no valid format)",filename) + end + instance.order[#instance.order+1]=instance.setups[pathname] + if instance.loaderror then + break + end end + elseif trace_locating then + report_resolving("warning: no lua configuration files found") + end end - --- scheme magic ... database loading - local function load_file_databases() - instance.loaderror, instance.files = false, allocate() - if not instance.renewcache then - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - resolvers.hashers.byscheme(hash.type,hash.name) - if instance.loaderror then break end - end + instance.loaderror,instance.files=false,allocate() + if not instance.renewcache then + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + resolvers.hashers.byscheme(hash.type,hash.name) + if instance.loaderror then break end end + end end - local function locate_file_databases() - -- todo: cache:// and tree:// (runtime) - local texmfpaths = resolvers.expandedpathlist("TEXMF") - if #texmfpaths > 0 then - for i=1,#texmfpaths do - local path = collapsepath(texmfpaths[i]) - path = gsub(path,"/+$","") -- in case $HOME expands to something with a trailing / - local stripped = lpegmatch(inhibitstripper,path) -- the !! thing - if stripped ~= "" then - local runtime = stripped == path - path = resolvers.cleanpath(path) - local spec = resolvers.splitmethod(stripped) - if runtime and (spec.noscheme or spec.scheme == "file") then - stripped = "tree:///" .. stripped - elseif spec.scheme == "cache" or spec.scheme == "file" then - stripped = spec.path - end - if trace_locating then - if runtime then - report_resolving("locating list of '%s' (runtime) (%s)",path,stripped) - else - report_resolving("locating list of '%s' (cached)",path) - end - end - methodhandler('locators',stripped) - end + local texmfpaths=resolvers.expandedpathlist("TEXMF") + if #texmfpaths>0 then + for i=1,#texmfpaths do + local path=collapsepath(texmfpaths[i]) + path=gsub(path,"/+$","") + local stripped=lpegmatch(inhibitstripper,path) + if stripped~="" then + local runtime=stripped==path + path=resolvers.cleanpath(path) + local spec=resolvers.splitmethod(stripped) + if runtime and (spec.noscheme or spec.scheme=="file") then + stripped="tree:///"..stripped + elseif spec.scheme=="cache" or spec.scheme=="file" then + stripped=spec.path end if trace_locating then - report_resolving() + if runtime then + report_resolving("locating list of '%s' (runtime) (%s)",path,stripped) + else + report_resolving("locating list of '%s' (cached)",path) + end end - elseif trace_locating then - report_resolving("no texmf paths are defined (using TEXMF)") - end -end - -local function generate_file_databases() - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - methodhandler('generators',hash.name) + methodhandler('locators',stripped) + end end if trace_locating then - report_resolving() + report_resolving() end + elseif trace_locating then + report_resolving("no texmf paths are defined (using TEXMF)") + end end - -local function save_file_databases() -- will become cachers - for i=1,#instance.hashes do - local hash = instance.hashes[i] - local cachename = hash.name - if hash.cache then - local content = instance.files[cachename] - caches.collapsecontent(content) - if trace_locating then - report_resolving("saving tree '%s'",cachename) - end - caches.savecontent(cachename,"files",content) - elseif trace_locating then - report_resolving("not saving runtime tree '%s'",cachename) - end +local function generate_file_databases() + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + methodhandler('generators',hash.name) + end + if trace_locating then + report_resolving() + end +end +local function save_file_databases() + for i=1,#instance.hashes do + local hash=instance.hashes[i] + local cachename=hash.name + if hash.cache then + local content=instance.files[cachename] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree '%s'",cachename) + end + caches.savecontent(cachename,"files",content) + elseif trace_locating then + report_resolving("not saving runtime tree '%s'",cachename) end + end end - function resolvers.renew(hashname) - if hashname and hashname ~= "" then - local expanded = resolvers.expansion(hashname) or "" - if expanded ~= "" then - if trace_locating then - report_resolving("identifying tree '%s' from '%s'",expanded,hashname) - end - hashname = expanded - else - if trace_locating then - report_resolving("identifying tree '%s'",hashname) - end - end - local realpath = resolvers.resolve(hashname) - if lfs.isdir(realpath) then - if trace_locating then - report_resolving("using path '%s'",realpath) - end - methodhandler('generators',hashname) - -- could be shared - local content = instance.files[hashname] - caches.collapsecontent(content) - if trace_locating then - report_resolving("saving tree '%s'",hashname) - end - caches.savecontent(hashname,"files",content) - -- till here - else - report_resolving("invalid path '%s'",realpath) - end + if hashname and hashname~="" then + local expanded=resolvers.expansion(hashname) or "" + if expanded~="" then + if trace_locating then + report_resolving("identifying tree '%s' from '%s'",expanded,hashname) + end + hashname=expanded + else + if trace_locating then + report_resolving("identifying tree '%s'",hashname) + end + end + local realpath=resolvers.resolve(hashname) + if lfs.isdir(realpath) then + if trace_locating then + report_resolving("using path '%s'",realpath) + end + methodhandler('generators',hashname) + local content=instance.files[hashname] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree '%s'",hashname) + end + caches.savecontent(hashname,"files",content) + else + report_resolving("invalid path '%s'",realpath) end + end end - local function load_databases() - locate_file_databases() - if instance.diskcache and not instance.renewcache then - load_file_databases() - if instance.loaderror then - generate_file_databases() - save_file_databases() - end - else - generate_file_databases() - if instance.renewcache then - save_file_databases() - end + locate_file_databases() + if instance.diskcache and not instance.renewcache then + load_file_databases() + if instance.loaderror then + generate_file_databases() + save_file_databases() end + else + generate_file_databases() + if instance.renewcache then + save_file_databases() + end + end end - function resolvers.appendhash(type,name,cache) - -- safeguard ... tricky as it's actually a bug when seen twice - if not instance.hashed[name] then - if trace_locating then - report_resolving("hash '%s' appended",name) - end - insert(instance.hashes, { type = type, name = name, cache = cache } ) - instance.hashed[name] = cache + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash '%s' appended",name) end + insert(instance.hashes,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end end - function resolvers.prependhash(type,name,cache) - -- safeguard ... tricky as it's actually a bug when seen twice - if not instance.hashed[name] then - if trace_locating then - report_resolving("hash '%s' prepended",name) - end - insert(instance.hashes, 1, { type = type, name = name, cache = cache } ) - instance.hashed[name] = cache - end -end - -function resolvers.extendtexmfvariable(specification) -- crap, we could better prepend the hash - local t = resolvers.splitpath(getenv("TEXMF")) -- okay? - insert(t,1,specification) - local newspec = concat(t,",") -- not ; - if instance.environment["TEXMF"] then - instance.environment["TEXMF"] = newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"] = newspec - else - -- weird - end - reset_hashes() + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash '%s' prepended",name) + end + insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end +end +function resolvers.extendtexmfvariable(specification) + local t=resolvers.splitpath(getenv("TEXMF")) + insert(t,1,specification) + local newspec=concat(t,",") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"]=newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"]=newspec + else + end + reset_hashes() end - function resolvers.splitexpansions() - local ie = instance.expansions - for k,v in next, ie do - local t, tn, h, p = { }, 0, { }, splitconfigurationpath(v) - for kk=1,#p do - local vv = p[kk] - if vv ~= "" and not h[vv] then - tn = tn + 1 - t[tn] = vv - h[vv] = true - end - end - if #t > 1 then - ie[k] = t - else - ie[k] = t[1] - end + local ie=instance.expansions + for k,v in next,ie do + local t,tn,h,p={},0,{},splitconfigurationpath(v) + for kk=1,#p do + local vv=p[kk] + if vv~="" and not h[vv] then + tn=tn+1 + t[tn]=vv + h[vv]=true + end + end + if #t>1 then + ie[k]=t + else + ie[k]=t[1] end + end end - --- end of split/join code - --- we used to have 'files' and 'configurations' so therefore the following --- shared function - function resolvers.datastate() - return caches.contentstate() + return caches.contentstate() end - function resolvers.variable(name) - local name = name and lpegmatch(dollarstripper,name) - local result = name and instance.variables[name] - return result ~= nil and result or "" + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.variables[name] + return result~=nil and result or "" end - function resolvers.expansion(name) - local name = name and lpegmatch(dollarstripper,name) - local result = name and instance.expansions[name] - return result ~= nil and result or "" + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.expansions[name] + return result~=nil and result or "" end - function resolvers.unexpandedpathlist(str) - local pth = resolvers.variable(str) - local lst = resolvers.splitpath(pth) - return expandedpathfromlist(lst) + local pth=resolvers.variable(str) + local lst=resolvers.splitpath(pth) + return expandedpathfromlist(lst) end - function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(resolvers.unexpandedpathlist(str)) end - -local done = { } - +local done={} function resolvers.resetextrapath() - local ep = instance.extra_paths - if not ep then - ep, done = { }, { } - instance.extra_paths = ep - elseif #ep > 0 then - instance.lists, done = { }, { } - end + local ep=instance.extra_paths + if not ep then + ep,done={},{} + instance.extra_paths=ep + elseif #ep>0 then + instance.lists,done={},{} + end end - function resolvers.registerextrapath(paths,subpaths) - paths = settings_to_array(paths) - subpaths = settings_to_array(subpaths) - local ep = instance.extra_paths or { } - local oldn = #ep - local newn = oldn - local nofpaths = #paths - local nofsubpaths = #subpaths - if nofpaths > 0 then - if nofsubpaths > 0 then - for i=1,nofpaths do - local p = paths[i] - for j=1,nofsubpaths do - local s = subpaths[j] - local ps = p .. "/" .. s - if not done[ps] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(ps) - done[ps] = true - end - end - end - else - for i=1,nofpaths do - local p = paths[i] - if not done[p] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(p) - done[p] = true - end - end + paths=settings_to_array(paths) + subpaths=settings_to_array(subpaths) + local ep=instance.extra_paths or {} + local oldn=#ep + local newn=oldn + local nofpaths=#paths + local nofsubpaths=#subpaths + if nofpaths>0 then + if nofsubpaths>0 then + for i=1,nofpaths do + local p=paths[i] + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=p.."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(ps) + done[ps]=true + end end - elseif nofsubpaths > 0 then - for i=1,oldn do - for j=1,nofsubpaths do - local s = subpaths[j] - local ps = ep[i] .. "/" .. s - if not done[ps] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(ps) - done[ps] = true - end - end + end + else + for i=1,nofpaths do + local p=paths[i] + if not done[p] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(p) + done[p]=true end + end end - if newn > 0 then - instance.extra_paths = ep -- register paths - end - if newn > oldn then - instance.lists = { } -- erase the cache + elseif nofsubpaths>0 then + for i=1,oldn do + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=ep[i].."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(ps) + done[ps]=true + end + end end + end + if newn>0 then + instance.extra_paths=ep + end + if newn>oldn then + instance.lists={} + end end - local function made_list(instance,list) - local ep = instance.extra_paths - if not ep or #ep == 0 then - return list - else - local done, new, newn = { }, { }, 0 - -- honour . .. ../.. but only when at the start - for k=1,#list do - local v = list[k] - if not done[v] then - if find(v,"^[%.%/]$") then - done[v] = true - newn = newn + 1 - new[newn] = v - else - break - end - end - end - -- first the extra paths - for k=1,#ep do - local v = ep[k] - if not done[v] then - done[v] = true - newn = newn + 1 - new[newn] = v - end - end - -- next the formal paths - for k=1,#list do - local v = list[k] - if not done[v] then - done[v] = true - newn = newn + 1 - new[newn] = v - end + local ep=instance.extra_paths + if not ep or #ep==0 then + return list + else + local done,new,newn={},{},0 + for k=1,#list do + local v=list[k] + if not done[v] then + if find(v,"^[%.%/]$") then + done[v]=true + newn=newn+1 + new[newn]=v + else + break end - return new + end + end + for k=1,#ep do + local v=ep[k] + if not done[v] then + done[v]=true + newn=newn+1 + new[newn]=v + end + end + for k=1,#list do + local v=list[k] + if not done[v] then + done[v]=true + newn=newn+1 + new[newn]=v + end end + return new + end end - function resolvers.cleanpathlist(str) - local t = resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i] = collapsepath(resolvers.cleanpath(t[i])) - end + local t=resolvers.expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(resolvers.cleanpath(t[i])) end - return t + end + return t end - function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) + return joinpath(resolvers.expandedpathlist(str)) end - function resolvers.expandedpathlist(str) - if not str then - return { } - elseif instance.savelists then - str = lpegmatch(dollarstripper,str) - local lists = instance.lists - local lst = lists[str] - if not lst then - local l = made_list(instance,resolvers.splitpath(resolvers.expansion(str))) - lst = expandedpathfromlist(l) - lists[str] = lst - end - return lst - else - local lst = resolvers.splitpath(resolvers.expansion(str)) - return made_list(instance,expandedpathfromlist(lst)) + if not str then + return {} + elseif instance.savelists then + str=lpegmatch(dollarstripper,str) + local lists=instance.lists + local lst=lists[str] + if not lst then + local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str))) + lst=expandedpathfromlist(l) + lists[str]=lst end + return lst + else + local lst=resolvers.splitpath(resolvers.expansion(str)) + return made_list(instance,expandedpathfromlist(lst)) + end end - -function resolvers.expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ // - str = lpegmatch(dollarstripper,str) - local tmp = resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp ~= "" and tmp or str) +function resolvers.expandedpathlistfromvariable(str) + str=lpegmatch(dollarstripper,str) + local tmp=resolvers.variableofformatorsuffix(str) + return resolvers.expandedpathlist(tmp~="" and tmp or str) end - function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(resolvers.expandedpathlistfromvariable(str)) end - -function resolvers.expandbraces(str) -- output variable and brace expansion of STRING --- local ori = resolvers.variable(str) --- if ori == "" then - local ori = str --- end - local pth = expandedpathfromlist(resolvers.splitpath(ori)) - return joinpath(pth) +function resolvers.expandbraces(str) + local ori=str + local pth=expandedpathfromlist(resolvers.splitpath(ori)) + return joinpath(pth) end - function resolvers.registerfilehash(name,content,someerror) - if content then - instance.files[name] = content - else - instance.files[name] = { } - if somerror == true then -- can be unset - instance.loaderror = someerror - end + if content then + instance.files[name]=content + else + instance.files[name]={} + if somerror==true then + instance.loaderror=someerror end + end end - local function isreadable(name) - local readable = lfs.isfile(name) -- not file.is_readable(name) asit can be a dir - if trace_detail then - if readable then - report_resolving("file '%s' is readable",name) - else - report_resolving("file '%s' is not readable", name) - end + local readable=lfs.isfile(name) + if trace_detail then + if readable then + report_resolving("file '%s' is readable",name) + else + report_resolving("file '%s' is not readable",name) end - return readable + end + return readable end - --- name --- name/name - local function collect_files(names) - local filelist, noffiles = { }, 0 - for k=1,#names do - local fname = names[k] + local filelist,noffiles={},0 + for k=1,#names do + local fname=names[k] + if trace_detail then + report_resolving("checking name '%s'",fname) + end + local bname=filebasename(fname) + local dname=filedirname(fname) + if dname=="" or find(dname,"^%.") then + dname=false + else + dname=gsub(dname,"%*",".*") + dname="/"..dname.."$" + end + local hashes=instance.hashes + for h=1,#hashes do + local hash=hashes[h] + local blobpath=hash.name + local files=blobpath and instance.files[blobpath] + if files then if trace_detail then - report_resolving("checking name '%s'",fname) - end - local bname = filebasename(fname) - local dname = filedirname(fname) - if dname == "" or find(dname,"^%.") then - dname = false - else - dname = gsub(dname,"*","%.*") - dname = "/" .. dname .. "$" + report_resolving("deep checking '%s' (%s)",blobpath,bname) + end + local blobfile=files[bname] + if not blobfile then + local rname="remap:"..bname + blobfile=files[rname] + if blobfile then + bname=files[rname] + blobfile=files[bname] + end end - local hashes = instance.hashes - for h=1,#hashes do - local hash = hashes[h] - local blobpath = hash.name - local files = blobpath and instance.files[blobpath] - if files then + if blobfile then + local blobroot=files.__path__ or blobpath + if type(blobfile)=='string' then + if not dname or find(blobfile,dname) then + local variant=hash.type + local search=filejoin(blobroot,blobfile,bname) + local result=methodhandler('concatinators',hash.type,blobroot,blobfile,bname) + if trace_detail then + report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + else + for kk=1,#blobfile do + local vv=blobfile[kk] + if not dname or find(vv,dname) then + local variant=hash.type + local search=filejoin(blobroot,vv,bname) + local result=methodhandler('concatinators',hash.type,blobroot,vv,bname) if trace_detail then - report_resolving("deep checking '%s' (%s)",blobpath,bname) - end - local blobfile = files[bname] - if not blobfile then - local rname = "remap:"..bname - blobfile = files[rname] - if blobfile then - bname = files[rname] - blobfile = files[bname] - end - end - if blobfile then - local blobroot = files.__path__ or blobpath - if type(blobfile) == 'string' then - if not dname or find(blobfile,dname) then - local variant = hash.type - -- local search = filejoin(blobpath,blobfile,bname) - local search = filejoin(blobroot,blobfile,bname) - local result = methodhandler('concatinators',hash.type,blobroot,blobfile,bname) - if trace_detail then - report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) - end - noffiles = noffiles + 1 - filelist[noffiles] = { variant, search, result } - end - else - for kk=1,#blobfile do - local vv = blobfile[kk] - if not dname or find(vv,dname) then - local variant = hash.type - -- local search = filejoin(blobpath,vv,bname) - local search = filejoin(blobroot,vv,bname) - local result = methodhandler('concatinators',hash.type,blobroot,vv,bname) - if trace_detail then - report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) - end - noffiles = noffiles + 1 - filelist[noffiles] = { variant, search, result } - end - end - end + report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) end - elseif trace_locating then - report_resolving("no match in '%s' (%s)",blobpath,bname) + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end end + end end + elseif trace_locating then + report_resolving("no match in '%s' (%s)",blobpath,bname) + end end - return noffiles > 0 and filelist or nil + end + return noffiles>0 and filelist or nil end - -local fit = { } - +local fit={} function resolvers.registerintrees(filename,format,filetype,usedmethod,foundname) - local foundintrees = instance.foundintrees - if usedmethod == "direct" and filename == foundname and fit[foundname] then - -- just an extra lookup after a test on presence - else - local t = { - filename = filename, - format = format ~= "" and format or nil, - filetype = filetype ~= "" and filetype or nil, - usedmethod = usedmethod, - foundname = foundname, - } - fit[foundname] = t - foundintrees[#foundintrees+1] = t - end + local foundintrees=instance.foundintrees + if usedmethod=="direct" and filename==foundname and fit[foundname] then + else + local t={ + filename=filename, + format=format~="" and format or nil, + filetype=filetype~="" and filetype or nil, + usedmethod=usedmethod, + foundname=foundname, + } + fit[foundname]=t + foundintrees[#foundintrees+1]=t + end end - --- split the next one up for readability (but this module needs a cleanup anyway) - -local function can_be_dir(name) -- can become local - local fakepaths = instance.fakepaths - if not fakepaths[name] then - if lfs.isdir(name) then - fakepaths[name] = 1 -- directory - else - fakepaths[name] = 2 -- no directory - end +local function can_be_dir(name) + local fakepaths=instance.fakepaths + if not fakepaths[name] then + if lfs.isdir(name) then + fakepaths[name]=1 + else + fakepaths[name]=2 end - return fakepaths[name] == 1 + end + return fakepaths[name]==1 end - -local preparetreepattern = Cs((P(".")/"%%." + P("-")/"%%-" + P(1))^0 * Cc("$")) - --- -- -- begin of main file search routing -- -- -- needs checking as previous has been patched - +local preparetreepattern=Cs((P(".")/"%%."+P("-")/"%%-"+P(1))^0*Cc("$")) local collect_instance_files - local function find_analyze(filename,askedformat,allresults) - local filetype, wantedfiles, ext = '', { }, suffixonly(filename) - -- too tricky as filename can be bla.1.2.3: - -- - -- if not suffixmap[ext] then - -- wantedfiles[#wantedfiles+1] = filename - -- end - wantedfiles[#wantedfiles+1] = filename - if askedformat == "" then - if ext == "" or not suffixmap[ext] then - local defaultsuffixes = resolvers.defaultsuffixes - for i=1,#defaultsuffixes do - local forcedname = filename .. '.' .. defaultsuffixes[i] - wantedfiles[#wantedfiles+1] = forcedname - filetype = resolvers.formatofsuffix(forcedname) - if trace_locating then - report_resolving("forcing filetype '%s'",filetype) - end - end - else - filetype = resolvers.formatofsuffix(filename) - if trace_locating then - report_resolving("using suffix based filetype '%s'",filetype) - end + local filetype,wantedfiles,ext='',{},suffixonly(filename) + wantedfiles[#wantedfiles+1]=filename + if askedformat=="" then + if ext=="" or not suffixmap[ext] then + local defaultsuffixes=resolvers.defaultsuffixes + for i=1,#defaultsuffixes do + local forcedname=filename..'.'..defaultsuffixes[i] + wantedfiles[#wantedfiles+1]=forcedname + filetype=resolvers.formatofsuffix(forcedname) + if trace_locating then + report_resolving("forcing filetype '%s'",filetype) end + end else - if ext == "" or not suffixmap[ext] then - local format_suffixes = suffixes[askedformat] - if format_suffixes then - for i=1,#format_suffixes do - wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i] - end - end - end - filetype = askedformat - if trace_locating then - report_resolving("using given filetype '%s'",filetype) + filetype=resolvers.formatofsuffix(filename) + if trace_locating then + report_resolving("using suffix based filetype '%s'",filetype) + end + end + else + if ext=="" or not suffixmap[ext] then + local format_suffixes=suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + wantedfiles[#wantedfiles+1]=filename.."."..format_suffixes[i] end + end end - return filetype, wantedfiles + filetype=askedformat + if trace_locating then + report_resolving("using given filetype '%s'",filetype) + end + end + return filetype,wantedfiles end - local function find_direct(filename,allresults) - if not dangerous[askedformat] and isreadable(filename) then - if trace_detail then - report_resolving("file '%s' found directly",filename) - end - return "direct", { filename } + if not dangerous[askedformat] and isreadable(filename) then + if trace_detail then + report_resolving("file '%s' found directly",filename) end + return "direct",{ filename } + end end - local function find_wildcard(filename,allresults) - if find(filename,'%*') then - if trace_locating then - report_resolving("checking wildcard '%s'", filename) - end - local method, result = resolvers.findwildcardfiles(filename) - if result then - return "wildcard", result - end - end -end - -local function find_qualified(filename,allresults) -- this one will be split too - if not file.is_qualified_path(filename) then - return - end + if find(filename,'%*') then if trace_locating then - report_resolving("checking qualified name '%s'", filename) + report_resolving("checking wildcard '%s'",filename) end - if isreadable(filename) then - if trace_detail then - report_resolving("qualified file '%s' found", filename) - end - return "qualified", { filename } + local method,result=resolvers.findwildcardfiles(filename) + if result then + return "wildcard",result end + end +end +local function find_qualified(filename,allresults) + if not file.is_qualified_path(filename) then + return + end + if trace_locating then + report_resolving("checking qualified name '%s'",filename) + end + if isreadable(filename) then if trace_detail then - report_resolving("locating qualified file '%s'", filename) - end - local forcedname, suffix = "", suffixonly(filename) - if suffix == "" then -- why - local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat] - if format_suffixes then - for i=1,#format_suffixes do - local s = format_suffixes[i] - forcedname = filename .. "." .. s - if isreadable(forcedname) then - if trace_locating then - report_resolving("no suffix, forcing format filetype '%s'", s) - end - return "qualified", { forcedname } - end - end + report_resolving("qualified file '%s' found",filename) + end + return "qualified",{ filename } + end + if trace_detail then + report_resolving("locating qualified file '%s'",filename) + end + local forcedname,suffix="",suffixonly(filename) + if suffix=="" then + local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + local s=format_suffixes[i] + forcedname=filename.."."..s + if isreadable(forcedname) then + if trace_locating then + report_resolving("no suffix, forcing format filetype '%s'",s) + end + return "qualified",{ forcedname } end + end end - if suffix and suffix ~= "" then - -- try to find in tree (no suffix manipulation), here we search for the - -- matching last part of the name - local basename = filebasename(filename) - local pattern = lpegmatch(preparetreepattern,filename) - -- messy .. to be sorted out - local savedformat = askedformat - local format = savedformat or "" - if format == "" then - askedformat = resolvers.formatofsuffix(suffix) + end + if suffix and suffix~="" then + local basename=filebasename(filename) + local pattern=lpegmatch(preparetreepattern,filename) + local savedformat=askedformat + local format=savedformat or "" + if format=="" then + askedformat=resolvers.formatofsuffix(suffix) + end + if not format then + askedformat="othertextfiles" + end + if basename~=filename then + local resolved=collect_instance_files(basename,askedformat,allresults) + if #resolved==0 then + local lowered=lower(basename) + if filename~=lowered then + resolved=collect_instance_files(lowered,askedformat,allresults) end - if not format then - askedformat = "othertextfiles" -- kind of everything, maybe all + end + resolvers.format=savedformat + if #resolved>0 then + local result={} + for r=1,#resolved do + local rr=resolved[r] + if find(rr,pattern) then + result[#result+1]=rr + end end - -- - if basename ~= filename then - local resolved = collect_instance_files(basename,askedformat,allresults) - if #resolved == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered,askedformat,allresults) - end - end - resolvers.format = savedformat - -- - if #resolved > 0 then - local result = { } - for r=1,#resolved do - local rr = resolved[r] - if find(rr,pattern) then - result[#result+1] = rr - end - end - if #result > 0 then - return "qualified", result - end - end + if #result>0 then + return "qualified",result end - -- a real wildcard: - -- - -- local filelist = collect_files({basename}) - -- result = { } - -- for f=1,#filelist do - -- local ff = filelist[f][3] or "" - -- if find(ff,pattern) then - -- result[#result+1], ok = ff, true - -- end - -- end - -- if #result > 0 then - -- return "qualified", result - -- end + end end + end end - local function check_subpath(fname) - if isreadable(fname) then - if trace_detail then - report_resolving("found '%s' by deep scanning",fname) - end - return fname + if isreadable(fname) then + if trace_detail then + report_resolving("found '%s' by deep scanning",fname) end + return fname + end end - local function find_intree(filename,filetype,wantedfiles,allresults) - local typespec = resolvers.variableofformat(filetype) - local pathlist = resolvers.expandedpathlist(typespec) - local method = "intree" - if pathlist and #pathlist > 0 then - -- list search - local filelist = collect_files(wantedfiles) - local dirlist = { } - if filelist then - for i=1,#filelist do - dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble - end - end + local typespec=resolvers.variableofformat(filetype) + local pathlist=resolvers.expandedpathlist(typespec) + local method="intree" + if pathlist and #pathlist>0 then + local filelist=collect_files(wantedfiles) + local dirlist={} + if filelist then + for i=1,#filelist do + dirlist[i]=filedirname(filelist[i][3]).."/" + end + end + if trace_detail then + report_resolving("checking filename '%s'",filename) + end + local result={} + for k=1,#pathlist do + local path=pathlist[k] + local pathname=lpegmatch(inhibitstripper,path) + local doscan=path==pathname + if not find (pathname,'//$') then + doscan=false + end + local done=false + if filelist then + local expression=makepathexpression(pathname) if trace_detail then - report_resolving("checking filename '%s'",filename) - end - local result = { } - for k=1,#pathlist do - local path = pathlist[k] - local pathname = lpegmatch(inhibitstripper,path) - local doscan = path == pathname -- no ^!! - if not find (pathname,'//$') then - doscan = false -- we check directly on the path + report_resolving("using pattern '%s' for path '%s'",expression,pathname) + end + for k=1,#filelist do + local fl=filelist[k] + local f=fl[2] + local d=dirlist[k] + if find(d,expression) then + result[#result+1]=resolvers.resolve(fl[3]) + done=true + if allresults then + if trace_detail then + report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d) + end + else + if trace_detail then + report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d) + end + break end - local done = false - -- using file list - if filelist then -- database - -- compare list entries with permitted pattern -- /xx /xx// - local expression = makepathexpression(pathname) - if trace_detail then - report_resolving("using pattern '%s' for path '%s'",expression,pathname) - end - for k=1,#filelist do - local fl = filelist[k] - local f = fl[2] - local d = dirlist[k] - if find(d,expression) then - -- todo, test for readable - result[#result+1] = resolvers.resolve(fl[3]) -- no shortcut - done = true - if allresults then - if trace_detail then - report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d) - end - else - if trace_detail then - report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d) - end + elseif trace_detail then + report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d) + end + end + end + if done then + method="database" + else + method="filesystem" + pathname=gsub(pathname,"/+$","") + pathname=resolvers.resolve(pathname) + local scheme=url.hasscheme(pathname) + if not scheme or scheme=="file" then + local pname=gsub(pathname,"%.%*$",'') + if not find(pname,"%*") then + if can_be_dir(pname) then + for k=1,#wantedfiles do + local w=wantedfiles[k] + local fname=check_subpath(filejoin(pname,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end + if not done and doscan then + local files=resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w=wantedfiles[k] + local subpath=files[w] + if not subpath or subpath=="" then + elseif type(subpath)=="string" then + local fname=check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp=subpath[i] + if sp=="" then + else + local fname=check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then break + end end - elseif trace_detail then - report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d) + end end - end - end - if done then - method = "database" - else - method = "filesystem" -- bonus, even when !! is specified - pathname = gsub(pathname,"/+$","") - pathname = resolvers.resolve(pathname) - local scheme = url.hasscheme(pathname) - if not scheme or scheme == "file" then - local pname = gsub(pathname,"%.%*$",'') - if not find(pname,"%*") then - if can_be_dir(pname) then - -- quick root scan first - for k=1,#wantedfiles do - local w = wantedfiles[k] - local fname = check_subpath(filejoin(pname,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - end - if not done and doscan then - -- collect files in path (and cache the result) - local files = resolvers.simplescanfiles(pname,false,true) - for k=1,#wantedfiles do - local w = wantedfiles[k] - local subpath = files[w] - if not subpath or subpath == "" then - -- rootscan already done - elseif type(subpath) == "string" then - local fname = check_subpath(filejoin(pname,subpath,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - else - for i=1,#subpath do - local sp = subpath[i] - if sp == "" then - -- roottest already done - else - local fname = check_subpath(filejoin(pname,sp,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - end - end - if done and not allresults then - break - end - end - end - end - end - else - -- no access needed for non existing path, speedup (esp in large tree with lots of fake) + if done and not allresults then + break end + end end + end end - -- todo recursive scanning - if done and not allresults then - break - end - end - if #result > 0 then - return method, result + else + end end + end + if done and not allresults then + break + end end + if #result>0 then + return method,result + end + end end - local function find_onpath(filename,filetype,wantedfiles,allresults) - if trace_detail then - report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) - end - local result = { } - for k=1,#wantedfiles do - local fname = wantedfiles[k] - if fname and isreadable(fname) then - filename = fname - result[#result+1] = filejoin('.',fname) - if not allresults then - break - end - end + if trace_detail then + report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename,filetype or '?',concat(wantedfiles," | ")) + end + local result={} + for k=1,#wantedfiles do + local fname=wantedfiles[k] + if fname and isreadable(fname) then + filename=fname + result[#result+1]=filejoin('.',fname) + if not allresults then + break + end end - if #result > 0 then - return "onpath", result - end -end - -local function find_otherwise(filename,filetype,wantedfiles,allresults) -- other text files | any | whatever - local filelist = collect_files(wantedfiles) - local fl = filelist and filelist[1] - if fl then - return "otherwise", { resolvers.resolve(fl[3]) } -- filename - end -end - --- we could have a loop over the 6 functions but then we'd have to --- always analyze - -collect_instance_files = function(filename,askedformat,allresults) -- uses nested - askedformat = askedformat or "" - filename = collapsepath(filename) - if allresults then - -- no need for caching, only used for tracing - local filetype, wantedfiles = find_analyze(filename,askedformat) - local results = { - { find_direct (filename,true) }, - { find_wildcard (filename,true) }, - { find_qualified(filename,true) }, - { find_intree (filename,filetype,wantedfiles,true) }, - { find_onpath (filename,filetype,wantedfiles,true) }, - { find_otherwise(filename,filetype,wantedfiles,true) }, - } - local result, status, done = { }, { }, { } - for k, r in next, results do - local method, list = r[1], r[2] - if method and list then - for i=1,#list do - local c = collapsepath(list[i]) - if not done[c] then - result[#result+1] = c - done[c] = true - end - status[#status+1] = format("%-10s: %s",method,c) - end - end - end - if trace_detail then - report_resolving("lookup status: %s",table.serialize(status,filename)) + end + if #result>0 then + return "onpath",result + end +end +local function find_otherwise(filename,filetype,wantedfiles,allresults) + local filelist=collect_files(wantedfiles) + local fl=filelist and filelist[1] + if fl then + return "otherwise",{ resolvers.resolve(fl[3]) } + end +end +collect_instance_files=function(filename,askedformat,allresults) + askedformat=askedformat or "" + filename=collapsepath(filename) + if allresults then + local filetype,wantedfiles=find_analyze(filename,askedformat) + local results={ + { find_direct (filename,true) }, + { find_wildcard (filename,true) }, + { find_qualified(filename,true) }, + { find_intree (filename,filetype,wantedfiles,true) }, + { find_onpath (filename,filetype,wantedfiles,true) }, + { find_otherwise(filename,filetype,wantedfiles,true) }, + } + local result,status,done={},{},{} + for k,r in next,results do + local method,list=r[1],r[2] + if method and list then + for i=1,#list do + local c=collapsepath(list[i]) + if not done[c] then + result[#result+1]=c + done[c]=true + end + status[#status+1]=format("%-10s: %s",method,c) end - return result, status - else - local method, result, stamp, filetype, wantedfiles - if instance.remember then - stamp = format("%s--%s", filename, askedformat) - result = stamp and instance.found[stamp] - if result then - if trace_locating then - report_resolving("remembered file '%s'",filename) - end - return result - end + end + end + if trace_detail then + report_resolving("lookup status: %s",table.serialize(status,filename)) + end + return result,status + else + local method,result,stamp,filetype,wantedfiles + if instance.remember then + stamp=format("%s--%s",filename,askedformat) + result=stamp and instance.found[stamp] + if result then + if trace_locating then + report_resolving("remembered file '%s'",filename) end - method, result = find_direct(filename) + return result + end + end + method,result=find_direct(filename) + if not result then + method,result=find_wildcard(filename) + if not result then + method,result=find_qualified(filename) if not result then - method, result = find_wildcard(filename) + filetype,wantedfiles=find_analyze(filename,askedformat) + method,result=find_intree(filename,filetype,wantedfiles) + if not result then + method,result=find_onpath(filename,filetype,wantedfiles) if not result then - method, result = find_qualified(filename) - if not result then - filetype, wantedfiles = find_analyze(filename,askedformat) - method, result = find_intree(filename,filetype,wantedfiles) - if not result then - method, result = find_onpath(filename,filetype,wantedfiles) - if not result then - method, result = find_otherwise(filename,filetype,wantedfiles) - end - end - end - end - end - if result and #result > 0 then - local foundname = collapsepath(result[1]) - resolvers.registerintrees(filename,askedformat,filetype,method,foundname) - result = { foundname } - else - result = { } -- maybe false - end - if stamp then - if trace_locating then - report_resolving("remembering file '%s'",filename) + method,result=find_otherwise(filename,filetype,wantedfiles) end - instance.found[stamp] = result + end end - return result + end + end + if result and #result>0 then + local foundname=collapsepath(result[1]) + resolvers.registerintrees(filename,askedformat,filetype,method,foundname) + result={ foundname } + else + result={} + end + if stamp then + if trace_locating then + report_resolving("remembering file '%s'",filename) + end + instance.found[stamp]=result end + return result + end end - --- -- -- end of main file search routing -- -- -- - - local function findfiles(filename,filetype,allresults) - local result, status = collect_instance_files(filename,filetype or "",allresults) - if not result or #result == 0 then - local lowered = lower(filename) - if filename ~= lowered then - result, status = collect_instance_files(lowered,filetype or "",allresults) - end + local result,status=collect_instance_files(filename,filetype or "",allresults) + if not result or #result==0 then + local lowered=lower(filename) + if filename~=lowered then + result,status=collect_instance_files(lowered,filetype or "",allresults) end - return result or { }, status + end + return result or {},status end - function resolvers.findfiles(filename,filetype) - return findfiles(filename,filetype,true) + return findfiles(filename,filetype,true) end - function resolvers.findfile(filename,filetype) - return findfiles(filename,filetype,false)[1] or "" + return findfiles(filename,filetype,false)[1] or "" end - function resolvers.findpath(filename,filetype) - return filedirname(findfiles(filename,filetype,false)[1] or "") + return filedirname(findfiles(filename,filetype,false)[1] or "") end - local function findgivenfiles(filename,allresults) - local bname, result = filebasename(filename), { } - local hashes = instance.hashes - local noffound = 0 - for k=1,#hashes do - local hash = hashes[k] - local files = instance.files[hash.name] or { } - local blist = files[bname] - if not blist then - local rname = "remap:"..bname - blist = files[rname] - if blist then - bname = files[rname] - blist = files[bname] - end - end - if blist then - if type(blist) == 'string' then - local found = methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" - if found ~= "" then - noffound = noffound + 1 - result[noffound] = resolvers.resolve(found) - if not allresults then break end - end - else - for kk=1,#blist do - local vv = blist[kk] - local found = methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" - if found ~= "" then - noffound = noffound + 1 - result[noffound] = resolvers.resolve(found) - if not allresults then break end - end - end - end + local bname,result=filebasename(filename),{} + local hashes=instance.hashes + local noffound=0 + for k=1,#hashes do + local hash=hashes[k] + local files=instance.files[hash.name] or {} + local blist=files[bname] + if not blist then + local rname="remap:"..bname + blist=files[rname] + if blist then + bname=files[rname] + blist=files[bname] + end + end + if blist then + if type(blist)=='string' then + local found=methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" + if found~="" then + noffound=noffound+1 + result[noffound]=resolvers.resolve(found) + if not allresults then break end + end + else + for kk=1,#blist do + local vv=blist[kk] + local found=methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" + if found~="" then + noffound=noffound+1 + result[noffound]=resolvers.resolve(found) + if not allresults then break end + end end + end end - return result + end + return result end - function resolvers.findgivenfiles(filename) - return findgivenfiles(filename,true) + return findgivenfiles(filename,true) end - function resolvers.findgivenfile(filename) - return findgivenfiles(filename,false)[1] or "" + return findgivenfiles(filename,false)[1] or "" end - local function doit(path,blist,bname,tag,variant,result,allresults) - local done = false - if blist and variant then - local resolve = resolvers.resolve -- added - if type(blist) == 'string' then - -- make function and share code - if find(lower(blist),path) then - local full = methodhandler('concatinators',variant,tag,blist,bname) or "" - result[#result+1] = resolve(full) - done = true - end - else - for kk=1,#blist do - local vv = blist[kk] - if find(lower(vv),path) then - local full = methodhandler('concatinators',variant,tag,vv,bname) or "" - result[#result+1] = resolve(full) - done = true - if not allresults then break end - end - end + local done=false + if blist and variant then + local resolve=resolvers.resolve + if type(blist)=='string' then + if find(lower(blist),path) then + local full=methodhandler('concatinators',variant,tag,blist,bname) or "" + result[#result+1]=resolve(full) + done=true + end + else + for kk=1,#blist do + local vv=blist[kk] + if find(lower(vv),path) then + local full=methodhandler('concatinators',variant,tag,vv,bname) or "" + result[#result+1]=resolve(full) + done=true + if not allresults then break end end + end end - return done + end + return done end - - -local makewildcard = Cs( - (P("^")^0 * P("/") * P(-1) + P(-1)) /".*" - + (P("^")^0 * P("/") / "")^0 * (P("*")/".*" + P("-")/"%%-" + P(".")/"%%." + P("?")/"."+ P("\\")/"/" + P(1))^0 +local makewildcard=Cs( + (P("^")^0*P("/")*P(-1)+P(-1))/".*"+(P("^")^0*P("/")/"")^0*(P("*")/".*"+P("-")/"%%-"+P(".")/"%%."+P("?")/"."+P("\\")/"/"+P(1))^0 ) - function resolvers.wildcardpattern(pattern) - return lpegmatch(makewildcard,pattern) or pattern -end - -local function findwildcardfiles(filename,allresults,result) -- todo: remap: and lpeg - result = result or { } - local base = filebasename(filename) - local dirn = filedirname(filename) - local path = lower(lpegmatch(makewildcard,dirn) or dirn) - local name = lower(lpegmatch(makewildcard,base) or base) - local files, done = instance.files, false - if find(name,"%*") then - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - local hashname, hashtype = hash.name, hash.type - for kk, hh in next, files[hashname] do - if not find(kk,"^remap:") then - if find(lower(kk),name) then - if doit(path,hh,kk,hashname,hashtype,result,allresults) then done = true end - if done and not allresults then break end - end - end - end - end - else - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - local hashname, hashtype = hash.name, hash.type - if doit(path,files[hashname][bname],bname,hashname,hashtype,result,allresults) then done = true end + return lpegmatch(makewildcard,pattern) or pattern +end +local function findwildcardfiles(filename,allresults,result) + result=result or {} + local base=filebasename(filename) + local dirn=filedirname(filename) + local path=lower(lpegmatch(makewildcard,dirn) or dirn) + local name=lower(lpegmatch(makewildcard,base) or base) + local files,done=instance.files,false + if find(name,"%*") then + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + local hashname,hashtype=hash.name,hash.type + for kk,hh in next,files[hashname] do + if not find(kk,"^remap:") then + if find(lower(kk),name) then + if doit(path,hh,kk,hashname,hashtype,result,allresults) then done=true end if done and not allresults then break end + end end + end end - -- we can consider also searching the paths not in the database, but then - -- we end up with a messy search (all // in all path specs) - return result + else + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + local hashname,hashtype=hash.name,hash.type + if doit(path,files[hashname][bname],bname,hashname,hashtype,result,allresults) then done=true end + if done and not allresults then break end + end + end + return result end - function resolvers.findwildcardfiles(filename,result) - return findwildcardfiles(filename,true,result) + return findwildcardfiles(filename,true,result) end - function resolvers.findwildcardfile(filename) - return findwildcardfiles(filename,false)[1] or "" + return findwildcardfiles(filename,false)[1] or "" end - --- main user functions - function resolvers.automount() - -- implemented later end - function resolvers.load(option) - statistics.starttiming(instance) - identify_configuration_files() - load_configuration_files() - if option ~= "nofiles" then - load_databases() - resolvers.automount() - end - statistics.stoptiming(instance) - local files = instance.files - return files and next(files) and true + statistics.starttiming(instance) + identify_configuration_files() + load_configuration_files() + if option~="nofiles" then + load_databases() + resolvers.automount() + end + statistics.stoptiming(instance) + local files=instance.files + return files and next(files) and true end - function resolvers.loadtime() - return statistics.elapsedtime(instance) + return statistics.elapsedtime(instance) end - local function report(str) + if trace_locating then + report_resolving(str) + else + print(str) + end +end +function resolvers.dowithfilesandreport(command,files,...) + if files and #files>0 then if trace_locating then - report_resolving(str) -- has already verbose - else - print(str) + report('') end -end - -function resolvers.dowithfilesandreport(command, files, ...) -- will move - if files and #files > 0 then - if trace_locating then - report('') -- ? - end - if type(files) == "string" then - files = { files } - end - for f=1,#files do - local file = files[f] - local result = command(file,...) - if type(result) == 'string' then - report(result) - else - for i=1,#result do - report(result[i]) -- could be unpack - end - end + if type(files)=="string" then + files={ files } + end + for f=1,#files do + local file=files[f] + local result=command(file,...) + if type(result)=='string' then + report(result) + else + for i=1,#result do + report(result[i]) end + end end + end end - --- obsolete - --- resolvers.varvalue = resolvers.variable -- output the value of variable $STRING. --- resolvers.expandvar = resolvers.expansion -- output variable expansion of STRING. - -function resolvers.showpath(str) -- output search path for file type NAME - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) +function resolvers.showpath(str) + return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) end - -function resolvers.registerfile(files, name, path) - if files[name] then - if type(files[name]) == 'string' then - files[name] = { files[name], path } - else - files[name] = path - end +function resolvers.registerfile(files,name,path) + if files[name] then + if type(files[name])=='string' then + files[name]={ files[name],path } else - files[name] = path + files[name]=path end + else + files[name]=path + end end - function resolvers.dowithpath(name,func) - local pathlist = resolvers.expandedpathlist(name) - for i=1,#pathlist do - func("^"..resolvers.cleanpath(pathlist[i])) - end + local pathlist=resolvers.expandedpathlist(name) + for i=1,#pathlist do + func("^"..resolvers.cleanpath(pathlist[i])) + end end - function resolvers.dowithvariable(name,func) - func(expandedvariable(name)) + func(expandedvariable(name)) end - function resolvers.locateformat(name) - local engine = environment.ownmain or "luatex" - local barename = file.removesuffix(name) - local fullname = file.addsuffix(barename,"fmt") - local fmtname = caches.getfirstreadablefile(fullname,"formats",engine) or "" - if fmtname == "" then - fmtname = resolvers.findfile(fullname) - fmtname = resolvers.cleanpath(fmtname) - end - if fmtname ~= "" then - local barename = file.removesuffix(fmtname) - local luaname = file.addsuffix(barename,luasuffixes.lua) - local lucname = file.addsuffix(barename,luasuffixes.luc) - local luiname = file.addsuffix(barename,luasuffixes.lui) - if lfs.isfile(luiname) then - return barename, luiname - elseif lfs.isfile(lucname) then - return barename, lucname - elseif lfs.isfile(luaname) then - return barename, luaname - end - end - return nil, nil + local engine=environment.ownmain or "luatex" + local barename=file.removesuffix(name) + local fullname=file.addsuffix(barename,"fmt") + local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" + if fmtname=="" then + fmtname=resolvers.findfile(fullname) + fmtname=resolvers.cleanpath(fmtname) + end + if fmtname~="" then + local barename=file.removesuffix(fmtname) + local luaname=file.addsuffix(barename,luasuffixes.lua) + local lucname=file.addsuffix(barename,luasuffixes.luc) + local luiname=file.addsuffix(barename,luasuffixes.lui) + if lfs.isfile(luiname) then + return barename,luiname + elseif lfs.isfile(lucname) then + return barename,lucname + elseif lfs.isfile(luaname) then + return barename,luaname + end + end + return nil,nil end - function resolvers.booleanvariable(str,default) - local b = resolvers.expansion(str) - if b == "" then - return default - else - b = toboolean(b) - return (b == nil and default) or b - end -end - -function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead - local instance = resolvers.instance - local hashes = instance.hashes - for i=1,#hashes do - local hash = hashes[i] - local blobtype = hash.type - local blobpath = hash.name - if blobpath then - if before then - before(blobtype,blobpath,pattern) - end - local files = instance.files[blobpath] - local total, checked, done = 0, 0, 0 - if files then - for k,v in next, files do - total = total + 1 - if find(k,"^remap:") then - k = files[k] - v = k -- files[k] -- chained - end - if find(k,pattern) then - if type(v) == "string" then - checked = checked + 1 - if handle(blobtype,blobpath,v,k) then - done = done + 1 - end - else - checked = checked + #v - for i=1,#v do - if handle(blobtype,blobpath,v[i],k) then - done = done + 1 - end - end - end - end + local b=resolvers.expansion(str) + if b=="" then + return default + else + b=toboolean(b) + return (b==nil and default) or b + end +end +function resolvers.dowithfilesintree(pattern,handle,before,after) + local instance=resolvers.instance + local hashes=instance.hashes + for i=1,#hashes do + local hash=hashes[i] + local blobtype=hash.type + local blobpath=hash.name + if blobpath then + if before then + before(blobtype,blobpath,pattern) + end + local files=instance.files[blobpath] + local total,checked,done=0,0,0 + if files then + for k,v in next,files do + total=total+1 + if find(k,"^remap:") then + k=files[k] + v=k + end + if find(k,pattern) then + if type(v)=="string" then + checked=checked+1 + if handle(blobtype,blobpath,v,k) then + done=done+1 + end + else + checked=checked+#v + for i=1,#v do + if handle(blobtype,blobpath,v[i],k) then + done=done+1 end + end end - if after then - after(blobtype,blobpath,pattern,total,checked,done) - end + end end + end + if after then + after(blobtype,blobpath,pattern,total,checked,done) + end end + end end - -resolvers.obsolete = resolvers.obsolete or { } -local obsolete = resolvers.obsolete - -resolvers.find_file = resolvers.findfile obsolete.find_file = resolvers.findfile -resolvers.find_files = resolvers.findfiles obsolete.find_files = resolvers.findfiles +resolvers.obsolete=resolvers.obsolete or {} +local obsolete=resolvers.obsolete +resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile +resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-pre'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- It could be interesting to hook the resolver in the file --- opener so that unresolved prefixes travel around and we --- get more abstraction. - --- As we use this beforehand we will move this up in the chain --- of loading. - - -local resolvers = resolvers -local prefixes = utilities.storage.allocate() -resolvers.prefixes = prefixes +-- original size: 6430, stripped down to: 4219 -local cleanpath, findgivenfile, expansion = resolvers.cleanpath, resolvers.findgivenfile, resolvers.expansion -local getenv = resolvers.getenv -- we can probably also use resolvers.expansion -local P, S, R, C, Cs, lpegmatch = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cs, lpeg.match -local joinpath, basename, dirname = file.join, file.basename, file.dirname -local getmetatable, rawset, type = getmetatable, rawset, type - --- getenv = function(...) return resolvers.getenv(...) end -- needs checking (definitions changes later on) - -prefixes.environment = function(str) - return cleanpath(expansion(str)) -end - -prefixes.relative = function(str,n) -- lfs.isfile - if io.exists(str) then - -- nothing - elseif io.exists("./" .. str) then - str = "./" .. str - else - local p = "../" - for i=1,n or 2 do - if io.exists(p .. str) then - str = p .. str - break - else - p = p .. "../" - end - end +if not modules then modules={} end modules ['data-pre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers=resolvers +local prefixes=utilities.storage.allocate() +resolvers.prefixes=prefixes +local cleanpath,findgivenfile,expansion=resolvers.cleanpath,resolvers.findgivenfile,resolvers.expansion +local getenv=resolvers.getenv +local P,S,R,C,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.match +local joinpath,basename,dirname=file.join,file.basename,file.dirname +local getmetatable,rawset,type=getmetatable,rawset,type +prefixes.environment=function(str) + return cleanpath(expansion(str)) +end +prefixes.relative=function(str,n) + if io.exists(str) then + elseif io.exists("./"..str) then + str="./"..str + else + local p="../" + for i=1,n or 2 do + if io.exists(p..str) then + str=p..str + break + else + p=p.."../" + end end - return cleanpath(str) + end + return cleanpath(str) end - -prefixes.auto = function(str) - local fullname = prefixes.relative(str) - if not lfs.isfile(fullname) then - fullname = prefixes.locate(str) - end - return fullname +prefixes.auto=function(str) + local fullname=prefixes.relative(str) + if not lfs.isfile(fullname) then + fullname=prefixes.locate(str) + end + return fullname end - -prefixes.locate = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath((fullname ~= "" and fullname) or str) +prefixes.locate=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath((fullname~="" and fullname) or str) end - -prefixes.filename = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath(basename((fullname ~= "" and fullname) or str)) -- no cleanpath needed here +prefixes.filename=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(basename((fullname~="" and fullname) or str)) end - -prefixes.pathname = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath(dirname((fullname ~= "" and fullname) or str)) +prefixes.pathname=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(dirname((fullname~="" and fullname) or str)) end - -prefixes.selfautoloc = function(str) - return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) +prefixes.selfautoloc=function(str) + return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) end - -prefixes.selfautoparent = function(str) - return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) +prefixes.selfautoparent=function(str) + return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) end - -prefixes.selfautodir = function(str) - return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) +prefixes.selfautodir=function(str) + return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) end - -prefixes.home = function(str) - return cleanpath(joinpath(getenv('HOME'),str)) +prefixes.home=function(str) + return cleanpath(joinpath(getenv('HOME'),str)) end - local function toppath() - local inputstack = resolvers.inputstack -- dependency, actually the code should move but it's - if not inputstack then -- more convenient to keep it here - return "." - end - local pathname = dirname(inputstack[#inputstack] or "") - if pathname == "" then - return "." - else - return pathname - end -end - -resolvers.toppath = toppath - -prefixes.toppath = function(str) - return cleanpath(joinpath(toppath(),str)) -end - -prefixes.env = prefixes.environment -prefixes.rel = prefixes.relative -prefixes.loc = prefixes.locate -prefixes.kpse = prefixes.locate -prefixes.full = prefixes.locate -prefixes.file = prefixes.filename -prefixes.path = prefixes.pathname - + local inputstack=resolvers.inputstack + if not inputstack then + return "." + end + local pathname=dirname(inputstack[#inputstack] or "") + if pathname=="" then + return "." + else + return pathname + end +end +resolvers.toppath=toppath +prefixes.toppath=function(str) + return cleanpath(joinpath(toppath(),str)) +end +prefixes.env=prefixes.environment +prefixes.rel=prefixes.relative +prefixes.loc=prefixes.locate +prefixes.kpse=prefixes.locate +prefixes.full=prefixes.locate +prefixes.file=prefixes.filename +prefixes.path=prefixes.pathname function resolvers.allprefixes(separator) - local all = table.sortedkeys(prefixes) - if separator then - for i=1,#all do - all[i] = all[i] .. ":" - end + local all=table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i]=all[i]..":" end - return all + end + return all end - local function _resolve_(method,target) - local action = prefixes[method] - if action then - return action(target) - else - return method .. ":" .. target - end -end - -local resolved, abstract = { }, { } - + local action=prefixes[method] + if action then + return action(target) + else + return method..":"..target + end +end +local resolved,abstract={},{} function resolvers.resetresolve(str) - resolved, abstract = { }, { } + resolved,abstract={},{} end - --- todo: use an lpeg (see data-lua for !! / stripper) - --- local function resolve(str) -- use schemes, this one is then for the commandline only --- if type(str) == "table" then --- local t = { } --- for i=1,#str do --- t[i] = resolve(str[i]) --- end --- return t --- else --- local res = resolved[str] --- if not res then --- res = gsub(str,"([a-z][a-z]+):([^ \"\';,]*)",_resolve_) -- home:xx;selfautoparent:xx; etc (comma added) --- resolved[str] = res --- abstract[res] = str --- end --- return res --- end --- end - --- home:xx;selfautoparent:xx; - -local pattern = Cs((C(R("az")^2) * P(":") * C((1-S(" \"\';,"))^1) / _resolve_ + P(1))^0) - -local function resolve(str) -- use schemes, this one is then for the commandline only - if type(str) == "table" then - local t = { } - for i=1,#str do - t[i] = resolve(str[i]) - end - return t - else - local res = resolved[str] - if not res then - res = lpegmatch(pattern,str) - resolved[str] = res - abstract[res] = str - end - return res +local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) +local function resolve(str) + if type(str)=="table" then + local t={} + for i=1,#str do + t[i]=resolve(str[i]) + end + return t + else + local res=resolved[str] + if not res then + res=lpegmatch(pattern,str) + resolved[str]=res + abstract[res]=str end + return res + end end - local function unresolve(str) - return abstract[str] or str + return abstract[str] or str end - -resolvers.resolve = resolve -resolvers.unresolve = unresolve - -if type(os.uname) == "function" then - - for k, v in next, os.uname() do - if not prefixes[k] then - prefixes[k] = function() return v end - end +resolvers.resolve=resolve +resolvers.unresolve=unresolve +if type(os.uname)=="function" then + for k,v in next,os.uname() do + if not prefixes[k] then + prefixes[k]=function() return v end end - + end end - -if os.type == "unix" then - - -- We need to distringuish between a prefix and something else : so we - -- have a special repath variant for linux. Also, when a new prefix is - -- defined, we need to remake the matcher. - - local pattern - - local function makepattern(t,k,v) - if t then - rawset(t,k,v) - end - local colon = P(":") - for k, v in table.sortedpairs(prefixes) do - if p then - p = P(k) + p - else - p = P(k) - end - end - pattern = Cs((p * colon + colon/";" + P(1))^0) - end - - makepattern() - - getmetatable(prefixes).__newindex = makepattern - - function resolvers.repath(str) - return lpegmatch(pattern,str) - end - -else -- already the default: - - function resolvers.repath(str) - return str +if os.type=="unix" then + local pattern + local function makepattern(t,k,v) + if t then + rawset(t,k,v) + end + local colon=P(":") + for k,v in table.sortedpairs(prefixes) do + if p then + p=P(k)+p + else + p=P(k) + end end - + pattern=Cs((p*colon+colon/";"+P(1))^0) + end + makepattern() + getmetatable(prefixes).__newindex=makepattern + function resolvers.repath(str) + return lpegmatch(pattern,str) + end +else + function resolvers.repath(str) + return str + end end @@ -16917,172 +13133,153 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-inp'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local allocate = utilities.storage.allocate -local resolvers = resolvers - -local methodhandler = resolvers.methodhandler -local registermethod = resolvers.registermethod - -local finders = allocate { helpers = { }, notfound = function() end } -local openers = allocate { helpers = { }, notfound = function() end } -local loaders = allocate { helpers = { }, notfound = function() return false, nil, 0 end } +-- original size: 910, stripped down to: 823 -registermethod("finders", finders, "uri") -registermethod("openers", openers, "uri") -registermethod("loaders", loaders, "uri") - -resolvers.finders = finders -resolvers.openers = openers -resolvers.loaders = loaders +if not modules then modules={} end modules ['data-inp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local methodhandler=resolvers.methodhandler +local registermethod=resolvers.registermethod +local finders=allocate { helpers={},notfound=function() end } +local openers=allocate { helpers={},notfound=function() end } +local loaders=allocate { helpers={},notfound=function() return false,nil,0 end } +registermethod("finders",finders,"uri") +registermethod("openers",openers,"uri") +registermethod("loaders",loaders,"uri") +resolvers.finders=finders +resolvers.openers=openers +resolvers.loaders=loaders end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-out'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local allocate = utilities.storage.allocate -local resolvers = resolvers +-- original size: 530, stripped down to: 475 -local registermethod = resolvers.registermethod - -local savers = allocate { helpers = { } } - -resolvers.savers = savers - -registermethod("savers", savers, "uri") +if not modules then modules={} end modules ['data-out']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registermethod=resolvers.registermethod +local savers=allocate { helpers={} } +resolvers.savers=savers +registermethod("savers",savers,"uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-fil'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_files = logs.reporter("resolvers","files") - -local resolvers = resolvers - -local finders, openers, loaders, savers = resolvers.finders, resolvers.openers, resolvers.loaders, resolvers.savers -local locators, hashers, generators, concatinators = resolvers.locators, resolvers.hashers, resolvers.generators, resolvers.concatinators - -local checkgarbage = utilities.garbagecollector and utilities.garbagecollector.check +-- original size: 3818, stripped down to: 3248 +if not modules then modules={} end modules ['data-fil']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_files=logs.reporter("resolvers","files") +local resolvers=resolvers +local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers +local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check function locators.file(specification) - local name = specification.filename - local realname = resolvers.resolve(name) -- no shortcut - if realname and realname ~= '' and lfs.isdir(realname) then - if trace_locating then - report_files("file locator '%s' found as '%s'",name,realname) - end - resolvers.appendhash('file',name,true) -- cache - elseif trace_locating then - report_files("file locator '%s' not found",name) + local name=specification.filename + local realname=resolvers.resolve(name) + if realname and realname~='' and lfs.isdir(realname) then + if trace_locating then + report_files("file locator '%s' found as '%s'",name,realname) end + resolvers.appendhash('file',name,true) + elseif trace_locating then + report_files("file locator '%s' not found",name) + end end - function hashers.file(specification) - local name = specification.filename - local content = caches.loadcontent(name,'files') - resolvers.registerfilehash(name,content,content==nil) + local name=specification.filename + local content=caches.loadcontent(name,'files') + resolvers.registerfilehash(name,content,content==nil) end - function generators.file(specification) - local path = specification.filename - local content = resolvers.scanfiles(path,false,true) -- scan once - resolvers.registerfilehash(path,content,true) + local path=specification.filename + local content=resolvers.scanfiles(path,false,true) + resolvers.registerfilehash(path,content,true) end - -concatinators.file = file.join - +concatinators.file=file.join function finders.file(specification,filetype) - local filename = specification.filename - local foundname = resolvers.findfile(filename,filetype) - if foundname and foundname ~= "" then - if trace_locating then - report_files("file finder: '%s' found",filename) - end - return foundname - else - if trace_locating then - report_files("file finder: %s' not found",filename) - end - return finders.notfound() + local filename=specification.filename + local foundname=resolvers.findfile(filename,filetype) + if foundname and foundname~="" then + if trace_locating then + report_files("file finder: '%s' found",filename) + end + return foundname + else + if trace_locating then + report_files("file finder: %s' not found",filename) end + return finders.notfound() + end end - --- The default textopener will be overloaded later on. - function openers.helpers.textopener(tag,filename,f) - return { - reader = function() return f:read () end, - close = function() logs.show_close(filename) return f:close() end, - } + return { + reader=function() return f:read () end, + close=function() logs.show_close(filename) return f:close() end, + } end - function openers.file(specification,filetype) - local filename = specification.filename - if filename and filename ~= "" then - local f = io.open(filename,"r") - if f then - if trace_locating then - report_files("file opener, '%s' opened",filename) - end - return openers.helpers.textopener("file",filename,f) - end - end - if trace_locating then - report_files("file opener, '%s' not found",filename) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"r") + if f then + if trace_locating then + report_files("file opener, '%s' opened",filename) + end + return openers.helpers.textopener("file",filename,f) end - return openers.notfound() + end + if trace_locating then + report_files("file opener, '%s' not found",filename) + end + return openers.notfound() end - function loaders.file(specification,filetype) - local filename = specification.filename - if filename and filename ~= "" then - local f = io.open(filename,"rb") - if f then - logs.show_load(filename) - if trace_locating then - report_files("file loader, '%s' loaded",filename) - end - local s = f:read("*a") -- io.readall(f) is faster but we never have large files here - if checkgarbage then - checkgarbage(#s) - end - f:close() - if s then - return true, s, #s - end - end - end - if trace_locating then - report_files("file loader, '%s' not found",filename) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"rb") + if f then + logs.show_load(filename) + if trace_locating then + report_files("file loader, '%s' loaded",filename) + end + local s=f:read("*a") + if checkgarbage then + checkgarbage(#s) + end + f:close() + if s then + return true,s,#s + end end - return loaders.notfound() + end + if trace_locating then + report_files("file loader, '%s' not found",filename) + end + return loaders.notfound() end @@ -17090,129 +13287,113 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-con'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local format, lower, gsub = string.format, string.lower, string.gsub - -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) -local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) - - - -containers = containers or { } -local containers = containers -containers.usecache = true - -local report_containers = logs.reporter("resolvers","containers") +-- original size: 4651, stripped down to: 3330 +if not modules then modules={} end modules ['data-con']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub=string.format,string.lower,string.gsub +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) +local trace_storage=false trackers.register("resolvers.storage",function(v) trace_storage=v end) +containers=containers or {} +local containers=containers +containers.usecache=true +local report_containers=logs.reporter("resolvers","containers") local function report(container,tag,name) - if trace_cache or trace_containers then - report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') - end -end - -local allocated = { } - -local mt = { - __index = function(t,k) - if k == "writable" then - local writable = caches.getwritablepath(t.category,t.subcategory) or { "." } - t.writable = writable - return writable - elseif k == "readables" then - local readables = caches.getreadablepaths(t.category,t.subcategory) or { "." } - t.readables = readables - return readables - end - end, - __storage__ = true + if trace_cache or trace_containers then + report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') + end +end +local allocated={} +local mt={ + __index=function(t,k) + if k=="writable" then + local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + t.writable=writable + return writable + elseif k=="readables" then + local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + t.readables=readables + return readables + end + end, + __storage__=true } - -function containers.define(category, subcategory, version, enabled) - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or math.pi, -- after all, this is TeX - trace = false, - -- writable = caches.getwritablepath and caches.getwritablepath (category,subcategory) or { "." }, - -- readables = caches.getreadablepaths and caches.getreadablepaths(category,subcategory) or { "." }, - } - setmetatable(s,mt) - c[subcategory] = s - end - return s +function containers.define(category,subcategory,version,enabled) + if category and subcategory then + local c=allocated[category] + if not c then + c={} + allocated[category]=c end -end - -function containers.is_usable(container, name) - return container.enabled and caches and caches.is_writable(container.writable, name) -end - -function containers.is_valid(container, name) - if name and name ~= "" then - local storage = container.storage[name] - return storage and storage.cache_version == container.version - else - return false + local s=c[subcategory] + if not s then + s={ + category=category, + subcategory=subcategory, + storage={}, + enabled=enabled, + version=version or math.pi, + trace=false, + } + setmetatable(s,mt) + c[subcategory]=s end + return s + end end - -function containers.read(container,name) - local storage = container.storage - local stored = storage[name] - if not stored and container.enabled and caches and containers.usecache then - stored = caches.loaddata(container.readables,name) - if stored and stored.cache_version == container.version then - report(container,"loaded",name) - else - stored = nil - end - storage[name] = stored - elseif stored then - report(container,"reusing",name) - end - return stored +function containers.is_usable(container,name) + return container.enabled and caches and caches.is_writable(container.writable,name) end - -function containers.write(container, name, data) - if data then - data.cache_version = container.version - if container.enabled and caches then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.writable, name, data) - report(container,"saved",name) - data.unique, data.shared = unique, shared - end - report(container,"stored",name) - container.storage[name] = data - end - return data +function containers.is_valid(container,name) + if name and name~="" then + local storage=container.storage[name] + return storage and storage.cache_version==container.version + else + return false + end +end +function containers.read(container,name) + local storage=container.storage + local stored=storage[name] + if not stored and container.enabled and caches and containers.usecache then + stored=caches.loaddata(container.readables,name) + if stored and stored.cache_version==container.version then + report(container,"loaded",name) + else + stored=nil + end + storage[name]=stored + elseif stored then + report(container,"reusing",name) + end + return stored +end +function containers.write(container,name,data) + if data then + data.cache_version=container.version + if container.enabled and caches then + local unique,shared=data.unique,data.shared + data.unique,data.shared=nil,nil + caches.savedata(container.writable,name,data) + report(container,"saved",name) + data.unique,data.shared=unique,shared + end + report(container,"stored",name) + container.storage[name]=data + end + return data end - function containers.content(container,name) - return container.storage[name] + return container.storage[name] end - function containers.cleanname(name) - return (gsub(lower(name),"[^%w%d]+","-")) + return (gsub(lower(name),"[^%w%d]+","-")) end @@ -17220,106 +13401,88 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-use'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_mounts = logs.reporter("resolvers","mounts") - -local resolvers = resolvers - --- we will make a better format, maybe something xml or just text or lua - -resolvers.automounted = resolvers.automounted or { } +-- original size: 3913, stripped down to: 2998 +if not modules then modules={} end modules ['data-use']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_mounts=logs.reporter("resolvers","mounts") +local resolvers=resolvers +resolvers.automounted=resolvers.automounted or {} function resolvers.automount(usecache) - local mountpaths = resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths == 0) and usecache then - mountpaths = caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths > 0 then - statistics.starttiming(resolvers.instance) - for k=1,#mountpaths do - local root = mountpaths[k] - local f = io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then -- or %W - -- skip - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %s",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() + local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) + if (not mountpaths or #mountpaths==0) and usecache then + mountpaths=caches.getreadablepaths("mount") + end + if mountpaths and #mountpaths>0 then + statistics.starttiming(resolvers.instance) + for k=1,#mountpaths do + local root=mountpaths[k] + local f=io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if find(line,"^[%%#%-]") then + elseif find(line,"^zip://") then + if trace_locating then + report_mounts("mounting %s",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) end + end end - statistics.stoptiming(resolvers.instance) - end -end - --- status info - -statistics.register("used config file", function() return caches.configfiles() end) -statistics.register("used cache path", function() return caches.usedpaths() end) - --- experiment (code will move) - -function statistics.savefmtstatus(texname,formatbanner,sourcefile) -- texname == formatname - local enginebanner = status.list().banner - if formatbanner and enginebanner and sourcefile then - local luvname = file.replacesuffix(texname,"luv") -- utilities.lua.suffixes.luv - local luvdata = { - enginebanner = enginebanner, - formatbanner = formatbanner, - sourcehash = md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), - sourcefile = sourcefile, - } - io.savedata(luvname,table.serialize(luvdata,true)) + f:close() + end end + statistics.stoptiming(resolvers.instance) + end +end +statistics.register("used config file",function() return caches.configfiles() end) +statistics.register("used cache path",function() return caches.usedpaths() end) +function statistics.savefmtstatus(texname,formatbanner,sourcefile) + local enginebanner=status.list().banner + if formatbanner and enginebanner and sourcefile then + local luvname=file.replacesuffix(texname,"luv") + local luvdata={ + enginebanner=enginebanner, + formatbanner=formatbanner, + sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcefile=sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) + end end - --- todo: check this at startup and return (say) 999 as signal that the run --- was aborted due to a wrong format in which case mtx-context can trigger --- a remake - function statistics.checkfmtstatus(texname) - local enginebanner = status.list().banner - if enginebanner and texname then - local luvname = file.replacesuffix(texname,"luv") -- utilities.lua.suffixes.luv - if lfs.isfile(luvname) then - local luv = dofile(luvname) - if luv and luv.sourcefile then - local sourcehash = md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") - local luvbanner = luv.enginebanner or "?" - if luvbanner ~= enginebanner then - return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) - end - local luvhash = luv.sourcehash or "?" - if luvhash ~= sourcehash then - return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) - end - else - return "invalid status file" - end - else - return "missing status file" - end + local enginebanner=status.list().banner + if enginebanner and texname then + local luvname=file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv=dofile(luvname) + if luv and luv.sourcefile then + local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local luvbanner=luv.enginebanner or "?" + if luvbanner~=enginebanner then + return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) + end + local luvhash=luv.sourcehash or "?" + if luvhash~=sourcehash then + return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) + end + else + return "invalid status file" + end + else + return "missing status file" end - return true + end + return true end @@ -17327,259 +13490,233 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-zip'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- partly redone .. needs testing - -local format, find, match = string.format, string.find, string.match +-- original size: 8537, stripped down to: 6805 -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_zip = logs.reporter("resolvers","zip") - - - -local resolvers = resolvers - -zip = zip or { } -local zip = zip - -zip.archives = zip.archives or { } -local archives = zip.archives - -zip.registeredfiles = zip.registeredfiles or { } -local registeredfiles = zip.registeredfiles - -local limited = false - -directives.register("system.inputmode", function(v) - if not limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - zip.open = i_limiter.protect(zip.open) - limited = true - end - end +if not modules then modules={} end modules ['data-zip']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,find,match=string.format,string.find,string.match +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_zip=logs.reporter("resolvers","zip") +local resolvers=resolvers +zip=zip or {} +local zip=zip +zip.archives=zip.archives or {} +local archives=zip.archives +zip.registeredfiles=zip.registeredfiles or {} +local registeredfiles=zip.registeredfiles +local limited=false +directives.register("system.inputmode",function(v) + if not limited then + local i_limiter=io.i_limiter(v) + if i_limiter then + zip.open=i_limiter.protect(zip.open) + limited=true + end + end end) - -local function validzip(str) -- todo: use url splitter - if not find(str,"^zip://") then - return "zip:///" .. str - else - return str - end +local function validzip(str) + if not find(str,"^zip://") then + return "zip:///"..str + else + return str + end end - function zip.openarchive(name) - if not name or name == "" then - return nil - else - local arch = archives[name] - if not arch then - local full = resolvers.findfile(name) or "" - arch = (full ~= "" and zip.open(full)) or false - archives[name] = arch - end - return arch + if not name or name=="" then + return nil + else + local arch=archives[name] + if not arch then + local full=resolvers.findfile(name) or "" + arch=(full~="" and zip.open(full)) or false + archives[name]=arch end + return arch + end end - function zip.closearchive(name) - if not name or (name == "" and archives[name]) then - zip.close(archives[name]) - archives[name] = nil - end + if not name or (name=="" and archives[name]) then + zip.close(archives[name]) + archives[name]=nil + end end - function resolvers.locators.zip(specification) - local archive = specification.filename - local zipfile = archive and archive ~= "" and zip.openarchive(archive) -- tricky, could be in to be initialized tree - if trace_locating then - if zipfile then - report_zip("locator, archive '%s' found",archive) - else - report_zip("locator, archive '%s' not found",archive) - end + local archive=specification.filename + local zipfile=archive and archive~="" and zip.openarchive(archive) + if trace_locating then + if zipfile then + report_zip("locator, archive '%s' found",archive) + else + report_zip("locator, archive '%s' not found",archive) end + end end - function resolvers.hashers.zip(specification) - local archive = specification.filename - if trace_locating then - report_zip("loading file '%s'",archive) - end - resolvers.usezipfile(specification.original) -end - -function resolvers.concatinators.zip(zipfile,path,name) -- ok ? - if not path or path == "" then - return format('%s?name=%s',zipfile,name) - else - return format('%s?name=%s/%s',zipfile,path,name) - end + local archive=specification.filename + if trace_locating then + report_zip("loading file '%s'",archive) + end + resolvers.usezipfile(specification.original) +end +function resolvers.concatinators.zip(zipfile,path,name) + if not path or path=="" then + return format('%s?name=%s',zipfile,name) + else + return format('%s?name=%s/%s',zipfile,path,name) + end end - function resolvers.finders.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("finder, archive '%s' found",archive) - end - local dfile = zfile:open(queryname) - if dfile then - dfile = zfile:close() - if trace_locating then - report_zip("finder, file '%s' found",queryname) - end - return specification.original - elseif trace_locating then - report_zip("finder, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("finder, unknown archive '%s'",archive) - end + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("finder, archive '%s' found",archive) end + local dfile=zfile:open(queryname) + if dfile then + dfile=zfile:close() + if trace_locating then + report_zip("finder, file '%s' found",queryname) + end + return specification.original + elseif trace_locating then + report_zip("finder, file '%s' not found",queryname) + end + elseif trace_locating then + report_zip("finder, unknown archive '%s'",archive) + end end - if trace_locating then - report_zip("finder, '%s' not found",original) - end - return resolvers.finders.notfound() + end + if trace_locating then + report_zip("finder, '%s' not found",original) + end + return resolvers.finders.notfound() end - function resolvers.openers.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("opener, archive '%s' opened",archive) - end - local dfile = zfile:open(queryname) - if dfile then - if trace_locating then - report_zip("opener, file '%s' found",queryname) - end - return resolvers.openers.helpers.textopener('zip',original,dfile) - elseif trace_locating then - report_zip("opener, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("opener, unknown archive '%s'",archive) - end + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("opener, archive '%s' opened",archive) end + local dfile=zfile:open(queryname) + if dfile then + if trace_locating then + report_zip("opener, file '%s' found",queryname) + end + return resolvers.openers.helpers.textopener('zip',original,dfile) + elseif trace_locating then + report_zip("opener, file '%s' not found",queryname) + end + elseif trace_locating then + report_zip("opener, unknown archive '%s'",archive) + end end - if trace_locating then - report_zip("opener, '%s' not found",original) - end - return resolvers.openers.notfound() + end + if trace_locating then + report_zip("opener, '%s' not found",original) + end + return resolvers.openers.notfound() end - function resolvers.loaders.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("loader, archive '%s' opened",archive) - end - local dfile = zfile:open(queryname) - if dfile then - logs.show_load(original) - if trace_locating then - report_zip("loader, file '%s' loaded",original) - end - local s = dfile:read("*all") - dfile:close() - return true, s, #s - elseif trace_locating then - report_zip("loader, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("loader, unknown archive '%s'",archive) - end + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("loader, archive '%s' opened",archive) end + local dfile=zfile:open(queryname) + if dfile then + logs.show_load(original) + if trace_locating then + report_zip("loader, file '%s' loaded",original) + end + local s=dfile:read("*all") + dfile:close() + return true,s,#s + elseif trace_locating then + report_zip("loader, file '%s' not found",queryname) + end + elseif trace_locating then + report_zip("loader, unknown archive '%s'",archive) + end end - if trace_locating then - report_zip("loader, '%s' not found",original) - end - return resolvers.openers.notfound() + end + if trace_locating then + report_zip("loader, '%s' not found",original) + end + return resolvers.openers.notfound() end - --- zip:///somefile.zip --- zip:///somefile.zip?tree=texmf-local -> mount - function resolvers.usezipfile(archive) - local specification = resolvers.splitmethod(archive) -- to be sure - local archive = specification.filename - if archive and not registeredfiles[archive] then - local z = zip.openarchive(archive) - if z then - local instance = resolvers.instance - local tree = url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering, registering archive '%s'",archive) - end - statistics.starttiming(instance) - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) -- resets hashes too - registeredfiles[archive] = z - instance.files[archive] = resolvers.registerzipfile(z,tree) - statistics.stoptiming(instance) - elseif trace_locating then - report_zip("registering, unknown archive '%s'",archive) - end + local specification=resolvers.splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=zip.openarchive(archive) + if z then + local instance=resolvers.instance + local tree=url.query(specification.query).tree or "" + if trace_locating then + report_zip("registering, registering archive '%s'",archive) + end + statistics.starttiming(instance) + resolvers.prependhash('zip',archive) + resolvers.extendtexmfvariable(archive) + registeredfiles[archive]=z + instance.files[archive]=resolvers.registerzipfile(z,tree) + statistics.stoptiming(instance) elseif trace_locating then - report_zip("registering, '%s' not found",archive) + report_zip("registering, unknown archive '%s'",archive) end + elseif trace_locating then + report_zip("registering, '%s' not found",archive) + end end - function resolvers.registerzipfile(z,tree) - local files, filter = { }, "" - if tree == "" then - filter = "^(.+)/(.-)$" + local files,filter={},"" + if tree=="" then + filter="^(.+)/(.-)$" + else + filter=format("^%s/(.+)/(.-)$",tree) + end + if trace_locating then + report_zip("registering, using filter '%s'",filter) + end + local register,n=resolvers.registerfile,0 + for i in z:files() do + local path,name=match(i.filename,filter) + if path then + if name and name~='' then + register(files,name,path) + n=n+1 + else + end else - filter = format("^%s/(.+)/(.-)$",tree) - end - if trace_locating then - report_zip("registering, using filter '%s'",filter) - end - local register, n = resolvers.registerfile, 0 - for i in z:files() do - local path, name = match(i.filename,filter) - if path then - if name and name ~= '' then - register(files, name, path) - n = n + 1 - else - -- directory - end - else - register(files, i.filename, '') - n = n + 1 - end + register(files,i.filename,'') + n=n+1 end - report_zip("registering, %s files registered",n) - return files + end + report_zip("registering, %s files registered",n) + return files end @@ -17587,286 +13724,244 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tre'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- \input tree://oeps1/**/oeps.tex - -local find, gsub, format = string.find, string.gsub, string.format - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_trees = logs.reporter("resolvers","trees") - -local resolvers = resolvers - -local done, found, notfound = { }, { }, resolvers.finders.notfound +-- original size: 2514, stripped down to: 2080 +if not modules then modules={} end modules ['data-tre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,gsub,format=string.find,string.gsub,string.format +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_trees=logs.reporter("resolvers","trees") +local resolvers=resolvers +local done,found,notfound={},{},resolvers.finders.notfound function resolvers.finders.tree(specification) - local spec = specification.filename - local fnd = found[spec] - if fnd == nil then - if spec ~= "" then - local path, name = file.dirname(spec), file.basename(spec) - if path == "" then path = "." end - local hash = done[path] - if not hash then - local pattern = path .. "/*" -- we will use the proper splitter - hash = dir.glob(pattern) - done[path] = hash - end - local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$" - for k=1,#hash do - local v = hash[k] - if find(v,pattern) then - found[spec] = v - return v - end - end + local spec=specification.filename + local fnd=found[spec] + if fnd==nil then + if spec~="" then + local path,name=file.dirname(spec),file.basename(spec) + if path=="" then path="." end + local hash=done[path] + if not hash then + local pattern=path.."/*" + hash=dir.glob(pattern) + done[path]=hash + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for k=1,#hash do + local v=hash[k] + if find(v,pattern) then + found[spec]=v + return v end - fnd = notfound() -- false - found[spec] = fnd + end end - return fnd + fnd=notfound() + found[spec]=fnd + end + return fnd end - function resolvers.locators.tree(specification) - local name = specification.filename - local realname = resolvers.resolve(name) -- no shortcut - if realname and realname ~= '' and lfs.isdir(realname) then - if trace_locating then - report_trees("locator '%s' found",realname) - end - resolvers.appendhash('tree',name,false) -- don't cache - elseif trace_locating then - report_trees("locator '%s' not found",name) + local name=specification.filename + local realname=resolvers.resolve(name) + if realname and realname~='' and lfs.isdir(realname) then + if trace_locating then + report_trees("locator '%s' found",realname) end + resolvers.appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator '%s' not found",name) + end end - function resolvers.hashers.tree(specification) - local name = specification.filename - if trace_locating then - report_trees("analysing '%s'",name) - end - resolvers.methodhandler("hashers",name) - - resolvers.generators.file(specification) + local name=specification.filename + if trace_locating then + report_trees("analysing '%s'",name) + end + resolvers.methodhandler("hashers",name) + resolvers.generators.file(specification) end - -resolvers.concatinators.tree = resolvers.concatinators.file -resolvers.generators.tree = resolvers.generators.file -resolvers.openers.tree = resolvers.openers.file -resolvers.loaders.tree = resolvers.loaders.file +resolvers.concatinators.tree=resolvers.concatinators.file +resolvers.generators.tree=resolvers.generators.file +resolvers.openers.tree=resolvers.openers.file +resolvers.loaders.tree=resolvers.loaders.file end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-sch'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local load = load -local gsub, concat, format = string.gsub, table.concat, string.format -local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders - -local trace_schemes = false trackers.register("resolvers.schemes",function(v) trace_schemes = v end) -local report_schemes = logs.reporter("resolvers","schemes") - -local http = require("socket.http") -local ltn12 = require("ltn12") - -local resolvers = resolvers -local schemes = resolvers.schemes or { } -resolvers.schemes = schemes - -local cleaners = { } -schemes.cleaners = cleaners - -local threshold = 24 * 60 * 60 - -directives.register("schemes.threshold", function(v) threshold = tonumber(v) or threshold end) +-- original size: 6218, stripped down to: 5165 +if not modules then modules={} end modules ['data-sch']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local load=load +local gsub,concat,format=string.gsub,table.concat,string.format +local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) +local report_schemes=logs.reporter("resolvers","schemes") +local http=require("socket.http") +local ltn12=require("ltn12") +local resolvers=resolvers +local schemes=resolvers.schemes or {} +resolvers.schemes=schemes +local cleaners={} +schemes.cleaners=cleaners +local threshold=24*60*60 +directives.register("schemes.threshold",function(v) threshold=tonumber(v) or threshold end) function cleaners.none(specification) - return specification.original + return specification.original end - function cleaners.strip(specification) - return (gsub(specification.original,"[^%a%d%.]+","-")) -- so we keep periods + return (gsub(specification.original,"[^%a%d%.]+","-")) end - function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) end - -local cleaner = cleaners.strip - -directives.register("schemes.cleanmethod", function(v) cleaner = cleaners[v] or cleaners.strip end) - +local cleaner=cleaners.strip +directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) function resolvers.schemes.cleanname(specification) - local hash = cleaner(specification) - if trace_schemes then - report_schemes("hashing %s to %s",specification.original,hash) - end - return hash + local hash=cleaner(specification) + if trace_schemes then + report_schemes("hashing %s to %s",specification.original,hash) + end + return hash end - -local cached, loaded, reused, thresholds, handlers = { }, { }, { }, { }, { } - -local function runcurl(name,cachename) -- we use sockets instead or the curl library when possible - local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name - os.spawn(command) +local cached,loaded,reused,thresholds,handlers={},{},{},{},{} +local function runcurl(name,cachename) + local command="curl --silent --create-dirs --output "..cachename.." "..name + os.spawn(command) end - local function fetch(specification) - local original = specification.original - local scheme = specification.scheme - local cleanname = schemes.cleanname(specification) - local cachename = caches.setfirstwritablefile(cleanname,"schemes") - if not cached[original] then - statistics.starttiming(schemes) - if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > (thresholds[protocol] or threshold)) then - cached[original] = cachename - local handler = handlers[scheme] - if handler then - if trace_schemes then - report_schemes("fetching '%s', protocol '%s', method 'built-in'",original,scheme) - end - logs.flush() - handler(specification,cachename) - else - if trace_schemes then - report_schemes("fetching '%s', protocol '%s', method 'curl'",original,scheme) - end - logs.flush() - runcurl(original,cachename) - end - end - if io.exists(cachename) then - cached[original] = cachename - if trace_schemes then - report_schemes("using cached '%s', protocol '%s', cachename '%s'",original,scheme,cachename) - end - else - cached[original] = "" - if trace_schemes then - report_schemes("using missing '%s', protocol '%s'",original,scheme) - end + local original=specification.original + local scheme=specification.scheme + local cleanname=schemes.cleanname(specification) + local cachename=caches.setfirstwritablefile(cleanname,"schemes") + if not cached[original] then + statistics.starttiming(schemes) + if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification)>(thresholds[protocol] or threshold)) then + cached[original]=cachename + local handler=handlers[scheme] + if handler then + if trace_schemes then + report_schemes("fetching '%s', protocol '%s', method 'built-in'",original,scheme) end - loaded[scheme] = loaded[scheme] + 1 - statistics.stoptiming(schemes) - else + logs.flush() + handler(specification,cachename) + else if trace_schemes then - report_schemes("reusing '%s', protocol '%s'",original,scheme) + report_schemes("fetching '%s', protocol '%s', method 'curl'",original,scheme) end - reused[scheme] = reused[scheme] + 1 + logs.flush() + runcurl(original,cachename) + end + end + if io.exists(cachename) then + cached[original]=cachename + if trace_schemes then + report_schemes("using cached '%s', protocol '%s', cachename '%s'",original,scheme,cachename) + end + else + cached[original]="" + if trace_schemes then + report_schemes("using missing '%s', protocol '%s'",original,scheme) + end + end + loaded[scheme]=loaded[scheme]+1 + statistics.stoptiming(schemes) + else + if trace_schemes then + report_schemes("reusing '%s', protocol '%s'",original,scheme) end - return cached[original] + reused[scheme]=reused[scheme]+1 + end + return cached[original] end - local function finder(specification,filetype) - return resolvers.methodhandler("finders",fetch(specification),filetype) + return resolvers.methodhandler("finders",fetch(specification),filetype) end - -local opener = openers.file -local loader = loaders.file - +local opener=openers.file +local loader=loaders.file local function install(scheme,handler,newthreshold) - handlers [scheme] = handler - loaded [scheme] = 0 - reused [scheme] = 0 - finders [scheme] = finder - openers [scheme] = opener - loaders [scheme] = loader - thresholds[scheme] = newthreshold or threshold -end - -schemes.install = install - + handlers [scheme]=handler + loaded [scheme]=0 + reused [scheme]=0 + finders [scheme]=finder + openers [scheme]=opener + loaders [scheme]=loader + thresholds[scheme]=newthreshold or threshold +end +schemes.install=install local function http_handler(specification,cachename) - local tempname = cachename .. ".tmp" - local f = io.open(tempname,"wb") - local status, message = http.request { - url = specification.original, - sink = ltn12.sink.file(f) - } - if not status then - os.remove(tempname) - else - os.remove(cachename) - os.rename(tempname,cachename) - end - return cachename + local tempname=cachename..".tmp" + local f=io.open(tempname,"wb") + local status,message=http.request { + url=specification.original, + sink=ltn12.sink.file(f) + } + if not status then + os.remove(tempname) + else + os.remove(cachename) + os.rename(tempname,cachename) + end + return cachename end - install('http',http_handler) -install('https') -- see pod +install('https') install('ftp') - -statistics.register("scheme handling time", function() - local l, r, nl, nr = { }, { }, 0, 0 - for k, v in table.sortedhash(loaded) do - if v > 0 then - nl = nl + 1 - l[nl] = k .. ":" .. v - end - end - for k, v in table.sortedhash(reused) do - if v > 0 then - nr = nr + 1 - r[nr] = k .. ":" .. v - end - end - local n = nl + nr - if n > 0 then - l = nl > 0 and concat(l) or "none" - r = nr > 0 and concat(r) or "none" - return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes), n, threshold, l, r) - else - return nil - end +statistics.register("scheme handling time",function() + local l,r,nl,nr={},{},0,0 + for k,v in table.sortedhash(loaded) do + if v>0 then + nl=nl+1 + l[nl]=k..":"..v + end + end + for k,v in table.sortedhash(reused) do + if v>0 then + nr=nr+1 + r[nr]=k..":"..v + end + end + local n=nl+nr + if n>0 then + l=nl>0 and concat(l) or "none" + r=nr>0 and concat(r) or "none" + return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", + statistics.elapsedtime(schemes),n,threshold,l,r) + else + return nil + end end) - --- We provide a few more helpers: - ------ http = require("socket.http") -local httprequest = http.request -local toquery = url.toquery - --- local function httprequest(url) --- return os.resultof(format("curl --silent %q", url)) --- end - +local httprequest=http.request +local toquery=url.toquery local function fetchstring(url,data) - local q = data and toquery(data) - if q then - url = url .. "?" .. q - end - local reply = httprequest(url) - return reply -- just one argument -end - -schemes.fetchstring = fetchstring - + local q=data and toquery(data) + if q then + url=url.."?"..q + end + local reply=httprequest(url) + return reply +end +schemes.fetchstring=fetchstring function schemes.fetchtable(url,data) - local reply = fetchstring(url,data) - if reply then - local s = load("return " .. reply) - if s then - return s() - end + local reply=fetchstring(url,data) + if reply then + local s=load("return "..reply) + if s then + return s() end + end end @@ -17874,278 +13969,241 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- We overload the regular loader. We do so because we operate mostly in --- tds and use our own loader code. Alternatively we could use a more --- extensive definition of package.path and package.cpath but even then --- we're not done. Also, we now have better tracing. --- --- -- local mylib = require("libtest") --- -- local mysql = require("luasql.mysql") - -local searchers = package.searchers or package.loaders - -local concat = table.concat - -local trace_libraries = false - -trackers.register("resolvers.libraries", function(v) trace_libraries = v end) -trackers.register("resolvers.locating", function(v) trace_libraries = v end) - -local report_libraries = logs.reporter("resolvers","libraries") - -local gsub, insert = string.gsub, table.insert -local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match -local unpack = unpack or table.unpack -local is_readable = file.is_readable - -local resolvers, package = resolvers, package - -local libsuffixes = { 'tex', 'lua' } -local clibsuffixes = { 'lib' } -local libformats = { 'TEXINPUTS', 'LUAINPUTS' } -local clibformats = { 'CLUAINPUTS' } - -local libpaths = nil -local clibpaths = nil -local libhash = { } -local clibhash = { } -local libextras = { } -local clibextras = { } +-- original size: 6387, stripped down to: 5108 -local pattern = Cs(P("!")^0 / "" * (P("/") * P(-1) / "/" + P("/")^1 / "/" + 1)^0) - -local function cleanpath(path) --hm, don't we have a helper for this? - return resolvers.resolve(lpegmatch(pattern,path)) +if not modules then modules={} end modules ['data-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local searchers=package.searchers or package.loaders +local concat=table.concat +local trace_libraries=false +trackers.register("resolvers.libraries",function(v) trace_libraries=v end) +trackers.register("resolvers.locating",function(v) trace_libraries=v end) +local report_libraries=logs.reporter("resolvers","libraries") +local gsub,insert=string.gsub,table.insert +local P,Cs,lpegmatch=lpeg.P,lpeg.Cs,lpeg.match +local unpack=unpack or table.unpack +local is_readable=file.is_readable +local resolvers,package=resolvers,package +local libsuffixes={ 'tex','lua' } +local clibsuffixes={ 'lib' } +local libformats={ 'TEXINPUTS','LUAINPUTS' } +local clibformats={ 'CLUAINPUTS' } +local libpaths=nil +local clibpaths=nil +local libhash={} +local clibhash={} +local libextras={} +local clibextras={} +local pattern=Cs(P("!")^0/""*(P("/")*P(-1)/"/"+P("/")^1/"/"+1)^0) +local function cleanpath(path) + return resolvers.resolve(lpegmatch(pattern,path)) end - local function getlibpaths() - if not libpaths then - libpaths = { } - for i=1,#libformats do - local paths = resolvers.expandedpathlistfromvariable(libformats[i]) - for i=1,#paths do - local path = cleanpath(paths[i]) - if not libhash[path] then - libpaths[#libpaths+1] = path - libhash[path] = true - end - end + if not libpaths then + libpaths={} + for i=1,#libformats do + local paths=resolvers.expandedpathlistfromvariable(libformats[i]) + for i=1,#paths do + local path=cleanpath(paths[i]) + if not libhash[path] then + libpaths[#libpaths+1]=path + libhash[path]=true end + end end - return libpaths + end + return libpaths end - local function getclibpaths() - if not clibpaths then - clibpaths = { } - for i=1,#clibformats do - local paths = resolvers.expandedpathlistfromvariable(clibformats[i]) - for i=1,#paths do - local path = cleanpath(paths[i]) - if not clibhash[path] then - clibpaths[#clibpaths+1] = path - clibhash[path] = true - end - end + if not clibpaths then + clibpaths={} + for i=1,#clibformats do + local paths=resolvers.expandedpathlistfromvariable(clibformats[i]) + for i=1,#paths do + local path=cleanpath(paths[i]) + if not clibhash[path] then + clibpaths[#clibpaths+1]=path + clibhash[path]=true end + end end - return clibpaths + end + return clibpaths end - -package.libpaths = getlibpaths -package.clibpaths = getclibpaths - +package.libpaths=getlibpaths +package.clibpaths=getclibpaths function package.extralibpath(...) - local libpaths = getlibpaths() - local paths = table.flattened { ... } - for i=1,#paths do - local path = cleanpath(paths[i]) - if not libhash[path] then - if trace_libraries then - report_libraries("! extra lua path '%s'",path) - end - libextras[#libextras+1] = path - libpaths[#libpaths +1] = path - end + local libpaths=getlibpaths() + local paths=table.flattened {... } + for i=1,#paths do + local path=cleanpath(paths[i]) + if not libhash[path] then + if trace_libraries then + report_libraries("! extra lua path '%s'",path) + end + libextras[#libextras+1]=path + libpaths[#libpaths+1]=path end + end end - function package.extraclibpath(...) - local clibpaths = getclibpaths() - local paths = table.flattened { ... } - for i=1,#paths do - local path = cleanpath(paths[i]) - if not clibhash[path] then - if trace_libraries then - report_libraries("! extra lib path '%s'",path) - end - clibextras[#clibextras+1] = path - clibpaths[#clibpaths +1] = path - end + local clibpaths=getclibpaths() + local paths=table.flattened {... } + for i=1,#paths do + local path=cleanpath(paths[i]) + if not clibhash[path] then + if trace_libraries then + report_libraries("! extra lib path '%s'",path) + end + clibextras[#clibextras+1]=path + clibpaths[#clibpaths+1]=path end + end end - if not searchers[-2] then - -- use package-path and package-cpath - searchers[-2] = searchers[2] + searchers[-2]=searchers[2] end - local function loadedaslib(resolved,rawname) - return package.loadlib(resolved,"luaopen_" .. gsub(rawname,"%.","_")) + return package.loadlib(resolved,"luaopen_"..gsub(rawname,"%.","_")) end - local function loadedbylua(name) - if trace_libraries then - report_libraries("! locating %q using normal loader",name) - end - local resolved = searchers[-2](name) + if trace_libraries then + report_libraries("! locating %q using normal loader",name) + end + local resolved=searchers[-2](name) end - local function loadedbyformat(name,rawname,suffixes,islib) + if trace_libraries then + report_libraries("! locating %q as %q using formats %q",rawname,name,concat(suffixes)) + end + for i=1,#suffixes do + local format=suffixes[i] + local resolved=resolvers.findfile(name,format) or "" if trace_libraries then - report_libraries("! locating %q as %q using formats %q",rawname,name,concat(suffixes)) + report_libraries("! checking for %q' using format %q",name,format) end - for i=1,#suffixes do -- so we use findfile and not a lookup loop - local format = suffixes[i] - local resolved = resolvers.findfile(name,format) or "" - if trace_libraries then - report_libraries("! checking for %q' using format %q",name,format) - end - if resolved ~= "" then - if trace_libraries then - report_libraries("! lib %q located on %q",name,resolved) - end - if islib then - return loadedaslib(resolved,rawname) - else - return loadfile(resolved) - end - end + if resolved~="" then + if trace_libraries then + report_libraries("! lib %q located on %q",name,resolved) + end + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) + end end + end end - local function loadedbypath(name,rawname,paths,islib,what) - if trace_libraries then - report_libraries("! locating %q as %q on %q paths",rawname,name,what) - end - for p=1,#paths do - local path = paths[p] - local resolved = file.join(path,name) - if trace_libraries then -- mode detail - report_libraries("! checking for %q using %q path %q",name,what,path) - end - if is_readable(resolved) then - if trace_libraries then - report_libraries("! lib %q located on %q",name,resolved) - end - if islib then - return loadedaslib(resolved,rawname) - else - return loadfile(resolved) - end - end + if trace_libraries then + report_libraries("! locating %q as %q on %q paths",rawname,name,what) + end + for p=1,#paths do + local path=paths[p] + local resolved=file.join(path,name) + if trace_libraries then + report_libraries("! checking for %q using %q path %q",name,what,path) + end + if is_readable(resolved) then + if trace_libraries then + report_libraries("! lib %q located on %q",name,resolved) + end + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) + end end + end end - local function notloaded(name) - if trace_libraries then - report_libraries("? unable to locate library %q",name) - end -end - -searchers[2] = function(name) - local thename = gsub(name,"%.","/") - local luaname = file.addsuffix(thename,"lua") - local libname = file.addsuffix(thename,os.libsuffix) - return - loadedbyformat(luaname,name,libsuffixes, false) - or loadedbyformat(libname,name,clibsuffixes, true) - or loadedbypath (luaname,name,getlibpaths (),false,"lua") - or loadedbypath (luaname,name,getclibpaths(),false,"lua") - or loadedbypath (libname,name,getclibpaths(),true, "lib") - or loadedbylua (name) - or notloaded (name) -end - --- searchers[3] = nil --- searchers[4] = nil - -resolvers.loadlualib = require + if trace_libraries then + report_libraries("? unable to locate library %q",name) + end +end +searchers[2]=function(name) + local thename=gsub(name,"%.","/") + local luaname=file.addsuffix(thename,"lua") + local libname=file.addsuffix(thename,os.libsuffix) + return + loadedbyformat(luaname,name,libsuffixes,false) + or loadedbyformat(libname,name,clibsuffixes,true) + or loadedbypath (luaname,name,getlibpaths (),false,"lua") + or loadedbypath (luaname,name,getclibpaths(),false,"lua") + or loadedbypath (libname,name,getclibpaths(),true,"lib") + or loadedbylua (name) + or notloaded (name) +end +resolvers.loadlualib=require end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-aux'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local find = string.find -local type, next = type, next - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local resolvers = resolvers - -local report_scripts = logs.reporter("resolvers","scripts") +-- original size: 2394, stripped down to: 2005 -function resolvers.updatescript(oldname,newname) -- oldname -> own.name, not per se a suffix - local scriptpath = "scripts/context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = resolvers.cleanpath(oldname) +if not modules then modules={} end modules ['data-aux']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find=string.find +local type,next=type,next +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local resolvers=resolvers +local report_scripts=logs.reporter("resolvers","scripts") +function resolvers.updatescript(oldname,newname) + local scriptpath="scripts/context/lua" + newname=file.addsuffix(newname,"lua") + local oldscript=resolvers.cleanpath(oldname) + if trace_locating then + report_scripts("to be replaced old script %s",oldscript) + end + local newscripts=resolvers.findfiles(newname) or {} + if #newscripts==0 then if trace_locating then - report_scripts("to be replaced old script %s", oldscript) + report_scripts("unable to locate new script") end - local newscripts = resolvers.findfiles(newname) or { } - if #newscripts == 0 then + else + for i=1,#newscripts do + local newscript=resolvers.cleanpath(newscripts[i]) + if trace_locating then + report_scripts("checking new script %s",newscript) + end + if oldscript==newscript then if trace_locating then - report_scripts("unable to locate new script") + report_scripts("old and new script are the same") end - else - for i=1,#newscripts do - local newscript = resolvers.cleanpath(newscripts[i]) - if trace_locating then - report_scripts("checking new script %s", newscript) - end - if oldscript == newscript then - if trace_locating then - report_scripts("old and new script are the same") - end - elseif not find(newscript,scriptpath) then - if trace_locating then - report_scripts("new script should come from %s",scriptpath) - end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_locating then - report_scripts("invalid new script name") - end - else - local newdata = io.loaddata(newscript) - if newdata then - if trace_locating then - report_scripts("old script content replaced by new content") - end - io.savedata(oldscript,newdata) - break - elseif trace_locating then - report_scripts("unable to load new script") - end - end + elseif not find(newscript,scriptpath) then + if trace_locating then + report_scripts("new script should come from %s",scriptpath) + end + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_locating then + report_scripts("invalid new script name") + end + else + local newdata=io.loaddata(newscript) + if newdata then + if trace_locating then + report_scripts("old script content replaced by new content") + end + io.savedata(oldscript,newdata) + break + elseif trace_locating then + report_scripts("unable to load new script") end + end end + end end @@ -18153,78 +14211,53 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tmf'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local resolvers = resolvers - -local report_tds = logs.reporter("resolvers","tds") - --- = << --- ? ?? --- < += --- > =+ +-- original size: 2610, stripped down to: 1637 +if not modules then modules={} end modules ['data-tmf']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers=resolvers +local report_tds=logs.reporter("resolvers","tds") function resolvers.load_tree(tree,resolve) - if type(tree) == "string" and tree ~= "" then - - local getenv, setenv = resolvers.getenv, resolvers.setenv - - -- later might listen to the raw osenv var as well - local texos = "texmf-" .. os.platform - - local oldroot = environment.texroot - local newroot = file.collapsepath(tree) - - local newtree = file.join(newroot,texos) - local newpath = file.join(newtree,"bin") - - if not lfs.isdir(newtree) then - report_tds("no '%s' under tree %s",texos,tree) - os.exit() - end - if not lfs.isdir(newpath) then - report_tds("no '%s/bin' under tree %s",texos,tree) - os.exit() - end - - local texmfos = newtree - - environment.texroot = newroot - environment.texos = texos - environment.texmfos = texmfos - - -- Beware, we need to obey the relocatable autoparent so we - -- set TEXMFCNF to its raw value. This is somewhat tricky when - -- we run a mkii job from within. Therefore, in mtxrun, there - -- is a resolve applied when we're in mkii/kpse mode or when - -- --resolve is passed to mtxrun. Maybe we should also set the - -- local AUTOPARENT etc. although these are alwasy set new. - - if resolve then - -- resolvers.luacnfspec = resolvers.joinpath(resolvers.resolve(resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)))) - resolvers.luacnfspec = resolvers.resolve(resolvers.luacnfspec) - end - - setenv('SELFAUTOPARENT', newroot) - setenv('SELFAUTODIR', newtree) - setenv('SELFAUTOLOC', newpath) - setenv('TEXROOT', newroot) - setenv('TEXOS', texos) - setenv('TEXMFOS', texmfos) - setenv('TEXMFCNF', resolvers.luacnfspec,true) -- already resolved - setenv('PATH', newpath .. io.pathseparator .. getenv('PATH')) - - report_tds("changing from root '%s' to '%s'",oldroot,newroot) - report_tds("prepending '%s' to PATH",newpath) - report_tds("setting TEXMFCNF to '%s'",resolvers.luacnfspec) - report_tds() + if type(tree)=="string" and tree~="" then + local getenv,setenv=resolvers.getenv,resolvers.setenv + local texos="texmf-"..os.platform + local oldroot=environment.texroot + local newroot=file.collapsepath(tree) + local newtree=file.join(newroot,texos) + local newpath=file.join(newtree,"bin") + if not lfs.isdir(newtree) then + report_tds("no '%s' under tree %s",texos,tree) + os.exit() + end + if not lfs.isdir(newpath) then + report_tds("no '%s/bin' under tree %s",texos,tree) + os.exit() + end + local texmfos=newtree + environment.texroot=newroot + environment.texos=texos + environment.texmfos=texmfos + if resolve then + resolvers.luacnfspec=resolvers.resolve(resolvers.luacnfspec) end + setenv('SELFAUTOPARENT',newroot) + setenv('SELFAUTODIR',newtree) + setenv('SELFAUTOLOC',newpath) + setenv('TEXROOT',newroot) + setenv('TEXOS',texos) + setenv('TEXMFOS',texmfos) + setenv('TEXMFCNF',resolvers.luacnfspec,true) + setenv('PATH',newpath..io.pathseparator..getenv('PATH')) + report_tds("changing from root '%s' to '%s'",oldroot,newroot) + report_tds("prepending '%s' to PATH",newpath) + report_tds("setting TEXMFCNF to '%s'",resolvers.luacnfspec) + report_tds() + end end @@ -18232,81 +14265,74 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-lst'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- used in mtxrun, can be loaded later .. todo - -local find, concat, upper, format = string.find, table.concat, string.upper, string.format -local fastcopy, sortedpairs = table.fastcopy, table.sortedpairs - -resolvers.listers = resolvers.listers or { } - -local resolvers = resolvers - -local report_lists = logs.reporter("resolvers","lists") +-- original size: 2632, stripped down to: 2278 +if not modules then modules={} end modules ['data-lst']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,concat,upper,format=string.find,table.concat,string.upper,string.format +local fastcopy,sortedpairs=table.fastcopy,table.sortedpairs +resolvers.listers=resolvers.listers or {} +local resolvers=resolvers +local report_lists=logs.reporter("resolvers","lists") local function tabstr(str) - if type(str) == 'table' then - return concat(str," | ") - else - return str - end + if type(str)=='table' then + return concat(str," | ") + else + return str + end end - function resolvers.listers.variables(pattern) - local instance = resolvers.instance - local environment = instance.environment - local variables = instance.variables - local expansions = instance.expansions - local pattern = upper(pattern or "") - local configured = { } - local order = instance.order - for i=1,#order do - for k, v in next, order[i] do - if v ~= nil and configured[k] == nil then - configured[k] = v - end - end - end - local env = fastcopy(environment) - local var = fastcopy(variables) - local exp = fastcopy(expansions) - for key, value in sortedpairs(configured) do - if key ~= "" and (pattern == "" or find(upper(key),pattern)) then - report_lists(key) - report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") - report_lists(" var: %s",tabstr(configured[key]) or "unset") - report_lists(" exp: %s",tabstr(expansions[key]) or "unset") - report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") - end + local instance=resolvers.instance + local environment=instance.environment + local variables=instance.variables + local expansions=instance.expansions + local pattern=upper(pattern or "") + local configured={} + local order=instance.order + for i=1,#order do + for k,v in next,order[i] do + if v~=nil and configured[k]==nil then + configured[k]=v + end end - instance.environment = fastcopy(env) - instance.variables = fastcopy(var) - instance.expansions = fastcopy(exp) + end + local env=fastcopy(environment) + local var=fastcopy(variables) + local exp=fastcopy(expansions) + for key,value in sortedpairs(configured) do + if key~="" and (pattern=="" or find(upper(key),pattern)) then + report_lists(key) + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") + end + end + instance.environment=fastcopy(env) + instance.variables=fastcopy(var) + instance.expansions=fastcopy(exp) end - function resolvers.listers.configurations(report) - local configurations = resolvers.instance.specification - local report = report or texio.write_nl - for i=1,#configurations do - report(format("file : %s",resolvers.resolve(configurations[i]))) - end - report("") - local list = resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) - for i=1,#list do - local li = resolvers.resolve(list[i]) - if lfs.isdir(li) then - report(format("path - %s",li)) - else - report(format("path + %s",li)) - end + local configurations=resolvers.instance.specification + local report=report or texio.write_nl + for i=1,#configurations do + report(format("file : %s",resolvers.resolve(configurations[i]))) + end + report("") + local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + for i=1,#list do + local li=resolvers.resolve(list[i]) + if lfs.isdir(li) then + report(format("path - %s",li)) + else + report(format("path + %s",li)) end + end end @@ -18314,264 +14340,234 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-sta'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this code is used in the updater - -local gmatch, match = string.gmatch, string.match -local type = type - -states = states or { } -local states = states - -states.data = states.data or { } -local data = states.data - -states.hash = states.hash or { } -local hash = states.hash - -states.tag = states.tag or "" -states.filename = states.filename or "" +-- original size: 5703, stripped down to: 2507 +if not modules then modules={} end modules ['luat-sta']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gmatch,match=string.gmatch,string.match +local type=type +states=states or {} +local states=states +states.data=states.data or {} +local data=states.data +states.hash=states.hash or {} +local hash=states.hash +states.tag=states.tag or "" +states.filename=states.filename or "" function states.save(filename,tag) - tag = tag or states.tag - filename = file.addsuffix(filename or states.filename,'lus') - io.savedata(filename, - "-- generator : luat-sta.lua\n" .. - "-- state tag : " .. tag .. "\n\n" .. - table.serialize(data[tag or states.tag] or {},true) - ) + tag=tag or states.tag + filename=file.addsuffix(filename or states.filename,'lus') + io.savedata(filename, + "-- generator : luat-sta.lua\n".."-- state tag : "..tag.."\n\n"..table.serialize(data[tag or states.tag] or {},true) + ) end - function states.load(filename,tag) - states.filename = filename - states.tag = tag or "whatever" - states.filename = file.addsuffix(states.filename,'lus') - data[states.tag], hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } + states.filename=filename + states.tag=tag or "whatever" + states.filename=file.addsuffix(states.filename,'lus') + data[states.tag],hash[states.tag]=(io.exists(filename) and dofile(filename)) or {},{} end - local function set_by_tag(tag,key,value,default,persistent) - local d, h = data[tag], hash[tag] - if d then - if type(d) == "table" then - local dkey, hkey = key, key - local pre, post = match(key,"(.+)%.([^%.]+)$") - if pre and post then - for k in gmatch(pre,"[^%.]+") do - local dk = d[k] - if not dk then - dk = { } - d[k] = dk - elseif type(dk) == "string" then - -- invalid table, unable to upgrade structure - -- hope for the best or delete the state file - break - end - d = dk - end - dkey, hkey = post, key - end - if value == nil then - value = default - elseif value == false then - -- special case - elseif persistent then - value = value or d[dkey] or default - else - value = value or default - end - d[dkey], h[hkey] = value, value - elseif type(d) == "string" then - -- weird - data[tag], hash[tag] = value, value + local d,h=data[tag],hash[tag] + if d then + if type(d)=="table" then + local dkey,hkey=key,key + local pre,post=match(key,"(.+)%.([^%.]+)$") + if pre and post then + for k in gmatch(pre,"[^%.]+") do + local dk=d[k] + if not dk then + dk={} + d[k]=dk + elseif type(dk)=="string" then + break + end + d=dk end + dkey,hkey=post,key + end + if value==nil then + value=default + elseif value==false then + elseif persistent then + value=value or d[dkey] or default + else + value=value or default + end + d[dkey],h[hkey]=value,value + elseif type(d)=="string" then + data[tag],hash[tag]=value,value end + end end - local function get_by_tag(tag,key,default) - local h = hash[tag] - if h and h[key] then - return h[key] - else - local d = data[tag] - if d then - for k in gmatch(key,"[^%.]+") do - local dk = d[k] - if dk ~= nil then - d = dk - else - return default - end - end - if d == false then - return false - else - return d or default - end + local h=hash[tag] + if h and h[key] then + return h[key] + else + local d=data[tag] + if d then + for k in gmatch(key,"[^%.]+") do + local dk=d[k] + if dk~=nil then + d=dk + else + return default end + end + if d==false then + return false + else + return d or default + end end + end end - -states.set_by_tag = set_by_tag -states.get_by_tag = get_by_tag - +states.set_by_tag=set_by_tag +states.get_by_tag=get_by_tag function states.set(key,value,default,persistent) - set_by_tag(states.tag,key,value,default,persistent) + set_by_tag(states.tag,key,value,default,persistent) end - function states.get(key,default) - return get_by_tag(states.tag,key,default) + return get_by_tag(states.tag,key,default) end - - - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-fmt'] = { - version = 1.001, - comment = "companion to mtxrun", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - - -local format = string.format -local quoted = string.quoted -local luasuffixes = utilities.lua.suffixes - -local report_format = logs.reporter("resolvers","formats") +-- original size: 5954, stripped down to: 4923 -local function primaryflags() -- not yet ok - local trackers = environment.argument("trackers") - local directives = environment.argument("directives") - local flags = "" - if trackers and trackers ~= "" then - flags = flags .. "--trackers=" .. quoted(trackers) - end - if directives and directives ~= "" then - flags = flags .. "--directives=" .. quoted(directives) - end - return flags +if not modules then modules={} end modules ['luat-fmt']={ + version=1.001, + comment="companion to mtxrun", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format=string.format +local concat=table.concat +local quoted=string.quoted +local luasuffixes=utilities.lua.suffixes +local report_format=logs.reporter("resolvers","formats") +local function primaryflags() + local trackers=environment.argument("trackers") + local directives=environment.argument("directives") + local flags={} + if trackers and trackers~="" then + flags={ "--trackers="..quoted(trackers) } + end + if directives and directives~="" then + flags={ "--directives="..quoted(directives) } + end + if environment.argument("jit") then + flags={ "--jiton" } + end + return concat(flags," ") end - function environment.make_format(name) - local engine = environment.ownmain or "luatex" - -- change to format path (early as we need expanded paths) - local olddir = dir.current() - local path = caches.getwritablepath("formats",engine) or "" -- maybe platform - if path ~= "" then - lfs.chdir(path) - end - report_format("format path: %s",dir.current()) - -- check source file - local texsourcename = file.addsuffix(name,"mkiv") - local fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" - if fulltexsourcename == "" then - texsourcename = file.addsuffix(name,"tex") - fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" - end - if fulltexsourcename == "" then - report_format("no tex source file with name: %s (mkiv or tex)",name) - lfs.chdir(olddir) - return - else - report_format("using tex source file: %s",fulltexsourcename) - end - local texsourcepath = dir.expandname(file.dirname(fulltexsourcename)) -- really needed - -- check specification - local specificationname = file.replacesuffix(fulltexsourcename,"lus") - local fullspecificationname = resolvers.findfile(specificationname,"tex") or "" - if fullspecificationname == "" then - specificationname = file.join(texsourcepath,"context.lus") - fullspecificationname = resolvers.findfile(specificationname,"tex") or "" - end - if fullspecificationname == "" then - report_format("unknown stub specification: %s",specificationname) - lfs.chdir(olddir) - return - end - local specificationpath = file.dirname(fullspecificationname) - -- load specification - local usedluastub = nil - local usedlualibs = dofile(fullspecificationname) - if type(usedlualibs) == "string" then - usedluastub = file.join(file.dirname(fullspecificationname),usedlualibs) - elseif type(usedlualibs) == "table" then - report_format("using stub specification: %s",fullspecificationname) - local texbasename = file.basename(name) - local luastubname = file.addsuffix(texbasename,luasuffixes.lua) - local lucstubname = file.addsuffix(texbasename,luasuffixes.luc) - -- pack libraries in stub - report_format("creating initialization file: %s",luastubname) - utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) - -- compile stub file (does not save that much as we don't use this stub at startup any more) - if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then - report_format("using compiled initialization file: %s",lucstubname) - usedluastub = lucstubname - else - report_format("using uncompiled initialization file: %s",luastubname) - usedluastub = luastubname - end - else - report_format("invalid stub specification: %s",fullspecificationname) - lfs.chdir(olddir) - return - end - -- generate format - local command = format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform == "unix" and "\\\\" or "\\") - report_format("running command: %s\n",command) - os.spawn(command) - -- remove related mem files - local pattern = file.removesuffix(file.basename(usedluastub)).."-*.mem" - -- report_format("removing related mplib format with pattern '%s'", pattern) - local mp = dir.glob(pattern) - if mp then - for i=1,#mp do - local name = mp[i] - report_format("removing related mplib format %s", file.basename(name)) - os.remove(name) - end - end + local engine=environment.ownmain or "luatex" + local olddir=dir.current() + local path=caches.getwritablepath("formats",engine) or "" + if path~="" then + lfs.chdir(path) + end + report_format("format path: %s",dir.current()) + local texsourcename=file.addsuffix(name,"mkiv") + local fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + if fulltexsourcename=="" then + texsourcename=file.addsuffix(name,"tex") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename=="" then + report_format("no tex source file with name: %s (mkiv or tex)",name) + lfs.chdir(olddir) + return + else + report_format("using tex source file: %s",fulltexsourcename) + end + local texsourcepath=dir.expandname(file.dirname(fulltexsourcename)) + local specificationname=file.replacesuffix(fulltexsourcename,"lus") + local fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + if fullspecificationname=="" then + specificationname=file.join(texsourcepath,"context.lus") + fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + end + if fullspecificationname=="" then + report_format("unknown stub specification: %s",specificationname) + lfs.chdir(olddir) + return + end + local specificationpath=file.dirname(fullspecificationname) + local usedluastub=nil + local usedlualibs=dofile(fullspecificationname) + if type(usedlualibs)=="string" then + usedluastub=file.join(file.dirname(fullspecificationname),usedlualibs) + elseif type(usedlualibs)=="table" then + report_format("using stub specification: %s",fullspecificationname) + local texbasename=file.basename(name) + local luastubname=file.addsuffix(texbasename,luasuffixes.lua) + local lucstubname=file.addsuffix(texbasename,luasuffixes.luc) + report_format("creating initialization file: %s",luastubname) + utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) + if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then + report_format("using compiled initialization file: %s",lucstubname) + usedluastub=lucstubname + else + report_format("using uncompiled initialization file: %s",luastubname) + usedluastub=luastubname + end + else + report_format("invalid stub specification: %s",fullspecificationname) lfs.chdir(olddir) + return + end + local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") + report_format("running command: %s\n",command) + os.spawn(command) + local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" + local mp=dir.glob(pattern) + if mp then + for i=1,#mp do + local name=mp[i] + report_format("removing related mplib format %s",file.basename(name)) + os.remove(name) + end + end + lfs.chdir(olddir) end - function environment.run_format(name,data,more) - if name and name ~= "" then - local engine = environment.ownmain or "luatex" - local barename = file.removesuffix(name) - local fmtname = caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) - if fmtname == "" then - fmtname = resolvers.findfile(file.addsuffix(barename,"fmt")) or "" - end - fmtname = resolvers.cleanpath(fmtname) - if fmtname == "" then - report_format("no format with name: %s",name) - else - local barename = file.removesuffix(name) -- expanded name - local luaname = file.addsuffix(barename,"luc") - if not lfs.isfile(luaname) then - luaname = file.addsuffix(barename,"lua") - end - if not lfs.isfile(luaname) then - report_format("using format name: %s",fmtname) - report_format("no luc/lua with name: %s",barename) - else - local command = format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more ~= "" and quoted(more) or "") - report_format("running command: %s",command) - os.spawn(command) - end - end + if name and name~="" then + local engine=environment.ownmain or "luatex" + local barename=file.removesuffix(name) + local fmtname=caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) + if fmtname=="" then + fmtname=resolvers.findfile(file.addsuffix(barename,"fmt")) or "" + end + fmtname=resolvers.cleanpath(fmtname) + if fmtname=="" then + report_format("no format with name: %s",name) + else + local barename=file.removesuffix(name) + local luaname=file.addsuffix(barename,"luc") + if not lfs.isfile(luaname) then + luaname=file.addsuffix(barename,"lua") + end + if not lfs.isfile(luaname) then + report_format("using format name: %s",fmtname) + report_format("no luc/lua with name: %s",barename) + else + local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") + report_format("running command: %s",command) + os.spawn(command) + end end + end end @@ -18579,128 +14575,106 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-tpl'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This is experimental code. Coming from dos and windows, I've always used %whatever% --- as template variables so let's stick to it. After all, it's easy to parse and stands --- out well. A double %% is turned into a regular %. - -utilities.templates = utilities.templates or { } -local templates = utilities.templates - -local trace_template = false trackers.register("templates.trace",function(v) trace_template = v end) -local report_template = logs.reporter("template") - -local format = string.format -local P, C, Cs, Carg, lpegmatch = lpeg.P, lpeg.C, lpeg.Cs, lpeg.Carg, lpeg.match - --- todo: make installable template.new +-- original size: 3570, stripped down to: 2441 +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local format=string.format +local P,C,Cs,Carg,lpegmatch=lpeg.P,lpeg.C,lpeg.Cs,lpeg.Carg,lpeg.match local replacer - local function replacekey(k,t,recursive) - local v = t[k] - if not v then - if trace_template then - report_template("unknown key %q",k) - end - return "" + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %q",k) + end + return "" + else + if trace_template then + report_template("setting key %q to value %q",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t) else - if trace_template then - report_template("setting key %q to value %q",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t) - else - return v - end + return v end + end end - -local sqlescape = lpeg.replacer { - { "'", "''" }, - { "\\", "\\\\" }, - { "\r\n", "\\n" }, - { "\r", "\\n" }, - -- { "\t", "\\t" }, +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, } - -local escapers = { - lua = function(s) - return format("%q",s) - end, - sql = function(s) - return lpegmatch(sqlescape,s) - end, +local escapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, } - -lpeg.patterns.sqlescape = sqlescape - -local function replacekeyunquoted(s,t,how,recurse) -- ".. \" " - local escaper = how and escapers[how] or escapers.lua - return escaper(replacekey(s,t,recurse)) -end - -local single = P("%") -- test %test% test : resolves test -local double = P("%%") -- test 10%% test : %% becomes % -local lquoted = P("%[") -- test %[test]" test : resolves test with escaped "'s -local rquoted = P("]%") -- - -local escape = double / '%%' -local nosingle = single / '' -local nodouble = double / '' -local nolquoted = lquoted / '' -local norquoted = rquoted / '' - -local key = nosingle * (C((1-nosingle)^1 * Carg(1) * Carg(2) * Carg(3))/replacekey) * nosingle -local unquoted = nolquoted * ((C((1 - norquoted)^1) * Carg(1) * Carg(2) * Carg(3))/replacekeyunquoted) * norquoted -local any = P(1) - - replacer = Cs((unquoted + escape + key + any)^0) - +lpeg.patterns.sqlescape=sqlescape +local function replacekeyunquoted(s,t,how,recurse) + local escaper=how and escapers[how] or escapers.lua + return escaper(replacekey(s,t,recurse)) +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local key=nosingle*(C((1-nosingle)^1*Carg(1)*Carg(2)*Carg(3))/replacekey)*nosingle +local unquoted=nolquoted*((C((1-norquoted)^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyunquoted)*norquoted +local any=P(1) + replacer=Cs((unquoted+escape+key+any)^0) local function replace(str,mapping,how,recurse) - if mapping then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end + if mapping then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end end - --- print(replace("test '%[x]%' test",{ x = [[a 'x'  a]] })) --- print(replace("test '%[x]%' test",{ x = [[a 'x'  a]] },'sql')) - -templates.replace = replace - +templates.replace=replace function templates.load(filename,mapping,how,recurse) - local data = io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end end - function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping = t - end - for k, v in next, t do - t[k] = replace(v,mapping,how,recurse) - end - return t + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t end --- inspect(utilities.templates.replace("test %one% test", { one = "%two%", two = "two" })) --- inspect(utilities.templates.resolve({ one = "%two%", two = "two", three = "%three%" })) - end -- of closure + +-- used libraries : l-lua.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-tab.lua util-sto.lua util-str.lua util-mrg.lua util-lua.lua util-prs.lua util-fmt.lua util-deb.lua trac-inf.lua trac-set.lua trac-log.lua trac-pro.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua luat-sta.lua luat-fmt.lua util-tpl.lua +-- skipped libraries : - +-- original bytes : 589196 +-- stripped bytes : 198197 + -- end library merge own = { } -- not local, might change diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun index 731f8ca21..4d5fd447d 100755 --- a/scripts/context/stubs/unix/mtxrun +++ b/scripts/context/stubs/unix/mtxrun @@ -54,112 +54,54 @@ if not modules then modules = { } end modules ['mtxrun'] = { do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +-- original size: 2319, stripped down to: 1038 + +if not modules then modules={} end modules ['l-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } - --- compatibility hacks ... try to avoid usage - -local major, minor = string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") - -_MAJORVERSION = tonumber(major) or 5 -_MINORVERSION = tonumber(minor) or 1 - --- basics: - +local major,minor=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") +_MAJORVERSION=tonumber(major) or 5 +_MINORVERSION=tonumber(minor) or 1 if loadstring then - - local loadnormal = load - - function load(first,...) - if type(first) == "string" then - return loadstring(first,...) - else - return loadnormal(first,...) - end + local loadnormal=load + function load(first,...) + if type(first)=="string" then + return loadstring(first,...) + else + return loadnormal(first,...) end - + end else - - loadstring = load - + loadstring=load end - --- table: - --- Starting with version 5.2 Lua no longer provide ipairs, which makes --- sense. As we already used the for loop and # in most places the --- impact on ConTeXt was not that large; the remaining ipairs already --- have been replaced. In a similar fashion we also hardly used pairs. --- --- Hm, actually ipairs was retained, but we no longer use it anyway. --- --- Just in case, we provide the fallbacks as discussed in Programming --- in Lua (http://www.lua.org/pil/7.3.html): - if not ipairs then - - -- for k, v in ipairs(t) do ... end - -- for k=1,#t do local v = t[k] ... end - - local function iterate(a,i) - i = i + 1 - local v = a[i] - if v ~= nil then - return i, v --, nil - end - end - - function ipairs(a) - return iterate, a, 0 + local function iterate(a,i) + i=i+1 + local v=a[i] + if v~=nil then + return i,v end - + end + function ipairs(a) + return iterate,a,0 + end end - if not pairs then - - -- for k, v in pairs(t) do ... end - -- for k, v in next, t do ... end - - function pairs(t) - return next, t -- , nil - end - + function pairs(t) + return next,t + end end - --- The unpack function has been moved to the table table, and for compatiility --- reasons we provide both now. - if not table.unpack then - - table.unpack = _G.unpack - + table.unpack=_G.unpack elseif not unpack then - - _G.unpack = table.unpack - + _G.unpack=table.unpack end - --- package: - --- if not package.seachers then --- --- package.searchers = package.loaders -- 5.2 --- --- elseif not package.loaders then --- --- package.loaders = package.searchers --- --- end - -if not package.loaders then -- brr, searchers is a special "loadlib function" userdata type - - package.loaders = package.searchers - +if not package.loaders then + package.loaders=package.searchers end @@ -167,7359 +109,3309 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-lpeg'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1) - --- move utf -> l-unicode --- move string -> l-string or keep it here - -local lpeg = require("lpeg") - --- tracing (only used when we encounter a problem in integration of lpeg in luatex) - --- some code will move to unicode and string - -local report = texio and texio.write_nl or print - --- local lpmatch = lpeg.match --- local lpprint = lpeg.print --- local lpp = lpeg.P --- local lpr = lpeg.R --- local lps = lpeg.S --- local lpc = lpeg.C --- local lpb = lpeg.B --- local lpv = lpeg.V --- local lpcf = lpeg.Cf --- local lpcb = lpeg.Cb --- local lpcg = lpeg.Cg --- local lpct = lpeg.Ct --- local lpcs = lpeg.Cs --- local lpcc = lpeg.Cc --- local lpcmt = lpeg.Cmt --- local lpcarg = lpeg.Carg - --- function lpeg.match(l,...) report("LPEG MATCH") lpprint(l) return lpmatch(l,...) end - --- function lpeg.P (l) local p = lpp (l) report("LPEG P =") lpprint(l) return p end --- function lpeg.R (l) local p = lpr (l) report("LPEG R =") lpprint(l) return p end --- function lpeg.S (l) local p = lps (l) report("LPEG S =") lpprint(l) return p end --- function lpeg.C (l) local p = lpc (l) report("LPEG C =") lpprint(l) return p end --- function lpeg.B (l) local p = lpb (l) report("LPEG B =") lpprint(l) return p end --- function lpeg.V (l) local p = lpv (l) report("LPEG V =") lpprint(l) return p end --- function lpeg.Cf (l) local p = lpcf (l) report("LPEG Cf =") lpprint(l) return p end --- function lpeg.Cb (l) local p = lpcb (l) report("LPEG Cb =") lpprint(l) return p end --- function lpeg.Cg (l) local p = lpcg (l) report("LPEG Cg =") lpprint(l) return p end --- function lpeg.Ct (l) local p = lpct (l) report("LPEG Ct =") lpprint(l) return p end --- function lpeg.Cs (l) local p = lpcs (l) report("LPEG Cs =") lpprint(l) return p end --- function lpeg.Cc (l) local p = lpcc (l) report("LPEG Cc =") lpprint(l) return p end --- function lpeg.Cmt (l) local p = lpcmt (l) report("LPEG Cmt =") lpprint(l) return p end --- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end - -local type, next = type, next -local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format - --- Beware, we predefine a bunch of patterns here and one reason for doing so --- is that we get consistent behaviour in some of the visualizers. - -lpeg.patterns = lpeg.patterns or { } -- so that we can share -local patterns = lpeg.patterns - -local P, R, S, V, Ct, C, Cs, Cc, Cp, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Cp, lpeg.Cmt -local lpegtype, lpegmatch = lpeg.type, lpeg.match - -local anything = P(1) -local endofstring = P(-1) -local alwaysmatched = P(true) - -patterns.anything = anything -patterns.endofstring = endofstring -patterns.beginofstring = alwaysmatched -patterns.alwaysmatched = alwaysmatched - -local digit, sign = R('09'), S('+-') -local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") -local newline = crlf + S("\r\n") -- cr + lf -local escaped = P("\\") * anything -local squote = P("'") -local dquote = P('"') -local space = P(" ") - -local utfbom_32_be = P('\000\000\254\255') -local utfbom_32_le = P('\255\254\000\000') -local utfbom_16_be = P('\255\254') -local utfbom_16_le = P('\254\255') -local utfbom_8 = P('\239\187\191') -local utfbom = utfbom_32_be + utfbom_32_le - + utfbom_16_be + utfbom_16_le - + utfbom_8 -local utftype = utfbom_32_be * Cc("utf-32-be") + utfbom_32_le * Cc("utf-32-le") - + utfbom_16_be * Cc("utf-16-be") + utfbom_16_le * Cc("utf-16-le") - + utfbom_8 * Cc("utf-8") + alwaysmatched * Cc("utf-8") -- assume utf8 -local utfoffset = utfbom_32_be * Cc(4) + utfbom_32_le * Cc(4) - + utfbom_16_be * Cc(2) + utfbom_16_le * Cc(2) - + utfbom_8 * Cc(3) + Cc(0) - -local utf8next = R("\128\191") - -patterns.utf8one = R("\000\127") -patterns.utf8two = R("\194\223") * utf8next -patterns.utf8three = R("\224\239") * utf8next * utf8next -patterns.utf8four = R("\240\244") * utf8next * utf8next * utf8next -patterns.utfbom = utfbom -patterns.utftype = utftype -patterns.utfoffset = utfoffset - -local utf8char = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four -local validutf8char = utf8char^0 * endofstring * Cc(true) + Cc(false) - -patterns.utf8 = utf8char -patterns.utf8char = utf8char -patterns.validutf8 = validutf8char -patterns.validutf8char = validutf8char - -local eol = S("\n\r") -local spacer = S(" \t\f\v") -- + char(0xc2, 0xa0) if we want utf (cf mail roberto) -local whitespace = eol + spacer -local nonspacer = 1 - spacer -local nonwhitespace = 1 - whitespace - -patterns.eol = eol -patterns.spacer = spacer -patterns.whitespace = whitespace -patterns.nonspacer = nonspacer -patterns.nonwhitespace = nonwhitespace - -local stripper = spacer^0 * C((spacer^0 * nonspacer^1)^0) -- from example by roberto - ------ collapser = Cs(spacer^0/"" * ((spacer^1 * P(-1) / "") + (spacer^1/" ") + P(1))^0) -local collapser = Cs(spacer^0/"" * nonspacer^0 * ((spacer^0/" " * nonspacer^1)^0)) - -patterns.stripper = stripper -patterns.collapser = collapser - -patterns.digit = digit -patterns.sign = sign -patterns.cardinal = sign^0 * digit^1 -patterns.integer = sign^0 * digit^1 -patterns.unsigned = digit^0 * P('.') * digit^1 -patterns.float = sign^0 * patterns.unsigned -patterns.cunsigned = digit^0 * P(',') * digit^1 -patterns.cfloat = sign^0 * patterns.cunsigned -patterns.number = patterns.float + patterns.integer -patterns.cnumber = patterns.cfloat + patterns.integer -patterns.oct = P("0") * R("07")^1 -patterns.octal = patterns.oct -patterns.HEX = P("0x") * R("09","AF")^1 -patterns.hex = P("0x") * R("09","af")^1 -patterns.hexadecimal = P("0x") * R("09","AF","af")^1 -patterns.lowercase = R("az") -patterns.uppercase = R("AZ") -patterns.letter = patterns.lowercase + patterns.uppercase -patterns.space = space -patterns.tab = P("\t") -patterns.spaceortab = patterns.space + patterns.tab -patterns.newline = newline -patterns.emptyline = newline^1 -patterns.equal = P("=") -patterns.comma = P(",") -patterns.commaspacer = P(",") * spacer^0 -patterns.period = P(".") -patterns.colon = P(":") -patterns.semicolon = P(";") -patterns.underscore = P("_") -patterns.escaped = escaped -patterns.squote = squote -patterns.dquote = dquote -patterns.nosquote = (escaped + (1-squote))^0 -patterns.nodquote = (escaped + (1-dquote))^0 -patterns.unsingle = (squote/"") * patterns.nosquote * (squote/"") -- will change to C in the middle -patterns.undouble = (dquote/"") * patterns.nodquote * (dquote/"") -- will change to C in the middle -patterns.unquoted = patterns.undouble + patterns.unsingle -- more often undouble -patterns.unspacer = ((patterns.spacer^1)/"")^0 - -patterns.singlequoted = squote * patterns.nosquote * squote -patterns.doublequoted = dquote * patterns.nodquote * dquote -patterns.quoted = patterns.doublequoted + patterns.singlequoted - -patterns.propername = R("AZ","az","__") * R("09","AZ","az", "__")^0 * P(-1) - -patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1 -patterns.beginline = #(1-newline) - -patterns.longtostring = Cs(whitespace^0/"" * nonwhitespace^0 * ((whitespace^0/" " * (patterns.quoted + nonwhitespace)^1)^0)) - -local function anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * V(1) } -end - -lpeg.anywhere = anywhere +-- original size: 25285, stripped down to: 13969 +if not modules then modules={} end modules ['l-lpeg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local lpeg=require("lpeg") +local report=texio and texio.write_nl or print +local type,next,tostring=type,next,tostring +local byte,char,gmatch,format=string.byte,string.char,string.gmatch,string.format +local floor=math.floor +lpeg.patterns=lpeg.patterns or {} +local patterns=lpeg.patterns +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt +local lpegtype,lpegmatch=lpeg.type,lpeg.match +local anything=P(1) +local endofstring=P(-1) +local alwaysmatched=P(true) +patterns.anything=anything +patterns.endofstring=endofstring +patterns.beginofstring=alwaysmatched +patterns.alwaysmatched=alwaysmatched +local digit,sign=R('09'),S('+-') +local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") +local newline=crlf+S("\r\n") +local escaped=P("\\")*anything +local squote=P("'") +local dquote=P('"') +local space=P(" ") +local utfbom_32_be=P('\000\000\254\255') +local utfbom_32_le=P('\255\254\000\000') +local utfbom_16_be=P('\255\254') +local utfbom_16_le=P('\254\255') +local utfbom_8=P('\239\187\191') +local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8 +local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8") +local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0) +local utf8next=R("\128\191") +patterns.utf8one=R("\000\127") +patterns.utf8two=R("\194\223")*utf8next +patterns.utf8three=R("\224\239")*utf8next*utf8next +patterns.utf8four=R("\240\244")*utf8next*utf8next*utf8next +patterns.utfbom=utfbom +patterns.utftype=utftype +patterns.utfoffset=utfoffset +local utf8char=patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four +local validutf8char=utf8char^0*endofstring*Cc(true)+Cc(false) +patterns.utf8=utf8char +patterns.utf8char=utf8char +patterns.validutf8=validutf8char +patterns.validutf8char=validutf8char +local eol=S("\n\r") +local spacer=S(" \t\f\v") +local whitespace=eol+spacer +local nonspacer=1-spacer +local nonwhitespace=1-whitespace +patterns.eol=eol +patterns.spacer=spacer +patterns.whitespace=whitespace +patterns.nonspacer=nonspacer +patterns.nonwhitespace=nonwhitespace +local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +patterns.stripper=stripper +patterns.collapser=collapser +patterns.digit=digit +patterns.sign=sign +patterns.cardinal=sign^0*digit^1 +patterns.integer=sign^0*digit^1 +patterns.unsigned=digit^0*P('.')*digit^1 +patterns.float=sign^0*patterns.unsigned +patterns.cunsigned=digit^0*P(',')*digit^1 +patterns.cfloat=sign^0*patterns.cunsigned +patterns.number=patterns.float+patterns.integer +patterns.cnumber=patterns.cfloat+patterns.integer +patterns.oct=P("0")*R("07")^1 +patterns.octal=patterns.oct +patterns.HEX=P("0x")*R("09","AF")^1 +patterns.hex=P("0x")*R("09","af")^1 +patterns.hexadecimal=P("0x")*R("09","AF","af")^1 +patterns.lowercase=R("az") +patterns.uppercase=R("AZ") +patterns.letter=patterns.lowercase+patterns.uppercase +patterns.space=space +patterns.tab=P("\t") +patterns.spaceortab=patterns.space+patterns.tab +patterns.newline=newline +patterns.emptyline=newline^1 +patterns.equal=P("=") +patterns.comma=P(",") +patterns.commaspacer=P(",")*spacer^0 +patterns.period=P(".") +patterns.colon=P(":") +patterns.semicolon=P(";") +patterns.underscore=P("_") +patterns.escaped=escaped +patterns.squote=squote +patterns.dquote=dquote +patterns.nosquote=(escaped+(1-squote))^0 +patterns.nodquote=(escaped+(1-dquote))^0 +patterns.unsingle=(squote/"")*patterns.nosquote*(squote/"") +patterns.undouble=(dquote/"")*patterns.nodquote*(dquote/"") +patterns.unquoted=patterns.undouble+patterns.unsingle +patterns.unspacer=((patterns.spacer^1)/"")^0 +patterns.singlequoted=squote*patterns.nosquote*squote +patterns.doublequoted=dquote*patterns.nodquote*dquote +patterns.quoted=patterns.doublequoted+patterns.singlequoted +patterns.propername=R("AZ","az","__")*R("09","AZ","az","__")^0*P(-1) +patterns.somecontent=(anything-newline-space)^1 +patterns.beginline=#(1-newline) +patterns.longtostring=Cs(whitespace^0/""*nonwhitespace^0*((whitespace^0/" "*(patterns.quoted+nonwhitespace)^1)^0)) +local function anywhere(pattern) + return P { P(pattern)+1*V(1) } +end +lpeg.anywhere=anywhere function lpeg.instringchecker(p) - p = anywhere(p) - return function(str) - return lpegmatch(p,str) and true or false - end + p=anywhere(p) + return function(str) + return lpegmatch(p,str) and true or false + end end - -function lpeg.splitter(pattern, action) - return (((1-P(pattern))^1)/action+1)^0 +function lpeg.splitter(pattern,action) + return (((1-P(pattern))^1)/action+1)^0 end - -function lpeg.tsplitter(pattern, action) - return Ct((((1-P(pattern))^1)/action+1)^0) +function lpeg.tsplitter(pattern,action) + return Ct((((1-P(pattern))^1)/action+1)^0) end - --- probleem: separator can be lpeg and that does not hash too well, but --- it's quite okay as the key is then not garbage collected - -local splitters_s, splitters_m, splitters_t = { }, { }, { } - +local splitters_s,splitters_m,splitters_t={},{},{} local function splitat(separator,single) - local splitter = (single and splitters_s[separator]) or splitters_m[separator] - if not splitter then - separator = P(separator) - local other = C((1 - separator)^0) - if single then - local any = anything - splitter = other * (separator * C(any^0) + "") -- ? - splitters_s[separator] = splitter - else - splitter = other * (separator * other)^0 - splitters_m[separator] = splitter - end + local splitter=(single and splitters_s[separator]) or splitters_m[separator] + if not splitter then + separator=P(separator) + local other=C((1-separator)^0) + if single then + local any=anything + splitter=other*(separator*C(any^0)+"") + splitters_s[separator]=splitter + else + splitter=other*(separator*other)^0 + splitters_m[separator]=splitter end - return splitter + end + return splitter end - local function tsplitat(separator) - local splitter = splitters_t[separator] - if not splitter then - splitter = Ct(splitat(separator)) - splitters_t[separator] = splitter - end - return splitter -end - -lpeg.splitat = splitat -lpeg.tsplitat = tsplitat - + local splitter=splitters_t[separator] + if not splitter then + splitter=Ct(splitat(separator)) + splitters_t[separator]=splitter + end + return splitter +end +lpeg.splitat=splitat +lpeg.tsplitat=tsplitat function string.splitup(str,separator) - if not separator then - separator = "," - end - return lpegmatch(splitters_m[separator] or splitat(separator),str) + if not separator then + separator="," + end + return lpegmatch(splitters_m[separator] or splitat(separator),str) end - --- local p = splitat("->",false) print(lpegmatch(p,"oeps->what->more")) -- oeps what more --- local p = splitat("->",true) print(lpegmatch(p,"oeps->what->more")) -- oeps what->more --- local p = splitat("->",false) print(lpegmatch(p,"oeps")) -- oeps --- local p = splitat("->",true) print(lpegmatch(p,"oeps")) -- oeps - -local cache = { } - +local cache={} function lpeg.split(separator,str) - local c = cache[separator] - if not c then - c = tsplitat(separator) - cache[separator] = c - end - return lpegmatch(c,str) + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c + end + return lpegmatch(c,str) end - function string.split(str,separator) - if separator then - local c = cache[separator] - if not c then - c = tsplitat(separator) - cache[separator] = c - end - return lpegmatch(c,str) - else - return { str } + if separator then + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c end -end - -local spacing = patterns.spacer^0 * newline -- sort of strip -local empty = spacing * Cc("") -local nonempty = Cs((1-spacing)^1) * spacing^-1 -local content = (empty + nonempty)^1 - -patterns.textline = content - -local linesplitter = tsplitat(newline) - -patterns.linesplitter = linesplitter - + return lpegmatch(c,str) + else + return { str } + end +end +local spacing=patterns.spacer^0*newline +local empty=spacing*Cc("") +local nonempty=Cs((1-spacing)^1)*spacing^-1 +local content=(empty+nonempty)^1 +patterns.textline=content +local linesplitter=tsplitat(newline) +patterns.linesplitter=linesplitter function string.splitlines(str) - return lpegmatch(linesplitter,str) + return lpegmatch(linesplitter,str) end - --- lpeg.splitters = cache -- no longer public - -local cache = { } - +local cache={} function lpeg.checkedsplit(separator,str) - local c = cache[separator] - if not c then - separator = P(separator) - local other = C((1 - separator)^1) - c = Ct(separator^0 * other * (separator^1 * other)^0) - cache[separator] = c - end - return lpegmatch(c,str) + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) end - function string.checkedsplit(str,separator) - local c = cache[separator] - if not c then - separator = P(separator) - local other = C((1 - separator)^1) - c = Ct(separator^0 * other * (separator^1 * other)^0) - cache[separator] = c - end - return lpegmatch(c,str) -end - --- from roberto's site: - -local function f2(s) local c1, c2 = byte(s,1,2) return c1 * 64 + c2 - 12416 end -local function f3(s) local c1, c2, c3 = byte(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end -local function f4(s) local c1, c2, c3, c4 = byte(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end - -local utf8byte = patterns.utf8one/byte + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 - -patterns.utf8byte = utf8byte - - - -local cache = { } - + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) +end +local function f2(s) local c1,c2=byte(s,1,2) return c1*64+c2-12416 end +local function f3(s) local c1,c2,c3=byte(s,1,3) return (c1*64+c2)*64+c3-925824 end +local function f4(s) local c1,c2,c3,c4=byte(s,1,4) return ((c1*64+c2)*64+c3)*64+c4-63447168 end +local utf8byte=patterns.utf8one/byte+patterns.utf8two/f2+patterns.utf8three/f3+patterns.utf8four/f4 +patterns.utf8byte=utf8byte +local cache={} function lpeg.stripper(str) - if type(str) == "string" then - local s = cache[str] - if not s then - s = Cs(((S(str)^1)/"" + 1)^0) - cache[str] = s - end - return s - else - return Cs(((str^1)/"" + 1)^0) + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs(((S(str)^1)/""+1)^0) + cache[str]=s end + return s + else + return Cs(((str^1)/""+1)^0) + end end - -local cache = { } - +local cache={} function lpeg.keeper(str) - if type(str) == "string" then - local s = cache[str] - if not s then - s = Cs((((1-S(str))^1)/"" + 1)^0) - cache[str] = s - end - return s + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs((((1-S(str))^1)/""+1)^0) + cache[str]=s + end + return s + else + return Cs((((1-str)^1)/""+1)^0) + end +end +function lpeg.frontstripper(str) + return (P(str)+P(true))*Cs(anything^0) +end +function lpeg.endstripper(str) + return Cs((1-P(str)*endofstring)^0) +end +function lpeg.replacer(one,two,makefunction,isutf) + local pattern + local u=isutf and utf8char or 1 + if type(one)=="table" then + local no=#one + local p=P(false) + if no==0 then + for k,v in next,one do + p=p+P(k)/v + end + pattern=Cs((p+u)^0) + elseif no==1 then + local o=one[1] + one,two=P(o[1]),o[2] + pattern=Cs((one/two+u)^0) + else + for i=1,no do + local o=one[i] + p=p+P(o[1])/o[2] + end + pattern=Cs((p+u)^0) + end + else + pattern=Cs((P(one)/(two or "")+u)^0) + end + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +function lpeg.finder(lst,makefunction) + local pattern + if type(lst)=="table" then + pattern=P(false) + if #lst==0 then + for k,v in next,lst do + pattern=pattern+P(k) + end else - return Cs((((1-str)^1)/"" + 1)^0) + for i=1,#lst do + pattern=pattern+P(lst[i]) + end end + else + pattern=P(lst) + end + pattern=(1-pattern)^0*pattern + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +local splitters_f,splitters_s={},{} +function lpeg.firstofsplit(separator) + local splitter=splitters_f[separator] + if not splitter then + separator=P(separator) + splitter=C((1-separator)^0) + splitters_f[separator]=splitter + end + return splitter +end +function lpeg.secondofsplit(separator) + local splitter=splitters_s[separator] + if not splitter then + separator=P(separator) + splitter=(1-separator)^0*separator*C(anything^0) + splitters_s[separator]=splitter + end + return splitter end - -function lpeg.frontstripper(str) -- or pattern (yet undocumented) - return (P(str) + P(true)) * Cs(anything^0) +function lpeg.balancer(left,right) + left,right=P(left),P(right) + return P { left*((1-left-right)+V(1))^0*right } end - -function lpeg.endstripper(str) -- or pattern (yet undocumented) - return Cs((1 - P(str) * endofstring)^0) +local nany=utf8char/"" +function lpeg.counter(pattern) + pattern=Cs((P(pattern)/" "+nany)^0) + return function(str) + return #lpegmatch(pattern,str) + end +end +local utfcharacters=utf and utf.characters or string.utfcharacters +local utfgmatch=unicode and unicode.utf8.gmatch +local utfchar=utf and utf.char or (unicode and unicode.utf8 and unicode.utf8.char) +lpeg.UP=lpeg.P +if utfcharacters then + function lpeg.US(str) + local p=P(false) + for uc in utfcharacters(str) do + p=p+P(uc) + end + return p + end +elseif utfgmatch then + function lpeg.US(str) + local p=P(false) + for uc in utfgmatch(str,".") do + p=p+P(uc) + end + return p + end +else + function lpeg.US(str) + local p=P(false) + local f=function(uc) + p=p+P(uc) + end + lpegmatch((utf8char/f)^0,str) + return p + end end - --- Just for fun I looked at the used bytecode and --- p = (p and p + pp) or pp gets one more (testset). - --- todo: cache when string - -function lpeg.replacer(one,two,makefunction,isutf) -- in principle we should sort the keys - local pattern - local u = isutf and utf8char or 1 - if type(one) == "table" then - local no = #one - local p = P(false) - if no == 0 then - for k, v in next, one do - p = p + P(k) / v - end - pattern = Cs((p + u)^0) - elseif no == 1 then - local o = one[1] - one, two = P(o[1]), o[2] - -- pattern = Cs(((1-one)^1 + one/two)^0) - pattern = Cs((one/two + u)^0) +local range=utf8byte*utf8byte+Cc(false) +function lpeg.UR(str,more) + local first,last + if type(str)=="number" then + first=str + last=more or first + else + first,last=lpegmatch(range,str) + if not last then + return P(str) + end + end + if first==last then + return P(str) + elseif utfchar and (last-first<8) then + local p=P(false) + for i=first,last do + p=p+P(utfchar(i)) + end + return p + else + local f=function(b) + return b>=first and b<=last + end + return utf8byte/f + end +end +function lpeg.is_lpeg(p) + return p and lpegtype(p)=="pattern" +end +function lpeg.oneof(list,...) + if type(list)~="table" then + list={ list,... } + end + local p=P(list[1]) + for l=2,#list do + p=p+P(list[l]) + end + return p +end +local sort=table.sort +local function copyindexed(old) + local new={} + for i=1,#old do + new[i]=old + end + return new +end +local function sortedkeys(tab) + local keys,s={},0 + for key,_ in next,tab do + s=s+1 + keys[s]=key + end + sort(keys) + return keys +end +function lpeg.append(list,pp,delayed,checked) + local p=pp + if #list>0 then + local keys=copyindexed(list) + sort(keys) + for i=#keys,1,-1 do + local k=keys[i] + if p then + p=P(k)+p + else + p=P(k) + end + end + elseif delayed then + local keys=sortedkeys(list) + if p then + for i=1,#keys,1 do + local k=keys[i] + local v=list[k] + p=P(k)/list+p + end + else + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)+p else - for i=1,no do - local o = one[i] - p = p + P(o[1]) / o[2] - end - pattern = Cs((p + u)^0) + p=P(k) end - else - pattern = Cs((P(one)/(two or "") + u)^0) + end + if p then + p=p/list + end end - if makefunction then - return function(str) - return lpegmatch(pattern,str) + elseif checked then + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + if k==v then + p=P(k)+p + else + p=P(k)/v+p end - else - return pattern - end -end - -function lpeg.finder(lst,makefunction) - local pattern - if type(lst) == "table" then - pattern = P(false) - if #lst == 0 then - for k, v in next, lst do - pattern = pattern + P(k) -- ignore key, so we can use a replacer table - end + else + if k==v then + p=P(k) else - for i=1,#lst do - pattern = pattern + P(lst[i]) - end + p=P(k)/v end - else - pattern = P(lst) + end end - pattern = (1-pattern)^0 * pattern - if makefunction then - return function(str) - return lpegmatch(pattern,str) - end - else - return pattern + else + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)/v+p + else + p=P(k)/v + end end + end + return p end - --- print(lpeg.match(lpeg.replacer("e","a"),"test test")) --- print(lpeg.match(lpeg.replacer{{"e","a"}},"test test")) --- print(lpeg.match(lpeg.replacer({ e = "a", t = "x" }),"test test")) - -local splitters_f, splitters_s = { }, { } - -function lpeg.firstofsplit(separator) -- always return value - local splitter = splitters_f[separator] - if not splitter then - separator = P(separator) - splitter = C((1 - separator)^0) - splitters_f[separator] = splitter +local function make(t) + local p + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + local v=t[k] + if not p then + if next(v) then + p=P(k)*make(v) + else + p=P(k) + end + else + if next(v) then + p=p+P(k)*make(v) + else + p=p+P(k) + end end - return splitter + end + return p end - -function lpeg.secondofsplit(separator) -- nil if not split - local splitter = splitters_s[separator] - if not splitter then - separator = P(separator) - splitter = (1 - separator)^0 * separator * C(anything^0) - splitters_s[separator] = splitter +function lpeg.utfchartabletopattern(list) + local tree={} + for i=1,#list do + local t=tree + for c in gmatch(list[i],".") do + if not t[c] then + t[c]={} + end + t=t[c] + end + end + return make(tree) +end +patterns.containseol=lpeg.finder(eol) +local function nextstep(n,step,result) + local m=n%step + local d=floor(n/step) + if d>0 then + local v=V(tostring(step)) + local s=result.start + for i=1,d do + if s then + s=v*s + else + s=v + end end - return splitter + result.start=s + end + if step>1 and result.start then + local v=V(tostring(step/2)) + result[tostring(step)]=v*v + end + if step>0 then + return nextstep(m,step/2,result) + else + return result + end end - -function lpeg.balancer(left,right) - left, right = P(left), P(right) - return P { left * ((1 - left - right) + V(1))^0 * right } +function lpeg.times(pattern,n) + return P(nextstep(n,2^16,{ "start",["1"]=pattern })) end --- print(1,lpegmatch(lpeg.firstofsplit(":"),"bc:de")) --- print(2,lpegmatch(lpeg.firstofsplit(":"),":de")) -- empty --- print(3,lpegmatch(lpeg.firstofsplit(":"),"bc")) --- print(4,lpegmatch(lpeg.secondofsplit(":"),"bc:de")) --- print(5,lpegmatch(lpeg.secondofsplit(":"),"bc:")) -- empty --- print(6,lpegmatch(lpeg.secondofsplit(":",""),"bc")) --- print(7,lpegmatch(lpeg.secondofsplit(":"),"bc")) --- print(9,lpegmatch(lpeg.secondofsplit(":","123"),"bc")) - --- -- slower: --- --- function lpeg.counter(pattern) --- local n, pattern = 0, (lpeg.P(pattern)/function() n = n + 1 end + lpeg.anything)^0 --- return function(str) n = 0 ; lpegmatch(pattern,str) ; return n end --- end - -local nany = utf8char/"" -function lpeg.counter(pattern) - pattern = Cs((P(pattern)/" " + nany)^0) - return function(str) - return #lpegmatch(pattern,str) - end -end +end -- of closure --- utf extensies +do -- create closure to overcome 200 locals limit -local utfcharacters = utf and utf.characters or string.utfcharacters -local utfgmatch = unicode and unicode.utf8.gmatch -local utfchar = utf and utf.char or (unicode and unicode.utf8 and unicode.utf8.char) +-- original size: 361, stripped down to: 322 -lpeg.UP = lpeg.P +if not modules then modules={} end modules ['l-functions']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +functions=functions or {} +function functions.dummy() end -if utfcharacters then - function lpeg.US(str) - local p = P(false) - for uc in utfcharacters(str) do - p = p + P(uc) - end - return p - end +end -- of closure +do -- create closure to overcome 200 locals limit -elseif utfgmatch then +-- original size: 5491, stripped down to: 2685 - function lpeg.US(str) - local p = P(false) - for uc in utfgmatch(str,".") do - p = p + P(uc) - end - return p - end +if not modules then modules={} end modules ['l-string']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local string=string +local sub,gmatch,format,char,byte,rep,lower=string.sub,string.gmatch,string.format,string.char,string.byte,string.rep,string.lower +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local P,S,C,Ct,Cc,Cs=lpeg.P,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cc,lpeg.Cs +local unquoted=patterns.squote*C(patterns.nosquote)*patterns.squote+patterns.dquote*C(patterns.nodquote)*patterns.dquote +function string.unquoted(str) + return lpegmatch(unquoted,str) or str +end +function string.quoted(str) + return format("%q",str) +end +function string.count(str,pattern) + local n=0 + for _ in gmatch(str,pattern) do + n=n+1 + end + return n +end +function string.limit(str,n,sentinel) + if #str>n then + sentinel=sentinel or "..." + return sub(str,1,(n-#sentinel))..sentinel + else + return str + end +end +local stripper=patterns.stripper +local collapser=patterns.collapser +local longtostring=patterns.longtostring +function string.strip(str) + return lpegmatch(stripper,str) or "" +end +function string.collapsespaces(str) + return lpegmatch(collapser,str) or "" +end +function string.longtostring(str) + return lpegmatch(longtostring,str) or "" +end +local pattern=P(" ")^0*P(-1) +function string.is_empty(str) + if str=="" then + return true + else + return lpegmatch(pattern,str) and true or false + end +end +local anything=patterns.anything +local allescapes=Cc("%")*S(".-+%?()[]*") +local someescapes=Cc("%")*S(".-+%()[]") +local matchescapes=Cc(".")*S("*?") +local pattern_a=Cs ((allescapes+anything )^0 ) +local pattern_b=Cs ((someescapes+matchescapes+anything )^0 ) +local pattern_c=Cs (Cc("^")*(someescapes+matchescapes+anything )^0*Cc("$") ) +function string.escapedpattern(str,simple) + return lpegmatch(simple and pattern_b or pattern_a,str) +end +function string.topattern(str,lowercase,strict) + if str=="" then + return ".*" + elseif strict then + str=lpegmatch(pattern_c,str) + else + str=lpegmatch(pattern_b,str) + end + if lowercase then + return lower(str) + else + return str + end +end +function string.valid(str,default) + return (type(str)=="string" and str~="" and str) or default or nil +end +string.itself=function(s) return s end +local pattern=Ct(C(1)^0) +function string.totable(str) + return lpegmatch(pattern,str) +end +local replacer=lpeg.replacer("@","%%") +function string.tformat(fmt,...) + return format(lpegmatch(replacer,fmt),...) +end +string.quote=string.quoted +string.unquote=string.unquoted -else - function lpeg.US(str) - local p = P(false) - local f = function(uc) - p = p + P(uc) - end - lpegmatch((utf8char/f)^0,str) - return p - end +end -- of closure -end +do -- create closure to overcome 200 locals limit -local range = utf8byte * utf8byte + Cc(false) -- utf8byte is already a capture +-- original size: 28973, stripped down to: 19400 -function lpeg.UR(str,more) - local first, last - if type(str) == "number" then - first = str - last = more or first +if not modules then modules={} end modules ['l-table']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring,tonumber,ipairs,select=type,next,tostring,tonumber,ipairs,select +local table,string=table,string +local concat,sort,insert,remove=table.concat,table.sort,table.insert,table.remove +local format,lower,dump=string.format,string.lower,string.dump +local getmetatable,setmetatable=getmetatable,setmetatable +local getinfo=debug.getinfo +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local floor=math.floor +local stripper=patterns.stripper +function table.strip(tab) + local lst,l={},0 + for i=1,#tab do + local s=lpegmatch(stripper,tab[i]) or "" + if s=="" then else - first, last = lpegmatch(range,str) - if not last then - return P(str) - end + l=l+1 + lst[l]=s end - if first == last then - return P(str) - elseif utfchar and (last - first < 8) then -- a somewhat arbitrary criterium - local p = P(false) - for i=first,last do - p = p + P(utfchar(i)) - end - return p -- nil when invalid range - else - local f = function(b) - return b >= first and b <= last - end - -- tricky, these nested captures - return utf8byte / f -- nil when invalid range + end + return lst +end +function table.keys(t) + if t then + local keys,k={},0 + for key,_ in next,t do + k=k+1 + keys[k]=key end + return keys + else + return {} + end end - --- print(lpeg.match(lpeg.Cs((C(lpeg.UR("αω"))/{ ["χ"] = "OEPS" })^0),"αωχαω")) - --- lpeg.print(lpeg.R("ab","cd","gh")) --- lpeg.print(lpeg.P("a","b","c")) --- lpeg.print(lpeg.S("a","b","c")) - --- print(lpeg.count("äáàa",lpeg.P("á") + lpeg.P("à"))) --- print(lpeg.count("äáàa",lpeg.UP("áà"))) --- print(lpeg.count("äáàa",lpeg.US("àá"))) --- print(lpeg.count("äáàa",lpeg.UR("aá"))) --- print(lpeg.count("äáàa",lpeg.UR("àá"))) --- print(lpeg.count("äáàa",lpeg.UR(0x0000,0xFFFF))) - -function lpeg.is_lpeg(p) - return p and lpegtype(p) == "pattern" -end - -function lpeg.oneof(list,...) -- lpeg.oneof("elseif","else","if","then") -- assume proper order - if type(list) ~= "table" then - list = { list, ... } - end - -- table.sort(list) -- longest match first - local p = P(list[1]) - for l=2,#list do - p = p + P(list[l]) - end - return p -end - --- For the moment here, but it might move to utilities. Beware, we need to --- have the longest keyword first, so 'aaa' comes beforte 'aa' which is why we --- loop back from the end cq. prepend. - -local sort = table.sort - -local function copyindexed(old) - local new = { } - for i=1,#old do - new[i] = old - end - return new +local function compare(a,b) + local ta,tb=type(a),type(b) + if ta==tb then + return a 0 then - local keys = copyindexed(list) - sort(keys) - for i=#keys,1,-1 do - local k = keys[i] - if p then - p = P(k) + p - else - p = P(k) - end - end - elseif delayed then -- hm, it looks like the lpeg parser resolves anyway - local keys = sortedkeys(list) - if p then - for i=1,#keys,1 do - local k = keys[i] - local v = list[k] - p = P(k)/list + p - end - else - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - p = P(k) + p - else - p = P(k) - end - end - if p then - p = p / list - end - end - elseif checked then - -- problem: substitution gives a capture - local keys = sortedkeys(list) - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - if k == v then - p = P(k) + p - else - p = P(k)/v + p - end - else - if k == v then - p = P(k) - else - p = P(k)/v - end - end + if tab then + local srt,category,s={},0,0 + for key,_ in next,tab do + s=s+1 + srt[s]=key + if category==3 then + else + local tkey=type(key) + if tkey=="string" then + category=(category==2 and 3) or 1 + elseif tkey=="number" then + category=(category==1 and 3) or 2 + else + category=3 end + end + end + if category==0 or category==3 then + sort(srt,compare) else - local keys = sortedkeys(list) - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - p = P(k)/v + p - else - p = P(k)/v - end - end + sort(srt) end - return p + return srt + else + return {} + end end - --- inspect(lpeg.append({ a = "1", aa = "1", aaa = "1" } ,nil,true)) --- inspect(lpeg.append({ ["degree celsius"] = "1", celsius = "1", degree = "1" } ,nil,true)) - --- function lpeg.exact_match(words,case_insensitive) --- local pattern = concat(words) --- if case_insensitive then --- local pattern = S(upper(characters)) + S(lower(characters)) --- local list = { } --- for i=1,#words do --- list[lower(words[i])] = true --- end --- return Cmt(pattern^1, function(_,i,s) --- return list[lower(s)] and i --- end) --- else --- local pattern = S(concat(words)) --- local list = { } --- for i=1,#words do --- list[words[i]] = true --- end --- return Cmt(pattern^1, function(_,i,s) --- return list[s] and i --- end) --- end --- end - --- experiment: - -local function make(t) - local p - local keys = sortedkeys(t) - for i=1,#keys do - local k = keys[i] - local v = t[k] - if not p then - if next(v) then - p = P(k) * make(v) - else - p = P(k) - end - else - if next(v) then - p = p + P(k) * make(v) - else - p = p + P(k) - end - end +local function sortedhashkeys(tab) + if tab then + local srt,s={},0 + for key,_ in next,tab do + if key then + s=s+1 + srt[s]=key + end end - return p + sort(srt) + return srt + else + return {} + end end - -function lpeg.utfchartabletopattern(list) -- goes to util-lpg - local tree = { } - for i=1,#list do - local t = tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c] = { } - end - t = t[c] - end +function table.allkeys(t) + local keys={} + for i=1,#t do + for k,v in next,t[i] do + keys[k]=true end - return make(tree) -end - --- inspect ( lpeg.utfchartabletopattern { --- utfchar(0x00A0), -- nbsp --- utfchar(0x2000), -- enquad --- utfchar(0x2001), -- emquad --- utfchar(0x2002), -- enspace --- utfchar(0x2003), -- emspace --- utfchar(0x2004), -- threeperemspace --- utfchar(0x2005), -- fourperemspace --- utfchar(0x2006), -- sixperemspace --- utfchar(0x2007), -- figurespace --- utfchar(0x2008), -- punctuationspace --- utfchar(0x2009), -- breakablethinspace --- utfchar(0x200A), -- hairspace --- utfchar(0x200B), -- zerowidthspace --- utfchar(0x202F), -- narrownobreakspace --- utfchar(0x205F), -- math thinspace --- } ) - --- a few handy ones: --- --- faster than find(str,"[\n\r]") when match and # > 7 and always faster when # > 3 - -patterns.containseol = lpeg.finder(eol) -- (1-eol)^0 * eol - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-functions'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -functions = functions or { } - -function functions.dummy() end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-string'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local string = string -local sub, gmatch, format, char, byte, rep, lower = string.sub, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower -local lpegmatch, patterns = lpeg.match, lpeg.patterns -local P, S, C, Ct, Cc, Cs = lpeg.P, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cs - --- Some functions are already defined in l-lpeg and maybe some from here will --- move there (unless we also expose caches). - --- if not string.split then --- --- function string.split(str,pattern) --- local t = { } --- if #str > 0 then --- local n = 1 --- for s in gmatch(str..pattern,"(.-)"..pattern) do --- t[n] = s --- n = n + 1 --- end --- end --- return t --- end --- --- end - --- function string.unquoted(str) --- return (gsub(str,"^([\"\'])(.*)%1$","%2")) -- interesting pattern --- end - -local unquoted = patterns.squote * C(patterns.nosquote) * patterns.squote - + patterns.dquote * C(patterns.nodquote) * patterns.dquote - -function string.unquoted(str) - return lpegmatch(unquoted,str) or str + end + return sortedkeys(keys) end - --- print(string.unquoted("test")) --- print(string.unquoted([["t\"est"]])) --- print(string.unquoted([["t\"est"x]])) --- print(string.unquoted("\'test\'")) --- print(string.unquoted('"test"')) --- print(string.unquoted('"test"')) - -function string.quoted(str) - return format("%q",str) -- always " +table.sortedkeys=sortedkeys +table.sortedhashkeys=sortedhashkeys +local function nothing() end +local function sortedhash(t) + if t then + local n,s=0,sortedkeys(t) + local function kv(s) + n=n+1 + local k=s[n] + return k,t[k] + end + return kv,s + else + return nothing + end +end +table.sortedhash=sortedhash +table.sortedpairs=sortedhash +function table.append(t,list) + local n=#t + for i=1,#list do + n=n+1 + t[n]=list[i] + end + return t +end +function table.prepend(t,list) + local nl=#list + local nt=nl+#t + for i=#t,1,-1 do + t[nt]=t[i] + nt=nt-1 + end + for i=1,#list do + t[i]=list[i] + end + return t +end +function table.merge(t,...) + t=t or {} + for i=1,select("#",...) do + for k,v in next,(select(i,...)) do + t[k]=v + end + end + return t end - -function string.count(str,pattern) -- variant 3 - local n = 0 - for _ in gmatch(str,pattern) do -- not for utf - n = n + 1 +function table.merged(...) + local t={} + for i=1,select("#",...) do + for k,v in next,(select(i,...)) do + t[k]=v end - return n + end + return t end - -function string.limit(str,n,sentinel) -- not utf proof - if #str > n then - sentinel = sentinel or "..." - return sub(str,1,(n-#sentinel)) .. sentinel - else - return str +function table.imerge(t,...) + local nt=#t + for i=1,select("#",...) do + local nst=select(i,...) + for j=1,#nst do + nt=nt+1 + t[nt]=nst[j] end + end + return t end - -local stripper = patterns.stripper -local collapser = patterns.collapser -local longtostring = patterns.longtostring - -function string.strip(str) - return lpegmatch(stripper,str) or "" -end - -function string.collapsespaces(str) - return lpegmatch(collapser,str) or "" -end - -function string.longtostring(str) - return lpegmatch(longtostring,str) or "" -end - --- function string.is_empty(str) --- return not find(str,"%S") --- end - -local pattern = P(" ")^0 * P(-1) - -function string.is_empty(str) - if str == "" then - return true - else - return lpegmatch(pattern,str) and true or false +function table.imerged(...) + local tmp,ntmp={},0 + for i=1,select("#",...) do + local nst=select(i,...) + for j=1,#nst do + ntmp=ntmp+1 + tmp[ntmp]=nst[j] + end + end + return tmp +end +local function fastcopy(old,metatabletoo) + if old then + local new={} + for k,v in next,old do + if type(v)=="table" then + new[k]=fastcopy(v,metatabletoo) + else + new[k]=v + end end -end - - --- if not string.escapedpattern then --- --- local patterns_escapes = { --- ["%"] = "%%", --- ["."] = "%.", --- ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", --- ["["] = "%[", ["]"] = "%]", --- ["("] = "%(", [")"] = "%)", --- -- ["{"] = "%{", ["}"] = "%}" --- -- ["^"] = "%^", ["$"] = "%$", --- } --- --- local simple_escapes = { --- ["-"] = "%-", --- ["."] = "%.", --- ["?"] = ".", --- ["*"] = ".*", --- } --- --- function string.escapedpattern(str,simple) --- return (gsub(str,".",simple and simple_escapes or patterns_escapes)) --- end --- --- function string.topattern(str,lowercase,strict) --- if str == "" then --- return ".*" --- else --- str = gsub(str,".",simple_escapes) --- if lowercase then --- str = lower(str) --- end --- if strict then --- return "^" .. str .. "$" --- else --- return str --- end --- end --- end --- --- end - ---- needs checking - -local anything = patterns.anything -local allescapes = Cc("%") * S(".-+%?()[]*") -- also {} and ^$ ? -local someescapes = Cc("%") * S(".-+%()[]") -- also {} and ^$ ? -local matchescapes = Cc(".") * S("*?") -- wildcard and single match - -local pattern_a = Cs ( ( allescapes + anything )^0 ) -local pattern_b = Cs ( ( someescapes + matchescapes + anything )^0 ) -local pattern_c = Cs ( Cc("^") * ( someescapes + matchescapes + anything )^0 * Cc("$") ) - -function string.escapedpattern(str,simple) - return lpegmatch(simple and pattern_b or pattern_a,str) -end - -function string.topattern(str,lowercase,strict) - if str == "" then - return ".*" - elseif strict then - str = lpegmatch(pattern_c,str) - else - str = lpegmatch(pattern_b,str) + if metatabletoo then + local mt=getmetatable(old) + if mt then + setmetatable(new,mt) + end end - if lowercase then - return lower(str) - else - return str + return new + else + return {} + end +end +local function copy(t,tables) + tables=tables or {} + local tcopy={} + if not tables[t] then + tables[t]=tcopy + end + for i,v in next,t do + if type(i)=="table" then + if tables[i] then + i=tables[i] + else + i=copy(i,tables) + end end + if type(v)~="table" then + tcopy[i]=v + elseif tables[v] then + tcopy[i]=tables[v] + else + tcopy[i]=copy(v,tables) + end + end + local mt=getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy +end +table.fastcopy=fastcopy +table.copy=copy +function table.derive(parent) + local child={} + if parent then + setmetatable(child,{ __index=parent }) + end + return child end - --- print(string.escapedpattern("12+34*.tex",false)) --- print(string.escapedpattern("12+34*.tex",true)) --- print(string.topattern ("12+34*.tex",false,false)) --- print(string.topattern ("12+34*.tex",false,true)) - -function string.valid(str,default) - return (type(str) == "string" and str ~= "" and str) or default or nil -end - --- handy fallback - -string.itself = function(s) return s end - --- also handy (see utf variant) - -local pattern = Ct(C(1)^0) -- string and not utf ! - -function string.totable(str) - return lpegmatch(pattern,str) +function table.tohash(t,value) + local h={} + if t then + if value==nil then value=true end + for _,v in next,t do + h[v]=value + end + end + return h end - --- handy from within tex: - -local replacer = lpeg.replacer("@","%%") -- Watch the escaped % in lpeg! - -function string.tformat(fmt,...) - return format(lpegmatch(replacer,fmt),...) +function table.fromhash(t) + local hsh,h={},0 + for k,v in next,t do + if v then + h=h+1 + hsh[h]=k + end + end + return hsh end - --- obsolete names: - -string.quote = string.quoted -string.unquote = string.unquoted - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-table'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +local noquotes,hexify,handle,reduce,compact,inline,functions +local reserved=table.tohash { + 'and','break','do','else','elseif','end','false','for','function','if', + 'in','local','nil','not','or','repeat','return','then','true','until','while', } - -local type, next, tostring, tonumber, ipairs, select = type, next, tostring, tonumber, ipairs, select -local table, string = table, string -local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, lower, dump = string.format, string.lower, string.dump -local getmetatable, setmetatable = getmetatable, setmetatable -local getinfo = debug.getinfo -local lpegmatch, patterns = lpeg.match, lpeg.patterns -local floor = math.floor - --- extra functions, some might go (when not used) - -local stripper = patterns.stripper - -function table.strip(tab) - local lst, l = { }, 0 - for i=1,#tab do - local s = lpegmatch(stripper,tab[i]) or "" - if s == "" then - -- skip this one +local function simple_table(t) + if #t>0 then + local n=0 + for _,v in next,t do + n=n+1 + end + if n==#t then + local tt,nt={},0 + for i=1,#t do + local v=t[i] + local tv=type(v) + if tv=="number" then + nt=nt+1 + if hexify then + tt[nt]=format("0x%04X",v) + else + tt[nt]=tostring(v) + end + elseif tv=="boolean" then + nt=nt+1 + tt[nt]=tostring(v) + elseif tv=="string" then + nt=nt+1 + tt[nt]=format("%q",v) else - l = l + 1 - lst[l] = s + tt=nil + break end + end + return tt end - return lst + end + return nil end - -function table.keys(t) - if t then - local keys, k = { }, 0 - for key, _ in next, t do - k = k + 1 - keys[k] = key - end - return keys +local propername=patterns.propername +local function dummy() end +local function do_serialize(root,name,depth,level,indexed) + if level>0 then + depth=depth.." " + if indexed then + handle(format("%s{",depth)) else - return { } + local tn=type(name) + if tn=="number" then + if hexify then + handle(format("%s[0x%04X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) + end + elseif tn=="string" then + if noquotes and not reserved[name] and lpegmatch(propername,name) then + handle(format("%s%s={",depth,name)) + else + handle(format("%s[%q]={",depth,name)) + end + elseif tn=="boolean" then + handle(format("%s[%s]={",depth,tostring(name))) + else + handle(format("%s{",depth)) + end end -end - -local function compare(a,b) - local ta, tb = type(a), type(b) -- needed, else 11 < 2 - if ta == tb then - return a < b - else - return tostring(a) < tostring(b) + end + if root and next(root) then + local first,last=nil,0 + if compact then + last=#root + for k=1,last do + if root[k]==nil then + last=k-1 + break + end + end + if last>0 then + first=1 + end end -end - -local function sortedkeys(tab) - if tab then - local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed - for key,_ in next, tab do - s = s + 1 - srt[s] = key - if category == 3 then - -- no further check + local sk=sortedkeys(root) + for i=1,#sk do + local k=sk[i] + local v=root[k] + local t,tk=type(v),type(k) + if compact and first and tk=="number" and k>=first and k<=last then + if t=="number" then + if hexify then + handle(format("%s 0x%04X,",depth,v)) + else + handle(format("%s %s,",depth,v)) + end + elseif t=="string" then + if reduce and tonumber(v) then + handle(format("%s %s,",depth,v)) + else + handle(format("%s %q,",depth,v)) + end + elseif t=="table" then + if not next(v) then + handle(format("%s {},",depth)) + elseif inline then + local st=simple_table(v) + if st then + handle(format("%s { %s },",depth,concat(st,", "))) else - local tkey = type(key) - if tkey == "string" then - category = (category == 2 and 3) or 1 - elseif tkey == "number" then - category = (category == 1 and 3) or 2 - else - category = 3 - end + do_serialize(v,k,depth,level+1,true) end + else + do_serialize(v,k,depth,level+1,true) + end + elseif t=="boolean" then + handle(format("%s %s,",depth,tostring(v))) + elseif t=="function" then + if functions then + handle(format('%s load(%q),',depth,dump(v))) + else + handle(format('%s "function",',depth)) + end + else + handle(format("%s %q,",depth,tostring(v))) + end + elseif k=="__p__" then + if false then + handle(format("%s __p__=nil,",depth)) end - if category == 0 or category == 3 then - sort(srt,compare) + elseif t=="number" then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif tk=="boolean" then + if hexify then + handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) + else + handle(format("%s [%s]=%s,",depth,tostring(k),v)) + end + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + if hexify then + handle(format("%s %s=0x%04X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) + end else - sort(srt) + if hexify then + handle(format("%s [%q]=0x%04X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end end - return srt - else - return { } - end -end - -local function sortedhashkeys(tab) -- fast one - if tab then - local srt, s = { }, 0 - for key,_ in next, tab do - if key then - s= s + 1 - srt[s] = key + elseif t=="string" then + if reduce and tonumber(v) then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) end - end - sort(srt) - return srt - else - return { } - end -end - -function table.allkeys(t) - local keys = { } - for i=1,#t do - for k, v in next, t[i] do - keys[k] = true - end - end - return sortedkeys(keys) -end - -table.sortedkeys = sortedkeys -table.sortedhashkeys = sortedhashkeys - -local function nothing() end - -local function sortedhash(t) - if t then - local n, s = 0, sortedkeys(t) -- the robust one - local function kv(s) - n = n + 1 - local k = s[n] - return k, t[k] - end - return kv, s - else - return nothing - end -end - -table.sortedhash = sortedhash -table.sortedpairs = sortedhash -- obsolete - -function table.append(t,list) - local n = #t - for i=1,#list do - n = n + 1 - t[n] = list[i] - end - return t -end - -function table.prepend(t, list) - local nl = #list - local nt = nl + #t - for i=#t,1,-1 do - t[nt] = t[i] - nt = nt - 1 - end - for i=1,#list do - t[i] = list[i] - end - return t -end - --- function table.merge(t, ...) -- first one is target --- t = t or { } --- local lst = { ... } --- for i=1,#lst do --- for k, v in next, lst[i] do --- t[k] = v --- end --- end --- return t --- end - -function table.merge(t, ...) -- first one is target - t = t or { } - for i=1,select("#",...) do - for k, v in next, (select(i,...)) do - t[k] = v - end - end - return t -end - --- function table.merged(...) --- local tmp, lst = { }, { ... } --- for i=1,#lst do --- for k, v in next, lst[i] do --- tmp[k] = v --- end --- end --- return tmp --- end - -function table.merged(...) - local t = { } - for i=1,select("#",...) do - for k, v in next, (select(i,...)) do - t[k] = v - end - end - return t -end - --- function table.imerge(t, ...) --- local lst, nt = { ... }, #t --- for i=1,#lst do --- local nst = lst[i] --- for j=1,#nst do --- nt = nt + 1 --- t[nt] = nst[j] --- end --- end --- return t --- end - -function table.imerge(t, ...) - local nt = #t - for i=1,select("#",...) do - local nst = select(i,...) - for j=1,#nst do - nt = nt + 1 - t[nt] = nst[j] - end - end - return t -end - --- function table.imerged(...) --- local tmp, ntmp, lst = { }, 0, {...} --- for i=1,#lst do --- local nst = lst[i] --- for j=1,#nst do --- ntmp = ntmp + 1 --- tmp[ntmp] = nst[j] --- end --- end --- return tmp --- end - -function table.imerged(...) - local tmp, ntmp = { }, 0 - for i=1,select("#",...) do - local nst = select(i,...) - for j=1,#nst do - ntmp = ntmp + 1 - tmp[ntmp] = nst[j] - end - end - return tmp -end - -local function fastcopy(old,metatabletoo) -- fast one - if old then - local new = { } - for k, v in next, old do - if type(v) == "table" then - new[k] = fastcopy(v,metatabletoo) -- was just table.copy + elseif tk=="boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%s,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end + else + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,v)) else - new[k] = v + handle(format("%s [%s]=%q,",depth,k,v)) end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,tostring(k),v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end end - if metatabletoo then - -- optional second arg - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) + elseif t=="table" then + if not next(v) then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]={},",depth,k)) + else + handle(format("%s [%s]={},",depth,k)) end - end - return new - else - return { } - end -end - --- todo : copy without metatable - -local function copy(t, tables) -- taken from lua wiki, slightly adapted - tables = tables or { } - local tcopy = {} - if not tables[t] then - tables[t] = tcopy - end - for i,v in next, t do -- brrr, what happens with sparse indexed - if type(i) == "table" then - if tables[i] then - i = tables[i] + elseif tk=="boolean" then + handle(format("%s [%s]={},",depth,tostring(k))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={},",depth,k)) + else + handle(format("%s [%q]={},",depth,k)) + end + elseif inline then + local st=simple_table(v) + if st then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif tk=="boolean" then + handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else - i = copy(i, tables) + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) end - end - if type(v) ~= "table" then - tcopy[i] = v - elseif tables[v] then - tcopy[i] = tables[v] + else + do_serialize(v,k,depth,level+1) + end else - tcopy[i] = copy(v, tables) - end - end - local mt = getmetatable(t) - if mt then - setmetatable(tcopy,mt) - end - return tcopy -end - -table.fastcopy = fastcopy -table.copy = copy - -function table.derive(parent) -- for the moment not public - local child = { } - if parent then - setmetatable(child,{ __index = parent }) - end - return child -end - -function table.tohash(t,value) - local h = { } - if t then - if value == nil then value = true end - for _, v in next, t do -- no ipairs here - h[v] = value - end - end - return h -end - -function table.fromhash(t) - local hsh, h = { }, 0 - for k, v in next, t do -- no ipairs here - if v then - h = h + 1 - hsh[h] = k - end - end - return hsh -end - -local noquotes, hexify, handle, reduce, compact, inline, functions - -local reserved = table.tohash { -- intercept a language inconvenience: no reserved words as key - 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', - 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', -} - -local function simple_table(t) - if #t > 0 then - local n = 0 - for _,v in next, t do - n = n + 1 - end - if n == #t then - local tt, nt = { }, 0 - for i=1,#t do - local v = t[i] - local tv = type(v) - if tv == "number" then - nt = nt + 1 - if hexify then - tt[nt] = format("0x%04X",v) - else - tt[nt] = tostring(v) -- tostring not needed - end - elseif tv == "boolean" then - nt = nt + 1 - tt[nt] = tostring(v) - elseif tv == "string" then - nt = nt + 1 - tt[nt] = format("%q",v) - else - tt = nil - break - end - end - return tt + do_serialize(v,k,depth,level+1) end - end - return nil -end - --- Because this is a core function of mkiv I moved some function calls --- inline. --- --- twice as fast in a test: --- --- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) - --- problem: there no good number_to_string converter with the best resolution - --- probably using .. is faster than format --- maybe split in a few cases (yes/no hexify) - --- todo: %g faster on numbers than %s - -local propername = patterns.propername -- was find(name,"^%a[%w%_]*$") - -local function dummy() end - -local function do_serialize(root,name,depth,level,indexed) - if level > 0 then - depth = depth .. " " - if indexed then - handle(format("%s{",depth)) + elseif t=="boolean" then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%s,",depth,k,tostring(v))) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%s,",depth,k,tostring(v))) else - local tn = type(name) - if tn == "number" then - if hexify then - handle(format("%s[0x%04X]={",depth,name)) - else - handle(format("%s[%s]={",depth,name)) - end - elseif tn == "string" then - if noquotes and not reserved[name] and lpegmatch(propername,name) then - handle(format("%s%s={",depth,name)) - else - handle(format("%s[%q]={",depth,name)) - end - elseif tn == "boolean" then - handle(format("%s[%s]={",depth,tostring(name))) - else - handle(format("%s{",depth)) - end - end - end - -- we could check for k (index) being number (cardinal) - if root and next(root) then - -- local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) - -- if compact then - -- -- NOT: for k=1,#root do (we need to quit at nil) - -- for k,v in ipairs(root) do -- can we use next? - -- if not first then first = k end - -- last = last + 1 - -- end - -- end - local first, last = nil, 0 - if compact then - last = #root - for k=1,last do - if root[k] == nil then - last = k - 1 - break - end - end - if last > 0 then - first = 1 - end + handle(format("%s [%q]=%s,",depth,k,tostring(v))) end - local sk = sortedkeys(root) - for i=1,#sk do - local k = sk[i] - local v = root[k] - -- circular - local t, tk = type(v), type(k) - if compact and first and tk == "number" and k >= first and k <= last then - if t == "number" then - if hexify then - handle(format("%s 0x%04X,",depth,v)) - else - handle(format("%s %s,",depth,v)) -- %.99g - end - elseif t == "string" then - if reduce and tonumber(v) then - handle(format("%s %s,",depth,v)) - else - handle(format("%s %q,",depth,v)) - end - elseif t == "table" then - if not next(v) then - handle(format("%s {},",depth)) - elseif inline then -- and #t > 0 - local st = simple_table(v) - if st then - handle(format("%s { %s },",depth,concat(st,", "))) - else - do_serialize(v,k,depth,level+1,true) - end - else - do_serialize(v,k,depth,level+1,true) - end - elseif t == "boolean" then - handle(format("%s %s,",depth,tostring(v))) - elseif t == "function" then - if functions then - handle(format('%s load(%q),',depth,dump(v))) - else - handle(format('%s "function",',depth)) - end - else - handle(format("%s %q,",depth,tostring(v))) - end - elseif k == "__p__" then -- parent - if false then - handle(format("%s __p__=nil,",depth)) - end - elseif t == "number" then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g - end - elseif tk == "boolean" then - if hexify then - handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) - else - handle(format("%s [%s]=%s,",depth,tostring(k),v)) -- %.99g - end - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - if hexify then - handle(format("%s %s=0x%04X,",depth,k,v)) - else - handle(format("%s %s=%s,",depth,k,v)) -- %.99g - end - else - if hexify then - handle(format("%s [%q]=0x%04X,",depth,k,v)) - else - handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g - end - end - elseif t == "string" then - if reduce and tonumber(v) then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%s,",depth,tostring(k),v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%s,",depth,k,v)) - else - handle(format("%s [%q]=%s,",depth,k,v)) - end - else - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,v)) - else - handle(format("%s [%s]=%q,",depth,k,v)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%q,",depth,tostring(k),v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%q,",depth,k,v)) - else - handle(format("%s [%q]=%q,",depth,k,v)) - end - end - elseif t == "table" then - if not next(v) then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]={},",depth,k)) - else - handle(format("%s [%s]={},",depth,k)) - end - elseif tk == "boolean" then - handle(format("%s [%s]={},",depth,tostring(k))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s={},",depth,k)) - else - handle(format("%s [%q]={},",depth,k)) - end - elseif inline then - local st = simple_table(v) - if st then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) - else - handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) - end - elseif tk == "boolean" then - handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s={ %s },",depth,k,concat(st,", "))) - else - handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) - end - else - do_serialize(v,k,depth,level+1) - end - else - do_serialize(v,k,depth,level+1) - end - elseif t == "boolean" then - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) - else - handle(format("%s [%s]=%s,",depth,k,tostring(v))) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%s,",depth,k,tostring(v))) - else - handle(format("%s [%q]=%s,",depth,k,tostring(v))) - end - elseif t == "function" then - if functions then - local f = getinfo(v).what == "C" and dump(dummy) or dump(v) - -- local f = getinfo(v).what == "C" and dump(function(...) return v(...) end) or dump(v) - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=load(%q),",depth,k,f)) - else - handle(format("%s [%s]=load(%q),",depth,k,f)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=load(%q),",depth,tostring(k),f)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=load(%q),",depth,k,f)) - else - handle(format("%s [%q]=load(%q),",depth,k,f)) - end - end + elseif t=="function" then + if functions then + local f=getinfo(v).what=="C" and dump(dummy) or dump(v) + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=load(%q),",depth,k,f)) else - if tk == "number" then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) - else - handle(format("%s [%s]=%q,",depth,k,tostring(v))) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%q,",depth,k,tostring(v))) - else - handle(format("%s [%q]=%q,",depth,k,tostring(v))) - end + handle(format("%s [%s]=load(%q),",depth,k,f)) end - end - end - if level > 0 then - handle(format("%s},",depth)) - end -end - --- replacing handle by a direct t[#t+1] = ... (plus test) is not much --- faster (0.03 on 1.00 for zapfino.tma) - -local function serialize(_handle,root,name,specification) -- handle wins - local tname = type(name) - if type(specification) == "table" then - noquotes = specification.noquotes - hexify = specification.hexify - handle = _handle or specification.handle or print - reduce = specification.reduce or false - functions = specification.functions - compact = specification.compact - inline = specification.inline and compact - if functions == nil then - functions = true - end - if compact == nil then - compact = true - end - if inline == nil then - inline = compact - end - else - noquotes = false - hexify = false - handle = _handle or print - reduce = false - compact = true - inline = true - functions = true - end - if tname == "string" then - if name == "return" then - handle("return {") - else - handle(name .. "={") - end - elseif tname == "number" then - if hexify then - handle(format("[0x%04X]={",name)) - else - handle("[" .. name .. "]={") + elseif tk=="boolean" then + handle(format("%s [%s]=load(%q),",depth,tostring(k),f)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=load(%q),",depth,k,f)) + else + handle(format("%s [%q]=load(%q),",depth,k,f)) + end end - elseif tname == "boolean" then - if name then - handle("return {") + else + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%q,",depth,k,tostring(v))) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,tostring(v))) else - handle("{") + handle(format("%s [%q]=%q,",depth,k,tostring(v))) end + end + end + end + if level>0 then + handle(format("%s},",depth)) + end +end +local function serialize(_handle,root,name,specification) + local tname=type(name) + if type(specification)=="table" then + noquotes=specification.noquotes + hexify=specification.hexify + handle=_handle or specification.handle or print + reduce=specification.reduce or false + functions=specification.functions + compact=specification.compact + inline=specification.inline and compact + if functions==nil then + functions=true + end + if compact==nil then + compact=true + end + if inline==nil then + inline=compact + end + else + noquotes=false + hexify=false + handle=_handle or print + reduce=false + compact=true + inline=true + functions=true + end + if tname=="string" then + if name=="return" then + handle("return {") + else + handle(name.."={") + end + elseif tname=="number" then + if hexify then + handle(format("[0x%04X]={",name)) + else + handle("["..name.."]={") + end + elseif tname=="boolean" then + if name then + handle("return {") else - handle("t={") - end - if root then - -- The dummy access will initialize a table that has a delayed initialization - -- using a metatable. (maybe explicitly test for metatable) - if getmetatable(root) then -- todo: make this an option, maybe even per subtable - local dummy = root._w_h_a_t_e_v_e_r_ - root._w_h_a_t_e_v_e_r_ = nil - end - -- Let's forget about empty tables. - if next(root) then - do_serialize(root,name,"",0) - end + handle("{") end - handle("}") -end - --- name: --- --- true : return { } --- false : { } --- nil : t = { } --- string : string = { } --- "return" : return { } --- number : [number] = { } - -function table.serialize(root,name,specification) - local t, n = { }, 0 - local function flush(s) - n = n + 1 - t[n] = s + else + handle("t={") + end + if root then + if getmetatable(root) then + local dummy=root._w_h_a_t_e_v_e_r_ + root._w_h_a_t_e_v_e_r_=nil end - serialize(flush,root,name,specification) - return concat(t,"\n") -end - -table.tohandle = serialize - --- sometimes tables are real use (zapfino extra pro is some 85M) in which --- case a stepwise serialization is nice; actually, we could consider: --- --- for line in table.serializer(root,name,reduce,noquotes) do --- ...(line) --- end --- --- so this is on the todo list - -local maxtab = 2*1024 - -function table.tofile(filename,root,name,specification) - local f = io.open(filename,'w') - if f then - if maxtab > 1 then - local t, n = { }, 0 - local function flush(s) - n = n + 1 - t[n] = s - if n > maxtab then - f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice - t, n = { }, 0 -- we could recycle t if needed - end - end - serialize(flush,root,name,specification) - f:write(concat(t,"\n"),"\n") - else - local function flush(s) - f:write(s,"\n") - end - serialize(flush,root,name,specification) - end - f:close() - io.flush() + if next(root) then + do_serialize(root,name,"",0) end + end + handle("}") end - -local function flattened(t,f,depth) - if f == nil then - f = { } - depth = 0xFFFF - elseif tonumber(f) then - -- assume that only two arguments are given - depth = f - f = { } - elseif not depth then - depth = 0xFFFF - end - for k, v in next, t do - if type(k) ~= "number" then - if depth > 0 and type(v) == "table" then - flattened(v,f,depth-1) - else - f[k] = v - end - end - end - local n = #f - for k=1,#t do - local v = t[k] - if depth > 0 and type(v) == "table" then - flattened(v,f,depth-1) - n = #f - else - n = n + 1 - f[n] = v - end - end - return f -end - -table.flattened = flattened - -local function unnest(t,f) -- only used in mk, for old times sake - if not f then -- and only relevant for token lists - f = { } -- this one can become obsolete - end - for i=1,#t do - local v = t[i] - if type(v) == "table" then - if type(v[1]) == "table" then - unnest(v,f) - else - f[#f+1] = v - end - else - f[#f+1] = v - end - end - return f -end - -function table.unnest(t) -- bad name - return unnest(t) -end - -local function are_equal(a,b,n,m) -- indexed - if a and b and #a == #b then - n = n or 1 - m = m or #a - for i=n,m do - local ai, bi = a[i], b[i] - if ai==bi then - -- same - elseif type(ai) == "table" and type(bi) == "table" then - if not are_equal(ai,bi) then - return false - end - else - return false - end - end - return true - else - return false - end -end - -local function identical(a,b) -- assumes same structure - for ka, va in next, a do - local vb = b[ka] - if va == vb then - -- same - elseif type(va) == "table" and type(vb) == "table" then - if not identical(va,vb) then - return false - end - else - return false - end - end - return true -end - -table.identical = identical -table.are_equal = are_equal - --- maybe also make a combined one - -function table.compact(t) -- remove empty tables, assumes subtables - if t then - for k, v in next, t do - if not next(v) then -- no type checking - t[k] = nil - end - end - end -end - -function table.contains(t, v) - if t then - for i=1, #t do - if t[i] == v then - return i - end - end - end - return false -end - -function table.count(t) - local n = 0 - for k, v in next, t do - n = n + 1 - end - return n -end - -function table.swapped(t,s) -- hash - local n = { } - if s then - for k, v in next, s do - n[k] = v - end - end - for k, v in next, t do - n[v] = k - end - return n -end - -function table.mirrored(t) -- hash - local n = { } - for k, v in next, t do - n[v] = k - n[k] = v - end - return n -end - -function table.reversed(t) - if t then - local tt, tn = { }, #t - if tn > 0 then - local ttn = 0 - for i=tn,1,-1 do - ttn = ttn + 1 - tt[ttn] = t[i] - end - end - return tt - end -end - -function table.reverse(t) - if t then - local n = #t - for i=1,floor(n/2) do - local j = n - i + 1 - t[i], t[j] = t[j], t[i] - end - return t - end -end - -function table.sequenced(t,sep) -- hash only - if t then - local s, n = { }, 0 - for k, v in sortedhash(t) do - if simple then - if v == true then - n = n + 1 - s[n] = k - elseif v and v~= "" then - n = n + 1 - s[n] = k .. "=" .. tostring(v) - end - else - n = n + 1 - s[n] = k .. "=" .. tostring(v) - end - end - return concat(s, sep or " | ") - else - return "" - end -end - -function table.print(t,...) - if type(t) ~= "table" then - print(tostring(t)) - else - table.tohandle(print,t,...) - end -end - --- -- -- obsolete but we keep them for a while and might comment them later -- -- -- - --- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) - -function table.sub(t,i,j) - return { unpack(t,i,j) } -end - --- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) - -function table.is_empty(t) - return not t or not next(t) -end - -function table.has_one_entry(t) - return t and not next(t,next(t)) -end - --- new - -function table.loweredkeys(t) -- maybe utf - local l = { } - for k, v in next, t do - l[lower(k)] = v - end - return l -end - --- new, might move (maybe duplicate) - -function table.unique(old) - local hash = { } - local new = { } - local n = 0 - for i=1,#old do - local oi = old[i] - if not hash[oi] then - n = n + 1 - new[n] = oi - hash[oi] = true - end - end - return new -end - -function table.sorted(t,...) - sort(t,...) - return t -- still sorts in-place -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-io'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local io = io -local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format -local concat = table.concat -local floor = math.floor -local type = type - -if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator = "\\", ";" -else - io.fileseparator, io.pathseparator = "/" , ":" -end - -local function readall(f) - return f:read("*all") -end - --- The next one is upto 50% faster on large files and less memory consumption due --- to less intermediate large allocations. This phenomena was discussed on the --- luatex dev list. - -local function readall(f) - local size = f:seek("end") - if size == 0 then - return "" - elseif size < 1024*1024 then - f:seek("set",0) - return f:read('*all') - else - local done = f:seek("set",0) - if size < 1024*1024 then - step = 1024 * 1024 - elseif size > 16*1024*1024 then - step = 16*1024*1024 - else - step = floor(size/(1024*1024)) * 1024 * 1024 / 8 - end - local data = { } - while true do - local r = f:read(step) - if not r then - return concat(data) - else - data[#data+1] = r - end - end - end -end - -io.readall = readall - -function io.loaddata(filename,textmode) -- return nil if empty - local f = io.open(filename,(textmode and 'r') or 'rb') - if f then --- local data = f:read('*all') - local data = readall(f) - f:close() - if #data > 0 then - return data - end - end -end - -function io.savedata(filename,data,joiner) - local f = io.open(filename,"wb") - if f then - if type(data) == "table" then - f:write(concat(data,joiner or "")) - elseif type(data) == "function" then - data(f) - else - f:write(data or "") - end - f:close() - io.flush() - return true - else - return false - end -end - -function io.loadlines(filename,n) -- return nil if empty - local f = io.open(filename,'r') - if not f then - -- no file - elseif n then - local lines = { } - for i=1,n do - local line = f:read("*lines") - if line then - lines[#lines+1] = line - else - break - end - end - f:close() - lines = concat(lines,"\n") - if #lines > 0 then - return lines - end - else - local line = f:read("*line") or "" - f:close() - if #line > 0 then - return line - end - end -end - -function io.loadchunk(filename,n) - local f = io.open(filename,'rb') - if f then - local data = f:read(n or 1024) - f:close() - if #data > 0 then - return data - end - end -end - -function io.exists(filename) - local f = io.open(filename) - if f == nil then - return false - else - f:close() - return true - end -end - -function io.size(filename) - local f = io.open(filename) - if f == nil then - return 0 - else - local s = f:seek("end") - f:close() - return s - end -end - -function io.noflines(f) - if type(f) == "string" then - local f = io.open(filename) - if f then - local n = f and io.noflines(f) or 0 - f:close() - return n - else - return 0 - end - else - local n = 0 - for _ in f:lines() do - n = n + 1 - end - f:seek('set',0) - return n - end -end - -local nextchar = { - [ 4] = function(f) - return f:read(1,1,1,1) - end, - [ 2] = function(f) - return f:read(1,1) - end, - [ 1] = function(f) - return f:read(1) - end, - [-2] = function(f) - local a, b = f:read(1,1) - return b, a - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - return d, c, b, a - end -} - -function io.characters(f,n) - if f then - return nextchar[n or 1], f - end -end - -local nextbyte = { - [4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return byte(a), byte(b), byte(c), byte(d) - end - end, - [3] = function(f) - local a, b, c = f:read(1,1,1) - if b then - return byte(a), byte(b), byte(c) - end - end, - [2] = function(f) - local a, b = f:read(1,1) - if b then - return byte(a), byte(b) - end - end, - [1] = function (f) - local a = f:read(1) - if a then - return byte(a) - end - end, - [-2] = function (f) - local a, b = f:read(1,1) - if b then - return byte(b), byte(a) - end - end, - [-3] = function(f) - local a, b, c = f:read(1,1,1) - if b then - return byte(c), byte(b), byte(a) - end - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return byte(d), byte(c), byte(b), byte(a) - end - end -} - -function io.bytes(f,n) - if f then - return nextbyte[n or 1], f - else - return nil, nil - end -end - -function io.ask(question,default,options) - while true do - io.write(question) - if options then - io.write(format(" [%s]",concat(options,"|"))) - end - if default then - io.write(format(" [%s]",default)) - end - io.write(format(" ")) - io.flush() - local answer = io.read() - answer = gsub(answer,"^%s*(.*)%s*$","%1") - if answer == "" and default then - return default - elseif not options then - return answer - else - for k=1,#options do - if options[k] == answer then - return answer - end - end - local pattern = "^" .. answer - for k=1,#options do - local v = options[k] - if find(v,pattern) then - return v - end - end - end - end -end - -local function readnumber(f,n,m) - if m then - f:seek("set",n) - n = m - end - if n == 1 then - return byte(f:read(1)) - elseif n == 2 then - local a, b = byte(f:read(2),1,2) - return 256 * a + b - elseif n == 3 then - local a, b, c = byte(f:read(3),1,3) - return 256*256 * a + 256 * b + c - elseif n == 4 then - local a, b, c, d = byte(f:read(4),1,4) - return 256*256*256 * a + 256*256 * b + 256 * c + d - elseif n == 8 then - local a, b = readnumber(f,4), readnumber(f,4) - return 256 * a + b - elseif n == 12 then - local a, b, c = readnumber(f,4), readnumber(f,4), readnumber(f,4) - return 256*256 * a + 256 * b + c - elseif n == -2 then - local b, a = byte(f:read(2),1,2) - return 256*a + b - elseif n == -3 then - local c, b, a = byte(f:read(3),1,3) - return 256*256 * a + 256 * b + c - elseif n == -4 then - local d, c, b, a = byte(f:read(4),1,4) - return 256*256*256 * a + 256*256 * b + 256*c + d - elseif n == -8 then - local h, g, f, e, d, c, b, a = byte(f:read(8),1,8) - return 256*256*256*256*256*256*256 * a + - 256*256*256*256*256*256 * b + - 256*256*256*256*256 * c + - 256*256*256*256 * d + - 256*256*256 * e + - 256*256 * f + - 256 * g + - h - else - return 0 - end -end - -io.readnumber = readnumber - -function io.readstring(f,n,m) - if m then - f:seek("set",n) - n = m - end - local str = gsub(f:read(n),"\000","") - return str -end - --- - -if not io.i_limiter then function io.i_limiter() end end -- dummy so we can test safely -if not io.o_limiter then function io.o_limiter() end end -- dummy so we can test safely - --- This works quite ok: --- --- function io.piped(command,writer) --- local pipe = io.popen(command) --- -- for line in pipe:lines() do --- -- print(line) --- -- end --- while true do --- local line = pipe:read(1) --- if not line then --- break --- elseif line ~= "\n" then --- writer(line) --- end --- end --- return pipe:close() -- ok, status, (error)code --- end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-number'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module will be replaced when we have the bit library - -local tostring, tonumber = tostring, tonumber -local format, floor, match, rep = string.format, math.floor, string.match, string.rep -local concat, insert = table.concat, table.insert -local lpegmatch = lpeg.match - -number = number or { } -local number = number - -if bit32 then - - local btest, bor = bit32.btest, bit32.bor - - function number.bit(p) - return 2 ^ (p - 1) -- 1-based indexing - end - - number.hasbit = btest - number.setbit = bor - - function number.setbit(x,p) - return btest(x,p) and x or x + p - end - - function number.clearbit(x,p) - return btest(x,p) and x - p or x - end - -else - - -- http://ricilake.blogspot.com/2007/10/iterating-bits-in-lua.html - - function number.bit(p) - return 2 ^ (p - 1) -- 1-based indexing - end - - function number.hasbit(x, p) -- typical call: if hasbit(x, bit(3)) then ... - return x % (p + p) >= p - end - - function number.setbit(x, p) - return (x % (p + p) >= p) and x or x + p - end - - function number.clearbit(x, p) - return (x % (p + p) >= p) and x - p or x - end - -end - --- print(number.tobitstring(8)) --- print(number.tobitstring(14)) --- print(number.tobitstring(66)) --- print(number.tobitstring(0x00)) --- print(number.tobitstring(0xFF)) --- print(number.tobitstring(46260767936,4)) - -if bit32 then - - local bextract = bit32.extract - - local t = { - "0", "0", "0", "0", "0", "0", "0", "0", - "0", "0", "0", "0", "0", "0", "0", "0", - "0", "0", "0", "0", "0", "0", "0", "0", - "0", "0", "0", "0", "0", "0", "0", "0", - } - - function number.tobitstring(b,m) - -- if really needed we can speed this one up - -- because small numbers need less extraction - local n = 32 - for i=0,31 do - local v = bextract(b,i) - local k = 32 - i - if v == 1 then - n = k - t[k] = "1" - else - t[k] = "0" - end - end - if m then - m = 33 - m * 8 - if m < 1 then - m = 1 - end - return concat(t,"",m) - elseif n < 8 then - return concat(t) - elseif n < 16 then - return concat(t,"",9) - elseif n < 24 then - return concat(t,"",17) - else - return concat(t,"",25) - end - end - -else - - function number.tobitstring(n,m) - if n > 0 then - local t = { } - while n > 0 do - insert(t,1,n % 2 > 0 and 1 or 0) - n = floor(n/2) - end - local nn = 8 - #t % 8 - if nn > 0 and nn < 8 then - for i=1,nn do - insert(t,1,0) - end - end - if m then - m = m * 8 - #t - if m > 0 then - insert(t,1,rep("0",m)) - end - end - return concat(t) - elseif m then - rep("00000000",m) - else - return "00000000" - end - end - -end - -function number.valid(str,default) - return tonumber(str) or default or nil -end - -function number.toevenhex(n) - local s = format("%X",n) - if #s % 2 == 0 then - return s - else - return "0" .. s - end -end - --- a,b,c,d,e,f = number.toset(100101) --- --- function number.toset(n) --- return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") --- end --- --- -- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5% --- -- on --- --- for i=1,1000000 do --- local a,b,c,d,e,f,g,h = number.toset(12345678) --- local a,b,c,d = number.toset(1234) --- local a,b,c = number.toset(123) --- local a,b,c = number.toset("123") --- end - -local one = lpeg.C(1-lpeg.S('')/tonumber)^1 - -function number.toset(n) - return lpegmatch(one,tostring(n)) -end - --- function number.bits(n,zero) --- local t, i = { }, (zero and 0) or 1 --- while n > 0 do --- local m = n % 2 --- if m > 0 then --- insert(t,1,i) --- end --- n = floor(n/2) --- i = i + 1 --- end --- return t --- end --- --- -- a bit faster - -local function bits(n,i,...) - if n > 0 then - local m = n % 2 - local n = floor(n/2) - if m > 0 then - return bits(n, i+1, i, ...) - else - return bits(n, i+1, ...) - end - else - return ... - end -end - -function number.bits(n) - return { bits(n,1) } -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-set'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This will become obsolete when we have the bitset library embedded. - -set = set or { } - -local nums = { } -local tabs = { } -local concat = table.concat -local next, type = next, type - -set.create = table.tohash - -function set.tonumber(t) - if next(t) then - local s = "" - -- we could save mem by sorting, but it slows down - for k, v in next, t do - if v then - -- why bother about the leading space - s = s .. " " .. k - end - end - local n = nums[s] - if not n then - n = #tabs + 1 - tabs[n] = t - nums[s] = n - end - return n - else - return 0 - end -end - -function set.totable(n) - if n == 0 then - return { } - else - return tabs[n] or { } - end -end - -function set.tolist(n) - if n == 0 or not tabs[n] then - return "" - else - local t, n = { }, 0 - for k, v in next, tabs[n] do - if v then - n = n + 1 - t[n] = k - end - end - return concat(t," ") - end -end - -function set.contains(n,s) - if type(n) == "table" then - return n[s] - elseif n == 0 then - return false - else - local t = tabs[n] - return t and t[s] - end -end - - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-os'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This file deals with some operating system issues. Please don't bother me --- with the pros and cons of operating systems as they all have their flaws --- and benefits. Bashing one of them won't help solving problems and fixing --- bugs faster and is a waste of time and energy. --- --- path separators: / or \ ... we can use / everywhere --- suffixes : dll so exe ... no big deal --- quotes : we can use "" in most cases --- expansion : unless "" are used * might give side effects --- piping/threads : somewhat different for each os --- locations : specific user file locations and settings can change over time --- --- os.type : windows | unix (new, we already guessed os.platform) --- os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) --- os.platform : extended os.name with architecture - --- os.sleep() => socket.sleep() --- math.randomseed(tonumber(string.sub(string.reverse(tostring(math.floor(socket.gettime()*10000))),1,6))) - --- maybe build io.flush in os.execute - -local os = os -local date, time = os.date, os.time -local find, format, gsub, upper, gmatch = string.find, string.format, string.gsub, string.upper, string.gmatch -local concat = table.concat -local random, ceil, randomseed = math.random, math.ceil, math.randomseed -local rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring = rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring - --- The following code permits traversing the environment table, at least --- in luatex. Internally all environment names are uppercase. - --- The randomseed in Lua is not that random, although this depends on the operating system as well --- as the binary (Luatex is normally okay). But to be sure we set the seed anyway. - -math.initialseed = tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6)) - -randomseed(math.initialseed) - -if not os.__getenv__ then - - os.__getenv__ = os.getenv - os.__setenv__ = os.setenv - - if os.env then - - local osgetenv = os.getenv - local ossetenv = os.setenv - local osenv = os.env local _ = osenv.PATH -- initialize the table - - function os.setenv(k,v) - if v == nil then - v = "" - end - local K = upper(k) - osenv[K] = v - if type(v) == "table" then - v = concat(v,";") -- path - end - ossetenv(K,v) - end - - function os.getenv(k) - local K = upper(k) - local v = osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) - if v == "" then - return nil - else - return v - end - end - - else - - local ossetenv = os.setenv - local osgetenv = os.getenv - local osenv = { } - - function os.setenv(k,v) - if v == nil then - v = "" - end - local K = upper(k) - osenv[K] = v - end - - function os.getenv(k) - local K = upper(k) - local v = osenv[K] or osgetenv(K) or osgetenv(k) - if v == "" then - return nil - else - return v - end - end - - local function __index(t,k) - return os.getenv(k) - end - local function __newindex(t,k,v) - os.setenv(k,v) - end - - os.env = { } - - setmetatable(os.env, { __index = __index, __newindex = __newindex } ) - - end - -end - --- end of environment hack - -local execute, spawn, exec, iopopen, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.popen, io.flush - -function os.execute(...) ioflush() return execute(...) end -function os.spawn (...) ioflush() return spawn (...) end -function os.exec (...) ioflush() return exec (...) end -function io.popen (...) ioflush() return iopopen(...) end - -function os.resultof(command) - local handle = io.popen(command,"r") - return handle and handle:read("*all") or "" -end - -if not io.fileseparator then - if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" - else - io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" - end -end - -os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" -os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" - -if os.type == "windows" then - os.libsuffix, os.binsuffix, os.binsuffixes = 'dll', 'exe', { 'exe', 'cmd', 'bat' } -else - os.libsuffix, os.binsuffix, os.binsuffixes = 'so', '', { '' } -end - -local launchers = { - windows = "start %s", - macosx = "open %s", - unix = "$BROWSER %s &> /dev/null &", -} - -function os.launch(str) - os.execute(format(launchers[os.name] or launchers.unix,str)) -end - -if not os.times then - -- utime = user time - -- stime = system time - -- cutime = children user time - -- cstime = children system time - function os.times() - return { - utime = os.gettimeofday(), -- user - stime = 0, -- system - cutime = 0, -- children user - cstime = 0, -- children system - } - end -end - -os.gettimeofday = os.gettimeofday or os.clock - -local startuptime = os.gettimeofday() - -function os.runtime() - return os.gettimeofday() - startuptime -end - - --- no need for function anymore as we have more clever code and helpers now --- this metatable trickery might as well disappear - -os.resolvers = os.resolvers or { } -- will become private - -local resolvers = os.resolvers - -local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil -local osix = osmt.__index - -osmt.__index = function(t,k) - return (resolvers[k] or osix)(t,k) -end - -setmetatable(os,osmt) - --- we can use HOSTTYPE on some platforms - -local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" - -local function guess() - local architecture = os.resultof("uname -m") or "" - if architecture ~= "" then - return architecture - end - architecture = os.getenv("HOSTTYPE") or "" - if architecture ~= "" then - return architecture - end - return os.resultof("echo $HOSTTYPE") or "" -end - -if platform ~= "" then - - os.platform = platform - -elseif os.type == "windows" then - - -- we could set the variable directly, no function needed here - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" - if find(architecture,"AMD64") then - platform = "mswin-64" - else - platform = "mswin" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "linux" then - - function os.resolvers.platform(t,k) - -- we sometimes have HOSTTYPE set so let's check that first - local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "macosx" then - - --[[ - Identifying the architecture of OSX is quite a mess and this - is the best we can come up with. For some reason $HOSTTYPE is - a kind of pseudo environment variable, not known to the current - environment. And yes, uname cannot be trusted either, so there - is a change that you end up with a 32 bit run on a 64 bit system. - Also, some proper 64 bit intel macs are too cheap (low-end) and - therefore not permitted to run the 64 bit kernel. - ]]-- - - function os.resolvers.platform(t,k) - -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" - -- if architecture == "" then - -- architecture = os.resultof("echo $HOSTTYPE") or "" - -- end - local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" - if architecture == "" then - -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") - platform = "osx-intel" - elseif find(architecture,"i386") then - platform = "osx-intel" - elseif find(architecture,"x86_64") then - platform = "osx-64" - else - platform = "osx-ppc" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "sunos" then - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.resultof("uname -m") or "" - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "freebsd" then - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.resultof("uname -m") or "" - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "kfreebsd" then - - function os.resolvers.platform(t,k) - -- we sometimes have HOSTTYPE set so let's check that first - local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then - platform = "kfreebsd-amd64" - else - platform = "kfreebsd-i386" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -else - - -- platform = "linux" - -- os.setenv("MTX_PLATFORM",platform) - -- os.platform = platform - - function os.resolvers.platform(t,k) - local platform = "linux" - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -end - --- beware, we set the randomseed - --- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the --- version number as well as two reserved bits. All other bits are set using a random or pseudorandom --- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal --- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. --- --- as we don't call this function too often there is not so much risk on repetition - -local t = { 8, 9, "a", "b" } - -function os.uuid() - return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", - random(0xFFFF),random(0xFFFF), - random(0x0FFF), - t[ceil(random(4))] or 8,random(0x0FFF), - random(0xFFFF), - random(0xFFFF),random(0xFFFF),random(0xFFFF) - ) -end - -local d - -function os.timezone(delta) - d = d or tonumber(tonumber(date("%H")-date("!%H"))) - if delta then - if d > 0 then - return format("+%02i:00",d) - else - return format("-%02i:00",-d) - end - else - return 1 - end -end - -local timeformat = format("%%s%s",os.timezone(true)) -local dateformat = "!%Y-%m-%d %H:%M:%S" - -function os.fulltime(t,default) - t = tonumber(t) or 0 - if t > 0 then - -- valid time - elseif default then - return default - else - t = nil - end - return format(timeformat,date(dateformat,t)) -end - -local dateformat = "%Y-%m-%d %H:%M:%S" - -function os.localtime(t,default) - t = tonumber(t) or 0 - if t > 0 then - -- valid time - elseif default then - return default - else - t = nil - end - return date(dateformat,t) -end - -function os.converttime(t,default) - local t = tonumber(t) - if t and t > 0 then - return date(dateformat,t) - else - return default or "-" - end -end - -local memory = { } - -local function which(filename) - local fullname = memory[filename] - if fullname == nil then - local suffix = file.suffix(filename) - local suffixes = suffix == "" and os.binsuffixes or { suffix } - for directory in gmatch(os.getenv("PATH"),"[^" .. io.pathseparator .."]+") do - local df = file.join(directory,filename) - for i=1,#suffixes do - local dfs = file.addsuffix(df,suffixes[i]) - if io.exists(dfs) then - fullname = dfs - break - end - end - end - if not fullname then - fullname = false - end - memory[filename] = fullname - end - return fullname -end - -os.which = which -os.where = which - -function os.today() - return date("!*t") -- table with values -end - -function os.now() - return date("!%Y-%m-%d %H:%M:%S") -- 2011-12-04 14:59:12 -end - - --- print(os.which("inkscape.exe")) --- print(os.which("inkscape")) --- print(os.which("gs.exe")) --- print(os.which("ps2pdf")) - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-file'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- needs a cleanup - -file = file or { } -local file = file - -local insert, concat = table.insert, table.concat -local match = string.match -local lpegmatch = lpeg.match -local getcurrentdir, attributes = lfs.currentdir, lfs.attributes -local checkedsplit = string.checkedsplit - --- local patterns = file.patterns or { } --- file.patterns = patterns - -local P, R, S, C, Cs, Cp, Cc, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpeg.Cp, lpeg.Cc, lpeg.Ct - -local colon = P(":") -local period = P(".") -local periods = P("..") -local fwslash = P("/") -local bwslash = P("\\") -local slashes = S("\\/") -local noperiod = 1-period -local noslashes = 1-slashes -local name = noperiod^1 -local suffix = period/"" * (1-period-slashes)^1 * -1 - -local pattern = C((noslashes^0 * slashes^1)^1) - -local function pathpart(name,default) - return name and lpegmatch(pattern,name) or default or "" -end - -local pattern = (noslashes^0 * slashes)^1 * C(noslashes^1) * -1 - -local function basename(name) - return name and lpegmatch(pattern,name) or name -end - -local pattern = (noslashes^0 * slashes^1)^0 * Cs((1-suffix)^1) * suffix^0 - -local function nameonly(name) - return name and lpegmatch(pattern,name) or name -end - -local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * C(noperiod^1) * -1 - -local function suffixonly(name) - return name and lpegmatch(pattern,name) or "" -end - -file.pathpart = pathpart -file.basename = basename -file.nameonly = nameonly -file.suffixonly = suffixonly -file.suffix = suffixonly - -file.dirname = pathpart -- obsolete -file.extname = suffixonly -- obsolete - --- actually these are schemes - -local drive = C(R("az","AZ")) * colon -local path = C(((1-slashes)^0 * slashes)^0) -local suffix = period * C(P(1-period)^0 * P(-1)) -local base = C((1-suffix)^0) -local rest = C(P(1)^0) - -drive = drive + Cc("") -path = path + Cc("") -base = base + Cc("") -suffix = suffix + Cc("") - -local pattern_a = drive * path * base * suffix -local pattern_b = path * base * suffix -local pattern_c = C(drive * path) * C(base * suffix) -- trick: two extra captures -local pattern_d = path * rest - -function file.splitname(str,splitdrive) - if not str then - -- error - elseif splitdrive then - return lpegmatch(pattern_a,str) -- returns drive, path, base, suffix - else - return lpegmatch(pattern_b,str) -- returns path, base, suffix - end -end - -function file.splitbase(str) - return str and lpegmatch(pattern_d,str) -- returns path, base+suffix -end - -function file.nametotable(str,splitdrive) -- returns table - if str then - local path, drive, subpath, name, base, suffix = lpegmatch(pattern_c,str) - if splitdrive then - return { - path = path, - drive = drive, - subpath = subpath, - name = name, - base = base, - suffix = suffix, - } - else - return { - path = path, - name = name, - base = base, - suffix = suffix, - } - end - end -end - -local pattern = Cs(((period * noperiod^1 * -1)/"" + 1)^1) - -function file.removesuffix(name) - return name and lpegmatch(pattern,name) -end - --- local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * Cp() * noperiod^1 * -1 --- --- function file.addsuffix(name, suffix) --- local p = lpegmatch(pattern,name) --- if p then --- return name --- else --- return name .. "." .. suffix --- end --- end - -local suffix = period/"" * (1-period-slashes)^1 * -1 -local pattern = Cs((noslashes^0 * slashes^1)^0 * ((1-suffix)^1)) * Cs(suffix) - -function file.addsuffix(filename,suffix,criterium) - if not filename or not suffix or suffix == "" then - return filename - elseif criterium == true then - return filename .. "." .. suffix - elseif not criterium then - local n, s = lpegmatch(pattern,filename) - if not s or s == "" then - return filename .. "." .. suffix - else - return filename - end - else - local n, s = lpegmatch(pattern,filename) - if s and s ~= "" then - local t = type(criterium) - if t == "table" then - -- keep if in criterium - for i=1,#criterium do - if s == criterium[i] then - return filename - end - end - elseif t == "string" then - -- keep if criterium - if s == criterium then - return filename - end - end - end - return (n or filename) .. "." .. suffix - end -end - --- print("1 " .. file.addsuffix("name","new") .. " -> name.new") --- print("2 " .. file.addsuffix("name.old","new") .. " -> name.old") --- print("3 " .. file.addsuffix("name.old","new",true) .. " -> name.old.new") --- print("4 " .. file.addsuffix("name.old","new","new") .. " -> name.new") --- print("5 " .. file.addsuffix("name.old","new","old") .. " -> name.old") --- print("6 " .. file.addsuffix("name.old","new","foo") .. " -> name.new") --- print("7 " .. file.addsuffix("name.old","new",{"foo","bar"}) .. " -> name.new") --- print("8 " .. file.addsuffix("name.old","new",{"old","bar"}) .. " -> name.old") - -local suffix = period * (1-period-slashes)^1 * -1 -local pattern = Cs((1-suffix)^0) - -function file.replacesuffix(name,suffix) - if name and suffix and suffix ~= "" then - return lpegmatch(pattern,name) .. "." .. suffix - else - return name - end -end - --- - -local reslasher = lpeg.replacer(P("\\"),"/") - -function file.reslash(str) - return str and lpegmatch(reslasher,str) -end - --- We should be able to use: --- --- local writable = P(1) * P("w") * Cc(true) --- --- function file.is_writable(name) --- local a = attributes(name) or attributes(pathpart(name,".")) --- return a and lpegmatch(writable,a.permissions) or false --- end --- --- But after some testing Taco and I came up with the more robust --- variant: - -function file.is_writable(name) - if not name then - -- error - elseif lfs.isdir(name) then - name = name .. "/m_t_x_t_e_s_t.tmp" - local f = io.open(name,"wb") - if f then - f:close() - os.remove(name) - return true - end - elseif lfs.isfile(name) then - local f = io.open(name,"ab") - if f then - f:close() - return true - end - else - local f = io.open(name,"ab") - if f then - f:close() - os.remove(name) - return true - end - end - return false -end - -local readable = P("r") * Cc(true) - -function file.is_readable(name) - if name then - local a = attributes(name) - return a and lpegmatch(readable,a.permissions) or false - else - return false - end -end - -file.isreadable = file.is_readable -- depricated -file.iswritable = file.is_writable -- depricated - -function file.size(name) - if name then - local a = attributes(name) - return a and a.size or 0 - else - return 0 - end -end - -function file.splitpath(str,separator) -- string .. reslash is a bonus (we could do a direct split) - return str and checkedsplit(lpegmatch(reslasher,str),separator or io.pathseparator) -end - -function file.joinpath(tab,separator) -- table - return tab and concat(tab,separator or io.pathseparator) -- can have trailing // -end - -local stripper = Cs(P(fwslash)^0/"" * reslasher) -local isnetwork = fwslash * fwslash * (1-fwslash) + (1-fwslash-colon)^1 * colon -local isroot = fwslash^1 * -1 -local hasroot = fwslash^1 - -local deslasher = lpeg.replacer(S("\\/")^1,"/") - --- If we have a network or prefix then there is a change that we end up with two --- // in the middle ... we could prevent this if we (1) expand prefixes: and (2) --- split and rebuild as url. Of course we could assume no network paths (which --- makes sense) adn assume either mapped drives (windows) or mounts (unix) but --- then we still have to deal with urls ... anyhow, multiple // are never a real --- problem but just ugly. - -function file.join(...) - local lst = { ... } - local one = lst[1] - if lpegmatch(isnetwork,one) then - local two = lpegmatch(deslasher,concat(lst,"/",2)) - return one .. "/" .. two - elseif lpegmatch(isroot,one) then - local two = lpegmatch(deslasher,concat(lst,"/",2)) - if lpegmatch(hasroot,two) then - return two - else - return "/" .. two - end - elseif one == "" then - return lpegmatch(stripper,concat(lst,"/",2)) - else - return lpegmatch(deslasher,concat(lst,"/")) - end -end - --- print(file.join("c:/whatever","name")) --- print(file.join("//","/y")) --- print(file.join("/","/y")) --- print(file.join("","/y")) --- print(file.join("/x/","/y")) --- print(file.join("x/","/y")) --- print(file.join("http://","/y")) --- print(file.join("http://a","/y")) --- print(file.join("http:///a","/y")) --- print(file.join("//nas-1","/y")) - --- The previous one fails on "a.b/c" so Taco came up with a split based --- variant. After some skyping we got it sort of compatible with the old --- one. After that the anchoring to currentdir was added in a better way. --- Of course there are some optimizations too. Finally we had to deal with --- windows drive prefixes and things like sys://. Eventually gsubs and --- finds were replaced by lpegs. - -local drivespec = R("az","AZ")^1 * colon -local anchors = fwslash + drivespec -local untouched = periods + (1-period)^1 * P(-1) -local splitstarter = (Cs(drivespec * (bwslash/"/" + fwslash)^0) + Cc(false)) * Ct(lpeg.splitat(S("/\\")^1)) -local absolute = fwslash - -function file.collapsepath(str,anchor) - if not str then - return - end - if anchor and not lpegmatch(anchors,str) then - str = getcurrentdir() .. "/" .. str - end - if str == "" or str =="." then - return "." - elseif lpegmatch(untouched,str) then - return lpegmatch(reslasher,str) - end - local starter, oldelements = lpegmatch(splitstarter,str) - local newelements = { } - local i = #oldelements - while i > 0 do - local element = oldelements[i] - if element == '.' then - -- do nothing - elseif element == '..' then - local n = i - 1 - while n > 0 do - local element = oldelements[n] - if element ~= '..' and element ~= '.' then - oldelements[n] = '.' - break - else - n = n - 1 - end - end - if n < 1 then - insert(newelements,1,'..') - end - elseif element ~= "" then - insert(newelements,1,element) - end - i = i - 1 - end - if #newelements == 0 then - return starter or "." - elseif starter then - return starter .. concat(newelements, '/') - elseif lpegmatch(absolute,str) then - return "/" .. concat(newelements,'/') - else - return concat(newelements, '/') - end -end - --- local function test(str) --- print(string.format("%-20s %-15s %-15s",str,file.collapsepath(str),file.collapsepath(str,true))) --- end --- test("a/b.c/d") test("b.c/d") test("b.c/..") --- test("/") test("c:/..") test("sys://..") --- test("") test("./") test(".") test("..") test("./..") test("../..") --- test("a") test("./a") test("/a") test("a/../..") --- test("a/./b/..") test("a/aa/../b/bb") test("a/.././././b/..") test("a/./././b/..") --- test("a/b/c/../..") test("./a/b/c/../..") test("a/b/c/../..") - -local validchars = R("az","09","AZ","--","..") -local pattern_a = lpeg.replacer(1-validchars) -local pattern_a = Cs((validchars + P(1)/"-")^1) -local whatever = P("-")^0 / "" -local pattern_b = Cs(whatever * (1 - whatever * -1)^1) - -function file.robustname(str,strict) - if str then - str = lpegmatch(pattern_a,str) or str - if strict then - return lpegmatch(pattern_b,str) or str -- two step is cleaner (less backtracking) - else - return str - end - end -end - -file.readdata = io.loaddata -file.savedata = io.savedata - -function file.copy(oldname,newname) - if oldname and newname then - file.savedata(newname,io.loaddata(oldname)) - end -end - --- also rewrite previous - -local letter = R("az","AZ") + S("_-+") -local separator = P("://") - -local qualified = period^0 * fwslash - + letter * colon - + letter^1 * separator - + letter^1 * fwslash -local rootbased = fwslash - + letter * colon - -lpeg.patterns.qualified = qualified -lpeg.patterns.rootbased = rootbased - --- ./name ../name /name c: :// name/name - -function file.is_qualified_path(filename) - return filename and lpegmatch(qualified,filename) ~= nil -end - -function file.is_rootbased_path(filename) - return filename and lpegmatch(rootbased,filename) ~= nil -end - --- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end --- --- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } --- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } --- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } --- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } - --- -- maybe: --- --- if os.type == "windows" then --- local currentdir = getcurrentdir --- function getcurrentdir() --- return lpegmatch(reslasher,currentdir()) --- end --- end - --- for myself: - -function file.strip(name,dir) - if name then - local b, a = match(name,"^(.-)" .. dir .. "(.*)$") - return a ~= "" and a or name - end -end - --- local debuglist = { --- "pathpart", "basename", "nameonly", "suffixonly", "suffix", "dirname", "extname", --- "addsuffix", "removesuffix", "replacesuffix", "join", --- "strip","collapsepath", "joinpath", "splitpath", --- } - --- for i=1,#debuglist do --- local name = debuglist[i] --- local f = file[name] --- file[name] = function(...) --- print(name,f(...)) --- return f(...) --- end --- end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-md5'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This also provides file checksums and checkers. - -local md5, file = md5, file -local gsub, format, byte = string.gsub, string.format, string.byte - -local function convert(str,fmt) - return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) -end - -if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end -if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end -if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end - --- local P, Cs, lpegmatch = lpeg.P, lpeg.Cs,lpeg.match --- --- if not md5.HEX then --- local function remap(chr) return format("%02X",byte(chr)) end --- function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end --- end --- --- if not md5.hex then --- local function remap(chr) return format("%02x",byte(chr)) end --- function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end --- end --- --- if not md5.dec then --- local function remap(chr) return format("%03i",byte(chr)) end --- function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end --- end - --- if not md5.HEX then --- local pattern_HEX = Cs( ( P(1) / function(chr) return format("%02X",byte(chr)) end)^0 ) --- function md5.HEX(str) return lpegmatch(pattern_HEX,md5.sum(str)) end --- end --- --- if not md5.hex then --- local pattern_hex = Cs( ( P(1) / function(chr) return format("%02x",byte(chr)) end)^0 ) --- function md5.hex(str) return lpegmatch(pattern_hex,md5.sum(str)) end --- end --- --- if not md5.dec then --- local pattern_dec = Cs( ( P(1) / function(chr) return format("%02i",byte(chr)) end)^0 ) --- function md5.dec(str) return lpegmatch(pattern_dec,md5.sum(str)) end --- end - -function file.needsupdating(oldname,newname,threshold) -- size modification access change - local oldtime = lfs.attributes(oldname,"modification") - if oldtime then - local newtime = lfs.attributes(newname,"modification") - if not newtime then - return true -- no new file, so no updating needed - elseif newtime >= oldtime then - return false -- new file definitely needs updating - elseif oldtime - newtime < (threshold or 1) then - return false -- new file is probably still okay - else - return true -- new file has to be updated - end - else - return false -- no old file, so no updating needed - end -end - -file.needs_updating = file.needsupdating - -function file.syncmtimes(oldname,newname) - local oldtime = lfs.attributes(oldname,"modification") - if oldtime and lfs.isfile(newname) then - lfs.touch(newname,oldtime,oldtime) - end -end - -function file.checksum(name) - if md5 then - local data = io.loaddata(name) - if data then - return md5.HEX(data) - end - end - return nil -end - -function file.loadchecksum(name) - if md5 then - local data = io.loaddata(name .. ".md5") - return data and (gsub(data,"%s","")) - end - return nil -end - -function file.savechecksum(name,checksum) - if not checksum then checksum = file.checksum(name) end - if checksum then - io.savedata(name .. ".md5",checksum) - return checksum - end - return nil -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-url'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local char, format, byte = string.char, string.format, string.byte -local concat = table.concat -local tonumber, type = tonumber, type -local P, C, R, S, Cs, Cc, Ct, Cf, Cg, V = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.Cf, lpeg.Cg, lpeg.V -local lpegmatch, lpegpatterns, replacer = lpeg.match, lpeg.patterns, lpeg.replacer - --- from wikipedia: --- --- foo://username:password@example.com:8042/over/there/index.dtb?type=animal;name=narwhal#nose --- \_/ \_______________/ \_________/ \__/ \___/ \_/ \______________________/ \__/ --- | | | | | | | | --- | userinfo hostname port | | query fragment --- | \________________________________/\_____________|____|/ --- scheme | | | | --- | authority path | | --- | | | --- | path interpretable as filename --- | ___________|____________ | --- / \ / \ | --- urn:example:animal:ferret:nose interpretable as extension - -url = url or { } -local url = url - -local tochar = function(s) return char(tonumber(s,16)) end - -local colon = P(":") -local qmark = P("?") -local hash = P("#") -local slash = P("/") -local percent = P("%") -local endofstring = P(-1) - -local hexdigit = R("09","AF","af") -local plus = P("+") -local nothing = Cc("") -local escapedchar = (percent * C(hexdigit * hexdigit)) / tochar -local escaped = (plus / " ") + escapedchar - -local noslash = P("/") / "" - --- we assume schemes with more than 1 character (in order to avoid problems with windows disks) --- we also assume that when we have a scheme, we also have an authority --- --- maybe we should already split the query (better for unescaping as = & can be part of a value - -local schemestr = Cs((escaped+(1-colon-slash-qmark-hash))^2) -local authoritystr = Cs((escaped+(1- slash-qmark-hash))^0) -local pathstr = Cs((escaped+(1- qmark-hash))^0) ------ querystr = Cs((escaped+(1- hash))^0) -local querystr = Cs(( (1- hash))^0) -local fragmentstr = Cs((escaped+(1- endofstring))^0) - -local scheme = schemestr * colon + nothing -local authority = slash * slash * authoritystr + nothing -local path = slash * pathstr + nothing -local query = qmark * querystr + nothing -local fragment = hash * fragmentstr + nothing - -local validurl = scheme * authority * path * query * fragment -local parser = Ct(validurl) - -lpegpatterns.url = validurl -lpegpatterns.urlsplitter = parser - -local escapes = { } - -setmetatable(escapes, { __index = function(t,k) - local v = format("%%%02X",byte(k)) - t[k] = v - return v -end }) - -local escaper = Cs((R("09","AZ","az")^1 + P(" ")/"%%20" + S("-./_")^1 + P(1) / escapes)^0) -- space happens most -local unescaper = Cs((escapedchar + 1)^0) - -lpegpatterns.urlunescaped = escapedchar -lpegpatterns.urlescaper = escaper -lpegpatterns.urlunescaper = unescaper - --- todo: reconsider Ct as we can as well have five return values (saves a table) --- so we can have two parsers, one with and one without - -local function split(str) - return (type(str) == "string" and lpegmatch(parser,str)) or str -end - -local isscheme = schemestr * colon * slash * slash -- this test also assumes authority - -local function hasscheme(str) - if str then - local scheme = lpegmatch(isscheme,str) -- at least one character - return scheme ~= "" and scheme or false - else - return false - end -end - - --- todo: cache them - -local rootletter = R("az","AZ") - + S("_-+") -local separator = P("://") -local qualified = P(".")^0 * P("/") - + rootletter * P(":") - + rootletter^1 * separator - + rootletter^1 * P("/") -local rootbased = P("/") - + rootletter * P(":") - -local barswapper = replacer("|",":") -local backslashswapper = replacer("\\","/") - --- queries: - -local equal = P("=") -local amp = P("&") -local key = Cs(((escapedchar+1)-equal )^0) -local value = Cs(((escapedchar+1)-amp -endofstring)^0) - -local splitquery = Cf ( Ct("") * P { "sequence", - sequence = V("pair") * (amp * V("pair"))^0, - pair = Cg(key * equal * value), -}, rawset) - --- hasher - -local function hashed(str) -- not yet ok (/test?test) - if str == "" then - return { - scheme = "invalid", - original = str, - } - end - local s = split(str) - local rawscheme = s[1] - local rawquery = s[4] - local somescheme = rawscheme ~= "" - local somequery = rawquery ~= "" - if not somescheme and not somequery then - s = { - scheme = "file", - authority = "", - path = str, - query = "", - fragment = "", - original = str, - noscheme = true, - filename = str, - } - else -- not always a filename but handy anyway - local authority, path, filename = s[2], s[3] - if authority == "" then - filename = path - elseif path == "" then - filename = "" - else - filename = authority .. "/" .. path - end - s = { - scheme = rawscheme, - authority = authority, - path = path, - query = lpegmatch(unescaper,rawquery), -- unescaped, but possible conflict with & and = - queries = lpegmatch(splitquery,rawquery), -- split first and then unescaped - fragment = s[5], - original = str, - noscheme = false, - filename = filename, - } - end - return s -end - --- inspect(hashed("template://test")) - --- Here we assume: --- --- files: /// = relative --- files: //// = absolute (!) - - - -url.split = split -url.hasscheme = hasscheme -url.hashed = hashed - -function url.addscheme(str,scheme) -- no authority - if hasscheme(str) then - return str - elseif not scheme then - return "file:///" .. str - else - return scheme .. ":///" .. str - end -end - -function url.construct(hash) -- dodo: we need to escape ! - local fullurl, f = { }, 0 - local scheme, authority, path, query, fragment = hash.scheme, hash.authority, hash.path, hash.query, hash.fragment - if scheme and scheme ~= "" then - f = f + 1 ; fullurl[f] = scheme .. "://" - end - if authority and authority ~= "" then - f = f + 1 ; fullurl[f] = authority - end - if path and path ~= "" then - f = f + 1 ; fullurl[f] = "/" .. path - end - if query and query ~= "" then - f = f + 1 ; fullurl[f] = "?".. query - end - if fragment and fragment ~= "" then - f = f + 1 ; fullurl[f] = "#".. fragment - end - return lpegmatch(escaper,concat(fullurl)) -end - -local pattern = Cs(noslash * R("az","AZ") * (S(":|")/":") * noslash * P(1)^0) - -function url.filename(filename) - local spec = hashed(filename) - local path = spec.path - return (spec.scheme == "file" and path and lpegmatch(pattern,path)) or filename -end - --- print(url.filename("/c|/test")) --- print(url.filename("/c/test")) - -local function escapestring(str) - return lpegmatch(escaper,str) -end - -url.escape = escapestring - -function url.query(str) - if type(str) == "string" then - return lpegmatch(splitquery,str) or "" - else - return str - end -end - -function url.toquery(data) - local td = type(data) - if td == "string" then - return #str and escape(data) or nil -- beware of double escaping - elseif td == "table" then - if next(data) then - local t = { } - for k, v in next, data do - t[#t+1] = format("%s=%s",k,escapestring(v)) - end - return concat(t,"&") - end - else - -- nil is a signal that no query - end -end - --- /test/ | /test | test/ | test => test - -local pattern = Cs(noslash^0 * (1 - noslash * P(-1))^0) - -function url.barepath(path) - if not path or path == "" then - return "" - else - return lpegmatch(pattern,path) - end -end - --- print(url.barepath("/test"),url.barepath("test/"),url.barepath("/test/"),url.barepath("test")) --- print(url.barepath("/x/yz"),url.barepath("x/yz/"),url.barepath("/x/yz/"),url.barepath("x/yz")) - - - - - - - - - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-dir'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- dir.expandname will be merged with cleanpath and collapsepath - -local type, select = type, select -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub -local concat, insert, remove = table.concat, table.insert, table.remove -local lpegmatch = lpeg.match - -local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V - -dir = dir or { } -local dir = dir -local lfs = lfs - -local attributes = lfs.attributes -local walkdir = lfs.dir -local isdir = lfs.isdir -local isfile = lfs.isfile -local currentdir = lfs.currentdir - --- in case we load outside luatex - -if not isdir then - function isdir(name) - local a = attributes(name) - return a and a.mode == "directory" - end - lfs.isdir = isdir -end - -if not isfile then - function isfile(name) - local a = attributes(name) - return a and a.mode == "file" - end - lfs.isfile = isfile -end - --- handy - -function dir.current() - return (gsub(currentdir(),"\\","/")) -end - --- optimizing for no find (*) does not save time - - -local lfsisdir = isdir - -local function isdir(path) - path = gsub(path,"[/\\]+$","") - return lfsisdir(path) -end - -lfs.isdir = isdir - -local function globpattern(path,patt,recurse,action) - if path == "/" then - path = path .. "." - elseif not find(path,"/$") then - path = path .. '/' - end - if isdir(path) then -- lfs.isdir does not like trailing / - for name in walkdir(path) do -- lfs.dir accepts trailing / - local full = path .. name - local mode = attributes(full,'mode') - if mode == 'file' then - if find(full,patt) then - action(full) - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - globpattern(full,patt,recurse,action) - end - end - end -end - -dir.globpattern = globpattern - -local function collectpattern(path,patt,recurse,result) - local ok, scanner - result = result or { } - if path == "/" then - ok, scanner, first = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe - else - ok, scanner, first = xpcall(function() return walkdir(path) end, function() end) -- kepler safe - end - if ok and type(scanner) == "function" then - if not find(path,"/$") then path = path .. '/' end - for name in scanner, first do - local full = path .. name - local attr = attributes(full) - local mode = attr.mode - if mode == 'file' then - if find(full,patt) then - result[name] = attr - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - attr.list = collectpattern(full,patt,recurse) - result[name] = attr - end - end - end - return result -end - -dir.collectpattern = collectpattern - -local pattern = Ct { - [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), - [2] = C(((1-S("*?/"))^0 * P("/"))^0), - [3] = C(P(1)^0) -} - -local filter = Cs ( ( - P("**") / ".*" + - P("*") / "[^/]*" + - P("?") / "[^/]" + - P(".") / "%%." + - P("+") / "%%+" + - P("-") / "%%-" + - P(1) -)^0 ) - -local function glob(str,t) - if type(t) == "function" then - if type(str) == "table" then - for s=1,#str do - glob(str[s],t) - end - elseif isfile(str) then - t(str) - else - local split = lpegmatch(pattern,str) -- we could use the file splitter - if split then - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = lpegmatch(filter,start .. base) - globpattern(start,result,recurse,t) - end - end - else - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) - end - return t - elseif isfile(str) then - if t then - t[#t+1] = str - return t - else - return { str } - end - else - local split = lpegmatch(pattern,str) -- we could use the file splitter - if split then - local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = lpegmatch(filter,start .. base) - globpattern(start,result,recurse,action) - return t - else - return { } - end - end - end -end - -dir.glob = glob - - -local function globfiles(path,recurse,func,files) -- func == pattern or function - if type(func) == "string" then - local s = func - func = function(name) return find(name,s) end - end - files = files or { } - local noffiles = #files - for name in walkdir(path) do - if find(name,"^%.") then - --- skip - else - local mode = attributes(name,'mode') - if mode == "directory" then - if recurse then - globfiles(path .. "/" .. name,recurse,func,files) - end - elseif mode == "file" then - if not func or func(name) then - noffiles = noffiles + 1 - files[noffiles] = path .. "/" .. name - end - end - end - end - return files -end - -dir.globfiles = globfiles - --- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") --- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") --- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") --- t = dir.glob("f:/minimal/tex/**/*") --- print(dir.ls("f:/minimal/tex/**/*")) --- print(dir.ls("*.tex")) - -function dir.ls(pattern) - return concat(glob(pattern),"\n") -end - - -local make_indeed = true -- false - -local onwindows = os.type == "windows" or find(os.getenv("PATH"),";") - -if onwindows then - - function dir.mkdirs(...) - local str, pth = "", "" - for i=1,select("#",...) do - local s = select(i,...) - if s == "" then - -- skip - elseif str == "" then - str = s - else - str = str .. "/" .. s - end - end - local first, middle, last - local drive = false - first, middle, last = match(str,"^(//)(//*)(.*)$") - if first then - -- empty network path == local path - else - first, last = match(str,"^(//)/*(.-)$") - if first then - middle, last = match(str,"([^/]+)/+(.-)$") - if middle then - pth = "//" .. middle - else - pth = "//" .. last - last = "" - end - else - first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") - if first then - pth, drive = first .. middle, true - else - middle, last = match(str,"^(/*)(.-)$") - if not middle then - last = str - end - end - end - end - for s in gmatch(last,"[^/]+") do - if pth == "" then - pth = s - elseif drive then - pth, drive = pth .. s, false - else - pth = pth .. "/" .. s - end - if make_indeed and not isdir(pth) then - lfs.mkdir(pth) - end - end - return pth, (isdir(pth) == true) - end - - -else - - function dir.mkdirs(...) - local str, pth = "", "" - for i=1,select("#",...) do - local s = select(i,...) - if s and s ~= "" then -- we catch nil and false - if str ~= "" then - str = str .. "/" .. s - else - str = s - end - end - end - str = gsub(str,"/+","/") - if find(str,"^/") then - pth = "/" - for s in gmatch(str,"[^/]+") do - local first = (pth == "/") - if first then - pth = pth .. s - else - pth = pth .. "/" .. s - end - if make_indeed and not first and not isdir(pth) then - lfs.mkdir(pth) - end - end - else - pth = "." - for s in gmatch(str,"[^/]+") do - pth = pth .. "/" .. s - if make_indeed and not isdir(pth) then - lfs.mkdir(pth) - end - end - end - return pth, (isdir(pth) == true) - end - - -end - -dir.makedirs = dir.mkdirs - --- we can only define it here as it uses dir.current - -if onwindows then - - function dir.expandname(str) -- will be merged with cleanpath and collapsepath - local first, nothing, last = match(str,"^(//)(//*)(.*)$") - if first then - first = dir.current() .. "/" - end - if not first then - first, last = match(str,"^(//)/*(.*)$") - end - if not first then - first, last = match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d = currentdir() - if lfs.chdir(first) then - first = dir.current() - end - lfs.chdir(d) - end - end - if not first then - first, last = dir.current(), str - end - last = gsub(last,"//","/") - last = gsub(last,"/%./","/") - last = gsub(last,"^/*","") - first = gsub(first,"/*$","") - if last == "" or last == "." then - return first - else - return first .. "/" .. last - end - end - -else - - function dir.expandname(str) -- will be merged with cleanpath and collapsepath - if not find(str,"^/") then - str = currentdir() .. "/" .. str - end - str = gsub(str,"//","/") - str = gsub(str,"/%./","/") - str = gsub(str,"(.)/%.$","%1") - return str - end - -end - -file.expandname = dir.expandname -- for convenience - -local stack = { } - -function dir.push(newdir) - insert(stack,lfs.currentdir()) -end - -function dir.pop() - local d = remove(stack) - if d then - lfs.chdir(d) - end - return d -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-boolean'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local type, tonumber = type, tonumber - -boolean = boolean or { } -local boolean = boolean - -function boolean.tonumber(b) - if b then return 1 else return 0 end -- test and return or return -end - -function toboolean(str,tolerant) - if str == nil then - return false - elseif str == false then - return false - elseif str == true then - return true - elseif str == "true" then - return true - elseif str == "false" then - return false - elseif not tolerant then - return false - elseif str == 0 then - return false - elseif (tonumber(str) or 0) > 0 then - return true - else - return str == "yes" or str == "on" or str == "t" - end -end - -string.toboolean = toboolean - -function string.booleanstring(str) - if str == nil then - return false - elseif str == false then - return false - elseif str == true then - return true - elseif str == "true" then - return true - elseif str == "false" then - return false - elseif str == 0 then - return false - elseif (tonumber(str) or 0) > 0 then - return true - else - return str == "yes" or str == "on" or str == "t" - end -end - -function string.is_boolean(str,default) - if type(str) == "string" then - if str == "true" or str == "yes" or str == "on" or str == "t" then - return true - elseif str == "false" or str == "no" or str == "off" or str == "f" then - return false - end - end - return default -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-unicode'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module will be reorganized - --- todo: utf.sub replacement (used in syst-aux) - --- we put these in the utf namespace: - -utf = utf or (unicode and unicode.utf8) or { } - -utf.characters = utf.characters or string.utfcharacters -utf.values = utf.values or string.utfvalues - --- string.utfvalues --- string.utfcharacters --- string.characters --- string.characterpairs --- string.bytes --- string.bytepairs - -local type = type -local char, byte, format, sub = string.char, string.byte, string.format, string.sub -local concat = table.concat -local P, C, R, Cs, Ct, Cmt, Cc, Carg = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Carg -local lpegmatch, patterns = lpeg.match, lpeg.patterns - -local bytepairs = string.bytepairs - -local finder = lpeg.finder -local replacer = lpeg.replacer - -local utfvalues = utf.values -local utfgmatch = utf.gmatch -- not always present - -local p_utftype = patterns.utftype -local p_utfoffset = patterns.utfoffset -local p_utf8char = patterns.utf8char -local p_utf8byte = patterns.utf8byte -local p_utfbom = patterns.utfbom -local p_newline = patterns.newline -local p_whitespace = patterns.whitespace - -if not unicode then - - unicode = { utf = utf } -- for a while - -end - -if not utf.char then - - local floor, char = math.floor, string.char - - function utf.char(n) - if n < 0x80 then - -- 0aaaaaaa : 0x80 - return char(n) - elseif n < 0x800 then - -- 110bbbaa : 0xC0 : n >> 6 - -- 10aaaaaa : 0x80 : n & 0x3F - return char( - 0xC0 + floor(n/0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x10000 then - -- 1110bbbb : 0xE0 : n >> 12 - -- 10bbbbaa : 0x80 : (n >> 6) & 0x3F - -- 10aaaaaa : 0x80 : n & 0x3F - return char( - 0xE0 + floor(n/0x1000), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x200000 then - -- 11110ccc : 0xF0 : n >> 18 - -- 10ccbbbb : 0x80 : (n >> 12) & 0x3F - -- 10bbbbaa : 0x80 : (n >> 6) & 0x3F - -- 10aaaaaa : 0x80 : n & 0x3F - -- dddd : ccccc - 1 - return char( - 0xF0 + floor(n/0x40000), - 0x80 + (floor(n/0x1000) % 0x40), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - else - return "" - end - end - -end - -if not utf.byte then - - local utf8byte = patterns.utf8byte - - function utf.byte(c) - return lpegmatch(utf8byte,c) - end - -end - -local utfchar, utfbyte = utf.char, utf.byte - --- As we want to get rid of the (unmaintained) utf library we implement our own --- variants (in due time an independent module): - -function utf.filetype(data) - return data and lpegmatch(p_utftype,data) or "unknown" -end - -local toentities = Cs ( - ( - patterns.utf8one - + ( - patterns.utf8two - + patterns.utf8three - + patterns.utf8four - ) / function(s) local b = utfbyte(s) if b < 127 then return s else return format("&#%X;",b) end end - )^0 -) - -patterns.toentities = toentities - -function utf.toentities(str) - return lpegmatch(toentities,str) -end - - - - -local one = P(1) -local two = C(1) * C(1) -local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1) - --- actually one of them is already utf ... sort of useless this one - --- function utf.char(n) --- if n < 0x80 then --- return char(n) --- elseif n < 0x800 then --- return char( --- 0xC0 + floor(n/0x40), --- 0x80 + (n % 0x40) --- ) --- elseif n < 0x10000 then --- return char( --- 0xE0 + floor(n/0x1000), --- 0x80 + (floor(n/0x40) % 0x40), --- 0x80 + (n % 0x40) --- ) --- elseif n < 0x40000 then --- return char( --- 0xF0 + floor(n/0x40000), --- 0x80 + floor(n/0x1000), --- 0x80 + (floor(n/0x40) % 0x40), --- 0x80 + (n % 0x40) --- ) --- else --- -- return char( --- -- 0xF1 + floor(n/0x1000000), --- -- 0x80 + floor(n/0x40000), --- -- 0x80 + floor(n/0x1000), --- -- 0x80 + (floor(n/0x40) % 0x40), --- -- 0x80 + (n % 0x40) --- -- ) --- return "?" --- end --- end --- --- merge into: - -local pattern = P("\254\255") * Cs( ( - four / function(a,b,c,d) - local ab = 0xFF * byte(a) + byte(b) - local cd = 0xFF * byte(c) + byte(d) - return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) - end - + two / function(a,b) - return utfchar(byte(a)*256 + byte(b)) - end - + one - )^1 ) - + P("\255\254") * Cs( ( - four / function(b,a,d,c) - local ab = 0xFF * byte(a) + byte(b) - local cd = 0xFF * byte(c) + byte(d) - return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) - end - + two / function(b,a) - return utfchar(byte(a)*256 + byte(b)) - end - + one - )^1 ) - -function string.toutf(s) -- in string namespace - return lpegmatch(pattern,s) or s -- todo: utf32 -end - -local validatedutf = Cs ( - ( - patterns.utf8one - + patterns.utf8two - + patterns.utf8three - + patterns.utf8four - + P(1) / "�" - )^0 -) - -patterns.validatedutf = validatedutf - -function utf.is_valid(str) - return type(str) == "string" and lpegmatch(validatedutf,str) or false -end - -if not utf.len then - - -- -- alternative 1: 0.77 - -- - -- local utfcharcounter = utfbom^-1 * Cs((p_utf8char/'!')^0) - -- - -- function utf.len(str) - -- return #lpegmatch(utfcharcounter,str or "") - -- end - -- - -- -- alternative 2: 1.70 - -- - -- local n = 0 - -- - -- local utfcharcounter = utfbom^-1 * (p_utf8char/function() n = n + 1 end)^0 -- slow - -- - -- function utf.length(str) - -- n = 0 - -- lpegmatch(utfcharcounter,str or "") - -- return n - -- end - -- - -- -- alternative 3: 0.24 (native unicode.utf8.len: 0.047) - - -- local n = 0 - -- - -- -- local utfcharcounter = lpeg.patterns.utfbom^-1 * P ( ( Cp() * ( - -- -- patterns.utf8one ^1 * Cc(1) - -- -- + patterns.utf8two ^1 * Cc(2) - -- -- + patterns.utf8three^1 * Cc(3) - -- -- + patterns.utf8four ^1 * Cc(4) ) * Cp() / function(f,d,t) n = n + (t - f)/d end - -- -- )^0 ) -- just as many captures as below - -- - -- -- local utfcharcounter = lpeg.patterns.utfbom^-1 * P ( ( - -- -- (Cmt(patterns.utf8one ^1,function(_,_,s) n = n + #s return true end)) - -- -- + (Cmt(patterns.utf8two ^1,function(_,_,s) n = n + #s/2 return true end)) - -- -- + (Cmt(patterns.utf8three^1,function(_,_,s) n = n + #s/3 return true end)) - -- -- + (Cmt(patterns.utf8four ^1,function(_,_,s) n = n + #s/4 return true end)) - -- -- )^0 ) -- not interesting as it creates strings but sometimes faster - -- - -- -- The best so far: - -- - -- local utfcharcounter = utfbom^-1 * P ( ( - -- Cp() * (patterns.utf8one )^1 * Cp() / function(f,t) n = n + t - f end - -- + Cp() * (patterns.utf8two )^1 * Cp() / function(f,t) n = n + (t - f)/2 end - -- + Cp() * (patterns.utf8three)^1 * Cp() / function(f,t) n = n + (t - f)/3 end - -- + Cp() * (patterns.utf8four )^1 * Cp() / function(f,t) n = n + (t - f)/4 end - -- )^0 ) - - -- function utf.len(str) - -- n = 0 - -- lpegmatch(utfcharcounter,str or "") - -- return n - -- end - - local n, f = 0, 1 - - local utfcharcounter = patterns.utfbom^-1 * Cmt ( - Cc(1) * patterns.utf8one ^1 - + Cc(2) * patterns.utf8two ^1 - + Cc(3) * patterns.utf8three^1 - + Cc(4) * patterns.utf8four ^1, - function(_,t,d) -- due to Cc no string captures, so faster - n = n + (t - f)/d - f = t - return true - end - )^0 - - function utf.len(str) - n, f = 0, 1 - lpegmatch(utfcharcounter,str or "") - return n - end - -end - -utf.length = utf.len - -if not utf.sub then - - -- inefficient as lpeg just copies ^n - - -- local function sub(str,start,stop) - -- local pattern = p_utf8char^-(start-1) * C(p_utf8char^-(stop-start+1)) - -- inspect(pattern) - -- return lpegmatch(pattern,str) or "" - -- end - - -- local b, e, n, first, last = 0, 0, 0, 0, 0 - -- - -- local function slide(s,p) - -- n = n + 1 - -- if n == first then - -- b = p - -- if not last then - -- return nil - -- end - -- end - -- if n == last then - -- e = p - -- return nil - -- else - -- return p - -- end - -- end - -- - -- local pattern = Cmt(p_utf8char,slide)^0 - -- - -- function utf.sub(str,start,stop) -- todo: from the end - -- if not start then - -- return str - -- end - -- b, e, n, first, last = 0, 0, 0, start, stop - -- lpegmatch(pattern,str) - -- if not stop then - -- return sub(str,b) - -- else - -- return sub(str,b,e-1) - -- end - -- end - - -- print(utf.sub("Hans Hagen is my name")) - -- print(utf.sub("Hans Hagen is my name",5)) - -- print(utf.sub("Hans Hagen is my name",5,10)) - - local utflength = utf.length - - -- also negative indices, upto 10 times slower than a c variant - - local b, e, n, first, last = 0, 0, 0, 0, 0 - - local function slide_zero(s,p) - n = n + 1 - if n >= last then - e = p - 1 - else - return p - end - end - - local function slide_one(s,p) - n = n + 1 - if n == first then - b = p - end - if n >= last then - e = p - 1 - else - return p - end - end - - local function slide_two(s,p) - n = n + 1 - if n == first then - b = p - else - return true - end - end - - local pattern_zero = Cmt(p_utf8char,slide_zero)^0 - local pattern_one = Cmt(p_utf8char,slide_one )^0 - local pattern_two = Cmt(p_utf8char,slide_two )^0 - - function utf.sub(str,start,stop) - if not start then - return str - end - if start == 0 then - start = 1 - end - if not stop then - if start < 0 then - local l = utflength(str) -- we can inline this function if needed - start = l + start - else - start = start - 1 - end - b, n, first = 0, 0, start - lpegmatch(pattern_two,str) - if n >= first then - return sub(str,b) - else - return "" - end - end - if start < 0 or stop < 0 then - local l = utf.length(str) - if start < 0 then - start = l + start - if start <= 0 then - start = 1 - else - start = start + 1 - end - end - if stop < 0 then - stop = l + stop - if stop == 0 then - stop = 1 - else - stop = stop + 1 - end - end - end - if start > stop then - return "" - elseif start > 1 then - b, e, n, first, last = 0, 0, 0, start - 1, stop - lpegmatch(pattern_one,str) - if n >= first and e == 0 then - e = #str - end - return sub(str,b,e) - else - b, e, n, last = 1, 0, 0, stop - lpegmatch(pattern_zero,str) - if e == 0 then - e = #str - end - return sub(str,b,e) - end - end - - -- local n = 100000 - -- local str = string.rep("123456àáâãäå",100) - -- - -- for i=-15,15,1 do - -- for j=-15,15,1 do - -- if utf.xsub(str,i,j) ~= utf.sub(str,i,j) then - -- print("error",i,j,"l>"..utf.xsub(str,i,j),"s>"..utf.sub(str,i,j)) - -- end - -- end - -- if utf.xsub(str,i) ~= utf.sub(str,i) then - -- print("error",i,"l>"..utf.xsub(str,i),"s>"..utf.sub(str,i)) - -- end - -- end - - -- print(" 1, 7",utf.xsub(str, 1, 7),utf.sub(str, 1, 7)) - -- print(" 0, 7",utf.xsub(str, 0, 7),utf.sub(str, 0, 7)) - -- print(" 0, 9",utf.xsub(str, 0, 9),utf.sub(str, 0, 9)) - -- print(" 4 ",utf.xsub(str, 4 ),utf.sub(str, 4 )) - -- print(" 0 ",utf.xsub(str, 0 ),utf.sub(str, 0 )) - -- print(" 0, 0",utf.xsub(str, 0, 0),utf.sub(str, 0, 0)) - -- print(" 4, 4",utf.xsub(str, 4, 4),utf.sub(str, 4, 4)) - -- print(" 4, 0",utf.xsub(str, 4, 0),utf.sub(str, 4, 0)) - -- print("-3, 0",utf.xsub(str,-3, 0),utf.sub(str,-3, 0)) - -- print(" 0,-3",utf.xsub(str, 0,-3),utf.sub(str, 0,-3)) - -- print(" 5,-3",utf.xsub(str,-5,-3),utf.sub(str,-5,-3)) - -- print("-3 ",utf.xsub(str,-3 ),utf.sub(str,-3 )) - -end - --- a replacement for simple gsubs: - -function utf.remapper(mapping) - local pattern = Cs((p_utf8char/mapping)^0) - return function(str) - if not str or str == "" then - return "" - else - return lpegmatch(pattern,str) - end - end, pattern -end - --- local remap = utf.remapper { a = 'd', b = "c", c = "b", d = "a" } --- print(remap("abcd 1234 abcd")) - --- - -function utf.replacer(t) -- no precheck, always string builder - local r = replacer(t,false,false,true) - return function(str) - return lpegmatch(r,str) - end -end - -function utf.subtituter(t) -- with precheck and no building if no match - local f = finder (t) - local r = replacer(t,false,false,true) - return function(str) - local i = lpegmatch(f,str) - if not i then - return str - elseif i > #str then - return str - else - -- return sub(str,1,i-2) .. lpegmatch(r,str,i-1) -- slower - return lpegmatch(r,str) - end - end -end - --- inspect(utf.split("a b c d")) --- inspect(utf.split("a b c d",true)) - -local utflinesplitter = p_utfbom^-1 * lpeg.tsplitat(p_newline) -local utfcharsplitter_ows = p_utfbom^-1 * Ct(C(p_utf8char)^0) -local utfcharsplitter_iws = p_utfbom^-1 * Ct((p_whitespace^1 + C(p_utf8char))^0) -local utfcharsplitter_raw = Ct(C(p_utf8char)^0) - -patterns.utflinesplitter = utflinesplitter - -function utf.splitlines(str) - return lpegmatch(utflinesplitter,str or "") -end - -function utf.split(str,ignorewhitespace) -- new - if ignorewhitespace then - return lpegmatch(utfcharsplitter_iws,str or "") - else - return lpegmatch(utfcharsplitter_ows,str or "") - end -end - -function utf.totable(str) -- keeps bom - return lpegmatch(utfcharsplitter_raw,str) -end - --- 0 EF BB BF UTF-8 --- 1 FF FE UTF-16-little-endian --- 2 FE FF UTF-16-big-endian --- 3 FF FE 00 00 UTF-32-little-endian --- 4 00 00 FE FF UTF-32-big-endian --- --- \000 fails in <= 5.0 but is valid in >=5.1 where %z is depricated - --- utf.name = { --- [0] = 'utf-8', --- [1] = 'utf-16-le', --- [2] = 'utf-16-be', --- [3] = 'utf-32-le', --- [4] = 'utf-32-be' --- } --- --- function utf.magic(f) --- local str = f:read(4) --- if not str then --- f:seek('set') --- return 0 --- -- elseif find(str,"^%z%z\254\255") then -- depricated --- -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged --- elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) --- return 4 --- -- elseif find(str,"^\255\254%z%z") then -- depricated --- -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged --- elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) --- return 3 --- elseif find(str,"^\254\255") then --- f:seek('set',2) --- return 2 --- elseif find(str,"^\255\254") then --- f:seek('set',2) --- return 1 --- elseif find(str,"^\239\187\191") then --- f:seek('set',3) --- return 0 --- else --- f:seek('set') --- return 0 --- end --- end - -function utf.magic(f) -- not used - local str = f:read(4) or "" - local off = lpegmatch(p_utfoffset,str) - if off < 4 then - f:seek('set',off) - end - return lpegmatch(p_utftype,str) -end - -local function utf16_to_utf8_be(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, 0 - for left, right in bytepairs(t[i]) do - if right then - local now = 256*left + right - if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong - more = 0 - r = r + 1 - result[r] = utfchar(now) - elseif now >= 0xD800 and now <= 0xDBFF then - more = now - else - r = r + 1 - result[r] = utfchar(now) - end - end - end - t[i] = concat(result,"",1,r) -- we reused tmp, hence t - end - return t -end - -local function utf16_to_utf8_le(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, 0 - for left, right in bytepairs(t[i]) do - if right then - local now = 256*right + left - if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong - more = 0 - r = r + 1 - result[r] = utfchar(now) - elseif now >= 0xD800 and now <= 0xDBFF then - more = now - else - r = r + 1 - result[r] = utfchar(now) - end - end - end - t[i] = concat(result,"",1,r) -- we reused tmp, hence t - end - return t -end - -local function utf32_to_utf8_be(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, -1 - for a,b in bytepairs(t[i]) do - if a and b then - if more < 0 then - more = 256*256*256*a + 256*256*b - else - r = r + 1 - result[t] = utfchar(more + 256*a + b) - more = -1 - end - else - break - end - end - t[i] = concat(result,"",1,r) - end - return t -end - -local function utf32_to_utf8_le(t) - if type(t) == "string" then - t = lpegmatch(utflinesplitter,t) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, -1 - for a,b in bytepairs(t[i]) do - if a and b then - if more < 0 then - more = 256*b + a - else - r = r + 1 - result[t] = utfchar(more + 256*256*256*b + 256*256*a) - more = -1 - end - else - break - end - end - t[i] = concat(result,"",1,r) - end - return t -end - -utf.utf32_to_utf8_be = utf32_to_utf8_be -utf.utf32_to_utf8_le = utf32_to_utf8_le -utf.utf16_to_utf8_be = utf16_to_utf8_be -utf.utf16_to_utf8_le = utf16_to_utf8_le - -function utf.utf8_to_utf8(t) - return type(t) == "string" and lpegmatch(utflinesplitter,t) or t -end - -function utf.utf16_to_utf8(t,endian) - return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t -end - -function utf.utf32_to_utf8(t,endian) - return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t -end - -local function little(c) - local b = byte(c) - if b < 0x10000 then - return char(b%256,b/256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1%256,b1/256,b2%256,b2/256) - end -end - -local function big(c) - local b = byte(c) - if b < 0x10000 then - return char(b/256,b%256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1/256,b1%256,b2/256,b2%256) - end -end - --- function utf.utf8_to_utf16(str,littleendian) --- if littleendian then --- return char(255,254) .. utfgsub(str,".",little) --- else --- return char(254,255) .. utfgsub(str,".",big) --- end --- end - -local _, l_remap = utf.remapper(little) -local _, b_remap = utf.remapper(big) - -function utf.utf8_to_utf16(str,littleendian) - if littleendian then - return char(255,254) .. lpegmatch(l_remap,str) - else - return char(254,255) .. lpegmatch(b_remap,str) - end -end - --- function utf.tocodes(str,separator) -- can be sped up with an lpeg --- local t, n = { }, 0 --- for u in utfvalues(str) do --- n = n + 1 --- t[n] = format("0x%04X",u) --- end --- return concat(t,separator or " ") --- end - -local pattern = Cs ( - (p_utf8byte / function(unicode ) return format( "0x%04X", unicode) end) * - (p_utf8byte * Carg(1) / function(unicode,separator) return format("%s0x%04X",separator,unicode) end)^0 -) - -function utf.tocodes(str,separator) - return lpegmatch(pattern,str,1,separator or " ") -end - -function utf.ustring(s) - return format("U+%05X",type(s) == "number" and s or utfbyte(s)) -end - -function utf.xstring(s) - return format("0x%05X",type(s) == "number" and s or utfbyte(s)) -end - --- - -local p_nany = p_utf8char / "" - -if utfgmatch then - - function utf.count(str,what) - if type(what) == "string" then - local n = 0 - for _ in utfgmatch(str,what) do - n = n + 1 - end - return n - else -- 4 times slower but still faster than / function - return #lpegmatch(Cs((P(what)/" " + p_nany)^0),str) - end - end - -else - - local cache = { } - - function utf.count(str,what) - if type(what) == "string" then - local p = cache[what] - if not p then - p = Cs((P(what)/" " + p_nany)^0) - cache[p] = p - end - return #lpegmatch(p,str) - else -- 4 times slower but still faster than / function - return #lpegmatch(Cs((P(what)/" " + p_nany)^0),str) - end - end - -end - --- maybe also register as string.utf* - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-math'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan - -if not math.round then - function math.round(x) return floor(x + 0.5) end -end - -if not math.div then - function math.div(n,m) return floor(n/m) end -end - -if not math.mod then - function math.mod(n,m) return n % m end -end - -local pipi = 2*math.pi/360 - -if not math.sind then - function math.sind(d) return sin(d*pipi) end - function math.cosd(d) return cos(d*pipi) end - function math.tand(d) return tan(d*pipi) end -end - -if not math.odd then - function math.odd (n) return n % 2 ~= 0 end - function math.even(n) return n % 2 == 0 end -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-tab'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or {} -utilities.tables = utilities.tables or { } -local tables = utilities.tables - -local format, gmatch, rep, gsub = string.format, string.gmatch, string.rep, string.gsub -local concat, insert, remove = table.concat, table.insert, table.remove -local setmetatable, getmetatable, tonumber, tostring = setmetatable, getmetatable, tonumber, tostring -local type, next, rawset, tonumber, load, select = type, next, rawset, tonumber, load, select -local lpegmatch, P, Cs = lpeg.match, lpeg.P, lpeg.Cs -local serialize = table.serialize - -local splitter = lpeg.tsplitat(".") - -function tables.definetable(target,nofirst,nolast) -- defines undefined tables - local composed, shortcut, t = nil, nil, { } - local snippets = lpegmatch(splitter,target) - for i=1,#snippets - (nolast and 1 or 0) do - local name = snippets[i] - if composed then - composed = shortcut .. "." .. name - shortcut = shortcut .. "_" .. name - t[#t+1] = format("local %s = %s if not %s then %s = { } %s = %s end",shortcut,composed,shortcut,shortcut,composed,shortcut) - else - composed = name - shortcut = name - if not nofirst then - t[#t+1] = format("%s = %s or { }",composed,composed) - end - end - end - if nolast then - composed = shortcut .. "." .. snippets[#snippets] - end - return concat(t,"\n"), composed -end - --- local t = tables.definedtable("a","b","c","d") - -function tables.definedtable(...) - local t = _G - for i=1,select("#",...) do - local li = select(i,...) - local tl = t[li] - if not tl then - tl = { } - t[li] = tl - end - t = tl - end - return t -end - -function tables.accesstable(target,root) - local t = root or _G - for name in gmatch(target,"([^%.]+)") do - t = t[name] - if not t then - return - end - end - return t -end - -function tables.migratetable(target,v,root) - local t = root or _G - local names = string.split(target,".") - for i=1,#names-1 do - local name = names[i] - t[name] = t[name] or { } - t = t[name] - if not t then - return - end - end - t[names[#names]] = v -end - -function tables.removevalue(t,value) -- todo: n - if value then - for i=1,#t do - if t[i] == value then - remove(t,i) - -- remove all, so no: return - end - end - end -end - -function tables.insertbeforevalue(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i,extra) - return - end - end - insert(t,1,extra) -end - -function tables.insertaftervalue(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i+1,extra) - return - end - end - insert(t,#t+1,extra) -end - --- experimental - -local function toxml(t,d,result,step) - for k, v in table.sortedpairs(t) do - if type(v) == "table" then - if type(k) == "number" then - result[#result+1] = format("%s",d,k) - toxml(v,d..step,result,step) - result[#result+1] = format("%s",d,k) - else - result[#result+1] = format("%s<%s>",d,k) - toxml(v,d..step,result,step) - result[#result+1] = format("%s",d,k) - end - elseif type(k) == "number" then - result[#result+1] = format("%s%s",d,k,v,k) - else - result[#result+1] = format("%s<%s>%s",d,k,tostring(v),k) - end - end -end - -function table.toxml(t,name,nobanner,indent,spaces) - local noroot = name == false - local result = (nobanner or noroot) and { } or { "" } - local indent = rep(" ",indent or 0) - local spaces = rep(" ",spaces or 1) - if noroot then - toxml( t, inndent, result, spaces) - else - toxml( { [name or "root"] = t }, indent, result, spaces) - end - return concat(result,"\n") -end - --- also experimental - --- encapsulate(table,utilities.tables) --- encapsulate(table,utilities.tables,true) --- encapsulate(table,true) - -function tables.encapsulate(core,capsule,protect) - if type(capsule) ~= "table" then - protect = true - capsule = { } - end - for key, value in next, core do - if capsule[key] then - print(format("\ninvalid inheritance '%s' in '%s': %s",key,tostring(core))) - os.exit() - else - capsule[key] = value - end - end - if protect then - for key, value in next, core do - core[key] = nil - end - setmetatable(core, { - __index = capsule, - __newindex = function(t,key,value) - if capsule[key] then - print(format("\ninvalid overload '%s' in '%s'",key,tostring(core))) - os.exit() - else - rawset(t,key,value) - end - end - } ) - end -end - -local function fastserialize(t,r,outer) -- no mixes - r[#r+1] = "{" - local n = #t - if n > 0 then - for i=1,n do - local v = t[i] - local tv = type(v) - if tv == "string" then - r[#r+1] = format("%q,",v) - elseif tv == "number" then - r[#r+1] = format("%s,",v) - elseif tv == "table" then - fastserialize(v,r) - elseif tv == "boolean" then - r[#r+1] = format("%s,",tostring(v)) - end - end - else - for k, v in next, t do - local tv = type(v) - if tv == "string" then - r[#r+1] = format("[%q]=%q,",k,v) - elseif tv == "number" then - r[#r+1] = format("[%q]=%s,",k,v) - elseif tv == "table" then - r[#r+1] = format("[%q]=",k) - fastserialize(v,r) - elseif tv == "boolean" then - r[#r+1] = format("[%q]=%s,",k,tostring(v)) - end - end - end - if outer then - r[#r+1] = "}" - else - r[#r+1] = "}," - end - return r -end - --- local f_hashed_string = formatters["[%q]=%q,"] --- local f_hashed_number = formatters["[%q]=%s,"] --- local f_hashed_table = formatters["[%q]="] --- local f_hashed_true = formatters["[%q]=true,"] --- local f_hashed_false = formatters["[%q]=false,"] --- --- local f_indexed_string = formatters["%q,"] --- local f_indexed_number = formatters["%s,"] --- ----- f_indexed_true = formatters["true,"] --- ----- f_indexed_false = formatters["false,"] --- --- local function fastserialize(t,r,outer) -- no mixes --- r[#r+1] = "{" --- local n = #t --- if n > 0 then --- for i=1,n do --- local v = t[i] --- local tv = type(v) --- if tv == "string" then --- r[#r+1] = f_indexed_string(v) --- elseif tv == "number" then --- r[#r+1] = f_indexed_number(v) --- elseif tv == "table" then --- fastserialize(v,r) --- elseif tv == "boolean" then --- -- r[#r+1] = v and f_indexed_true(k) or f_indexed_false(k) --- r[#r+1] = v and "true," or "false," --- end --- end --- else --- for k, v in next, t do --- local tv = type(v) --- if tv == "string" then --- r[#r+1] = f_hashed_string(k,v) --- elseif tv == "number" then --- r[#r+1] = f_hashed_number(k,v) --- elseif tv == "table" then --- r[#r+1] = f_hashed_table(k) --- fastserialize(v,r) --- elseif tv == "boolean" then --- r[#r+1] = v and f_hashed_true(k) or f_hashed_false(k) --- end --- end --- end --- if outer then --- r[#r+1] = "}" --- else --- r[#r+1] = "}," --- end --- return r --- end - -function table.fastserialize(t,prefix) -- so prefix should contain the = - return concat(fastserialize(t,{ prefix or "return" },true)) -end - -function table.deserialize(str) - if not str or str == "" then - return - end - local code = load(str) - if not code then - return - end - code = code() - if not code then - return - end - return code -end - --- inspect(table.fastserialize { a = 1, b = { 4, { 5, 6 } }, c = { d = 7, e = 'f"g\nh' } }) - -function table.load(filename) - if filename then - local t = io.loaddata(filename) - if t and t ~= "" then - t = load(t) - if type(t) == "function" then - t = t() - if type(t) == "table" then - return t - end - end - end - end -end - -function table.save(filename,t,n,...) - io.savedata(filename,serialize(t,n == nil and true or n,...)) -end - -local function slowdrop(t) - local r = { } - local l = { } - for i=1,#t do - local ti = t[i] - local j = 0 - for k, v in next, ti do - j = j + 1 - l[j] = format("%s=%q",k,v) - end - r[i] = format(" {%s},\n",concat(l)) - end - return format("return {\n%s}",concat(r)) -end - -local function fastdrop(t) - local r = { "return {\n" } - for i=1,#t do - local ti = t[i] - r[#r+1] = " {" - for k, v in next, ti do - r[#r+1] = format("%s=%q",k,v) - end - r[#r+1] = "},\n" - end - r[#r+1] = "}" - return concat(r) -end - -function table.drop(t,slow) - if #t == 0 then - return "return { }" - elseif slow == true then - return slowdrop(t) -- less memory - else - return fastdrop(t) -- some 15% faster - end -end - -function table.autokey(t,k) - local v = { } - t[k] = v - return v -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-sto'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local setmetatable, getmetatable = setmetatable, getmetatable - -utilities = utilities or { } -utilities.storage = utilities.storage or { } -local storage = utilities.storage - -local report = texio and texio.write_nl or print - -function storage.mark(t) - if not t then - report("fatal error: storage cannot be marked") - return -- os.exit() - end - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) - end - m.__storage__ = true - return t -end - -function storage.allocate(t) - t = t or { } - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) - end - m.__storage__ = true - return t -end - -function storage.marked(t) - local m = getmetatable(t) - return m and m.__storage__ -end - -function storage.checked(t) - if not t then - report("fatal error: storage has not been allocated") - return -- os.exit() - end - return t -end - --- function utilities.storage.delay(parent,name,filename) --- local m = getmetatable(parent) --- m.__list[name] = filename --- end --- --- function utilities.storage.predefine(parent) --- local list = { } --- local m = getmetatable(parent) or { --- __list = list, --- __index = function(t,k) --- local l = require(list[k]) --- t[k] = l --- return l --- end --- } --- setmetatable(parent,m) --- end --- --- bla = { } --- utilities.storage.predefine(bla) --- utilities.storage.delay(bla,"test","oepsoeps") --- local t = bla.test --- table.print(t) --- print(t.a) - -function storage.setinitializer(data,initialize) - local m = getmetatable(data) or { } - m.__index = function(data,k) - m.__index = nil -- so that we can access the entries during initializing - initialize() - return data[k] - end - setmetatable(data, m) -end - -local keyisvalue = { __index = function(t,k) - t[k] = k - return k -end } - -function storage.sparse(t) - t = t or { } - setmetatable(t,keyisvalue) - return t -end - --- table namespace ? - -local function f_empty () return "" end -- t,k -local function f_self (t,k) t[k] = k return k end -local function f_table (t,k) local v = { } t[k] = v return v end -local function f_ignore() end -- t,k,v - -local t_empty = { __index = f_empty } -local t_self = { __index = f_self } -local t_table = { __index = f_table } -local t_ignore = { __newindex = f_ignore } - -function table.setmetatableindex(t,f) - local m = getmetatable(t) - if m then - if f == "empty" then - m.__index = f_empty - elseif f == "key" then - m.__index = f_self - elseif f == "table" then - m.__index = f_table - else - m.__index = f - end - else - if f == "empty" then - setmetatable(t, t_empty) - elseif f == "key" then - setmetatable(t, t_self) - elseif f == "table" then - setmetatable(t, t_table) - else - setmetatable(t,{ __index = f }) - end - end - return t -end - -function table.setmetatablenewindex(t,f) - local m = getmetatable(t) - if m then - if f == "ignore" then - m.__newindex = f_ignore - else - m.__newindex = f - end - else - if f == "ignore" then - setmetatable(t, t_ignore) - else - setmetatable(t,{ __newindex = f }) - end - end - return t -end - -function table.setmetatablecall(t,f) - local m = getmetatable(t) - if m then - m.__call = f - else - setmetatable(t,{ __call = f }) - end - return t -end - -function table.setmetatablekey(t,key,value) - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) - end - m[key] = value - return t -end - -function table.getmetatablekey(t,key,value) - local m = getmetatable(t) - return m and m[key] -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-str'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or {} -utilities.strings = utilities.strings or { } -local strings = utilities.strings - -local load = load -local format, gsub, rep, sub = string.format, string.gsub, string.rep, string.sub -local concat = table.concat -local P, V, C, S, R, Ct, Cs, Cp, Carg = lpeg.P, lpeg.V, lpeg.C, lpeg.S, lpeg.R, lpeg.Ct, lpeg.Cs, lpeg.Cp, lpeg.Carg -local patterns, lpegmatch = lpeg.patterns, lpeg.match -local utfchar, utfbyte = utf.char, utf.byte -local setmetatableindex = table.setmetatableindex --- - -local stripper = patterns.stripzeros - -local function points(n) - return (not n or n == 0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) -end - -local function basepoints(n) - return (not n or n == 0) and "0bp" or lpegmatch(stripper,format("%.5fbp", n*(7200/7227)/65536)) -end - -number.points = points -number.basepoints = basepoints - --- str = " \n \ntest \n test\ntest " --- print("["..string.gsub(string.collapsecrlf(str),"\n","+").."]") - -local rubish = patterns.spaceortab^0 * patterns.newline -local anyrubish = patterns.spaceortab + patterns.newline -local anything = patterns.anything -local stripped = (patterns.spaceortab^1 / "") * patterns.newline -local leading = rubish^0 / "" -local trailing = (anyrubish^1 * patterns.endofstring) / "" -local redundant = rubish^3 / "\n" - -local pattern = Cs(leading * (trailing + redundant + stripped + anything)^0) - -function strings.collapsecrlf(str) - return lpegmatch(pattern,str) -end - --- The following functions might end up in another namespace. - -local repeaters = { } -- watch how we also moved the -1 in depth-1 to the creator - -function strings.newrepeater(str,offset) - offset = offset or 0 - local s = repeaters[str] - if not s then - s = { } - repeaters[str] = s - end - local t = s[offset] - if t then - return t - end - t = { } - setmetatableindex(t, function(t,k) - if not k then - return "" - end - local n = k + offset - local s = n > 0 and rep(str,n) or "" - t[k] = s - return s - end) - s[offset] = t - return t -end - --- local dashes = strings.newrepeater("--",-1) --- print(dashes[2],dashes[3],dashes[1]) - -local extra, tab, start = 0, 0, 4, 0 - -local nspaces = strings.newrepeater(" ") - -local pattern = - Carg(1) / function(t) - extra, tab, start = 0, t or 7, 1 - end - * Cs(( - Cp() * patterns.tab / function(position) - local current = (position - start + 1) + extra - local spaces = tab-(current-1) % tab - if spaces > 0 then - extra = extra + spaces - 1 - return nspaces[spaces] -- rep(" ",spaces) - else - return "" - end - end - + patterns.newline * Cp() / function(position) - extra, start = 0, position - end - + patterns.anything - )^1) - -function strings.tabtospace(str,tab) - return lpegmatch(pattern,str,1,tab or 7) -end - --- local t = { --- "1234567123456712345671234567", --- "\tb\tc", --- "a\tb\tc", --- "aa\tbb\tcc", --- "aaa\tbbb\tccc", --- "aaaa\tbbbb\tcccc", --- "aaaaa\tbbbbb\tccccc", --- "aaaaaa\tbbbbbb\tcccccc\n aaaaaa\tbbbbbb\tcccccc", --- "one\n two\nxxx three\nxx four\nx five\nsix", --- } --- for k=1,#t do --- print(strings.tabtospace(t[k])) --- end - -function strings.striplong(str) -- strips all leading spaces - str = gsub(str,"^%s*","") - str = gsub(str,"[\n\r]+ *","\n") - return str -end - --- local template = string.striplong([[ --- aaaa --- bb --- cccccc --- ]]) - -function strings.nice(str) - str = gsub(str,"[:%-+_]+"," ") -- maybe more - return str -end - --- Work in progress. Interesting is that compared to the built-in this --- is faster in luatex than in luajittex where we have a comparable speed. - -local n = 0 - --- we are somewhat sloppy in parsing prefixes as it's not that critical --- --- this does not work out ok: --- --- function fnc(...) -- 1,2,3 --- print(...,...,...) -- 1,1,1,2,3 --- end - -local prefix_any = C((S("+- .") + R("09"))^0) -local prefix_tab = C((1-R("az","AZ","09","%%"))^0) - --- we've split all cases as then we can optimize them (let's omit the fuzzy u) - -local format_s = function(f) - n = n + 1 - if f and f ~= "" then - return format("format('%%%ss',(select(%s,...)))",f,n) - else - return format("(select(%s,...))",n) - end -end - -local format_q = function() - n = n + 1 - return format("format('%%q',(select(%s,...)))",n) -- maybe an own lpeg -end - -local format_i = function(f) - n = n + 1 - if f and f ~= "" then - return format("format('%%%si',(select(%s,...)))",f,n) - else - return format("(select(%s,...))",n) - end -end - -local format_d = format_i - -local format_f = function(f) - n = n + 1 - return format("format('%%%sf',(select(%s,...)))",f,n) -end - -local format_g = function(f) - n = n + 1 - return format("format('%%%sg',(select(%s,...)))",f,n) -end - -local format_G = function(f) - n = n + 1 - return format("format('%%%sG',(select(%s,...)))",f,n) -end - -local format_e = function(f) - n = n + 1 - return format("format('%%%se',(select(%s,...)))",f,n) -end - -local format_E = function(f) - n = n + 1 - return format("format('%%%sE',(select(%s,...)))",f,n) -end - -local format_x = function(f) - n = n + 1 - return format("format('%%%sx',(select(%s,...)))",f,n) -end - -local format_X = function(f) - n = n + 1 - return format("format('%%%sX',(select(%s,...)))",f,n) -end - -local format_o = function(f) - n = n + 1 - return format("format('%%%so',(select(%s,...)))",f,n) -end - -local format_c = function() - n = n + 1 - return format("utfchar((select(%s,...)))",n) -end - -local format_r = function(f) - n = n + 1 - return format("format('%%%s.0f',(select(%s,...)))",f,n) -end - -local format_v = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +function table.serialize(root,name,specification) + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + end + serialize(flush,root,name,specification) + return concat(t,"\n") +end +table.tohandle=serialize +local maxtab=2*1024 +function table.tofile(filename,root,name,specification) + local f=io.open(filename,'w') + if f then + if maxtab>1 then + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + if n>maxtab then + f:write(concat(t,"\n"),"\n") + t,n={},0 + end + end + serialize(flush,root,name,specification) + f:write(concat(t,"\n"),"\n") else - return format("format('0x%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) + local function flush(s) + f:write(s,"\n") + end + serialize(flush,root,name,specification) end + f:close() + io.flush() + end end - -local format_V = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +local function flattened(t,f,depth) + if f==nil then + f={} + depth=0xFFFF + elseif tonumber(f) then + depth=f + f={} + elseif not depth then + depth=0xFFFF + end + for k,v in next,t do + if type(k)~="number" then + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + else + f[k]=v + end + end + end + local n=#f + for k=1,#t do + local v=t[k] + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + n=#f + else + n=n+1 + f[n]=v + end + end + return f +end +table.flattened=flattened +local function unnest(t,f) + if not f then + f={} + end + for i=1,#t do + local v=t[i] + if type(v)=="table" then + if type(v[1])=="table" then + unnest(v,f) + else + f[#f+1]=v + end else - return format("format('0x%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) + f[#f+1]=v end + end + return f end - -local format_u = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +function table.unnest(t) + return unnest(t) +end +local function are_equal(a,b,n,m) + if a and b and #a==#b then + n=n or 1 + m=m or #a + for i=n,m do + local ai,bi=a[i],b[i] + if ai==bi then + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end + else + return false + end + end + return true + else + return false + end +end +local function identical(a,b) + for ka,va in next,a do + local vb=b[ka] + if va==vb then + elseif type(va)=="table" and type(vb)=="table" then + if not identical(va,vb) then + return false + end else - return format("format('u+%%%sx',utfbyte((select(%s,...))))",f == "" and "05" or f,n) + return false end + end + return true end - -local format_U = function(f) - n = n + 1 - if f == "-" then - f = sub(f,2) - return format("format('%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) - else - return format("format('U+%%%sX',utfbyte((select(%s,...))))",f == "" and "05" or f,n) +table.identical=identical +table.are_equal=are_equal +function table.compact(t) + if t then + for k,v in next,t do + if not next(v) then + t[k]=nil + end end + end end - -local format_p = function() - n = n + 1 - return format("points((select(%s,...)))",n) +function table.contains(t,v) + if t then + for i=1,#t do + if t[i]==v then + return i + end + end + end + return false end - -local format_b = function() - n = n + 1 - return format("basepoints((select(%s,...)))",n) +function table.count(t) + local n=0 + for k,v in next,t do + n=n+1 + end + return n +end +function table.swapped(t,s) + local n={} + if s then + for k,v in next,s do + n[k]=v + end + end + for k,v in next,t do + n[v]=k + end + return n +end +function table.mirrored(t) + local n={} + for k,v in next,t do + n[v]=k + n[k]=v + end + return n end - -local format_t = function(f) - n = n + 1 - if f and f ~= "" then - return format("concat((select(%s,...)),%q)",n,f) - else - return format("concat((select(%s,...)))",n) +function table.reversed(t) + if t then + local tt,tn={},#t + if tn>0 then + local ttn=0 + for i=tn,1,-1 do + ttn=ttn+1 + tt[ttn]=t[i] + end end + return tt + end end - -local format_l = function() - n = n + 1 - return format("(select(%s,...) and 'true' or 'false')",n) +function table.reverse(t) + if t then + local n=#t + for i=1,floor(n/2) do + local j=n-i+1 + t[i],t[j]=t[j],t[i] + end + return t + end +end +function table.sequenced(t,sep) + if t then + local s,n={},0 + for k,v in sortedhash(t) do + if simple then + if v==true then + n=n+1 + s[n]=k + elseif v and v~="" then + n=n+1 + s[n]=k.."="..tostring(v) + end + else + n=n+1 + s[n]=k.."="..tostring(v) + end + end + return concat(s,sep or " | ") + else + return "" + end end - -local format_a = function(s) - return format("%q",s) +function table.print(t,...) + if type(t)~="table" then + print(tostring(t)) + else + table.tohandle(print,t,...) + end end - -local builder = Ct { "start", - start = (P("%") * ( - V("s") + V("q") - + V("i") + V("d") - + V("f") + V("g") + V("G") + V("e") + V("E") - + V("x") + V("X") + V("o") - -- - + V("c") - -- - + V("r") - + V("v") + V("V") + V("u") + V("U") - + V("p") + V("b") - + V("t") - + V("l") - ) - + V("a") - )^0, - -- - ["s"] = (prefix_any * P("s")) / format_s, -- %s => regular %s (string) - ["q"] = (prefix_any * P("q")) / format_q, -- %q => regular %q (quoted string) - ["i"] = (prefix_any * P("i")) / format_i, -- %i => regular %i (integer) - ["d"] = (prefix_any * P("d")) / format_d, -- %d => regular %d (integer) - ["f"] = (prefix_any * P("f")) / format_f, -- %f => regular %f (float) - ["g"] = (prefix_any * P("g")) / format_g, -- %g => regular %g (float) - ["G"] = (prefix_any * P("G")) / format_G, -- %G => regular %G (float) - ["e"] = (prefix_any * P("e")) / format_e, -- %e => regular %e (float) - ["E"] = (prefix_any * P("E")) / format_E, -- %E => regular %E (float) - ["x"] = (prefix_any * P("x")) / format_x, -- %x => regular %x (hexadecimal) - ["X"] = (prefix_any * P("X")) / format_X, -- %X => regular %X (HEXADECIMAL) - ["o"] = (prefix_any * P("o")) / format_o, -- %o => regular %o (octal) - -- - ["c"] = (prefix_any * P("c")) / format_c, -- %c => utf character (extension to regular) - -- - ["r"] = (prefix_any * P("r")) / format_r, -- %r => round - ["v"] = (prefix_any * P("v")) / format_v, -- %v => 0x0a1b2 (when - no 0x) - ["V"] = (prefix_any * P("V")) / format_V, -- %V => 0x0A1B2 (when - no 0x) - ["u"] = (prefix_any * P("u")) / format_u, -- %u => u+0a1b2 (when - no u+) - ["U"] = (prefix_any * P("U")) / format_U, -- %U => U+0A1B2 (when - no U+) - ["p"] = (prefix_any * P("p")) / format_p, -- %p => 12.345pt / maybe: P (and more units) - ["b"] = (prefix_any * P("b")) / format_b, -- %b => 12.342bp / maybe: B (and more units) - ["t"] = (prefix_tab * P("t")) / format_t, -- %t => concat - ["l"] = (prefix_tab * P("l")) / format_l, -- %l => boolean - -- - ["a"] = Cs(((1-P("%"))^1 + P("%%")/"%%")^1) / format_a, -- %a => text (including %%) -} - --- we can be clever and only alias what is needed - -local template = [[ -local format = string.format -local concat = table.concat -local points = number.points -local basepoints = number.basepoints -local utfchar = utf.char -local utfbyte = utf.byte -return function(...) - return %s +function table.sub(t,i,j) + return { unpack(t,i,j) } end -]] - -local function make(t,str) - n = 0 - local p = lpegmatch(builder,str) --- inspect(p) - local c = format(template,concat(p,"..")) --- inspect(c) - formatter = load(c)() - t[str] = formatter - return formatter +function table.is_empty(t) + return not t or not next(t) end - -local formatters = string.formatters or { } -string.formatters = formatters - -setmetatableindex(formatters,make) - -function string.makeformatter(str) - return formatters[str] +function table.has_one_entry(t) + return t and not next(t,next(t)) end - -function string.formatter(str,...) - return formatters[str](...) +function table.loweredkeys(t) + local l={} + for k,v in next,t do + l[lower(k)]=v + end + return l +end +function table.unique(old) + local hash={} + local new={} + local n=0 + for i=1,#old do + local oi=old[i] + if not hash[oi] then + n=n+1 + new[n]=oi + hash[oi]=true + end + end + return new +end +function table.sorted(t,...) + sort(t,...) + return t end - --- local p1 = "%s test %f done %p and %c and %V or %+t or %%" --- local p2 = "%s test %f done %s and %s and 0x%05X or %s or %%" --- --- local t = { 1,2,3,4 } --- local r = "" --- --- local format, formatter, formatters = string.format, string.formatter, string.formatters --- local utfchar, utfbyte, concat, points = utf.char, utf.byte, table.concat, number.points --- --- local c = os.clock() --- local f = formatters[p1] --- for i=1,500000 do --- -- r = formatters[p1]("hans",123.45,123.45,123,"a",t) --- r = formatter(p1,"hans",123.45,123.45,123,"a",t) --- -- r = f("hans",123.45,123.45,123,"a",t) --- end --- print(os.clock()-c,r) --- --- local c = os.clock() --- for i=1,500000 do --- r = format(p2,"hans",123.45,points(123.45),utfchar(123),utfbyte("a"),concat(t,"+")) --- end --- print(os.clock()-c,r) - --- local f = format --- function string.format(fmt,...) --- print(fmt,...) --- return f(fmt,...) --- end end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-mrg'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- hm, quite unreadable - -local gsub, format = string.gsub, string.format -local concat = table.concat -local type, next = type, next +-- original size: 8723, stripped down to: 6325 -utilities = utilities or {} -local merger = utilities.merger or { } -utilities.merger = merger -utilities.report = logs and logs.reporter("system") or print - -merger.strip_comment = true - -local m_begin_merge = "begin library merge" -local m_end_merge = "end library merge" -local m_begin_closure = "do -- create closure to overcome 200 locals limit" -local m_end_closure = "end -- of closure" - -local m_pattern = - "%c+" .. - "%-%-%s+" .. m_begin_merge .. - "%c+(.-)%c+" .. - "%-%-%s+" .. m_end_merge .. - "%c+" - -local m_format = - "\n\n-- " .. m_begin_merge .. - "\n%s\n" .. - "-- " .. m_end_merge .. "\n\n" - -local m_faked = - "-- " .. "created merged file" .. "\n\n" .. - "-- " .. m_begin_merge .. "\n\n" .. - "-- " .. m_end_merge .. "\n\n" - -local function self_fake() - return m_faked +if not modules then modules={} end modules ['l-io']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local io=io +local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format +local concat=table.concat +local floor=math.floor +local type=type +if string.find(os.getenv("PATH"),";") then + io.fileseparator,io.pathseparator="\\",";" +else + io.fileseparator,io.pathseparator="/",":" end - -local function self_nothing() +local function readall(f) + return f:read("*all") +end +local function readall(f) + local size=f:seek("end") + if size==0 then return "" + elseif size<1024*1024 then + f:seek("set",0) + return f:read('*all') + else + local done=f:seek("set",0) + if size<1024*1024 then + step=1024*1024 + elseif size>16*1024*1024 then + step=16*1024*1024 + else + step=floor(size/(1024*1024))*1024*1024/8 + end + local data={} + while true do + local r=f:read(step) + if not r then + return concat(data) + else + data[#data+1]=r + end + end + end end - -local function self_load(name) - local data = io.loaddata(name) or "" - if data == "" then - utilities.report("merge: unknown file %s",name) +io.readall=readall +function io.loaddata(filename,textmode) + local f=io.open(filename,(textmode and 'r') or 'rb') + if f then + local data=readall(f) + f:close() + if #data>0 then + return data + end + end +end +function io.savedata(filename,data,joiner) + local f=io.open(filename,"wb") + if f then + if type(data)=="table" then + f:write(concat(data,joiner or "")) + elseif type(data)=="function" then + data(f) else - utilities.report("merge: inserting %s",name) + f:write(data or "") + end + f:close() + io.flush() + return true + else + return false + end +end +function io.loadlines(filename,n) + local f=io.open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[#lines+1]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines end - return data or "" + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end + end end - -local function self_save(name, data) - if data ~= "" then - if merger.strip_comment then - local n = #data - -- saves some 20K .. scite comments - data = gsub(data,"%-%-~[^\n\r]*[\r\n]","") - -- saves some 20K .. ldx comments - data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-ldx%]%]%-%-","") - utilities.report("merge: %s bytes of comment stripped, %s bytes of code left",n-#data,#data) - end - io.savedata(name,data) - utilities.report("merge: saving %s",name) +function io.loadchunk(filename,n) + local f=io.open(filename,'rb') + if f then + local data=f:read(n or 1024) + f:close() + if #data>0 then + return data end + end end - -local function self_swap(data,code) - return data ~= "" and (gsub(data,m_pattern, function() return format(m_format,code) end, 1)) or "" +function io.exists(filename) + local f=io.open(filename) + if f==nil then + return false + else + f:close() + return true + end end - -local function self_libs(libs,list) - local result, f, frozen, foundpath = { }, nil, false, nil - result[#result+1] = "\n" - if type(libs) == 'string' then libs = { libs } end - if type(list) == 'string' then list = { list } end - for i=1,#libs do - local lib = libs[i] - for j=1,#list do - local pth = gsub(list[j],"\\","/") -- file.clean_path - utilities.report("merge: checking library path %s",pth) - local name = pth .. "/" .. lib - if lfs.isfile(name) then - foundpath = pth - end - end - if foundpath then break end - end - if foundpath then - utilities.report("merge: using library path %s",foundpath) - local right, wrong = { }, { } - for i=1,#libs do - local lib = libs[i] - local fullname = foundpath .. "/" .. lib - if lfs.isfile(fullname) then - utilities.report("merge: using library %s",fullname) - right[#right+1] = lib - result[#result+1] = m_begin_closure - result[#result+1] = io.loaddata(fullname,true) - result[#result+1] = m_end_closure - else - utilities.report("merge: skipping library %s",fullname) - wrong[#wrong+1] = lib - end - end - if #right > 0 then - utilities.report("merge: used libraries: %s",concat(right," ")) - end - if #wrong > 0 then - utilities.report("merge: skipped libraries: %s",concat(wrong," ")) - end +function io.size(filename) + local f=io.open(filename) + if f==nil then + return 0 + else + local s=f:seek("end") + f:close() + return s + end +end +function io.noflines(f) + if type(f)=="string" then + local f=io.open(filename) + if f then + local n=f and io.noflines(f) or 0 + f:close() + return n else - utilities.report("merge: no valid library path found") + return 0 + end + else + local n=0 + for _ in f:lines() do + n=n+1 end - return concat(result, "\n\n") + f:seek('set',0) + return n + end +end +local nextchar={ + [ 4]=function(f) + return f:read(1,1,1,1) + end, + [ 2]=function(f) + return f:read(1,1) + end, + [ 1]=function(f) + return f:read(1) + end, + [-2]=function(f) + local a,b=f:read(1,1) + return b,a + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + return d,c,b,a + end +} +function io.characters(f,n) + if f then + return nextchar[n or 1],f + end end - -function merger.selfcreate(libs,list,target) - if target then - self_save(target,self_swap(self_fake(),self_libs(libs,list))) +local nextbyte={ + [4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(a),byte(b),byte(c),byte(d) + end + end, + [3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(a),byte(b),byte(c) + end + end, + [2]=function(f) + local a,b=f:read(1,1) + if b then + return byte(a),byte(b) + end + end, + [1]=function (f) + local a=f:read(1) + if a then + return byte(a) + end + end, + [-2]=function (f) + local a,b=f:read(1,1) + if b then + return byte(b),byte(a) + end + end, + [-3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(c),byte(b),byte(a) + end + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(d),byte(c),byte(b),byte(a) end + end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1],f + else + return nil,nil + end end - -function merger.selfmerge(name,libs,list,target) - self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) +function io.ask(question,default,options) + while true do + io.write(question) + if options then + io.write(format(" [%s]",concat(options,"|"))) + end + if default then + io.write(format(" [%s]",default)) + end + io.write(format(" ")) + io.flush() + local answer=io.read() + answer=gsub(answer,"^%s*(.*)%s*$","%1") + if answer=="" and default then + return default + elseif not options then + return answer + else + for k=1,#options do + if options[k]==answer then + return answer + end + end + local pattern="^"..answer + for k=1,#options do + local v=options[k] + if find(v,pattern) then + return v + end + end + end + end end - -function merger.selfclean(name) - self_save(name,self_swap(self_load(name),self_nothing())) +local function readnumber(f,n,m) + if m then + f:seek("set",n) + n=m + end + if n==1 then + return byte(f:read(1)) + elseif n==2 then + local a,b=byte(f:read(2),1,2) + return 256*a+b + elseif n==3 then + local a,b,c=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==4 then + local a,b,c,d=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==8 then + local a,b=readnumber(f,4),readnumber(f,4) + return 256*a+b + elseif n==12 then + local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) + return 256*256*a+256*b+c + elseif n==-2 then + local b,a=byte(f:read(2),1,2) + return 256*a+b + elseif n==-3 then + local c,b,a=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==-4 then + local d,c,b,a=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==-8 then + local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) + return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + else + return 0 + end end +io.readnumber=readnumber +function io.readstring(f,n,m) + if m then + f:seek("set",n) + n=m + end + local str=gsub(f:read(n),"\000","") + return str +end +if not io.i_limiter then function io.i_limiter() end end +if not io.o_limiter then function io.o_limiter() end end end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - comment = "the strip code is written by Peter Cawley", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +-- original size: 4851, stripped down to: 2828 -local rep, sub, byte, dump, format = string.rep, string.sub, string.byte, string.dump, string.format -local load, loadfile, type = load, loadfile, type - -utilities = utilities or {} -utilities.lua = utilities.lua or { } -local luautilities = utilities.lua - -utilities.report = logs and logs.reporter("system") or print -- can be overloaded later - -local tracestripping = false -local forcestupidcompile = true -- use internal bytecode compiler -luautilities.stripcode = true -- support stripping when asked for -luautilities.alwaysstripcode = false -- saves 1 meg on 7 meg compressed format file (2012.08.12) -luautilities.nofstrippedchunks = 0 -luautilities.nofstrippedbytes = 0 -local strippedchunks = { } -- allocate() -luautilities.strippedchunks = strippedchunks - -luautilities.suffixes = { - tma = "tma", - tmc = jit and "tmb" or "tmc", - lua = "lua", - luc = jit and "lub" or "luc", - lui = "lui", - luv = "luv", - luj = "luj", - tua = "tua", - tuc = "tuc", +if not modules then modules={} end modules ['l-number']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } - -local function fatalerror(name) - utilities.report(format("fatal error in %q",name or "unknown")) +local tostring,tonumber=tostring,tonumber +local format,floor,match,rep=string.format,math.floor,string.match,string.rep +local concat,insert=table.concat,table.insert +local lpegmatch=lpeg.match +number=number or {} +local number=number +if bit32 then + local btest,bor=bit32.btest,bit32.bor + function number.bit(p) + return 2^(p-1) + end + number.hasbit=btest + number.setbit=bor + function number.setbit(x,p) + return btest(x,p) and x or x+p + end + function number.clearbit(x,p) + return btest(x,p) and x-p or x + end +else + function number.bit(p) + return 2^(p-1) + end + function number.hasbit(x,p) + return x%(p+p)>=p + end + function number.setbit(x,p) + return (x%(p+p)>=p) and x or x+p + end + function number.clearbit(x,p) + return (x%(p+p)>=p) and x-p or x + end end - -if jit or status.luatex_version >= 74 then - - local function register(name) - if tracestripping then - utilities.report("stripped bytecode: %s",name or "unknown") - end - strippedchunks[#strippedchunks+1] = name - luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1 - end - - local function stupidcompile(luafile,lucfile,strip) - local code = io.loaddata(luafile) - if code and code ~= "" then - code = load(code) - if code then - code = dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) - if code and code ~= "" then - register(name) - io.savedata(lucfile,code) - return true, 0 - end - else - fatalerror() - end - else - fatalerror() - end - return false, 0 - end - - -- quite subtle ... doing this wrong incidentally can give more bytes - - function luautilities.loadedluacode(fullname,forcestrip,name) - -- quite subtle ... doing this wrong incidentally can give more bytes - name = name or fullname - local code = loadfile(fullname) - if code then - code() - end - if forcestrip and luautilities.stripcode then - if type(forcestrip) == "function" then - forcestrip = forcestrip(fullname) - end - if forcestrip or luautilities.alwaysstripcode then - register(name) - return load(dump(code,true)), 0 - else - return code, 0 - end - elseif luautilities.alwaysstripcode then - register(name) - return load(dump(code,true)), 0 - else - return code, 0 - end +if bit32 then + local bextract=bit32.extract + local t={ + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + } + function number.tobitstring(b,m) + local n=32 + for i=0,31 do + local v=bextract(b,i) + local k=32-i + if v==1 then + n=k + t[k]="1" + else + t[k]="0" + end end - - function luautilities.strippedloadstring(code,forcestrip,name) -- not executed - if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then - code = load(code) - if not code then - fatalerror(name) - end - register(name) - code = dump(code,true) - end - return load(code), 0 - end - - function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true - utilities.report("lua: compiling %s into %s",luafile,lucfile) - os.remove(lucfile) - local done = stupidcompile(luafile,lucfile,strip ~= false) - if done then - utilities.report("lua: %s dumped into %s (stripped)",luafile,lucfile) - if cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - utilities.report("lua: removing %s",luafile) - os.remove(luafile) - end - end - return done + if m then + m=33-m*8 + if m<1 then + m=1 + end + return concat(t,"",m) + elseif n<8 then + return concat(t) + elseif n<16 then + return concat(t,"",9) + elseif n<24 then + return concat(t,"",17) + else + return concat(t,"",25) end - + end else - - -- The next function was posted by Peter Cawley on the lua list and strips line - -- number information etc. from the bytecode data blob. We only apply this trick - -- when we store data tables. Stripping makes the compressed format file about - -- 1MB smaller (and uncompressed we save at least 6MB). - -- - -- You can consider this feature an experiment, so it might disappear. There is - -- no noticeable gain in runtime although the memory footprint should be somewhat - -- smaller (and the file system has a bit less to deal with). - -- - -- Begin of borrowed code ... works for Lua 5.1 which LuaTeX currently uses ... - - local function register(name,before,after) - local delta = before - after - if tracestripping then - utilities.report("stripped bytecode: %s, before %s, after %s, delta %s",name or "unknown",before,after,delta) + function number.tobitstring(n,m) + if n>0 then + local t={} + while n>0 do + insert(t,1,n%2>0 and 1 or 0) + n=floor(n/2) + end + local nn=8-#t%8 + if nn>0 and nn<8 then + for i=1,nn do + insert(t,1,0) end - strippedchunks[#strippedchunks+1] = name - luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1 - luautilities.nofstrippedbytes = luautilities.nofstrippedbytes + delta - return delta - end - - local strip_code_pc - - if _MAJORVERSION == 5 and _MINORVERSION == 1 then - - strip_code_pc = function(dump,name) - local before = #dump - local version, format, endian, int, size, ins, num = byte(dump,5,11) - local subint - if endian == 1 then - subint = function(dump, i, l) - local val = 0 - for n = l, 1, -1 do - val = val * 256 + byte(dump,i + n - 1) - end - return val, i + l - end - else - subint = function(dump, i, l) - local val = 0 - for n = 1, l, 1 do - val = val * 256 + byte(dump,i + n - 1) - end - return val, i + l - end - end - local strip_function - strip_function = function(dump) - local count, offset = subint(dump, 1, size) - local stripped, dirty = rep("\0", size), offset + count - offset = offset + count + int * 2 + 4 - offset = offset + int + subint(dump, offset, int) * ins - count, offset = subint(dump, offset, int) - for n = 1, count do - local t - t, offset = subint(dump, offset, 1) - if t == 1 then - offset = offset + 1 - elseif t == 4 then - offset = offset + size + subint(dump, offset, size) - elseif t == 3 then - offset = offset + num - end - end - count, offset = subint(dump, offset, int) - stripped = stripped .. sub(dump,dirty, offset - 1) - for n = 1, count do - local proto, off = strip_function(sub(dump,offset, -1)) - stripped, offset = stripped .. proto, offset + off - 1 - end - offset = offset + subint(dump, offset, int) * int + int - count, offset = subint(dump, offset, int) - for n = 1, count do - offset = offset + subint(dump, offset, size) + size + int * 2 - end - count, offset = subint(dump, offset, int) - for n = 1, count do - offset = offset + subint(dump, offset, size) + size - end - stripped = stripped .. rep("\0", int * 3) - return stripped, offset - end - dump = sub(dump,1,12) .. strip_function(sub(dump,13,-1)) - local after = #dump - local delta = register(name,before,after) - return dump, delta + end + if m then + m=m*8-#t + if m>0 then + insert(t,1,rep("0",m)) end - + end + return concat(t) + elseif m then + rep("00000000",m) else - - strip_code_pc = function(dump,name) - return dump, 0 - end - - end - - -- ... end of borrowed code. - - -- quite subtle ... doing this wrong incidentally can give more bytes - - function luautilities.loadedluacode(fullname,forcestrip,name) - -- quite subtle ... doing this wrong incidentally can give more bytes - name = name or fullname - local code = loadfile(fullname) - if code then - code() - end - if forcestrip and luautilities.stripcode then - if type(forcestrip) == "function" then - forcestrip = forcestrip(fullname) - end - if forcestrip then - local code, n = strip_code_pc(dump(code),name) - return load(code), n - elseif luautilities.alwaysstripcode then - return load(strip_code_pc(dump(code),name)) - else - return code, 0 - end - elseif luautilities.alwaysstripcode then - return load(strip_code_pc(dump(code),name)) - else - return code, 0 - end - end - - function luautilities.strippedloadstring(code,forcestrip,name) -- not executed - local n = 0 - if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then - code = load(code) - if not code then - fatalerror(name) - end - code, n = strip_code_pc(dump(code),name) - end - return load(code), n - end - - local function stupidcompile(luafile,lucfile,strip) - local code = io.loaddata(luafile) - local n = 0 - if code and code ~= "" then - code = load(code) - if not code then - fatalerror() - end - code = dump(code) - if strip then - code, n = strip_code_pc(code,luautilities.stripcode or luautilities.alwaysstripcode,luafile) -- last one is reported - end - if code and code ~= "" then - io.savedata(lucfile,code) - end - end - return n + return "00000000" end - - local luac_normal = "texluac -o %q %q" - local luac_strip = "texluac -s -o %q %q" - - function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true - utilities.report("lua: compiling %s into %s",luafile,lucfile) - os.remove(lucfile) - local done = false - if strip ~= false then - strip = true - end - if forcestupidcompile then - fallback = true - elseif strip then - done = os.spawn(format(luac_strip, lucfile,luafile)) == 0 - else - done = os.spawn(format(luac_normal,lucfile,luafile)) == 0 - end - if not done and fallback then - local n = stupidcompile(luafile,lucfile,strip) - if n > 0 then - utilities.report("lua: %s dumped into %s (%i bytes stripped)",luafile,lucfile,n) - else - utilities.report("lua: %s dumped into %s (unstripped)",luafile,lucfile) - end - cleanup = false -- better see how bad it is - done = true -- hm - end - if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - utilities.report("lua: removing %s",luafile) - os.remove(luafile) - end - return done + end +end +function number.valid(str,default) + return tonumber(str) or default or nil +end +function number.toevenhex(n) + local s=format("%X",n) + if #s%2==0 then + return s + else + return "0"..s + end +end +local one=lpeg.C(1-lpeg.S('')/tonumber)^1 +function number.toset(n) + return lpegmatch(one,tostring(n)) +end +local function bits(n,i,...) + if n>0 then + local m=n%2 + local n=floor(n/2) + if m>0 then + return bits(n,i+1,i,...) + else + return bits(n,i+1,...) end - + else + return... + end +end +function number.bits(n) + return { bits(n,1) } end - --- local getmetatable, type = getmetatable, type --- --- local types = { } --- --- function luautilities.registerdatatype(d,name) --- types[getmetatable(d)] = name --- end --- --- function luautilities.datatype(d) --- local t = type(d) --- if t == "userdata" then --- local m = getmetatable(d) --- return m and types[m] or "userdata" --- else --- return t --- end --- end --- --- luautilities.registerdatatype(lpeg.P("!"),"lpeg") --- --- print(luautilities.datatype(lpeg.P("oeps"))) end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-prs'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +-- original size: 1923, stripped down to: 1133 -local lpeg, table, string = lpeg, table, string -local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf, Cp = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.Cp -local lpegmatch, patterns = lpeg.match, lpeg.patterns -local concat, format, gmatch, find = table.concat, string.format, string.gmatch, string.find -local tostring, type, next, rawset = tostring, type, next, rawset - -utilities = utilities or {} -utilities.parsers = utilities.parsers or { } -local parsers = utilities.parsers -parsers.patterns = parsers.patterns or { } - -local setmetatableindex = table.setmetatableindex -local sortedhash = table.sortedhash - --- we share some patterns - -local digit = R("09") -local space = P(' ') -local equal = P("=") -local comma = P(",") -local lbrace = P("{") -local rbrace = P("}") -local lparent = P("(") -local rparent = P(")") -local period = S(".") -local punctuation = S(".,:;") -local spacer = patterns.spacer -local whitespace = patterns.whitespace -local newline = patterns.newline -local anything = patterns.anything -local endofstring = patterns.endofstring - -local nobrace = 1 - ( lbrace + rbrace ) -local noparent = 1 - ( lparent + rparent) - --- we could use a Cf Cg construct - -local escape, left, right = P("\\"), P('{'), P('}') - -patterns.balanced = P { - [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, - [2] = left * V(1) * right +if not modules then modules={} end modules ['l-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } +set=set or {} +local nums={} +local tabs={} +local concat=table.concat +local next,type=next,type +set.create=table.tohash +function set.tonumber(t) + if next(t) then + local s="" + for k,v in next,t do + if v then + s=s.." "..k + end + end + local n=nums[s] + if not n then + n=#tabs+1 + tabs[n]=t + nums[s]=n + end + return n + else + return 0 + end +end +function set.totable(n) + if n==0 then + return {} + else + return tabs[n] or {} + end +end +function set.tolist(n) + if n==0 or not tabs[n] then + return "" + else + local t,n={},0 + for k,v in next,tabs[n] do + if v then + n=n+1 + t[n]=k + end + end + return concat(t," ") + end +end +function set.contains(n,s) + if type(n)=="table" then + return n[s] + elseif n==0 then + return false + else + local t=tabs[n] + return t and t[s] + end +end -local nestedbraces = P { lbrace * (nobrace + V(1))^0 * rbrace } -local nestedparents = P { lparent * (noparent + V(1))^0 * rparent } -local spaces = space^0 -local argument = Cs((lbrace/"") * ((nobrace + nestedbraces)^0) * (rbrace/"")) -local content = (1-endofstring)^0 - -patterns.nestedbraces = nestedbraces -- no capture -patterns.nestedparents = nestedparents -- no capture -patterns.nested = nestedbraces -- no capture -patterns.argument = argument -- argument after e.g. = -patterns.content = content -- rest after e.g = - -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) + C((nestedbraces + (1-comma))^0) - -local key = C((1-equal-comma)^1) -local pattern_a = (space+comma)^0 * (key * equal * value + key * C("")) -local pattern_c = (space+comma)^0 * (key * equal * value) - -local key = C((1-space-equal-comma)^1) -local pattern_b = spaces * comma^0 * spaces * (key * ((spaces * equal * spaces * value) + C(""))) - --- "a=1, b=2, c=3, d={a{b,c}d}, e=12345, f=xx{a{b,c}d}xx, g={}" : outer {} removes, leading spaces ignored - --- todo: rewrite to fold etc --- --- parse = lpeg.Cf(lpeg.Carg(1) * lpeg.Cg(key * equal * value) * separator^0,rawset)^0 -- lpeg.match(parse,"...",1,hash) - -local hash = { } -local function set(key,value) - hash[key] = value -end +end -- of closure -local pattern_a_s = (pattern_a/set)^1 -local pattern_b_s = (pattern_b/set)^1 -local pattern_c_s = (pattern_c/set)^1 +do -- create closure to overcome 200 locals limit -parsers.patterns.settings_to_hash_a = pattern_a_s -parsers.patterns.settings_to_hash_b = pattern_b_s -parsers.patterns.settings_to_hash_c = pattern_c_s +-- original size: 13731, stripped down to: 8450 -function parsers.make_settings_to_hash_pattern(set,how) - if how == "strict" then - return (pattern_c/set)^1 - elseif how == "tolerant" then - return (pattern_b/set)^1 - else - return (pattern_a/set)^1 +if not modules then modules={} end modules ['l-os']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local os=os +local date,time=os.date,os.time +local find,format,gsub,upper,gmatch=string.find,string.format,string.gsub,string.upper,string.gmatch +local concat=table.concat +local random,ceil,randomseed=math.random,math.ceil,math.randomseed +local rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring=rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring +math.initialseed=tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6)) +randomseed(math.initialseed) +if not os.__getenv__ then + os.__getenv__=os.getenv + os.__setenv__=os.setenv + if os.env then + local osgetenv=os.getenv + local ossetenv=os.setenv + local osenv=os.env local _=osenv.PATH + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + if type(v)=="table" then + v=concat(v,";") + end + ossetenv(K,v) end -end - -function parsers.settings_to_hash(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_a_s,str) - return hash - else - return { } + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + else + local ossetenv=os.setenv + local osgetenv=os.getenv + local osenv={} + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + end + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + local function __index(t,k) + return os.getenv(k) + end + local function __newindex(t,k,v) + os.setenv(k,v) end + os.env={} + setmetatable(os.env,{ __index=__index,__newindex=__newindex } ) + end end - -function parsers.settings_to_hash_tolerant(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_b_s,str) - return hash +local execute,spawn,exec,iopopen,ioflush=os.execute,os.spawn or os.execute,os.exec or os.execute,io.popen,io.flush +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end +function io.popen (...) ioflush() return iopopen(...) end +function os.resultof(command) + local handle=io.popen(command,"r") + return handle and handle:read("*all") or "" +end +if not io.fileseparator then + if find(os.getenv("PATH"),";") then + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + else + io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" + end +end +os.type=os.type or (io.pathseparator==";" and "windows") or "unix" +os.name=os.name or (os.type=="windows" and "mswin" ) or "linux" +if os.type=="windows" then + os.libsuffix,os.binsuffix,os.binsuffixes='dll','exe',{ 'exe','cmd','bat' } +else + os.libsuffix,os.binsuffix,os.binsuffixes='so','',{ '' } +end +local launchers={ + windows="start %s", + macosx="open %s", + unix="$BROWSER %s &> /dev/null &", +} +function os.launch(str) + os.execute(format(launchers[os.name] or launchers.unix,str)) +end +if not os.times then + function os.times() + return { + utime=os.gettimeofday(), + stime=0, + cutime=0, + cstime=0, + } + end +end +os.gettimeofday=os.gettimeofday or os.clock +local startuptime=os.gettimeofday() +function os.runtime() + return os.gettimeofday()-startuptime +end +os.resolvers=os.resolvers or {} +local resolvers=os.resolvers +local osmt=getmetatable(os) or { __index=function(t,k) t[k]="unset" return "unset" end } +local osix=osmt.__index +osmt.__index=function(t,k) + return (resolvers[k] or osix)(t,k) +end +setmetatable(os,osmt) +local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" +local function guess() + local architecture=os.resultof("uname -m") or "" + if architecture~="" then + return architecture + end + architecture=os.getenv("HOSTTYPE") or "" + if architecture~="" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end +if platform~="" then + os.platform=platform +elseif os.type=="windows" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform="mswin-64" + else + platform="mswin" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="linux" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform="linux-64" + elseif find(architecture,"ppc") then + platform="linux-ppc" + else + platform="linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="macosx" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + if architecture=="" then + platform="osx-intel" + elseif find(architecture,"i386") then + platform="osx-intel" + elseif find(architecture,"x86_64") then + platform="osx-64" + else + platform="osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="sunos" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform="solaris-sparc" + else + platform="solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="freebsd" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform="freebsd-amd64" + else + platform="freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="kfreebsd" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform="kfreebsd-amd64" + else + platform="kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +else + function os.resolvers.platform(t,k) + local platform="linux" + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +end +local t={ 8,9,"a","b" } +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end +local d +function os.timezone(delta) + d=d or tonumber(tonumber(date("%H")-date("!%H"))) + if delta then + if d>0 then + return format("+%02i:00",d) else - return { } + return format("-%02i:00",-d) end + else + return 1 + end end - -function parsers.settings_to_hash_strict(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_c_s,str) - return next(hash) and hash - else - return nil +local timeformat=format("%%s%s",os.timezone(true)) +local dateformat="!%Y-%m-%d %H:%M:%S" +function os.fulltime(t,default) + t=tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=nil + end + return format(timeformat,date(dateformat,t)) +end +local dateformat="%Y-%m-%d %H:%M:%S" +function os.localtime(t,default) + t=tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=nil + end + return date(dateformat,t) +end +function os.converttime(t,default) + local t=tonumber(t) + if t and t>0 then + return date(dateformat,t) + else + return default or "-" + end +end +local memory={} +local function which(filename) + local fullname=memory[filename] + if fullname==nil then + local suffix=file.suffix(filename) + local suffixes=suffix=="" and os.binsuffixes or { suffix } + for directory in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local df=file.join(directory,filename) + for i=1,#suffixes do + local dfs=file.addsuffix(df,suffixes[i]) + if io.exists(dfs) then + fullname=dfs + break + end + end + end + if not fullname then + fullname=false end + memory[filename]=fullname + end + return fullname +end +os.which=which +os.where=which +function os.today() + return date("!*t") +end +function os.now() + return date("!%Y-%m-%d %H:%M:%S") end -local separator = comma * space^0 -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) - + C((nestedbraces + (1-comma))^0) -local pattern = spaces * Ct(value*(separator*value)^0) --- "aap, {noot}, mies" : outer {} removes, leading spaces ignored +end -- of closure -parsers.patterns.settings_to_array = pattern +do -- create closure to overcome 200 locals limit --- we could use a weak table as cache +-- original size: 15501, stripped down to: 8354 -function parsers.settings_to_array(str,strict) - if not str or str == "" then - return { } - elseif strict then - if find(str,"{") then - return lpegmatch(pattern,str) - else - return { str } +if not modules then modules={} end modules ['l-file']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +file=file or {} +local file=file +local insert,concat=table.insert,table.concat +local match=string.match +local lpegmatch=lpeg.match +local getcurrentdir,attributes=lfs.currentdir,lfs.attributes +local checkedsplit=string.checkedsplit +local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local colon=P(":") +local period=P(".") +local periods=P("..") +local fwslash=P("/") +local bwslash=P("\\") +local slashes=S("\\/") +local noperiod=1-period +local noslashes=1-slashes +local name=noperiod^1 +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=C((1-(slashes^1*noslashes^1*-1))^1)*P(1) +local function pathpart(name,default) + return name and lpegmatch(pattern,name) or default or "" +end +local pattern=(noslashes^0*slashes)^1*C(noslashes^1)*-1 +local function basename(name) + return name and lpegmatch(pattern,name) or name +end +local pattern=(noslashes^0*slashes^1)^0*Cs((1-suffix)^1)*suffix^0 +local function nameonly(name) + return name and lpegmatch(pattern,name) or name +end +local pattern=(noslashes^0*slashes)^0*(noperiod^1*period)^1*C(noperiod^1)*-1 +local function suffixonly(name) + return name and lpegmatch(pattern,name) or "" +end +file.pathpart=pathpart +file.basename=basename +file.nameonly=nameonly +file.suffixonly=suffixonly +file.suffix=suffixonly +file.dirname=pathpart +file.extname=suffixonly +local drive=C(R("az","AZ"))*colon +local path=C((noslashes^0*slashes)^0) +local suffix=period*C(P(1-period)^0*P(-1)) +local base=C((1-suffix)^0) +local rest=C(P(1)^0) +drive=drive+Cc("") +path=path+Cc("") +base=base+Cc("") +suffix=suffix+Cc("") +local pattern_a=drive*path*base*suffix +local pattern_b=path*base*suffix +local pattern_c=C(drive*path)*C(base*suffix) +local pattern_d=path*rest +function file.splitname(str,splitdrive) + if not str then + elseif splitdrive then + return lpegmatch(pattern_a,str) + else + return lpegmatch(pattern_b,str) + end +end +function file.splitbase(str) + return str and lpegmatch(pattern_d,str) +end +function file.nametotable(str,splitdrive) + if str then + local path,drive,subpath,name,base,suffix=lpegmatch(pattern_c,str) + if splitdrive then + return { + path=path, + drive=drive, + subpath=subpath, + name=name, + base=base, + suffix=suffix, + } + else + return { + path=path, + name=name, + base=base, + suffix=suffix, + } + end + end +end +local pattern=Cs(((period*noperiod^1*-1)/""+1)^1) +function file.removesuffix(name) + return name and lpegmatch(pattern,name) +end +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=Cs((noslashes^0*slashes^1)^0*((1-suffix)^1))*Cs(suffix) +function file.addsuffix(filename,suffix,criterium) + if not filename or not suffix or suffix=="" then + return filename + elseif criterium==true then + return filename.."."..suffix + elseif not criterium then + local n,s=lpegmatch(pattern,filename) + if not s or s=="" then + return filename.."."..suffix + else + return filename + end + else + local n,s=lpegmatch(pattern,filename) + if s and s~="" then + local t=type(criterium) + if t=="table" then + for i=1,#criterium do + if s==criterium[i] then + return filename + end end - else - return lpegmatch(pattern,str) + elseif t=="string" then + if s==criterium then + return filename + end + end end + return (n or filename).."."..suffix + end end - -local function set(t,v) - t[#t+1] = v +local suffix=period*(1-period-slashes)^1*-1 +local pattern=Cs((1-suffix)^0) +function file.replacesuffix(name,suffix) + if name and suffix and suffix~="" then + return lpegmatch(pattern,name).."."..suffix + else + return name + end end - -local value = P(Carg(1)*value) / set -local pattern = value*(separator*value)^0 * Carg(1) - -function parsers.add_settings_to_array(t,str) - return lpegmatch(pattern,str,nil,t) +local reslasher=lpeg.replacer(P("\\"),"/") +function file.reslash(str) + return str and lpegmatch(reslasher,str) end - -function parsers.hash_to_string(h,separator,yes,no,strict,omit) - if h then - local t, tn, s = { }, 0, table.sortedkeys(h) - omit = omit and table.tohash(omit) - for i=1,#s do - local key = s[i] - if not omit or not omit[key] then - local value = h[key] - if type(value) == "boolean" then - if yes and no then - if value then - tn = tn + 1 - t[tn] = key .. '=' .. yes - elseif not strict then - tn = tn + 1 - t[tn] = key .. '=' .. no - end - elseif value or not strict then - tn = tn + 1 - t[tn] = key .. '=' .. tostring(value) - end - else - tn = tn + 1 - t[tn] = key .. '=' .. value - end - end - end - return concat(t,separator or ",") - else - return "" +function file.is_writable(name) + if not name then + elseif lfs.isdir(name) then + name=name.."/m_t_x_t_e_s_t.tmp" + local f=io.open(name,"wb") + if f then + f:close() + os.remove(name) + return true + end + elseif lfs.isfile(name) then + local f=io.open(name,"ab") + if f then + f:close() + return true + end + else + local f=io.open(name,"ab") + if f then + f:close() + os.remove(name) + return true end + end + return false end - -function parsers.array_to_string(a,separator) - if a then - return concat(a,separator or ",") +local readable=P("r")*Cc(true) +function file.is_readable(name) + if name then + local a=attributes(name) + return a and lpegmatch(readable,a.permissions) or false + else + return false + end +end +file.isreadable=file.is_readable +file.iswritable=file.is_writable +function file.size(name) + if name then + local a=attributes(name) + return a and a.size or 0 + else + return 0 + end +end +function file.splitpath(str,separator) + return str and checkedsplit(lpegmatch(reslasher,str),separator or io.pathseparator) +end +function file.joinpath(tab,separator) + return tab and concat(tab,separator or io.pathseparator) +end +local stripper=Cs(P(fwslash)^0/""*reslasher) +local isnetwork=fwslash*fwslash*(1-fwslash)+(1-fwslash-colon)^1*colon +local isroot=fwslash^1*-1 +local hasroot=fwslash^1 +local deslasher=lpeg.replacer(S("\\/")^1,"/") +function file.join(...) + local lst={... } + local one=lst[1] + if lpegmatch(isnetwork,one) then + local two=lpegmatch(deslasher,concat(lst,"/",2)) + return one.."/"..two + elseif lpegmatch(isroot,one) then + local two=lpegmatch(deslasher,concat(lst,"/",2)) + if lpegmatch(hasroot,two) then + return two + else + return "/"..two + end + elseif one=="" then + return lpegmatch(stripper,concat(lst,"/",2)) + else + return lpegmatch(deslasher,concat(lst,"/")) + end +end +local drivespec=R("az","AZ")^1*colon +local anchors=fwslash+drivespec +local untouched=periods+(1-period)^1*P(-1) +local splitstarter=(Cs(drivespec*(bwslash/"/"+fwslash)^0)+Cc(false))*Ct(lpeg.splitat(S("/\\")^1)) +local absolute=fwslash +function file.collapsepath(str,anchor) + if not str then + return + end + if anchor and not lpegmatch(anchors,str) then + str=getcurrentdir().."/"..str + end + if str=="" or str=="." then + return "." + elseif lpegmatch(untouched,str) then + return lpegmatch(reslasher,str) + end + local starter,oldelements=lpegmatch(splitstarter,str) + local newelements={} + local i=#oldelements + while i>0 do + local element=oldelements[i] + if element=='.' then + elseif element=='..' then + local n=i-1 + while n>0 do + local element=oldelements[n] + if element~='..' and element~='.' then + oldelements[n]='.' + break + else + n=n-1 + end + end + if n<1 then + insert(newelements,1,'..') + end + elseif element~="" then + insert(newelements,1,element) + end + i=i-1 + end + if #newelements==0 then + return starter or "." + elseif starter then + return starter..concat(newelements,'/') + elseif lpegmatch(absolute,str) then + return "/"..concat(newelements,'/') + else + return concat(newelements,'/') + end +end +local validchars=R("az","09","AZ","--","..") +local pattern_a=lpeg.replacer(1-validchars) +local pattern_a=Cs((validchars+P(1)/"-")^1) +local whatever=P("-")^0/"" +local pattern_b=Cs(whatever*(1-whatever*-1)^1) +function file.robustname(str,strict) + if str then + str=lpegmatch(pattern_a,str) or str + if strict then + return lpegmatch(pattern_b,str) or str else - return "" + return str end + end end - -function parsers.settings_to_set(str,t) -- tohash? -- todo: lpeg -- duplicate anyway - t = t or { } --- for s in gmatch(str,"%s*([^, ]+)") do -- space added - for s in gmatch(str,"[^, ]+") do -- space added - t[s] = true - end - return t +file.readdata=io.loaddata +file.savedata=io.savedata +function file.copy(oldname,newname) + if oldname and newname then + file.savedata(newname,io.loaddata(oldname)) + end +end +local letter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=period^0*fwslash+letter*colon+letter^1*separator+letter^1*fwslash +local rootbased=fwslash+letter*colon +lpeg.patterns.qualified=qualified +lpeg.patterns.rootbased=rootbased +function file.is_qualified_path(filename) + return filename and lpegmatch(qualified,filename)~=nil end - -function parsers.simple_hash_to_string(h, separator) - local t, tn = { }, 0 - for k, v in sortedhash(h) do - if v then - tn = tn + 1 - t[tn] = k - end - end - return concat(t,separator or ",") +function file.is_rootbased_path(filename) + return filename and lpegmatch(rootbased,filename)~=nil +end +function file.strip(name,dir) + if name then + local b,a=match(name,"^(.-)"..dir.."(.*)$") + return a~="" and a or name + end end --- for chem (currently one level) -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) - + C(digit^1 * lparent * (noparent + nestedparents)^1 * rparent) - + C((nestedbraces + (1-comma))^1) -local pattern_a = spaces * Ct(value*(separator*value)^0) +end -- of closure -local function repeater(n,str) - if not n then - return str - else - local s = lpegmatch(pattern_a,str) - if n == 1 then - return unpack(s) - else - local t, tn = { }, 0 - for i=1,n do - for j=1,#s do - tn = tn + 1 - t[tn] = s[j] - end - end - return unpack(t) - end +do -- create closure to overcome 200 locals limit + +-- original size: 3445, stripped down to: 1803 + +if not modules then modules={} end modules ['l-md5']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local md5,file=md5,file +local gsub,format,byte=string.gsub,string.format,string.byte +local function convert(str,fmt) + return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +end +if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end +if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end +if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end +function file.needsupdating(oldname,newname,threshold) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime then + local newtime=lfs.attributes(newname,"modification") + if not newtime then + return true + elseif newtime>=oldtime then + return false + elseif oldtime-newtime<(threshold or 1) then + return false + else + return true + end + else + return false + end +end +file.needs_updating=file.needsupdating +function file.syncmtimes(oldname,newname) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime and lfs.isfile(newname) then + lfs.touch(newname,oldtime,oldtime) + end +end +function file.checksum(name) + if md5 then + local data=io.loaddata(name) + if data then + return md5.HEX(data) end + end + return nil +end +function file.loadchecksum(name) + if md5 then + local data=io.loaddata(name..".md5") + return data and (gsub(data,"%s","")) + end + return nil +end +function file.savechecksum(name,checksum) + if not checksum then checksum=file.checksum(name) end + if checksum then + io.savedata(name..".md5",checksum) + return checksum + end + return nil end -local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) - + (C(digit^1)/tonumber * lparent * Cs((noparent + nestedparents)^1) * rparent) / repeater - + C((nestedbraces + (1-comma))^1) -local pattern_b = spaces * Ct(value*(separator*value)^0) -function parsers.settings_to_array_with_repeat(str,expand) -- beware: "" => { } - if expand then - return lpegmatch(pattern_b,str) or { } - else - return lpegmatch(pattern_a,str) or { } +end -- of closure + +do -- create closure to overcome 200 locals limit + +-- original size: 11806, stripped down to: 5417 + +if not modules then modules={} end modules ['l-url']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local char,format,byte=string.char,string.format,string.byte +local concat=table.concat +local tonumber,type=tonumber,type +local P,C,R,S,Cs,Cc,Ct,Cf,Cg,V=lpeg.P,lpeg.C,lpeg.R,lpeg.S,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.Cf,lpeg.Cg,lpeg.V +local lpegmatch,lpegpatterns,replacer=lpeg.match,lpeg.patterns,lpeg.replacer +url=url or {} +local url=url +local tochar=function(s) return char(tonumber(s,16)) end +local colon=P(":") +local qmark=P("?") +local hash=P("#") +local slash=P("/") +local percent=P("%") +local endofstring=P(-1) +local hexdigit=R("09","AF","af") +local plus=P("+") +local nothing=Cc("") +local escapedchar=(percent*C(hexdigit*hexdigit))/tochar +local escaped=(plus/" ")+escapedchar +local noslash=P("/")/"" +local schemestr=Cs((escaped+(1-colon-slash-qmark-hash))^2) +local authoritystr=Cs((escaped+(1- slash-qmark-hash))^0) +local pathstr=Cs((escaped+(1- qmark-hash))^0) +local querystr=Cs(((1- hash))^0) +local fragmentstr=Cs((escaped+(1- endofstring))^0) +local scheme=schemestr*colon+nothing +local authority=slash*slash*authoritystr+nothing +local path=slash*pathstr+nothing +local query=qmark*querystr+nothing +local fragment=hash*fragmentstr+nothing +local validurl=scheme*authority*path*query*fragment +local parser=Ct(validurl) +lpegpatterns.url=validurl +lpegpatterns.urlsplitter=parser +local escapes={} +setmetatable(escapes,{ __index=function(t,k) + local v=format("%%%02X",byte(k)) + t[k]=v + return v +end }) +local escaper=Cs((R("09","AZ","az")^1+P(" ")/"%%20"+S("-./_")^1+P(1)/escapes)^0) +local unescaper=Cs((escapedchar+1)^0) +lpegpatterns.urlunescaped=escapedchar +lpegpatterns.urlescaper=escaper +lpegpatterns.urlunescaper=unescaper +local function split(str) + return (type(str)=="string" and lpegmatch(parser,str)) or str +end +local isscheme=schemestr*colon*slash*slash +local function hasscheme(str) + if str then + local scheme=lpegmatch(isscheme,str) + return scheme~="" and scheme or false + else + return false + end +end +local rootletter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=P(".")^0*P("/")+rootletter*P(":")+rootletter^1*separator+rootletter^1*P("/") +local rootbased=P("/")+rootletter*P(":") +local barswapper=replacer("|",":") +local backslashswapper=replacer("\\","/") +local equal=P("=") +local amp=P("&") +local key=Cs(((escapedchar+1)-equal )^0) +local value=Cs(((escapedchar+1)-amp -endofstring)^0) +local splitquery=Cf (Ct("")*P { "sequence", + sequence=V("pair")*(amp*V("pair"))^0, + pair=Cg(key*equal*value), +},rawset) +local function hashed(str) + if str=="" then + return { + scheme="invalid", + original=str, + } + end + local s=split(str) + local rawscheme=s[1] + local rawquery=s[4] + local somescheme=rawscheme~="" + local somequery=rawquery~="" + if not somescheme and not somequery then + s={ + scheme="file", + authority="", + path=str, + query="", + fragment="", + original=str, + noscheme=true, + filename=str, + } + else + local authority,path,filename=s[2],s[3] + if authority=="" then + filename=path + elseif path=="" then + filename="" + else + filename=authority.."/"..path + end + s={ + scheme=rawscheme, + authority=authority, + path=path, + query=lpegmatch(unescaper,rawquery), + queries=lpegmatch(splitquery,rawquery), + fragment=s[5], + original=str, + noscheme=false, + filename=filename, + } + end + return s +end +url.split=split +url.hasscheme=hasscheme +url.hashed=hashed +function url.addscheme(str,scheme) + if hasscheme(str) then + return str + elseif not scheme then + return "file:///"..str + else + return scheme..":///"..str + end +end +function url.construct(hash) + local fullurl,f={},0 + local scheme,authority,path,query,fragment=hash.scheme,hash.authority,hash.path,hash.query,hash.fragment + if scheme and scheme~="" then + f=f+1;fullurl[f]=scheme.."://" + end + if authority and authority~="" then + f=f+1;fullurl[f]=authority + end + if path and path~="" then + f=f+1;fullurl[f]="/"..path + end + if query and query~="" then + f=f+1;fullurl[f]="?"..query + end + if fragment and fragment~="" then + f=f+1;fullurl[f]="#"..fragment + end + return lpegmatch(escaper,concat(fullurl)) +end +local pattern=Cs(noslash*R("az","AZ")*(S(":|")/":")*noslash*P(1)^0) +function url.filename(filename) + local spec=hashed(filename) + local path=spec.path + return (spec.scheme=="file" and path and lpegmatch(pattern,path)) or filename +end +local function escapestring(str) + return lpegmatch(escaper,str) +end +url.escape=escapestring +function url.query(str) + if type(str)=="string" then + return lpegmatch(splitquery,str) or "" + else + return str + end +end +function url.toquery(data) + local td=type(data) + if td=="string" then + return #str and escape(data) or nil + elseif td=="table" then + if next(data) then + local t={} + for k,v in next,data do + t[#t+1]=format("%s=%s",k,escapestring(v)) + end + return concat(t,"&") end + else + end +end +local pattern=Cs(noslash^0*(1-noslash*P(-1))^0) +function url.barepath(path) + if not path or path=="" then + return "" + else + return lpegmatch(pattern,path) + end end --- -local value = lbrace * C((nobrace + nestedbraces)^0) * rbrace -local pattern = Ct((space + value)^0) +end -- of closure -function parsers.arguments_to_table(str) - return lpegmatch(pattern,str) -end +do -- create closure to overcome 200 locals limit --- temporary here (unoptimized) +-- original size: 13035, stripped down to: 8133 -function parsers.getparameters(self,class,parentclass,settings) - local sc = self[class] - if not sc then - sc = { } - self[class] = sc - if parentclass then - local sp = self[parentclass] - if not sp then - sp = { } - self[parentclass] = sp - end - setmetatableindex(sc,sp) - end +if not modules then modules={} end modules ['l-dir']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,select=type,select +local find,gmatch,match,gsub=string.find,string.gmatch,string.match,string.gsub +local concat,insert,remove=table.concat,table.insert,table.remove +local lpegmatch=lpeg.match +local P,S,R,C,Cc,Cs,Ct,Cv,V=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Cv,lpeg.V +dir=dir or {} +local dir=dir +local lfs=lfs +local attributes=lfs.attributes +local walkdir=lfs.dir +local isdir=lfs.isdir +local isfile=lfs.isfile +local currentdir=lfs.currentdir +if not isdir then + function isdir(name) + local a=attributes(name) + return a and a.mode=="directory" + end + lfs.isdir=isdir +end +if not isfile then + function isfile(name) + local a=attributes(name) + return a and a.mode=="file" + end + lfs.isfile=isfile +end +function dir.current() + return (gsub(currentdir(),"\\","/")) +end +local lfsisdir=isdir +local function isdir(path) + path=gsub(path,"[/\\]+$","") + return lfsisdir(path) +end +lfs.isdir=isdir +local function globpattern(path,patt,recurse,action) + if path=="/" then + path=path.."." + elseif not find(path,"/$") then + path=path..'/' + end + if isdir(path) then + for name in walkdir(path) do + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if find(full,patt) then + action(full) + end + elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then + globpattern(full,patt,recurse,action) + end end - parsers.settings_to_hash(settings,sc) + end end - -function parsers.listitem(str) - return gmatch(str,"[^, ]+") +dir.globpattern=globpattern +local function collectpattern(path,patt,recurse,result) + local ok,scanner + result=result or {} + if path=="/" then + ok,scanner,first=xpcall(function() return walkdir(path..".") end,function() end) + else + ok,scanner,first=xpcall(function() return walkdir(path) end,function() end) + end + if ok and type(scanner)=="function" then + if not find(path,"/$") then path=path..'/' end + for name in scanner,first do + local full=path..name + local attr=attributes(full) + local mode=attr.mode + if mode=='file' then + if find(full,patt) then + result[name]=attr + end + elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then + attr.list=collectpattern(full,patt,recurse) + result[name]=attr + end + end + end + return result end - --- - -local pattern = Cs { "start", - start = V("one") + V("two") + V("three"), - rest = (Cc(",") * V("thousand"))^0 * (P(".") + endofstring) * anything^0, - thousand = digit * digit * digit, - one = digit * V("rest"), - two = digit * digit * V("rest"), - three = V("thousand") * V("rest"), +dir.collectpattern=collectpattern +local pattern=Ct { + [1]=(C(P(".")+P("/")^1)+C(R("az","AZ")*P(":")*P("/")^0)+Cc("./"))*V(2)*V(3), + [2]=C(((1-S("*?/"))^0*P("/"))^0), + [3]=C(P(1)^0) } - -patterns.splitthousands = pattern -- maybe better in the parsers namespace ? - -function parsers.splitthousands(str) - return lpegmatch(pattern,str) or str -end - --- print(parsers.splitthousands("11111111111.11")) - -local optionalwhitespace = whitespace^0 - -patterns.words = Ct((Cs((1-punctuation-whitespace)^1) + anything)^1) -patterns.sentences = Ct((optionalwhitespace * Cs((1-period)^0 * period))^1) -patterns.paragraphs = Ct((optionalwhitespace * Cs((whitespace^1*endofstring/"" + 1 - (spacer^0*newline*newline))^1))^1) - --- local str = " Word1 word2. \n Word3 word4. \n\n Word5 word6.\n " --- inspect(lpegmatch(patterns.paragraphs,str)) --- inspect(lpegmatch(patterns.sentences,str)) --- inspect(lpegmatch(patterns.words,str)) - --- handy for k="v" [, ] k="v" - -local dquote = P('"') -local equal = P('=') -local escape = P('\\') -local separator = S(' ,') - -local key = C((1-equal)^1) -local value = dquote * C((1-dquote-escape*dquote)^0) * dquote - -local pattern = Cf(Ct("") * Cg(key * equal * value) * separator^0,rawset)^0 - -parsers.patterns.keq_to_hash_c = pattern - -function parsers.keq_to_hash(str) - if str and str ~= "" then - return lpegmatch(pattern,str) +local filter=Cs (( + P("**")/".*"+P("*")/"[^/]*"+P("?")/"[^/]"+P(".")/"%%."+P("+")/"%%+"+P("-")/"%%-"+P(1) +)^0 ) +local function glob(str,t) + if type(t)=="function" then + if type(str)=="table" then + for s=1,#str do + glob(str[s],t) + end + elseif isfile(str) then + t(str) + else + local split=lpegmatch(pattern,str) + if split then + local root,path,base=split[1],split[2],split[3] + local recurse=find(base,"%*%*") + local start=root..path + local result=lpegmatch(filter,start..base) + globpattern(start,result,recurse,t) + end + end + else + if type(str)=="table" then + local t=t or {} + for s=1,#str do + glob(str[s],t) + end + return t + elseif isfile(str) then + if t then + t[#t+1]=str + return t + else + return { str } + end else - return { } + local split=lpegmatch(pattern,str) + if split then + local t=t or {} + local action=action or function(name) t[#t+1]=name end + local root,path,base=split[1],split[2],split[3] + local recurse=find(base,"%*%*") + local start=root..path + local result=lpegmatch(filter,start..base) + globpattern(start,result,recurse,action) + return t + else + return {} + end + end + end +end +dir.glob=glob +local function globfiles(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if recurse then + globfiles(path.."/"..name,recurse,func,files) + end + elseif mode=="file" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name + end + end end + end + return files end - --- inspect(lpeg.match(pattern,[[key="value"]])) - -local defaultspecification = { separator = ",", quote = '"' } - --- this version accepts multiple separators and quotes as used in the --- database module - -function parsers.csvsplitter(specification) - specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification - local separator = specification.separator - local quotechar = specification.quote - local separator = S(separator ~= "" and separator or ",") - local whatever = C((1 - separator - newline)^0) - if quotechar and quotechar ~= "" then - local quotedata = nil - for chr in gmatch(quotechar,".") do - local quotechar = P(chr) - local quoteword = quotechar * C((1 - quotechar)^0) * quotechar - if quotedata then - quotedata = quotedata + quoteword - else - quotedata = quoteword - end +dir.globfiles=globfiles +function dir.ls(pattern) + return concat(glob(pattern),"\n") +end +local make_indeed=true +local onwindows=os.type=="windows" or find(os.getenv("PATH"),";") +if onwindows then + function dir.mkdirs(...) + local str,pth="","" + for i=1,select("#",...) do + local s=select(i,...) + if s=="" then + elseif str=="" then + str=s + else + str=str.."/"..s + end + end + local first,middle,last + local drive=false + first,middle,last=match(str,"^(//)(//*)(.*)$") + if first then + else + first,last=match(str,"^(//)/*(.-)$") + if first then + middle,last=match(str,"([^/]+)/+(.-)$") + if middle then + pth="//"..middle + else + pth="//"..last + last="" end - whatever = quotedata + whatever + else + first,middle,last=match(str,"^([a-zA-Z]:)(/*)(.-)$") + if first then + pth,drive=first..middle,true + else + middle,last=match(str,"^(/*)(.-)$") + if not middle then + last=str + end + end + end end - local parser = Ct((Ct(whatever * (separator * whatever)^0) * S("\n\r"))^0 ) - return function(data) - return lpegmatch(parser,data) + for s in gmatch(last,"[^/]+") do + if pth=="" then + pth=s + elseif drive then + pth,drive=pth..s,false + else + pth=pth.."/"..s + end + if make_indeed and not isdir(pth) then + lfs.mkdir(pth) + end end -end - --- and this is a slightly patched version of a version posted by Philipp Gesang - --- local mycsvsplitter = utilities.parsers.rfc4180splitter() --- --- local crap = [[ --- first,second,third,fourth --- "1","2","3","4" --- "a","b","c","d" --- "foo","bar""baz","boogie","xyzzy" --- ]] --- --- local list, names = mycsvsplitter(crap,true) inspect(list) inspect(names) --- local list, names = mycsvsplitter(crap) inspect(list) inspect(names) - -function parsers.rfc4180splitter(specification) - specification = specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification - local separator = specification.separator --> rfc: COMMA - local quotechar = P(specification.quote) --> DQUOTE - local dquotechar = quotechar * quotechar --> 2DQUOTE - / specification.quote - local separator = S(separator ~= "" and separator or ",") - local escaped = quotechar - * Cs((dquotechar + (1 - quotechar))^0) - * quotechar - local non_escaped = C((1 - quotechar - newline - separator)^1) - local field = escaped + non_escaped - local record = Ct((field * separator^-1)^1) - local headerline = record * Cp() - local wholeblob = Ct((newline^-1 * record)^0) - return function(data,getheader) - if getheader then - local header, position = lpegmatch(headerline,data) - local data = lpegmatch(wholeblob,data,position) - return data, header + return pth,(isdir(pth)==true) + end +else + function dir.mkdirs(...) + local str,pth="","" + for i=1,select("#",...) do + local s=select(i,...) + if s and s~="" then + if str~="" then + str=str.."/"..s else - return lpegmatch(wholeblob,data) + str=s end + end end -end - --- utilities.parsers.stepper("1,7-",9,function(i) print(">>>",i) end) --- utilities.parsers.stepper("1-3,7,8,9") --- utilities.parsers.stepper("1-3,6,7",function(i) print(">>>",i) end) --- utilities.parsers.stepper(" 1 : 3, ,7 ") --- utilities.parsers.stepper("1:4,9:13,24:*",30) - -local function ranger(first,last,n,action) - if not first then - -- forget about it - elseif last == true then - for i=first,n or first do - action(i) + str=gsub(str,"/+","/") + if find(str,"^/") then + pth="/" + for s in gmatch(str,"[^/]+") do + local first=(pth=="/") + if first then + pth=pth..s + else + pth=pth.."/"..s end - elseif last then - for i=first,last do - action(i) + if make_indeed and not first and not isdir(pth) then + lfs.mkdir(pth) end + end else - action(first) + pth="." + for s in gmatch(str,"[^/]+") do + pth=pth.."/"..s + if make_indeed and not isdir(pth) then + lfs.mkdir(pth) + end + end end + return pth,(isdir(pth)==true) + end end - -local cardinal = patterns.cardinal / tonumber -local spacers = patterns.spacer^0 -local endofstring = patterns.endofstring - -local stepper = spacers * ( C(cardinal) * ( spacers * S(":-") * spacers * ( C(cardinal) + Cc(true) ) + Cc(false) ) - * Carg(1) * Carg(2) / ranger * S(", ")^0 )^1 - -local stepper = spacers * ( C(cardinal) * ( spacers * S(":-") * spacers * ( C(cardinal) + (P("*") + endofstring) * Cc(true) ) + Cc(false) ) - * Carg(1) * Carg(2) / ranger * S(", ")^0 )^1 * endofstring -- we're sort of strict (could do without endofstring) - -function utilities.parsers.stepper(str,n,action) - if type(n) == "function" then - lpegmatch(stepper,str,1,false,n or print) +dir.makedirs=dir.mkdirs +if onwindows then + function dir.expandname(str) + local first,nothing,last=match(str,"^(//)(//*)(.*)$") + if first then + first=dir.current().."/" + end + if not first then + first,last=match(str,"^(//)/*(.*)$") + end + if not first then + first,last=match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d=currentdir() + if lfs.chdir(first) then + first=dir.current() + end + lfs.chdir(d) + end + end + if not first then + first,last=dir.current(),str + end + last=gsub(last,"//","/") + last=gsub(last,"/%./","/") + last=gsub(last,"^/*","") + first=gsub(first,"/*$","") + if last=="" or last=="." then + return first else - lpegmatch(stepper,str,1,n,action or print) + return first.."/"..last + end + end +else + function dir.expandname(str) + if not find(str,"^/") then + str=currentdir().."/"..str end + str=gsub(str,"//","/") + str=gsub(str,"/%./","/") + str=gsub(str,"(.)/%.$","%1") + return str + end +end +file.expandname=dir.expandname +local stack={} +function dir.push(newdir) + insert(stack,lfs.currentdir()) +end +function dir.pop() + local d=remove(stack) + if d then + lfs.chdir(d) + end + return d end @@ -7527,95 +3419,71 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-fmt'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or { } -utilities.formatters = utilities.formatters or { } -local formatters = utilities.formatters +-- original size: 1822, stripped down to: 1544 -local concat, format = table.concat, string.format -local tostring, type = tostring, type -local strip = string.strip - -local P, R, Cs = lpeg.P, lpeg.R, lpeg.Cs -local lpegmatch = lpeg.match - --- temporary here - -local digit = R("09") -local period = P(".") -local zero = P("0") -local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") -local number = digit^1 * (case_1 + case_2) -local stripper = Cs((number + 1)^0) - - -lpeg.patterns.stripzeros = stripper - -function formatters.stripzeros(str) - return lpegmatch(stripper,str) +if not modules then modules={} end modules ['l-boolean']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,tonumber=type,tonumber +boolean=boolean or {} +local boolean=boolean +function boolean.tonumber(b) + if b then return 1 else return 0 end end - -function formatters.formatcolumns(result,between) - if result and #result > 0 then - between = between or " " - local widths, numbers = { }, { } - local first = result[1] - local n = #first - for i=1,n do - widths[i] = 0 - end - for i=1,#result do - local r = result[i] - for j=1,n do - local rj = r[j] - local tj = type(rj) - if tj == "number" then - numbers[j] = true - end - if tj ~= "string" then - rj = tostring(rj) - r[j] = rj - end - local w = #rj - if w > widths[j] then - widths[j] = w - end - end - end - for i=1,n do - local w = widths[i] - if numbers[i] then - if w > 80 then - widths[i] = "%s" .. between - else - widths[i] = "%0" .. w .. "i" .. between - end - else - if w > 80 then - widths[i] = "%s" .. between - elseif w > 0 then - widths[i] = "%-" .. w .. "s" .. between - else - widths[i] = "%s" - end - end - end - local template = strip(concat(widths)) - for i=1,#result do - local str = format(template,unpack(result[i])) - result[i] = strip(str) - end +function toboolean(str,tolerant) + if str==nil then + return false + elseif str==false then + return false + elseif str==true then + return true + elseif str=="true" then + return true + elseif str=="false" then + return false + elseif not tolerant then + return false + elseif str==0 then + return false + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +string.toboolean=toboolean +function string.booleanstring(str) + if str==nil then + return false + elseif str==false then + return false + elseif str==true then + return true + elseif str=="true" then + return true + elseif str=="false" then + return false + elseif str==0 then + return false + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +function string.is_boolean(str,default) + if type(str)=="string" then + if str=="true" or str=="yes" or str=="on" or str=="t" then + return true + elseif str=="false" or str=="no" or str=="off" or str=="f" then + return false end - return result + end + return default end @@ -7623,136 +3491,468 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-deb'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- the tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) - -local debug = require "debug" - -local getinfo = debug.getinfo -local type, next, tostring = type, next, tostring -local format, find = string.format, string.find -local is_boolean = string.is_boolean - -utilities = utilities or { } -utilities.debugger = utilities.debugger or { } -local debugger = utilities.debugger +-- original size: 24092, stripped down to: 11311 -local counters = { } -local names = { } - --- one - -local function hook() - local f = getinfo(2) -- "nS" - if f then - local n = "unknown" - if f.what == "C" then - n = f.name or '' - if not names[n] then - names[n] = format("%42s",n) - end +if not modules then modules={} end modules ['l-unicode']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utf=utf or (unicode and unicode.utf8) or {} +utf.characters=utf.characters or string.utfcharacters +utf.values=utf.values or string.utfvalues +local type=type +local char,byte,format,sub=string.char,string.byte,string.format,string.sub +local concat=table.concat +local P,C,R,Cs,Ct,Cmt,Cc,Carg=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Ct,lpeg.Cmt,lpeg.Cc,lpeg.Carg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local bytepairs=string.bytepairs +local finder=lpeg.finder +local replacer=lpeg.replacer +local utfvalues=utf.values +local utfgmatch=utf.gmatch +local p_utftype=patterns.utftype +local p_utfoffset=patterns.utfoffset +local p_utf8char=patterns.utf8char +local p_utf8byte=patterns.utf8byte +local p_utfbom=patterns.utfbom +local p_newline=patterns.newline +local p_whitespace=patterns.whitespace +if not unicode then + unicode={ utf=utf } +end +if not utf.char then + local floor,char=math.floor,string.char + function utf.char(n) + if n<0x80 then + return char(n) + elseif n<0x800 then + return char( + 0xC0+floor(n/0x40), + 0x80+(n%0x40) + ) + elseif n<0x10000 then + return char( + 0xE0+floor(n/0x1000), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) + elseif n<0x200000 then + return char( + 0xF0+floor(n/0x40000), + 0x80+(floor(n/0x1000)%0x40), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) + else + return "" + end + end +end +if not utf.byte then + local utf8byte=patterns.utf8byte + function utf.byte(c) + return lpegmatch(utf8byte,c) + end +end +local utfchar,utfbyte=utf.char,utf.byte +function utf.filetype(data) + return data and lpegmatch(p_utftype,data) or "unknown" +end +local toentities=Cs ( + ( + patterns.utf8one+( + patterns.utf8two+patterns.utf8three+patterns.utf8four + )/function(s) local b=utfbyte(s) if b<127 then return s else return format("&#%X;",b) end end + )^0 +) +patterns.toentities=toentities +function utf.toentities(str) + return lpegmatch(toentities,str) +end +local one=P(1) +local two=C(1)*C(1) +local four=C(R(utfchar(0xD8),utfchar(0xFF)))*C(1)*C(1)*C(1) +local pattern=P("\254\255")*Cs(( + four/function(a,b,c,d) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(a,b) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 )+P("\255\254")*Cs(( + four/function(b,a,d,c) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(b,a) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 ) +function string.toutf(s) + return lpegmatch(pattern,s) or s +end +local validatedutf=Cs ( + ( + patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four+P(1)/"�" + )^0 +) +patterns.validatedutf=validatedutf +function utf.is_valid(str) + return type(str)=="string" and lpegmatch(validatedutf,str) or false +end +if not utf.len then + local n,f=0,1 + local utfcharcounter=patterns.utfbom^-1*Cmt ( + Cc(1)*patterns.utf8one^1+Cc(2)*patterns.utf8two^1+Cc(3)*patterns.utf8three^1+Cc(4)*patterns.utf8four^1, + function(_,t,d) + n=n+(t-f)/d + f=t + return true + end + )^0 + function utf.len(str) + n,f=0,1 + lpegmatch(utfcharcounter,str or "") + return n + end +end +utf.length=utf.len +if not utf.sub then + local utflength=utf.length + local b,e,n,first,last=0,0,0,0,0 + local function slide_zero(s,p) + n=n+1 + if n>=last then + e=p-1 + else + return p + end + end + local function slide_one(s,p) + n=n+1 + if n==first then + b=p + end + if n>=last then + e=p-1 + else + return p + end + end + local function slide_two(s,p) + n=n+1 + if n==first then + b=p + else + return true + end + end + local pattern_zero=Cmt(p_utf8char,slide_zero)^0 + local pattern_one=Cmt(p_utf8char,slide_one )^0 + local pattern_two=Cmt(p_utf8char,slide_two )^0 + function utf.sub(str,start,stop) + if not start then + return str + end + if start==0 then + start=1 + end + if not stop then + if start<0 then + local l=utflength(str) + start=l+start + else + start=start-1 + end + b,n,first=0,0,start + lpegmatch(pattern_two,str) + if n>=first then + return sub(str,b) + else + return "" + end + end + if start<0 or stop<0 then + local l=utf.length(str) + if start<0 then + start=l+start + if start<=0 then + start=1 else - -- source short_src linedefined what name namewhat nups func - n = f.name or f.namewhat or f.what - if not n or n == "" then - n = "?" - end - if not names[n] then - names[n] = format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") - end + start=start+1 end - counters[n] = (counters[n] or 0) + 1 - end -end - -function debugger.showstats(printer,threshold) -- hm, something has changed, rubish now - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - local dataset = { } - for name, count in next, counters do - dataset[#dataset+1] = { name, count } - end - table.sort(dataset,function(a,b) return a[2] == b[2] and b[1] > a[1] or a[2] > b[2] end) - for i=1,#dataset do - local d = dataset[i] - local name = d[1] - local count = d[2] - if count > threshold and not find(name,"for generator") then -- move up - printer(format("%8i %s\n", count, names[name])) - total = total + count + end + if stop<0 then + stop=l+stop + if stop==0 then + stop=1 + else + stop=stop+1 end - grandtotal = grandtotal + count - functions = functions + 1 + end + end + if start>stop then + return "" + elseif start>1 then + b,e,n,first,last=0,0,0,start-1,stop + lpegmatch(pattern_one,str) + if n>=first and e==0 then + e=#str + end + return sub(str,b,e) + else + b,e,n,last=1,0,0,stop + lpegmatch(pattern_zero,str) + if e==0 then + e=#str + end + return sub(str,b,e) end - printer("\n") - printer(format("functions : % 10i\n", functions)) - printer(format("total : % 10i\n", total)) - printer(format("grand total: % 10i\n", grandtotal)) - printer(format("threshold : % 10i\n", threshold)) + end end - -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() +function utf.remapper(mapping) + local pattern=Cs((p_utf8char/mapping)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern +end +function utf.replacer(t) + local r=replacer(t,false,false,true) + return function(str) + return lpegmatch(r,str) + end +end +function utf.subtituter(t) + local f=finder (t) + local r=replacer(t,false,false,true) + return function(str) + local i=lpegmatch(f,str) + if not i then + return str + elseif i>#str then + return str + else + return lpegmatch(r,str) + end + end +end +local utflinesplitter=p_utfbom^-1*lpeg.tsplitat(p_newline) +local utfcharsplitter_ows=p_utfbom^-1*Ct(C(p_utf8char)^0) +local utfcharsplitter_iws=p_utfbom^-1*Ct((p_whitespace^1+C(p_utf8char))^0) +local utfcharsplitter_raw=Ct(C(p_utf8char)^0) +patterns.utflinesplitter=utflinesplitter +function utf.splitlines(str) + return lpegmatch(utflinesplitter,str or "") +end +function utf.split(str,ignorewhitespace) + if ignorewhitespace then + return lpegmatch(utfcharsplitter_iws,str or "") + else + return lpegmatch(utfcharsplitter_ows,str or "") + end +end +function utf.totable(str) + return lpegmatch(utfcharsplitter_raw,str) +end +function utf.magic(f) + local str=f:read(4) or "" + local off=lpegmatch(p_utfoffset,str) + if off<4 then + f:seek('set',off) + end + return lpegmatch(p_utftype,str) +end +local function utf16_to_utf8_be(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,0 + for left,right in bytepairs(t[i]) do + if right then + local now=256*left+right + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + r=r+1 + result[r]=utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + else + r=r+1 + result[r]=utfchar(now) + end + end end + t[i]=concat(result,"",1,r) + end + return t end - -function debugger.enable() - debug.sethook(hook,"c") +local function utf16_to_utf8_le(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,0 + for left,right in bytepairs(t[i]) do + if right then + local now=256*right+left + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + r=r+1 + result[r]=utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + else + r=r+1 + result[r]=utfchar(now) + end + end + end + t[i]=concat(result,"",1,r) + end + return t +end +local function utf32_to_utf8_be(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,-1 + for a,b in bytepairs(t[i]) do + if a and b then + if more<0 then + more=256*256*256*a+256*256*b + else + r=r+1 + result[t]=utfchar(more+256*a+b) + more=-1 + end + else + break + end + end + t[i]=concat(result,"",1,r) + end + return t +end +local function utf32_to_utf8_le(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,-1 + for a,b in bytepairs(t[i]) do + if a and b then + if more<0 then + more=256*b+a + else + r=r+1 + result[t]=utfchar(more+256*256*256*b+256*256*a) + more=-1 + end + else + break + end + end + t[i]=concat(result,"",1,r) + end + return t +end +utf.utf32_to_utf8_be=utf32_to_utf8_be +utf.utf32_to_utf8_le=utf32_to_utf8_le +utf.utf16_to_utf8_be=utf16_to_utf8_be +utf.utf16_to_utf8_le=utf16_to_utf8_le +function utf.utf8_to_utf8(t) + return type(t)=="string" and lpegmatch(utflinesplitter,t) or t +end +function utf.utf16_to_utf8(t,endian) + return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +end +function utf.utf32_to_utf8(t,endian) + return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +end +local function little(c) + local b=byte(c) + if b<0x10000 then + return char(b%256,b/256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end +end +local function big(c) + local b=byte(c) + if b<0x10000 then + return char(b/256,b%256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end +end +local _,l_remap=utf.remapper(little) +local _,b_remap=utf.remapper(big) +function utf.utf8_to_utf16(str,littleendian) + if littleendian then + return char(255,254)..lpegmatch(l_remap,str) + else + return char(254,255)..lpegmatch(b_remap,str) + end +end +local pattern=Cs ( + (p_utf8byte/function(unicode ) return format("0x%04X",unicode) end)*(p_utf8byte*Carg(1)/function(unicode,separator) return format("%s0x%04X",separator,unicode) end)^0 +) +function utf.tocodes(str,separator) + return lpegmatch(pattern,str,1,separator or " ") end - -function debugger.disable() - debug.sethook() +function utf.ustring(s) + return format("U+%05X",type(s)=="number" and s or utfbyte(s)) end - - - - - -local is_node = node and node.is_node -local is_lpeg = lpeg and lpeg.type - -function inspect(i) -- global function - local ti = type(i) - if ti == "table" then - table.print(i,"table") - elseif is_node and is_node(i) then - table.print(nodes.astable(i),tostring(i)) - elseif is_lpeg and is_lpeg(i) then - lpeg.print(i) - else - print(tostring(i)) - end - return i -- so that we can inline the inspect +function utf.xstring(s) + return format("0x%05X",type(s)=="number" and s or utfbyte(s)) end - --- from the lua book: - -function traceback() - local level = 1 - while true do - local info = debug.getinfo(level, "Sl") - if not info then - break - elseif info.what == "C" then - print(format("%3i : C function",level)) - else - print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) - end - level = level + 1 +local p_nany=p_utf8char/"" +if utfgmatch then + function utf.count(str,what) + if type(what)=="string" then + local n=0 + for _ in utfgmatch(str,what) do + n=n+1 + end + return n + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) + end + end +else + local cache={} + function utf.count(str,what) + if type(what)=="string" then + local p=cache[what] + if not p then + p=Cs((P(what)/" "+p_nany)^0) + cache[p]=p + end + return #lpegmatch(p,str) + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) end + end end @@ -7760,198 +3960,473 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-inf'] = { - version = 1.001, - comment = "companion to trac-inf.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- As we want to protect the global tables, we no longer store the timing --- in the tables themselves but in a hidden timers table so that we don't --- get warnings about assignments. This is more efficient than using rawset --- and rawget. +-- original size: 915, stripped down to: 836 -local format, lower = string.format, string.lower -local clock = os.gettimeofday or os.clock -- should go in environment -local write_nl = texio and texio.write_nl or print +if not modules then modules={} end modules ['l-math']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local floor,sin,cos,tan=math.floor,math.sin,math.cos,math.tan +if not math.round then + function math.round(x) return floor(x+0.5) end +end +if not math.div then + function math.div(n,m) return floor(n/m) end +end +if not math.mod then + function math.mod(n,m) return n%m end +end +local pipi=2*math.pi/360 +if not math.sind then + function math.sind(d) return sin(d*pipi) end + function math.cosd(d) return cos(d*pipi) end + function math.tand(d) return tan(d*pipi) end +end +if not math.odd then + function math.odd (n) return n%2~=0 end + function math.even(n) return n%2==0 end +end -statistics = statistics or { } -local statistics = statistics -statistics.enable = true -statistics.threshold = 0.01 +end -- of closure -local statusinfo, n, registered, timers = { }, 0, { }, { } +do -- create closure to overcome 200 locals limit -table.setmetatableindex(timers,function(t,k) - local v = { timing = 0, loadtime = 0 } - t[k] = v - return v -end) +-- original size: 10334, stripped down to: 6756 -local function hastiming(instance) - return instance and timers[instance] +if not modules then modules={} end modules ['util-tab']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.tables=utilities.tables or {} +local tables=utilities.tables +local format,gmatch,rep,gsub=string.format,string.gmatch,string.rep,string.gsub +local concat,insert,remove=table.concat,table.insert,table.remove +local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local type,next,rawset,tonumber,load,select=type,next,rawset,tonumber,load,select +local lpegmatch,P,Cs=lpeg.match,lpeg.P,lpeg.Cs +local serialize=table.serialize +local splitter=lpeg.tsplitat(".") +function tables.definetable(target,nofirst,nolast) + local composed,shortcut,t=nil,nil,{} + local snippets=lpegmatch(splitter,target) + for i=1,#snippets-(nolast and 1 or 0) do + local name=snippets[i] + if composed then + composed=shortcut.."."..name + shortcut=shortcut.."_"..name + t[#t+1]=format("local %s = %s if not %s then %s = { } %s = %s end",shortcut,composed,shortcut,shortcut,composed,shortcut) + else + composed=name + shortcut=name + if not nofirst then + t[#t+1]=format("%s = %s or { }",composed,composed) + end + end + end + if nolast then + composed=shortcut.."."..snippets[#snippets] + end + return concat(t,"\n"),composed end - -local function resettiming(instance) - timers[instance or "notimer"] = { timing = 0, loadtime = 0 } +function tables.definedtable(...) + local t=_G + for i=1,select("#",...) do + local li=select(i,...) + local tl=t[li] + if not tl then + tl={} + t[li]=tl + end + t=tl + end + return t end - -local function starttiming(instance) - local timer = timers[instance or "notimer"] - local it = timer.timing or 0 - if it == 0 then - timer.starttime = clock() - if not timer.loadtime then - timer.loadtime = 0 - end +function tables.accesstable(target,root) + local t=root or _G + for name in gmatch(target,"([^%.]+)") do + t=t[name] + if not t then + return end - timer.timing = it + 1 + end + return t end - -local function stoptiming(instance, report) - local timer = timers[instance or "notimer"] - local it = timer.timing - if it > 1 then - timer.timing = it - 1 - else - local starttime = timer.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - timer.stoptime = stoptime - timer.loadtime = timer.loadtime + loadtime - if report then - statistics.report("load time %0.3f",loadtime) - end - timer.timing = 0 - return loadtime - end +function tables.migratetable(target,v,root) + local t=root or _G + local names=string.split(target,".") + for i=1,#names-1 do + local name=names[i] + t[name]=t[name] or {} + t=t[name] + if not t then + return end - return 0 + end + t[names[#names]]=v end - -local function elapsedtime(instance) - local timer = timers[instance or "notimer"] - return format("%0.3f",timer and timer.loadtime or 0) +function tables.removevalue(t,value) + if value then + for i=1,#t do + if t[i]==value then + remove(t,i) + end + end + end end - -local function elapsedindeed(instance) - local timer = timers[instance or "notimer"] - return (timer and timer.loadtime or 0) > statistics.threshold +function tables.insertbeforevalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) end - -local function elapsedseconds(instance,rest) -- returns nil if 0 seconds - if elapsedindeed(instance) then - return format("%s seconds %s", elapsedtime(instance),rest or "") +function tables.insertaftervalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i+1,extra) + return end + end + insert(t,#t+1,extra) end - -statistics.hastiming = hastiming -statistics.resettiming = resettiming -statistics.starttiming = starttiming -statistics.stoptiming = stoptiming -statistics.elapsedtime = elapsedtime -statistics.elapsedindeed = elapsedindeed -statistics.elapsedseconds = elapsedseconds - --- general function .. we might split this module - -function statistics.register(tag,fnc) - if statistics.enable and type(fnc) == "function" then - local rt = registered[tag] or (#statusinfo + 1) - statusinfo[rt] = { tag, fnc } - registered[tag] = rt - if #tag > n then n = #tag end +local function toxml(t,d,result,step) + for k,v in table.sortedpairs(t) do + if type(v)=="table" then + if type(k)=="number" then + result[#result+1]=format("%s",d,k) + toxml(v,d..step,result,step) + result[#result+1]=format("%s",d,k) + else + result[#result+1]=format("%s<%s>",d,k) + toxml(v,d..step,result,step) + result[#result+1]=format("%s",d,k) + end + elseif type(k)=="number" then + result[#result+1]=format("%s%s",d,k,v,k) + else + result[#result+1]=format("%s<%s>%s",d,k,tostring(v),k) end + end end - -function statistics.show(reporter) - if statistics.enable then - if not reporter then reporter = function(tag,data,n) write_nl(tag .. " " .. data) end end - -- this code will move - local register = statistics.register - register("luatex banner", function() - return lower(status.banner) - end) - register("control sequences", function() - return format("%s of %s + %s", status.cs_count, status.hash_size,status.hash_extra) - end) - register("callbacks", function() - local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 - return format("%s direct, %s indirect, %s total", total-indirect, indirect, total) - end) - collectgarbage("collect") - register("current memory usage", statistics.memused) - register("runtime",statistics.runtime) - for i=1,#statusinfo do - local s = statusinfo[i] - local r = s[2]() - if r then - reporter(s[1],r,n) - end +function table.toxml(t,name,nobanner,indent,spaces) + local noroot=name==false + local result=(nobanner or noroot) and {} or { "" } + local indent=rep(" ",indent or 0) + local spaces=rep(" ",spaces or 1) + if noroot then + toxml(t,inndent,result,spaces) + else + toxml({ [name or "root"]=t },indent,result,spaces) + end + return concat(result,"\n") +end +function tables.encapsulate(core,capsule,protect) + if type(capsule)~="table" then + protect=true + capsule={} + end + for key,value in next,core do + if capsule[key] then + print(format("\ninvalid inheritance '%s' in '%s': %s",key,tostring(core))) + os.exit() + else + capsule[key]=value + end + end + if protect then + for key,value in next,core do + core[key]=nil + end + setmetatable(core,{ + __index=capsule, + __newindex=function(t,key,value) + if capsule[key] then + print(format("\ninvalid overload '%s' in '%s'",key,tostring(core))) + os.exit() + else + rawset(t,key,value) end - write_nl("") -- final newline - statistics.enable = false + end + } ) + end +end +local function fastserialize(t,r,outer) + r[#r+1]="{" + local n=#t + if n>0 then + for i=1,n do + local v=t[i] + local tv=type(v) + if tv=="string" then + r[#r+1]=format("%q,",v) + elseif tv=="number" then + r[#r+1]=format("%s,",v) + elseif tv=="table" then + fastserialize(v,r) + elseif tv=="boolean" then + r[#r+1]=format("%s,",tostring(v)) + end + end + else + for k,v in next,t do + local tv=type(v) + if tv=="string" then + r[#r+1]=format("[%q]=%q,",k,v) + elseif tv=="number" then + r[#r+1]=format("[%q]=%s,",k,v) + elseif tv=="table" then + r[#r+1]=format("[%q]=",k) + fastserialize(v,r) + elseif tv=="boolean" then + r[#r+1]=format("[%q]=%s,",k,tostring(v)) + end end + end + if outer then + r[#r+1]="}" + else + r[#r+1]="}," + end + return r end - -local template, report_statistics, nn = nil, nil, 0 -- we only calcute it once - -function statistics.showjobstat(tag,data,n) - if not logs then - -- sorry - elseif type(data) == "table" then - for i=1,#data do - statistics.showjobstat(tag,data[i],n) - end - else - if not template or n > nn then - template, n = format("%%-%ss - %%s",n), nn - report_statistics = logs.reporter("mkiv lua stats") +function table.fastserialize(t,prefix) + return concat(fastserialize(t,{ prefix or "return" },true)) +end +function table.deserialize(str) + if not str or str=="" then + return + end + local code=load(str) + if not code then + return + end + code=code() + if not code then + return + end + return code +end +function table.load(filename) + if filename then + local t=io.loaddata(filename) + if t and t~="" then + t=load(t) + if type(t)=="function" then + t=t() + if type(t)=="table" then + return t end - report_statistics(format(template,tag,data)) + end end + end end - -function statistics.memused() -- no math.round yet -) - local round = math.round or math.floor - return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) +function table.save(filename,t,n,...) + io.savedata(filename,serialize(t,n==nil and true or n,...)) end - -starttiming(statistics) - -function statistics.formatruntime(runtime) -- indirect so it can be overloaded and - return format("%s seconds", runtime) -- indeed that happens in cure-uti.lua +local function slowdrop(t) + local r={} + local l={} + for i=1,#t do + local ti=t[i] + local j=0 + for k,v in next,ti do + j=j+1 + l[j]=format("%s=%q",k,v) + end + r[i]=format(" {%s},\n",concat(l)) + end + return format("return {\n%s}",concat(r)) end - -function statistics.runtime() - stoptiming(statistics) - return statistics.formatruntime(elapsedtime(statistics)) +local function fastdrop(t) + local r={ "return {\n" } + for i=1,#t do + local ti=t[i] + r[#r+1]=" {" + for k,v in next,ti do + r[#r+1]=format("%s=%q",k,v) + end + r[#r+1]="},\n" + end + r[#r+1]="}" + return concat(r) end - -function statistics.timed(action,report) - report = report or logs.reporter("system") - starttiming("run") - action() - stoptiming("run") - report("total runtime: %s",elapsedtime("run")) +function table.drop(t,slow) + if #t==0 then + return "return { }" + elseif slow==true then + return slowdrop(t) + else + return fastdrop(t) + end +end +function table.autokey(t,k) + local v={} + t[k]=v + return v end --- where, not really the best spot for this: -commands = commands or { } +end -- of closure + +do -- create closure to overcome 200 locals limit -function commands.resettimer(name) - resettiming(name or "whatever") - starttiming(name or "whatever") -end +-- original size: 4270, stripped down to: 2989 -function commands.elapsedtime(name) - stoptiming(name or "whatever") - context(elapsedtime(name or "whatever")) +if not modules then modules={} end modules ['util-sto']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local setmetatable,getmetatable=setmetatable,getmetatable +utilities=utilities or {} +utilities.storage=utilities.storage or {} +local storage=utilities.storage +local report=texio and texio.write_nl or print +function storage.mark(t) + if not t then + report("fatal error: storage cannot be marked") + return + end + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.allocate(t) + t=t or {} + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.marked(t) + local m=getmetatable(t) + return m and m.__storage__ +end +function storage.checked(t) + if not t then + report("fatal error: storage has not been allocated") + return + end + return t +end +function storage.setinitializer(data,initialize) + local m=getmetatable(data) or {} + m.__index=function(data,k) + m.__index=nil + initialize() + return data[k] + end + setmetatable(data,m) +end +local keyisvalue={ __index=function(t,k) + t[k]=k + return k +end } +function storage.sparse(t) + t=t or {} + setmetatable(t,keyisvalue) + return t +end +local function f_empty () return "" end +local function f_self (t,k) t[k]=k return k end +local function f_table (t,k) local v={} t[k]=v return v end +local function f_ignore() end +local t_empty={ __index=f_empty } +local t_self={ __index=f_self } +local t_table={ __index=f_table } +local t_ignore={ __newindex=f_ignore } +function table.setmetatableindex(t,f) + local m=getmetatable(t) + if m then + if f=="empty" then + m.__index=f_empty + elseif f=="key" then + m.__index=f_self + elseif f=="table" then + m.__index=f_table + else + m.__index=f + end + else + if f=="empty" then + setmetatable(t,t_empty) + elseif f=="key" then + setmetatable(t,t_self) + elseif f=="table" then + setmetatable(t,t_table) + else + setmetatable(t,{ __index=f }) + end + end + return t +end +function table.setmetatablenewindex(t,f) + local m=getmetatable(t) + if m then + if f=="ignore" then + m.__newindex=f_ignore + else + m.__newindex=f + end + else + if f=="ignore" then + setmetatable(t,t_ignore) + else + setmetatable(t,{ __newindex=f }) + end + end + return t +end +function table.setmetatablecall(t,f) + local m=getmetatable(t) + if m then + m.__call=f + else + setmetatable(t,{ __call=f }) + end + return t +end +function table.setmetatablekey(t,key,value) + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m[key]=value + return t +end +function table.getmetatablekey(t,key,value) + local m=getmetatable(t) + return m and m[key] end @@ -7959,382 +4434,1165 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-set'] = { -- might become util-set.lua - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +-- original size: 11610, stripped down to: 7440 + +if not modules then modules={} end modules ['util-str']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.strings=utilities.strings or {} +local strings=utilities.strings +local load=load +local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local concat=table.concat +local P,V,C,S,R,Ct,Cs,Cp,Carg=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg +local patterns,lpegmatch=lpeg.patterns,lpeg.match +local utfchar,utfbyte=utf.char,utf.byte +local setmetatableindex=table.setmetatableindex +local stripper=patterns.stripzeros +local function points(n) + return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) +end +local function basepoints(n) + return (not n or n==0) and "0bp" or lpegmatch(stripper,format("%.5fbp",n*(7200/7227)/65536)) +end +number.points=points +number.basepoints=basepoints +local rubish=patterns.spaceortab^0*patterns.newline +local anyrubish=patterns.spaceortab+patterns.newline +local anything=patterns.anything +local stripped=(patterns.spaceortab^1/"")*patterns.newline +local leading=rubish^0/"" +local trailing=(anyrubish^1*patterns.endofstring)/"" +local redundant=rubish^3/"\n" +local pattern=Cs(leading*(trailing+redundant+stripped+anything)^0) +function strings.collapsecrlf(str) + return lpegmatch(pattern,str) +end +local repeaters={} +function strings.newrepeater(str,offset) + offset=offset or 0 + local s=repeaters[str] + if not s then + s={} + repeaters[str]=s + end + local t=s[offset] + if t then + return t + end + t={} + setmetatableindex(t,function(t,k) + if not k then + return "" + end + local n=k+offset + local s=n>0 and rep(str,n) or "" + t[k]=s + return s + end) + s[offset]=t + return t +end +local extra,tab,start=0,0,4,0 +local nspaces=strings.newrepeater(" ") +local pattern=Carg(1)/function(t) + extra,tab,start=0,t or 7,1 + end*Cs(( + Cp()*patterns.tab/function(position) + local current=(position-start+1)+extra + local spaces=tab-(current-1)%tab + if spaces>0 then + extra=extra+spaces-1 + return nspaces[spaces] + else + return "" + end + end+patterns.newline*Cp()/function(position) + extra,start=0,position + end+patterns.anything + )^1) +function strings.tabtospace(str,tab) + return lpegmatch(pattern,str,1,tab or 7) +end +function strings.striplong(str) + str=gsub(str,"^%s*","") + str=gsub(str,"[\n\r]+ *","\n") + return str +end +function strings.nice(str) + str=gsub(str,"[:%-+_]+"," ") + return str +end +local n=0 +local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_tab=C((1-R("az","AZ","09","%%"))^0) +local format_s=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%ss',(select(%s,...)))",f,n) + else + return format("(select(%s,...))",n) + end +end +local format_q=function() + n=n+1 + return format("format('%%q',(select(%s,...)))",n) +end +local format_i=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%si',(select(%s,...)))",f,n) + else + return format("(select(%s,...))",n) + end +end +local format_d=format_i +local format_f=function(f) + n=n+1 + return format("format('%%%sf',(select(%s,...)))",f,n) +end +local format_g=function(f) + n=n+1 + return format("format('%%%sg',(select(%s,...)))",f,n) +end +local format_G=function(f) + n=n+1 + return format("format('%%%sG',(select(%s,...)))",f,n) +end +local format_e=function(f) + n=n+1 + return format("format('%%%se',(select(%s,...)))",f,n) +end +local format_E=function(f) + n=n+1 + return format("format('%%%sE',(select(%s,...)))",f,n) +end +local format_x=function(f) + n=n+1 + return format("format('%%%sx',(select(%s,...)))",f,n) +end +local format_X=function(f) + n=n+1 + return format("format('%%%sX',(select(%s,...)))",f,n) +end +local format_o=function(f) + n=n+1 + return format("format('%%%so',(select(%s,...)))",f,n) +end +local format_c=function() + n=n+1 + return format("utfchar((select(%s,...)))",n) +end +local format_r=function(f) + n=n+1 + return format("format('%%%s.0f',(select(%s,...)))",f,n) +end +local format_v=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('0x%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_V=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('0x%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_u=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('u+%%%sx',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_U=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + else + return format("format('U+%%%sX',utfbyte((select(%s,...))))",f=="" and "05" or f,n) + end +end +local format_p=function() + n=n+1 + return format("points((select(%s,...)))",n) +end +local format_b=function() + n=n+1 + return format("basepoints((select(%s,...)))",n) +end +local format_t=function(f) + n=n+1 + if f and f~="" then + return format("concat((select(%s,...)),%q)",n,f) + else + return format("concat((select(%s,...)))",n) + end +end +local format_l=function() + n=n+1 + return format("(select(%s,...) and 'true' or 'false')",n) +end +local format_a=function(s) + return format("%q",s) +end +local builder=Ct { "start", + start=(P("%")*( + V("s")+V("q")+V("i")+V("d")+V("f")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") ++V("c") ++V("r")+V("v")+V("V")+V("u")+V("U")+V("p")+V("b")+V("t")+V("l") + )+V("a") + )^0, + ["s"]=(prefix_any*P("s"))/format_s, + ["q"]=(prefix_any*P("q"))/format_q, + ["i"]=(prefix_any*P("i"))/format_i, + ["d"]=(prefix_any*P("d"))/format_d, + ["f"]=(prefix_any*P("f"))/format_f, + ["g"]=(prefix_any*P("g"))/format_g, + ["G"]=(prefix_any*P("G"))/format_G, + ["e"]=(prefix_any*P("e"))/format_e, + ["E"]=(prefix_any*P("E"))/format_E, + ["x"]=(prefix_any*P("x"))/format_x, + ["X"]=(prefix_any*P("X"))/format_X, + ["o"]=(prefix_any*P("o"))/format_o, + ["c"]=(prefix_any*P("c"))/format_c, + ["r"]=(prefix_any*P("r"))/format_r, + ["v"]=(prefix_any*P("v"))/format_v, + ["V"]=(prefix_any*P("V"))/format_V, + ["u"]=(prefix_any*P("u"))/format_u, + ["U"]=(prefix_any*P("U"))/format_U, + ["p"]=(prefix_any*P("p"))/format_p, + ["b"]=(prefix_any*P("b"))/format_b, + ["t"]=(prefix_tab*P("t"))/format_t, + ["l"]=(prefix_tab*P("l"))/format_l, + ["a"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_a, } +local template=[[ +local format = string.format +local concat = table.concat +local points = number.points +local basepoints = number.basepoints +local utfchar = utf.char +local utfbyte = utf.byte +return function(...) + return %s +end +]] +local function make(t,str) + n=0 + local p=lpegmatch(builder,str) + local c=format(template,concat(p,"..")) + formatter=load(c)() + t[str]=formatter + return formatter +end +local formatters=string.formatters or {} +string.formatters=formatters +setmetatableindex(formatters,make) +function string.makeformatter(str) + return formatters[str] +end +function string.formatter(str,...) + return formatters[str](...) +end --- maybe this should be util-set.lua -local type, next, tostring = type, next, tostring -local concat = table.concat -local format, find, lower, gsub, topattern = string.format, string.find, string.lower, string.gsub, string.topattern -local is_boolean = string.is_boolean -local settings_to_hash = utilities.parsers.settings_to_hash -local allocate = utilities.storage.allocate - -utilities = utilities or { } -local utilities = utilities -utilities.setters = utilities.setters or { } -local setters = utilities.setters - -local data = { } -- maybe just local - --- We can initialize from the cnf file. This is sort of tricky as --- later defined setters also need to be initialized then. If set --- this way, we need to ensure that they are not reset later on. - -local trace_initialize = false -- only for testing during development - -function setters.initialize(filename,name,values) -- filename only for diagnostics - local setter = data[name] - if setter then - frozen = true -- don't permitoverload --- trace_initialize = true - local data = setter.data - if data then - for key, newvalue in next, values do - local newvalue = is_boolean(newvalue,newvalue) - local functions = data[key] - if functions then - local oldvalue = functions.value - if functions.frozen then - if trace_initialize then - setter.report("%s: %q is frozen to %q",filename,key,tostring(oldvalue)) - end - elseif #functions > 0 and not oldvalue then --- elseif #functions > 0 and oldvalue == nil then - if trace_initialize then - setter.report("%s: %q is set to %q",filename,key,tostring(newvalue)) - end - for i=1,#functions do - functions[i](newvalue) - end - functions.value = newvalue - functions.frozen = functions.frozen or frozen - else - if trace_initialize then - setter.report("%s: %q is kept as %q",filename,key,tostring(oldvalue)) - end - end - else - -- we do a simple preregistration i.e. not in the - -- list as it might be an obsolete entry - functions = { default = newvalue, frozen = frozen } - data[key] = functions - if trace_initialize then - setter.report("%s: %q default to %q",filename,key,tostring(newvalue)) - end - end - end - return true - end +end -- of closure + +do -- create closure to overcome 200 locals limit + +-- original size: 7050, stripped down to: 5641 + +if not modules then modules={} end modules ['util-mrg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gsub,format=string.gsub,string.format +local concat=table.concat +local type,next=type,next +utilities=utilities or {} +local merger=utilities.merger or {} +utilities.merger=merger +utilities.report=logs and logs.reporter("system") or print +merger.strip_comment=true +local m_begin_merge="begin library merge" +local m_end_merge="end library merge" +local m_begin_closure="do -- create closure to overcome 200 locals limit" +local m_end_closure="end -- of closure" +local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" +local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" +local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" +local m_report=[[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] +local function self_fake() + return m_faked +end +local function self_nothing() + return "" +end +local function self_load(name) + local data=io.loaddata(name) or "" + if data=="" then + utilities.report("merge: unknown file %s",name) + else + utilities.report("merge: inserting %s",name) + end + return data or "" +end +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local equals=P("=")^0 +local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 +local close=P("]")*C(equals)*P("]") +local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) +local longstring=open*(1-closeeq)^0*close +local space=patterns.space +local eol=patterns.newline +local quoted=patterns.quoted +local emptyline=space^0*eol +local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") +local operator2=S("*+/") +local operator3=S("-") +local separator=S(",;") +local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" +local strings=quoted +local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" +local longstr=longstring +local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" +local pack=((eol+space)^0/"")*operator1*((eol+space)^0/"")+((eol+space)^0/"")*operator2*((space)^0/"")+((eol+space)^1/"")*operator3*((space)^1/"")+((space)^0/"")*separator*((space)^0/"") +local lines=emptyline^2/"\n" +local spaces=(space*space)/" " +local compact=Cs (( + ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 +)^1 ) +local strip=Cs((emptyline^2/"\n"+1)^0) +local function self_compact(data) + if merger.strip_comment then + local before=#data + data=lpeg.match(compact,data) + data=lpeg.match(strip,data) + local after=#data + local delta=before-after + utilities.report("merge: %s bytes compacted to %s (%s bytes stripped)",before,after,delta) + data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + return data,delta + else + return data,0 + end +end +local function self_save(name,data) + if data~="" then + io.savedata(name,data) + utilities.report("merge: saving %s bytes in %s",#data,name) + end +end +local function self_swap(data,code) + return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +end +local function self_libs(libs,list) + local result,f,frozen,foundpath={},nil,false,nil + result[#result+1]="\n" + if type(libs)=='string' then libs={ libs } end + if type(list)=='string' then list={ list } end + for i=1,#libs do + local lib=libs[i] + for j=1,#list do + local pth=gsub(list[j],"\\","/") + utilities.report("merge: checking library path %s",pth) + local name=pth.."/"..lib + if lfs.isfile(name) then + foundpath=pth + end end + if foundpath then break end + end + if foundpath then + utilities.report("merge: using library path %s",foundpath) + local right,wrong,original,stripped={},{},0,0 + for i=1,#libs do + local lib=libs[i] + local fullname=foundpath.."/"..lib + if lfs.isfile(fullname) then + utilities.report("merge: using library %s",fullname) + local data=io.loaddata(fullname,true) + original=original+#data + local data,delta=self_compact(data) + right[#right+1]=lib + result[#result+1]=m_begin_closure + result[#result+1]=data + result[#result+1]=m_end_closure + stripped=stripped+delta + else + utilities.report("merge: skipping library %s",fullname) + wrong[#wrong+1]=lib + end + end + right=#right>0 and concat(right," ") or "-" + wrong=#wrong>0 and concat(wrong," ") or "-" + utilities.report("merge: used libraries: %s",right) + utilities.report("merge: skipped libraries: %s",wrong) + utilities.report("merge: original bytes: %s",original) + utilities.report("merge: stripped bytes: %s",stripped) + result[#result+1]=format(m_report,right,wrong,original,stripped) + else + utilities.report("merge: no valid library path found") + end + return concat(result,"\n\n") +end +function merger.selfcreate(libs,list,target) + if target then + self_save(target,self_swap(self_fake(),self_libs(libs,list))) + end +end +function merger.selfmerge(name,libs,list,target) + self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) +end +function merger.selfclean(name) + self_save(name,self_swap(self_load(name),self_nothing())) end --- user interface code -local function set(t,what,newvalue) - local data = t.data - if not data.frozen then - local done = t.done - if type(what) == "string" then - what = settings_to_hash(what) -- inefficient but ok +end -- of closure + +do -- create closure to overcome 200 locals limit + +-- original size: 12211, stripped down to: 8441 + +if not modules then modules={} end modules ['util-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + comment="the strip code is written by Peter Cawley", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local rep,sub,byte,dump,format=string.rep,string.sub,string.byte,string.dump,string.format +local load,loadfile,type=load,loadfile,type +utilities=utilities or {} +utilities.lua=utilities.lua or {} +local luautilities=utilities.lua +utilities.report=logs and logs.reporter("system") or print +local tracestripping=false +local forcestupidcompile=true +luautilities.stripcode=true +luautilities.alwaysstripcode=false +luautilities.nofstrippedchunks=0 +luautilities.nofstrippedbytes=0 +local strippedchunks={} +luautilities.strippedchunks=strippedchunks +luautilities.suffixes={ + tma="tma", + tmc=jit and "tmb" or "tmc", + lua="lua", + luc=jit and "lub" or "luc", + lui="lui", + luv="luv", + luj="luj", + tua="tua", + tuc="tuc", +} +local function fatalerror(name) + utilities.report(format("fatal error in %q",name or "unknown")) +end +if jit or status.luatex_version>=74 then + local function register(name) + if tracestripping then + utilities.report("stripped bytecode: %s",name or "unknown") + end + strippedchunks[#strippedchunks+1]=name + luautilities.nofstrippedchunks=luautilities.nofstrippedchunks+1 + end + local function stupidcompile(luafile,lucfile,strip) + local code=io.loaddata(luafile) + if code and code~="" then + code=load(code) + if code then + code=dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) + if code and code~="" then + register(name) + io.savedata(lucfile,code) + return true,0 + end + else + fatalerror() + end + else + fatalerror() + end + return false,0 + end + function luautilities.loadedluacode(fullname,forcestrip,name) + name=name or fullname + local code=loadfile(fullname) + if code then + code() + end + if forcestrip and luautilities.stripcode then + if type(forcestrip)=="function" then + forcestrip=forcestrip(fullname) + end + if forcestrip or luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end + elseif luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end + end + function luautilities.strippedloadstring(code,forcestrip,name) + if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then + code=load(code) + if not code then + fatalerror(name) + end + register(name) + code=dump(code,true) + end + return load(code),0 + end + function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) + utilities.report("lua: compiling %s into %s",luafile,lucfile) + os.remove(lucfile) + local done=stupidcompile(luafile,lucfile,strip~=false) + if done then + utilities.report("lua: %s dumped into %s (stripped)",luafile,lucfile) + if cleanup==true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + utilities.report("lua: removing %s",luafile) + os.remove(luafile) + end + end + return done + end +else + local function register(name,before,after) + local delta=before-after + if tracestripping then + utilities.report("stripped bytecode: %s, before %s, after %s, delta %s",name or "unknown",before,after,delta) + end + strippedchunks[#strippedchunks+1]=name + luautilities.nofstrippedchunks=luautilities.nofstrippedchunks+1 + luautilities.nofstrippedbytes=luautilities.nofstrippedbytes+delta + return delta + end + local strip_code_pc + if _MAJORVERSION==5 and _MINORVERSION==1 then + strip_code_pc=function(dump,name) + local before=#dump + local version,format,endian,int,size,ins,num=byte(dump,5,11) + local subint + if endian==1 then + subint=function(dump,i,l) + local val=0 + for n=l,1,-1 do + val=val*256+byte(dump,i+n-1) + end + return val,i+l + end + else + subint=function(dump,i,l) + local val=0 + for n=1,l,1 do + val=val*256+byte(dump,i+n-1) + end + return val,i+l end - if type(what) ~= "table" then - return + end + local strip_function + strip_function=function(dump) + local count,offset=subint(dump,1,size) + local stripped,dirty=rep("\0",size),offset+count + offset=offset+count+int*2+4 + offset=offset+int+subint(dump,offset,int)*ins + count,offset=subint(dump,offset,int) + for n=1,count do + local t + t,offset=subint(dump,offset,1) + if t==1 then + offset=offset+1 + elseif t==4 then + offset=offset+size+subint(dump,offset,size) + elseif t==3 then + offset=offset+num + end end - if not done then -- catch ... why not set? - done = { } - t.done = done + count,offset=subint(dump,offset,int) + stripped=stripped..sub(dump,dirty,offset-1) + for n=1,count do + local proto,off=strip_function(sub(dump,offset,-1)) + stripped,offset=stripped..proto,offset+off-1 end - for w, value in next, what do - if value == "" then - value = newvalue - elseif not value then - value = false -- catch nil - else - value = is_boolean(value,value) - end - w = topattern(w,true,true) - for name, functions in next, data do - if done[name] then - -- prevent recursion due to wildcards - elseif find(name,w) then - done[name] = true - for i=1,#functions do - functions[i](value) - end - functions.value = value - end - end + offset=offset+subint(dump,offset,int)*int+int + count,offset=subint(dump,offset,int) + for n=1,count do + offset=offset+subint(dump,offset,size)+size+int*2 end - end -end - -local function reset(t) - local data = t.data - if not data.frozen then - for name, functions in next, data do - for i=1,#functions do - functions[i](false) - end - functions.value = false + count,offset=subint(dump,offset,int) + for n=1,count do + offset=offset+subint(dump,offset,size)+size end + stripped=stripped..rep("\0",int*3) + return stripped,offset + end + dump=sub(dump,1,12)..strip_function(sub(dump,13,-1)) + local after=#dump + local delta=register(name,before,after) + return dump,delta + end + else + strip_code_pc=function(dump,name) + return dump,0 + end + end + function luautilities.loadedluacode(fullname,forcestrip,name) + name=name or fullname + local code=loadfile(fullname) + if code then + code() + end + if forcestrip and luautilities.stripcode then + if type(forcestrip)=="function" then + forcestrip=forcestrip(fullname) + end + if forcestrip then + local code,n=strip_code_pc(dump(code),name) + return load(code),n + elseif luautilities.alwaysstripcode then + return load(strip_code_pc(dump(code),name)) + else + return code,0 + end + elseif luautilities.alwaysstripcode then + return load(strip_code_pc(dump(code),name)) + else + return code,0 + end + end + function luautilities.strippedloadstring(code,forcestrip,name) + local n=0 + if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then + code=load(code) + if not code then + fatalerror(name) + end + code,n=strip_code_pc(dump(code),name) + end + return load(code),n + end + local function stupidcompile(luafile,lucfile,strip) + local code=io.loaddata(luafile) + local n=0 + if code and code~="" then + code=load(code) + if not code then + fatalerror() + end + code=dump(code) + if strip then + code,n=strip_code_pc(code,luautilities.stripcode or luautilities.alwaysstripcode,luafile) + end + if code and code~="" then + io.savedata(lucfile,code) + end + end + return n + end + local luac_normal="texluac -o %q %q" + local luac_strip="texluac -s -o %q %q" + function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) + utilities.report("lua: compiling %s into %s",luafile,lucfile) + os.remove(lucfile) + local done=false + if strip~=false then + strip=true + end + if forcestupidcompile then + fallback=true + elseif strip then + done=os.spawn(format(luac_strip,lucfile,luafile))==0 + else + done=os.spawn(format(luac_normal,lucfile,luafile))==0 + end + if not done and fallback then + local n=stupidcompile(luafile,lucfile,strip) + if n>0 then + utilities.report("lua: %s dumped into %s (%i bytes stripped)",luafile,lucfile,n) + else + utilities.report("lua: %s dumped into %s (unstripped)",luafile,lucfile) + end + cleanup=false + done=true end + if done and cleanup==true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + utilities.report("lua: removing %s",luafile) + os.remove(luafile) + end + return done + end end -local function enable(t,what) - set(t,what,true) -end -local function disable(t,what) - local data = t.data - if not what or what == "" then - t.done = { } - reset(t) - else - set(t,what,false) - end -end +end -- of closure -function setters.register(t,what,...) - local data = t.data - what = lower(what) - local functions = data[what] - if not functions then - functions = { } - data[what] = functions - if trace_initialize then - t.report("defining %s",what) - end - end - local default = functions.default -- can be set from cnf file - for i=1,select("#",...) do - local fnc = select(i,...) - local typ = type(fnc) - if typ == "string" then - if trace_initialize then - t.report("coupling %s to %s",what,fnc) - end - local s = fnc -- else wrong reference - fnc = function(value) set(t,s,value) end - elseif typ ~= "function" then - fnc = nil - end - if fnc then - functions[#functions+1] = fnc - -- default: set at command line or in cnf file - -- value : set in tex run (needed when loading runtime) - local value = functions.value or default - if value ~= nil then - fnc(value) - functions.value = value - end - end - end - return false -- so we can use it in an assignment -end +do -- create closure to overcome 200 locals limit -function setters.enable(t,what) - local e = t.enable - t.enable, t.done = enable, { } - enable(t,what) - t.enable, t.done = e, { } -end +-- original size: 14514, stripped down to: 10374 -function setters.disable(t,what) - local e = t.disable - t.disable, t.done = disable, { } - disable(t,what) - t.disable, t.done = e, { } +if not modules then modules={} end modules ['util-prs']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local lpeg,table,string=lpeg,table,string +local P,R,V,S,C,Ct,Cs,Carg,Cc,Cg,Cf,Cp=lpeg.P,lpeg.R,lpeg.V,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cs,lpeg.Carg,lpeg.Cc,lpeg.Cg,lpeg.Cf,lpeg.Cp +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local concat,format,gmatch,find=table.concat,string.format,string.gmatch,string.find +local tostring,type,next,rawset=tostring,type,next,rawset +utilities=utilities or {} +utilities.parsers=utilities.parsers or {} +local parsers=utilities.parsers +parsers.patterns=parsers.patterns or {} +local setmetatableindex=table.setmetatableindex +local sortedhash=table.sortedhash +local digit=R("09") +local space=P(' ') +local equal=P("=") +local comma=P(",") +local lbrace=P("{") +local rbrace=P("}") +local lparent=P("(") +local rparent=P(")") +local period=S(".") +local punctuation=S(".,:;") +local spacer=patterns.spacer +local whitespace=patterns.whitespace +local newline=patterns.newline +local anything=patterns.anything +local endofstring=patterns.endofstring +local nobrace=1-(lbrace+rbrace ) +local noparent=1-(lparent+rparent) +local escape,left,right=P("\\"),P('{'),P('}') +patterns.balanced=P { + [1]=((escape*(left+right))+(1-(left+right))+V(2))^0, + [2]=left*V(1)*right +} +local nestedbraces=P { lbrace*(nobrace+V(1))^0*rbrace } +local nestedparents=P { lparent*(noparent+V(1))^0*rparent } +local spaces=space^0 +local argument=Cs((lbrace/"")*((nobrace+nestedbraces)^0)*(rbrace/"")) +local content=(1-endofstring)^0 +patterns.nestedbraces=nestedbraces +patterns.nestedparents=nestedparents +patterns.nested=nestedbraces +patterns.argument=argument +patterns.content=content +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local key=C((1-equal-comma)^1) +local pattern_a=(space+comma)^0*(key*equal*value+key*C("")) +local pattern_c=(space+comma)^0*(key*equal*value) +local key=C((1-space-equal-comma)^1) +local pattern_b=spaces*comma^0*spaces*(key*((spaces*equal*spaces*value)+C(""))) +local hash={} +local function set(key,value) + hash[key]=value +end +local pattern_a_s=(pattern_a/set)^1 +local pattern_b_s=(pattern_b/set)^1 +local pattern_c_s=(pattern_c/set)^1 +parsers.patterns.settings_to_hash_a=pattern_a_s +parsers.patterns.settings_to_hash_b=pattern_b_s +parsers.patterns.settings_to_hash_c=pattern_c_s +function parsers.make_settings_to_hash_pattern(set,how) + if how=="strict" then + return (pattern_c/set)^1 + elseif how=="tolerant" then + return (pattern_b/set)^1 + else + return (pattern_a/set)^1 + end end - -function setters.reset(t) - t.done = { } - reset(t) +function parsers.settings_to_hash(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_a_s,str) + return hash + else + return {} + end end - -function setters.list(t) -- pattern - local list = table.sortedkeys(t.data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what +function parsers.settings_to_hash_tolerant(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_b_s,str) + return hash + else + return {} + end +end +function parsers.settings_to_hash_strict(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_c_s,str) + return next(hash) and hash + else + return nil + end +end +local separator=comma*space^0 +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local pattern=spaces*Ct(value*(separator*value)^0) +parsers.patterns.settings_to_array=pattern +function parsers.settings_to_array(str,strict) + if not str or str=="" then + return {} + elseif strict then + if find(str,"{") then + return lpegmatch(pattern,str) + else + return { str } + end + else + return lpegmatch(pattern,str) + end +end +local function set(t,v) + t[#t+1]=v +end +local value=P(Carg(1)*value)/set +local pattern=value*(separator*value)^0*Carg(1) +function parsers.add_settings_to_array(t,str) + return lpegmatch(pattern,str,nil,t) +end +function parsers.hash_to_string(h,separator,yes,no,strict,omit) + if h then + local t,tn,s={},0,table.sortedkeys(h) + omit=omit and table.tohash(omit) + for i=1,#s do + local key=s[i] + if not omit or not omit[key] then + local value=h[key] + if type(value)=="boolean" then + if yes and no then + if value then + tn=tn+1 + t[tn]=key..'='..yes + elseif not strict then + tn=tn+1 + t[tn]=key..'='..no + end + elseif value or not strict then + tn=tn+1 + t[tn]=key..'='..tostring(value) + end else - user[#user+1] = what + tn=tn+1 + t[tn]=key..'='..value end + end end - return user, system + return concat(t,separator or ",") + else + return "" + end end - -function setters.show(t) - local category = t.name - local list = setters.list(t) - t.report() - for k=1,#list do - local name = list[k] - local functions = t.data[name] - if functions then - local value, default, modules = functions.value, functions.default, #functions - value = value == nil and "unset" or tostring(value) - default = default == nil and "unset" or tostring(default) - t.report("%-50s modules: %2i default: %-12s value: %-12s",name,modules,default,value) - end +function parsers.array_to_string(a,separator) + if a then + return concat(a,separator or ",") + else + return "" + end +end +function parsers.settings_to_set(str,t) + t=t or {} + for s in gmatch(str,"[^, ]+") do + t[s]=true + end + return t +end +function parsers.simple_hash_to_string(h,separator) + local t,tn={},0 + for k,v in sortedhash(h) do + if v then + tn=tn+1 + t[tn]=k end - t.report() + end + return concat(t,separator or ",") end - --- we could have used a bit of oo and the trackers:enable syntax but --- there is already a lot of code around using the singular tracker - --- we could make this into a module but we also want the rest avaliable - -local enable, disable, register, list, show = setters.enable, setters.disable, setters.register, setters.list, setters.show - -local write_nl = texio and texio.write_nl or print - -local function report(setter,...) - local report = logs and logs.report - if report then - report(setter.name,...) - else -- fallback, as this module is loaded before the logger - write_nl(format("%-15s : %s\n",setter.name,format(...))) +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) +local pattern_a=spaces*Ct(value*(separator*value)^0) +local function repeater(n,str) + if not n then + return str + else + local s=lpegmatch(pattern_a,str) + if n==1 then + return unpack(s) + else + local t,tn={},0 + for i=1,n do + for j=1,#s do + tn=tn+1 + t[tn]=s[j] + end + end + return unpack(t) end + end end - -local function default(setter,name) - local d = setter.data[name] - return d and d.default +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+(C(digit^1)/tonumber*lparent*Cs((noparent+nestedparents)^1)*rparent)/repeater+C((nestedbraces+(1-comma))^1) +local pattern_b=spaces*Ct(value*(separator*value)^0) +function parsers.settings_to_array_with_repeat(str,expand) + if expand then + return lpegmatch(pattern_b,str) or {} + else + return lpegmatch(pattern_a,str) or {} + end end - -local function value(setter,name) - local d = setter.data[name] - return d and (d.value or d.default) -end - -function setters.new(name) -- we could use foo:bar syntax (but not used that often) - local setter -- we need to access it in setter itself - setter = { - data = allocate(), -- indexed, but also default and value fields - name = name, - report = function(...) report (setter,...) end, - enable = function(...) enable (setter,...) end, - disable = function(...) disable (setter,...) end, - register = function(...) register(setter,...) end, - list = function(...) list (setter,...) end, - show = function(...) show (setter,...) end, - default = function(...) return default (setter,...) end, - value = function(...) return value (setter,...) end, - } - data[name] = setter - return setter +local value=lbrace*C((nobrace+nestedbraces)^0)*rbrace +local pattern=Ct((space+value)^0) +function parsers.arguments_to_table(str) + return lpegmatch(pattern,str) end - -trackers = setters.new("trackers") -directives = setters.new("directives") -experiments = setters.new("experiments") - -local t_enable, t_disable, t_report = trackers .enable, trackers .disable, trackers .report -local d_enable, d_disable, d_report = directives .enable, directives .disable, directives .report -local e_enable, e_disable, e_report = experiments.enable, experiments.disable, experiments.report - --- nice trick: we overload two of the directives related functions with variants that --- do tracing (itself using a tracker) .. proof of concept - -local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) -local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) - -function directives.enable(...) - if trace_directives then - d_report("enabling: %s",concat({...}," ")) +function parsers.getparameters(self,class,parentclass,settings) + local sc=self[class] + if not sc then + sc={} + self[class]=sc + if parentclass then + local sp=self[parentclass] + if not sp then + sp={} + self[parentclass]=sp + end + setmetatableindex(sc,sp) end - d_enable(...) + end + parsers.settings_to_hash(settings,sc) end - -function directives.disable(...) - if trace_directives then - d_report("disabling: %s",concat({...}," ")) - end - d_disable(...) +function parsers.listitem(str) + return gmatch(str,"[^, ]+") +end +local pattern=Cs { "start", + start=V("one")+V("two")+V("three"), + rest=(Cc(",")*V("thousand"))^0*(P(".")+endofstring)*anything^0, + thousand=digit*digit*digit, + one=digit*V("rest"), + two=digit*digit*V("rest"), + three=V("thousand")*V("rest"), +} +patterns.splitthousands=pattern +function parsers.splitthousands(str) + return lpegmatch(pattern,str) or str +end +local optionalwhitespace=whitespace^0 +patterns.words=Ct((Cs((1-punctuation-whitespace)^1)+anything)^1) +patterns.sentences=Ct((optionalwhitespace*Cs((1-period)^0*period))^1) +patterns.paragraphs=Ct((optionalwhitespace*Cs((whitespace^1*endofstring/""+1-(spacer^0*newline*newline))^1))^1) +local dquote=P('"') +local equal=P('=') +local escape=P('\\') +local separator=S(' ,') +local key=C((1-equal)^1) +local value=dquote*C((1-dquote-escape*dquote)^0)*dquote +local pattern=Cf(Ct("")*Cg(key*equal*value)*separator^0,rawset)^0 +parsers.patterns.keq_to_hash_c=pattern +function parsers.keq_to_hash(str) + if str and str~="" then + return lpegmatch(pattern,str) + else + return {} + end end - -function experiments.enable(...) - if trace_experiments then - e_report("enabling: %s",concat({...}," ")) +local defaultspecification={ separator=",",quote='"' } +function parsers.csvsplitter(specification) + specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=specification.quote + local separator=S(separator~="" and separator or ",") + local whatever=C((1-separator-newline)^0) + if quotechar and quotechar~="" then + local quotedata=nil + for chr in gmatch(quotechar,".") do + local quotechar=P(chr) + local quoteword=quotechar*C((1-quotechar)^0)*quotechar + if quotedata then + quotedata=quotedata+quoteword + else + quotedata=quoteword + end end - e_enable(...) + whatever=quotedata+whatever + end + local parser=Ct((Ct(whatever*(separator*whatever)^0)*S("\n\r"))^0 ) + return function(data) + return lpegmatch(parser,data) + end end - -function experiments.disable(...) - if trace_experiments then - e_report("disabling: %s",concat({...}," ")) - end - e_disable(...) +function parsers.rfc4180splitter(specification) + specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=P(specification.quote) + local dquotechar=quotechar*quotechar +/specification.quote + local separator=S(separator~="" and separator or ",") + local escaped=quotechar*Cs((dquotechar+(1-quotechar))^0)*quotechar + local non_escaped=C((1-quotechar-newline-separator)^1) + local field=escaped+non_escaped + local record=Ct((field*separator^-1)^1) + local headerline=record*Cp() + local wholeblob=Ct((newline^-1*record)^0) + return function(data,getheader) + if getheader then + local header,position=lpegmatch(headerline,data) + local data=lpegmatch(wholeblob,data,position) + return data,header + else + return lpegmatch(wholeblob,data) + end + end +end +local function ranger(first,last,n,action) + if not first then + elseif last==true then + for i=first,n or first do + action(i) + end + elseif last then + for i=first,last do + action(i) + end + else + action(first) + end +end +local cardinal=patterns.cardinal/tonumber +local spacers=patterns.spacer^0 +local endofstring=patterns.endofstring +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1 +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+(P("*")+endofstring)*Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1*endofstring +function utilities.parsers.stepper(str,n,action) + if type(n)=="function" then + lpegmatch(stepper,str,1,false,n or print) + else + lpegmatch(stepper,str,1,n,action or print) + end end --- a useful example - -directives.register("system.nostatistics", function(v) - statistics.enable = not v -end) - -directives.register("system.nolibraries", function(v) - libraries = nil -- we discard this tracing for security -end) - --- experiment -if environment then +end -- of closure - -- The engineflags are known earlier than environment.arguments but maybe we - -- need to handle them both as the later are parsed differently. The c: prefix - -- is used by mtx-context to isolate the flags from those that concern luatex. +do -- create closure to overcome 200 locals limit - local engineflags = environment.engineflags +-- original size: 3006, stripped down to: 2072 - if engineflags then - local list = engineflags["c:trackers"] or engineflags["trackers"] - if type(list) == "string" then - setters.initialize("commandline flags","trackers",settings_to_hash(list)) - -- t_enable(list) - end - local list = engineflags["c:directives"] or engineflags["directives"] - if type(list) == "string" then - setters.initialize("commandline flags","directives", settings_to_hash(list)) - -- d_enable(list) +if not modules then modules={} end modules ['util-fmt']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.formatters=utilities.formatters or {} +local formatters=utilities.formatters +local concat,format=table.concat,string.format +local tostring,type=tostring,type +local strip=string.strip +local P,R,Cs=lpeg.P,lpeg.R,lpeg.Cs +local lpegmatch=lpeg.match +local digit=R("09") +local period=P(".") +local zero=P("0") +local trailingzeros=zero^0*-digit +local case_1=period*trailingzeros/"" +local case_2=period*(digit-trailingzeros)^1*(trailingzeros/"") +local number=digit^1*(case_1+case_2) +local stripper=Cs((number+1)^0) +lpeg.patterns.stripzeros=stripper +function formatters.stripzeros(str) + return lpegmatch(stripper,str) +end +function formatters.formatcolumns(result,between) + if result and #result>0 then + between=between or " " + local widths,numbers={},{} + local first=result[1] + local n=#first + for i=1,n do + widths[i]=0 + end + for i=1,#result do + local r=result[i] + for j=1,n do + local rj=r[j] + local tj=type(rj) + if tj=="number" then + numbers[j]=true + end + if tj~="string" then + rj=tostring(rj) + r[j]=rj + end + local w=#rj + if w>widths[j] then + widths[j]=w end + end end - -end - --- here - -if texconfig then - - -- this happens too late in ini mode but that is no problem - - local function set(k,v) - v = tonumber(v) - if v then - texconfig[k] = v + for i=1,n do + local w=widths[i] + if numbers[i] then + if w>80 then + widths[i]="%s"..between + else + widths[i]="%0"..w.."i"..between + end + else + if w>80 then + widths[i]="%s"..between + elseif w>0 then + widths[i]="%-"..w.."s"..between + else + widths[i]="%s" end + end end - - directives.register("luatex.expanddepth", function(v) set("expand_depth",v) end) - directives.register("luatex.hashextra", function(v) set("hash_extra",v) end) - directives.register("luatex.nestsize", function(v) set("nest_size",v) end) - directives.register("luatex.maxinopen", function(v) set("max_in_open",v) end) - directives.register("luatex.maxprintline", function(v) set("max_print_line",v) end) - directives.register("luatex.maxstrings", function(v) set("max_strings",v) end) - directives.register("luatex.paramsize", function(v) set("param_size",v) end) - directives.register("luatex.savesize", function(v) set("save_size",v) end) - directives.register("luatex.stacksize", function(v) set("stack_size",v) end) - + local template=strip(concat(widths)) + for i=1,#result do + local str=format(template,unpack(result[i])) + result[i]=strip(str) + end + end + return result end @@ -8342,1234 +5600,1558 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-log'] = { - version = 1.001, - comment = "companion to trac-log.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- todo: less categories, more subcategories (e.g. nodes) - - -local write_nl, write = texio and texio.write_nl or print, texio and texio.write or io.write -local format, gmatch, find = string.format, string.gmatch, string.find -local concat, insert, remove = table.concat, table.insert, table.remove -local topattern = string.topattern -local texcount = tex and tex.count -local next, type, select = next, type, select - -local setmetatableindex = table.setmetatableindex -local formatters = string.formatters - - - -logs = logs or { } -local logs = logs - -local moreinfo = [[ -More information about ConTeXt and the tools that come with it can be found at: - -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] - --- basic loggers - -local function ignore() end - -setmetatableindex(logs, function(t,k) t[k] = ignore ; return ignore end) - -local report, subreport, status, settarget, setformats, settranslations +-- original size: 4118, stripped down to: 2901 -local direct, subdirect, writer, pushtarget, poptarget - -if tex and (tex.jobname or tex.formatname) then +if not modules then modules={} end modules ['util-deb']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local debug=require "debug" +local getinfo=debug.getinfo +local type,next,tostring=type,next,tostring +local format,find=string.format,string.find +local is_boolean=string.is_boolean +utilities=utilities or {} +utilities.debugger=utilities.debugger or {} +local debugger=utilities.debugger +local counters={} +local names={} +local function hook() + local f=getinfo(2) + if f then + local n="unknown" + if f.what=="C" then + n=f.name or '' + if not names[n] then + names[n]=format("%42s",n) + end + else + n=f.name or f.namewhat or f.what + if not n or n=="" then + n="?" + end + if not names[n] then + names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + end + counters[n]=(counters[n] or 0)+1 + end +end +function debugger.showstats(printer,threshold) + printer=printer or texio.write or print + threshold=threshold or 0 + local total,grandtotal,functions=0,0,0 + local dataset={} + for name,count in next,counters do + dataset[#dataset+1]={ name,count } + end + table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + for i=1,#dataset do + local d=dataset[i] + local name=d[1] + local count=d[2] + if count>threshold and not find(name,"for generator") then + printer(format("%8i %s\n",count,names[name])) + total=total+count + end + grandtotal=grandtotal+count + functions=functions+1 + end + printer("\n") + printer(format("functions : % 10i\n",functions)) + printer(format("total : % 10i\n",total)) + printer(format("grand total: % 10i\n",grandtotal)) + printer(format("threshold : % 10i\n",threshold)) +end +function debugger.savestats(filename,threshold) + local f=io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end +function debugger.enable() + debug.sethook(hook,"c") +end +function debugger.disable() + debug.sethook() +end +local is_node=node and node.is_node +local is_lpeg=lpeg and lpeg.type +function inspect(i) + local ti=type(i) + if ti=="table" then + table.print(i,"table") + elseif is_node and is_node(i) then + table.print(nodes.astable(i),tostring(i)) + elseif is_lpeg and is_lpeg(i) then + lpeg.print(i) + else + print(tostring(i)) + end + return i +end +function traceback() + local level=1 + while true do + local info=debug.getinfo(level,"Sl") + if not info then + break + elseif info.what=="C" then + print(format("%3i : C function",level)) + else + print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) + end + level=level+1 + end +end - local valueiskey = { __index = function(t,k) t[k] = k return k end } -- will be helper - local target = "term and log" +end -- of closure - logs.flush = io.flush +do -- create closure to overcome 200 locals limit - local formats = { } setmetatable(formats, valueiskey) - local translations = { } setmetatable(translations,valueiskey) +-- original size: 6209, stripped down to: 4958 - writer = function(...) - write_nl(target,...) +if not modules then modules={} end modules ['trac-inf']={ + version=1.001, + comment="companion to trac-inf.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower=string.format,string.lower +local concat=table.concat +local clock=os.gettimeofday or os.clock +local write_nl=texio and texio.write_nl or print +statistics=statistics or {} +local statistics=statistics +statistics.enable=true +statistics.threshold=0.01 +local statusinfo,n,registered,timers={},0,{},{} +table.setmetatableindex(timers,function(t,k) + local v={ timing=0,loadtime=0 } + t[k]=v + return v +end) +local function hastiming(instance) + return instance and timers[instance] +end +local function resettiming(instance) + timers[instance or "notimer"]={ timing=0,loadtime=0 } +end +local function starttiming(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing or 0 + if it==0 then + timer.starttime=clock() + if not timer.loadtime then + timer.loadtime=0 + end + end + timer.timing=it+1 +end +local function stoptiming(instance,report) + local timer=timers[instance or "notimer"] + local it=timer.timing + if it>1 then + timer.timing=it-1 + else + local starttime=timer.starttime + if starttime then + local stoptime=clock() + local loadtime=stoptime-starttime + timer.stoptime=stoptime + timer.loadtime=timer.loadtime+loadtime + if report then + statistics.report("load time %0.3f",loadtime) + end + timer.timing=0 + return loadtime end - - newline = function() - write_nl(target,"\n") + end + return 0 +end +local function elapsedtime(instance) + local timer=timers[instance or "notimer"] + return format("%0.3f",timer and timer.loadtime or 0) +end +local function elapsedindeed(instance) + local timer=timers[instance or "notimer"] + return (timer and timer.loadtime or 0)>statistics.threshold +end +local function elapsedseconds(instance,rest) + if elapsedindeed(instance) then + return format("%s seconds %s",elapsedtime(instance),rest or "") + end +end +statistics.hastiming=hastiming +statistics.resettiming=resettiming +statistics.starttiming=starttiming +statistics.stoptiming=stoptiming +statistics.elapsedtime=elapsedtime +statistics.elapsedindeed=elapsedindeed +statistics.elapsedseconds=elapsedseconds +function statistics.register(tag,fnc) + if statistics.enable and type(fnc)=="function" then + local rt=registered[tag] or (#statusinfo+1) + statusinfo[rt]={ tag,fnc } + registered[tag]=rt + if #tag>n then n=#tag end + end +end +function statistics.show(reporter) + if statistics.enable then + if not reporter then reporter=function(tag,data,n) write_nl(tag.." "..data) end end + local register=statistics.register + register("luatex banner",function() + return lower(status.banner) + end) + register("control sequences",function() + return format("%s of %s + %s",status.cs_count,status.hash_size,status.hash_extra) + end) + register("callbacks",function() + local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 + return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) + end) + if jit then + local status={ jit.status() } + if status[1] then + register("luajit status",function() + return concat(status," ",2) + end) + end end - - local f_one = formatters["%-15s > %s\n"] - local f_two = formatters["%-15s >\n"] - - report = function(a,b,c,...) - if c then - write_nl(target,f_one(translations[a],format(formats[b],c,...))) - elseif b then - write_nl(target,f_one(translations[a],formats[b])) - elseif a then - write_nl(target,f_two(translations[a])) - else - write_nl(target,"\n") - end + collectgarbage("collect") + register("current memory usage",statistics.memused) + register("runtime",statistics.runtime) + for i=1,#statusinfo do + local s=statusinfo[i] + local r=s[2]() + if r then + reporter(s[1],r,n) + end end - - local f_one = formatters["%-15s > %s"] - local f_two = formatters["%-15s >"] - - direct = function(a,b,c,...) - if c then - return f_one(translations[a],format(formats[b],c,...)) - elseif b then - return f_one(translations[a],formats[b]) - elseif a then - return f_two(translations[a]) - else - return "" - end + write_nl("") + statistics.enable=false + end +end +local template,report_statistics,nn=nil,nil,0 +function statistics.showjobstat(tag,data,n) + if not logs then + elseif type(data)=="table" then + for i=1,#data do + statistics.showjobstat(tag,data[i],n) end - - local f_one = formatters["%-15s > %s > %s\n"] - local f_two = formatters["%-15s > %s >\n"] - - subreport = function(a,s,b,c,...) - if c then - write_nl(target,f_one(translations[a],translations[s],format(formats[b],c,...))) - elseif b then - write_nl(target,f_one(translations[a],translations[s],formats[b])) - elseif a then - write_nl(target,f_two(translations[a],translations[s])) - else - write_nl(target,"\n") - end + else + if not template or n>nn then + template,n=format("%%-%ss - %%s",n),nn + report_statistics=logs.reporter("mkiv lua stats") end + report_statistics(format(template,tag,data)) + end +end +function statistics.memused() + local round=math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000),round(status.luastate_bytes/1000000)) +end +starttiming(statistics) +function statistics.formatruntime(runtime) + return format("%s seconds",runtime) +end +function statistics.runtime() + stoptiming(statistics) + return statistics.formatruntime(elapsedtime(statistics)) +end +function statistics.timed(action,report) + report=report or logs.reporter("system") + starttiming("run") + action() + stoptiming("run") + report("total runtime: %s",elapsedtime("run")) +end +commands=commands or {} +function commands.resettimer(name) + resettiming(name or "whatever") + starttiming(name or "whatever") +end +function commands.elapsedtime(name) + stoptiming(name or "whatever") + context(elapsedtime(name or "whatever")) +end - local f_one = formatters["%-15s > %s > %s"] - local f_two = formatters["%-15s > %s >"] - - subdirect = function(a,s,b,c,...) - if c then - return f_one(translations[a],translations[s],format(formats[b],c,...)) - elseif b then - return f_one(translations[a],translations[s],formats[b]) - elseif a then - return f_two(translations[a],translations[s]) - else - return "" - end - end - local f_one = formatters["%-15s : %s\n"] - local f_two = formatters["%-15s :\n"] +end -- of closure - status = function(a,b,c,...) - if c then - write_nl(target,f_one(translations[a],format(formats[b],c,...))) - elseif b then - write_nl(target,f_one(translations[a],formats[b])) - elseif a then - write_nl(target,f_two(translations[a])) - else - write_nl(target,"\n") - end - end +do -- create closure to overcome 200 locals limit - local targets = { - logfile = "log", - log = "log", - file = "log", - console = "term", - terminal = "term", - both = "term and log", - } +-- original size: 12560, stripped down to: 8979 - settarget = function(whereto) - target = targets[whereto or "both"] or targets.both - if target == "term" or target == "term and log" then - logs.flush = io.flush +if not modules then modules={} end modules ['trac-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring=type,next,tostring +local concat=table.concat +local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local is_boolean=string.is_boolean +local settings_to_hash=utilities.parsers.settings_to_hash +local allocate=utilities.storage.allocate +utilities=utilities or {} +local utilities=utilities +utilities.setters=utilities.setters or {} +local setters=utilities.setters +local data={} +local trace_initialize=false +function setters.initialize(filename,name,values) + local setter=data[name] + if setter then + frozen=true + local data=setter.data + if data then + for key,newvalue in next,values do + local newvalue=is_boolean(newvalue,newvalue) + local functions=data[key] + if functions then + local oldvalue=functions.value + if functions.frozen then + if trace_initialize then + setter.report("%s: %q is frozen to %q",filename,key,tostring(oldvalue)) + end + elseif #functions>0 and not oldvalue then + if trace_initialize then + setter.report("%s: %q is set to %q",filename,key,tostring(newvalue)) + end + for i=1,#functions do + functions[i](newvalue) + end + functions.value=newvalue + functions.frozen=functions.frozen or frozen + else + if trace_initialize then + setter.report("%s: %q is kept as %q",filename,key,tostring(oldvalue)) + end + end else - logs.flush = ignore + functions={ default=newvalue,frozen=frozen } + data[key]=functions + if trace_initialize then + setter.report("%s: %q default to %q",filename,key,tostring(newvalue)) + end end + end + return true end - - local stack = { } - - pushtarget = function(newtarget) - insert(stack,target) - settarget(newtarget) - end - - poptarget = function() - if #stack > 0 then - settarget(remove(stack)) + end +end +local function set(t,what,newvalue) + local data=t.data + if not data.frozen then + local done=t.done + if type(what)=="string" then + what=settings_to_hash(what) + end + if type(what)~="table" then + return + end + if not done then + done={} + t.done=done + end + for w,value in next,what do + if value=="" then + value=newvalue + elseif not value then + value=false + else + value=is_boolean(value,value) + end + w=topattern(w,true,true) + for name,functions in next,data do + if done[name] then + elseif find(name,w) then + done[name]=true + for i=1,#functions do + functions[i](value) + end + functions.value=value end + end end - - setformats = function(f) - formats = f + end +end +local function reset(t) + local data=t.data + if not data.frozen then + for name,functions in next,data do + for i=1,#functions do + functions[i](false) + end + functions.value=false end - - settranslations = function(t) - translations = t + end +end +local function enable(t,what) + set(t,what,true) +end +local function disable(t,what) + local data=t.data + if not what or what=="" then + t.done={} + reset(t) + else + set(t,what,false) + end +end +function setters.register(t,what,...) + local data=t.data + what=lower(what) + local functions=data[what] + if not functions then + functions={} + data[what]=functions + if trace_initialize then + t.report("defining %s",what) + end + end + local default=functions.default + for i=1,select("#",...) do + local fnc=select(i,...) + local typ=type(fnc) + if typ=="string" then + if trace_initialize then + t.report("coupling %s to %s",what,fnc) + end + local s=fnc + fnc=function(value) set(t,s,value) end + elseif typ~="function" then + fnc=nil + end + if fnc then + functions[#functions+1]=fnc + local value=functions.value or default + if value~=nil then + fnc(value) + functions.value=value + end end - -else - - logs.flush = ignore - - writer = write_nl - - newline = function() - write_nl("\n") + end + return false +end +function setters.enable(t,what) + local e=t.enable + t.enable,t.done=enable,{} + enable(t,what) + t.enable,t.done=e,{} +end +function setters.disable(t,what) + local e=t.disable + t.disable,t.done=disable,{} + disable(t,what) + t.disable,t.done=e,{} +end +function setters.reset(t) + t.done={} + reset(t) +end +function setters.list(t) + local list=table.sortedkeys(t.data) + local user,system={},{} + for l=1,#list do + local what=list[l] + if find(what,"^%*") then + system[#system+1]=what + else + user[#user+1]=what end - - local f_one = formatters["%-15s | %s"] - local f_two = formatters["%-15s |"] - - report = function(a,b,c,...) - if c then - write_nl(f_one(a,format(b,c,...))) - elseif b then - write_nl(f_one(a,b)) - elseif a then - write_nl(f_two(a)) - else - write_nl("") - end + end + return user,system +end +function setters.show(t) + local category=t.name + local list=setters.list(t) + t.report() + for k=1,#list do + local name=list[k] + local functions=t.data[name] + if functions then + local value,default,modules=functions.value,functions.default,#functions + value=value==nil and "unset" or tostring(value) + default=default==nil and "unset" or tostring(default) + t.report("%-50s modules: %2i default: %-12s value: %-12s",name,modules,default,value) + end + end + t.report() +end +local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show +local write_nl=texio and texio.write_nl or print +local function report(setter,...) + local report=logs and logs.report + if report then + report(setter.name,...) + else + write_nl(format("%-15s : %s\n",setter.name,format(...))) + end +end +local function default(setter,name) + local d=setter.data[name] + return d and d.default +end +local function value(setter,name) + local d=setter.data[name] + return d and (d.value or d.default) +end +function setters.new(name) + local setter + setter={ + data=allocate(), + name=name, + report=function(...) report (setter,...) end, + enable=function(...) enable (setter,...) end, + disable=function(...) disable (setter,...) end, + register=function(...) register(setter,...) end, + list=function(...) list (setter,...) end, + show=function(...) show (setter,...) end, + default=function(...) return default (setter,...) end, + value=function(...) return value (setter,...) end, + } + data[name]=setter + return setter +end +trackers=setters.new("trackers") +directives=setters.new("directives") +experiments=setters.new("experiments") +local t_enable,t_disable,t_report=trackers .enable,trackers .disable,trackers .report +local d_enable,d_disable,d_report=directives .enable,directives .disable,directives .report +local e_enable,e_disable,e_report=experiments.enable,experiments.disable,experiments.report +local trace_directives=false local trace_directives=false trackers.register("system.directives",function(v) trace_directives=v end) +local trace_experiments=false local trace_experiments=false trackers.register("system.experiments",function(v) trace_experiments=v end) +function directives.enable(...) + if trace_directives then + d_report("enabling: %s",concat({...}," ")) + end + d_enable(...) +end +function directives.disable(...) + if trace_directives then + d_report("disabling: %s",concat({...}," ")) + end + d_disable(...) +end +function experiments.enable(...) + if trace_experiments then + e_report("enabling: %s",concat({...}," ")) + end + e_enable(...) +end +function experiments.disable(...) + if trace_experiments then + e_report("disabling: %s",concat({...}," ")) + end + e_disable(...) +end +directives.register("system.nostatistics",function(v) + statistics.enable=not v +end) +directives.register("system.nolibraries",function(v) + libraries=nil +end) +if environment then + local engineflags=environment.engineflags + if engineflags then + local list=engineflags["c:trackers"] or engineflags["trackers"] + if type(list)=="string" then + setters.initialize("commandline flags","trackers",settings_to_hash(list)) end - - local f_one = formatters["%-15s | %s | %s"] - local f_two = formatters["%-15s | %s |"] - - subreport = function(a,sub,b,c,...) - if c then - write_nl(f_one(a,sub,format(b,c,...))) - elseif b then - write_nl(f_one(a,sub,b)) - elseif a then - write_nl(f_two(a,sub)) - else - write_nl("") - end + local list=engineflags["c:directives"] or engineflags["directives"] + if type(list)=="string" then + setters.initialize("commandline flags","directives",settings_to_hash(list)) end - - local f_one = formatters["%-15s : %s\n"] - local f_two = formatters["%-15s :\n"] - - status = function(a,b,c,...) -- not to be used in lua anyway - if c then - write_nl(f_one(a,format(b,c,...))) - elseif b then - write_nl(f_one(a,b)) -- b can have %'s - elseif a then - write_nl(f_two(a)) - else - write_nl("\n") - end + end +end +if texconfig then + local function set(k,v) + v=tonumber(v) + if v then + texconfig[k]=v end - - direct = ignore - subdirect = ignore - - settarget = ignore - pushtarget = ignore - poptarget = ignore - setformats = ignore - settranslations = ignore - + end + directives.register("luatex.expanddepth",function(v) set("expand_depth",v) end) + directives.register("luatex.hashextra",function(v) set("hash_extra",v) end) + directives.register("luatex.nestsize",function(v) set("nest_size",v) end) + directives.register("luatex.maxinopen",function(v) set("max_in_open",v) end) + directives.register("luatex.maxprintline",function(v) set("max_print_line",v) end) + directives.register("luatex.maxstrings",function(v) set("max_strings",v) end) + directives.register("luatex.paramsize",function(v) set("param_size",v) end) + directives.register("luatex.savesize",function(v) set("save_size",v) end) + directives.register("luatex.stacksize",function(v) set("stack_size",v) end) end -logs.report = report -logs.subreport = subreport -logs.status = status -logs.settarget = settarget -logs.pushtarget = pushtarget -logs.poptarget = poptarget -logs.setformats = setformats -logs.settranslations = settranslations -logs.direct = direct -logs.subdirect = subdirect -logs.writer = writer -logs.newline = newline - --- installer +end -- of closure --- todo: renew (un) locks when a new one is added and wildcard +do -- create closure to overcome 200 locals limit -local data, states = { }, nil +-- original size: 17885, stripped down to: 13242 +if not modules then modules={} end modules ['trac-log']={ + version=1.001, + comment="companion to trac-log.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local write_nl,write=texio and texio.write_nl or print,texio and texio.write or io.write +local format,gmatch,find=string.format,string.gmatch,string.find +local concat,insert,remove=table.concat,table.insert,table.remove +local topattern=string.topattern +local texcount=tex and tex.count +local next,type,select=next,type,select +local setmetatableindex=table.setmetatableindex +local formatters=string.formatters +logs=logs or {} +local logs=logs +local moreinfo=[[ +More information about ConTeXt and the tools that come with it can be found at: +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] +local function ignore() end +setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) +local report,subreport,status,settarget,setformats,settranslations +local direct,subdirect,writer,pushtarget,poptarget +if tex and (tex.jobname or tex.formatname) then + local valueiskey={ __index=function(t,k) t[k]=k return k end } + local target="term and log" + logs.flush=io.flush + local formats={} setmetatable(formats,valueiskey) + local translations={} setmetatable(translations,valueiskey) + writer=function(...) + write_nl(target,...) + end + newline=function() + write_nl(target,"\n") + end + local f_one=formatters["%-15s > %s\n"] + local f_two=formatters["%-15s >\n"] + report=function(a,b,c,...) + if c then + write_nl(target,f_one(translations[a],format(formats[b],c,...))) + elseif b then + write_nl(target,f_one(translations[a],formats[b])) + elseif a then + write_nl(target,f_two(translations[a])) + else + write_nl(target,"\n") + end + end + local f_one=formatters["%-15s > %s"] + local f_two=formatters["%-15s >"] + direct=function(a,b,c,...) + if c then + return f_one(translations[a],format(formats[b],c,...)) + elseif b then + return f_one(translations[a],formats[b]) + elseif a then + return f_two(translations[a]) + else + return "" + end + end + local f_one=formatters["%-15s > %s > %s\n"] + local f_two=formatters["%-15s > %s >\n"] + subreport=function(a,s,b,c,...) + if c then + write_nl(target,f_one(translations[a],translations[s],format(formats[b],c,...))) + elseif b then + write_nl(target,f_one(translations[a],translations[s],formats[b])) + elseif a then + write_nl(target,f_two(translations[a],translations[s])) + else + write_nl(target,"\n") + end + end + local f_one=formatters["%-15s > %s > %s"] + local f_two=formatters["%-15s > %s >"] + subdirect=function(a,s,b,c,...) + if c then + return f_one(translations[a],translations[s],format(formats[b],c,...)) + elseif b then + return f_one(translations[a],translations[s],formats[b]) + elseif a then + return f_two(translations[a],translations[s]) + else + return "" + end + end + local f_one=formatters["%-15s : %s\n"] + local f_two=formatters["%-15s :\n"] + status=function(a,b,c,...) + if c then + write_nl(target,f_one(translations[a],format(formats[b],c,...))) + elseif b then + write_nl(target,f_one(translations[a],formats[b])) + elseif a then + write_nl(target,f_two(translations[a])) + else + write_nl(target,"\n") + end + end + local targets={ + logfile="log", + log="log", + file="log", + console="term", + terminal="term", + both="term and log", + } + settarget=function(whereto) + target=targets[whereto or "both"] or targets.both + if target=="term" or target=="term and log" then + logs.flush=io.flush + else + logs.flush=ignore + end + end + local stack={} + pushtarget=function(newtarget) + insert(stack,target) + settarget(newtarget) + end + poptarget=function() + if #stack>0 then + settarget(remove(stack)) + end + end + setformats=function(f) + formats=f + end + settranslations=function(t) + translations=t + end +else + logs.flush=ignore + writer=write_nl + newline=function() + write_nl("\n") + end + local f_one=formatters["%-15s | %s"] + local f_two=formatters["%-15s |"] + report=function(a,b,c,...) + if c then + write_nl(f_one(a,format(b,c,...))) + elseif b then + write_nl(f_one(a,b)) + elseif a then + write_nl(f_two(a)) + else + write_nl("") + end + end + local f_one=formatters["%-15s | %s | %s"] + local f_two=formatters["%-15s | %s |"] + subreport=function(a,sub,b,c,...) + if c then + write_nl(f_one(a,sub,format(b,c,...))) + elseif b then + write_nl(f_one(a,sub,b)) + elseif a then + write_nl(f_two(a,sub)) + else + write_nl("") + end + end + local f_one=formatters["%-15s : %s\n"] + local f_two=formatters["%-15s :\n"] + status=function(a,b,c,...) + if c then + write_nl(f_one(a,format(b,c,...))) + elseif b then + write_nl(f_one(a,b)) + elseif a then + write_nl(f_two(a)) + else + write_nl("\n") + end + end + direct=ignore + subdirect=ignore + settarget=ignore + pushtarget=ignore + poptarget=ignore + setformats=ignore + settranslations=ignore +end +logs.report=report +logs.subreport=subreport +logs.status=status +logs.settarget=settarget +logs.pushtarget=pushtarget +logs.poptarget=poptarget +logs.setformats=setformats +logs.settranslations=settranslations +logs.direct=direct +logs.subdirect=subdirect +logs.writer=writer +logs.newline=newline +local data,states={},nil function logs.reporter(category,subcategory) - local logger = data[category] - if not logger then - local state = false - if states == true then - state = true - elseif type(states) == "table" then - for c, _ in next, states do - if find(category,c) then - state = true - break - end - end + local logger=data[category] + if not logger then + local state=false + if states==true then + state=true + elseif type(states)=="table" then + for c,_ in next,states do + if find(category,c) then + state=true + break end - logger = { - reporters = { }, - state = state, - } - data[category] = logger - end - local reporter = logger.reporters[subcategory or "default"] - if not reporter then - if subcategory then - reporter = function(...) - if not logger.state then - subreport(category,subcategory,...) - end - end - logger.reporters[subcategory] = reporter - else - local tag = category - reporter = function(...) - if not logger.state then - report(category,...) - end - end - logger.reporters.default = reporter + end + end + logger={ + reporters={}, + state=state, + } + data[category]=logger + end + local reporter=logger.reporters[subcategory or "default"] + if not reporter then + if subcategory then + reporter=function(...) + if not logger.state then + subreport(category,subcategory,...) end + end + logger.reporters[subcategory]=reporter + else + local tag=category + reporter=function(...) + if not logger.state then + report(category,...) + end + end + logger.reporters.default=reporter end - return reporter + end + return reporter end - -logs.new = logs.reporter -- for old times sake - --- context specicific: this ends up in the macro stream - -local ctxreport = logs.writer - +logs.new=logs.reporter +local ctxreport=logs.writer function logs.setmessenger(m) - ctxreport = m + ctxreport=m end - function logs.messenger(category,subcategory) - -- we need to avoid catcode mess (todo: fast context) - if subcategory then - return function(...) - ctxreport(subdirect(category,subcategory,...)) - end - else - return function(...) - ctxreport(direct(category,...)) - end + if subcategory then + return function(...) + ctxreport(subdirect(category,subcategory,...)) + end + else + return function(...) + ctxreport(direct(category,...)) end + end end - --- so far - local function setblocked(category,value) - if category == true then - -- lock all - category, value = "*", true - elseif category == false then - -- unlock all - category, value = "*", false - elseif value == nil then - -- lock selective - value = true - end - if category == "*" then - states = value - for k, v in next, data do - v.state = value - end - else - states = utilities.parsers.settings_to_hash(category) - for c, _ in next, states do - if data[c] then - v.state = value - else - c = topattern(c,true,true) - for k, v in next, data do - if find(k,c) then - v.state = value - end - end - end + if category==true then + category,value="*",true + elseif category==false then + category,value="*",false + elseif value==nil then + value=true + end + if category=="*" then + states=value + for k,v in next,data do + v.state=value + end + else + states=utilities.parsers.settings_to_hash(category) + for c,_ in next,states do + if data[c] then + v.state=value + else + c=topattern(c,true,true) + for k,v in next,data do + if find(k,c) then + v.state=value + end end + end end + end end - function logs.disable(category,value) - setblocked(category,value == nil and true or value) + setblocked(category,value==nil and true or value) end - function logs.enable(category) - setblocked(category,false) + setblocked(category,false) end - function logs.categories() - return table.sortedkeys(data) + return table.sortedkeys(data) end - function logs.show() - local n, c, s, max = 0, 0, 0, 0 - for category, v in table.sortedpairs(data) do - n = n + 1 - local state = v.state - local reporters = v.reporters - local nc = #category - if nc > c then - c = nc - end - for subcategory, _ in next, reporters do - local ns = #subcategory - if ns > c then - s = ns - end - local m = nc + ns - if m > max then - max = m - end - end - local subcategories = concat(table.sortedkeys(reporters),", ") - if state == true then - state = "disabled" - elseif state == false then - state = "enabled" - else - state = "unknown" - end - -- no new here - report("logging","category: '%s', subcategories: '%s', state: '%s'",category,subcategories,state) + local n,c,s,max=0,0,0,0 + for category,v in table.sortedpairs(data) do + n=n+1 + local state=v.state + local reporters=v.reporters + local nc=#category + if nc>c then + c=nc + end + for subcategory,_ in next,reporters do + local ns=#subcategory + if ns>c then + s=ns + end + local m=nc+ns + if m>max then + max=m + end end - report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) + local subcategories=concat(table.sortedkeys(reporters),", ") + if state==true then + state="disabled" + elseif state==false then + state="enabled" + else + state="unknown" + end + report("logging","category: '%s', subcategories: '%s', state: '%s'",category,subcategories,state) + end + report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) end - -directives.register("logs.blocked", function(v) - setblocked(v,true) +directives.register("logs.blocked",function(v) + setblocked(v,true) end) - -directives.register("logs.target", function(v) - settarget(v) +directives.register("logs.target",function(v) + settarget(v) end) - --- tex specific loggers (might move elsewhere) - -local report_pages = logs.reporter("pages") -- not needed but saves checking when we grep for it - -local real, user, sub - +local report_pages=logs.reporter("pages") +local real,user,sub function logs.start_page_number() - real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno --- real, user, sub = 0, 0, 0 -end - -local timing = false -local starttime = nil -local lasttime = nil - -trackers.register("pages.timing", function(v) -- only for myself (diagnostics) - starttime = os.clock() - timing = true + real,user,sub=texcount.realpageno,texcount.userpageno,texcount.subpageno +end +local timing=false +local starttime=nil +local lasttime=nil +trackers.register("pages.timing",function(v) + starttime=os.clock() + timing=true end) - -function logs.stop_page_number() -- the first page can includes the initialization so we omit this in average - if timing then - local elapsed, average - local stoptime = os.clock() - if not lasttime or real < 2 then - elapsed = stoptime - average = stoptime - starttime = stoptime - else - elapsed = stoptime - lasttime - average = (stoptime - starttime) / (real - 1) - end - lasttime = stoptime - if real > 0 then - if user > 0 then - if sub > 0 then - report_pages("flushing realpage %s, userpage %s, subpage %s, time %0.04f / %0.04f",real,user,sub,elapsed,average) - else - report_pages("flushing realpage %s, userpage %s, time %0.04f / %0.04f",real,user,elapsed,average) - end - else - report_pages("flushing realpage %s, time %0.04f / %0.04f",real,elapsed,average) - end - else - report_pages("flushing page, time %0.04f / %0.04f",elapsed,average) - end +function logs.stop_page_number() + if timing then + local elapsed,average + local stoptime=os.clock() + if not lasttime or real<2 then + elapsed=stoptime + average=stoptime + starttime=stoptime + else + elapsed=stoptime-lasttime + average=(stoptime-starttime)/(real-1) + end + lasttime=stoptime + if real>0 then + if user>0 then + if sub>0 then + report_pages("flushing realpage %s, userpage %s, subpage %s, time %0.04f / %0.04f",real,user,sub,elapsed,average) + else + report_pages("flushing realpage %s, userpage %s, time %0.04f / %0.04f",real,user,elapsed,average) + end + else + report_pages("flushing realpage %s, time %0.04f / %0.04f",real,elapsed,average) + end else - if real > 0 then - if user > 0 then - if sub > 0 then - report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) - else - report_pages("flushing realpage %s, userpage %s",real,user) - end - else - report_pages("flushing realpage %s",real) - end + report_pages("flushing page, time %0.04f / %0.04f",elapsed,average) + end + else + if real>0 then + if user>0 then + if sub>0 then + report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - report_pages("flushing page") + report_pages("flushing realpage %s, userpage %s",real,user) end + else + report_pages("flushing realpage %s",real) + end + else + report_pages("flushing page") end - logs.flush() + end + logs.flush() end - -logs.report_job_stat = statistics and statistics.showjobstat - -local report_files = logs.reporter("files") - -local nesting = 0 -local verbose = false -local hasscheme = url.hasscheme - --- we don't have show_open and show_close callbacks yet - +logs.report_job_stat=statistics and statistics.showjobstat +local report_files=logs.reporter("files") +local nesting=0 +local verbose=false +local hasscheme=url.hasscheme function logs.show_open(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- nesting = nesting + 1 - -- report_files("level %s, opening %s",nesting,name) - -- else - -- write(format("(%s",name)) -- tex adds a space - -- end - -- end end - function logs.show_close(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, closing %s",nesting,name) - -- nesting = nesting - 1 - -- else - -- write(")") -- tex adds a space - -- end - -- end end - function logs.show_load(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, loading %s",nesting+1,name) - -- else - -- write(format("(%s)",name)) - -- end - -- end end - --- there may be scripts out there using this: - -local simple = logs.reporter("comment") - -logs.simple = simple -logs.simpleline = simple - --- obsolete - -function logs.setprogram () end -- obsolete -function logs.extendbanner() end -- obsolete -function logs.reportlines () end -- obsolete -function logs.reportbanner() end -- obsolete -function logs.reportline () end -- obsolete -function logs.simplelines () end -- obsolete -function logs.help () end -- obsolete - --- applications - +local simple=logs.reporter("comment") +logs.simple=simple +logs.simpleline=simple +function logs.setprogram () end +function logs.extendbanner() end +function logs.reportlines () end +function logs.reportbanner() end +function logs.reportline () end +function logs.simplelines () end +function logs.help () end local function reportlines(t,str) - if str then - for line in gmatch(str,"(.-)[\n\r]") do - t.report(line) - end + if str then + for line in gmatch(str,"(.-)[\n\r]") do + t.report(line) end + end end - local function reportbanner(t) - local banner = t.banner - if banner then - t.report(banner) - t.report() - end + local banner=t.banner + if banner then + t.report(banner) + t.report() + end end - local function reportversion(t) - local banner = t.banner - if banner then - t.report(banner) - end + local banner=t.banner + if banner then + t.report(banner) + end end - local function reporthelp(t,...) - local helpinfo = t.helpinfo - if type(helpinfo) == "string" then - reportlines(t,helpinfo) - elseif type(helpinfo) == "table" then - local n = select("#",...) - for i=1,n do - reportlines(t,t.helpinfo[select(i,...)]) - if i < n then - t.report() - end - end + local helpinfo=t.helpinfo + if type(helpinfo)=="string" then + reportlines(t,helpinfo) + elseif type(helpinfo)=="table" then + local n=select("#",...) + for i=1,n do + reportlines(t,t.helpinfo[select(i,...)]) + if i %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) - for i=1,10 do - local f = io.open(whereto,"a") -- we can consider keepint the file open - if f then - f:write(message) - f:close() - break - else - sleep(0.1) - end + local message=format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) + for i=1,10 do + local f=io.open(whereto,"a") + if f then + f:write(message) + f:close() + break + else + sleep(0.1) end + end end - -local report_system = logs.reporter("system","logs") - +local report_system=logs.reporter("system","logs") function logs.obsolete(old,new) - local o = loadstring("return " .. new)() - if type(o) == "function" then - return function(...) - report_system("function %s is obsolete, use %s",old,new) - loadstring(old .. "=" .. new .. " return ".. old)()(...) - end - elseif type(o) == "table" then - local t, m = { }, { } - m.__index = function(t,k) - report_system("table %s is obsolete, use %s",old,new) - m.__index, m.__newindex = o, o - return o[k] - end - m.__newindex = function(t,k,v) - report_system("table %s is obsolete, use %s",old,new) - m.__index, m.__newindex = o, o - o[k] = v - end - if libraries then - libraries.obsolete[old] = t -- true - end - setmetatable(t,m) - return t - end + local o=loadstring("return "..new)() + if type(o)=="function" then + return function(...) + report_system("function %s is obsolete, use %s",old,new) + loadstring(old.."="..new.." return "..old)()(...) + end + elseif type(o)=="table" then + local t,m={},{} + m.__index=function(t,k) + report_system("table %s is obsolete, use %s",old,new) + m.__index,m.__newindex=o,o + return o[k] + end + m.__newindex=function(t,k,v) + report_system("table %s is obsolete, use %s",old,new) + m.__index,m.__newindex=o,o + o[k]=v + end + if libraries then + libraries.obsolete[old]=t + end + setmetatable(t,m) + return t + end end - if utilities then - utilities.report = report_system + utilities.report=report_system end - if tex and tex.error then - function logs.texerrormessage(...) -- for the moment we put this function here - tex.error(format(...), { }) - end + function logs.texerrormessage(...) + tex.error(format(...),{}) + end else - function logs.texerrormessage(...) - print(format(...)) - end + function logs.texerrormessage(...) + print(format(...)) + end end +io.stdout:setvbuf('no') +io.stderr:setvbuf('no') --- do we still need io.flush then? - -io.stdout:setvbuf('no') -io.stderr:setvbuf('no') - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-pro'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local getmetatable, setmetatable, rawset, type = getmetatable, setmetatable, rawset, type - --- The protection implemented here is probably not that tight but good enough to catch --- problems due to naive usage. --- --- There's a more extensive version (trac-xxx.lua) that supports nesting. --- --- This will change when we have _ENV in lua 5.2+ - -local trace_namespaces = false trackers.register("system.namespaces", function(v) trace_namespaces = v end) -local report_system = logs.reporter("system","protection") +end -- of closure -namespaces = namespaces or { } -local namespaces = namespaces +do -- create closure to overcome 200 locals limit -local registered = { } +-- original size: 5789, stripped down to: 3469 +if not modules then modules={} end modules ['trac-pro']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local getmetatable,setmetatable,rawset,type=getmetatable,setmetatable,rawset,type +local trace_namespaces=false trackers.register("system.namespaces",function(v) trace_namespaces=v end) +local report_system=logs.reporter("system","protection") +namespaces=namespaces or {} +local namespaces=namespaces +local registered={} local function report_index(k,name) - if trace_namespaces then - report_system("reference to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) - else - report_system("reference to '%s' in protected namespace '%s'",k,name) - end + if trace_namespaces then + report_system("reference to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) + else + report_system("reference to '%s' in protected namespace '%s'",k,name) + end end - local function report_newindex(k,name) - if trace_namespaces then - report_system("assignment to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) - else - report_system("assignment to '%s' in protected namespace '%s'",k,name) - end + if trace_namespaces then + report_system("assignment to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) + else + report_system("assignment to '%s' in protected namespace '%s'",k,name) + end end - local function register(name) - local data = name == "global" and _G or _G[name] - if not data then - return -- error - end - registered[name] = data - local m = getmetatable(data) - if not m then - m = { } - setmetatable(data,m) - end - local index, newindex = { }, { } - m.__saved__index = m.__index - m.__no__index = function(t,k) - if not index[k] then - index[k] = true - report_index(k,name) - end - return nil - end - m.__saved__newindex = m.__newindex - m.__no__newindex = function(t,k,v) - if not newindex[k] then - newindex[k] = true - report_newindex(k,name) - end - rawset(t,k,v) + local data=name=="global" and _G or _G[name] + if not data then + return + end + registered[name]=data + local m=getmetatable(data) + if not m then + m={} + setmetatable(data,m) + end + local index,newindex={},{} + m.__saved__index=m.__index + m.__no__index=function(t,k) + if not index[k] then + index[k]=true + report_index(k,name) end - m.__protection__depth = 0 -end - -local function private(name) -- maybe save name - local data = registered[name] + return nil + end + m.__saved__newindex=m.__newindex + m.__no__newindex=function(t,k,v) + if not newindex[k] then + newindex[k]=true + report_newindex(k,name) + end + rawset(t,k,v) + end + m.__protection__depth=0 +end +local function private(name) + local data=registered[name] + if not data then + data=_G[name] if not data then - data = _G[name] - if not data then - data = { } - _G[name] = data - end - register(name) + data={} + _G[name]=data end - return data + register(name) + end + return data end - local function protect(name) - local data = registered[name] - if not data then - return - end - local m = getmetatable(data) - local pd = m.__protection__depth - if pd > 0 then - m.__protection__depth = pd + 1 - else - m.__save_d_index, m.__saved__newindex = m.__index, m.__newindex - m.__index, m.__newindex = m.__no__index, m.__no__newindex - m.__protection__depth = 1 - end + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>0 then + m.__protection__depth=pd+1 + else + m.__save_d_index,m.__saved__newindex=m.__index,m.__newindex + m.__index,m.__newindex=m.__no__index,m.__no__newindex + m.__protection__depth=1 + end end - local function unprotect(name) - local data = registered[name] - if not data then - return - end - local m = getmetatable(data) - local pd = m.__protection__depth - if pd > 1 then - m.__protection__depth = pd - 1 - else - m.__index, m.__newindex = m.__saved__index, m.__saved__newindex - m.__protection__depth = 0 - end + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>1 then + m.__protection__depth=pd-1 + else + m.__index,m.__newindex=m.__saved__index,m.__saved__newindex + m.__protection__depth=0 + end end - local function protectall() - for name, _ in next, registered do - if name ~= "global" then - protect(name) - end + for name,_ in next,registered do + if name~="global" then + protect(name) end + end end - local function unprotectall() - for name, _ in next, registered do - if name ~= "global" then - unprotect(name) - end - end -end - -namespaces.register = register -- register when defined -namespaces.private = private -- allocate and register if needed -namespaces.protect = protect -namespaces.unprotect = unprotect -namespaces.protectall = protectall -namespaces.unprotectall = unprotectall - -namespaces.private("namespaces") registered = { } register("global") -- unreachable - -directives.register("system.protect", function(v) - if v then - protectall() - else - unprotectall() - end + for name,_ in next,registered do + if name~="global" then + unprotect(name) + end + end +end +namespaces.register=register +namespaces.private=private +namespaces.protect=protect +namespaces.unprotect=unprotect +namespaces.protectall=protectall +namespaces.unprotectall=unprotectall +namespaces.private("namespaces") registered={} register("global") +directives.register("system.protect",function(v) + if v then + protectall() + else + unprotectall() + end end) - -directives.register("system.checkglobals", function(v) - if v then - report_system("enabling global namespace guard") - protect("global") - else - report_system("disabling global namespace guard") - unprotect("global") - end +directives.register("system.checkglobals",function(v) + if v then + report_system("enabling global namespace guard") + protect("global") + else + report_system("disabling global namespace guard") + unprotect("global") + end end) --- dummy section (will go to luat-dum.lua) - - - - - - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-env'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- A former version provided functionality for non embeded core --- scripts i.e. runtime library loading. Given the amount of --- Lua code we use now, this no longer makes sense. Much of this --- evolved before bytecode arrays were available and so a lot of --- code has disappeared already. - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_lua = logs.reporter("resolvers","lua") - -local allocate, mark = utilities.storage.allocate, utilities.storage.mark - -local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find -local unquoted, quoted = string.unquoted, string.quoted -local concat, insert, remove = table.concat, table.insert, table.remove -local loadedluacode = utilities.lua.loadedluacode -local luasuffixes = utilities.lua.suffixes - -environment = environment or { } -local environment = environment - --- precautions - -os.setlocale(nil,nil) -- useless feature and even dangerous in luatex +-- original size: 12260, stripped down to: 8100 +if not modules then modules={} end modules ['luat-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_lua=logs.reporter("resolvers","lua") +local allocate,mark=utilities.storage.allocate,utilities.storage.mark +local format,sub,match,gsub,find=string.format,string.sub,string.match,string.gsub,string.find +local unquoted,quoted=string.unquoted,string.quoted +local concat,insert,remove=table.concat,table.insert,table.remove +local loadedluacode=utilities.lua.loadedluacode +local luasuffixes=utilities.lua.suffixes +environment=environment or {} +local environment=environment +os.setlocale(nil,nil) function os.setlocale() - -- no way you can mess with it end - --- dirty tricks (we will replace the texlua call by luatex --luaonly) - -local validengines = allocate { - ["luatex"] = true, - ["luajittex"] = true, - -- ["luatex.exe"] = true, - -- ["luajittex.exe"] = true, +local validengines=allocate { + ["luatex"]=true, + ["luajittex"]=true, } - -local basicengines = allocate { - ["luatex"] = "luatex", - ["texlua"] = "luatex", - ["texluac"] = "luatex", - ["luajittex"] = "luajittex", - ["texluajit"] = "luajittex", - -- ["texlua.exe"] = "luatex", - -- ["texluajit.exe"] = "luajittex", +local basicengines=allocate { + ["luatex"]="luatex", + ["texlua"]="luatex", + ["texluac"]="luatex", + ["luajittex"]="luajittex", + ["texluajit"]="luajittex", } - -environment.validengines = validengines -environment.basicengines = basicengines - -if arg and validengines[file.removesuffix(arg[0])] and arg[1] == "--luaonly" then - arg[-1] = arg[0] - arg[ 0] = arg[2] - for k=3,#arg do - arg[k-2] = arg[k] - end - remove(arg) -- last - remove(arg) -- pre-last +environment.validengines=validengines +environment.basicengines=basicengines +if arg and validengines[file.removesuffix(arg[0])] and arg[1]=="--luaonly" then + arg[-1]=arg[0] + arg[ 0]=arg[2] + for k=3,#arg do + arg[k-2]=arg[k] + end + remove(arg) + remove(arg) end - --- This is an ugly hack but it permits symlinking a script (say 'context') to 'mtxrun' as in: --- --- ln -s /opt/minimals/tex/texmf-linux-64/bin/mtxrun context --- --- The special mapping hack is needed because 'luatools' boils down to 'mtxrun --script base' --- but it's unlikely that there will be more of this - do - - local originalzero = file.basename(arg[0]) - local specialmapping = { luatools == "base" } - - if originalzero ~= "mtxrun" and originalzero ~= "mtxrun.lua" then - arg[0] = specialmapping[originalzero] or originalzero - insert(arg,0,"--script") - insert(arg,0,"mtxrun") - end - -end - --- environment - -environment.arguments = allocate() -environment.files = allocate() -environment.sortedflags = nil - -local mt = { - __index = function(_,k) - if k == "version" then - local version = tex.toks and tex.toks.contextversiontoks - if version and version ~= "" then - rawset(environment,"version",version) - return version - else - return "unknown" - end - elseif k == "jobname" or k == "formatname" then - local name = tex and tex[k] - if name or name== "" then - rawset(environment,k,name) - return name - else - return "unknown" - end - elseif k == "outputfilename" then - local name = environment.jobname - rawset(environment,k,name) - return name - end + local originalzero=file.basename(arg[0]) + local specialmapping={ luatools=="base" } + if originalzero~="mtxrun" and originalzero~="mtxrun.lua" then + arg[0]=specialmapping[originalzero] or originalzero + insert(arg,0,"--script") + insert(arg,0,"mtxrun") + end +end +environment.arguments=allocate() +environment.files=allocate() +environment.sortedflags=nil +local mt={ + __index=function(_,k) + if k=="version" then + local version=tex.toks and tex.toks.contextversiontoks + if version and version~="" then + rawset(environment,"version",version) + return version + else + return "unknown" + end + elseif k=="jobname" or k=="formatname" then + local name=tex and tex[k] + if name or name=="" then + rawset(environment,k,name) + return name + else + return "unknown" + end + elseif k=="outputfilename" then + local name=environment.jobname + rawset(environment,k,name) + return name end + end } - setmetatable(environment,mt) - --- context specific arguments (in order not to confuse the engine) - function environment.initializearguments(arg) - local arguments, files = { }, { } - environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index=1,#arg do - local argument = arg[index] - if index > 0 then - local flag, value = match(argument,"^%-+(.-)=(.-)$") - if flag then - flag = gsub(flag,"^c:","") - arguments[flag] = unquoted(value or "") - else - flag = match(argument,"^%-+(.+)") - if flag then - flag = gsub(flag,"^c:","") - arguments[flag] = true - else - files[#files+1] = argument - end - end + local arguments,files={},{} + environment.arguments,environment.files,environment.sortedflags=arguments,files,nil + for index=1,#arg do + local argument=arg[index] + if index>0 then + local flag,value=match(argument,"^%-+(.-)=(.-)$") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=unquoted(value or "") + else + flag=match(argument,"^%-+(.+)") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=true + else + files[#files+1]=argument end + end end - environment.ownname = file.reslash(environment.ownname or arg[0] or 'unknown.lua') + end + environment.ownname=file.reslash(environment.ownname or arg[0] or 'unknown.lua') end - function environment.setargument(name,value) - environment.arguments[name] = value + environment.arguments[name]=value end - --- todo: defaults, better checks e.g on type (boolean versus string) --- --- tricky: too many hits when we support partials unless we add --- a registration of arguments so from now on we have 'partial' - function environment.getargument(name,partial) - local arguments, sortedflags = environment.arguments, environment.sortedflags - if arguments[name] then - return arguments[name] - elseif partial then - if not sortedflags then - sortedflags = allocate(table.sortedkeys(arguments)) - for k=1,#sortedflags do - sortedflags[k] = "^" .. sortedflags[k] - end - environment.sortedflags = sortedflags - end - -- example of potential clash: ^mode ^modefile - for k=1,#sortedflags do - local v = sortedflags[k] - if find(name,v) then - return arguments[sub(v,2,#v)] - end - end + local arguments,sortedflags=environment.arguments,environment.sortedflags + if arguments[name] then + return arguments[name] + elseif partial then + if not sortedflags then + sortedflags=allocate(table.sortedkeys(arguments)) + for k=1,#sortedflags do + sortedflags[k]="^"..sortedflags[k] + end + environment.sortedflags=sortedflags end - return nil + for k=1,#sortedflags do + local v=sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] + end + end + end + return nil end - -environment.argument = environment.getargument - -function environment.splitarguments(separator) -- rather special, cut-off before separator - local done, before, after = false, { }, { } - local originalarguments = environment.originalarguments - for k=1,#originalarguments do - local v = originalarguments[k] - if not done and v == separator then - done = true - elseif done then - after[#after+1] = v - else - before[#before+1] = v - end +environment.argument=environment.getargument +function environment.splitarguments(separator) + local done,before,after=false,{},{} + local originalarguments=environment.originalarguments + for k=1,#originalarguments do + local v=originalarguments[k] + if not done and v==separator then + done=true + elseif done then + after[#after+1]=v + else + before[#before+1]=v end - return before, after + end + return before,after end - function environment.reconstructcommandline(arg,noquote) - arg = arg or environment.originalarguments - if noquote and #arg == 1 then - -- we could just do: return unquoted(resolvers.resolve(arg[i])) - local a = arg[1] - a = resolvers.resolve(a) - a = unquoted(a) - return a - elseif #arg > 0 then - local result = { } - for i=1,#arg do - -- we could just do: result[#result+1] = format("%q",unquoted(resolvers.resolve(arg[i]))) - local a = arg[i] - a = resolvers.resolve(a) - a = unquoted(a) - a = gsub(a,'"','\\"') -- tricky - if find(a," ") then - result[#result+1] = quoted(a) - else - result[#result+1] = a - end - end - return concat(result," ") - else - return "" + arg=arg or environment.originalarguments + if noquote and #arg==1 then + local a=arg[1] + a=resolvers.resolve(a) + a=unquoted(a) + return a + elseif #arg>0 then + local result={} + for i=1,#arg do + local a=arg[i] + a=resolvers.resolve(a) + a=unquoted(a) + a=gsub(a,'"','\\"') + if find(a," ") then + result[#result+1]=quoted(a) + else + result[#result+1]=a + end end + return concat(result," ") + else + return "" + end end - --- -- to be tested: --- --- function environment.reconstructcommandline(arg,noquote) --- arg = arg or environment.originalarguments --- if noquote and #arg == 1 then --- return unquoted(resolvers.resolve(arg[1])) --- elseif #arg > 0 then --- local result = { } --- for i=1,#arg do --- result[#result+1] = format("%q",unquoted(resolvers.resolve(arg[i]))) -- always quote --- end --- return concat(result," ") --- else --- return "" --- end --- end - if arg then - - -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) - local newarg, instring = { }, false - - for index=1,#arg do - local argument = arg[index] - if find(argument,"^\"") then - newarg[#newarg+1] = gsub(argument,"^\"","") - if not find(argument,"\"$") then - instring = true - end - elseif find(argument,"\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") - instring = false - elseif instring then - newarg[#newarg] = newarg[#newarg] .. " " .. argument - else - newarg[#newarg+1] = argument - end - end - for i=1,-5,-1 do - newarg[i] = arg[i] - end - - environment.initializearguments(newarg) - - environment.originalarguments = mark(newarg) - environment.rawarguments = mark(arg) - - arg = { } -- prevent duplicate handling - + local newarg,instring={},false + for index=1,#arg do + local argument=arg[index] + if find(argument,"^\"") then + newarg[#newarg+1]=gsub(argument,"^\"","") + if not find(argument,"\"$") then + instring=true + end + elseif find(argument,"\"$") then + newarg[#newarg]=newarg[#newarg].." "..gsub(argument,"\"$","") + instring=false + elseif instring then + newarg[#newarg]=newarg[#newarg].." "..argument + else + newarg[#newarg+1]=argument + end + end + for i=1,-5,-1 do + newarg[i]=arg[i] + end + environment.initializearguments(newarg) + environment.originalarguments=mark(newarg) + environment.rawarguments=mark(arg) + arg={} end - --- weird place ... depends on a not yet loaded module - function environment.texfile(filename) - return resolvers.findfile(filename,'tex') + return resolvers.findfile(filename,'tex') +end +function environment.luafile(filename) + local resolved=resolvers.findfile(filename,'tex') or "" + if resolved~="" then + return resolved + end + resolved=resolvers.findfile(filename,'texmfscripts') or "" + if resolved~="" then + return resolved + end + return resolvers.findfile(filename,'luatexlibs') or "" +end +local stripindeed=false directives.register("system.compile.strip",function(v) stripindeed=v end) +local function strippable(filename) + if stripindeed then + local modu=modules[file.nameonly(filename)] + return modu and modu.dataonly + else + return false + end end - -function environment.luafile(filename) -- needs checking - local resolved = resolvers.findfile(filename,'tex') or "" - if resolved ~= "" then - return resolved - end - resolved = resolvers.findfile(filename,'texmfscripts') or "" - if resolved ~= "" then - return resolved +function environment.luafilechunk(filename,silent) + filename=file.replacesuffix(filename,"lua") + local fullname=environment.luafile(filename) + if fullname and fullname~="" then + local data=loadedluacode(fullname,strippable,filename) + if trace_locating then + report_lua("loading file %s%s",fullname,not data and " failed" or "") + elseif not silent then + texio.write("<",data and "+ " or "- ",fullname,">") end - return resolvers.findfile(filename,'luatexlibs') or "" -end - --- local function checkstrip(filename) --- local modu = modules[file.nameonly(filename)] --- return modu and modu.dataonly --- end - -local stripindeed = false directives.register("system.compile.strip", function(v) stripindeed = v end) - -local function strippable(filename) - if stripindeed then - local modu = modules[file.nameonly(filename)] - return modu and modu.dataonly - else - return false + return data + else + if trace_locating then + report_lua("unknown file %s",filename) end -end - -function environment.luafilechunk(filename,silent) -- used for loading lua bytecode in the format - filename = file.replacesuffix(filename, "lua") - local fullname = environment.luafile(filename) - if fullname and fullname ~= "" then - local data = loadedluacode(fullname,strippable,filename) - if trace_locating then - report_lua("loading file %s%s", fullname, not data and " failed" or "") - elseif not silent then - texio.write("<",data and "+ " or "- ",fullname,">") - end - return data - else + return nil + end +end +function environment.loadluafile(filename,version) + local lucname,luaname,chunk + local basename=file.removesuffix(filename) + if basename==filename then + luaname=file.addsuffix(basename,luasuffixes.lua) + lucname=file.addsuffix(basename,luasuffixes.luc) + else + luaname=basename + lucname=nil + end + local fullname=(lucname and environment.luafile(lucname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %s",fullname) + end + chunk=loadfile(fullname) + end + if chunk then + assert(chunk)() + if version then + local v=version + if modules and modules[filename] then + v=modules[filename].version + elseif versions and versions[filename] then + v=versions[filename] + end + if v==version then + return true + else if trace_locating then - report_lua("unknown file %s", filename) + report_lua("version mismatch for %s: lua=%s, luc=%s",filename,v,version) end - return nil - end -end - --- the next ones can use the previous ones / combine - -function environment.loadluafile(filename, version) - local lucname, luaname, chunk - local basename = file.removesuffix(filename) - if basename == filename then - luaname = fiule.addsuffix(basename,luasuffixes.lua) - lucname = fiule.addsuffix(basename,luasuffixes.luc) + environment.loadluafile(filename) + end else - luaname = basename -- forced suffix - lucname = nil + return true end - -- when not overloaded by explicit suffix we look for a luc file first - local fullname = (lucname and environment.luafile(lucname)) or "" - if fullname ~= "" then - if trace_locating then - report_lua("loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - end - if chunk then - assert(chunk)() - if version then - -- we check of the version number of this chunk matches - local v = version -- can be nil - if modules and modules[filename] then - v = modules[filename].version -- new method - elseif versions and versions[filename] then - v = versions[filename] -- old method - end - if v == version then - return true - else - if trace_locating then - report_lua("version mismatch for %s: lua=%s, luc=%s", filename, v, version) - end - environment.loadluafile(filename) - end - else - return true - end + end + fullname=(luaname and environment.luafile(luaname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %s",fullname) end - fullname = (luaname and environment.luafile(luaname)) or "" - if fullname ~= "" then - if trace_locating then - report_lua("loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - if not chunk then - if trace_locating then - report_lua("unknown file %s", filename) - end - else - assert(chunk)() - return true - end + chunk=loadfile(fullname) + if not chunk then + if trace_locating then + report_lua("unknown file %s",filename) + end + else + assert(chunk)() + return true end - return false + end + return false end @@ -9577,1226 +7159,978 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-tab'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc --- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the --- trouble - --- todo: when serializing optionally remap named entities to hex (if known in char-ent.lua) --- maybe when letter -> utf, else name .. then we need an option to the serializer .. a bit --- of work so we delay this till we cleanup - -local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) - -local report_xml = logs and logs.reporter("xml","core") or function(...) print(format(...)) end - - - -xml = xml or { } -local xml = xml - - -local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber -local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub -local utfchar = utf.char -local lpegmatch = lpeg.match -local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs - - - -xml.xmlns = xml.xmlns or { } - -local check = P(false) -local parse = check - +-- original size: 42438, stripped down to: 26556 - -function xml.registerns(namespace, pattern) -- pattern can be an lpeg - check = check + C(P(lower(pattern))) / namespace - parse = P { P(check) + 1 * V(1) } +if not modules then modules={} end modules ['lxml-tab']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_entities=false trackers.register("xml.entities",function(v) trace_entities=v end) +local report_xml=logs and logs.reporter("xml","core") or function(...) print(format(...)) end +xml=xml or {} +local xml=xml +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,setmetatable,getmetatable,tonumber=type,next,setmetatable,getmetatable,tonumber +local format,lower,find,match,gsub=string.format,string.lower,string.find,string.match,string.gsub +local utfchar=utf.char +local lpegmatch=lpeg.match +local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs +xml.xmlns=xml.xmlns or {} +local check=P(false) +local parse=check +function xml.registerns(namespace,pattern) + check=check+C(P(lower(pattern)))/namespace + parse=P { P(check)+1*V(1) } end - - - function xml.checkns(namespace,url) - local ns = lpegmatch(parse,lower(url)) - if ns and namespace ~= ns then - xml.xmlns[namespace] = ns - end + local ns=lpegmatch(parse,lower(url)) + if ns and namespace~=ns then + xml.xmlns[namespace]=ns + end end - - - function xml.resolvens(url) - return lpegmatch(parse,lower(url)) or "" -end - - - - - --- not just one big nested table capture (lpeg overflow) - -local nsremap, resolvens = xml.xmlns, xml.resolvens - -local stack = { } -local top = { } -local dt = { } -local at = { } -local xmlns = { } -local errorstr = nil -local entities = { } -local strip = false -local cleanup = false -local utfize = false -local resolve_predefined = false -local unify_predefined = false - -local dcache = { } -local hcache = { } -local acache = { } - -local mt = { } - -local function initialize_mt(root) - mt = { __index = root } -- will be redefined later -end - -function xml.setproperty(root,k,v) - getmetatable(root).__index[k] = v -end - -function xml.checkerror(top,toclose) - return "" -- can be set -end - -local function add_attribute(namespace,tag,value) - if cleanup and #value > 0 then - value = cleanup(value) -- new - end - if tag == "xmlns" then - xmlns[#xmlns+1] = resolvens(value) - at[tag] = value - elseif namespace == "" then - at[tag] = value - elseif namespace == "xmlns" then - xml.checkns(tag,value) - at["xmlns:" .. tag] = value - else - -- for the moment this way: - at[namespace .. ":" .. tag] = value - end + return lpegmatch(parse,lower(url)) or "" +end +local nsremap,resolvens=xml.xmlns,xml.resolvens +local stack={} +local top={} +local dt={} +local at={} +local xmlns={} +local errorstr=nil +local entities={} +local strip=false +local cleanup=false +local utfize=false +local resolve_predefined=false +local unify_predefined=false +local dcache={} +local hcache={} +local acache={} +local mt={} +local function initialize_mt(root) + mt={ __index=root } end - -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } +function xml.setproperty(root,k,v) + getmetatable(root).__index[k]=v end - -local function add_begin(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } - setmetatable(top, mt) - dt = top.dt - stack[#stack+1] = top - at = { } +function xml.checkerror(top,toclose) + return "" end - -local function add_end(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local toclose = remove(stack) - top = stack[#stack] - if #stack < 1 then - errorstr = format("nothing to close with %s %s", tag, xml.checkerror(top,toclose) or "") - elseif toclose.tg ~= tag then -- no namespace check - errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.checkerror(top,toclose) or "") - end - dt = top.dt - dt[#dt+1] = toclose - -- dt[0] = top -- nasty circular reference when serializing table - if toclose.at.xmlns then - remove(xmlns) - end +local function add_attribute(namespace,tag,value) + if cleanup and #value>0 then + value=cleanup(value) + end + if tag=="xmlns" then + xmlns[#xmlns+1]=resolvens(value) + at[tag]=value + elseif namespace=="" then + at[tag]=value + elseif namespace=="xmlns" then + xml.checkns(tag,value) + at["xmlns:"..tag]=value + else + at[namespace..":"..tag]=value + end +end +local function add_empty(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=(namespace=="" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top=stack[#stack] + dt=top.dt + local t={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=top } + dt[#dt+1]=t + setmetatable(t,mt) + if at.xmlns then + remove(xmlns) + end + at={} +end +local function add_begin(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=(namespace=="" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=stack[#stack] } + setmetatable(top,mt) + dt=top.dt + stack[#stack+1]=top + at={} +end +local function add_end(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local toclose=remove(stack) + top=stack[#stack] + if #stack<1 then + errorstr=format("nothing to close with %s %s",tag,xml.checkerror(top,toclose) or "") + elseif toclose.tg~=tag then + errorstr=format("unable to close %s with %s %s",toclose.tg,tag,xml.checkerror(top,toclose) or "") + end + dt=top.dt + dt[#dt+1]=toclose + if toclose.at.xmlns then + remove(xmlns) + end end - local function add_text(text) - if cleanup and #text > 0 then - dt[#dt+1] = cleanup(text) - else - dt[#dt+1] = text - end + if cleanup and #text>0 then + dt[#dt+1]=cleanup(text) + else + dt[#dt+1]=text + end +end +local function add_special(what,spacing,text) + if #spacing>0 then + dt[#dt+1]=spacing + end + if strip and (what=="@cm@" or what=="@dt@") then + else + dt[#dt+1]={ special=true,ns="",tg=what,dt={ text } } + end end - -local function add_special(what, spacing, text) - if #spacing > 0 then - dt[#dt+1] = spacing - end - if strip and (what == "@cm@" or what == "@dt@") then - -- forget it - else - dt[#dt+1] = { special=true, ns="", tg=what, dt={ text } } - end -end - local function set_message(txt) - errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") + errorstr="garbage at the end of the file: "..gsub(txt,"([ \n\r\t]*)","") end - -local reported_attribute_errors = { } - +local reported_attribute_errors={} local function attribute_value_error(str) - if not reported_attribute_errors[str] then - report_xml("invalid attribute value: %q",str) - reported_attribute_errors[str] = true - at._error_ = str - end - return str + if not reported_attribute_errors[str] then + report_xml("invalid attribute value: %q",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str end - local function attribute_specification_error(str) - if not reported_attribute_errors[str] then - report_xml("invalid attribute specification: %q",str) - reported_attribute_errors[str] = true - at._error_ = str - end - return str -end - -xml.placeholders = { - unknown_dec_entity = function(str) return (str == "" and "&error;") or format("&%s;",str) end, - unknown_hex_entity = function(str) return format("&#x%s;",str) end, - unknown_any_entity = function(str) return format("&#x%s;",str) end, + if not reported_attribute_errors[str] then + report_xml("invalid attribute specification: %q",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str +end +xml.placeholders={ + unknown_dec_entity=function(str) return (str=="" and "&error;") or format("&%s;",str) end, + unknown_hex_entity=function(str) return format("&#x%s;",str) end, + unknown_any_entity=function(str) return format("&#x%s;",str) end, } - -local placeholders = xml.placeholders - +local placeholders=xml.placeholders local function fromhex(s) - local n = tonumber(s,16) - if n then - return utfchar(n) - else - return format("h:%s",s), true - end + local n=tonumber(s,16) + if n then + return utfchar(n) + else + return format("h:%s",s),true + end end - local function fromdec(s) - local n = tonumber(s) - if n then - return utfchar(n) - else - return format("d:%s",s), true - end -end - --- one level expansion (simple case), no checking done - -local rest = (1-P(";"))^0 -local many = P(1)^0 - -local parsedentity = - P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + - (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) - --- parsing in the xml file - -local predefined_unified = { - [38] = "&", - [42] = """, - [47] = "'", - [74] = "<", - [76] = ">", + local n=tonumber(s) + if n then + return utfchar(n) + else + return format("d:%s",s),true + end +end +local rest=(1-P(";"))^0 +local many=P(1)^0 +local parsedentity=P("&")*(P("#x")*(rest/fromhex)+P("#")*(rest/fromdec))*P(";")*P(-1)+(P("#x")*(many/fromhex)+P("#")*(many/fromdec)) +local predefined_unified={ + [38]="&", + [42]=""", + [47]="'", + [74]="<", + [76]=">", } - -local predefined_simplified = { - [38] = "&", amp = "&", - [42] = '"', quot = '"', - [47] = "'", apos = "'", - [74] = "<", lt = "<", - [76] = ">", gt = ">", -} - -local nofprivates = 0xF0000 -- shared but seldom used - -local privates_u = { -- unescaped - [ [[&]] ] = "&", - [ [["]] ] = """, - [ [[']] ] = "'", - [ [[<]] ] = "<", - [ [[>]] ] = ">", +local predefined_simplified={ + [38]="&",amp="&", + [42]='"',quot='"', + [47]="'",apos="'", + [74]="<",lt="<", + [76]=">",gt=">", } - -local privates_p = { +local nofprivates=0xF0000 +local privates_u={ + [ [[&]] ]="&", + [ [["]] ]=""", + [ [[']] ]="'", + [ [[<]] ]="<", + [ [[>]] ]=">", } - -local privates_n = { - -- keeps track of defined ones +local privates_p={} +local privates_n={ } - -local escaped = utf.remapper(privates_u) - +local escaped=utf.remapper(privates_u) local function unescaped(s) - local p = privates_n[s] - if not p then - nofprivates = nofprivates + 1 - p = utfchar(nofprivates) - privates_n[s] = p - s = "&" .. s .. ";" -- todo: use char-ent to map to hex - privates_u[p] = s - privates_p[p] = s - end - return p -end - -local unprivatized = utf.remapper(privates_p) - -xml.privatetoken = unescaped -xml.unprivatized = unprivatized -xml.privatecodes = privates_n - + local p=privates_n[s] + if not p then + nofprivates=nofprivates+1 + p=utfchar(nofprivates) + privates_n[s]=p + s="&"..s..";" + privates_u[p]=s + privates_p[p]=s + end + return p +end +local unprivatized=utf.remapper(privates_p) +xml.privatetoken=unescaped +xml.unprivatized=unprivatized +xml.privatecodes=privates_n local function handle_hex_entity(str) - local h = hcache[str] - if not h then - local n = tonumber(str,16) - h = unify_predefined and predefined_unified[n] - if h then - if trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %s",str,h) - end - elseif utfize then - h = (n and utfchar(n)) or xml.unknown_hex_entity(str) or "" - if not n then - report_xml("utfize, ignoring hex entity &#x%s;",str) - elseif trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %s",str,h) - end - else - if trace_entities then - report_xml("found entity &#x%s;",str) - end - h = "&#x" .. str .. ";" - end - hcache[str] = h + local h=hcache[str] + if not h then + local n=tonumber(str,16) + h=unify_predefined and predefined_unified[n] + if h then + if trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %s",str,h) + end + elseif utfize then + h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" + if not n then + report_xml("utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %s",str,h) + end + else + if trace_entities then + report_xml("found entity &#x%s;",str) + end + h="&#x"..str..";" end - return h + hcache[str]=h + end + return h end - local function handle_dec_entity(str) - local d = dcache[str] - if not d then - local n = tonumber(str) - d = unify_predefined and predefined_unified[n] - if d then + local d=dcache[str] + if not d then + local n=tonumber(str) + d=unify_predefined and predefined_unified[n] + if d then + if trace_entities then + report_xml("utfize, converting dec entity &#%s; into %s",str,d) + end + elseif utfize then + d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" + if not n then + report_xml("utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + report_xml("utfize, converting dec entity &#%s; into %s",str,d) + end + else + if trace_entities then + report_xml("found entity &#%s;",str) + end + d="&#"..str..";" + end + dcache[str]=d + end + return d +end +xml.parsedentitylpeg=parsedentity +local function handle_any_entity(str) + if resolve then + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + if trace_entities then + report_xml("resolved entity &%s; -> %s (predefined)",str,a) + end + else + if type(resolve)=="function" then + a=resolve(str) or entities[str] + else + a=entities[str] + end + if a then + if type(a)=="function" then if trace_entities then - report_xml("utfize, converting dec entity &#%s; into %s",str,d) - end - elseif utfize then - d = (n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" - if not n then - report_xml("utfize, ignoring dec entity &#%s;",str) - elseif trace_entities then - report_xml("utfize, converting dec entity &#%s; into %s",str,d) + report_xml("expanding entity &%s; (function)",str) end + a=a(str) or "" + end + a=lpegmatch(parsedentity,a) or a + if trace_entities then + report_xml("resolved entity &%s; -> %s (internal)",str,a) + end else + local unknown_any_entity=placeholders.unknown_any_entity + if unknown_any_entity then + a=unknown_any_entity(str) or "" + end + if a then if trace_entities then - report_xml("found entity &#%s;",str) - end - d = "&#" .. str .. ";" - end - dcache[str] = d - end - return d -end - -xml.parsedentitylpeg = parsedentity - -local function handle_any_entity(str) - if resolve then - local a = acache[str] -- per instance ! todo - if not a then - a = resolve_predefined and predefined_simplified[str] - if a then - if trace_entities then - report_xml("resolved entity &%s; -> %s (predefined)",str,a) - end - else - if type(resolve) == "function" then - a = resolve(str) or entities[str] - else - a = entities[str] - end - if a then - if type(a) == "function" then - if trace_entities then - report_xml("expanding entity &%s; (function)",str) - end - a = a(str) or "" - end - a = lpegmatch(parsedentity,a) or a -- for nested - if trace_entities then - report_xml("resolved entity &%s; -> %s (internal)",str,a) - end - else - local unknown_any_entity = placeholders.unknown_any_entity - if unknown_any_entity then - a = unknown_any_entity(str) or "" - end - if a then - if trace_entities then - report_xml("resolved entity &%s; -> %s (external)",str,a) - end - else - if trace_entities then - report_xml("keeping entity &%s;",str) - end - if str == "" then - a = "&error;" - else - a = "&" .. str .. ";" - end - end - end + report_xml("resolved entity &%s; -> %s (external)",str,a) end - acache[str] = a - elseif trace_entities then - if not acache[str] then - report_xml("converting entity &%s; into %s",str,a) - acache[str] = a + else + if trace_entities then + report_xml("keeping entity &%s;",str) end - end - return a - else - local a = acache[str] - if not a then - a = resolve_predefined and predefined_simplified[str] - if a then - -- one of the predefined - acache[str] = a - if trace_entities then - report_xml("entity &%s; becomes %s",str,tostring(a)) - end - elseif str == "" then - if trace_entities then - report_xml("invalid entity &%s;",str) - end - a = "&error;" - acache[str] = a + if str=="" then + a="&error;" else - if trace_entities then - report_xml("entity &%s; is made private",str) - end - -- a = "&" .. str .. ";" - a = unescaped(str) - acache[str] = a + a="&"..str..";" end + end end - return a + end + acache[str]=a + elseif trace_entities then + if not acache[str] then + report_xml("converting entity &%s; into %s",str,a) + acache[str]=a + end + end + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %s",str,tostring(a)) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a="&error;" + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a + end end + return a + end end - local function handle_end_entity(chr) - report_xml("error in entity, %q found instead of ';'",chr) -end - -local space = S(' \r\n\t') -local open = P('<') -local close = P('>') -local squote = S("'") -local dquote = S('"') -local equal = P('=') -local slash = P('/') -local colon = P(':') -local semicolon = P(';') -local ampersand = P('&') -local valid = R('az', 'AZ', '09') + S('_-.') -local name_yes = C(valid^1) * colon * C(valid^1) -local name_nop = C(P(true)) * C(valid^1) -local name = name_yes + name_nop -local utfbom = lpeg.patterns.utfbom -- no capture -local spacing = C(space^0) - ------ entitycontent = (1-open-semicolon)^0 -local anyentitycontent = (1-open-semicolon-space-close)^0 -local hexentitycontent = R("AF","af","09")^0 -local decentitycontent = R("09")^0 -local parsedentity = P("#")/"" * ( - P("x")/"" * (hexentitycontent/handle_hex_entity) + - (decentitycontent/handle_dec_entity) - ) + (anyentitycontent/handle_any_entity) -local entity = ampersand/"" * parsedentity * ( (semicolon/"") + #(P(1)/handle_end_entity)) - -local text_unparsed = C((1-open)^1) -local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) - -local somespace = space^1 -local optionalspace = space^0 - ------ value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value -local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value - -local endofattributes = slash * close + close -- recovery of flacky html -local whatever = space * name * optionalspace * equal ------ wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error ------ wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error ------ wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error -local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error - -local attributevalue = value + wrongvalue - -local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute ------ attributes = (attribute)^0 - -local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 - -local parsedtext = text_parsed / add_text -local unparsedtext = text_unparsed / add_text -local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example - -local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty -local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin -local endelement = (spacing * open * slash * name * optionalspace * close) / add_end - -local begincomment = open * P("!--") -local endcomment = P("--") * close -local begininstruction = open * P("?") -local endinstruction = P("?") * close -local begincdata = open * P("![CDATA[") -local endcdata = P("]]") * close - -local someinstruction = C((1 - endinstruction)^0) -local somecomment = C((1 - endcomment )^0) -local somecdata = C((1 - endcdata )^0) - -local function normalentity(k,v ) entities[k] = v end -local function systementity(k,v,n) entities[k] = v end -local function publicentity(k,v,n) entities[k] = v end - --- todo: separate dtd parser - -local begindoctype = open * P("!DOCTYPE") -local enddoctype = close -local beginset = P("[") -local endset = P("]") -local doctypename = C((1-somespace-close)^0) -local elementdoctype = optionalspace * P("') +local squote=S("'") +local dquote=S('"') +local equal=P('=') +local slash=P('/') +local colon=P(':') +local semicolon=P(';') +local ampersand=P('&') +local valid=R('az','AZ','09')+S('_-.') +local name_yes=C(valid^1)*colon*C(valid^1) +local name_nop=C(P(true))*C(valid^1) +local name=name_yes+name_nop +local utfbom=lpeg.patterns.utfbom +local spacing=C(space^0) +local anyentitycontent=(1-open-semicolon-space-close)^0 +local hexentitycontent=R("AF","af","09")^0 +local decentitycontent=R("09")^0 +local parsedentity=P("#")/""*( + P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) + )+(anyentitycontent/handle_any_entity) +local entity=ampersand/""*parsedentity*((semicolon/"")+#(P(1)/handle_end_entity)) +local text_unparsed=C((1-open)^1) +local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) +local somespace=space^1 +local optionalspace=space^0 +local value=(squote*Cs((entity+(1-squote))^0)*squote)+(dquote*Cs((entity+(1-dquote))^0)*dquote) +local endofattributes=slash*close+close +local whatever=space*name*optionalspace*equal +local wrongvalue=Cs(P(entity+(1-space-endofattributes))^1)/attribute_value_error +local attributevalue=value+wrongvalue +local attribute=(somespace*name*optionalspace*equal*optionalspace*attributevalue)/add_attribute +local attributes=(attribute+somespace^-1*(((1-endofattributes)^1)/attribute_specification_error))^0 +local parsedtext=text_parsed/add_text +local unparsedtext=text_unparsed/add_text +local balanced=P { "["*((1-S"[]")+V(1))^0*"]" } +local emptyelement=(spacing*open*name*attributes*optionalspace*slash*close)/add_empty +local beginelement=(spacing*open*name*attributes*optionalspace*close)/add_begin +local endelement=(spacing*open*slash*name*optionalspace*close)/add_end +local begincomment=open*P("!--") +local endcomment=P("--")*close +local begininstruction=open*P("?") +local endinstruction=P("?")*close +local begincdata=open*P("![CDATA[") +local endcdata=P("]]")*close +local someinstruction=C((1-endinstruction)^0) +local somecomment=C((1-endcomment )^0) +local somecdata=C((1-endcdata )^0) +local function normalentity(k,v ) entities[k]=v end +local function systementity(k,v,n) entities[k]=v end +local function publicentity(k,v,n) entities[k]=v end +local begindoctype=open*P("!DOCTYPE") +local enddoctype=close +local beginset=P("[") +local endset=P("]") +local doctypename=C((1-somespace-close)^0) +local elementdoctype=optionalspace*P(" & - cleanup = settings.text_cleanup - entities = settings.entities or { } - -- - if utfize == nil then - settings.utfize_entities = true - utfize = true - end - if resolve_predefined == nil then - settings.resolve_predefined_entities = true - resolve_predefined = true - end - -- - -- - stack, top, at, xmlns, errorstr = { }, { }, { }, { }, nil - acache, hcache, dcache = { }, { }, { } -- not stored - reported_attribute_errors = { } - if settings.parent_root then - mt = getmetatable(settings.parent_root) - else - initialize_mt(top) - end - stack[#stack+1] = top - top.dt = { } - dt = top.dt - if not data or data == "" then - errorstr = "empty xml file" - elseif utfize or resolve then - if lpegmatch(grammar_parsed_text,data) then - errorstr = "" - else - errorstr = "invalid xml file - parsed text" - end - elseif type(data) == "string" then - if lpegmatch(grammar_unparsed_text,data) then - errorstr = "" - else - errorstr = "invalid xml file - unparsed text" - end - else - errorstr = "invalid xml file - no text at all" - end - local result - if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={ }, er = true } } } - setmetatable(stack, mt) - local errorhandler = settings.error_handler - if errorhandler == false then - -- no error message - else - errorhandler = errorhandler or xml.errorhandler - if errorhandler then - local currentresource = settings.currentresource - if currentresource and currentresource ~= "" then - xml.errorhandler(format("load error in [%s]: %s",currentresource,errorstr)) - else - xml.errorhandler(format("load error: %s",errorstr)) - end - end - end - else - result = stack[1] - end - if not settings.no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={ }, entities = entities, settings = settings } - setmetatable(result, mt) - local rdt = result.dt - for k=1,#rdt do - local v = rdt[k] - if type(v) == "table" and not v.special then -- always table -) - result.ri = k -- rootindex - v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! - break - end +local function _xmlconvert_(data,settings) + settings=settings or {} + strip=settings.strip_cm_and_dt + utfize=settings.utfize_entities + resolve=settings.resolve_entities + resolve_predefined=settings.resolve_predefined_entities + unify_predefined=settings.unify_predefined_entities + cleanup=settings.text_cleanup + entities=settings.entities or {} + if utfize==nil then + settings.utfize_entities=true + utfize=true + end + if resolve_predefined==nil then + settings.resolve_predefined_entities=true + resolve_predefined=true + end + stack,top,at,xmlns,errorstr={},{},{},{},nil + acache,hcache,dcache={},{},{} + reported_attribute_errors={} + if settings.parent_root then + mt=getmetatable(settings.parent_root) + else + initialize_mt(top) + end + stack[#stack+1]=top + top.dt={} + dt=top.dt + if not data or data=="" then + errorstr="empty xml file" + elseif utfize or resolve then + if lpegmatch(grammar_parsed_text,data) then + errorstr="" + else + errorstr="invalid xml file - parsed text" + end + elseif type(data)=="string" then + if lpegmatch(grammar_unparsed_text,data) then + errorstr="" + else + errorstr="invalid xml file - unparsed text" + end + else + errorstr="invalid xml file - no text at all" + end + local result + if errorstr and errorstr~="" then + result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } + setmetatable(stack,mt) + local errorhandler=settings.error_handler + if errorhandler==false then + else + errorhandler=errorhandler or xml.errorhandler + if errorhandler then + local currentresource=settings.currentresource + if currentresource and currentresource~="" then + xml.errorhandler(format("load error in [%s]: %s",currentresource,errorstr)) + else + xml.errorhandler(format("load error: %s",errorstr)) end + end end - if errorstr and errorstr ~= "" then - result.error = true + else + result=stack[1] + end + if not settings.no_root then + result={ special=true,ns="",tg='@rt@',dt=result.dt,at={},entities=entities,settings=settings } + setmetatable(result,mt) + local rdt=result.dt + for k=1,#rdt do + local v=rdt[k] + if type(v)=="table" and not v.special then + result.ri=k + v.__p__=result + break + end end - result.statistics = { - entities = { - decimals = dcache, - hexadecimals = hcache, - names = acache, - } + end + if errorstr and errorstr~="" then + result.error=true + end + result.statistics={ + entities={ + decimals=dcache, + hexadecimals=hcache, + names=acache, } - strip, utfize, resolve, resolve_predefined = nil, nil, nil, nil - unify_predefined, cleanup, entities = nil, nil, nil - stack, top, at, xmlns, errorstr = nil, nil, nil, nil, nil - acache, hcache, dcache = nil, nil, nil - reported_attribute_errors, mt, errorhandler = nil, nil, nil - return result + } + strip,utfize,resolve,resolve_predefined=nil,nil,nil,nil + unify_predefined,cleanup,entities=nil,nil,nil + stack,top,at,xmlns,errorstr=nil,nil,nil,nil,nil + acache,hcache,dcache=nil,nil,nil + reported_attribute_errors,mt,errorhandler=nil,nil,nil + return result end - --- Because we can have a crash (stack issues) with faulty xml, we wrap this one --- in a protector: - function xmlconvert(data,settings) - local ok, result = pcall(function() return _xmlconvert_(data,settings) end) - if ok then - return result - else - return _xmlconvert_("",settings) - end -end - -xml.convert = xmlconvert - -function xml.inheritedconvert(data,xmldata) -- xmldata is parent - local settings = xmldata.settings - if settings then - settings.parent_root = xmldata -- to be tested - end - -- settings.no_root = true - local xc = xmlconvert(data,settings) -- hm, we might need to locate settings - -- xc.settings = nil - -- xc.entities = nil - -- xc.special = nil - -- xc.ri = nil - -- print(xc.tg) - return xc + local ok,result=pcall(function() return _xmlconvert_(data,settings) end) + if ok then + return result + else + return _xmlconvert_("",settings) + end +end +xml.convert=xmlconvert +function xml.inheritedconvert(data,xmldata) + local settings=xmldata.settings + if settings then + settings.parent_root=xmldata + end + local xc=xmlconvert(data,settings) + return xc end - - - function xml.is_valid(root) - return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er + return root and root.dt and root.dt[1] and type(root.dt[1])=="table" and not root.dt[1].er end - function xml.package(tag,attributes,data) - local ns, tg = match(tag,"^(.-):?([^:]+)$") - local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } - setmetatable(t, mt) - return t + local ns,tg=match(tag,"^(.-):?([^:]+)$") + local t={ ns=ns,tg=tg,dt=data or "",at=attributes or {} } + setmetatable(t,mt) + return t end - function xml.is_valid(root) - return root and not root.error + return root and not root.error end - -xml.errorhandler = report_xml - - - +xml.errorhandler=report_xml function xml.load(filename,settings) - local data = "" - if type(filename) == "string" then - -- local data = io.loaddata(filename) - -todo: check type in io.loaddata - local f = io.open(filename,'r') -- why not 'rb' - if f then - data = f:read("*all") -- io.readall(f) ... only makes sense for large files - f:close() - end - elseif filename then -- filehandle - data = filename:read("*all") -- io.readall(f) ... only makes sense for large files - end - if settings then - settings.currentresource = filename - local result = xmlconvert(data,settings) - settings.currentresource = nil - return result - else - return xmlconvert(data,{ currentresource = filename }) - end + local data="" + if type(filename)=="string" then + local f=io.open(filename,'r') + if f then + data=f:read("*all") + f:close() + end + elseif filename then + data=filename:read("*all") + end + if settings then + settings.currentresource=filename + local result=xmlconvert(data,settings) + settings.currentresource=nil + return result + else + return xmlconvert(data,{ currentresource=filename }) + end end - - - -local no_root = { no_root = true } - +local no_root={ no_root=true } function xml.toxml(data) - if type(data) == "string" then - local root = { xmlconvert(data,no_root) } - return (#root > 1 and root) or root[1] - else - return data - end + if type(data)=="string" then + local root={ xmlconvert(data,no_root) } + return (#root>1 and root) or root[1] + else + return data + end end - - - local function copy(old,tables) - if old then - tables = tables or { } - local new = { } - if not tables[old] then - tables[old] = new - end - for k,v in next, old do - new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) - end - return new - else - return { } + if old then + tables=tables or {} + local new={} + if not tables[old] then + tables[old]=new end -end - -xml.copy = copy - - - --- todo: add when not present - -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt = root.dt - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then - return - end - end - insert(dt, 1, { special = true, ns = "", tg = "@pi@", dt = { "xml version='1.0' standalone='yes'" } } ) - insert(dt, 2, "\n" ) + for k,v in next,old do + new[k]=(type(v)=="table" and (tables[v] or copy(v,tables))) or v end -end - - - --- new experimental reorganized serialize - -local function verbose_element(e,handlers) -- options - local handle = handlers.handle - local serialize = handlers.serialize - local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } - if ats then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,escaped(v)) - end + local mt=getmetatable(old) + if mt then + setmetatable(new,mt) end - if ern and trace_entities and ern ~= ens then - ens = ern + return new + else + return {} + end +end +xml.copy=copy +function xml.checkbom(root) + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" and find(v.dt[1],"xml.*version=") then + return + end end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - handle("<",ens,":",etg," ",concat(ats," "),">") - else - handle("<",ens,":",etg,">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - handle(escaped(e)) - else - serialize(e,handlers) - end - end - handle("") + insert(dt,1,{ special=true,ns="",tg="@pi@",dt={ "xml version='1.0' standalone='yes'" } } ) + insert(dt,2,"\n" ) + end +end +local function verbose_element(e,handlers) + local handle=handlers.handle + local serialize=handlers.serialize + local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn + local ats=eat and next(eat) and {} + if ats then + for k,v in next,eat do + ats[#ats+1]=format('%s=%q',k,escaped(v)) + end + end + if ern and trace_entities and ern~=ens then + ens=ern + end + if ens~="" then + if edt and #edt>0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) else - if ats then - handle("<",ens,":",etg," ",concat(ats," "),"/>") - else - handle("<",ens,":",etg,"/>") - end + serialize(e,handlers) end + end + handle("") else - if edt and #edt > 0 then - if ats then - handle("<",etg," ",concat(ats," "),">") - else - handle("<",etg,">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - handle(escaped(e)) -- option: hexify escaped entities - else - serialize(e,handlers) - end - end - handle("") + if ats then + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") + end + end + else + if edt and #edt>0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) else - if ats then - handle("<",etg," ",concat(ats," "),"/>") - else - handle("<",etg,"/>") - end + serialize(e,handlers) end + end + handle("") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") + else + handle("<",etg,"/>") + end end + end end - local function verbose_pi(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_comment(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_cdata(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_doctype(e,handlers) - handlers.handle("") + handlers.handle("") end - local function verbose_root(e,handlers) - handlers.serialize(e.dt,handlers) + handlers.serialize(e.dt,handlers) end - local function verbose_text(e,handlers) - handlers.handle(escaped(e)) + handlers.handle(escaped(e)) end - local function verbose_document(e,handlers) - local serialize = handlers.serialize - local functions = handlers.functions - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - functions["@tx@"](ei,handlers) - else - serialize(ei,handlers) - end + local serialize=handlers.serialize + local functions=handlers.functions + for i=1,#e do + local ei=e[i] + if type(ei)=="string" then + functions["@tx@"](ei,handlers) + else + serialize(ei,handlers) end + end end - local function serialize(e,handlers,...) - local initialize = handlers.initialize - local finalize = handlers.finalize - local functions = handlers.functions - if initialize then - local state = initialize(...) - if not state == true then - return state - end - end - local etg = e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - -- elseif type(e) == "string" then - -- functions["@tx@"](e,handlers) - else - functions["@dc@"](e,handlers) -- dc ? - end - if finalize then - return finalize() - end + local initialize=handlers.initialize + local finalize=handlers.finalize + local functions=handlers.functions + if initialize then + local state=initialize(...) + if not state==true then + return state + end + end + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end - local function xserialize(e,handlers) - local functions = handlers.functions - local etg = e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - -- elseif type(e) == "string" then - -- functions["@tx@"](e,handlers) - else - functions["@dc@"](e,handlers) - end -end - -local handlers = { } - + local functions=handlers.functions + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end +end +local handlers={} local function newhandlers(settings) - local t = table.copy(handlers.verbose or { }) -- merge - if settings then - for k,v in next, settings do - if type(v) == "table" then - local tk = t[k] if not tk then tk = { } t[k] = tk end - for kk,vv in next, v do - tk[kk] = vv - end - else - t[k] = v - end - end - if settings.name then - handlers[settings.name] = t - end + local t=table.copy(handlers.verbose or {}) + if settings then + for k,v in next,settings do + if type(v)=="table" then + local tk=t[k] if not tk then tk={} t[k]=tk end + for kk,vv in next,v do + tk[kk]=vv + end + else + t[k]=v + end end - utilities.storage.mark(t) - return t + if settings.name then + handlers[settings.name]=t + end + end + utilities.storage.mark(t) + return t end - -local nofunction = function() end - +local nofunction=function() end function xml.sethandlersfunction(handler,name,fnc) - handler.functions[name] = fnc or nofunction + handler.functions[name]=fnc or nofunction end - function xml.gethandlersfunction(handler,name) - return handler.functions[name] + return handler.functions[name] end - function xml.gethandlers(name) - return handlers[name] + return handlers[name] end - newhandlers { - name = "verbose", - initialize = false, -- faster than nil and mt lookup - finalize = false, -- faster than nil and mt lookup - serialize = xserialize, - handle = print, - functions = { - ["@dc@"] = verbose_document, - ["@dt@"] = verbose_doctype, - ["@rt@"] = verbose_root, - ["@el@"] = verbose_element, - ["@pi@"] = verbose_pi, - ["@cm@"] = verbose_comment, - ["@cd@"] = verbose_cdata, - ["@tx@"] = verbose_text, - } + name="verbose", + initialize=false, + finalize=false, + serialize=xserialize, + handle=print, + functions={ + ["@dc@"]=verbose_document, + ["@dt@"]=verbose_doctype, + ["@rt@"]=verbose_root, + ["@el@"]=verbose_element, + ["@pi@"]=verbose_pi, + ["@cm@"]=verbose_comment, + ["@cd@"]=verbose_cdata, + ["@tx@"]=verbose_text, + } } - - - --- maybe this will move to lxml-xml - local result - -local xmlfilehandler = newhandlers { - name = "file", - initialize = function(name) - result = io.open(name,"wb") - return result - end, - finalize = function() - result:close() - return true - end, - handle = function(...) - result:write(...) - end, +local xmlfilehandler=newhandlers { + name="file", + initialize=function(name) + result=io.open(name,"wb") + return result + end, + finalize=function() + result:close() + return true + end, + handle=function(...) + result:write(...) + end, } - --- no checking on writeability here but not faster either --- --- local xmlfilehandler = newhandlers { --- initialize = function(name) --- io.output(name,"wb") --- return true --- end, --- finalize = function() --- io.close() --- return true --- end, --- handle = io.write, --- } - function xml.save(root,name) - serialize(root,xmlfilehandler,name) + serialize(root,xmlfilehandler,name) end - local result - -local xmlstringhandler = newhandlers { - name = "string", - initialize = function() - result = { } - return result - end, - finalize = function() - return concat(result) - end, - handle = function(...) - result[#result+1] = concat { ... } - end, +local xmlstringhandler=newhandlers { + name="string", + initialize=function() + result={} + return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, } - -local function xmltostring(root) -- 25% overhead due to collecting - if not root then - return "" - elseif type(root) == 'string' then - return root - else -- if next(root) then -- next is faster than type (and >0 test) - return serialize(root,xmlstringhandler) or "" - end +local function xmltostring(root) + if not root then + return "" + elseif type(root)=='string' then + return root + else + return serialize(root,xmlstringhandler) or "" + end end - -local function __tostring(root) -- inline - return (root and xmltostring(root)) or "" +local function __tostring(root) + return (root and xmltostring(root)) or "" end - -initialize_mt = function(root) -- redefinition - mt = { __tostring = __tostring, __index = root } +initialize_mt=function(root) + mt={ __tostring=__tostring,__index=root } end - -xml.defaulthandlers = handlers -xml.newhandlers = newhandlers -xml.serialize = serialize -xml.tostring = xmltostring - - - +xml.defaulthandlers=handlers +xml.newhandlers=newhandlers +xml.serialize=serialize +xml.tostring=xmltostring local function xmlstring(e,handle) - if not handle or (e.special and e.tg ~= "@rt@") then - -- nothing - elseif e.tg then - local edt = e.dt - if edt then - for i=1,#edt do - xmlstring(edt[i],handle) - end - end - else - handle(e) + if not handle or (e.special and e.tg~="@rt@") then + elseif e.tg then + local edt=e.dt + if edt then + for i=1,#edt do + xmlstring(edt[i],handle) + end end + else + handle(e) + end end - -xml.string = xmlstring - - - - +xml.string=xmlstring function xml.settings(e) - while e do - local s = e.settings - if s then - return s - else - e = e.__p__ - end + while e do + local s=e.settings + if s then + return s + else + e=e.__p__ end - return nil + end + return nil end - function xml.root(e) - local r = e - while e do - e = e.__p__ - if e then - r = e - end + local r=e + while e do + e=e.__p__ + if e then + r=e end - return r + end + return r end - function xml.parent(root) - return root.__p__ + return root.__p__ end - function xml.body(root) - return root.ri and root.dt[root.ri] or root -- not ok yet + return root.ri and root.dt[root.ri] or root end - function xml.name(root) - if not root then - return "" - end - local ns = root.ns - local tg = root.tg - if ns == "" then - return tg - else - return ns .. ":" .. tg - end + if not root then + return "" + end + local ns=root.ns + local tg=root.tg + if ns=="" then + return tg + else + return ns..":"..tg + end end - - - function xml.erase(dt,k) - if dt then - if k then - dt[k] = "" - else for k=1,#dt do - dt[1] = { "" } - end end - end + if dt then + if k then + dt[k]="" + else for k=1,#dt do + dt[1]={ "" } + end end + end end - - - function xml.assign(dt,k,root) - if dt and k then - dt[k] = type(root) == "table" and xml.body(root) or root - return dt[k] - else - return xml.body(root) - end + if dt and k then + dt[k]=type(root)=="table" and xml.body(root) or root + return dt[k] + else + return xml.body(root) + end +end +function xml.tocdata(e,wrapper) + local whatever=type(e)=="table" and xmltostring(e.dt) or e or "" + if wrapper then + whatever=format("<%s>%s",wrapper,whatever,wrapper) + end + local t={ special=true,ns="",tg="@cd@",at={},rn="",dt={ whatever },__p__=e } + setmetatable(t,getmetatable(e)) + e.dt={ t } end - --- the following helpers may move - - - -function xml.tocdata(e,wrapper) -- a few more in the aux module - local whatever = type(e) == "table" and xmltostring(e.dt) or e or "" - if wrapper then - whatever = format("<%s>%s",wrapper,whatever,wrapper) - end - local t = { special = true, ns = "", tg = "@cd@", at = { }, rn = "", dt = { whatever }, __p__ = e } - setmetatable(t,getmetatable(e)) - e.dt = { t } -end - function xml.makestandalone(root) - if root.ri then - local dt = root.dt - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi@" then - local txt = v.dt[1] - if find(txt,"xml.*version=") then - v.dt[1] = txt .. " standalone='yes'" - break - end - end + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" then + local txt=v.dt[1] + if find(txt,"xml.*version=") then + v.dt[1]=txt.." standalone='yes'" + break end + end end - return root + end + return root end - function xml.kind(e) - local dt = e and e.dt - if dt then - local n = #dt - if n == 1 then - local d = dt[1] - if d.special then - local tg = d.tg - if tg == "@cd@" then - return "cdata" - elseif tg == "@cm" then - return "comment" - elseif tg == "@pi@" then - return "instruction" - elseif tg == "@dt@" then - return "declaration" - end - elseif type(d) == "string" then - return "text" - end - return "element" - elseif n > 0 then - return "mixed" - end + local dt=e and e.dt + if dt then + local n=#dt + if n==1 then + local d=dt[1] + if d.special then + local tg=d.tg + if tg=="@cd@" then + return "cdata" + elseif tg=="@cm" then + return "comment" + elseif tg=="@pi@" then + return "instruction" + elseif tg=="@dt@" then + return "declaration" + end + elseif type(d)=="string" then + return "text" + end + return "element" + elseif n>0 then + return "mixed" end - return "empty" + end + return "empty" end @@ -10804,1308 +8138,1036 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-lpt'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- e.ni is only valid after a filter run --- todo: B/C/[get first match] - -local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, tonumber, tostring, setmetatable, load, select = type, next, tonumber, tostring, setmetatable, load, select -local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns - -local setmetatableindex = table.setmetatableindex - --- beware, this is not xpath ... e.g. position is different (currently) and --- we have reverse-sibling as reversed preceding sibling - - - - - -local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end -local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end -local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end - -local report_lpath = logs.reporter("xml","lpath") - - - -local xml = xml - -local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end -local lpathcached = 0 function xml.lpathcached() return lpathcached end - -xml.functions = xml.functions or { } -- internal -local functions = xml.functions - -xml.expressions = xml.expressions or { } -- in expressions -local expressions = xml.expressions - -xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) -local finalizers = xml.finalizers - -xml.specialhandler = xml.specialhandler or { } -local specialhandler = xml.specialhandler - -lpegpatterns.xml = lpegpatterns.xml or { } -local xmlpatterns = lpegpatterns.xml - -finalizers.xml = finalizers.xml or { } -finalizers.tex = finalizers.tex or { } - -local function fallback (t, name) - local fn = finalizers[name] - if fn then - t[name] = fn - else - report_lpath("unknown sub finalizer '%s'",tostring(name)) - fn = function() end - end - return fn -end - -setmetatableindex(finalizers.xml, fallback) -setmetatableindex(finalizers.tex, fallback) - -xml.defaultprotocol = "xml" - --- as xsl does not follow xpath completely here we will also --- be more liberal especially with regards to the use of | and --- the rootpath: --- --- test : all 'test' under current --- /test : 'test' relative to current --- a|b|c : set of names --- (a|b|c) : idem --- ! : not --- --- after all, we're not doing transformations but filtering. in --- addition we provide filter functions (last bit) --- --- todo: optimizer --- --- .. : parent --- * : all kids --- / : anchor here --- // : /**/ --- ** : all in between --- --- so far we had (more practical as we don't transform) --- --- {/test} : kids 'test' under current node --- {test} : any kid with tag 'test' --- {//test} : same as above - --- evaluator (needs to be redone, for the moment copied) +-- original size: 47510, stripped down to: 30425 --- todo: apply_axis(list,notable) and collection vs single - -local apply_axis = { } - -apply_axis['root'] = function(list) - local collected = { } - for l=1,#list do - local ll = list[l] - local rt = ll - while ll do - ll = ll.__p__ - if ll then - rt = ll - end - end - collected[l] = rt +if not modules then modules={} end modules ['lxml-lpt']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,tonumber,tostring,setmetatable,load,select=type,next,tonumber,tostring,setmetatable,load,select +local format,upper,lower,gmatch,gsub,find,rep=string.format,string.upper,string.lower,string.gmatch,string.gsub,string.find,string.rep +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local setmetatableindex=table.setmetatableindex +local trace_lpath=false if trackers then trackers.register("xml.path",function(v) trace_lpath=v end) end +local trace_lparse=false if trackers then trackers.register("xml.parse",function(v) trace_lparse=v end) end +local trace_lprofile=false if trackers then trackers.register("xml.profile",function(v) trace_lpath=v trace_lparse=v trace_lprofile=v end) end +local report_lpath=logs.reporter("xml","lpath") +local xml=xml +local lpathcalls=0 function xml.lpathcalls () return lpathcalls end +local lpathcached=0 function xml.lpathcached() return lpathcached end +xml.functions=xml.functions or {} +local functions=xml.functions +xml.expressions=xml.expressions or {} +local expressions=xml.expressions +xml.finalizers=xml.finalizers or {} +local finalizers=xml.finalizers +xml.specialhandler=xml.specialhandler or {} +local specialhandler=xml.specialhandler +lpegpatterns.xml=lpegpatterns.xml or {} +local xmlpatterns=lpegpatterns.xml +finalizers.xml=finalizers.xml or {} +finalizers.tex=finalizers.tex or {} +local function fallback (t,name) + local fn=finalizers[name] + if fn then + t[name]=fn + else + report_lpath("unknown sub finalizer '%s'",tostring(name)) + fn=function() end + end + return fn +end +setmetatableindex(finalizers.xml,fallback) +setmetatableindex(finalizers.tex,fallback) +xml.defaultprotocol="xml" +local apply_axis={} +apply_axis['root']=function(list) + local collected={} + for l=1,#list do + local ll=list[l] + local rt=ll + while ll do + ll=ll.__p__ + if ll then + rt=ll + end end - return collected -end - -apply_axis['self'] = function(list) - return list -end - -apply_axis['child'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local dt = ll.dt - if dt then -- weird that this is needed - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - end - end - ll.en = en + collected[l]=rt + end + return collected +end +apply_axis['self']=function(list) + return list +end +apply_axis['child']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local dt=ll.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en end + end + ll.en=en end - return collected + end + return collected end - local function collect(list,collected,c) - local dt = list.dt - if dt then - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - c = collect(dk,collected,c) - end - end - list.en = en + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end end - return c + list.en=en + end + return c end - -apply_axis['descendant'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - c = collect(list[l],collected,c) - end - return collected +apply_axis['descendant']=function(list) + local collected,c={},0 + for l=1,#list do + c=collect(list[l],collected,c) + end + return collected end - local function collect(list,collected,c) - local dt = list.dt - if dt then - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - c = collect(dk,collected,c) - end - end - list.en = en - end - return c -end -apply_axis['descendant-or-self'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - if ll.special ~= true then -- catch double root - c = c + 1 - collected[c] = ll - end - c = collect(ll,collected,c) - end - return collected -end - -apply_axis['ancestor'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - while ll do - ll = ll.__p__ - if ll then - c = c + 1 - collected[c] = ll - end - end + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end end - return collected -end - -apply_axis['ancestor-or-self'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - c = c + 1 - collected[c] = ll - while ll do - ll = ll.__p__ - if ll then - c = c + 1 - collected[c] = ll - end - end + list.en=en + end + return c +end +apply_axis['descendant-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + if ll.special~=true then + c=c+1 + collected[c]=ll + end + c=collect(ll,collected,c) + end + return collected +end +apply_axis['ancestor']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end end - return collected -end - -apply_axis['parent'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local pl = list[l].__p__ - if pl then - c = c + 1 - collected[c] = pl - end + end + return collected +end +apply_axis['ancestor-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + c=c+1 + collected[c]=ll + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end end - return collected -end - -apply_axis['attribute'] = function(list) - return { } -end - -apply_axis['namespace'] = function(list) - return { } -end - -apply_axis['following'] = function(list) -- incomplete - return { } -end - -apply_axis['preceding'] = function(list) -- incomplete - return { } -end - -apply_axis['following-sibling'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=ll.ni+1,#d do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end + end + return collected +end +apply_axis['parent']=function(list) + local collected,c={},0 + for l=1,#list do + local pl=list[l].__p__ + if pl then + c=c+1 + collected[c]=pl + end + end + return collected +end +apply_axis['attribute']=function(list) + return {} +end +apply_axis['namespace']=function(list) + return {} +end +apply_axis['following']=function(list) + return {} +end +apply_axis['preceding']=function(list) + return {} +end +apply_axis['following-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni+1,#d do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end end - return collected -end - -apply_axis['preceding-sibling'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=1,ll.ni-1 do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end + end + return collected +end +apply_axis['preceding-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=1,ll.ni-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end end - return collected -end - -apply_axis['reverse-sibling'] = function(list) -- reverse preceding - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=ll.ni-1,1,-1 do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end + end + return collected +end +apply_axis['reverse-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni-1,1,-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end end - return collected + end + return collected end - -apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] -apply_axis['auto-descendant'] = apply_axis['descendant'] -apply_axis['auto-child'] = apply_axis['child'] -apply_axis['auto-self'] = apply_axis['self'] -apply_axis['initial-child'] = apply_axis['child'] - +apply_axis['auto-descendant-or-self']=apply_axis['descendant-or-self'] +apply_axis['auto-descendant']=apply_axis['descendant'] +apply_axis['auto-child']=apply_axis['child'] +apply_axis['auto-self']=apply_axis['self'] +apply_axis['initial-child']=apply_axis['child'] local function apply_nodes(list,directive,nodes) - -- todo: nodes[1] etc ... negated node name in set ... when needed - -- ... currently ignored - local maxn = #nodes - if maxn == 3 then --optimized loop - local nns, ntg = nodes[2], nodes[3] - if not nns and not ntg then -- wildcard + local maxn=#nodes + if maxn==3 then + local nns,ntg=nodes[2],nodes[3] + if not nns and not ntg then + if directive then + return list + else + return {} + end + else + local collected,c,m,p={},0,0,nil + if not nns then + for l=1,#list do + local ll=list[l] + local ltg=ll.tg + if ltg then if directive then - return list - else - return { } + if ntg==ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif ntg~=ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end - else - local collected, c, m, p = { }, 0, 0, nil - if not nns then -- only check tag - for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - if directive then - if ntg == ltg then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif ntg ~= ltg then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - elseif not ntg then -- only check namespace - for l=1,#list do - local ll = list[l] - local lns = ll.rn or ll.ns - if lns then - if directive then - if lns == nns then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif lns ~= nns then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - else -- check both - for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - local lns = ll.rn or ll.ns - local ok = ltg == ntg and lns == nns - if directive then - if ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif not ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end + end + end + elseif not ntg then + for l=1,#list do + local ll=list[l] + local lns=ll.rn or ll.ns + if lns then + if directive then + if lns==nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif lns~=nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end - return collected + end end - else - local collected, c, m, p = { }, 0, 0, nil + else for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - local lns = ll.rn or ll.ns - local ok = false - for n=1,maxn,3 do - local nns, ntg = nodes[n+1], nodes[n+2] - ok = (not ntg or ltg == ntg) and (not nns or lns == nns) - if ok then - break - end - end - if directive then - if ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif not ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=ltg==ntg and lns==nns + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end + end end - return collected + end + return collected end -end - -local quit_expression = false - -local function apply_expression(list,expression,order) - local collected, c = { }, 0 - quit_expression = false + else + local collected,c,m,p={},0,0,nil for l=1,#list do - local ll = list[l] - if expression(list,ll,l,order) then -- nasty, order alleen valid als n=1 - c = c + 1 - collected[c] = ll - end - if quit_expression then + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=false + for n=1,maxn,3 do + local nns,ntg=nodes[n+1],nodes[n+2] + ok=(not ntg or ltg==ntg) and (not nns or lns==nns) + if ok then break + end + end + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m end + end end return collected + end end - -local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb - -local spaces = S(" \n\r\t\f")^0 -local lp_space = S(" \n\r\t\f") -local lp_any = P(1) -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_or = P("|") / " or " -local lp_and = P("&") / " and " - -local lp_builtin = P ( - P("text") / "(ll.dt[1] or '')" + -- fragile - P("content") / "ll.dt" + - -- P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + - P("name") / "((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)" + - P("tag") / "ll.tg" + - P("position") / "l" + -- is element in finalizer - P("firstindex") / "1" + - P("lastindex") / "(#ll.__p__.dt or 1)" + - P("firstelement") / "1" + - P("lastelement") / "(ll.__p__.en or 1)" + - P("first") / "1" + - P("last") / "#list" + - P("rootposition") / "order" + - P("order") / "order" + - P("element") / "(ll.ei or 1)" + - P("index") / "(ll.ni or 1)" + - P("match") / "(ll.mi or 1)" + - -- P("namespace") / "ll.ns" + - P("ns") / "ll.ns" - ) * ((spaces * P("(") * spaces * P(")"))/"") - --- for the moment we keep namespaces with attributes - -local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * ((R("az","AZ") + S("-_:"))^1) * Cc("'])") - --- lp_fastpos_p = (P("+")^0 * R("09")^1 * P(-1)) / function(s) return "l==" .. s end --- lp_fastpos_n = (P("-") * R("09")^1 * P(-1)) / function(s) return "(" .. s .. "<0 and (#list+".. s .. "==l))" end - -lp_fastpos_p = P("+")^0 * R("09")^1 * P(-1) / "l==%0" -lp_fastpos_n = P("-") * R("09")^1 * P(-1) / "(%0<0 and (#list+%0==l))" - -local lp_fastpos = lp_fastpos_n + lp_fastpos_p - -local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") - --- local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling --- return t .. "(" --- end - --- local lp_lua_function = (R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / "%0(" -local lp_lua_function = Cs((R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(")) / "%0" - -local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling - if expressions[t] then - return "expr." .. t .. "(" - else - return "expr.error(" - end -end - -local lparent = P("(") -local rparent = P(")") -local noparent = 1 - (lparent+rparent) -local nested = P{lparent * (noparent + V(1))^0 * rparent} -local value = P(lparent * C((noparent + nested)^0) * rparent) -- P{"("*C(((1-S("()"))+V(1))^0)*")"} - -local lp_child = Cc("expr.child(ll,'") * R("az","AZ","--","__")^1 * Cc("')") -local lp_number = S("+-") * R("09")^1 -local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") -local lp_content = (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) - +local quit_expression=false +local function apply_expression(list,expression,order) + local collected,c={},0 + quit_expression=false + for l=1,#list do + local ll=list[l] + if expression(list,ll,l,order) then + c=c+1 + collected[c]=ll + end + if quit_expression then + break + end + end + return collected +end +local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb +local spaces=S(" \n\r\t\f")^0 +local lp_space=S(" \n\r\t\f") +local lp_any=P(1) +local lp_noequal=P("!=")/"~="+P("<=")+P(">=")+P("==") +local lp_doequal=P("=")/"==" +local lp_or=P("|")/" or " +local lp_and=P("&")/" and " +local lp_builtin=P ( + P("text")/"(ll.dt[1] or '')"+ + P("content")/"ll.dt"+ + P("name")/"((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)"+P("tag")/"ll.tg"+P("position")/"l"+ + P("firstindex")/"1"+P("lastindex")/"(#ll.__p__.dt or 1)"+P("firstelement")/"1"+P("lastelement")/"(ll.__p__.en or 1)"+P("first")/"1"+P("last")/"#list"+P("rootposition")/"order"+P("order")/"order"+P("element")/"(ll.ei or 1)"+P("index")/"(ll.ni or 1)"+P("match")/"(ll.mi or 1)"+ + P("ns")/"ll.ns" + )*((spaces*P("(")*spaces*P(")"))/"") +local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") +lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" +lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" +local lp_fastpos=lp_fastpos_n+lp_fastpos_p +local lp_reserved=C("and")+C("or")+C("not")+C("div")+C("mod")+C("true")+C("false") +local lp_lua_function=Cs((R("az","AZ","__")^1*(P(".")*R("az","AZ","__")^1)^1)*("("))/"%0" +local lp_function=C(R("az","AZ","__")^1)*P("(")/function(t) + if expressions[t] then + return "expr."..t.."(" + else + return "expr.error(" + end +end +local lparent=P("(") +local rparent=P(")") +local noparent=1-(lparent+rparent) +local nested=P{lparent*(noparent+V(1))^0*rparent} +local value=P(lparent*C((noparent+nested)^0)*rparent) +local lp_child=Cc("expr.child(ll,'")*R("az","AZ","--","__")^1*Cc("')") +local lp_number=S("+-")*R("09")^1 +local lp_string=Cc("'")*R("az","AZ","--","__")^1*Cc("'") +local lp_content=(P("'")*(1-P("'"))^0*P("'")+P('"')*(1-P('"'))^0*P('"')) local cleaner - -local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) - if expressions[t] then - s = s and s ~= "" and lpegmatch(cleaner,s) - if s and s ~= "" then - return "expr." .. t .. "(ll," .. s ..")" - else - return "expr." .. t .. "(ll)" - end - else - return "expr.error(" .. t .. ")" - end -end - -local content = - lp_builtin + - lp_attribute + - lp_special + - lp_noequal + lp_doequal + - lp_or + lp_and + - lp_reserved + - lp_lua_function + lp_function + - lp_content + -- too fragile - lp_child + - lp_any - -local converter = Cs ( - lp_fastpos + (P { lparent * (V(1))^0 * rparent + content } )^0 +local lp_special=(C(P("name")+P("text")+P("tag")+P("count")+P("child")))*value/function(t,s) + if expressions[t] then + s=s and s~="" and lpegmatch(cleaner,s) + if s and s~="" then + return "expr."..t.."(ll,"..s..")" + else + return "expr."..t.."(ll)" + end + else + return "expr.error("..t..")" + end +end +local content=lp_builtin+lp_attribute+lp_special+lp_noequal+lp_doequal+lp_or+lp_and+lp_reserved+lp_lua_function+lp_function+lp_content+ + lp_child+lp_any +local converter=Cs ( + lp_fastpos+(P { lparent*(V(1))^0*rparent+content } )^0 ) - -cleaner = Cs ( ( - lp_reserved + - lp_number + - lp_string + -1 )^1 ) - - - -local template_e = [[ +cleaner=Cs (( + lp_reserved+lp_number+lp_string+1 )^1 ) +local template_e=[[ local expr = xml.expressions return function(list,ll,l,order) return %s end ]] - -local template_f_y = [[ +local template_f_y=[[ local finalizer = xml.finalizers['%s']['%s'] return function(collection) return finalizer(collection,%s) end ]] - -local template_f_n = [[ +local template_f_n=[[ return xml.finalizers['%s']['%s'] ]] - --- - -local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } -local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } -local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } -local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } -local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } -local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } -local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } -local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } -local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } -local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } -local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } -local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } -local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } -local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } -local register_reverse_sibling = { kind = "axis", axis = "reverse-sibling" } -- , apply = apply_axis["reverse-sibling"] } - -local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } -local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } -local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } -local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } - -local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } - -local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } - -local skip = { } - +local register_self={ kind="axis",axis="self" } +local register_parent={ kind="axis",axis="parent" } +local register_descendant={ kind="axis",axis="descendant" } +local register_child={ kind="axis",axis="child" } +local register_descendant_or_self={ kind="axis",axis="descendant-or-self" } +local register_root={ kind="axis",axis="root" } +local register_ancestor={ kind="axis",axis="ancestor" } +local register_ancestor_or_self={ kind="axis",axis="ancestor-or-self" } +local register_attribute={ kind="axis",axis="attribute" } +local register_namespace={ kind="axis",axis="namespace" } +local register_following={ kind="axis",axis="following" } +local register_following_sibling={ kind="axis",axis="following-sibling" } +local register_preceding={ kind="axis",axis="preceding" } +local register_preceding_sibling={ kind="axis",axis="preceding-sibling" } +local register_reverse_sibling={ kind="axis",axis="reverse-sibling" } +local register_auto_descendant_or_self={ kind="axis",axis="auto-descendant-or-self" } +local register_auto_descendant={ kind="axis",axis="auto-descendant" } +local register_auto_self={ kind="axis",axis="auto-self" } +local register_auto_child={ kind="axis",axis="auto-child" } +local register_initial_child={ kind="axis",axis="initial-child" } +local register_all_nodes={ kind="nodes",nodetest=true,nodes={ true,false,false } } +local skip={} local function errorrunner_e(str,cnv) - if not skip[str] then - report_lpath("error in expression: %s => %s",str,cnv) - skip[str] = cnv or str - end - return false + if not skip[str] then + report_lpath("error in expression: %s => %s",str,cnv) + skip[str]=cnv or str + end + return false end local function errorrunner_f(str,arg) - report_lpath("error in finalizer: %s(%s)",str,arg or "") - return false + report_lpath("error in finalizer: %s(%s)",str,arg or "") + return false end - local function register_nodes(nodetest,nodes) - return { kind = "nodes", nodetest = nodetest, nodes = nodes } + return { kind="nodes",nodetest=nodetest,nodes=nodes } end - local function register_expression(expression) - local converted = lpegmatch(converter,expression) - local runner = load(format(template_e,converted)) - runner = (runner and runner()) or function() errorrunner_e(expression,converted) end - return { kind = "expression", expression = expression, converted = converted, evaluator = runner } + local converted=lpegmatch(converter,expression) + local runner=load(format(template_e,converted)) + runner=(runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind="expression",expression=expression,converted=converted,evaluator=runner } end - local function register_finalizer(protocol,name,arguments) - local runner - if arguments and arguments ~= "" then - runner = load(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) - else - runner = load(format(template_f_n,protocol or xml.defaultprotocol,name)) - end - runner = (runner and runner()) or function() errorrunner_f(name,arguments) end - return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } -end - -local expression = P { "ex", - ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", - sq = "'" * (1 - S("'"))^0 * "'", - dq = '"' * (1 - S('"'))^0 * '"', + local runner + if arguments and arguments~="" then + runner=load(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) + else + runner=load(format(template_f_n,protocol or xml.defaultprotocol,name)) + end + runner=(runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind="finalizer",name=name,arguments=arguments,finalizer=runner } +end +local expression=P { "ex", + ex="["*C((V("sq")+V("dq")+(1-S("[]"))+V("ex"))^0)*"]", + sq="'"*(1-S("'"))^0*"'", + dq='"'*(1-S('"'))^0*'"', } - -local arguments = P { "ar", - ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", - nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, - sq = P("'") * (1 - P("'"))^0 * P("'"), - dq = P('"') * (1 - P('"'))^0 * P('"'), +local arguments=P { "ar", + ar="("*Cs((V("sq")+V("dq")+V("nq")+P(1-P(")")))^0)*")", + nq=((1-S("),'\""))^1)/function(s) return format("%q",s) end, + sq=P("'")*(1-P("'"))^0*P("'"), + dq=P('"')*(1-P('"'))^0*P('"'), } - --- todo: better arg parser - local function register_error(str) - return { kind = "error", error = format("unparsed: %s",str) } -end - --- there is a difference in * and /*/ and so we need to catch a few special cases - -local special_1 = P("*") * Cc(register_auto_descendant) * Cc(register_all_nodes) -- last one not needed -local special_2 = P("/") * Cc(register_auto_self) -local special_3 = P("") * Cc(register_auto_self) - -local no_nextcolon = P(-1) + #(1-P(":")) -- newer lpeg needs the P(-1) -local no_nextlparent = P(-1) + #(1-P("(")) -- newer lpeg needs the P(-1) - -local pathparser = Ct { "patterns", -- can be made a bit faster by moving some patterns outside - - patterns = spaces * V("protocol") * spaces * ( - ( V("special") * spaces * P(-1) ) + - ( V("initial") * spaces * V("step") * spaces * (P("/") * spaces * V("step") * spaces)^0 ) - ), - - protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), - - -- the / is needed for // as descendant or self is somewhat special - -- step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, - step = ((V("shortcuts") + P("/") + V("axis")) * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, - - axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + - V("descendant_or_self") + V("following_sibling") + V("following") + - V("reverse_sibling") + V("preceding_sibling") + V("preceding") + V("ancestor_or_self") + - #(1-P(-1)) * Cc(register_auto_child), - - special = special_1 + special_2 + special_3, - - initial = (P("/") * spaces * Cc(register_initial_child))^-1, - - error = (P(1)^1) / register_error, - - shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), - - shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, - - s_descendant_or_self = (P("***/") + P("/")) * Cc(register_descendant_or_self), --- *** is a bonus - s_descendant = P("**") * Cc(register_descendant), - s_child = P("*") * no_nextcolon * Cc(register_child ), - s_parent = P("..") * Cc(register_parent ), - s_self = P("." ) * Cc(register_self ), - s_root = P("^^") * Cc(register_root ), - s_ancestor = P("^") * Cc(register_ancestor ), - - descendant = P("descendant::") * Cc(register_descendant ), - child = P("child::") * Cc(register_child ), - parent = P("parent::") * Cc(register_parent ), - self = P("self::") * Cc(register_self ), - root = P('root::') * Cc(register_root ), - ancestor = P('ancestor::') * Cc(register_ancestor ), - descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), - ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), - -- attribute = P('attribute::') * Cc(register_attribute ), - -- namespace = P('namespace::') * Cc(register_namespace ), - following = P('following::') * Cc(register_following ), - following_sibling = P('following-sibling::') * Cc(register_following_sibling ), - preceding = P('preceding::') * Cc(register_preceding ), - preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), - reverse_sibling = P('reverse-sibling::') * Cc(register_reverse_sibling ), - - nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, - - expressions = expression / register_expression, - - letters = R("az")^1, - name = (1-S("/[]()|:*!"))^1, -- make inline - negate = P("!") * Cc(false), - - nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), - nodetest = V("negate") + Cc(true), - nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), - wildnodename = (C(V("name")) + P("*") * Cc(false)) * no_nextlparent, - nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, - - finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, - + return { kind="error",error=format("unparsed: %s",str) } +end +local special_1=P("*")*Cc(register_auto_descendant)*Cc(register_all_nodes) +local special_2=P("/")*Cc(register_auto_self) +local special_3=P("")*Cc(register_auto_self) +local no_nextcolon=P(-1)+#(1-P(":")) +local no_nextlparent=P(-1)+#(1-P("(")) +local pathparser=Ct { "patterns", + patterns=spaces*V("protocol")*spaces*( + (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) + ), + protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), + step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + axis=V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), + special=special_1+special_2+special_3, + initial=(P("/")*spaces*Cc(register_initial_child))^-1, + error=(P(1)^1)/register_error, + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, + s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), + s_descendant=P("**")*Cc(register_descendant), + s_child=P("*")*no_nextcolon*Cc(register_child ), + s_parent=P("..")*Cc(register_parent ), + s_self=P("." )*Cc(register_self ), + s_root=P("^^")*Cc(register_root ), + s_ancestor=P("^")*Cc(register_ancestor ), + descendant=P("descendant::")*Cc(register_descendant ), + child=P("child::")*Cc(register_child ), + parent=P("parent::")*Cc(register_parent ), + self=P("self::")*Cc(register_self ), + root=P('root::')*Cc(register_root ), + ancestor=P('ancestor::')*Cc(register_ancestor ), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), + following=P('following::')*Cc(register_following ), + following_sibling=P('following-sibling::')*Cc(register_following_sibling ), + preceding=P('preceding::')*Cc(register_preceding ), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), + nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, + expressions=expression/register_expression, + letters=R("az")^1, + name=(1-S("/[]()|:*!"))^1, + negate=P("!")*Cc(false), + nodefunction=V("negate")+P("not")*Cc(false)+Cc(true), + nodetest=V("negate")+Cc(true), + nodename=(V("negate")+Cc(true))*spaces*((V("wildnodename")*P(":")*V("wildnodename"))+(Cc(false)*V("wildnodename"))), + wildnodename=(C(V("name"))+P("*")*Cc(false))*no_nextlparent, + nodeset=spaces*Ct(V("nodename")*(spaces*P("|")*spaces*V("nodename"))^0)*spaces, + finalizer=(Cb("protocol")*P("/")^-1*C(V("name"))*arguments*P(-1))/register_finalizer, } - -xmlpatterns.pathparser = pathparser - -local cache = { } - +xmlpatterns.pathparser=pathparser +local cache={} local function nodesettostring(set,nodetest) - local t = { } - for i=1,#set,3 do - local directive, ns, tg = set[i], set[i+1], set[i+2] - if not ns or ns == "" then ns = "*" end - if not tg or tg == "" then tg = "*" end - tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) - t[i] = (directive and tg) or format("not(%s)",tg) - end - if nodetest == false then - return format("not(%s)",concat(t,"|")) - else - return concat(t,"|") - end + local t={} + for i=1,#set,3 do + local directive,ns,tg=set[i],set[i+1],set[i+2] + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + tg=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + t[i]=(directive and tg) or format("not(%s)",tg) + end + if nodetest==false then + return format("not(%s)",concat(t,"|")) + else + return concat(t,"|") + end end - local function tagstostring(list) - if #list == 0 then - return "no elements" - else - local t = { } - for i=1, #list do - local li = list[i] - local ns, tg = li.ns, li.tg - if not ns or ns == "" then ns = "*" end - if not tg or tg == "" then tg = "*" end - t[i] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) - end - return concat(t," ") - end -end - -xml.nodesettostring = nodesettostring - -local lpath -- we have a harmless kind of circular reference - -local lshowoptions = { functions = false } - + if #list==0 then + return "no elements" + else + local t={} + for i=1,#list do + local li=list[i] + local ns,tg=li.ns,li.tg + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + t[i]=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end +end +xml.nodesettostring=nodesettostring +local lpath +local lshowoptions={ functions=false } local function lshow(parsed) - if type(parsed) == "string" then - parsed = lpath(parsed) - end - report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, - table.serialize(parsed,false,lshowoptions)) + if type(parsed)=="string" then + parsed=lpath(parsed) + end + report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, + table.serialize(parsed,false,lshowoptions)) end - -xml.lshow = lshow - +xml.lshow=lshow local function add_comment(p,str) - local pc = p.comment - if not pc then - p.comment = { str } - else - pc[#pc+1] = str - end -end - -lpath = function (pattern) -- the gain of caching is rather minimal - lpathcalls = lpathcalls + 1 - if type(pattern) == "table" then - return pattern - else - local parsed = cache[pattern] - if parsed then - lpathcached = lpathcached + 1 - else - parsed = lpegmatch(pathparser,pattern) - if parsed then - parsed.pattern = pattern - local np = #parsed - if np == 0 then - parsed = { pattern = pattern, register_self, state = "parsing error" } - report_lpath("parsing error in '%s'",pattern) - lshow(parsed) - else - -- we could have done this with a more complex parser but this - -- is cleaner - local pi = parsed[1] - if pi.axis == "auto-child" then - if false then - add_comment(parsed, "auto-child replaced by auto-descendant-or-self") - parsed[1] = register_auto_descendant_or_self - else - add_comment(parsed, "auto-child replaced by auto-descendant") - parsed[1] = register_auto_descendant - end - elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then - add_comment(parsed, "initial-child removed") -- we could also make it a auto-self - remove(parsed,1) - end - local np = #parsed -- can have changed - if np > 1 then - local pnp = parsed[np] - if pnp.kind == "nodes" and pnp.nodetest == true then - local nodes = pnp.nodes - if nodes[1] == true and nodes[2] == false and nodes[3] == false then - add_comment(parsed, "redundant final wildcard filter removed") - remove(parsed,np) - end - end - end - end + local pc=p.comment + if not pc then + p.comment={ str } + else + pc[#pc+1]=str + end +end +lpath=function (pattern) + lpathcalls=lpathcalls+1 + if type(pattern)=="table" then + return pattern + else + local parsed=cache[pattern] + if parsed then + lpathcached=lpathcached+1 + else + parsed=lpegmatch(pathparser,pattern) + if parsed then + parsed.pattern=pattern + local np=#parsed + if np==0 then + parsed={ pattern=pattern,register_self,state="parsing error" } + report_lpath("parsing error in '%s'",pattern) + lshow(parsed) + else + local pi=parsed[1] + if pi.axis=="auto-child" then + if false then + add_comment(parsed,"auto-child replaced by auto-descendant-or-self") + parsed[1]=register_auto_descendant_or_self else - parsed = { pattern = pattern } + add_comment(parsed,"auto-child replaced by auto-descendant") + parsed[1]=register_auto_descendant end - cache[pattern] = parsed - if trace_lparse and not trace_lprofile then - lshow(parsed) + elseif pi.axis=="initial-child" and np>1 and parsed[2].axis then + add_comment(parsed,"initial-child removed") + remove(parsed,1) + end + local np=#parsed + if np>1 then + local pnp=parsed[np] + if pnp.kind=="nodes" and pnp.nodetest==true then + local nodes=pnp.nodes + if nodes[1]==true and nodes[2]==false and nodes[3]==false then + add_comment(parsed,"redundant final wildcard filter removed") + remove(parsed,np) + end end + end end - return parsed + else + parsed={ pattern=pattern } + end + cache[pattern]=parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) + end end + return parsed + end end - -xml.lpath = lpath - --- we can move all calls inline and then merge the trace back --- technically we can combine axis and the next nodes which is --- what we did before but this a bit cleaner (but slower too) --- but interesting is that it's not that much faster when we --- go inline --- --- beware: we need to return a collection even when we filter --- else the (simple) cache gets messed up - --- caching found lookups saves not that much (max .1 sec on a 8 sec run) --- and it also messes up finalizers - --- watch out: when there is a finalizer, it's always called as there --- can be cases that a finalizer returns (or does) something in case --- there is no match; an example of this is count() - -local profiled = { } xml.profiled = profiled - +xml.lpath=lpath +local profiled={} xml.profiled=profiled local function profiled_apply(list,parsed,nofparsed,order) - local p = profiled[parsed.pattern] - if p then - p.tested = p.tested + 1 - else - p = { tested = 1, matched = 0, finalized = 0 } - profiled[parsed.pattern] = p - end - local collected = list - for i=1,nofparsed do - local pi = parsed[i] - local kind = pi.kind - if kind == "axis" then - collected = apply_axis[pi.axis](collected) - elseif kind == "nodes" then - collected = apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind == "expression" then - collected = apply_expression(collected,pi.evaluator,order) - elseif kind == "finalizer" then - collected = pi.finalizer(collected) -- no check on # here - p.matched = p.matched + 1 - p.finalized = p.finalized + 1 - return collected - end - if not collected or #collected == 0 then - local pn = i < nofparsed and parsed[nofparsed] - if pn and pn.kind == "finalizer" then - collected = pn.finalizer(collected) - p.finalized = p.finalized + 1 - return collected - end - return nil - end - end - if collected then - p.matched = p.matched + 1 + local p=profiled[parsed.pattern] + if p then + p.tested=p.tested+1 + else + p={ tested=1,matched=0,finalized=0 } + profiled[parsed.pattern]=p + end + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + p.matched=p.matched+1 + p.finalized=p.finalized+1 + return collected + end + if not collected or #collected==0 then + local pn=i %s",(collected and #collected) or 0,pi.expression,pi.converted) - elseif kind == "finalizer" then - collected = pi.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") - return collected - end - if not collected or #collected == 0 then - local pn = i < nofparsed and parsed[nofparsed] - if pn and pn.kind == "finalizer" then - collected = pn.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") - return collected - end - return nil - end + if trace_lparse then + lshow(parsed) + end + report_lpath("collecting: %s",parsed.pattern) + report_lpath("root tags : %s",tagstostring(list)) + report_lpath("order : %s",order or "unset") + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected==0 then + local pn=i oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps oeps oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -xmlpatterns.escaped = escaped -xmlpatterns.unescaped = unescaped -xmlpatterns.cleansed = cleansed - -function xml.escaped (str) return lpegmatch(escaped,str) end + end +end +function xml.stripleadingspaces(dk,d,k) + if d and k then + local dkm=d[k-1] + if dkm and type(dkm)=="string" then + local s=match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") + end + end +end +local normal=(1-S("<&>"))^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local escaped=Cs(normal*(special*normal)^0) +local normal=(1-S"&")^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local unescaped=Cs(normal*(special*normal)^0) +local cleansed=Cs(((P("<")*(1-P(">"))^0*P(">"))/""+1)^0) +xmlpatterns.escaped=escaped +xmlpatterns.unescaped=unescaped +xmlpatterns.cleansed=cleansed +function xml.escaped (str) return lpegmatch(escaped,str) end function xml.unescaped(str) return lpegmatch(unescaped,str) end -function xml.cleansed (str) return lpegmatch(cleansed,str) end - --- this might move - +function xml.cleansed (str) return lpegmatch(cleansed,str) end function xml.fillin(root,pattern,str,check) - local e = xml.first(root,pattern) - if e then - local n = #e.dt - if not check or n == 0 or (n == 1 and e.dt[1] == "") then - e.dt = { str } - end + local e=xml.first(root,pattern) + if e then + local n=#e.dt + if not check or n==0 or (n==1 and e.dt[1]=="") then + e.dt={ str } end + end end @@ -12210,800 +9242,690 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-aux'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- not all functions here make sense anymore vbut we keep them for --- compatibility reasons - -local trace_manipulations = false trackers.register("lxml.manipulations", function(v) trace_manipulations = v end) - -local report_xml = logs.reporter("xml") - -local xml = xml - -local xmlconvert, xmlcopy, xmlname = xml.convert, xml.copy, xml.name -local xmlinheritedconvert = xml.inheritedconvert -local xmlapplylpath = xml.applylpath -local xmlfilter = xml.filter - -local type, setmetatable, getmetatable = type, setmetatable, getmetatable -local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat -local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip -local utfbyte = utf.byte +-- original size: 23813, stripped down to: 16826 +if not modules then modules={} end modules ['lxml-aux']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_manipulations=false trackers.register("lxml.manipulations",function(v) trace_manipulations=v end) +local report_xml=logs.reporter("xml") +local xml=xml +local xmlconvert,xmlcopy,xmlname=xml.convert,xml.copy,xml.name +local xmlinheritedconvert=xml.inheritedconvert +local xmlapplylpath=xml.applylpath +local xmlfilter=xml.filter +local type,setmetatable,getmetatable=type,setmetatable,getmetatable +local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat +local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip +local utfbyte=utf.byte local function report(what,pattern,c,e) - report_xml("%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) + report_xml("%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end - local function withelements(e,handle,depth) - if e and handle then - local edt = e.dt - if edt then - depth = depth or 0 - for i=1,#edt do - local e = edt[i] - if type(e) == "table" then - handle(e,depth) - withelements(e,handle,depth+1) - end - end + if e and handle then + local edt=e.dt + if edt then + depth=depth or 0 + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + handle(e,depth) + withelements(e,handle,depth+1) end + end end + end end - -xml.withelements = withelements - -function xml.withelement(e,n,handle) -- slow - if e and n ~= 0 and handle then - local edt = e.dt - if edt then - if n > 0 then - for i=1,#edt do - local ei = edt[i] - if type(ei) == "table" then - if n == 1 then - handle(ei) - return - else - n = n - 1 - end - end - end - elseif n < 0 then - for i=#edt,1,-1 do - local ei = edt[i] - if type(ei) == "table" then - if n == -1 then - handle(ei) - return - else - n = n + 1 - end - end - end +xml.withelements=withelements +function xml.withelement(e,n,handle) + if e and n~=0 and handle then + local edt=e.dt + if edt then + if n>0 then + for i=1,#edt do + local ei=edt[i] + if type(ei)=="table" then + if n==1 then + handle(ei) + return + else + n=n-1 + end + end + end + elseif n<0 then + for i=#edt,1,-1 do + local ei=edt[i] + if type(ei)=="table" then + if n==-1 then + handle(ei) + return + else + n=n+1 end + end end + end end + end end - function xml.each(root,pattern,handle,reverse) - local collected = xmlapplylpath(root,pattern) - if collected then - if reverse then - for c=#collected,1,-1 do - handle(collected[c]) - end - else - for c=1,#collected do - handle(collected[c]) - end - end - return collected + local collected=xmlapplylpath(root,pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end end + return collected + end end - function xml.processattributes(root,pattern,handle) - local collected = xmlapplylpath(root,pattern) - if collected and handle then - for c=1,#collected do - handle(collected[c].at) - end + local collected=xmlapplylpath(root,pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) end - return collected + end + return collected end - - - --- are these still needed -> lxml-cmp.lua - -function xml.collect(root, pattern) - return xmlapplylpath(root,pattern) +function xml.collect(root,pattern) + return xmlapplylpath(root,pattern) end - -function xml.collecttexts(root, pattern, flatten) -- todo: variant with handle - local collected = xmlapplylpath(root,pattern) - if collected and flatten then - local xmltostring = xml.tostring - for c=1,#collected do - collected[c] = xmltostring(collected[c].dt) - end +function xml.collecttexts(root,pattern,flatten) + local collected=xmlapplylpath(root,pattern) + if collected and flatten then + local xmltostring=xml.tostring + for c=1,#collected do + collected[c]=xmltostring(collected[c].dt) end - return collected or { } + end + return collected or {} end - -function xml.collect_tags(root, pattern, nonamespace) - local collected = xmlapplylpath(root,pattern) - if collected then - local t, n = { }, 0 - for c=1,#collected do - local e = collected[c] - local ns, tg = e.ns, e.tg - n = n + 1 - if nonamespace then - t[n] = tg - elseif ns == "" then - t[n] = tg - else - t[n] = ns .. ":" .. tg - end - end - return t +function xml.collect_tags(root,pattern,nonamespace) + local collected=xmlapplylpath(root,pattern) + if collected then + local t,n={},0 + for c=1,#collected do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace then + t[n]=tg + elseif ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg + end end + return t + end end - - - -local no_root = { no_root = true } - +local no_root={ no_root=true } local function redo_ni(d) - for k=1,#d do - local dk = d[k] - if type(dk) == "table" then - dk.ni = k - end + for k=1,#d do + local dk=d[k] + if type(dk)=="table" then + dk.ni=k end + end end - local function xmltoelement(whatever,root) - if not whatever then - return nil - end - local element - if type(whatever) == "string" then - element = xmlinheritedconvert(whatever,root) -- beware, not really a root - else - element = whatever -- we assume a table - end - if element.error then - return whatever -- string - end - if element then - end - return element -end - -xml.toelement = xmltoelement - + if not whatever then + return nil + end + local element + if type(whatever)=="string" then + element=xmlinheritedconvert(whatever,root) + else + element=whatever + end + if element.error then + return whatever + end + if element then + end + return element +end +xml.toelement=xmltoelement local function copiedelement(element,newparent) - if type(element) == "string" then - return element - else - element = xmlcopy(element).dt - if newparent and type(element) == "table" then - element.__p__ = newparent - end - return element + if type(element)=="string" then + return element + else + element=xmlcopy(element).dt + if newparent and type(element)=="table" then + element.__p__=newparent end + return element + end end - function xml.delete(root,pattern) - if not pattern or pattern == "" then - local p = root.__p__ + if not pattern or pattern=="" then + local p=root.__p__ + if p then + if trace_manipulations then + report('deleting',"--",c,root) + end + local d=p.dt + remove(d,root.ni) + redo_ni(d) + end + else + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ if p then - if trace_manipulations then - report('deleting',"--",c,root) - end - local d = p.dt - remove(d,root.ni) - redo_ni(d) -- can be made faster and inlined - end - else - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local p = e.__p__ - if p then - if trace_manipulations then - report('deleting',pattern,c,e) - end - local d = p.dt - remove(d,e.ni) - redo_ni(d) -- can be made faster and inlined - end - end + if trace_manipulations then + report('deleting',pattern,c,e) + end + local d=p.dt + remove(d,e.ni) + redo_ni(d) end + end end + end end - function xml.replace(root,pattern,whatever) - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local p = e.__p__ - if p then - if trace_manipulations then - report('replacing',pattern,c,e) - end - local d = p.dt - d[e.ni] = copiedelement(element,p) - redo_ni(d) -- probably not needed - end - end + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ + if p then + if trace_manipulations then + report('replacing',pattern,c,e) + end + local d=p.dt + d[e.ni]=copiedelement(element,p) + redo_ni(d) + end end + end end - local function wrap(e,wrapper) - local t = { - rn = e.rn, - tg = e.tg, - ns = e.ns, - at = e.at, - dt = e.dt, - __p__ = e, - } - setmetatable(t,getmetatable(e)) - e.rn = wrapper.rn or e.rn or "" - e.tg = wrapper.tg or e.tg or "" - e.ns = wrapper.ns or e.ns or "" - e.at = fastcopy(wrapper.at) - e.dt = { t } + local t={ + rn=e.rn, + tg=e.tg, + ns=e.ns, + at=e.at, + dt=e.dt, + __p__=e, + } + setmetatable(t,getmetatable(e)) + e.rn=wrapper.rn or e.rn or "" + e.tg=wrapper.tg or e.tg or "" + e.ns=wrapper.ns or e.ns or "" + e.at=fastcopy(wrapper.at) + e.dt={ t } end - function xml.wrap(root,pattern,whatever) - if whatever then - local wrapper = xmltoelement(whatever,root) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - if trace_manipulations then - report('wrapping',pattern,c,e) - end - wrap(e,wrapper) - end + if whatever then + local wrapper=xmltoelement(whatever,root) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if trace_manipulations then + report('wrapping',pattern,c,e) end - else - wrap(root,xmltoelement(pattern)) + wrap(e,wrapper) + end end + else + wrap(root,xmltoelement(pattern)) + end end - local function inject_element(root,pattern,whatever,prepend) - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - local function inject_e(e) - local r = e.__p__ - local d, k, rri = r.dt, e.ni, r.ri - local edt = (rri and d[rri].dt) or (d and d[k] and d[k].dt) - if edt then - local be, af - local cp = copiedelement(element,e) - if prepend then - be, af = cp, edt - else - be, af = edt, cp - end - local bn = #be - for i=1,#af do - bn = bn + 1 - be[bn] = af[i] - end - if rri then - r.dt[rri].dt = be - else - d[k].dt = be - end - redo_ni(d) - end - end - if not collected then - -- nothing - elseif collected.tg then - -- first or so - inject_e(collected) - else - for c=1,#collected do - inject_e(collected[c]) - end + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function inject_e(e) + local r=e.__p__ + local d,k,rri=r.dt,e.ni,r.ri + local edt=(rri and d[rri].dt) or (d and d[k] and d[k].dt) + if edt then + local be,af + local cp=copiedelement(element,e) + if prepend then + be,af=cp,edt + else + be,af=edt,cp + end + local bn=#be + for i=1,#af do + bn=bn+1 + be[bn]=af[i] + end + if rri then + r.dt[rri].dt=be + else + d[k].dt=be + end + redo_ni(d) end -end - -local function insert_element(root,pattern,whatever,before) -- todo: element als functie - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - local function insert_e(e) - local r = e.__p__ - local d, k = r.dt, e.ni - if not before then - k = k + 1 - end - insert(d,k,copiedelement(element,r)) - redo_ni(d) - end - if not collected then - -- nothing - elseif collected.tg then - -- first or so - insert_e(collected) - else - for c=1,#collected do - insert_e(collected[c]) - end + end + if not collected then + elseif collected.tg then + inject_e(collected) + else + for c=1,#collected do + inject_e(collected[c]) + end + end +end +local function insert_element(root,pattern,whatever,before) + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function insert_e(e) + local r=e.__p__ + local d,k=r.dt,e.ni + if not before then + k=k+1 + end + insert(d,k,copiedelement(element,r)) + redo_ni(d) + end + if not collected then + elseif collected.tg then + insert_e(collected) + else + for c=1,#collected do + insert_e(collected[c]) end + end end - -xml.insert_element = insert_element -xml.insertafter = insert_element -xml.insertbefore = function(r,p,e) insert_element(r,p,e,true) end -xml.injectafter = inject_element -xml.injectbefore = function(r,p,e) inject_element(r,p,e,true) end - +xml.insert_element=insert_element +xml.insertafter=insert_element +xml.insertbefore=function(r,p,e) insert_element(r,p,e,true) end +xml.injectafter=inject_element +xml.injectbefore=function(r,p,e) inject_element(r,p,e,true) end local function include(xmldata,pattern,attribute,recursive,loaddata) - -- parse="text" (default: xml), encoding="" (todo) - -- attribute = attribute or 'href' - pattern = pattern or 'include' - loaddata = loaddata or io.loaddata - local collected = xmlapplylpath(xmldata,pattern) - if collected then - for c=1,#collected do - local ek = collected[c] - local name = nil - local ekdt = ek.dt - local ekat = ek.at - local epdt = ek.__p__.dt - if not attribute or attribute == "" then - name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- check, probably always tab or str - end - if not name then - for a in gmatch(attribute or "href","([^|]+)") do - name = ekat[a] - if name then break end - end - end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - epdt[ek.ni] = "" -- xml.empty(d,k) - elseif ekat["parse"] == "text" then - -- for the moment hard coded - epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) - else - local xi = xmlinheritedconvert(data,xmldata) - if not xi then - epdt[ek.ni] = "" -- xml.empty(d,k) - else - if recursive then - include(xi,pattern,attribute,recursive,loaddata) - end - epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) - end - end + pattern=pattern or 'include' + loaddata=loaddata or io.loaddata + local collected=xmlapplylpath(xmldata,pattern) + if collected then + for c=1,#collected do + local ek=collected[c] + local name=nil + local ekdt=ek.dt + local ekat=ek.at + local epdt=ek.__p__.dt + if not attribute or attribute=="" then + name=(type(ekdt)=="table" and ekdt[1]) or ekdt + end + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name=ekat[a] + if name then break end end + end + local data=(name and name~="" and loaddata(name)) or "" + if data=="" then + epdt[ek.ni]="" + elseif ekat["parse"]=="text" then + epdt[ek.ni]=xml.escaped(data) + else + local xi=xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni]="" + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni]=xml.body(xi) + end + end end + end end - -xml.include = include - +xml.include=include local function stripelement(e,nolines,anywhere) - local edt = e.dt - if edt then - if anywhere then - local t, n = { }, 0 - for e=1,#edt do - local str = edt[e] - if type(str) ~= "string" then - n = n + 1 - t[n] = str - elseif str ~= "" then - -- todo: lpeg for each case - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s*(.-)%s*$","%1") - if str ~= "" then - n = n + 1 - t[n] = str - end - end - end - e.dt = t + local edt=e.dt + if edt then + if anywhere then + local t,n={},0 + for e=1,#edt do + local str=edt[e] + if type(str)~="string" then + n=n+1 + t[n]=str + elseif str~="" then + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"^%s*(.-)%s*$","%1") + if str~="" then + n=n+1 + t[n]=str + end + end + end + e.dt=t + else + if #edt>0 then + local str=edt[1] + if type(str)~="string" then + elseif str=="" then + remove(edt,1) else - -- we can assume a regular sparse xml table with no successive strings - -- otherwise we should use a while loop - if #edt > 0 then - -- strip front - local str = edt[1] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt,1) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s+","") - if str == "" then - remove(edt,1) - else - edt[1] = str - end - end - end - local nedt = #edt - if nedt > 0 then - -- strip end - local str = edt[nedt] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"%s+$","") - if str == "" then - remove(edt) - else - edt[nedt] = str - end - end - end + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"^%s+","") + if str=="" then + remove(edt,1) + else + edt[1]=str + end + end + end + local nedt=#edt + if nedt>0 then + local str=edt[nedt] + if type(str)~="string" then + elseif str=="" then + remove(edt) + else + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"%s+$","") + if str=="" then + remove(edt) + else + edt[nedt]=str + end end + end end - return e -- convenient + end + return e end - -xml.stripelement = stripelement - -function xml.strip(root,pattern,nolines,anywhere) -- strips all leading and trailing spacing - local collected = xmlapplylpath(root,pattern) -- beware, indices no longer are valid now - if collected then - for i=1,#collected do - stripelement(collected[i],nolines,anywhere) - end +xml.stripelement=stripelement +function xml.strip(root,pattern,nolines,anywhere) + local collected=xmlapplylpath(root,pattern) + if collected then + for i=1,#collected do + stripelement(collected[i],nolines,anywhere) end + end end - -local function renamespace(root, oldspace, newspace) -- fast variant - local ndt = #root.dt - for i=1,ndt or 0 do - local e = root[i] - if type(e) == "table" then - if e.ns == oldspace then - e.ns = newspace - if e.rn then - e.rn = newspace - end - end - local edt = e.dt - if edt then - renamespace(edt, oldspace, newspace) - end +local function renamespace(root,oldspace,newspace) + local ndt=#root.dt + for i=1,ndt or 0 do + local e=root[i] + if type(e)=="table" then + if e.ns==oldspace then + e.ns=newspace + if e.rn then + e.rn=newspace end + end + local edt=e.dt + if edt then + renamespace(edt,oldspace,newspace) + end end + end end - -xml.renamespace = renamespace - -function xml.remaptag(root, pattern, newtg) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - collected[c].tg = newtg - end +xml.renamespace=renamespace +function xml.remaptag(root,pattern,newtg) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].tg=newtg end + end end - -function xml.remapnamespace(root, pattern, newns) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - collected[c].ns = newns - end +function xml.remapnamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].ns=newns end + end end - -function xml.checknamespace(root, pattern, newns) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - if (not e.rn or e.rn == "") and e.ns == "" then - e.rn = newns - end - end +function xml.checknamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if (not e.rn or e.rn=="") and e.ns=="" then + e.rn=newns + end end + end end - -function xml.remapname(root, pattern, newtg, newns, newrn) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - e.tg, e.ns, e.rn = newtg, newns, newrn - end +function xml.remapname(root,pattern,newtg,newns,newrn) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + e.tg,e.ns,e.rn=newtg,newns,newrn end + end end - - - function xml.cdatatotext(e) - local dt = e.dt - if #dt == 1 then - local first = dt[1] - if first.tg == "@cd@" then - e.dt = first.dt - end - else - -- maybe option - end -end - --- local x = xml.convert("123") --- xml.texttocdata(xml.first(x,"a")) --- print(x) -- 23]]> - -function xml.texttocdata(e) -- could be a finalizer - local dt = e.dt - local s = xml.tostring(dt) -- no shortcut? - e.tg = "@cd@" - e.special = true - e.ns = "" - e.rn = "" - e.dt = { s } - e.at = nil -end - --- local x = xml.convert("123") --- xml.tocdata(xml.first(x,"a")) --- print(x) -- 123]]> - -function xml.elementtocdata(e) -- could be a finalizer - local dt = e.dt - local s = xml.tostring(e) -- no shortcut? - e.tg = "@cd@" - e.special = true - e.ns = "" - e.rn = "" - e.dt = { s } - e.at = nil -end - -xml.builtinentities = table.tohash { "amp", "quot", "apos", "lt", "gt" } -- used often so share - -local entities = characters and characters.entities or nil -local builtinentities = xml.builtinentities - -function xml.addentitiesdoctype(root,option) -- we could also have a 'resolve' i.e. inline hex - if not entities then - require("char-ent") - entities = characters.entities - end - if entities and root and root.tg == "@rt@" and root.statistics then - local list = { } - local hexify = option == "hexadecimal" - for k, v in table.sortedhash(root.statistics.entities.names) do - if not builtinentities[k] then - local e = entities[k] - if not e then - e = format("[%s]",k) - elseif hexify then - e = format("&#%05X;",utfbyte(k)) - end - list[#list+1] = format(" ",k,e) - end - end - local dt = root.dt - local n = dt[1].tg == "@pi@" and 2 or 1 - if #list > 0 then - insert(dt, n, { "\n" }) - insert(dt, n, { - tg = "@dt@", -- beware, doctype is unparsed - dt = { format("Something [\n%s\n] ",concat(list)) }, - ns = "", - special = true, - }) - insert(dt, n, { "\n\n" }) - else - -- insert(dt, n, { table.serialize(root.statistics) }) - end + local dt=e.dt + if #dt==1 then + local first=dt[1] + if first.tg=="@cd@" then + e.dt=first.dt + end + else + end +end +function xml.texttocdata(e) + local dt=e.dt + local s=xml.tostring(dt) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +function xml.elementtocdata(e) + local dt=e.dt + local s=xml.tostring(e) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +xml.builtinentities=table.tohash { "amp","quot","apos","lt","gt" } +local entities=characters and characters.entities or nil +local builtinentities=xml.builtinentities +function xml.addentitiesdoctype(root,option) + if not entities then + require("char-ent") + entities=characters.entities + end + if entities and root and root.tg=="@rt@" and root.statistics then + local list={} + local hexify=option=="hexadecimal" + for k,v in table.sortedhash(root.statistics.entities.names) do + if not builtinentities[k] then + local e=entities[k] + if not e then + e=format("[%s]",k) + elseif hexify then + e=format("&#%05X;",utfbyte(k)) + end + list[#list+1]=format(" ",k,e) + end end -end - --- local str = [==[ --- --- --- test   test { test --- --- --- ]==] --- --- local x = xml.convert(str) --- xml.addentitiesdoctype(x,"hexadecimal") --- print(x) - - - -xml.all = xml.each -xml.insert = xml.insertafter -xml.inject = xml.injectafter -xml.after = xml.insertafter -xml.before = xml.insertbefore -xml.process = xml.each - --- obsolete - -xml.obsolete = xml.obsolete or { } -local obsolete = xml.obsolete - -xml.strip_whitespace = xml.strip obsolete.strip_whitespace = xml.strip -xml.collect_elements = xml.collect obsolete.collect_elements = xml.collect -xml.delete_element = xml.delete obsolete.delete_element = xml.delete -xml.replace_element = xml.replace obsolete.replace_element = xml.replacet -xml.each_element = xml.each obsolete.each_element = xml.each -xml.process_elements = xml.process obsolete.process_elements = xml.process -xml.insert_element_after = xml.insertafter obsolete.insert_element_after = xml.insertafter -xml.insert_element_before = xml.insertbefore obsolete.insert_element_before = xml.insertbefore -xml.inject_element_after = xml.injectafter obsolete.inject_element_after = xml.injectafter -xml.inject_element_before = xml.injectbefore obsolete.inject_element_before = xml.injectbefore -xml.process_attributes = xml.processattributes obsolete.process_attributes = xml.processattributes -xml.collect_texts = xml.collecttexts obsolete.collect_texts = xml.collecttexts -xml.inject_element = xml.inject obsolete.inject_element = xml.inject -xml.remap_tag = xml.remaptag obsolete.remap_tag = xml.remaptag -xml.remap_name = xml.remapname obsolete.remap_name = xml.remapname -xml.remap_namespace = xml.remapnamespace obsolete.remap_namespace = xml.remapnamespace - --- new (probably ok) - + local dt=root.dt + local n=dt[1].tg=="@pi@" and 2 or 1 + if #list>0 then + insert(dt,n,{ "\n" }) + insert(dt,n,{ + tg="@dt@", + dt={ format("Something [\n%s\n] ",concat(list)) }, + ns="", + special=true, + }) + insert(dt,n,{ "\n\n" }) + else + end + end +end +xml.all=xml.each +xml.insert=xml.insertafter +xml.inject=xml.injectafter +xml.after=xml.insertafter +xml.before=xml.insertbefore +xml.process=xml.each +xml.obsolete=xml.obsolete or {} +local obsolete=xml.obsolete +xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip +xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect +xml.delete_element=xml.delete obsolete.delete_element=xml.delete +xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.each_element=xml.each obsolete.each_element=xml.each +xml.process_elements=xml.process obsolete.process_elements=xml.process +xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter +xml.insert_element_before=xml.insertbefore obsolete.insert_element_before=xml.insertbefore +xml.inject_element_after=xml.injectafter obsolete.inject_element_after=xml.injectafter +xml.inject_element_before=xml.injectbefore obsolete.inject_element_before=xml.injectbefore +xml.process_attributes=xml.processattributes obsolete.process_attributes=xml.processattributes +xml.collect_texts=xml.collecttexts obsolete.collect_texts=xml.collecttexts +xml.inject_element=xml.inject obsolete.inject_element=xml.inject +xml.remap_tag=xml.remaptag obsolete.remap_tag=xml.remaptag +xml.remap_name=xml.remapname obsolete.remap_name=xml.remapname +xml.remap_namespace=xml.remapnamespace obsolete.remap_namespace=xml.remapnamespace function xml.cdata(e) - if e then - local dt = e.dt - if dt and #dt == 1 then - local first = dt[1] - return first.tg == "@cd@" and first.dt[1] or "" - end + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" end - return "" + end + return "" end - function xml.finalizers.xml.cdata(collected) - if collected then - local e = collected[1] - if e then - local dt = e.dt - if dt and #dt == 1 then - local first = dt[1] - return first.tg == "@cd@" and first.dt[1] or "" - end - end + if collected then + local e=collected[1] + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" + end end - return "" -end - -function xml.insertcomment(e,str,n) -- also insertcdata - table.insert(e.dt,n or 1,{ - tg = "@cm@", - ns = "", - special = true, - at = { }, - dt = { str }, - }) -end - -function xml.setcdata(e,str) -- also setcomment - e.dt = { { - tg = "@cd@", - ns = "", - special = true, - at = { }, - dt = { str }, - } } + end + return "" +end +function xml.insertcomment(e,str,n) + table.insert(e.dt,n or 1,{ + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.setcdata(e,str) + e.dt={ { + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + } } end - --- maybe helpers like this will move to an autoloader - function xml.separate(x,pattern) - local collected = xmlapplylpath(x,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local d = e.dt - if d == x then - report_xml("warning: xml.separate changes root") - x = d - end - local t, n = { "\n" }, 1 - local i, nd = 1, #d - while i <= nd do - while i <= nd do - local di = d[i] - if type(di) == "string" then - if di == "\n" or find(di,"^%s+$") then -- first test is speedup - i = i + 1 - else - d[i] = strip(di) - break - end - else - break - end - end - if i > nd then - break - end - t[n+1] = "\n" - t[n+2] = d[i] - t[n+3] = "\n" - n = n + 3 - i = i + 1 + local collected=xmlapplylpath(x,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local d=e.dt + if d==x then + report_xml("warning: xml.separate changes root") + x=d + end + local t,n={ "\n" },1 + local i,nd=1,#d + while i<=nd do + while i<=nd do + local di=d[i] + if type(di)=="string" then + if di=="\n" or find(di,"^%s+$") then + i=i+1 + else + d[i]=strip(di) + break end - t[n+1] = "\n" - setmetatable(t,getmetatable(d)) - e.dt = t + else + break + end + end + if i>nd then + break end + t[n+1]="\n" + t[n+2]=d[i] + t[n+3]="\n" + n=n+3 + i=i+1 + end + t[n+1]="\n" + setmetatable(t,getmetatable(d)) + e.dt=t end - return x + end + return x end - --- - -local helpers = xml.helpers or { } -xml.helpers = helpers - +local helpers=xml.helpers or {} +xml.helpers=helpers local function normal(e,action) - local edt = e.dt - if edt then - for i=1,#edt do - local str = edt[i] - if type(str) == "string" and str ~= "" then - edt[i] = action(str) - end - end + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)=="string" and str~="" then + edt[i]=action(str) + end end + end end - local function recurse(e,action) - local edt = e.dt - if edt then - for i=1,#edt do - local str = edt[i] - if type(str) ~= "string" then - recurse(str,action,recursive) - elseif str ~= "" then - edt[i] = action(str) - end - end + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)~="string" then + recurse(str,action,recursive) + elseif str~="" then + edt[i]=action(str) + end end + end end - function helpers.recursetext(collected,action,recursive) - if recursive then - for i=1,#collected do - recurse(collected[i],action) - end - else - for i=1,#collected do - normal(collected[i],action) - end + if recursive then + for i=1,#collected do + recurse(collected[i],action) + end + else + for i=1,#collected do + normal(collected[i],action) end + end end @@ -13011,450 +9933,375 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-xml'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local concat = table.concat -local find, lower, upper = string.find, string.lower, string.upper - -local xml = xml +-- original size: 10274, stripped down to: 7538 -local finalizers = xml.finalizers.xml -local xmlfilter = xml.filter -- we could inline this one for speed -local xmltostring = xml.tostring -local xmlserialize = xml.serialize -local xmlcollected = xml.collected -local xmlnewhandlers = xml.newhandlers - -local function first(collected) -- wrong ? - return collected and collected[1] +if not modules then modules={} end modules ['lxml-xml']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat=table.concat +local find,lower,upper=string.find,string.lower,string.upper +local xml=xml +local finalizers=xml.finalizers.xml +local xmlfilter=xml.filter +local xmltostring=xml.tostring +local xmlserialize=xml.serialize +local xmlcollected=xml.collected +local xmlnewhandlers=xml.newhandlers +local function first(collected) + return collected and collected[1] end - local function last(collected) - return collected and collected[#collected] + return collected and collected[#collected] end - local function all(collected) - return collected + return collected end - --- local function reverse(collected) --- if collected then --- local nc = #collected --- if nc > 0 then --- local reversed, r = { }, 0 --- for c=nc,1,-1 do --- r = r + 1 --- reversed[r] = collected[c] --- end --- return reversed --- else --- return collected --- end --- end --- end - -local reverse = table.reversed - +local reverse=table.reversed local function attribute(collected,name) - if collected and #collected > 0 then - local at = collected[1].at - return at and at[name] - end + if collected and #collected>0 then + local at=collected[1].at + return at and at[name] + end end - local function att(id,name) - local at = id.at - return at and at[name] + local at=id.at + return at and at[name] end - local function count(collected) - return collected and #collected or 0 + return collected and #collected or 0 end - local function position(collected,n) - if not collected then - return 0 - end - local nc = #collected - if nc == 0 then - return 0 - end - n = tonumber(n) or 0 - if n < 0 then - return collected[nc + n + 1] - elseif n > 0 then - return collected[n] - else - return collected[1].mi or 0 - end + if not collected then + return 0 + end + local nc=#collected + if nc==0 then + return 0 + end + n=tonumber(n) or 0 + if n<0 then + return collected[nc+n+1] + elseif n>0 then + return collected[n] + else + return collected[1].mi or 0 + end end - local function match(collected) - return collected and #collected > 0 and collected[1].mi or 0 -- match + return collected and #collected>0 and collected[1].mi or 0 end - local function index(collected) - return collected and #collected > 0 and collected[1].ni or 0 -- 0 is new + return collected and #collected>0 and collected[1].ni or 0 end - local function attributes(collected,arguments) - if collected and #collected > 0 then - local at = collected[1].at - if arguments then - return at[arguments] - elseif next(at) then - return at -- all of them + if collected and #collected>0 then + local at=collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at + end + end +end +local function chainattribute(collected,arguments) + if collected and #collected>0 then + local e=collected[1] + while e do + local at=e.at + if at then + local a=at[arguments] + if a then + return a end + else + break + end + e=e.__p__ end + end + return "" end - -local function chainattribute(collected,arguments) -- todo: optional levels - if collected and #collected > 0 then - local e = collected[1] - while e do - local at = e.at - if at then - local a = at[arguments] - if a then - return a - end - else - break -- error - end - e = e.__p__ - end - end +local function raw(collected) + if collected and #collected>0 then + local e=collected[1] or collected + return e and xmltostring(e) or "" + else return "" + end end - -local function raw(collected) -- hybrid (not much different from text so it might go) - if collected and #collected > 0 then - local e = collected[1] or collected - return e and xmltostring(e) or "" -- only first as we cannot concat function - else - return "" - end -end - --- - -local xmltexthandler = xmlnewhandlers { - name = "string", - initialize = function() - result = { } - return result - end, - finalize = function() - return concat(result) - end, - handle = function(...) - result[#result+1] = concat { ... } - end, - escape = false, +local xmltexthandler=xmlnewhandlers { + name="string", + initialize=function() + result={} + return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, + escape=false, } - local function xmltotext(root) - local dt = root.dt - if not dt then - return "" - end - local nt = #dt -- string or table - if nt == 0 then - return "" - elseif nt == 1 and type(dt[1]) == "string" then - return dt[1] -- no escaping of " ' < > & - else - return xmlserialize(root,xmltexthandler) or "" - end -end - --- - -local function text(collected) -- hybrid - if collected then -- no # test here ! - local e = collected[1] or collected -- why fallback to element, how about cdata - return e and xmltotext(e) or "" - else - return "" - end + local dt=root.dt + if not dt then + return "" + end + local nt=#dt + if nt==0 then + return "" + elseif nt==1 and type(dt[1])=="string" then + return dt[1] + else + return xmlserialize(root,xmltexthandler) or "" + end +end +local function text(collected) + if collected then + local e=collected[1] or collected + return e and xmltotext(e) or "" + else + return "" + end end - local function texts(collected) - if not collected then - return { } -- why no nil - end - local nc = #collected - if nc == 0 then - return { } -- why no nil - end - local t, n = { }, 0 - for c=1,nc do - local e = collected[c] - if e and e.dt then - n = n + 1 - t[n] = e.dt - end - end - return t + if not collected then + return {} + end + local nc=#collected + if nc==0 then + return {} + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + if e and e.dt then + n=n+1 + t[n]=e.dt + end + end + return t end - local function tag(collected,n) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local c - if n == 0 or not n then - c = collected[1] - elseif n > 1 then - c = collected[n] - else - c = collected[nc-n+1] - end - return c and c.tg + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + return c and c.tg end - local function name(collected,n) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local c - if n == 0 or not n then - c = collected[1] - elseif n > 1 then - c = collected[n] - else - c = collected[nc-n+1] - end - if not c then - -- sorry - elseif c.ns == "" then - return c.tg - else - return c.ns .. ":" .. c.tg - end + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + if not c then + elseif c.ns=="" then + return c.tg + else + return c.ns..":"..c.tg + end end - local function tags(collected,nonamespace) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local t, n = { }, 0 - for c=1,nc do - local e = collected[c] - local ns, tg = e.ns, e.tg - n = n + 1 - if nonamespace or ns == "" then - t[n] = tg - else - t[n] = ns .. ":" .. tg - end + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace or ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg end - return t + end + return t end - local function empty(collected,spacesonly) - if not collected then - return true - end - local nc = #collected - if nc == 0 then - return true - end - for c=1,nc do - local e = collected[c] - if e then - local edt = e.dt - if edt then - local n = #edt - if n == 1 then - local edk = edt[1] - local typ = type(edk) - if typ == "table" then - return false - elseif edk ~= "" then - return false - elseif spacesonly and not find(edk,"%S") then - return false - end - elseif n > 1 then - return false - end - end + if not collected then + return true + end + local nc=#collected + if nc==0 then + return true + end + for c=1,nc do + local e=collected[c] + if e then + local edt=e.dt + if edt then + local n=#edt + if n==1 then + local edk=edt[1] + local typ=type(edk) + if typ=="table" then + return false + elseif edk~="" then + return false + elseif spacesonly and not find(edk,"%S") then + return false + end + elseif n>1 then + return false end + end end - return true -end - -finalizers.first = first -finalizers.last = last -finalizers.all = all -finalizers.reverse = reverse -finalizers.elements = all -finalizers.default = all -finalizers.attribute = attribute -finalizers.att = att -finalizers.count = count -finalizers.position = position -finalizers.match = match -finalizers.index = index -finalizers.attributes = attributes -finalizers.chainattribute = chainattribute -finalizers.text = text -finalizers.texts = texts -finalizers.tag = tag -finalizers.name = name -finalizers.tags = tags -finalizers.empty = empty - --- shortcuts -- we could support xmlfilter(id,pattern,first) - + end + return true +end +finalizers.first=first +finalizers.last=last +finalizers.all=all +finalizers.reverse=reverse +finalizers.elements=all +finalizers.default=all +finalizers.attribute=attribute +finalizers.att=att +finalizers.count=count +finalizers.position=position +finalizers.match=match +finalizers.index=index +finalizers.attributes=attributes +finalizers.chainattribute=chainattribute +finalizers.text=text +finalizers.texts=texts +finalizers.tag=tag +finalizers.name=name +finalizers.tags=tags +finalizers.empty=empty function xml.first(id,pattern) - return first(xmlfilter(id,pattern)) + return first(xmlfilter(id,pattern)) end - function xml.last(id,pattern) - return last(xmlfilter(id,pattern)) + return last(xmlfilter(id,pattern)) end - function xml.count(id,pattern) - return count(xmlfilter(id,pattern)) + return count(xmlfilter(id,pattern)) end - function xml.attribute(id,pattern,a,default) - return attribute(xmlfilter(id,pattern),a,default) + return attribute(xmlfilter(id,pattern),a,default) end - function xml.raw(id,pattern) - if pattern then - return raw(xmlfilter(id,pattern)) - else - return raw(id) - end -end - -function xml.text(id,pattern) -- brrr either content or element (when cdata) - if pattern then - -- return text(xmlfilter(id,pattern)) - local collected = xmlfilter(id,pattern) - return collected and #collected > 0 and xmltotext(collected[1]) or "" - elseif id then - -- return text(id) - return xmltotext(id) or "" - else - return "" - end + if pattern then + return raw(xmlfilter(id,pattern)) + else + return raw(id) + end +end +function xml.text(id,pattern) + if pattern then + local collected=xmlfilter(id,pattern) + return collected and #collected>0 and xmltotext(collected[1]) or "" + elseif id then + return xmltotext(id) or "" + else + return "" + end end - -xml.content = text - --- - -function xml.position(id,pattern,n) -- element - return position(xmlfilter(id,pattern),n) +xml.content=text +function xml.position(id,pattern,n) + return position(xmlfilter(id,pattern),n) end - -function xml.match(id,pattern) -- number - return match(xmlfilter(id,pattern)) +function xml.match(id,pattern) + return match(xmlfilter(id,pattern)) end - function xml.empty(id,pattern,spacesonly) - return empty(xmlfilter(id,pattern),spacesonly) + return empty(xmlfilter(id,pattern),spacesonly) end - -xml.all = xml.filter -xml.index = xml.position -xml.found = xml.filter - --- a nice one: - +xml.all=xml.filter +xml.index=xml.position +xml.found=xml.filter local function totable(x) - local t = { } - for e in xmlcollected(x[1] or x,"/*") do - t[e.tg] = xmltostring(e.dt) or "" - end - return next(t) and t or nil -end - -xml.table = totable -finalizers.table = totable - + local t={} + for e in xmlcollected(x[1] or x,"/*") do + t[e.tg]=xmltostring(e.dt) or "" + end + return next(t) and t or nil +end +xml.table=totable +finalizers.table=totable local function textonly(e,t) - if e then - local edt = e.dt - if edt then - for i=1,#edt do - local e = edt[i] - if type(e) == "table" then - textonly(e,t) - else - t[#t+1] = e - end - end + if e then + local edt=e.dt + if edt then + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + textonly(e,t) + else + t[#t+1]=e end + end end - return t + end + return t end - -function xml.textonly(e) -- no pattern - return concat(textonly(e,{})) +function xml.textonly(e) + return concat(textonly(e,{})) end - --- - --- local x = xml.convert("123") --- xml.filter(x,"**/lowerall()") print(x) --- xml.filter(x,"**/upperall()") print(x) - function finalizers.lowerall(collected) - for c=1,#collected do - local e = collected[c] - if not e.special then - e.tg = lower(e.tg) - local eat = e.at - if eat then - local t = { } - for k,v in next, eat do - t[lower(k)] = v - end - e.at = t - end - end + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=lower(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[lower(k)]=v + end + e.at=t + end end + end end - function finalizers.upperall(collected) - for c=1,#collected do - local e = collected[c] - if not e.special then - e.tg = upper(e.tg) - local eat = e.at - if eat then - local t = { } - for k,v in next, eat do - t[upper(k)] = v - end - e.at = t - end - end + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=upper(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[upper(k)]=v + end + e.at=t + end end + end end @@ -13462,237 +10309,159 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-ini'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - -local gsub, find, gmatch, char = string.gsub, string.find, string.gmatch, string.char -local next, type = next, type - -local filedirname, filebasename, filejoin = file.dirname, file.basename, file.join - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_initialization = logs.reporter("resolvers","initialization") +-- original size: 7894, stripped down to: 5497 -local ostype, osname, ossetenv, osgetenv = os.type, os.name, os.setenv, os.getenv - --- The code here used to be part of a data-res but for convenience --- we now split it over multiple files. As this file is now the --- starting point we introduce resolvers here. - -resolvers = resolvers or { } -local resolvers = resolvers - --- We don't want the kpse library to kick in. Also, we want to be able to --- execute programs. Control over execution is implemented later. - -texconfig.kpse_init = false -texconfig.shell_escape = 't' - -if kpse and kpse.default_texmfcnf then - local default_texmfcnf = kpse.default_texmfcnf() - -- looks more like context: - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTODIR","selfautodir:") - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTOPARENT","selfautoparent:") - default_texmfcnf = gsub(default_texmfcnf,"$HOME","home:") - -- - environment.default_texmfcnf = default_texmfcnf -end - -kpse = { original = kpse } - -setmetatable(kpse, { - __index = function(kp,name) - report_initialization("fatal error: kpse library is accessed (key: %s)",name) - os.exit() - end +if not modules then modules={} end modules ['data-ini']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char +local next,type=next,type +local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_initialization=logs.reporter("resolvers","initialization") +local ostype,osname,ossetenv,osgetenv=os.type,os.name,os.setenv,os.getenv +resolvers=resolvers or {} +local resolvers=resolvers +texconfig.kpse_init=false +texconfig.shell_escape='t' +if not (environment and environment.default_texmfcnf) and kpse and kpse.default_texmfcnf then + local default_texmfcnf=kpse.default_texmfcnf() + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTODIR","selfautodir:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOPARENT","selfautoparent:") + default_texmfcnf=gsub(default_texmfcnf,"$HOME","home:") + environment.default_texmfcnf=default_texmfcnf +end +kpse={ original=kpse } +setmetatable(kpse,{ + __index=function(kp,name) + report_initialization("fatal error: kpse library is accessed (key: %s)",name) + os.exit() + end } ) - --- First we check a couple of environment variables. Some might be --- set already but we need then later on. We start with the system --- font path. - do - - local osfontdir = osgetenv("OSFONTDIR") - - if osfontdir and osfontdir ~= "" then - -- ok - elseif osname == "windows" then - ossetenv("OSFONTDIR","c:/windows/fonts//") - elseif osname == "macosx" then - ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") - end - + local osfontdir=osgetenv("OSFONTDIR") + if osfontdir and osfontdir~="" then + elseif osname=="windows" then + ossetenv("OSFONTDIR","c:/windows/fonts//") + elseif osname=="macosx" then + ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + end end - --- Next comes the user's home path. We need this as later on we have --- to replace ~ with its value. - do - - local homedir = osgetenv(ostype == "windows" and 'USERPROFILE' or 'HOME') or '' - - if not homedir or homedir == "" then - homedir = char(127) -- we need a value, later we wil trigger on it - end - - homedir = file.collapsepath(homedir) - - ossetenv("HOME", homedir) -- can be used in unix cnf files - ossetenv("USERPROFILE",homedir) -- can be used in windows cnf files - - environment.homedir = homedir - + local homedir=osgetenv(ostype=="windows" and 'USERPROFILE' or 'HOME') or '' + if not homedir or homedir=="" then + homedir=char(127) + end + homedir=file.collapsepath(homedir) + ossetenv("HOME",homedir) + ossetenv("USERPROFILE",homedir) + environment.homedir=homedir end - --- The following code sets the name of the own binary and its --- path. This is fallback code as we have os.selfdir now. - do - - local args = environment.originalarguments or arg -- this needs a cleanup - - if not environment.ownmain then - environment.ownmain = status and string.match(string.lower(status.banner),"this is ([%a]+)") or "luatex" - end - - local ownbin = environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" - local ownpath = environment.ownpath or os.selfdir - - ownbin = file.collapsepath(ownbin) - ownpath = file.collapsepath(ownpath) - - if not ownpath or ownpath == "" or ownpath == "unset" then - ownpath = args[-1] or arg[-1] - ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) - if not ownpath or ownpath == "" then - ownpath = args[-0] or arg[-0] - ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) - end - local binary = ownbin - if not ownpath or ownpath == "" then - ownpath = ownpath and filedirname(binary) - end - if not ownpath or ownpath == "" then - if os.binsuffix ~= "" then - binary = file.replacesuffix(binary,os.binsuffix) - end - local path = osgetenv("PATH") - if path then - for p in gmatch(path,"[^"..io.pathseparator.."]+") do - local b = filejoin(p,binary) - if lfs.isfile(b) then - -- we assume that after changing to the path the currentdir function - -- resolves to the real location and use this side effect here; this - -- trick is needed because on the mac installations use symlinks in the - -- path instead of real locations - local olddir = lfs.currentdir() - if lfs.chdir(p) then - local pp = lfs.currentdir() - if trace_locating and p ~= pp then - report_initialization("following symlink '%s' to '%s'",p,pp) - end - ownpath = pp - lfs.chdir(olddir) - else - if trace_locating then - report_initialization("unable to check path '%s'",p) - end - ownpath = p - end - break - end - end + local args=environment.originalarguments or arg + if not environment.ownmain then + environment.ownmain=status and string.match(string.lower(status.banner),"this is ([%a]+)") or "luatex" + end + local ownbin=environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" + local ownpath=environment.ownpath or os.selfdir + ownbin=file.collapsepath(ownbin) + ownpath=file.collapsepath(ownpath) + if not ownpath or ownpath=="" or ownpath=="unset" then + ownpath=args[-1] or arg[-1] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath=="" then + ownpath=args[-0] or arg[-0] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + end + local binary=ownbin + if not ownpath or ownpath=="" then + ownpath=ownpath and filedirname(binary) + end + if not ownpath or ownpath=="" then + if os.binsuffix~="" then + binary=file.replacesuffix(binary,os.binsuffix) + end + local path=osgetenv("PATH") + if path then + for p in gmatch(path,"[^"..io.pathseparator.."]+") do + local b=filejoin(p,binary) + if lfs.isfile(b) then + local olddir=lfs.currentdir() + if lfs.chdir(p) then + local pp=lfs.currentdir() + if trace_locating and p~=pp then + report_initialization("following symlink '%s' to '%s'",p,pp) + end + ownpath=pp + lfs.chdir(olddir) + else + if trace_locating then + report_initialization("unable to check path '%s'",p) + end + ownpath=p end + break + end end - if not ownpath or ownpath == "" then - ownpath = "." - report_initialization("forcing fallback ownpath .") - elseif trace_locating then - report_initialization("using ownpath '%s'",ownpath) - end + end end - - environment.ownbin = ownbin - environment.ownpath = ownpath - + if not ownpath or ownpath=="" then + ownpath="." + report_initialization("forcing fallback ownpath .") + elseif trace_locating then + report_initialization("using ownpath '%s'",ownpath) + end + end + environment.ownbin=ownbin + environment.ownpath=ownpath end - -resolvers.ownpath = environment.ownpath - +resolvers.ownpath=environment.ownpath function resolvers.getownpath() - return environment.ownpath + return environment.ownpath end - --- The self variables permit us to use only a few (or even no) --- environment variables. - do - - local ownpath = environment.ownpath or dir.current() - - if ownpath then - ossetenv('SELFAUTOLOC', file.collapsepath(ownpath)) - ossetenv('SELFAUTODIR', file.collapsepath(ownpath .. "/..")) - ossetenv('SELFAUTOPARENT', file.collapsepath(ownpath .. "/../..")) - else - report_initialization("error: unable to locate ownpath") - os.exit() - end - -end - --- The running os: - --- todo: check is context sits here os.platform is more trustworthy --- that the bin check as mtx-update runs from another path - -local texos = environment.texos or osgetenv("TEXOS") -local texmfos = environment.texmfos or osgetenv('SELFAUTODIR') - -if not texos or texos == "" then - texos = file.basename(texmfos) -end - -ossetenv('TEXMFOS', texmfos) -- full bin path -ossetenv('TEXOS', texos) -- partial bin parent -ossetenv('SELFAUTOSYSTEM',os.platform) -- bonus - -environment.texos = texos -environment.texmfos = texmfos - --- The current root: - -local texroot = environment.texroot or osgetenv("TEXROOT") - -if not texroot or texroot == "" then - texroot = osgetenv('SELFAUTOPARENT') - ossetenv('TEXROOT',texroot) -end - -environment.texroot = file.collapsepath(texroot) - + local ownpath=environment.ownpath or dir.current() + if ownpath then + ossetenv('SELFAUTOLOC',file.collapsepath(ownpath)) + ossetenv('SELFAUTODIR',file.collapsepath(ownpath.."/..")) + ossetenv('SELFAUTOPARENT',file.collapsepath(ownpath.."/../..")) + else + report_initialization("error: unable to locate ownpath") + os.exit() + end +end +local texos=environment.texos or osgetenv("TEXOS") +local texmfos=environment.texmfos or osgetenv('SELFAUTODIR') +if not texos or texos=="" then + texos=file.basename(texmfos) +end +ossetenv('TEXMFOS',texmfos) +ossetenv('TEXOS',texos) +ossetenv('SELFAUTOSYSTEM',os.platform) +environment.texos=texos +environment.texmfos=texmfos +local texroot=environment.texroot or osgetenv("TEXROOT") +if not texroot or texroot=="" then + texroot=osgetenv('SELFAUTOPARENT') + ossetenv('TEXROOT',texroot) +end +environment.texroot=file.collapsepath(texroot) if profiler then - directives.register("system.profile",function() - profiler.start("luatex-profile.log") - end) + directives.register("system.profile",function() + profiler.start("luatex-profile.log") + end) end - --- a forward definition - if not resolvers.resolve then - function resolvers.resolve (s) return s end - function resolvers.unresolve(s) return s end - function resolvers.repath (s) return s end + function resolvers.resolve (s) return s end + function resolvers.unresolve(s) return s end + function resolvers.repath (s) return s end end @@ -13700,3216 +10469,2663 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-exp'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - -local format, find, gmatch, lower, char, sub = string.format, string.find, string.gmatch, string.lower, string.char, string.sub -local concat, sort = table.concat, table.sort -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns -local Ct, Cs, Cc, P, C, S = lpeg.Ct, lpeg.Cs, lpeg.Cc, lpeg.P, lpeg.C, lpeg.S -local type, next = type, next - -local ostype = os.type -local collapsepath = file.collapsepath - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_expansions = logs.reporter("resolvers","expansions") - -local resolvers = resolvers - --- As this bit of code is somewhat special it gets its own module. After --- all, when working on the main resolver code, I don't want to scroll --- past this every time. See data-obs.lua for the gsub variant. +-- original size: 14663, stripped down to: 9537 +if not modules then modules={} end modules ['data-exp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local format,find,gmatch,lower,char,sub=string.format,string.find,string.gmatch,string.lower,string.char,string.sub +local concat,sort=table.concat,table.sort +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local Ct,Cs,Cc,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.P,lpeg.C,lpeg.S +local type,next=type,next +local ostype=os.type +local collapsepath=file.collapsepath +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_expansions=logs.reporter("resolvers","expansions") +local resolvers=resolvers local function f_first(a,b) - local t, n = { }, 0 - for s in gmatch(b,"[^,]+") do - n = n + 1 ; t[n] = a .. s - end - return concat(t,",") + local t,n={},0 + for s in gmatch(b,"[^,]+") do + n=n+1;t[n]=a..s + end + return concat(t,",") end - local function f_second(a,b) - local t, n = { }, 0 - for s in gmatch(a,"[^,]+") do - n = n + 1 ; t[n] = s .. b - end - return concat(t,",") + local t,n={},0 + for s in gmatch(a,"[^,]+") do + n=n+1;t[n]=s..b + end + return concat(t,",") end - --- kpsewhich --expand-braces '{a,b}{c,d}' --- ac:bc:ad:bd - --- old {a,b}{c,d} => ac ad bc bd --- --- local function f_both(a,b) --- local t, n = { }, 0 --- for sa in gmatch(a,"[^,]+") do --- for sb in gmatch(b,"[^,]+") do --- n = n + 1 ; t[n] = sa .. sb --- end --- end --- return concat(t,",") --- end --- --- new {a,b}{c,d} => ac bc ad bd - local function f_both(a,b) - local t, n = { }, 0 - for sb in gmatch(b,"[^,]+") do -- and not sa - for sa in gmatch(a,"[^,]+") do -- sb - n = n + 1 ; t[n] = sa .. sb - end - end - return concat(t,",") -end - -local left = P("{") -local right = P("}") -local var = P((1 - S("{}" ))^0) -local set = P((1 - S("{},"))^0) -local other = P(1) - -local l_first = Cs( ( Cc("{") * (C(set) * left * C(var) * right / f_first) * Cc("}") + other )^0 ) -local l_second = Cs( ( Cc("{") * (left * C(var) * right * C(set) / f_second) * Cc("}") + other )^0 ) -local l_both = Cs( ( Cc("{") * (left * C(var) * right * left * C(var) * right / f_both) * Cc("}") + other )^0 ) -local l_rest = Cs( ( left * var * (left/"") * var * (right/"") * var * right + other )^0 ) - -local stripper_1 = lpeg.stripper ("{}@") -local replacer_1 = lpeg.replacer { { ",}", ",@}" }, { "{,", "{@," }, } - -local function splitpathexpr(str, newlist, validate) -- I couldn't resist lpegging it (nice exercise). - if trace_expansions then - report_expansions("expanding variable '%s'",str) - end - local t, ok, done = newlist or { }, false, false - local n = #t - str = lpegmatch(replacer_1,str) + local t,n={},0 + for sb in gmatch(b,"[^,]+") do + for sa in gmatch(a,"[^,]+") do + n=n+1;t[n]=sa..sb + end + end + return concat(t,",") +end +local left=P("{") +local right=P("}") +local var=P((1-S("{}" ))^0) +local set=P((1-S("{},"))^0) +local other=P(1) +local l_first=Cs((Cc("{")*(C(set)*left*C(var)*right/f_first)*Cc("}")+other )^0 ) +local l_second=Cs((Cc("{")*(left*C(var)*right*C(set)/f_second)*Cc("}")+other )^0 ) +local l_both=Cs((Cc("{")*(left*C(var)*right*left*C(var)*right/f_both)*Cc("}")+other )^0 ) +local l_rest=Cs((left*var*(left/"")*var*(right/"")*var*right+other )^0 ) +local stripper_1=lpeg.stripper ("{}@") +local replacer_1=lpeg.replacer { { ",}",",@}" },{ "{,","{@," },} +local function splitpathexpr(str,newlist,validate) + if trace_expansions then + report_expansions("expanding variable '%s'",str) + end + local t,ok,done=newlist or {},false,false + local n=#t + str=lpegmatch(replacer_1,str) + repeat + local old=str repeat - local old = str - repeat - local old = str - str = lpegmatch(l_first, str) - until old == str - repeat - local old = str - str = lpegmatch(l_second,str) - until old == str - repeat - local old = str - str = lpegmatch(l_both, str) - until old == str - repeat - local old = str - str = lpegmatch(l_rest, str) - until old == str - until old == str -- or not find(str,"{") - str = lpegmatch(stripper_1,str) - if validate then - for s in gmatch(str,"[^,]+") do - s = validate(s) - if s then - n = n + 1 - t[n] = s - end - end - else - for s in gmatch(str,"[^,]+") do - n = n + 1 - t[n] = s - end + local old=str + str=lpegmatch(l_first,str) + until old==str + repeat + local old=str + str=lpegmatch(l_second,str) + until old==str + repeat + local old=str + str=lpegmatch(l_both,str) + until old==str + repeat + local old=str + str=lpegmatch(l_rest,str) + until old==str + until old==str + str=lpegmatch(stripper_1,str) + if validate then + for s in gmatch(str,"[^,]+") do + s=validate(s) + if s then + n=n+1 + t[n]=s + end end - if trace_expansions then - for k=1,#t do - report_expansions("% 4i: %s",k,t[k]) - end + else + for s in gmatch(str,"[^,]+") do + n=n+1 + t[n]=s end - return t + end + if trace_expansions then + for k=1,#t do + report_expansions("% 4i: %s",k,t[k]) + end + end + return t end - --- We could make the previous one public. - local function validate(s) - s = collapsepath(s) -- already keeps the trailing / and // - return s ~= "" and not find(s,"^!*unset/*$") and s + s=collapsepath(s) + return s~="" and not find(s,"^!*unset/*$") and s end - -resolvers.validatedpath = validate -- keeps the trailing // - +resolvers.validatedpath=validate function resolvers.expandedpathfromlist(pathlist) - local newlist = { } - for k=1,#pathlist do - splitpathexpr(pathlist[k],newlist,validate) - end - return newlist -end - --- {a,b,c,d} --- a,b,c/{p,q,r},d --- a,b,c/{p,q,r}/d/{x,y,z}// --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a{b,c}{d,e}f --- {a,b,c,d} --- {a,b,c/{p,q,r},d} --- {a,b,c/{p,q,r}/d/{x,y,z}//} --- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} --- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} --- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} - -local cleanup = lpeg.replacer { - { "!" , "" }, - { "\\" , "/" }, + local newlist={} + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) + end + return newlist +end +local cleanup=lpeg.replacer { + { "!","" }, + { "\\","/" }, } - -function resolvers.cleanpath(str) -- tricky, maybe only simple paths - local doslashes = (P("\\")/"/" + 1)^0 - local donegation = (P("!") /"" )^0 - local homedir = lpegmatch(Cs(donegation * doslashes),environment.homedir or "") - if homedir == "~" or homedir == "" or not lfs.isdir(homedir) then - if trace_expansions then - report_expansions("no home dir set, ignoring dependent paths") - end - function resolvers.cleanpath(str) - if not str or find(str,"~") then - return "" -- special case - else - return lpegmatch(cleanup,str) - end - end - else - local dohome = ((P("~")+P("$HOME"))/homedir)^0 - local cleanup = Cs(donegation * dohome * doslashes) - function resolvers.cleanpath(str) - return str and lpegmatch(cleanup,str) or "" - end +function resolvers.cleanpath(str) + local doslashes=(P("\\")/"/"+1)^0 + local donegation=(P("!")/"" )^0 + local homedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") + if homedir=="~" or homedir=="" or not lfs.isdir(homedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent paths") end - return resolvers.cleanpath(str) -end - --- print(resolvers.cleanpath("")) --- print(resolvers.cleanpath("!")) --- print(resolvers.cleanpath("~")) --- print(resolvers.cleanpath("~/test")) --- print(resolvers.cleanpath("!~/test")) --- print(resolvers.cleanpath("~/test~test")) - --- This one strips quotes and funny tokens. - -local expandhome = P("~") / "$HOME" -- environment.homedir - -local dodouble = P('"')/"" * (expandhome + (1 - P('"')))^0 * P('"')/"" -local dosingle = P("'")/"" * (expandhome + (1 - P("'")))^0 * P("'")/"" -local dostring = (expandhome + 1 )^0 - -local stripper = Cs( - lpegpatterns.unspacer * (dosingle + dodouble + dostring) * lpegpatterns.unspacer + function resolvers.cleanpath(str) + if not str or find(str,"~") then + return "" + else + return lpegmatch(cleanup,str) + end + end + else + local dohome=((P("~")+P("$HOME"))/homedir)^0 + local cleanup=Cs(donegation*dohome*doslashes) + function resolvers.cleanpath(str) + return str and lpegmatch(cleanup,str) or "" + end + end + return resolvers.cleanpath(str) +end +local expandhome=P("~")/"$HOME" +local dodouble=P('"')/""*(expandhome+(1-P('"')))^0*P('"')/"" +local dosingle=P("'")/""*(expandhome+(1-P("'")))^0*P("'")/"" +local dostring=(expandhome+1 )^0 +local stripper=Cs( + lpegpatterns.unspacer*(dosingle+dodouble+dostring)*lpegpatterns.unspacer ) - -function resolvers.checkedvariable(str) -- assumes str is a string - return type(str) == "string" and lpegmatch(stripper,str) or str -end - --- The path splitter: - --- A config (optionally) has the paths split in tables. Internally --- we join them and split them after the expansion has taken place. This --- is more convenient. - -local cache = { } - ------ splitter = lpeg.tsplitat(S(ostype == "windows" and ";" or ":;")) -- maybe add , -local splitter = lpeg.tsplitat(";") -- as we move towards urls, prefixes and use tables we no longer do : - -local backslashswapper = lpeg.replacer("\\","/") - -local function splitconfigurationpath(str) -- beware, this can be either a path or a { specification } - if str then - local found = cache[str] - if not found then - if str == "" then - found = { } - else - local split = lpegmatch(splitter,lpegmatch(backslashswapper,str)) -- can be combined - found = { } - local noffound = 0 - for i=1,#split do - local s = split[i] - if not find(s,"^{*unset}*") then - noffound = noffound + 1 - found[noffound] = s - end - end - if trace_expansions then - report_expansions("splitting path specification '%s'",str) - for k=1,noffound do - report_expansions("% 4i: %s",k,found[k]) - end - end - cache[str] = found - end +function resolvers.checkedvariable(str) + return type(str)=="string" and lpegmatch(stripper,str) or str +end +local cache={} +local splitter=lpeg.tsplitat(";") +local backslashswapper=lpeg.replacer("\\","/") +local function splitconfigurationpath(str) + if str then + local found=cache[str] + if not found then + if str=="" then + found={} + else + local split=lpegmatch(splitter,lpegmatch(backslashswapper,str)) + found={} + local noffound=0 + for i=1,#split do + local s=split[i] + if not find(s,"^{*unset}*") then + noffound=noffound+1 + found[noffound]=s + end + end + if trace_expansions then + report_expansions("splitting path specification '%s'",str) + for k=1,noffound do + report_expansions("% 4i: %s",k,found[k]) + end end - return found + cache[str]=found + end end + return found + end end - -resolvers.splitconfigurationpath = splitconfigurationpath - +resolvers.splitconfigurationpath=splitconfigurationpath function resolvers.splitpath(str) - if type(str) == 'table' then - return str - else - return splitconfigurationpath(str) - end + if type(str)=='table' then + return str + else + return splitconfigurationpath(str) + end end - function resolvers.joinpath(str) - if type(str) == 'table' then - return file.joinpath(str) - else - return str - end -end - --- The next function scans directories and returns a hash where the --- entries are either strings or tables. - --- starting with . or .. etc or funny char - - - - --- a lot of this caching can be stripped away when we have ssd's everywhere --- --- we could cache all the (sub)paths here if needed - -local attributes, directory = lfs.attributes, lfs.dir - -local weird = P(".")^1 + lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) -local timer = { } -local scanned = { } -local nofscans = 0 -local scancache = { } - + if type(str)=='table' then + return file.joinpath(str) + else + return str + end +end +local attributes,directory=lfs.attributes,lfs.dir +local weird=P(".")^1+lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local timer={} +local scanned={} +local nofscans=0 +local scancache={} local function scan(files,spec,path,n,m,r) - local full = (path == "" and spec) or (spec .. path .. '/') - local dirs = { } - local nofdirs = 0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode = attributes(full..name,'mode') - if mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - else -- probably unique anyway - files[name] = path - local lower = lower(name) - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end - end - elseif mode == 'directory' then - m = m + 1 - nofdirs = nofdirs + 1 - if path ~= "" then - dirs[nofdirs] = path..'/'..name - else - dirs[nofdirs] = name - end - end - end - end - if nofdirs > 0 then - sort(dirs) - for i=1,nofdirs do - files, n, m, r = scan(files,spec,dirs[i],n,m,r) - end - end - scancache[sub(full,1,-2)] = files - return files, n, m, r -end - -local fullcache = { } - -function resolvers.scanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath = resolvers.resolve(path) -- no shortcut - if usecache then - local files = fullcache[realpath] - if files then - if trace_locating then - report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) - end - return files - end - end - if trace_locating then - report_expansions("scanning path '%s', branch '%s'",path,branch or path) - end - local files, n, m, r = scan({ },realpath .. '/',"",0,0,0) - files.__path__ = path -- can be selfautoparent:texmf-whatever - files.__files__ = n - files.__directories__ = m - files.__remappings__ = r - if trace_locating then - report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) - end - if usecache then - scanned[#scanned+1] = realpath - fullcache[realpath] = files - end - nofscans = nofscans + 1 - statistics.stoptiming(timer) - return files -end - -local function simplescan(files,spec,path) -- first match only, no map and such - local full = (path == "" and spec) or (spec .. path .. '/') - local dirs = { } - local nofdirs = 0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode = attributes(full..name,'mode') - if mode == 'file' then - if not files[name] then - -- only first match - files[name] = path - end - elseif mode == 'directory' then - nofdirs = nofdirs + 1 - if path ~= "" then - dirs[nofdirs] = path..'/'..name - else - dirs[nofdirs] = name - end - end - end - end - if nofdirs > 0 then - sort(dirs) - for i=1,nofdirs do - files = simplescan(files,spec,dirs[i]) - end - end - return files -end - -local simplecache = { } -local nofsharedscans = 0 - -function resolvers.simplescanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath = resolvers.resolve(path) -- no shortcut - if usecache then - local files = simplecache[realpath] - if not files then - files = scancache[realpath] - if files then - nofsharedscans = nofsharedscans + 1 - end + local full=(path=="" and spec) or (spec..path..'/') + local dirs={} + local nofdirs=0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode=attributes(full..name,'mode') + if mode=='file' then + n=n+1 + local f=files[name] + if f then + if type(f)=='string' then + files[name]={ f,path } + else + f[#f+1]=path + end + else + files[name]=path + local lower=lower(name) + if name~=lower then + files["remap:"..lower]=name + r=r+1 + end end - if files then - if trace_locating then - report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) - end - return files + elseif mode=='directory' then + m=m+1 + nofdirs=nofdirs+1 + if path~="" then + dirs[nofdirs]=path..'/'..name + else + dirs[nofdirs]=name end + end end - if trace_locating then - report_expansions("scanning path '%s', branch '%s'",path,branch or path) + end + if nofdirs>0 then + sort(dirs) + for i=1,nofdirs do + files,n,m,r=scan(files,spec,dirs[i],n,m,r) end - local files = simplescan({ },realpath .. '/',"") - if trace_locating then - report_expansions("%s files found",table.count(files)) + end + scancache[sub(full,1,-2)]=files + return files,n,m,r +end +local fullcache={} +function resolvers.scanfiles(path,branch,usecache) + statistics.starttiming(timer) + local realpath=resolvers.resolve(path) + if usecache then + local files=fullcache[realpath] + if files then + if trace_locating then + report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) + end + return files + end + end + if trace_locating then + report_expansions("scanning path '%s', branch '%s'",path,branch or path) + end + local files,n,m,r=scan({},realpath..'/',"",0,0,0) + files.__path__=path + files.__files__=n + files.__directories__=m + files.__remappings__=r + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) + end + if usecache then + scanned[#scanned+1]=realpath + fullcache[realpath]=files + end + nofscans=nofscans+1 + statistics.stoptiming(timer) + return files +end +local function simplescan(files,spec,path) + local full=(path=="" and spec) or (spec..path..'/') + local dirs={} + local nofdirs=0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode=attributes(full..name,'mode') + if mode=='file' then + if not files[name] then + files[name]=path + end + elseif mode=='directory' then + nofdirs=nofdirs+1 + if path~="" then + dirs[nofdirs]=path..'/'..name + else + dirs[nofdirs]=name + end + end end - if usecache then - scanned[#scanned+1] = realpath - simplecache[realpath] = files + end + if nofdirs>0 then + sort(dirs) + for i=1,nofdirs do + files=simplescan(files,spec,dirs[i]) end - nofscans = nofscans + 1 - statistics.stoptiming(timer) - return files + end + return files +end +local simplecache={} +local nofsharedscans=0 +function resolvers.simplescanfiles(path,branch,usecache) + statistics.starttiming(timer) + local realpath=resolvers.resolve(path) + if usecache then + local files=simplecache[realpath] + if not files then + files=scancache[realpath] + if files then + nofsharedscans=nofsharedscans+1 + end + end + if files then + if trace_locating then + report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) + end + return files + end + end + if trace_locating then + report_expansions("scanning path '%s', branch '%s'",path,branch or path) + end + local files=simplescan({},realpath..'/',"") + if trace_locating then + report_expansions("%s files found",table.count(files)) + end + if usecache then + scanned[#scanned+1]=realpath + simplecache[realpath]=files + end + nofscans=nofscans+1 + statistics.stoptiming(timer) + return files end - function resolvers.scandata() - table.sort(scanned) - return { - n = nofscans, - shared = nofsharedscans, - time = statistics.elapsedtime(timer), - paths = scanned, - } + table.sort(scanned) + return { + n=nofscans, + shared=nofsharedscans, + time=statistics.elapsedtime(timer), + paths=scanned, + } end - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-env'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} +-- original size: 8762, stripped down to: 6484 -local lower, gsub = string.lower, string.gsub - -local resolvers = resolvers - -local allocate = utilities.storage.allocate -local setmetatableindex = table.setmetatableindex -local suffixonly = file.suffixonly - -local formats = allocate() -local suffixes = allocate() -local dangerous = allocate() -local suffixmap = allocate() - -resolvers.formats = formats -resolvers.suffixes = suffixes -resolvers.dangerous = dangerous -resolvers.suffixmap = suffixmap - -local luasuffixes = utilities.lua.suffixes - -local relations = allocate { -- todo: handlers also here - core = { - ofm = { -- will become obsolete - names = { "ofm", "omega font metric", "omega font metrics" }, - variable = 'OFMFONTS', - suffixes = { 'ofm', 'tfm' }, - }, - ovf = { -- will become obsolete - names = { "ovf", "omega virtual font", "omega virtual fonts" }, - variable = 'OVFFONTS', - suffixes = { 'ovf', 'vf' }, - }, - tfm = { - names = { "tfm", "tex font metric", "tex font metrics" }, - variable = 'TFMFONTS', - suffixes = { 'tfm' }, - }, - vf = { - names = { "vf", "virtual font", "virtual fonts" }, - variable = 'VFFONTS', - suffixes = { 'vf' }, - }, - otf = { - names = { "otf", "opentype", "opentype font", "opentype fonts"}, - variable = 'OPENTYPEFONTS', - suffixes = { 'otf' }, - }, - ttf = { - names = { "ttf", "truetype", "truetype font", "truetype fonts", "truetype collection", "truetype collections", "truetype dictionary", "truetype dictionaries" }, - variable = 'TTFONTS', - suffixes = { 'ttf', 'ttc', 'dfont' }, - }, - afm = { - names = { "afm", "adobe font metric", "adobe font metrics" }, - variable = "AFMFONTS", - suffixes = { "afm" }, - }, - pfb = { - names = { "pfb", "type1", "type 1", "type1 font", "type 1 font", "type1 fonts", "type 1 fonts" }, - variable = 'T1FONTS', - suffixes = { 'pfb', 'pfa' }, - }, - fea = { - names = { "fea", "font feature", "font features", "font feature file", "font feature files" }, - variable = 'FONTFEATURES', - suffixes = { 'fea' }, - }, - cid = { - names = { "cid", "cid map", "cid maps", "cid file", "cid files" }, - variable = 'FONTCIDMAPS', - suffixes = { 'cid', 'cidmap' }, - }, - fmt = { - names = { "fmt", "format", "tex format" }, - variable = 'TEXFORMATS', - suffixes = { 'fmt' }, - }, - mem = { -- will become obsolete - names = { 'mem', "metapost format" }, - variable = 'MPMEMS', - suffixes = { 'mem' }, - }, - mp = { - names = { "mp" }, - variable = 'MPINPUTS', - suffixes = { 'mp', 'mpvi', 'mpiv', 'mpii' }, - }, - tex = { - names = { "tex" }, - variable = 'TEXINPUTS', - suffixes = { 'tex', "mkvi", "mkiv", "mkii" }, - }, - icc = { - names = { "icc", "icc profile", "icc profiles" }, - variable = 'ICCPROFILES', - suffixes = { 'icc' }, - }, - texmfscripts = { - names = { "texmfscript", "texmfscripts", "script", "scripts" }, - variable = 'TEXMFSCRIPTS', - suffixes = { 'rb', 'pl', 'py' }, - }, - lua = { - names = { "lua" }, - variable = 'LUAINPUTS', - suffixes = { luasuffixes.lua, luasuffixes.luc, luasuffixes.tma, luasuffixes.tmc }, - }, - lib = { - names = { "lib" }, - variable = 'CLUAINPUTS', - suffixes = os.libsuffix and { os.libsuffix } or { 'dll', 'so' }, - }, - bib = { - names = { 'bib' }, - suffixes = { 'bib' }, - }, - bst = { - names = { 'bst' }, - suffixes = { 'bst' }, - }, - fontconfig = { - names = { 'fontconfig', 'fontconfig file', 'fontconfig files' }, - variable = 'FONTCONFIG_PATH', - }, +if not modules then modules={} end modules ['data-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local lower,gsub=string.lower,string.gsub +local resolvers=resolvers +local allocate=utilities.storage.allocate +local setmetatableindex=table.setmetatableindex +local suffixonly=file.suffixonly +local formats=allocate() +local suffixes=allocate() +local dangerous=allocate() +local suffixmap=allocate() +resolvers.formats=formats +resolvers.suffixes=suffixes +resolvers.dangerous=dangerous +resolvers.suffixmap=suffixmap +local luasuffixes=utilities.lua.suffixes +local relations=allocate { + core={ + ofm={ + names={ "ofm","omega font metric","omega font metrics" }, + variable='OFMFONTS', + suffixes={ 'ofm','tfm' }, + }, + ovf={ + names={ "ovf","omega virtual font","omega virtual fonts" }, + variable='OVFFONTS', + suffixes={ 'ovf','vf' }, + }, + tfm={ + names={ "tfm","tex font metric","tex font metrics" }, + variable='TFMFONTS', + suffixes={ 'tfm' }, + }, + vf={ + names={ "vf","virtual font","virtual fonts" }, + variable='VFFONTS', + suffixes={ 'vf' }, + }, + otf={ + names={ "otf","opentype","opentype font","opentype fonts"}, + variable='OPENTYPEFONTS', + suffixes={ 'otf' }, + }, + ttf={ + names={ "ttf","truetype","truetype font","truetype fonts","truetype collection","truetype collections","truetype dictionary","truetype dictionaries" }, + variable='TTFONTS', + suffixes={ 'ttf','ttc','dfont' }, + }, + afm={ + names={ "afm","adobe font metric","adobe font metrics" }, + variable="AFMFONTS", + suffixes={ "afm" }, + }, + pfb={ + names={ "pfb","type1","type 1","type1 font","type 1 font","type1 fonts","type 1 fonts" }, + variable='T1FONTS', + suffixes={ 'pfb','pfa' }, + }, + fea={ + names={ "fea","font feature","font features","font feature file","font feature files" }, + variable='FONTFEATURES', + suffixes={ 'fea' }, }, - obsolete = { - enc = { - names = { "enc", "enc files", "enc file", "encoding files", "encoding file" }, - variable = 'ENCFONTS', - suffixes = { 'enc' }, - }, - map = { - names = { "map", "map files", "map file" }, - variable = 'TEXFONTMAPS', - suffixes = { 'map' }, - }, - lig = { - names = { "lig files", "lig file", "ligature file", "ligature files" }, - variable = 'LIGFONTS', - suffixes = { 'lig' }, - }, - opl = { - names = { "opl" }, - variable = 'OPLFONTS', - suffixes = { 'opl' }, - }, - ovp = { - names = { "ovp" }, - variable = 'OVPFONTS', - suffixes = { 'ovp' }, - }, + cid={ + names={ "cid","cid map","cid maps","cid file","cid files" }, + variable='FONTCIDMAPS', + suffixes={ 'cid','cidmap' }, }, - kpse = { -- subset - base = { - names = { 'base', "metafont format" }, - variable = 'MFBASES', - suffixes = { 'base', 'bas' }, - }, - cmap = { - names = { 'cmap', 'cmap files', 'cmap file' }, - variable = 'CMAPFONTS', - suffixes = { 'cmap' }, - }, - cnf = { - names = { 'cnf' }, - suffixes = { 'cnf' }, - }, - web = { - names = { 'web' }, - suffixes = { 'web', 'ch' } - }, - cweb = { - names = { 'cweb' }, - suffixes = { 'w', 'web', 'ch' }, - }, - gf = { - names = { 'gf' }, - suffixes = { 'gf' }, - }, - mf = { - names = { 'mf' }, - variable = 'MFINPUTS', - suffixes = { 'mf' }, - }, - mft = { - names = { 'mft' }, - suffixes = { 'mft' }, - }, - pk = { - names = { 'pk' }, - suffixes = { 'pk' }, - }, + fmt={ + names={ "fmt","format","tex format" }, + variable='TEXFORMATS', + suffixes={ 'fmt' }, }, + mem={ + names={ 'mem',"metapost format" }, + variable='MPMEMS', + suffixes={ 'mem' }, + }, + mp={ + names={ "mp" }, + variable='MPINPUTS', + suffixes={ 'mp','mpvi','mpiv','mpii' }, + }, + tex={ + names={ "tex" }, + variable='TEXINPUTS', + suffixes={ 'tex',"mkvi","mkiv","mkii" }, + }, + icc={ + names={ "icc","icc profile","icc profiles" }, + variable='ICCPROFILES', + suffixes={ 'icc' }, + }, + texmfscripts={ + names={ "texmfscript","texmfscripts","script","scripts" }, + variable='TEXMFSCRIPTS', + suffixes={ 'rb','pl','py' }, + }, + lua={ + names={ "lua" }, + variable='LUAINPUTS', + suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + }, + lib={ + names={ "lib" }, + variable='CLUAINPUTS', + suffixes=os.libsuffix and { os.libsuffix } or { 'dll','so' }, + }, + bib={ + names={ 'bib' }, + suffixes={ 'bib' }, + }, + bst={ + names={ 'bst' }, + suffixes={ 'bst' }, + }, + fontconfig={ + names={ 'fontconfig','fontconfig file','fontconfig files' }, + variable='FONTCONFIG_PATH', + }, + }, + obsolete={ + enc={ + names={ "enc","enc files","enc file","encoding files","encoding file" }, + variable='ENCFONTS', + suffixes={ 'enc' }, + }, + map={ + names={ "map","map files","map file" }, + variable='TEXFONTMAPS', + suffixes={ 'map' }, + }, + lig={ + names={ "lig files","lig file","ligature file","ligature files" }, + variable='LIGFONTS', + suffixes={ 'lig' }, + }, + opl={ + names={ "opl" }, + variable='OPLFONTS', + suffixes={ 'opl' }, + }, + ovp={ + names={ "ovp" }, + variable='OVPFONTS', + suffixes={ 'ovp' }, + }, + }, + kpse={ + base={ + names={ 'base',"metafont format" }, + variable='MFBASES', + suffixes={ 'base','bas' }, + }, + cmap={ + names={ 'cmap','cmap files','cmap file' }, + variable='CMAPFONTS', + suffixes={ 'cmap' }, + }, + cnf={ + names={ 'cnf' }, + suffixes={ 'cnf' }, + }, + web={ + names={ 'web' }, + suffixes={ 'web','ch' } + }, + cweb={ + names={ 'cweb' }, + suffixes={ 'w','web','ch' }, + }, + gf={ + names={ 'gf' }, + suffixes={ 'gf' }, + }, + mf={ + names={ 'mf' }, + variable='MFINPUTS', + suffixes={ 'mf' }, + }, + mft={ + names={ 'mft' }, + suffixes={ 'mft' }, + }, + pk={ + names={ 'pk' }, + suffixes={ 'pk' }, + }, + }, } - -resolvers.relations = relations - --- formats: maps a format onto a variable - +resolvers.relations=relations function resolvers.updaterelations() - for category, categories in next, relations do - for name, relation in next, categories do - local rn = relation.names - local rv = relation.variable - local rs = relation.suffixes - if rn and rv then - for i=1,#rn do - local rni = lower(gsub(rn[i]," ","")) - formats[rni] = rv - if rs then - suffixes[rni] = rs - for i=1,#rs do - local rsi = rs[i] - suffixmap[rsi] = rni - end - end - end - end - if rs then + for category,categories in next,relations do + for name,relation in next,categories do + local rn=relation.names + local rv=relation.variable + local rs=relation.suffixes + if rn and rv then + for i=1,#rn do + local rni=lower(gsub(rn[i]," ","")) + formats[rni]=rv + if rs then + suffixes[rni]=rs + for i=1,#rs do + local rsi=rs[i] + suffixmap[rsi]=rni end + end end + end + if rs then + end end + end end - -resolvers.updaterelations() -- push this in the metatable -> newindex - +resolvers.updaterelations() local function simplified(t,k) - return k and rawget(t,lower(gsub(k," ",""))) or nil + return k and rawget(t,lower(gsub(k," ",""))) or nil end - -setmetatableindex(formats, simplified) -setmetatableindex(suffixes, simplified) -setmetatableindex(suffixmap, simplified) - --- A few accessors, mostly for command line tool. - +setmetatableindex(formats,simplified) +setmetatableindex(suffixes,simplified) +setmetatableindex(suffixmap,simplified) function resolvers.suffixofformat(str) - local s = suffixes[str] - return s and s[1] or "" + local s=suffixes[str] + return s and s[1] or "" end - function resolvers.suffixofformat(str) - return suffixes[str] or { } + return suffixes[str] or {} end - -for name, format in next, formats do - dangerous[name] = true -- still needed ? +for name,format in next,formats do + dangerous[name]=true end - --- because vf searching is somewhat dangerous, we want to prevent --- too liberal searching esp because we do a lookup on the current --- path anyway; only tex (or any) is safe - -dangerous.tex = nil - - --- more helpers - +dangerous.tex=nil function resolvers.formatofvariable(str) - return formats[str] or '' + return formats[str] or '' end - -function resolvers.formatofsuffix(str) -- of file - return suffixmap[suffixonly(str)] or 'tex' -- so many map onto tex (like mkiv, cld etc) +function resolvers.formatofsuffix(str) + return suffixmap[suffixonly(str)] or 'tex' end - function resolvers.variableofformat(str) - return formats[str] or '' + return formats[str] or '' end - function resolvers.variableofformatorsuffix(str) - local v = formats[str] - if v then - return v - end - v = suffixmap[suffixonly(str)] - if v then - return formats[v] - end - return '' + local v=formats[str] + if v then + return v + end + v=suffixmap[suffixonly(str)] + if v then + return formats[v] + end + return '' end - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tmp'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - - - -local format, lower, gsub, concat = string.format, string.lower, string.gsub, table.concat -local serialize, serializetofile = table.serialize, table.tofile -local mkdirs, isdir = dir.mkdirs, lfs.isdir -local addsuffix, is_writable, is_readable = file.addsuffix, file.is_writable, file.is_readable - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) - -local report_caches = logs.reporter("resolvers","caches") -local report_resolvers = logs.reporter("resolvers","caching") - -local resolvers = resolvers - --- intermezzo - -local directive_cleanup = false directives.register("system.compile.cleanup", function(v) directive_cleanup = v end) -local directive_strip = false directives.register("system.compile.strip", function(v) directive_strip = v end) - -local compile = utilities.lua.compile +-- original size: 14075, stripped down to: 10764 +if not modules then modules={} end modules ['data-tmp']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat +local serialize,serializetofile=table.serialize,table.tofile +local mkdirs,isdir=dir.mkdirs,lfs.isdir +local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local report_caches=logs.reporter("resolvers","caches") +local report_resolvers=logs.reporter("resolvers","caching") +local resolvers=resolvers +local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) +local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) +local compile=utilities.lua.compile function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup == nil then cleanup = directive_cleanup end - if strip == nil then strip = directive_strip end - return compile(luafile,lucfile,cleanup,strip) -end - --- end of intermezzo - -caches = caches or { } -local caches = caches - -local luasuffixes = utilities.lua.suffixes - -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.tree = false -caches.force = true -caches.ask = false -caches.relocate = false -caches.defaults = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } - -local writable, readables, usedreadables = nil, { }, { } - --- we could use a metatable for writable and readable but not yet - + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compile(luafile,lucfile,cleanup,strip) +end +caches=caches or {} +local caches=caches +local luasuffixes=utilities.lua.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" +caches.direct=false +caches.tree=false +caches.force=true +caches.ask=false +caches.relocate=false +caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } +local writable,readables,usedreadables=nil,{},{} local function identify() - -- Combining the loops makes it messy. First we check the format cache path - -- and when the last component is not present we try to create it. - local texmfcaches = resolvers.cleanpathlist("TEXMFCACHE") - if texmfcaches then - for k=1,#texmfcaches do - local cachepath = texmfcaches[k] - if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) - cachepath = file.collapsepath(cachepath) - local valid = isdir(cachepath) - if valid then - if is_readable(cachepath) then - readables[#readables+1] = cachepath - if not writable and is_writable(cachepath) then - writable = cachepath - end - end - elseif not writable and caches.force then - local cacheparent = file.dirname(cachepath) - if is_writable(cacheparent) and true then -- we go on anyway (needed for mojca's kind of paths) - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("created: %s",cachepath) - writable = cachepath - readables[#readables+1] = cachepath - end - end - end - end + local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + cachepath=file.collapsepath(cachepath) + local valid=isdir(cachepath) + if valid then + if is_readable(cachepath) then + readables[#readables+1]=cachepath + if not writable and is_writable(cachepath) then + writable=cachepath end - end - end - -- As a last resort we check some temporary paths but this time we don't - -- create them. - local texmfcaches = caches.defaults - if texmfcaches then - for k=1,#texmfcaches do - local cachepath = texmfcaches[k] - cachepath = resolvers.expansion(cachepath) -- was getenv - if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) - local valid = isdir(cachepath) - if valid and is_readable(cachepath) then - if not writable and is_writable(cachepath) then - readables[#readables+1] = cachepath - writable = cachepath - break - end - end + end + elseif not writable and caches.force then + local cacheparent=file.dirname(cachepath) + if is_writable(cacheparent) and true then + if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("created: %s",cachepath) + writable=cachepath + readables[#readables+1]=cachepath + end end + end end + end end - -- Some extra checking. If we have no writable or readable path then we simply - -- quit. - if not writable then - report_caches("fatal error: there is no valid writable cache path defined") - os.exit() - elseif #readables == 0 then - report_caches("fatal error: there is no valid readable cache path defined") - os.exit() - end - -- why here - writable = dir.expandname(resolvers.cleanpath(writable)) -- just in case - -- moved here - local base, more, tree = caches.base, caches.more, caches.tree or caches.treehash() -- we have only one writable tree - if tree then - caches.tree = tree - writable = mkdirs(writable,base,more,tree) - for i=1,#readables do - readables[i] = file.join(readables[i],base,more,tree) - end - else - writable = mkdirs(writable,base,more) - for i=1,#readables do - readables[i] = file.join(readables[i],base,more) + end + local texmfcaches=caches.defaults + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + cachepath=resolvers.expansion(cachepath) + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + local valid=isdir(cachepath) + if valid and is_readable(cachepath) then + if not writable and is_writable(cachepath) then + readables[#readables+1]=cachepath + writable=cachepath + break + end end + end end - -- end - if trace_cache then - for i=1,#readables do - report_caches("using readable path '%s' (order %s)",readables[i],i) - end - report_caches("using writable path '%s'",writable) + end + if not writable then + report_caches("fatal error: there is no valid writable cache path defined") + os.exit() + elseif #readables==0 then + report_caches("fatal error: there is no valid readable cache path defined") + os.exit() + end + writable=dir.expandname(resolvers.cleanpath(writable)) + local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + if tree then + caches.tree=tree + writable=mkdirs(writable,base,more,tree) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more,tree) + end + else + writable=mkdirs(writable,base,more) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more) end - identify = function() - return writable, readables + end + if trace_cache then + for i=1,#readables do + report_caches("using readable path '%s' (order %s)",readables[i],i) end - return writable, readables + report_caches("using writable path '%s'",writable) + end + identify=function() + return writable,readables + end + return writable,readables end - function caches.usedpaths() - local writable, readables = identify() - if #readables > 1 then - local result = { } - for i=1,#readables do - local readable = readables[i] - if usedreadables[i] or readable == writable then - result[#result+1] = format("readable: '%s' (order %s)",readable,i) - end - end - result[#result+1] = format("writable: '%s'",writable) - return result - else - return writable + local writable,readables=identify() + if #readables>1 then + local result={} + for i=1,#readables do + local readable=readables[i] + if usedreadables[i] or readable==writable then + result[#result+1]=format("readable: '%s' (order %s)",readable,i) + end end + result[#result+1]=format("writable: '%s'",writable) + return result + else + return writable + end end - function caches.configfiles() - return concat(resolvers.instance.specification,";") + return concat(resolvers.instance.specification,";") end - function caches.hashed(tree) - tree = gsub(tree,"[\\/]+$","") - tree = lower(tree) - local hash = md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %s, hash %s",tree,hash) - end - return hash + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5.hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %s, hash %s",tree,hash) + end + return hash end - function caches.treehash() - local tree = caches.configfiles() - if not tree or tree == "" then - return false - else - return caches.hashed(tree) - end + local tree=caches.configfiles() + if not tree or tree=="" then + return false + else + return caches.hashed(tree) + end end - -local r_cache, w_cache = { }, { } -- normally w in in r but who cares - +local r_cache,w_cache={},{} local function getreadablepaths(...) - local tags = { ... } - local hash = concat(tags,"/") - local done = r_cache[hash] - if not done then - local writable, readables = identify() -- exit if not found - if #tags > 0 then - done = { } - for i=1,#readables do - done[i] = file.join(readables[i],...) - end - else - done = readables - end - r_cache[hash] = done + local tags={... } + local hash=concat(tags,"/") + local done=r_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done={} + for i=1,#readables do + done[i]=file.join(readables[i],...) + end + else + done=readables end - return done + r_cache[hash]=done + end + return done end - local function getwritablepath(...) - local tags = { ... } - local hash = concat(tags,"/") - local done = w_cache[hash] - if not done then - local writable, readables = identify() -- exit if not found - if #tags > 0 then - done = mkdirs(writable,...) - else - done = writable - end - w_cache[hash] = done - end - return done -end - -caches.getreadablepaths = getreadablepaths -caches.getwritablepath = getwritablepath - + local tags={... } + local hash=concat(tags,"/") + local done=w_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done=mkdirs(writable,...) + else + done=writable + end + w_cache[hash]=done + end + return done +end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath function caches.getfirstreadablefile(filename,...) - local rd = getreadablepaths(...) - for i=1,#rd do - local path = rd[i] - local fullname = file.join(path,filename) - if is_readable(fullname) then - usedreadables[i] = true - return fullname, path - end + local rd=getreadablepaths(...) + for i=1,#rd do + local path=rd[i] + local fullname=file.join(path,filename) + if is_readable(fullname) then + usedreadables[i]=true + return fullname,path end - return caches.setfirstwritablefile(filename,...) + end + return caches.setfirstwritablefile(filename,...) end - function caches.setfirstwritablefile(filename,...) - local wr = getwritablepath(...) - local fullname = file.join(wr,filename) - return fullname, wr + local wr=getwritablepath(...) + local fullname=file.join(wr,filename) + return fullname,wr end - -function caches.define(category,subcategory) -- for old times sake - return function() - return getwritablepath(category,subcategory) - end +function caches.define(category,subcategory) + return function() + return getwritablepath(category,subcategory) + end end - function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma), format("%s/%s.%s",path,name,luasuffixes.tmc) + return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) end - function caches.loaddata(readables,name) - if type(readables) == "string" then - readables = { readables } - end - for i=1,#readables do - local path = readables[i] - local tmaname, tmcname = caches.setluanames(path,name) - local loader = loadfile(tmcname) - if not loader then - -- in case we have a different engine - utilities.lua.compile(tmaname,tmcname) - -- - loader = loadfile(tmaname) - end - if loader then - loader = loader() - collectgarbage("step") - return loader - end - end - return false + if type(readables)=="string" then + readables={ readables } + end + for i=1,#readables do + local path=readables[i] + local tmaname,tmcname=caches.setluanames(path,name) + local loader=loadfile(tmcname) + if not loader then + utilities.lua.compile(tmaname,tmcname) + loader=loadfile(tmaname) + end + if loader then + loader=loader() + collectgarbage("step") + return loader + end + end + return false end - function caches.is_writable(filepath,filename) - local tmaname, tmcname = caches.setluanames(filepath,filename) - return is_writable(tmaname) + local tmaname,tmcname=caches.setluanames(filepath,filename) + return is_writable(tmaname) end - -local saveoptions = { compact = true } - --- add some point we will only use the internal bytecode compiler and --- then we can flag success in the tma so that it can trigger a compile --- if the other engine - +local saveoptions={ compact=true } function caches.savedata(filepath,filename,data,raw) - local tmaname, tmcname = caches.setluanames(filepath,filename) - local reduce, simplify = true, true - if raw then - reduce, simplify = false, false - end - data.cache_uuid = os.uuid() - if caches.direct then - file.savedata(tmaname,serialize(data,true,saveoptions)) - else - serializetofile(tmaname,data,true,saveoptions) - end - utilities.lua.compile(tmaname,tmcname) -end - --- moved from data-res: - -local content_state = { } - + local tmaname,tmcname=caches.setluanames(filepath,filename) + local reduce,simplify=true,true + if raw then + reduce,simplify=false,false + end + data.cache_uuid=os.uuid() + if caches.direct then + file.savedata(tmaname,serialize(data,true,saveoptions)) + else + serializetofile(tmaname,data,true,saveoptions) + end + utilities.lua.compile(tmaname,tmcname) +end +local content_state={} function caches.contentstate() - return content_state or { } + return content_state or {} end - function caches.loadcontent(cachename,dataname) - local name = caches.hashed(cachename) - local full, path = caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename = file.join(path,name) - local blob = loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) - if blob then - local data = blob() - if data and data.content then - if data.type == dataname then - if data.version == resolvers.cacheversion then - content_state[#content_state+1] = data.uuid - if trace_locating then - report_resolvers("loading '%s' for '%s' from '%s'",dataname,cachename,filename) - end - return data.content - else - report_resolvers("skipping '%s' for '%s' from '%s' (version mismatch)",dataname,cachename,filename) - end - else - report_resolvers("skipping '%s' for '%s' from '%s' (datatype mismatch)",dataname,cachename,filename) - end - elseif trace_locating then - report_resolvers("skipping '%s' for '%s' from '%s' (no content)",dataname,cachename,filename) - end - elseif trace_locating then - report_resolvers("skipping '%s' for '%s' from '%s' (invalid file)",dataname,cachename,filename) - end -end - -function caches.collapsecontent(content) - for k, v in next, content do - if type(v) == "table" and #v == 1 then - content[k] = v[1] - end - end -end - -function caches.savecontent(cachename,dataname,content) - local name = caches.hashed(cachename) - local full, path = caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename = file.join(path,name) -- is full - local luaname = addsuffix(filename,luasuffixes.lua) - local lucname = addsuffix(filename,luasuffixes.luc) - if trace_locating then - report_resolvers("preparing '%s' for '%s'",dataname,cachename) - end - local data = { - type = dataname, - root = cachename, - version = resolvers.cacheversion, - date = os.date("%Y-%m-%d"), - time = os.date("%H:%M:%S"), - content = content, - uuid = os.uuid(), - } - local ok = io.savedata(luaname,serialize(data,true)) - if ok then - if trace_locating then - report_resolvers("category '%s', cachename '%s' saved in '%s'",dataname,cachename,luaname) - end - if utilities.lua.compile(luaname,lucname) then - if trace_locating then - report_resolvers("'%s' compiled to '%s'",dataname,lucname) - end - return true + local name=caches.hashed(cachename) + local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + local filename=file.join(path,name) + local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) + if blob then + local data=blob() + if data and data.content then + if data.type==dataname then + if data.version==resolvers.cacheversion then + content_state[#content_state+1]=data.uuid + if trace_locating then + report_resolvers("loading '%s' for '%s' from '%s'",dataname,cachename,filename) + end + return data.content else - if trace_locating then - report_resolvers("compiling failed for '%s', deleting file '%s'",dataname,lucname) - end - os.remove(lucname) + report_resolvers("skipping '%s' for '%s' from '%s' (version mismatch)",dataname,cachename,filename) end + else + report_resolvers("skipping '%s' for '%s' from '%s' (datatype mismatch)",dataname,cachename,filename) + end elseif trace_locating then - report_resolvers("unable to save '%s' in '%s' (access error)",dataname,luaname) + report_resolvers("skipping '%s' for '%s' from '%s' (no content)",dataname,cachename,filename) end + elseif trace_locating then + report_resolvers("skipping '%s' for '%s' from '%s' (invalid file)",dataname,cachename,filename) + end end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-met'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local find, format = string.find, string.format -local sequenced = table.sequenced -local addurlscheme, urlhashed = url.addscheme, url.hashed - -local trace_locating = false - -trackers.register("resolvers.locating", function(v) trace_methods = v end) -trackers.register("resolvers.methods", function(v) trace_methods = v end) - - -local report_methods = logs.reporter("resolvers","methods") - -local allocate = utilities.storage.allocate - -local resolvers = resolvers - -local registered = { } - -local function splitmethod(filename) -- todo: filetype in specification - if not filename then - return { scheme = "unknown", original = filename } - end - if type(filename) == "table" then - return filename -- already split - end - filename = file.collapsepath(filename) - if not find(filename,"://") then - return { scheme = "file", path = filename, original = filename, filename = filename } - end - local specification = url.hashed(filename) - if not specification.scheme or specification.scheme == "" then - return { scheme = "file", path = filename, original = filename, filename = filename } - else - return specification +function caches.collapsecontent(content) + for k,v in next,content do + if type(v)=="table" and #v==1 then + content[k]=v[1] end + end end - -resolvers.splitmethod = splitmethod -- bad name but ok - --- the second argument is always analyzed (saves time later on) and the original --- gets passed as original but also as argument - -local function methodhandler(what,first,...) -- filename can be nil or false - local method = registered[what] - if method then - local how, namespace = method.how, method.namespace - if how == "uri" or how == "url" then - local specification = splitmethod(first) - local scheme = specification.scheme - local resolver = namespace and namespace[scheme] - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, scheme=%s, argument=%s",what,how,scheme,first) - end - return resolver(specification,...) - else - resolver = namespace.default or namespace.file - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, default, argument=%s",what,how,first) - end - return resolver(specification,...) - elseif trace_methods then - report_methods("resolver: method=%s, how=%s, no handler",what,how) - end - end - elseif how == "tag" then - local resolver = namespace and namespace[first] - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, tag=%s",what,how,first) - end - return resolver(...) - else - resolver = namespace.default or namespace.file - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, default",what,how) - end - return resolver(...) - elseif trace_methods then - report_methods("resolver: method=%s, how=%s, unknown",what,how) - end - end - end +function caches.savecontent(cachename,dataname,content) + local name=caches.hashed(cachename) + local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + local filename=file.join(path,name) + local luaname=addsuffix(filename,luasuffixes.lua) + local lucname=addsuffix(filename,luasuffixes.luc) + if trace_locating then + report_resolvers("preparing '%s' for '%s'",dataname,cachename) + end + local data={ + type=dataname, + root=cachename, + version=resolvers.cacheversion, + date=os.date("%Y-%m-%d"), + time=os.date("%H:%M:%S"), + content=content, + uuid=os.uuid(), + } + local ok=io.savedata(luaname,serialize(data,true)) + if ok then + if trace_locating then + report_resolvers("category '%s', cachename '%s' saved in '%s'",dataname,cachename,luaname) + end + if utilities.lua.compile(luaname,lucname) then + if trace_locating then + report_resolvers("'%s' compiled to '%s'",dataname,lucname) + end + return true else - report_methods("resolver: method=%s, unknown",what) + if trace_locating then + report_resolvers("compiling failed for '%s', deleting file '%s'",dataname,lucname) + end + os.remove(lucname) end + elseif trace_locating then + report_resolvers("unable to save '%s' in '%s' (access error)",dataname,luaname) + end end -resolvers.methodhandler = methodhandler - -function resolvers.registermethod(name,namespace,how) - registered[name] = { how = how or "tag", namespace = namespace } - namespace["byscheme"] = function(scheme,filename,...) - if scheme == "file" then - return methodhandler(name,filename,...) - else - return methodhandler(name,addurlscheme(filename,scheme),...) - end - end -end -local concatinators = allocate { notfound = file.join } -- concatinate paths -local locators = allocate { notfound = function() end } -- locate databases -local hashers = allocate { notfound = function() end } -- load databases -local generators = allocate { notfound = function() end } -- generate databases +end -- of closure -resolvers.concatinators = concatinators -resolvers.locators = locators -resolvers.hashers = hashers -resolvers.generators = generators +do -- create closure to overcome 200 locals limit -local registermethod = resolvers.registermethod +-- original size: 4863, stripped down to: 3890 +if not modules then modules={} end modules ['data-met']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,format=string.find,string.format +local sequenced=table.sequenced +local addurlscheme,urlhashed=url.addscheme,url.hashed +local trace_locating=false +trackers.register("resolvers.locating",function(v) trace_methods=v end) +trackers.register("resolvers.methods",function(v) trace_methods=v end) +local report_methods=logs.reporter("resolvers","methods") +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registered={} +local function splitmethod(filename) + if not filename then + return { scheme="unknown",original=filename } + end + if type(filename)=="table" then + return filename + end + filename=file.collapsepath(filename) + if not find(filename,"://") then + return { scheme="file",path=filename,original=filename,filename=filename } + end + local specification=url.hashed(filename) + if not specification.scheme or specification.scheme=="" then + return { scheme="file",path=filename,original=filename,filename=filename } + else + return specification + end +end +resolvers.splitmethod=splitmethod +local function methodhandler(what,first,...) + local method=registered[what] + if method then + local how,namespace=method.how,method.namespace + if how=="uri" or how=="url" then + local specification=splitmethod(first) + local scheme=specification.scheme + local resolver=namespace and namespace[scheme] + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, scheme=%s, argument=%s",what,how,scheme,first) + end + return resolver(specification,...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, default, argument=%s",what,how,first) + end + return resolver(specification,...) + elseif trace_methods then + report_methods("resolver: method=%s, how=%s, no handler",what,how) + end + end + elseif how=="tag" then + local resolver=namespace and namespace[first] + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, tag=%s",what,how,first) + end + return resolver(...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, default",what,how) + end + return resolver(...) + elseif trace_methods then + report_methods("resolver: method=%s, how=%s, unknown",what,how) + end + end + end + else + report_methods("resolver: method=%s, unknown",what) + end +end +resolvers.methodhandler=methodhandler +function resolvers.registermethod(name,namespace,how) + registered[name]={ how=how or "tag",namespace=namespace } + namespace["byscheme"]=function(scheme,filename,...) + if scheme=="file" then + return methodhandler(name,filename,...) + else + return methodhandler(name,addurlscheme(filename,scheme),...) + end + end +end +local concatinators=allocate { notfound=file.join } +local locators=allocate { notfound=function() end } +local hashers=allocate { notfound=function() end } +local generators=allocate { notfound=function() end } +resolvers.concatinators=concatinators +resolvers.locators=locators +resolvers.hashers=hashers +resolvers.generators=generators +local registermethod=resolvers.registermethod registermethod("concatinators",concatinators,"tag") -registermethod("locators", locators, "uri") -registermethod("hashers", hashers, "uri") -registermethod("generators", generators, "uri") +registermethod("locators",locators,"uri") +registermethod("hashers",hashers,"uri") +registermethod("generators",generators,"uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-res'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - --- In practice we will work within one tds tree, but i want to keep --- the option open to build tools that look at multiple trees, which is --- why we keep the tree specific data in a table. We used to pass the --- instance but for practical purposes we now avoid this and use a --- instance variable. We always have one instance active (sort of global). - --- todo: cache:/// home:/// selfautoparent:/// (sometime end 2012) - -local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch -local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys -local next, type, rawget = next, type, rawget -local os = os - -local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns - -local filedirname = file.dirname -local filebasename = file.basename -local suffixonly = file.suffixonly -local filejoin = file.join -local collapsepath = file.collapsepath -local joinpath = file.joinpath -local allocate = utilities.storage.allocate -local settings_to_array = utilities.parsers.settings_to_array -local setmetatableindex = table.setmetatableindex -local luasuffixes = utilities.lua.suffixes - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_resolving = logs.reporter("resolvers","resolving") - -local resolvers = resolvers - -local expandedpathfromlist = resolvers.expandedpathfromlist -local checkedvariable = resolvers.checkedvariable -local splitconfigurationpath = resolvers.splitconfigurationpath -local methodhandler = resolvers.methodhandler - -local initializesetter = utilities.setters.initialize - -local ostype, osname, osenv, ossetenv, osgetenv = os.type, os.name, os.env, os.setenv, os.getenv - -resolvers.cacheversion = '1.0.1' -resolvers.configbanner = '' -resolvers.homedir = environment.homedir -resolvers.criticalvars = allocate { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF", "TEXMF", "TEXOS" } -resolvers.luacnfname = "texmfcnf.lua" -resolvers.luacnfstate = "unknown" - --- The web2c tex binaries as well as kpse have built in paths for the configuration --- files and there can be a depressing truckload of them. This is actually the weak --- spot of a distribution. So we don't want: --- --- resolvers.luacnfspec = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}' --- --- but instead use: --- --- resolvers.luacnfspec = 'selfautoparent:{/texmf{-local,}{,/web2c}}' --- --- which does not make texlive happy as there is a texmf-local tree one level up --- (sigh), so we need this. We can assume web2c as mkiv does not run on older --- texlives anyway. --- --- texlive: --- --- selfautodir: --- selfautoparent: --- selfautodir:share/texmf-local/web2c --- selfautodir:share/texmf/web2c --- selfautodir:texmf-local/web2c --- selfautodir:texmf/web2c --- selfautoparent:share/texmf-local/web2c --- selfautoparent:share/texmf/web2c --- selfautoparent:texmf-local/web2c --- selfautoparent:texmf/web2c --- --- minimals: --- --- home:texmf/web2c --- selfautoparent:texmf-local/web2c --- selfautoparent:texmf-context/web2c --- selfautoparent:texmf/web2c +-- original size: 60360, stripped down to: 42573 +if not modules then modules={} end modules ['data-res']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local format,gsub,find,lower,upper,match,gmatch=string.format,string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch +local concat,insert,sortedkeys=table.concat,table.insert,table.sortedkeys +local next,type,rawget=next,type,rawget +local os=os +local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local filedirname=file.dirname +local filebasename=file.basename +local suffixonly=file.suffixonly +local filejoin=file.join +local collapsepath=file.collapsepath +local joinpath=file.joinpath +local allocate=utilities.storage.allocate +local settings_to_array=utilities.parsers.settings_to_array +local setmetatableindex=table.setmetatableindex +local luasuffixes=utilities.lua.suffixes +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_resolving=logs.reporter("resolvers","resolving") +local resolvers=resolvers +local expandedpathfromlist=resolvers.expandedpathfromlist +local checkedvariable=resolvers.checkedvariable +local splitconfigurationpath=resolvers.splitconfigurationpath +local methodhandler=resolvers.methodhandler +local initializesetter=utilities.setters.initialize +local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os.getenv +resolvers.cacheversion='1.0.1' +resolvers.configbanner='' +resolvers.homedir=environment.homedir +resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } +resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnfstate="unknown" if environment.default_texmfcnf then - -- unfortunately we now have quite some overkill in the spec (not so nice on a network) - resolvers.luacnfspec = environment.default_texmfcnf + resolvers.luacnfspec=environment.default_texmfcnf else - -- resolvers.luacnfspec = "selfautoparent:texmf{-local,-context,}/web2c" - resolvers.luacnfspec = "{selfautoloc:,selfautodir:,selfautoparent:}{,/texmf{-local,}/web2c}" -end - -resolvers.luacnfspec = 'home:texmf/web2c;' .. resolvers.luacnfspec - --- which (as we want users to use the web2c path) be can be simplified to this: --- --- if environment and environment.ownpath and string.find(environment.ownpath,"[\\/]texlive[\\/]") then --- resolvers.luacnfspec = 'selfautodir:/texmf-local/web2c,selfautoparent:/texmf-local/web2c,selfautoparent:/texmf/web2c' --- else --- resolvers.luacnfspec = 'selfautoparent:/texmf-local/web2c,selfautoparent:/texmf/web2c' --- end - - - -local unset_variable = "unset" - -local formats = resolvers.formats -local suffixes = resolvers.suffixes -local dangerous = resolvers.dangerous -local suffixmap = resolvers.suffixmap - -resolvers.defaultsuffixes = { "tex" } -- "mkiv", "cld" -- too tricky - -resolvers.instance = resolvers.instance or nil -- the current one (slow access) -local instance = resolvers.instance or nil -- the current one (fast access) - --- An instance has an environment (coming from the outside, kept raw), variables --- (coming from the configuration file), and expansions (variables with nested --- variables replaced). One can push something into the outer environment and --- its internal copy, but only the later one will be the raw unprefixed variant. - + resolvers.luacnfspec="{selfautoloc:,selfautodir:,selfautoparent:}{,/texmf{-local,}/web2c}" +end +resolvers.luacnfspec='home:texmf/web2c;'..resolvers.luacnfspec +local unset_variable="unset" +local formats=resolvers.formats +local suffixes=resolvers.suffixes +local dangerous=resolvers.dangerous +local suffixmap=resolvers.suffixmap +resolvers.defaultsuffixes={ "tex" } +resolvers.instance=resolvers.instance or nil +local instance=resolvers.instance or nil function resolvers.setenv(key,value,raw) - if instance then - -- this one will be consulted first when we stay inside - -- the current environment (prefixes are not resolved here) - instance.environment[key] = value - -- we feed back into the environment, and as this is used - -- by other applications (via os.execute) we need to make - -- sure that prefixes are resolve - ossetenv(key,raw and value or resolvers.resolve(value)) - end + if instance then + instance.environment[key]=value + ossetenv(key,raw and value or resolvers.resolve(value)) + end end - --- Beware we don't want empty here as this one can be called early on --- and therefore we use rawget. - local function getenv(key) - local value = rawget(instance.environment,key) - if value and value ~= "" then - return value - else - local e = osgetenv(key) - return e ~= nil and e ~= "" and checkedvariable(e) or "" - end -end - -resolvers.getenv = getenv -resolvers.env = getenv - --- We are going to use some metatable trickery where we backtrack from --- expansion to variable to environment. - + local value=rawget(instance.environment,key) + if value and value~="" then + return value + else + local e=osgetenv(key) + return e~=nil and e~="" and checkedvariable(e) or "" + end +end +resolvers.getenv=getenv +resolvers.env=getenv local function resolve(k) - return instance.expansions[k] -end - -local dollarstripper = lpeg.stripper("$") -local inhibitstripper = P("!")^0 * Cs(P(1)^0) -local backslashswapper = lpeg.replacer("\\","/") - -local somevariable = P("$") / "" -local somekey = C(R("az","AZ","09","__","--")^1) -local somethingelse = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "") - + P(";") * (P(";") / "") - + P(1) -local variableexpander = Cs( (somevariable * (somekey/resolve) + somethingelse)^1 ) - -local cleaner = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";" -local variablecleaner = Cs((cleaner + P(1))^0) - -local somevariable = R("az","AZ","09","__","--")^1 / resolve -local variable = (P("$")/"") * (somevariable + (P("{")/"") * somevariable * (P("}")/"")) -local variableresolver = Cs((variable + P(1))^0) - + return instance.expansions[k] +end +local dollarstripper=lpeg.stripper("$") +local inhibitstripper=P("!")^0*Cs(P(1)^0) +local backslashswapper=lpeg.replacer("\\","/") +local somevariable=P("$")/"" +local somekey=C(R("az","AZ","09","__","--")^1) +local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) +local variableexpander=Cs((somevariable*(somekey/resolve)+somethingelse)^1 ) +local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" +local variablecleaner=Cs((cleaner+P(1))^0) +local somevariable=R("az","AZ","09","__","--")^1/resolve +local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) +local variableresolver=Cs((variable+P(1))^0) local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end - -function resolvers.newinstance() -- todo: all vars will become lowercase and alphanum only - - if trace_locating then - report_resolving("creating instance") - end - - local environment, variables, expansions, order = allocate(), allocate(), allocate(), allocate() - - local newinstance = { - environment = environment, - variables = variables, - expansions = expansions, - order = order, - files = allocate(), - setups = allocate(), - found = allocate(), - foundintrees = allocate(), - hashes = allocate(), - hashed = allocate(), - specification = allocate(), - lists = allocate(), - data = allocate(), -- only for loading - fakepaths = allocate(), - remember = true, - diskcache = true, - renewcache = false, - renewtree = false, - loaderror = false, - savelists = true, - pattern = nil, -- lists - force_suffixes = true, - } - - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v = order[i][k] - if v ~= nil then - t[k] = v - return v - end - end - if v == nil then - v = "" - end - t[k] = v - return v - end) - - setmetatableindex(environment, function(t,k) - local v = osgetenv(k) - if v == nil then - v = variables[k] - end - if v ~= nil then - v = checkedvariable(v) or "" - end - v = resolvers.repath(v) -- for taco who has a : separated osfontdir - t[k] = v - return v - end) - - setmetatableindex(expansions, function(t,k) - local v = environment[k] - if type(v) == "string" then - v = lpegmatch(variableresolver,v) - v = lpegmatch(variablecleaner,v) - end - t[k] = v + return lpegmatch(variableexpander,var) or var +end +function resolvers.newinstance() + if trace_locating then + report_resolving("creating instance") + end + local environment,variables,expansions,order=allocate(),allocate(),allocate(),allocate() + local newinstance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files=allocate(), + setups=allocate(), + found=allocate(), + foundintrees=allocate(), + hashes=allocate(), + hashed=allocate(), + specification=allocate(), + lists=allocate(), + data=allocate(), + fakepaths=allocate(), + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v return v - end) - - return newinstance - + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end + if v~=nil then + v=checkedvariable(v) or "" + end + v=resolvers.repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + return newinstance end - -function resolvers.setinstance(someinstance) -- only one instance is active - instance = someinstance - resolvers.instance = someinstance - return someinstance +function resolvers.setinstance(someinstance) + instance=someinstance + resolvers.instance=someinstance + return someinstance end - function resolvers.reset() - return resolvers.setinstance(resolvers.newinstance()) + return resolvers.setinstance(resolvers.newinstance()) end - local function reset_hashes() - instance.lists = { } - instance.found = { } -end - -local slash = P("/") - -local pathexpressionpattern = Cs ( - Cc("^") * ( - Cc("%") * S(".-") - + slash^2 * P(-1) / "/.*" - + slash^2 / "/.-/" - + (1-slash) * P(-1) * Cc("/") - + P(1) - )^1 * Cc("$") -- yes or no $ + instance.lists={} + instance.found={} +end +local slash=P("/") +local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*"+slash^2/"/.-/"+(1-slash)*P(-1)*Cc("/")+P(1) + )^1*Cc("$") ) - -local cache = { } - +local cache={} local function makepathexpression(str) - if str == "." then - return "^%./$" - else - local c = cache[str] - if not c then - c = lpegmatch(pathexpressionpattern,str) - cache[str] = c - end - return c + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c end + return c + end end - local function reportcriticalvariables(cnfspec) - if trace_locating then - for i=1,#resolvers.criticalvars do - local k = resolvers.criticalvars[i] - local v = resolvers.getenv(k) or "unknown" -- this one will not resolve ! - report_resolving("variable '%s' set to '%s'",k,v) - end - report_resolving() - if cnfspec then - if type(cnfspec) == "table" then - report_resolving("using configuration specification '%s'",concat(cnfspec,",")) - else - report_resolving("using configuration specification '%s'",cnfspec) - end - end - report_resolving() + if trace_locating then + for i=1,#resolvers.criticalvars do + local k=resolvers.criticalvars[i] + local v=resolvers.getenv(k) or "unknown" + report_resolving("variable '%s' set to '%s'",k,v) + end + report_resolving() + if cnfspec then + if type(cnfspec)=="table" then + report_resolving("using configuration specification '%s'",concat(cnfspec,",")) + else + report_resolving("using configuration specification '%s'",cnfspec) + end end - reportcriticalvariables = function() end + report_resolving() + end + reportcriticalvariables=function() end end - local function identify_configuration_files() - local specification = instance.specification - if #specification == 0 then - local cnfspec = getenv("TEXMFCNF") - if cnfspec == "" then - cnfspec = resolvers.luacnfspec - resolvers.luacnfstate = "default" - else - resolvers.luacnfstate = "environment" - end - reportcriticalvariables(cnfspec) - local cnfpaths = expandedpathfromlist(resolvers.splitpath(cnfspec)) - local luacnfname = resolvers.luacnfname - for i=1,#cnfpaths do - local filename = collapsepath(filejoin(cnfpaths[i],luacnfname)) - local realname = resolvers.resolve(filename) - if lfs.isfile(realname) then - specification[#specification+1] = filename - if trace_locating then - report_resolving("found configuration file '%s'",realname) - end - elseif trace_locating then - report_resolving("unknown configuration file '%s'",realname) - end - end + local specification=instance.specification + if #specification==0 then + local cnfspec=getenv("TEXMFCNF") + if cnfspec=="" then + cnfspec=resolvers.luacnfspec + resolvers.luacnfstate="default" + else + resolvers.luacnfstate="environment" + end + reportcriticalvariables(cnfspec) + local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local luacnfname=resolvers.luacnfname + for i=1,#cnfpaths do + local filename=collapsepath(filejoin(cnfpaths[i],luacnfname)) + local realname=resolvers.resolve(filename) + if lfs.isfile(realname) then + specification[#specification+1]=filename if trace_locating then - report_resolving() + report_resolving("found configuration file '%s'",realname) end - elseif trace_locating then - report_resolving("configuration files already identified") + elseif trace_locating then + report_resolving("unknown configuration file '%s'",realname) + end end + if trace_locating then + report_resolving() + end + elseif trace_locating then + report_resolving("configuration files already identified") + end end - local function load_configuration_files() - local specification = instance.specification - if #specification > 0 then - local luacnfname = resolvers.luacnfname - for i=1,#specification do - local filename = specification[i] - local pathname = filedirname(filename) - local filename = filejoin(pathname,luacnfname) - local realname = resolvers.resolve(filename) -- no shortcut - local blob = loadfile(realname) - if blob then - local setups = instance.setups - local data = blob() - local parent = data and data.parent - if parent then - local filename = filejoin(pathname,parent) - local realname = resolvers.resolve(filename) -- no shortcut - local blob = loadfile(realname) - if blob then - local parentdata = blob() - if parentdata then - report_resolving("loading configuration file '%s'",filename) - data = table.merged(parentdata,data) - end - end - end - data = data and data.content - if data then - if trace_locating then - report_resolving("loading configuration file '%s'",filename) - report_resolving() - end - local variables = data.variables or { } - local warning = false - for k, v in next, data do - local variant = type(v) - if variant == "table" then - initializesetter(filename,k,v) - elseif variables[k] == nil then - if trace_locating and not warning then - report_resolving("variables like '%s' in configuration file '%s' should move to the 'variables' subtable", - k,resolvers.resolve(filename)) - warning = true - end - variables[k] = v - end - end - setups[pathname] = variables - if resolvers.luacnfstate == "default" then - -- the following code is not tested - local cnfspec = variables["TEXMFCNF"] - if cnfspec then - if trace_locating then - report_resolving("reloading configuration due to TEXMF redefinition") - end - -- we push the value into the main environment (osenv) so - -- that it takes precedence over the default one and therefore - -- also over following definitions - resolvers.setenv("TEXMFCNF",cnfspec) -- resolves prefixes - -- we now identify and load the specified configuration files - instance.specification = { } - identify_configuration_files() - load_configuration_files() - -- we prevent further overload of the configuration variable - resolvers.luacnfstate = "configuration" - -- we quit the outer loop - break - end - end - - else - if trace_locating then - report_resolving("skipping configuration file '%s' (no content)",filename) - end - setups[pathname] = { } - instance.loaderror = true - end - elseif trace_locating then - report_resolving("skipping configuration file '%s' (no valid format)",filename) + local specification=instance.specification + if #specification>0 then + local luacnfname=resolvers.luacnfname + for i=1,#specification do + local filename=specification[i] + local pathname=filedirname(filename) + local filename=filejoin(pathname,luacnfname) + local realname=resolvers.resolve(filename) + local blob=loadfile(realname) + if blob then + local setups=instance.setups + local data=blob() + local parent=data and data.parent + if parent then + local filename=filejoin(pathname,parent) + local realname=resolvers.resolve(filename) + local blob=loadfile(realname) + if blob then + local parentdata=blob() + if parentdata then + report_resolving("loading configuration file '%s'",filename) + data=table.merged(parentdata,data) + end + end + end + data=data and data.content + if data then + if trace_locating then + report_resolving("loading configuration file '%s'",filename) + report_resolving() + end + local variables=data.variables or {} + local warning=false + for k,v in next,data do + local variant=type(v) + if variant=="table" then + initializesetter(filename,k,v) + elseif variables[k]==nil then + if trace_locating and not warning then + report_resolving("variables like '%s' in configuration file '%s' should move to the 'variables' subtable", + k,resolvers.resolve(filename)) + warning=true + end + variables[k]=v end - instance.order[#instance.order+1] = instance.setups[pathname] - if instance.loaderror then - break + end + setups[pathname]=variables + if resolvers.luacnfstate=="default" then + local cnfspec=variables["TEXMFCNF"] + if cnfspec then + if trace_locating then + report_resolving("reloading configuration due to TEXMF redefinition") + end + resolvers.setenv("TEXMFCNF",cnfspec) + instance.specification={} + identify_configuration_files() + load_configuration_files() + resolvers.luacnfstate="configuration" + break end + end + else + if trace_locating then + report_resolving("skipping configuration file '%s' (no content)",filename) + end + setups[pathname]={} + instance.loaderror=true end - elseif trace_locating then - report_resolving("warning: no lua configuration files found") + elseif trace_locating then + report_resolving("skipping configuration file '%s' (no valid format)",filename) + end + instance.order[#instance.order+1]=instance.setups[pathname] + if instance.loaderror then + break + end end + elseif trace_locating then + report_resolving("warning: no lua configuration files found") + end end - --- scheme magic ... database loading - local function load_file_databases() - instance.loaderror, instance.files = false, allocate() - if not instance.renewcache then - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - resolvers.hashers.byscheme(hash.type,hash.name) - if instance.loaderror then break end - end + instance.loaderror,instance.files=false,allocate() + if not instance.renewcache then + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + resolvers.hashers.byscheme(hash.type,hash.name) + if instance.loaderror then break end end + end end - local function locate_file_databases() - -- todo: cache:// and tree:// (runtime) - local texmfpaths = resolvers.expandedpathlist("TEXMF") - if #texmfpaths > 0 then - for i=1,#texmfpaths do - local path = collapsepath(texmfpaths[i]) - path = gsub(path,"/+$","") -- in case $HOME expands to something with a trailing / - local stripped = lpegmatch(inhibitstripper,path) -- the !! thing - if stripped ~= "" then - local runtime = stripped == path - path = resolvers.cleanpath(path) - local spec = resolvers.splitmethod(stripped) - if runtime and (spec.noscheme or spec.scheme == "file") then - stripped = "tree:///" .. stripped - elseif spec.scheme == "cache" or spec.scheme == "file" then - stripped = spec.path - end - if trace_locating then - if runtime then - report_resolving("locating list of '%s' (runtime) (%s)",path,stripped) - else - report_resolving("locating list of '%s' (cached)",path) - end - end - methodhandler('locators',stripped) - end + local texmfpaths=resolvers.expandedpathlist("TEXMF") + if #texmfpaths>0 then + for i=1,#texmfpaths do + local path=collapsepath(texmfpaths[i]) + path=gsub(path,"/+$","") + local stripped=lpegmatch(inhibitstripper,path) + if stripped~="" then + local runtime=stripped==path + path=resolvers.cleanpath(path) + local spec=resolvers.splitmethod(stripped) + if runtime and (spec.noscheme or spec.scheme=="file") then + stripped="tree:///"..stripped + elseif spec.scheme=="cache" or spec.scheme=="file" then + stripped=spec.path end if trace_locating then - report_resolving() + if runtime then + report_resolving("locating list of '%s' (runtime) (%s)",path,stripped) + else + report_resolving("locating list of '%s' (cached)",path) + end end - elseif trace_locating then - report_resolving("no texmf paths are defined (using TEXMF)") - end -end - -local function generate_file_databases() - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - methodhandler('generators',hash.name) + methodhandler('locators',stripped) + end end if trace_locating then - report_resolving() + report_resolving() end + elseif trace_locating then + report_resolving("no texmf paths are defined (using TEXMF)") + end end - -local function save_file_databases() -- will become cachers - for i=1,#instance.hashes do - local hash = instance.hashes[i] - local cachename = hash.name - if hash.cache then - local content = instance.files[cachename] - caches.collapsecontent(content) - if trace_locating then - report_resolving("saving tree '%s'",cachename) - end - caches.savecontent(cachename,"files",content) - elseif trace_locating then - report_resolving("not saving runtime tree '%s'",cachename) - end +local function generate_file_databases() + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + methodhandler('generators',hash.name) + end + if trace_locating then + report_resolving() + end +end +local function save_file_databases() + for i=1,#instance.hashes do + local hash=instance.hashes[i] + local cachename=hash.name + if hash.cache then + local content=instance.files[cachename] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree '%s'",cachename) + end + caches.savecontent(cachename,"files",content) + elseif trace_locating then + report_resolving("not saving runtime tree '%s'",cachename) end + end end - function resolvers.renew(hashname) - if hashname and hashname ~= "" then - local expanded = resolvers.expansion(hashname) or "" - if expanded ~= "" then - if trace_locating then - report_resolving("identifying tree '%s' from '%s'",expanded,hashname) - end - hashname = expanded - else - if trace_locating then - report_resolving("identifying tree '%s'",hashname) - end - end - local realpath = resolvers.resolve(hashname) - if lfs.isdir(realpath) then - if trace_locating then - report_resolving("using path '%s'",realpath) - end - methodhandler('generators',hashname) - -- could be shared - local content = instance.files[hashname] - caches.collapsecontent(content) - if trace_locating then - report_resolving("saving tree '%s'",hashname) - end - caches.savecontent(hashname,"files",content) - -- till here - else - report_resolving("invalid path '%s'",realpath) - end + if hashname and hashname~="" then + local expanded=resolvers.expansion(hashname) or "" + if expanded~="" then + if trace_locating then + report_resolving("identifying tree '%s' from '%s'",expanded,hashname) + end + hashname=expanded + else + if trace_locating then + report_resolving("identifying tree '%s'",hashname) + end + end + local realpath=resolvers.resolve(hashname) + if lfs.isdir(realpath) then + if trace_locating then + report_resolving("using path '%s'",realpath) + end + methodhandler('generators',hashname) + local content=instance.files[hashname] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree '%s'",hashname) + end + caches.savecontent(hashname,"files",content) + else + report_resolving("invalid path '%s'",realpath) end + end end - local function load_databases() - locate_file_databases() - if instance.diskcache and not instance.renewcache then - load_file_databases() - if instance.loaderror then - generate_file_databases() - save_file_databases() - end - else - generate_file_databases() - if instance.renewcache then - save_file_databases() - end + locate_file_databases() + if instance.diskcache and not instance.renewcache then + load_file_databases() + if instance.loaderror then + generate_file_databases() + save_file_databases() end + else + generate_file_databases() + if instance.renewcache then + save_file_databases() + end + end end - function resolvers.appendhash(type,name,cache) - -- safeguard ... tricky as it's actually a bug when seen twice - if not instance.hashed[name] then - if trace_locating then - report_resolving("hash '%s' appended",name) - end - insert(instance.hashes, { type = type, name = name, cache = cache } ) - instance.hashed[name] = cache + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash '%s' appended",name) end + insert(instance.hashes,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end end - function resolvers.prependhash(type,name,cache) - -- safeguard ... tricky as it's actually a bug when seen twice - if not instance.hashed[name] then - if trace_locating then - report_resolving("hash '%s' prepended",name) - end - insert(instance.hashes, 1, { type = type, name = name, cache = cache } ) - instance.hashed[name] = cache - end -end - -function resolvers.extendtexmfvariable(specification) -- crap, we could better prepend the hash - local t = resolvers.splitpath(getenv("TEXMF")) -- okay? - insert(t,1,specification) - local newspec = concat(t,",") -- not ; - if instance.environment["TEXMF"] then - instance.environment["TEXMF"] = newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"] = newspec - else - -- weird - end - reset_hashes() + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash '%s' prepended",name) + end + insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end +end +function resolvers.extendtexmfvariable(specification) + local t=resolvers.splitpath(getenv("TEXMF")) + insert(t,1,specification) + local newspec=concat(t,",") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"]=newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"]=newspec + else + end + reset_hashes() end - function resolvers.splitexpansions() - local ie = instance.expansions - for k,v in next, ie do - local t, tn, h, p = { }, 0, { }, splitconfigurationpath(v) - for kk=1,#p do - local vv = p[kk] - if vv ~= "" and not h[vv] then - tn = tn + 1 - t[tn] = vv - h[vv] = true - end - end - if #t > 1 then - ie[k] = t - else - ie[k] = t[1] - end + local ie=instance.expansions + for k,v in next,ie do + local t,tn,h,p={},0,{},splitconfigurationpath(v) + for kk=1,#p do + local vv=p[kk] + if vv~="" and not h[vv] then + tn=tn+1 + t[tn]=vv + h[vv]=true + end + end + if #t>1 then + ie[k]=t + else + ie[k]=t[1] end + end end - --- end of split/join code - --- we used to have 'files' and 'configurations' so therefore the following --- shared function - function resolvers.datastate() - return caches.contentstate() + return caches.contentstate() end - function resolvers.variable(name) - local name = name and lpegmatch(dollarstripper,name) - local result = name and instance.variables[name] - return result ~= nil and result or "" + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.variables[name] + return result~=nil and result or "" end - function resolvers.expansion(name) - local name = name and lpegmatch(dollarstripper,name) - local result = name and instance.expansions[name] - return result ~= nil and result or "" + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.expansions[name] + return result~=nil and result or "" end - function resolvers.unexpandedpathlist(str) - local pth = resolvers.variable(str) - local lst = resolvers.splitpath(pth) - return expandedpathfromlist(lst) + local pth=resolvers.variable(str) + local lst=resolvers.splitpath(pth) + return expandedpathfromlist(lst) end - function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(resolvers.unexpandedpathlist(str)) end - -local done = { } - +local done={} function resolvers.resetextrapath() - local ep = instance.extra_paths - if not ep then - ep, done = { }, { } - instance.extra_paths = ep - elseif #ep > 0 then - instance.lists, done = { }, { } - end + local ep=instance.extra_paths + if not ep then + ep,done={},{} + instance.extra_paths=ep + elseif #ep>0 then + instance.lists,done={},{} + end end - function resolvers.registerextrapath(paths,subpaths) - paths = settings_to_array(paths) - subpaths = settings_to_array(subpaths) - local ep = instance.extra_paths or { } - local oldn = #ep - local newn = oldn - local nofpaths = #paths - local nofsubpaths = #subpaths - if nofpaths > 0 then - if nofsubpaths > 0 then - for i=1,nofpaths do - local p = paths[i] - for j=1,nofsubpaths do - local s = subpaths[j] - local ps = p .. "/" .. s - if not done[ps] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(ps) - done[ps] = true - end - end - end - else - for i=1,nofpaths do - local p = paths[i] - if not done[p] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(p) - done[p] = true - end - end + paths=settings_to_array(paths) + subpaths=settings_to_array(subpaths) + local ep=instance.extra_paths or {} + local oldn=#ep + local newn=oldn + local nofpaths=#paths + local nofsubpaths=#subpaths + if nofpaths>0 then + if nofsubpaths>0 then + for i=1,nofpaths do + local p=paths[i] + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=p.."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(ps) + done[ps]=true + end end - elseif nofsubpaths > 0 then - for i=1,oldn do - for j=1,nofsubpaths do - local s = subpaths[j] - local ps = ep[i] .. "/" .. s - if not done[ps] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(ps) - done[ps] = true - end - end + end + else + for i=1,nofpaths do + local p=paths[i] + if not done[p] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(p) + done[p]=true end + end end - if newn > 0 then - instance.extra_paths = ep -- register paths - end - if newn > oldn then - instance.lists = { } -- erase the cache + elseif nofsubpaths>0 then + for i=1,oldn do + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=ep[i].."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(ps) + done[ps]=true + end + end end + end + if newn>0 then + instance.extra_paths=ep + end + if newn>oldn then + instance.lists={} + end end - local function made_list(instance,list) - local ep = instance.extra_paths - if not ep or #ep == 0 then - return list - else - local done, new, newn = { }, { }, 0 - -- honour . .. ../.. but only when at the start - for k=1,#list do - local v = list[k] - if not done[v] then - if find(v,"^[%.%/]$") then - done[v] = true - newn = newn + 1 - new[newn] = v - else - break - end - end - end - -- first the extra paths - for k=1,#ep do - local v = ep[k] - if not done[v] then - done[v] = true - newn = newn + 1 - new[newn] = v - end - end - -- next the formal paths - for k=1,#list do - local v = list[k] - if not done[v] then - done[v] = true - newn = newn + 1 - new[newn] = v - end + local ep=instance.extra_paths + if not ep or #ep==0 then + return list + else + local done,new,newn={},{},0 + for k=1,#list do + local v=list[k] + if not done[v] then + if find(v,"^[%.%/]$") then + done[v]=true + newn=newn+1 + new[newn]=v + else + break end - return new + end + end + for k=1,#ep do + local v=ep[k] + if not done[v] then + done[v]=true + newn=newn+1 + new[newn]=v + end + end + for k=1,#list do + local v=list[k] + if not done[v] then + done[v]=true + newn=newn+1 + new[newn]=v + end end + return new + end end - function resolvers.cleanpathlist(str) - local t = resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i] = collapsepath(resolvers.cleanpath(t[i])) - end + local t=resolvers.expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(resolvers.cleanpath(t[i])) end - return t + end + return t end - function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) + return joinpath(resolvers.expandedpathlist(str)) end - function resolvers.expandedpathlist(str) - if not str then - return { } - elseif instance.savelists then - str = lpegmatch(dollarstripper,str) - local lists = instance.lists - local lst = lists[str] - if not lst then - local l = made_list(instance,resolvers.splitpath(resolvers.expansion(str))) - lst = expandedpathfromlist(l) - lists[str] = lst - end - return lst - else - local lst = resolvers.splitpath(resolvers.expansion(str)) - return made_list(instance,expandedpathfromlist(lst)) + if not str then + return {} + elseif instance.savelists then + str=lpegmatch(dollarstripper,str) + local lists=instance.lists + local lst=lists[str] + if not lst then + local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str))) + lst=expandedpathfromlist(l) + lists[str]=lst end + return lst + else + local lst=resolvers.splitpath(resolvers.expansion(str)) + return made_list(instance,expandedpathfromlist(lst)) + end end - -function resolvers.expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ // - str = lpegmatch(dollarstripper,str) - local tmp = resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp ~= "" and tmp or str) +function resolvers.expandedpathlistfromvariable(str) + str=lpegmatch(dollarstripper,str) + local tmp=resolvers.variableofformatorsuffix(str) + return resolvers.expandedpathlist(tmp~="" and tmp or str) end - function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(resolvers.expandedpathlistfromvariable(str)) end - -function resolvers.expandbraces(str) -- output variable and brace expansion of STRING --- local ori = resolvers.variable(str) --- if ori == "" then - local ori = str --- end - local pth = expandedpathfromlist(resolvers.splitpath(ori)) - return joinpath(pth) +function resolvers.expandbraces(str) + local ori=str + local pth=expandedpathfromlist(resolvers.splitpath(ori)) + return joinpath(pth) end - function resolvers.registerfilehash(name,content,someerror) - if content then - instance.files[name] = content - else - instance.files[name] = { } - if somerror == true then -- can be unset - instance.loaderror = someerror - end + if content then + instance.files[name]=content + else + instance.files[name]={} + if somerror==true then + instance.loaderror=someerror end + end end - local function isreadable(name) - local readable = lfs.isfile(name) -- not file.is_readable(name) asit can be a dir - if trace_detail then - if readable then - report_resolving("file '%s' is readable",name) - else - report_resolving("file '%s' is not readable", name) - end + local readable=lfs.isfile(name) + if trace_detail then + if readable then + report_resolving("file '%s' is readable",name) + else + report_resolving("file '%s' is not readable",name) end - return readable + end + return readable end - --- name --- name/name - local function collect_files(names) - local filelist, noffiles = { }, 0 - for k=1,#names do - local fname = names[k] + local filelist,noffiles={},0 + for k=1,#names do + local fname=names[k] + if trace_detail then + report_resolving("checking name '%s'",fname) + end + local bname=filebasename(fname) + local dname=filedirname(fname) + if dname=="" or find(dname,"^%.") then + dname=false + else + dname=gsub(dname,"%*",".*") + dname="/"..dname.."$" + end + local hashes=instance.hashes + for h=1,#hashes do + local hash=hashes[h] + local blobpath=hash.name + local files=blobpath and instance.files[blobpath] + if files then if trace_detail then - report_resolving("checking name '%s'",fname) - end - local bname = filebasename(fname) - local dname = filedirname(fname) - if dname == "" or find(dname,"^%.") then - dname = false - else - dname = gsub(dname,"*","%.*") - dname = "/" .. dname .. "$" + report_resolving("deep checking '%s' (%s)",blobpath,bname) + end + local blobfile=files[bname] + if not blobfile then + local rname="remap:"..bname + blobfile=files[rname] + if blobfile then + bname=files[rname] + blobfile=files[bname] + end end - local hashes = instance.hashes - for h=1,#hashes do - local hash = hashes[h] - local blobpath = hash.name - local files = blobpath and instance.files[blobpath] - if files then + if blobfile then + local blobroot=files.__path__ or blobpath + if type(blobfile)=='string' then + if not dname or find(blobfile,dname) then + local variant=hash.type + local search=filejoin(blobroot,blobfile,bname) + local result=methodhandler('concatinators',hash.type,blobroot,blobfile,bname) + if trace_detail then + report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + else + for kk=1,#blobfile do + local vv=blobfile[kk] + if not dname or find(vv,dname) then + local variant=hash.type + local search=filejoin(blobroot,vv,bname) + local result=methodhandler('concatinators',hash.type,blobroot,vv,bname) if trace_detail then - report_resolving("deep checking '%s' (%s)",blobpath,bname) - end - local blobfile = files[bname] - if not blobfile then - local rname = "remap:"..bname - blobfile = files[rname] - if blobfile then - bname = files[rname] - blobfile = files[bname] - end - end - if blobfile then - local blobroot = files.__path__ or blobpath - if type(blobfile) == 'string' then - if not dname or find(blobfile,dname) then - local variant = hash.type - -- local search = filejoin(blobpath,blobfile,bname) - local search = filejoin(blobroot,blobfile,bname) - local result = methodhandler('concatinators',hash.type,blobroot,blobfile,bname) - if trace_detail then - report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) - end - noffiles = noffiles + 1 - filelist[noffiles] = { variant, search, result } - end - else - for kk=1,#blobfile do - local vv = blobfile[kk] - if not dname or find(vv,dname) then - local variant = hash.type - -- local search = filejoin(blobpath,vv,bname) - local search = filejoin(blobroot,vv,bname) - local result = methodhandler('concatinators',hash.type,blobroot,vv,bname) - if trace_detail then - report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) - end - noffiles = noffiles + 1 - filelist[noffiles] = { variant, search, result } - end - end - end + report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) end - elseif trace_locating then - report_resolving("no match in '%s' (%s)",blobpath,bname) + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end end + end end + elseif trace_locating then + report_resolving("no match in '%s' (%s)",blobpath,bname) + end end - return noffiles > 0 and filelist or nil + end + return noffiles>0 and filelist or nil end - -local fit = { } - +local fit={} function resolvers.registerintrees(filename,format,filetype,usedmethod,foundname) - local foundintrees = instance.foundintrees - if usedmethod == "direct" and filename == foundname and fit[foundname] then - -- just an extra lookup after a test on presence - else - local t = { - filename = filename, - format = format ~= "" and format or nil, - filetype = filetype ~= "" and filetype or nil, - usedmethod = usedmethod, - foundname = foundname, - } - fit[foundname] = t - foundintrees[#foundintrees+1] = t - end + local foundintrees=instance.foundintrees + if usedmethod=="direct" and filename==foundname and fit[foundname] then + else + local t={ + filename=filename, + format=format~="" and format or nil, + filetype=filetype~="" and filetype or nil, + usedmethod=usedmethod, + foundname=foundname, + } + fit[foundname]=t + foundintrees[#foundintrees+1]=t + end end - --- split the next one up for readability (but this module needs a cleanup anyway) - -local function can_be_dir(name) -- can become local - local fakepaths = instance.fakepaths - if not fakepaths[name] then - if lfs.isdir(name) then - fakepaths[name] = 1 -- directory - else - fakepaths[name] = 2 -- no directory - end +local function can_be_dir(name) + local fakepaths=instance.fakepaths + if not fakepaths[name] then + if lfs.isdir(name) then + fakepaths[name]=1 + else + fakepaths[name]=2 end - return fakepaths[name] == 1 + end + return fakepaths[name]==1 end - -local preparetreepattern = Cs((P(".")/"%%." + P("-")/"%%-" + P(1))^0 * Cc("$")) - --- -- -- begin of main file search routing -- -- -- needs checking as previous has been patched - +local preparetreepattern=Cs((P(".")/"%%."+P("-")/"%%-"+P(1))^0*Cc("$")) local collect_instance_files - local function find_analyze(filename,askedformat,allresults) - local filetype, wantedfiles, ext = '', { }, suffixonly(filename) - -- too tricky as filename can be bla.1.2.3: - -- - -- if not suffixmap[ext] then - -- wantedfiles[#wantedfiles+1] = filename - -- end - wantedfiles[#wantedfiles+1] = filename - if askedformat == "" then - if ext == "" or not suffixmap[ext] then - local defaultsuffixes = resolvers.defaultsuffixes - for i=1,#defaultsuffixes do - local forcedname = filename .. '.' .. defaultsuffixes[i] - wantedfiles[#wantedfiles+1] = forcedname - filetype = resolvers.formatofsuffix(forcedname) - if trace_locating then - report_resolving("forcing filetype '%s'",filetype) - end - end - else - filetype = resolvers.formatofsuffix(filename) - if trace_locating then - report_resolving("using suffix based filetype '%s'",filetype) - end + local filetype,wantedfiles,ext='',{},suffixonly(filename) + wantedfiles[#wantedfiles+1]=filename + if askedformat=="" then + if ext=="" or not suffixmap[ext] then + local defaultsuffixes=resolvers.defaultsuffixes + for i=1,#defaultsuffixes do + local forcedname=filename..'.'..defaultsuffixes[i] + wantedfiles[#wantedfiles+1]=forcedname + filetype=resolvers.formatofsuffix(forcedname) + if trace_locating then + report_resolving("forcing filetype '%s'",filetype) end + end else - if ext == "" or not suffixmap[ext] then - local format_suffixes = suffixes[askedformat] - if format_suffixes then - for i=1,#format_suffixes do - wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i] - end - end - end - filetype = askedformat - if trace_locating then - report_resolving("using given filetype '%s'",filetype) + filetype=resolvers.formatofsuffix(filename) + if trace_locating then + report_resolving("using suffix based filetype '%s'",filetype) + end + end + else + if ext=="" or not suffixmap[ext] then + local format_suffixes=suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + wantedfiles[#wantedfiles+1]=filename.."."..format_suffixes[i] end + end end - return filetype, wantedfiles + filetype=askedformat + if trace_locating then + report_resolving("using given filetype '%s'",filetype) + end + end + return filetype,wantedfiles end - local function find_direct(filename,allresults) - if not dangerous[askedformat] and isreadable(filename) then - if trace_detail then - report_resolving("file '%s' found directly",filename) - end - return "direct", { filename } + if not dangerous[askedformat] and isreadable(filename) then + if trace_detail then + report_resolving("file '%s' found directly",filename) end + return "direct",{ filename } + end end - local function find_wildcard(filename,allresults) - if find(filename,'%*') then - if trace_locating then - report_resolving("checking wildcard '%s'", filename) - end - local method, result = resolvers.findwildcardfiles(filename) - if result then - return "wildcard", result - end - end -end - -local function find_qualified(filename,allresults) -- this one will be split too - if not file.is_qualified_path(filename) then - return - end + if find(filename,'%*') then if trace_locating then - report_resolving("checking qualified name '%s'", filename) + report_resolving("checking wildcard '%s'",filename) end - if isreadable(filename) then - if trace_detail then - report_resolving("qualified file '%s' found", filename) - end - return "qualified", { filename } + local method,result=resolvers.findwildcardfiles(filename) + if result then + return "wildcard",result end + end +end +local function find_qualified(filename,allresults) + if not file.is_qualified_path(filename) then + return + end + if trace_locating then + report_resolving("checking qualified name '%s'",filename) + end + if isreadable(filename) then if trace_detail then - report_resolving("locating qualified file '%s'", filename) - end - local forcedname, suffix = "", suffixonly(filename) - if suffix == "" then -- why - local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat] - if format_suffixes then - for i=1,#format_suffixes do - local s = format_suffixes[i] - forcedname = filename .. "." .. s - if isreadable(forcedname) then - if trace_locating then - report_resolving("no suffix, forcing format filetype '%s'", s) - end - return "qualified", { forcedname } - end - end + report_resolving("qualified file '%s' found",filename) + end + return "qualified",{ filename } + end + if trace_detail then + report_resolving("locating qualified file '%s'",filename) + end + local forcedname,suffix="",suffixonly(filename) + if suffix=="" then + local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + local s=format_suffixes[i] + forcedname=filename.."."..s + if isreadable(forcedname) then + if trace_locating then + report_resolving("no suffix, forcing format filetype '%s'",s) + end + return "qualified",{ forcedname } end + end end - if suffix and suffix ~= "" then - -- try to find in tree (no suffix manipulation), here we search for the - -- matching last part of the name - local basename = filebasename(filename) - local pattern = lpegmatch(preparetreepattern,filename) - -- messy .. to be sorted out - local savedformat = askedformat - local format = savedformat or "" - if format == "" then - askedformat = resolvers.formatofsuffix(suffix) + end + if suffix and suffix~="" then + local basename=filebasename(filename) + local pattern=lpegmatch(preparetreepattern,filename) + local savedformat=askedformat + local format=savedformat or "" + if format=="" then + askedformat=resolvers.formatofsuffix(suffix) + end + if not format then + askedformat="othertextfiles" + end + if basename~=filename then + local resolved=collect_instance_files(basename,askedformat,allresults) + if #resolved==0 then + local lowered=lower(basename) + if filename~=lowered then + resolved=collect_instance_files(lowered,askedformat,allresults) end - if not format then - askedformat = "othertextfiles" -- kind of everything, maybe all + end + resolvers.format=savedformat + if #resolved>0 then + local result={} + for r=1,#resolved do + local rr=resolved[r] + if find(rr,pattern) then + result[#result+1]=rr + end end - -- - if basename ~= filename then - local resolved = collect_instance_files(basename,askedformat,allresults) - if #resolved == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered,askedformat,allresults) - end - end - resolvers.format = savedformat - -- - if #resolved > 0 then - local result = { } - for r=1,#resolved do - local rr = resolved[r] - if find(rr,pattern) then - result[#result+1] = rr - end - end - if #result > 0 then - return "qualified", result - end - end + if #result>0 then + return "qualified",result end - -- a real wildcard: - -- - -- local filelist = collect_files({basename}) - -- result = { } - -- for f=1,#filelist do - -- local ff = filelist[f][3] or "" - -- if find(ff,pattern) then - -- result[#result+1], ok = ff, true - -- end - -- end - -- if #result > 0 then - -- return "qualified", result - -- end + end end + end end - local function check_subpath(fname) - if isreadable(fname) then - if trace_detail then - report_resolving("found '%s' by deep scanning",fname) - end - return fname + if isreadable(fname) then + if trace_detail then + report_resolving("found '%s' by deep scanning",fname) end + return fname + end end - local function find_intree(filename,filetype,wantedfiles,allresults) - local typespec = resolvers.variableofformat(filetype) - local pathlist = resolvers.expandedpathlist(typespec) - local method = "intree" - if pathlist and #pathlist > 0 then - -- list search - local filelist = collect_files(wantedfiles) - local dirlist = { } - if filelist then - for i=1,#filelist do - dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble - end - end + local typespec=resolvers.variableofformat(filetype) + local pathlist=resolvers.expandedpathlist(typespec) + local method="intree" + if pathlist and #pathlist>0 then + local filelist=collect_files(wantedfiles) + local dirlist={} + if filelist then + for i=1,#filelist do + dirlist[i]=filedirname(filelist[i][3]).."/" + end + end + if trace_detail then + report_resolving("checking filename '%s'",filename) + end + local result={} + for k=1,#pathlist do + local path=pathlist[k] + local pathname=lpegmatch(inhibitstripper,path) + local doscan=path==pathname + if not find (pathname,'//$') then + doscan=false + end + local done=false + if filelist then + local expression=makepathexpression(pathname) if trace_detail then - report_resolving("checking filename '%s'",filename) - end - local result = { } - for k=1,#pathlist do - local path = pathlist[k] - local pathname = lpegmatch(inhibitstripper,path) - local doscan = path == pathname -- no ^!! - if not find (pathname,'//$') then - doscan = false -- we check directly on the path + report_resolving("using pattern '%s' for path '%s'",expression,pathname) + end + for k=1,#filelist do + local fl=filelist[k] + local f=fl[2] + local d=dirlist[k] + if find(d,expression) then + result[#result+1]=resolvers.resolve(fl[3]) + done=true + if allresults then + if trace_detail then + report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d) + end + else + if trace_detail then + report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d) + end + break end - local done = false - -- using file list - if filelist then -- database - -- compare list entries with permitted pattern -- /xx /xx// - local expression = makepathexpression(pathname) - if trace_detail then - report_resolving("using pattern '%s' for path '%s'",expression,pathname) - end - for k=1,#filelist do - local fl = filelist[k] - local f = fl[2] - local d = dirlist[k] - if find(d,expression) then - -- todo, test for readable - result[#result+1] = resolvers.resolve(fl[3]) -- no shortcut - done = true - if allresults then - if trace_detail then - report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d) - end - else - if trace_detail then - report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d) - end + elseif trace_detail then + report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d) + end + end + end + if done then + method="database" + else + method="filesystem" + pathname=gsub(pathname,"/+$","") + pathname=resolvers.resolve(pathname) + local scheme=url.hasscheme(pathname) + if not scheme or scheme=="file" then + local pname=gsub(pathname,"%.%*$",'') + if not find(pname,"%*") then + if can_be_dir(pname) then + for k=1,#wantedfiles do + local w=wantedfiles[k] + local fname=check_subpath(filejoin(pname,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end + if not done and doscan then + local files=resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w=wantedfiles[k] + local subpath=files[w] + if not subpath or subpath=="" then + elseif type(subpath)=="string" then + local fname=check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp=subpath[i] + if sp=="" then + else + local fname=check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then break + end end - elseif trace_detail then - report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d) + end end - end - end - if done then - method = "database" - else - method = "filesystem" -- bonus, even when !! is specified - pathname = gsub(pathname,"/+$","") - pathname = resolvers.resolve(pathname) - local scheme = url.hasscheme(pathname) - if not scheme or scheme == "file" then - local pname = gsub(pathname,"%.%*$",'') - if not find(pname,"%*") then - if can_be_dir(pname) then - -- quick root scan first - for k=1,#wantedfiles do - local w = wantedfiles[k] - local fname = check_subpath(filejoin(pname,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - end - if not done and doscan then - -- collect files in path (and cache the result) - local files = resolvers.simplescanfiles(pname,false,true) - for k=1,#wantedfiles do - local w = wantedfiles[k] - local subpath = files[w] - if not subpath or subpath == "" then - -- rootscan already done - elseif type(subpath) == "string" then - local fname = check_subpath(filejoin(pname,subpath,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - else - for i=1,#subpath do - local sp = subpath[i] - if sp == "" then - -- roottest already done - else - local fname = check_subpath(filejoin(pname,sp,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - end - end - if done and not allresults then - break - end - end - end - end - end - else - -- no access needed for non existing path, speedup (esp in large tree with lots of fake) + if done and not allresults then + break end + end end + end end - -- todo recursive scanning - if done and not allresults then - break - end - end - if #result > 0 then - return method, result + else + end end + end + if done and not allresults then + break + end end + if #result>0 then + return method,result + end + end end - local function find_onpath(filename,filetype,wantedfiles,allresults) - if trace_detail then - report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) - end - local result = { } - for k=1,#wantedfiles do - local fname = wantedfiles[k] - if fname and isreadable(fname) then - filename = fname - result[#result+1] = filejoin('.',fname) - if not allresults then - break - end - end + if trace_detail then + report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename,filetype or '?',concat(wantedfiles," | ")) + end + local result={} + for k=1,#wantedfiles do + local fname=wantedfiles[k] + if fname and isreadable(fname) then + filename=fname + result[#result+1]=filejoin('.',fname) + if not allresults then + break + end end - if #result > 0 then - return "onpath", result - end -end - -local function find_otherwise(filename,filetype,wantedfiles,allresults) -- other text files | any | whatever - local filelist = collect_files(wantedfiles) - local fl = filelist and filelist[1] - if fl then - return "otherwise", { resolvers.resolve(fl[3]) } -- filename - end -end - --- we could have a loop over the 6 functions but then we'd have to --- always analyze - -collect_instance_files = function(filename,askedformat,allresults) -- uses nested - askedformat = askedformat or "" - filename = collapsepath(filename) - if allresults then - -- no need for caching, only used for tracing - local filetype, wantedfiles = find_analyze(filename,askedformat) - local results = { - { find_direct (filename,true) }, - { find_wildcard (filename,true) }, - { find_qualified(filename,true) }, - { find_intree (filename,filetype,wantedfiles,true) }, - { find_onpath (filename,filetype,wantedfiles,true) }, - { find_otherwise(filename,filetype,wantedfiles,true) }, - } - local result, status, done = { }, { }, { } - for k, r in next, results do - local method, list = r[1], r[2] - if method and list then - for i=1,#list do - local c = collapsepath(list[i]) - if not done[c] then - result[#result+1] = c - done[c] = true - end - status[#status+1] = format("%-10s: %s",method,c) - end - end - end - if trace_detail then - report_resolving("lookup status: %s",table.serialize(status,filename)) + end + if #result>0 then + return "onpath",result + end +end +local function find_otherwise(filename,filetype,wantedfiles,allresults) + local filelist=collect_files(wantedfiles) + local fl=filelist and filelist[1] + if fl then + return "otherwise",{ resolvers.resolve(fl[3]) } + end +end +collect_instance_files=function(filename,askedformat,allresults) + askedformat=askedformat or "" + filename=collapsepath(filename) + if allresults then + local filetype,wantedfiles=find_analyze(filename,askedformat) + local results={ + { find_direct (filename,true) }, + { find_wildcard (filename,true) }, + { find_qualified(filename,true) }, + { find_intree (filename,filetype,wantedfiles,true) }, + { find_onpath (filename,filetype,wantedfiles,true) }, + { find_otherwise(filename,filetype,wantedfiles,true) }, + } + local result,status,done={},{},{} + for k,r in next,results do + local method,list=r[1],r[2] + if method and list then + for i=1,#list do + local c=collapsepath(list[i]) + if not done[c] then + result[#result+1]=c + done[c]=true + end + status[#status+1]=format("%-10s: %s",method,c) end - return result, status - else - local method, result, stamp, filetype, wantedfiles - if instance.remember then - stamp = format("%s--%s", filename, askedformat) - result = stamp and instance.found[stamp] - if result then - if trace_locating then - report_resolving("remembered file '%s'",filename) - end - return result - end + end + end + if trace_detail then + report_resolving("lookup status: %s",table.serialize(status,filename)) + end + return result,status + else + local method,result,stamp,filetype,wantedfiles + if instance.remember then + stamp=format("%s--%s",filename,askedformat) + result=stamp and instance.found[stamp] + if result then + if trace_locating then + report_resolving("remembered file '%s'",filename) end - method, result = find_direct(filename) + return result + end + end + method,result=find_direct(filename) + if not result then + method,result=find_wildcard(filename) + if not result then + method,result=find_qualified(filename) if not result then - method, result = find_wildcard(filename) + filetype,wantedfiles=find_analyze(filename,askedformat) + method,result=find_intree(filename,filetype,wantedfiles) + if not result then + method,result=find_onpath(filename,filetype,wantedfiles) if not result then - method, result = find_qualified(filename) - if not result then - filetype, wantedfiles = find_analyze(filename,askedformat) - method, result = find_intree(filename,filetype,wantedfiles) - if not result then - method, result = find_onpath(filename,filetype,wantedfiles) - if not result then - method, result = find_otherwise(filename,filetype,wantedfiles) - end - end - end - end - end - if result and #result > 0 then - local foundname = collapsepath(result[1]) - resolvers.registerintrees(filename,askedformat,filetype,method,foundname) - result = { foundname } - else - result = { } -- maybe false - end - if stamp then - if trace_locating then - report_resolving("remembering file '%s'",filename) + method,result=find_otherwise(filename,filetype,wantedfiles) end - instance.found[stamp] = result + end end - return result + end + end + if result and #result>0 then + local foundname=collapsepath(result[1]) + resolvers.registerintrees(filename,askedformat,filetype,method,foundname) + result={ foundname } + else + result={} + end + if stamp then + if trace_locating then + report_resolving("remembering file '%s'",filename) + end + instance.found[stamp]=result end + return result + end end - --- -- -- end of main file search routing -- -- -- - - local function findfiles(filename,filetype,allresults) - local result, status = collect_instance_files(filename,filetype or "",allresults) - if not result or #result == 0 then - local lowered = lower(filename) - if filename ~= lowered then - result, status = collect_instance_files(lowered,filetype or "",allresults) - end + local result,status=collect_instance_files(filename,filetype or "",allresults) + if not result or #result==0 then + local lowered=lower(filename) + if filename~=lowered then + result,status=collect_instance_files(lowered,filetype or "",allresults) end - return result or { }, status + end + return result or {},status end - function resolvers.findfiles(filename,filetype) - return findfiles(filename,filetype,true) + return findfiles(filename,filetype,true) end - function resolvers.findfile(filename,filetype) - return findfiles(filename,filetype,false)[1] or "" + return findfiles(filename,filetype,false)[1] or "" end - function resolvers.findpath(filename,filetype) - return filedirname(findfiles(filename,filetype,false)[1] or "") + return filedirname(findfiles(filename,filetype,false)[1] or "") end - local function findgivenfiles(filename,allresults) - local bname, result = filebasename(filename), { } - local hashes = instance.hashes - local noffound = 0 - for k=1,#hashes do - local hash = hashes[k] - local files = instance.files[hash.name] or { } - local blist = files[bname] - if not blist then - local rname = "remap:"..bname - blist = files[rname] - if blist then - bname = files[rname] - blist = files[bname] - end - end - if blist then - if type(blist) == 'string' then - local found = methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" - if found ~= "" then - noffound = noffound + 1 - result[noffound] = resolvers.resolve(found) - if not allresults then break end - end - else - for kk=1,#blist do - local vv = blist[kk] - local found = methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" - if found ~= "" then - noffound = noffound + 1 - result[noffound] = resolvers.resolve(found) - if not allresults then break end - end - end - end + local bname,result=filebasename(filename),{} + local hashes=instance.hashes + local noffound=0 + for k=1,#hashes do + local hash=hashes[k] + local files=instance.files[hash.name] or {} + local blist=files[bname] + if not blist then + local rname="remap:"..bname + blist=files[rname] + if blist then + bname=files[rname] + blist=files[bname] + end + end + if blist then + if type(blist)=='string' then + local found=methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" + if found~="" then + noffound=noffound+1 + result[noffound]=resolvers.resolve(found) + if not allresults then break end + end + else + for kk=1,#blist do + local vv=blist[kk] + local found=methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" + if found~="" then + noffound=noffound+1 + result[noffound]=resolvers.resolve(found) + if not allresults then break end + end end + end end - return result + end + return result end - function resolvers.findgivenfiles(filename) - return findgivenfiles(filename,true) + return findgivenfiles(filename,true) end - function resolvers.findgivenfile(filename) - return findgivenfiles(filename,false)[1] or "" + return findgivenfiles(filename,false)[1] or "" end - local function doit(path,blist,bname,tag,variant,result,allresults) - local done = false - if blist and variant then - local resolve = resolvers.resolve -- added - if type(blist) == 'string' then - -- make function and share code - if find(lower(blist),path) then - local full = methodhandler('concatinators',variant,tag,blist,bname) or "" - result[#result+1] = resolve(full) - done = true - end - else - for kk=1,#blist do - local vv = blist[kk] - if find(lower(vv),path) then - local full = methodhandler('concatinators',variant,tag,vv,bname) or "" - result[#result+1] = resolve(full) - done = true - if not allresults then break end - end - end + local done=false + if blist and variant then + local resolve=resolvers.resolve + if type(blist)=='string' then + if find(lower(blist),path) then + local full=methodhandler('concatinators',variant,tag,blist,bname) or "" + result[#result+1]=resolve(full) + done=true + end + else + for kk=1,#blist do + local vv=blist[kk] + if find(lower(vv),path) then + local full=methodhandler('concatinators',variant,tag,vv,bname) or "" + result[#result+1]=resolve(full) + done=true + if not allresults then break end end + end end - return done + end + return done end - - -local makewildcard = Cs( - (P("^")^0 * P("/") * P(-1) + P(-1)) /".*" - + (P("^")^0 * P("/") / "")^0 * (P("*")/".*" + P("-")/"%%-" + P(".")/"%%." + P("?")/"."+ P("\\")/"/" + P(1))^0 +local makewildcard=Cs( + (P("^")^0*P("/")*P(-1)+P(-1))/".*"+(P("^")^0*P("/")/"")^0*(P("*")/".*"+P("-")/"%%-"+P(".")/"%%."+P("?")/"."+P("\\")/"/"+P(1))^0 ) - function resolvers.wildcardpattern(pattern) - return lpegmatch(makewildcard,pattern) or pattern -end - -local function findwildcardfiles(filename,allresults,result) -- todo: remap: and lpeg - result = result or { } - local base = filebasename(filename) - local dirn = filedirname(filename) - local path = lower(lpegmatch(makewildcard,dirn) or dirn) - local name = lower(lpegmatch(makewildcard,base) or base) - local files, done = instance.files, false - if find(name,"%*") then - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - local hashname, hashtype = hash.name, hash.type - for kk, hh in next, files[hashname] do - if not find(kk,"^remap:") then - if find(lower(kk),name) then - if doit(path,hh,kk,hashname,hashtype,result,allresults) then done = true end - if done and not allresults then break end - end - end - end - end - else - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - local hashname, hashtype = hash.name, hash.type - if doit(path,files[hashname][bname],bname,hashname,hashtype,result,allresults) then done = true end + return lpegmatch(makewildcard,pattern) or pattern +end +local function findwildcardfiles(filename,allresults,result) + result=result or {} + local base=filebasename(filename) + local dirn=filedirname(filename) + local path=lower(lpegmatch(makewildcard,dirn) or dirn) + local name=lower(lpegmatch(makewildcard,base) or base) + local files,done=instance.files,false + if find(name,"%*") then + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + local hashname,hashtype=hash.name,hash.type + for kk,hh in next,files[hashname] do + if not find(kk,"^remap:") then + if find(lower(kk),name) then + if doit(path,hh,kk,hashname,hashtype,result,allresults) then done=true end if done and not allresults then break end + end end + end end - -- we can consider also searching the paths not in the database, but then - -- we end up with a messy search (all // in all path specs) - return result + else + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + local hashname,hashtype=hash.name,hash.type + if doit(path,files[hashname][bname],bname,hashname,hashtype,result,allresults) then done=true end + if done and not allresults then break end + end + end + return result end - function resolvers.findwildcardfiles(filename,result) - return findwildcardfiles(filename,true,result) + return findwildcardfiles(filename,true,result) end - function resolvers.findwildcardfile(filename) - return findwildcardfiles(filename,false)[1] or "" + return findwildcardfiles(filename,false)[1] or "" end - --- main user functions - function resolvers.automount() - -- implemented later end - function resolvers.load(option) - statistics.starttiming(instance) - identify_configuration_files() - load_configuration_files() - if option ~= "nofiles" then - load_databases() - resolvers.automount() - end - statistics.stoptiming(instance) - local files = instance.files - return files and next(files) and true + statistics.starttiming(instance) + identify_configuration_files() + load_configuration_files() + if option~="nofiles" then + load_databases() + resolvers.automount() + end + statistics.stoptiming(instance) + local files=instance.files + return files and next(files) and true end - function resolvers.loadtime() - return statistics.elapsedtime(instance) + return statistics.elapsedtime(instance) end - local function report(str) + if trace_locating then + report_resolving(str) + else + print(str) + end +end +function resolvers.dowithfilesandreport(command,files,...) + if files and #files>0 then if trace_locating then - report_resolving(str) -- has already verbose - else - print(str) + report('') end -end - -function resolvers.dowithfilesandreport(command, files, ...) -- will move - if files and #files > 0 then - if trace_locating then - report('') -- ? - end - if type(files) == "string" then - files = { files } - end - for f=1,#files do - local file = files[f] - local result = command(file,...) - if type(result) == 'string' then - report(result) - else - for i=1,#result do - report(result[i]) -- could be unpack - end - end + if type(files)=="string" then + files={ files } + end + for f=1,#files do + local file=files[f] + local result=command(file,...) + if type(result)=='string' then + report(result) + else + for i=1,#result do + report(result[i]) end + end end + end end - --- obsolete - --- resolvers.varvalue = resolvers.variable -- output the value of variable $STRING. --- resolvers.expandvar = resolvers.expansion -- output variable expansion of STRING. - -function resolvers.showpath(str) -- output search path for file type NAME - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) +function resolvers.showpath(str) + return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) end - -function resolvers.registerfile(files, name, path) - if files[name] then - if type(files[name]) == 'string' then - files[name] = { files[name], path } - else - files[name] = path - end +function resolvers.registerfile(files,name,path) + if files[name] then + if type(files[name])=='string' then + files[name]={ files[name],path } else - files[name] = path + files[name]=path end + else + files[name]=path + end end - function resolvers.dowithpath(name,func) - local pathlist = resolvers.expandedpathlist(name) - for i=1,#pathlist do - func("^"..resolvers.cleanpath(pathlist[i])) - end + local pathlist=resolvers.expandedpathlist(name) + for i=1,#pathlist do + func("^"..resolvers.cleanpath(pathlist[i])) + end end - function resolvers.dowithvariable(name,func) - func(expandedvariable(name)) + func(expandedvariable(name)) end - function resolvers.locateformat(name) - local engine = environment.ownmain or "luatex" - local barename = file.removesuffix(name) - local fullname = file.addsuffix(barename,"fmt") - local fmtname = caches.getfirstreadablefile(fullname,"formats",engine) or "" - if fmtname == "" then - fmtname = resolvers.findfile(fullname) - fmtname = resolvers.cleanpath(fmtname) - end - if fmtname ~= "" then - local barename = file.removesuffix(fmtname) - local luaname = file.addsuffix(barename,luasuffixes.lua) - local lucname = file.addsuffix(barename,luasuffixes.luc) - local luiname = file.addsuffix(barename,luasuffixes.lui) - if lfs.isfile(luiname) then - return barename, luiname - elseif lfs.isfile(lucname) then - return barename, lucname - elseif lfs.isfile(luaname) then - return barename, luaname - end - end - return nil, nil + local engine=environment.ownmain or "luatex" + local barename=file.removesuffix(name) + local fullname=file.addsuffix(barename,"fmt") + local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" + if fmtname=="" then + fmtname=resolvers.findfile(fullname) + fmtname=resolvers.cleanpath(fmtname) + end + if fmtname~="" then + local barename=file.removesuffix(fmtname) + local luaname=file.addsuffix(barename,luasuffixes.lua) + local lucname=file.addsuffix(barename,luasuffixes.luc) + local luiname=file.addsuffix(barename,luasuffixes.lui) + if lfs.isfile(luiname) then + return barename,luiname + elseif lfs.isfile(lucname) then + return barename,lucname + elseif lfs.isfile(luaname) then + return barename,luaname + end + end + return nil,nil end - function resolvers.booleanvariable(str,default) - local b = resolvers.expansion(str) - if b == "" then - return default - else - b = toboolean(b) - return (b == nil and default) or b - end -end - -function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead - local instance = resolvers.instance - local hashes = instance.hashes - for i=1,#hashes do - local hash = hashes[i] - local blobtype = hash.type - local blobpath = hash.name - if blobpath then - if before then - before(blobtype,blobpath,pattern) - end - local files = instance.files[blobpath] - local total, checked, done = 0, 0, 0 - if files then - for k,v in next, files do - total = total + 1 - if find(k,"^remap:") then - k = files[k] - v = k -- files[k] -- chained - end - if find(k,pattern) then - if type(v) == "string" then - checked = checked + 1 - if handle(blobtype,blobpath,v,k) then - done = done + 1 - end - else - checked = checked + #v - for i=1,#v do - if handle(blobtype,blobpath,v[i],k) then - done = done + 1 - end - end - end - end + local b=resolvers.expansion(str) + if b=="" then + return default + else + b=toboolean(b) + return (b==nil and default) or b + end +end +function resolvers.dowithfilesintree(pattern,handle,before,after) + local instance=resolvers.instance + local hashes=instance.hashes + for i=1,#hashes do + local hash=hashes[i] + local blobtype=hash.type + local blobpath=hash.name + if blobpath then + if before then + before(blobtype,blobpath,pattern) + end + local files=instance.files[blobpath] + local total,checked,done=0,0,0 + if files then + for k,v in next,files do + total=total+1 + if find(k,"^remap:") then + k=files[k] + v=k + end + if find(k,pattern) then + if type(v)=="string" then + checked=checked+1 + if handle(blobtype,blobpath,v,k) then + done=done+1 + end + else + checked=checked+#v + for i=1,#v do + if handle(blobtype,blobpath,v[i],k) then + done=done+1 end + end end - if after then - after(blobtype,blobpath,pattern,total,checked,done) - end + end end + end + if after then + after(blobtype,blobpath,pattern,total,checked,done) + end end + end end - -resolvers.obsolete = resolvers.obsolete or { } -local obsolete = resolvers.obsolete - -resolvers.find_file = resolvers.findfile obsolete.find_file = resolvers.findfile -resolvers.find_files = resolvers.findfiles obsolete.find_files = resolvers.findfiles +resolvers.obsolete=resolvers.obsolete or {} +local obsolete=resolvers.obsolete +resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile +resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-pre'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- It could be interesting to hook the resolver in the file --- opener so that unresolved prefixes travel around and we --- get more abstraction. - --- As we use this beforehand we will move this up in the chain --- of loading. - - -local resolvers = resolvers -local prefixes = utilities.storage.allocate() -resolvers.prefixes = prefixes +-- original size: 6430, stripped down to: 4219 -local cleanpath, findgivenfile, expansion = resolvers.cleanpath, resolvers.findgivenfile, resolvers.expansion -local getenv = resolvers.getenv -- we can probably also use resolvers.expansion -local P, S, R, C, Cs, lpegmatch = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cs, lpeg.match -local joinpath, basename, dirname = file.join, file.basename, file.dirname -local getmetatable, rawset, type = getmetatable, rawset, type - --- getenv = function(...) return resolvers.getenv(...) end -- needs checking (definitions changes later on) - -prefixes.environment = function(str) - return cleanpath(expansion(str)) -end - -prefixes.relative = function(str,n) -- lfs.isfile - if io.exists(str) then - -- nothing - elseif io.exists("./" .. str) then - str = "./" .. str - else - local p = "../" - for i=1,n or 2 do - if io.exists(p .. str) then - str = p .. str - break - else - p = p .. "../" - end - end +if not modules then modules={} end modules ['data-pre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers=resolvers +local prefixes=utilities.storage.allocate() +resolvers.prefixes=prefixes +local cleanpath,findgivenfile,expansion=resolvers.cleanpath,resolvers.findgivenfile,resolvers.expansion +local getenv=resolvers.getenv +local P,S,R,C,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.match +local joinpath,basename,dirname=file.join,file.basename,file.dirname +local getmetatable,rawset,type=getmetatable,rawset,type +prefixes.environment=function(str) + return cleanpath(expansion(str)) +end +prefixes.relative=function(str,n) + if io.exists(str) then + elseif io.exists("./"..str) then + str="./"..str + else + local p="../" + for i=1,n or 2 do + if io.exists(p..str) then + str=p..str + break + else + p=p.."../" + end end - return cleanpath(str) + end + return cleanpath(str) end - -prefixes.auto = function(str) - local fullname = prefixes.relative(str) - if not lfs.isfile(fullname) then - fullname = prefixes.locate(str) - end - return fullname +prefixes.auto=function(str) + local fullname=prefixes.relative(str) + if not lfs.isfile(fullname) then + fullname=prefixes.locate(str) + end + return fullname end - -prefixes.locate = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath((fullname ~= "" and fullname) or str) +prefixes.locate=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath((fullname~="" and fullname) or str) end - -prefixes.filename = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath(basename((fullname ~= "" and fullname) or str)) -- no cleanpath needed here +prefixes.filename=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(basename((fullname~="" and fullname) or str)) end - -prefixes.pathname = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath(dirname((fullname ~= "" and fullname) or str)) +prefixes.pathname=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(dirname((fullname~="" and fullname) or str)) end - -prefixes.selfautoloc = function(str) - return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) +prefixes.selfautoloc=function(str) + return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) end - -prefixes.selfautoparent = function(str) - return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) +prefixes.selfautoparent=function(str) + return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) end - -prefixes.selfautodir = function(str) - return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) +prefixes.selfautodir=function(str) + return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) end - -prefixes.home = function(str) - return cleanpath(joinpath(getenv('HOME'),str)) +prefixes.home=function(str) + return cleanpath(joinpath(getenv('HOME'),str)) end - local function toppath() - local inputstack = resolvers.inputstack -- dependency, actually the code should move but it's - if not inputstack then -- more convenient to keep it here - return "." - end - local pathname = dirname(inputstack[#inputstack] or "") - if pathname == "" then - return "." - else - return pathname - end -end - -resolvers.toppath = toppath - -prefixes.toppath = function(str) - return cleanpath(joinpath(toppath(),str)) -end - -prefixes.env = prefixes.environment -prefixes.rel = prefixes.relative -prefixes.loc = prefixes.locate -prefixes.kpse = prefixes.locate -prefixes.full = prefixes.locate -prefixes.file = prefixes.filename -prefixes.path = prefixes.pathname - + local inputstack=resolvers.inputstack + if not inputstack then + return "." + end + local pathname=dirname(inputstack[#inputstack] or "") + if pathname=="" then + return "." + else + return pathname + end +end +resolvers.toppath=toppath +prefixes.toppath=function(str) + return cleanpath(joinpath(toppath(),str)) +end +prefixes.env=prefixes.environment +prefixes.rel=prefixes.relative +prefixes.loc=prefixes.locate +prefixes.kpse=prefixes.locate +prefixes.full=prefixes.locate +prefixes.file=prefixes.filename +prefixes.path=prefixes.pathname function resolvers.allprefixes(separator) - local all = table.sortedkeys(prefixes) - if separator then - for i=1,#all do - all[i] = all[i] .. ":" - end + local all=table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i]=all[i]..":" end - return all + end + return all end - local function _resolve_(method,target) - local action = prefixes[method] - if action then - return action(target) - else - return method .. ":" .. target - end -end - -local resolved, abstract = { }, { } - + local action=prefixes[method] + if action then + return action(target) + else + return method..":"..target + end +end +local resolved,abstract={},{} function resolvers.resetresolve(str) - resolved, abstract = { }, { } + resolved,abstract={},{} end - --- todo: use an lpeg (see data-lua for !! / stripper) - --- local function resolve(str) -- use schemes, this one is then for the commandline only --- if type(str) == "table" then --- local t = { } --- for i=1,#str do --- t[i] = resolve(str[i]) --- end --- return t --- else --- local res = resolved[str] --- if not res then --- res = gsub(str,"([a-z][a-z]+):([^ \"\';,]*)",_resolve_) -- home:xx;selfautoparent:xx; etc (comma added) --- resolved[str] = res --- abstract[res] = str --- end --- return res --- end --- end - --- home:xx;selfautoparent:xx; - -local pattern = Cs((C(R("az")^2) * P(":") * C((1-S(" \"\';,"))^1) / _resolve_ + P(1))^0) - -local function resolve(str) -- use schemes, this one is then for the commandline only - if type(str) == "table" then - local t = { } - for i=1,#str do - t[i] = resolve(str[i]) - end - return t - else - local res = resolved[str] - if not res then - res = lpegmatch(pattern,str) - resolved[str] = res - abstract[res] = str - end - return res +local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) +local function resolve(str) + if type(str)=="table" then + local t={} + for i=1,#str do + t[i]=resolve(str[i]) + end + return t + else + local res=resolved[str] + if not res then + res=lpegmatch(pattern,str) + resolved[str]=res + abstract[res]=str end + return res + end end - local function unresolve(str) - return abstract[str] or str + return abstract[str] or str end - -resolvers.resolve = resolve -resolvers.unresolve = unresolve - -if type(os.uname) == "function" then - - for k, v in next, os.uname() do - if not prefixes[k] then - prefixes[k] = function() return v end - end +resolvers.resolve=resolve +resolvers.unresolve=unresolve +if type(os.uname)=="function" then + for k,v in next,os.uname() do + if not prefixes[k] then + prefixes[k]=function() return v end end - + end end - -if os.type == "unix" then - - -- We need to distringuish between a prefix and something else : so we - -- have a special repath variant for linux. Also, when a new prefix is - -- defined, we need to remake the matcher. - - local pattern - - local function makepattern(t,k,v) - if t then - rawset(t,k,v) - end - local colon = P(":") - for k, v in table.sortedpairs(prefixes) do - if p then - p = P(k) + p - else - p = P(k) - end - end - pattern = Cs((p * colon + colon/";" + P(1))^0) - end - - makepattern() - - getmetatable(prefixes).__newindex = makepattern - - function resolvers.repath(str) - return lpegmatch(pattern,str) - end - -else -- already the default: - - function resolvers.repath(str) - return str +if os.type=="unix" then + local pattern + local function makepattern(t,k,v) + if t then + rawset(t,k,v) + end + local colon=P(":") + for k,v in table.sortedpairs(prefixes) do + if p then + p=P(k)+p + else + p=P(k) + end end - + pattern=Cs((p*colon+colon/";"+P(1))^0) + end + makepattern() + getmetatable(prefixes).__newindex=makepattern + function resolvers.repath(str) + return lpegmatch(pattern,str) + end +else + function resolvers.repath(str) + return str + end end @@ -16917,172 +13133,153 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-inp'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local allocate = utilities.storage.allocate -local resolvers = resolvers - -local methodhandler = resolvers.methodhandler -local registermethod = resolvers.registermethod - -local finders = allocate { helpers = { }, notfound = function() end } -local openers = allocate { helpers = { }, notfound = function() end } -local loaders = allocate { helpers = { }, notfound = function() return false, nil, 0 end } +-- original size: 910, stripped down to: 823 -registermethod("finders", finders, "uri") -registermethod("openers", openers, "uri") -registermethod("loaders", loaders, "uri") - -resolvers.finders = finders -resolvers.openers = openers -resolvers.loaders = loaders +if not modules then modules={} end modules ['data-inp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local methodhandler=resolvers.methodhandler +local registermethod=resolvers.registermethod +local finders=allocate { helpers={},notfound=function() end } +local openers=allocate { helpers={},notfound=function() end } +local loaders=allocate { helpers={},notfound=function() return false,nil,0 end } +registermethod("finders",finders,"uri") +registermethod("openers",openers,"uri") +registermethod("loaders",loaders,"uri") +resolvers.finders=finders +resolvers.openers=openers +resolvers.loaders=loaders end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-out'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local allocate = utilities.storage.allocate -local resolvers = resolvers +-- original size: 530, stripped down to: 475 -local registermethod = resolvers.registermethod - -local savers = allocate { helpers = { } } - -resolvers.savers = savers - -registermethod("savers", savers, "uri") +if not modules then modules={} end modules ['data-out']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registermethod=resolvers.registermethod +local savers=allocate { helpers={} } +resolvers.savers=savers +registermethod("savers",savers,"uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-fil'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_files = logs.reporter("resolvers","files") - -local resolvers = resolvers - -local finders, openers, loaders, savers = resolvers.finders, resolvers.openers, resolvers.loaders, resolvers.savers -local locators, hashers, generators, concatinators = resolvers.locators, resolvers.hashers, resolvers.generators, resolvers.concatinators - -local checkgarbage = utilities.garbagecollector and utilities.garbagecollector.check +-- original size: 3818, stripped down to: 3248 +if not modules then modules={} end modules ['data-fil']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_files=logs.reporter("resolvers","files") +local resolvers=resolvers +local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers +local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check function locators.file(specification) - local name = specification.filename - local realname = resolvers.resolve(name) -- no shortcut - if realname and realname ~= '' and lfs.isdir(realname) then - if trace_locating then - report_files("file locator '%s' found as '%s'",name,realname) - end - resolvers.appendhash('file',name,true) -- cache - elseif trace_locating then - report_files("file locator '%s' not found",name) + local name=specification.filename + local realname=resolvers.resolve(name) + if realname and realname~='' and lfs.isdir(realname) then + if trace_locating then + report_files("file locator '%s' found as '%s'",name,realname) end + resolvers.appendhash('file',name,true) + elseif trace_locating then + report_files("file locator '%s' not found",name) + end end - function hashers.file(specification) - local name = specification.filename - local content = caches.loadcontent(name,'files') - resolvers.registerfilehash(name,content,content==nil) + local name=specification.filename + local content=caches.loadcontent(name,'files') + resolvers.registerfilehash(name,content,content==nil) end - function generators.file(specification) - local path = specification.filename - local content = resolvers.scanfiles(path,false,true) -- scan once - resolvers.registerfilehash(path,content,true) + local path=specification.filename + local content=resolvers.scanfiles(path,false,true) + resolvers.registerfilehash(path,content,true) end - -concatinators.file = file.join - +concatinators.file=file.join function finders.file(specification,filetype) - local filename = specification.filename - local foundname = resolvers.findfile(filename,filetype) - if foundname and foundname ~= "" then - if trace_locating then - report_files("file finder: '%s' found",filename) - end - return foundname - else - if trace_locating then - report_files("file finder: %s' not found",filename) - end - return finders.notfound() + local filename=specification.filename + local foundname=resolvers.findfile(filename,filetype) + if foundname and foundname~="" then + if trace_locating then + report_files("file finder: '%s' found",filename) + end + return foundname + else + if trace_locating then + report_files("file finder: %s' not found",filename) end + return finders.notfound() + end end - --- The default textopener will be overloaded later on. - function openers.helpers.textopener(tag,filename,f) - return { - reader = function() return f:read () end, - close = function() logs.show_close(filename) return f:close() end, - } + return { + reader=function() return f:read () end, + close=function() logs.show_close(filename) return f:close() end, + } end - function openers.file(specification,filetype) - local filename = specification.filename - if filename and filename ~= "" then - local f = io.open(filename,"r") - if f then - if trace_locating then - report_files("file opener, '%s' opened",filename) - end - return openers.helpers.textopener("file",filename,f) - end - end - if trace_locating then - report_files("file opener, '%s' not found",filename) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"r") + if f then + if trace_locating then + report_files("file opener, '%s' opened",filename) + end + return openers.helpers.textopener("file",filename,f) end - return openers.notfound() + end + if trace_locating then + report_files("file opener, '%s' not found",filename) + end + return openers.notfound() end - function loaders.file(specification,filetype) - local filename = specification.filename - if filename and filename ~= "" then - local f = io.open(filename,"rb") - if f then - logs.show_load(filename) - if trace_locating then - report_files("file loader, '%s' loaded",filename) - end - local s = f:read("*a") -- io.readall(f) is faster but we never have large files here - if checkgarbage then - checkgarbage(#s) - end - f:close() - if s then - return true, s, #s - end - end - end - if trace_locating then - report_files("file loader, '%s' not found",filename) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"rb") + if f then + logs.show_load(filename) + if trace_locating then + report_files("file loader, '%s' loaded",filename) + end + local s=f:read("*a") + if checkgarbage then + checkgarbage(#s) + end + f:close() + if s then + return true,s,#s + end end - return loaders.notfound() + end + if trace_locating then + report_files("file loader, '%s' not found",filename) + end + return loaders.notfound() end @@ -17090,129 +13287,113 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-con'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local format, lower, gsub = string.format, string.lower, string.gsub - -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) -local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) - - - -containers = containers or { } -local containers = containers -containers.usecache = true - -local report_containers = logs.reporter("resolvers","containers") +-- original size: 4651, stripped down to: 3330 +if not modules then modules={} end modules ['data-con']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub=string.format,string.lower,string.gsub +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) +local trace_storage=false trackers.register("resolvers.storage",function(v) trace_storage=v end) +containers=containers or {} +local containers=containers +containers.usecache=true +local report_containers=logs.reporter("resolvers","containers") local function report(container,tag,name) - if trace_cache or trace_containers then - report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') - end -end - -local allocated = { } - -local mt = { - __index = function(t,k) - if k == "writable" then - local writable = caches.getwritablepath(t.category,t.subcategory) or { "." } - t.writable = writable - return writable - elseif k == "readables" then - local readables = caches.getreadablepaths(t.category,t.subcategory) or { "." } - t.readables = readables - return readables - end - end, - __storage__ = true + if trace_cache or trace_containers then + report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') + end +end +local allocated={} +local mt={ + __index=function(t,k) + if k=="writable" then + local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + t.writable=writable + return writable + elseif k=="readables" then + local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + t.readables=readables + return readables + end + end, + __storage__=true } - -function containers.define(category, subcategory, version, enabled) - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or math.pi, -- after all, this is TeX - trace = false, - -- writable = caches.getwritablepath and caches.getwritablepath (category,subcategory) or { "." }, - -- readables = caches.getreadablepaths and caches.getreadablepaths(category,subcategory) or { "." }, - } - setmetatable(s,mt) - c[subcategory] = s - end - return s +function containers.define(category,subcategory,version,enabled) + if category and subcategory then + local c=allocated[category] + if not c then + c={} + allocated[category]=c end -end - -function containers.is_usable(container, name) - return container.enabled and caches and caches.is_writable(container.writable, name) -end - -function containers.is_valid(container, name) - if name and name ~= "" then - local storage = container.storage[name] - return storage and storage.cache_version == container.version - else - return false + local s=c[subcategory] + if not s then + s={ + category=category, + subcategory=subcategory, + storage={}, + enabled=enabled, + version=version or math.pi, + trace=false, + } + setmetatable(s,mt) + c[subcategory]=s end + return s + end end - -function containers.read(container,name) - local storage = container.storage - local stored = storage[name] - if not stored and container.enabled and caches and containers.usecache then - stored = caches.loaddata(container.readables,name) - if stored and stored.cache_version == container.version then - report(container,"loaded",name) - else - stored = nil - end - storage[name] = stored - elseif stored then - report(container,"reusing",name) - end - return stored +function containers.is_usable(container,name) + return container.enabled and caches and caches.is_writable(container.writable,name) end - -function containers.write(container, name, data) - if data then - data.cache_version = container.version - if container.enabled and caches then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.writable, name, data) - report(container,"saved",name) - data.unique, data.shared = unique, shared - end - report(container,"stored",name) - container.storage[name] = data - end - return data +function containers.is_valid(container,name) + if name and name~="" then + local storage=container.storage[name] + return storage and storage.cache_version==container.version + else + return false + end +end +function containers.read(container,name) + local storage=container.storage + local stored=storage[name] + if not stored and container.enabled and caches and containers.usecache then + stored=caches.loaddata(container.readables,name) + if stored and stored.cache_version==container.version then + report(container,"loaded",name) + else + stored=nil + end + storage[name]=stored + elseif stored then + report(container,"reusing",name) + end + return stored +end +function containers.write(container,name,data) + if data then + data.cache_version=container.version + if container.enabled and caches then + local unique,shared=data.unique,data.shared + data.unique,data.shared=nil,nil + caches.savedata(container.writable,name,data) + report(container,"saved",name) + data.unique,data.shared=unique,shared + end + report(container,"stored",name) + container.storage[name]=data + end + return data end - function containers.content(container,name) - return container.storage[name] + return container.storage[name] end - function containers.cleanname(name) - return (gsub(lower(name),"[^%w%d]+","-")) + return (gsub(lower(name),"[^%w%d]+","-")) end @@ -17220,106 +13401,88 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-use'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_mounts = logs.reporter("resolvers","mounts") - -local resolvers = resolvers - --- we will make a better format, maybe something xml or just text or lua - -resolvers.automounted = resolvers.automounted or { } +-- original size: 3913, stripped down to: 2998 +if not modules then modules={} end modules ['data-use']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_mounts=logs.reporter("resolvers","mounts") +local resolvers=resolvers +resolvers.automounted=resolvers.automounted or {} function resolvers.automount(usecache) - local mountpaths = resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths == 0) and usecache then - mountpaths = caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths > 0 then - statistics.starttiming(resolvers.instance) - for k=1,#mountpaths do - local root = mountpaths[k] - local f = io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then -- or %W - -- skip - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %s",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() + local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) + if (not mountpaths or #mountpaths==0) and usecache then + mountpaths=caches.getreadablepaths("mount") + end + if mountpaths and #mountpaths>0 then + statistics.starttiming(resolvers.instance) + for k=1,#mountpaths do + local root=mountpaths[k] + local f=io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if find(line,"^[%%#%-]") then + elseif find(line,"^zip://") then + if trace_locating then + report_mounts("mounting %s",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) end + end end - statistics.stoptiming(resolvers.instance) - end -end - --- status info - -statistics.register("used config file", function() return caches.configfiles() end) -statistics.register("used cache path", function() return caches.usedpaths() end) - --- experiment (code will move) - -function statistics.savefmtstatus(texname,formatbanner,sourcefile) -- texname == formatname - local enginebanner = status.list().banner - if formatbanner and enginebanner and sourcefile then - local luvname = file.replacesuffix(texname,"luv") -- utilities.lua.suffixes.luv - local luvdata = { - enginebanner = enginebanner, - formatbanner = formatbanner, - sourcehash = md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), - sourcefile = sourcefile, - } - io.savedata(luvname,table.serialize(luvdata,true)) + f:close() + end end + statistics.stoptiming(resolvers.instance) + end +end +statistics.register("used config file",function() return caches.configfiles() end) +statistics.register("used cache path",function() return caches.usedpaths() end) +function statistics.savefmtstatus(texname,formatbanner,sourcefile) + local enginebanner=status.list().banner + if formatbanner and enginebanner and sourcefile then + local luvname=file.replacesuffix(texname,"luv") + local luvdata={ + enginebanner=enginebanner, + formatbanner=formatbanner, + sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcefile=sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) + end end - --- todo: check this at startup and return (say) 999 as signal that the run --- was aborted due to a wrong format in which case mtx-context can trigger --- a remake - function statistics.checkfmtstatus(texname) - local enginebanner = status.list().banner - if enginebanner and texname then - local luvname = file.replacesuffix(texname,"luv") -- utilities.lua.suffixes.luv - if lfs.isfile(luvname) then - local luv = dofile(luvname) - if luv and luv.sourcefile then - local sourcehash = md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") - local luvbanner = luv.enginebanner or "?" - if luvbanner ~= enginebanner then - return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) - end - local luvhash = luv.sourcehash or "?" - if luvhash ~= sourcehash then - return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) - end - else - return "invalid status file" - end - else - return "missing status file" - end + local enginebanner=status.list().banner + if enginebanner and texname then + local luvname=file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv=dofile(luvname) + if luv and luv.sourcefile then + local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local luvbanner=luv.enginebanner or "?" + if luvbanner~=enginebanner then + return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) + end + local luvhash=luv.sourcehash or "?" + if luvhash~=sourcehash then + return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) + end + else + return "invalid status file" + end + else + return "missing status file" end - return true + end + return true end @@ -17327,259 +13490,233 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-zip'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- partly redone .. needs testing - -local format, find, match = string.format, string.find, string.match +-- original size: 8537, stripped down to: 6805 -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_zip = logs.reporter("resolvers","zip") - - - -local resolvers = resolvers - -zip = zip or { } -local zip = zip - -zip.archives = zip.archives or { } -local archives = zip.archives - -zip.registeredfiles = zip.registeredfiles or { } -local registeredfiles = zip.registeredfiles - -local limited = false - -directives.register("system.inputmode", function(v) - if not limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - zip.open = i_limiter.protect(zip.open) - limited = true - end - end +if not modules then modules={} end modules ['data-zip']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,find,match=string.format,string.find,string.match +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_zip=logs.reporter("resolvers","zip") +local resolvers=resolvers +zip=zip or {} +local zip=zip +zip.archives=zip.archives or {} +local archives=zip.archives +zip.registeredfiles=zip.registeredfiles or {} +local registeredfiles=zip.registeredfiles +local limited=false +directives.register("system.inputmode",function(v) + if not limited then + local i_limiter=io.i_limiter(v) + if i_limiter then + zip.open=i_limiter.protect(zip.open) + limited=true + end + end end) - -local function validzip(str) -- todo: use url splitter - if not find(str,"^zip://") then - return "zip:///" .. str - else - return str - end +local function validzip(str) + if not find(str,"^zip://") then + return "zip:///"..str + else + return str + end end - function zip.openarchive(name) - if not name or name == "" then - return nil - else - local arch = archives[name] - if not arch then - local full = resolvers.findfile(name) or "" - arch = (full ~= "" and zip.open(full)) or false - archives[name] = arch - end - return arch + if not name or name=="" then + return nil + else + local arch=archives[name] + if not arch then + local full=resolvers.findfile(name) or "" + arch=(full~="" and zip.open(full)) or false + archives[name]=arch end + return arch + end end - function zip.closearchive(name) - if not name or (name == "" and archives[name]) then - zip.close(archives[name]) - archives[name] = nil - end + if not name or (name=="" and archives[name]) then + zip.close(archives[name]) + archives[name]=nil + end end - function resolvers.locators.zip(specification) - local archive = specification.filename - local zipfile = archive and archive ~= "" and zip.openarchive(archive) -- tricky, could be in to be initialized tree - if trace_locating then - if zipfile then - report_zip("locator, archive '%s' found",archive) - else - report_zip("locator, archive '%s' not found",archive) - end + local archive=specification.filename + local zipfile=archive and archive~="" and zip.openarchive(archive) + if trace_locating then + if zipfile then + report_zip("locator, archive '%s' found",archive) + else + report_zip("locator, archive '%s' not found",archive) end + end end - function resolvers.hashers.zip(specification) - local archive = specification.filename - if trace_locating then - report_zip("loading file '%s'",archive) - end - resolvers.usezipfile(specification.original) -end - -function resolvers.concatinators.zip(zipfile,path,name) -- ok ? - if not path or path == "" then - return format('%s?name=%s',zipfile,name) - else - return format('%s?name=%s/%s',zipfile,path,name) - end + local archive=specification.filename + if trace_locating then + report_zip("loading file '%s'",archive) + end + resolvers.usezipfile(specification.original) +end +function resolvers.concatinators.zip(zipfile,path,name) + if not path or path=="" then + return format('%s?name=%s',zipfile,name) + else + return format('%s?name=%s/%s',zipfile,path,name) + end end - function resolvers.finders.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("finder, archive '%s' found",archive) - end - local dfile = zfile:open(queryname) - if dfile then - dfile = zfile:close() - if trace_locating then - report_zip("finder, file '%s' found",queryname) - end - return specification.original - elseif trace_locating then - report_zip("finder, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("finder, unknown archive '%s'",archive) - end + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("finder, archive '%s' found",archive) end + local dfile=zfile:open(queryname) + if dfile then + dfile=zfile:close() + if trace_locating then + report_zip("finder, file '%s' found",queryname) + end + return specification.original + elseif trace_locating then + report_zip("finder, file '%s' not found",queryname) + end + elseif trace_locating then + report_zip("finder, unknown archive '%s'",archive) + end end - if trace_locating then - report_zip("finder, '%s' not found",original) - end - return resolvers.finders.notfound() + end + if trace_locating then + report_zip("finder, '%s' not found",original) + end + return resolvers.finders.notfound() end - function resolvers.openers.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("opener, archive '%s' opened",archive) - end - local dfile = zfile:open(queryname) - if dfile then - if trace_locating then - report_zip("opener, file '%s' found",queryname) - end - return resolvers.openers.helpers.textopener('zip',original,dfile) - elseif trace_locating then - report_zip("opener, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("opener, unknown archive '%s'",archive) - end + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("opener, archive '%s' opened",archive) end + local dfile=zfile:open(queryname) + if dfile then + if trace_locating then + report_zip("opener, file '%s' found",queryname) + end + return resolvers.openers.helpers.textopener('zip',original,dfile) + elseif trace_locating then + report_zip("opener, file '%s' not found",queryname) + end + elseif trace_locating then + report_zip("opener, unknown archive '%s'",archive) + end end - if trace_locating then - report_zip("opener, '%s' not found",original) - end - return resolvers.openers.notfound() + end + if trace_locating then + report_zip("opener, '%s' not found",original) + end + return resolvers.openers.notfound() end - function resolvers.loaders.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("loader, archive '%s' opened",archive) - end - local dfile = zfile:open(queryname) - if dfile then - logs.show_load(original) - if trace_locating then - report_zip("loader, file '%s' loaded",original) - end - local s = dfile:read("*all") - dfile:close() - return true, s, #s - elseif trace_locating then - report_zip("loader, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("loader, unknown archive '%s'",archive) - end + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("loader, archive '%s' opened",archive) end + local dfile=zfile:open(queryname) + if dfile then + logs.show_load(original) + if trace_locating then + report_zip("loader, file '%s' loaded",original) + end + local s=dfile:read("*all") + dfile:close() + return true,s,#s + elseif trace_locating then + report_zip("loader, file '%s' not found",queryname) + end + elseif trace_locating then + report_zip("loader, unknown archive '%s'",archive) + end end - if trace_locating then - report_zip("loader, '%s' not found",original) - end - return resolvers.openers.notfound() + end + if trace_locating then + report_zip("loader, '%s' not found",original) + end + return resolvers.openers.notfound() end - --- zip:///somefile.zip --- zip:///somefile.zip?tree=texmf-local -> mount - function resolvers.usezipfile(archive) - local specification = resolvers.splitmethod(archive) -- to be sure - local archive = specification.filename - if archive and not registeredfiles[archive] then - local z = zip.openarchive(archive) - if z then - local instance = resolvers.instance - local tree = url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering, registering archive '%s'",archive) - end - statistics.starttiming(instance) - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) -- resets hashes too - registeredfiles[archive] = z - instance.files[archive] = resolvers.registerzipfile(z,tree) - statistics.stoptiming(instance) - elseif trace_locating then - report_zip("registering, unknown archive '%s'",archive) - end + local specification=resolvers.splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=zip.openarchive(archive) + if z then + local instance=resolvers.instance + local tree=url.query(specification.query).tree or "" + if trace_locating then + report_zip("registering, registering archive '%s'",archive) + end + statistics.starttiming(instance) + resolvers.prependhash('zip',archive) + resolvers.extendtexmfvariable(archive) + registeredfiles[archive]=z + instance.files[archive]=resolvers.registerzipfile(z,tree) + statistics.stoptiming(instance) elseif trace_locating then - report_zip("registering, '%s' not found",archive) + report_zip("registering, unknown archive '%s'",archive) end + elseif trace_locating then + report_zip("registering, '%s' not found",archive) + end end - function resolvers.registerzipfile(z,tree) - local files, filter = { }, "" - if tree == "" then - filter = "^(.+)/(.-)$" + local files,filter={},"" + if tree=="" then + filter="^(.+)/(.-)$" + else + filter=format("^%s/(.+)/(.-)$",tree) + end + if trace_locating then + report_zip("registering, using filter '%s'",filter) + end + local register,n=resolvers.registerfile,0 + for i in z:files() do + local path,name=match(i.filename,filter) + if path then + if name and name~='' then + register(files,name,path) + n=n+1 + else + end else - filter = format("^%s/(.+)/(.-)$",tree) - end - if trace_locating then - report_zip("registering, using filter '%s'",filter) - end - local register, n = resolvers.registerfile, 0 - for i in z:files() do - local path, name = match(i.filename,filter) - if path then - if name and name ~= '' then - register(files, name, path) - n = n + 1 - else - -- directory - end - else - register(files, i.filename, '') - n = n + 1 - end + register(files,i.filename,'') + n=n+1 end - report_zip("registering, %s files registered",n) - return files + end + report_zip("registering, %s files registered",n) + return files end @@ -17587,286 +13724,244 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tre'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- \input tree://oeps1/**/oeps.tex - -local find, gsub, format = string.find, string.gsub, string.format - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_trees = logs.reporter("resolvers","trees") - -local resolvers = resolvers - -local done, found, notfound = { }, { }, resolvers.finders.notfound +-- original size: 2514, stripped down to: 2080 +if not modules then modules={} end modules ['data-tre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,gsub,format=string.find,string.gsub,string.format +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_trees=logs.reporter("resolvers","trees") +local resolvers=resolvers +local done,found,notfound={},{},resolvers.finders.notfound function resolvers.finders.tree(specification) - local spec = specification.filename - local fnd = found[spec] - if fnd == nil then - if spec ~= "" then - local path, name = file.dirname(spec), file.basename(spec) - if path == "" then path = "." end - local hash = done[path] - if not hash then - local pattern = path .. "/*" -- we will use the proper splitter - hash = dir.glob(pattern) - done[path] = hash - end - local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$" - for k=1,#hash do - local v = hash[k] - if find(v,pattern) then - found[spec] = v - return v - end - end + local spec=specification.filename + local fnd=found[spec] + if fnd==nil then + if spec~="" then + local path,name=file.dirname(spec),file.basename(spec) + if path=="" then path="." end + local hash=done[path] + if not hash then + local pattern=path.."/*" + hash=dir.glob(pattern) + done[path]=hash + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for k=1,#hash do + local v=hash[k] + if find(v,pattern) then + found[spec]=v + return v end - fnd = notfound() -- false - found[spec] = fnd + end end - return fnd + fnd=notfound() + found[spec]=fnd + end + return fnd end - function resolvers.locators.tree(specification) - local name = specification.filename - local realname = resolvers.resolve(name) -- no shortcut - if realname and realname ~= '' and lfs.isdir(realname) then - if trace_locating then - report_trees("locator '%s' found",realname) - end - resolvers.appendhash('tree',name,false) -- don't cache - elseif trace_locating then - report_trees("locator '%s' not found",name) + local name=specification.filename + local realname=resolvers.resolve(name) + if realname and realname~='' and lfs.isdir(realname) then + if trace_locating then + report_trees("locator '%s' found",realname) end + resolvers.appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator '%s' not found",name) + end end - function resolvers.hashers.tree(specification) - local name = specification.filename - if trace_locating then - report_trees("analysing '%s'",name) - end - resolvers.methodhandler("hashers",name) - - resolvers.generators.file(specification) + local name=specification.filename + if trace_locating then + report_trees("analysing '%s'",name) + end + resolvers.methodhandler("hashers",name) + resolvers.generators.file(specification) end - -resolvers.concatinators.tree = resolvers.concatinators.file -resolvers.generators.tree = resolvers.generators.file -resolvers.openers.tree = resolvers.openers.file -resolvers.loaders.tree = resolvers.loaders.file +resolvers.concatinators.tree=resolvers.concatinators.file +resolvers.generators.tree=resolvers.generators.file +resolvers.openers.tree=resolvers.openers.file +resolvers.loaders.tree=resolvers.loaders.file end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-sch'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local load = load -local gsub, concat, format = string.gsub, table.concat, string.format -local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders - -local trace_schemes = false trackers.register("resolvers.schemes",function(v) trace_schemes = v end) -local report_schemes = logs.reporter("resolvers","schemes") - -local http = require("socket.http") -local ltn12 = require("ltn12") - -local resolvers = resolvers -local schemes = resolvers.schemes or { } -resolvers.schemes = schemes - -local cleaners = { } -schemes.cleaners = cleaners - -local threshold = 24 * 60 * 60 - -directives.register("schemes.threshold", function(v) threshold = tonumber(v) or threshold end) +-- original size: 6218, stripped down to: 5165 +if not modules then modules={} end modules ['data-sch']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local load=load +local gsub,concat,format=string.gsub,table.concat,string.format +local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) +local report_schemes=logs.reporter("resolvers","schemes") +local http=require("socket.http") +local ltn12=require("ltn12") +local resolvers=resolvers +local schemes=resolvers.schemes or {} +resolvers.schemes=schemes +local cleaners={} +schemes.cleaners=cleaners +local threshold=24*60*60 +directives.register("schemes.threshold",function(v) threshold=tonumber(v) or threshold end) function cleaners.none(specification) - return specification.original + return specification.original end - function cleaners.strip(specification) - return (gsub(specification.original,"[^%a%d%.]+","-")) -- so we keep periods + return (gsub(specification.original,"[^%a%d%.]+","-")) end - function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) end - -local cleaner = cleaners.strip - -directives.register("schemes.cleanmethod", function(v) cleaner = cleaners[v] or cleaners.strip end) - +local cleaner=cleaners.strip +directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) function resolvers.schemes.cleanname(specification) - local hash = cleaner(specification) - if trace_schemes then - report_schemes("hashing %s to %s",specification.original,hash) - end - return hash + local hash=cleaner(specification) + if trace_schemes then + report_schemes("hashing %s to %s",specification.original,hash) + end + return hash end - -local cached, loaded, reused, thresholds, handlers = { }, { }, { }, { }, { } - -local function runcurl(name,cachename) -- we use sockets instead or the curl library when possible - local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name - os.spawn(command) +local cached,loaded,reused,thresholds,handlers={},{},{},{},{} +local function runcurl(name,cachename) + local command="curl --silent --create-dirs --output "..cachename.." "..name + os.spawn(command) end - local function fetch(specification) - local original = specification.original - local scheme = specification.scheme - local cleanname = schemes.cleanname(specification) - local cachename = caches.setfirstwritablefile(cleanname,"schemes") - if not cached[original] then - statistics.starttiming(schemes) - if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > (thresholds[protocol] or threshold)) then - cached[original] = cachename - local handler = handlers[scheme] - if handler then - if trace_schemes then - report_schemes("fetching '%s', protocol '%s', method 'built-in'",original,scheme) - end - logs.flush() - handler(specification,cachename) - else - if trace_schemes then - report_schemes("fetching '%s', protocol '%s', method 'curl'",original,scheme) - end - logs.flush() - runcurl(original,cachename) - end - end - if io.exists(cachename) then - cached[original] = cachename - if trace_schemes then - report_schemes("using cached '%s', protocol '%s', cachename '%s'",original,scheme,cachename) - end - else - cached[original] = "" - if trace_schemes then - report_schemes("using missing '%s', protocol '%s'",original,scheme) - end + local original=specification.original + local scheme=specification.scheme + local cleanname=schemes.cleanname(specification) + local cachename=caches.setfirstwritablefile(cleanname,"schemes") + if not cached[original] then + statistics.starttiming(schemes) + if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification)>(thresholds[protocol] or threshold)) then + cached[original]=cachename + local handler=handlers[scheme] + if handler then + if trace_schemes then + report_schemes("fetching '%s', protocol '%s', method 'built-in'",original,scheme) end - loaded[scheme] = loaded[scheme] + 1 - statistics.stoptiming(schemes) - else + logs.flush() + handler(specification,cachename) + else if trace_schemes then - report_schemes("reusing '%s', protocol '%s'",original,scheme) + report_schemes("fetching '%s', protocol '%s', method 'curl'",original,scheme) end - reused[scheme] = reused[scheme] + 1 + logs.flush() + runcurl(original,cachename) + end + end + if io.exists(cachename) then + cached[original]=cachename + if trace_schemes then + report_schemes("using cached '%s', protocol '%s', cachename '%s'",original,scheme,cachename) + end + else + cached[original]="" + if trace_schemes then + report_schemes("using missing '%s', protocol '%s'",original,scheme) + end + end + loaded[scheme]=loaded[scheme]+1 + statistics.stoptiming(schemes) + else + if trace_schemes then + report_schemes("reusing '%s', protocol '%s'",original,scheme) end - return cached[original] + reused[scheme]=reused[scheme]+1 + end + return cached[original] end - local function finder(specification,filetype) - return resolvers.methodhandler("finders",fetch(specification),filetype) + return resolvers.methodhandler("finders",fetch(specification),filetype) end - -local opener = openers.file -local loader = loaders.file - +local opener=openers.file +local loader=loaders.file local function install(scheme,handler,newthreshold) - handlers [scheme] = handler - loaded [scheme] = 0 - reused [scheme] = 0 - finders [scheme] = finder - openers [scheme] = opener - loaders [scheme] = loader - thresholds[scheme] = newthreshold or threshold -end - -schemes.install = install - + handlers [scheme]=handler + loaded [scheme]=0 + reused [scheme]=0 + finders [scheme]=finder + openers [scheme]=opener + loaders [scheme]=loader + thresholds[scheme]=newthreshold or threshold +end +schemes.install=install local function http_handler(specification,cachename) - local tempname = cachename .. ".tmp" - local f = io.open(tempname,"wb") - local status, message = http.request { - url = specification.original, - sink = ltn12.sink.file(f) - } - if not status then - os.remove(tempname) - else - os.remove(cachename) - os.rename(tempname,cachename) - end - return cachename + local tempname=cachename..".tmp" + local f=io.open(tempname,"wb") + local status,message=http.request { + url=specification.original, + sink=ltn12.sink.file(f) + } + if not status then + os.remove(tempname) + else + os.remove(cachename) + os.rename(tempname,cachename) + end + return cachename end - install('http',http_handler) -install('https') -- see pod +install('https') install('ftp') - -statistics.register("scheme handling time", function() - local l, r, nl, nr = { }, { }, 0, 0 - for k, v in table.sortedhash(loaded) do - if v > 0 then - nl = nl + 1 - l[nl] = k .. ":" .. v - end - end - for k, v in table.sortedhash(reused) do - if v > 0 then - nr = nr + 1 - r[nr] = k .. ":" .. v - end - end - local n = nl + nr - if n > 0 then - l = nl > 0 and concat(l) or "none" - r = nr > 0 and concat(r) or "none" - return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes), n, threshold, l, r) - else - return nil - end +statistics.register("scheme handling time",function() + local l,r,nl,nr={},{},0,0 + for k,v in table.sortedhash(loaded) do + if v>0 then + nl=nl+1 + l[nl]=k..":"..v + end + end + for k,v in table.sortedhash(reused) do + if v>0 then + nr=nr+1 + r[nr]=k..":"..v + end + end + local n=nl+nr + if n>0 then + l=nl>0 and concat(l) or "none" + r=nr>0 and concat(r) or "none" + return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", + statistics.elapsedtime(schemes),n,threshold,l,r) + else + return nil + end end) - --- We provide a few more helpers: - ------ http = require("socket.http") -local httprequest = http.request -local toquery = url.toquery - --- local function httprequest(url) --- return os.resultof(format("curl --silent %q", url)) --- end - +local httprequest=http.request +local toquery=url.toquery local function fetchstring(url,data) - local q = data and toquery(data) - if q then - url = url .. "?" .. q - end - local reply = httprequest(url) - return reply -- just one argument -end - -schemes.fetchstring = fetchstring - + local q=data and toquery(data) + if q then + url=url.."?"..q + end + local reply=httprequest(url) + return reply +end +schemes.fetchstring=fetchstring function schemes.fetchtable(url,data) - local reply = fetchstring(url,data) - if reply then - local s = load("return " .. reply) - if s then - return s() - end + local reply=fetchstring(url,data) + if reply then + local s=load("return "..reply) + if s then + return s() end + end end @@ -17874,278 +13969,241 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- We overload the regular loader. We do so because we operate mostly in --- tds and use our own loader code. Alternatively we could use a more --- extensive definition of package.path and package.cpath but even then --- we're not done. Also, we now have better tracing. --- --- -- local mylib = require("libtest") --- -- local mysql = require("luasql.mysql") - -local searchers = package.searchers or package.loaders - -local concat = table.concat - -local trace_libraries = false - -trackers.register("resolvers.libraries", function(v) trace_libraries = v end) -trackers.register("resolvers.locating", function(v) trace_libraries = v end) - -local report_libraries = logs.reporter("resolvers","libraries") - -local gsub, insert = string.gsub, table.insert -local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match -local unpack = unpack or table.unpack -local is_readable = file.is_readable - -local resolvers, package = resolvers, package - -local libsuffixes = { 'tex', 'lua' } -local clibsuffixes = { 'lib' } -local libformats = { 'TEXINPUTS', 'LUAINPUTS' } -local clibformats = { 'CLUAINPUTS' } - -local libpaths = nil -local clibpaths = nil -local libhash = { } -local clibhash = { } -local libextras = { } -local clibextras = { } +-- original size: 6387, stripped down to: 5108 -local pattern = Cs(P("!")^0 / "" * (P("/") * P(-1) / "/" + P("/")^1 / "/" + 1)^0) - -local function cleanpath(path) --hm, don't we have a helper for this? - return resolvers.resolve(lpegmatch(pattern,path)) +if not modules then modules={} end modules ['data-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local searchers=package.searchers or package.loaders +local concat=table.concat +local trace_libraries=false +trackers.register("resolvers.libraries",function(v) trace_libraries=v end) +trackers.register("resolvers.locating",function(v) trace_libraries=v end) +local report_libraries=logs.reporter("resolvers","libraries") +local gsub,insert=string.gsub,table.insert +local P,Cs,lpegmatch=lpeg.P,lpeg.Cs,lpeg.match +local unpack=unpack or table.unpack +local is_readable=file.is_readable +local resolvers,package=resolvers,package +local libsuffixes={ 'tex','lua' } +local clibsuffixes={ 'lib' } +local libformats={ 'TEXINPUTS','LUAINPUTS' } +local clibformats={ 'CLUAINPUTS' } +local libpaths=nil +local clibpaths=nil +local libhash={} +local clibhash={} +local libextras={} +local clibextras={} +local pattern=Cs(P("!")^0/""*(P("/")*P(-1)/"/"+P("/")^1/"/"+1)^0) +local function cleanpath(path) + return resolvers.resolve(lpegmatch(pattern,path)) end - local function getlibpaths() - if not libpaths then - libpaths = { } - for i=1,#libformats do - local paths = resolvers.expandedpathlistfromvariable(libformats[i]) - for i=1,#paths do - local path = cleanpath(paths[i]) - if not libhash[path] then - libpaths[#libpaths+1] = path - libhash[path] = true - end - end + if not libpaths then + libpaths={} + for i=1,#libformats do + local paths=resolvers.expandedpathlistfromvariable(libformats[i]) + for i=1,#paths do + local path=cleanpath(paths[i]) + if not libhash[path] then + libpaths[#libpaths+1]=path + libhash[path]=true end + end end - return libpaths + end + return libpaths end - local function getclibpaths() - if not clibpaths then - clibpaths = { } - for i=1,#clibformats do - local paths = resolvers.expandedpathlistfromvariable(clibformats[i]) - for i=1,#paths do - local path = cleanpath(paths[i]) - if not clibhash[path] then - clibpaths[#clibpaths+1] = path - clibhash[path] = true - end - end + if not clibpaths then + clibpaths={} + for i=1,#clibformats do + local paths=resolvers.expandedpathlistfromvariable(clibformats[i]) + for i=1,#paths do + local path=cleanpath(paths[i]) + if not clibhash[path] then + clibpaths[#clibpaths+1]=path + clibhash[path]=true end + end end - return clibpaths + end + return clibpaths end - -package.libpaths = getlibpaths -package.clibpaths = getclibpaths - +package.libpaths=getlibpaths +package.clibpaths=getclibpaths function package.extralibpath(...) - local libpaths = getlibpaths() - local paths = table.flattened { ... } - for i=1,#paths do - local path = cleanpath(paths[i]) - if not libhash[path] then - if trace_libraries then - report_libraries("! extra lua path '%s'",path) - end - libextras[#libextras+1] = path - libpaths[#libpaths +1] = path - end + local libpaths=getlibpaths() + local paths=table.flattened {... } + for i=1,#paths do + local path=cleanpath(paths[i]) + if not libhash[path] then + if trace_libraries then + report_libraries("! extra lua path '%s'",path) + end + libextras[#libextras+1]=path + libpaths[#libpaths+1]=path end + end end - function package.extraclibpath(...) - local clibpaths = getclibpaths() - local paths = table.flattened { ... } - for i=1,#paths do - local path = cleanpath(paths[i]) - if not clibhash[path] then - if trace_libraries then - report_libraries("! extra lib path '%s'",path) - end - clibextras[#clibextras+1] = path - clibpaths[#clibpaths +1] = path - end + local clibpaths=getclibpaths() + local paths=table.flattened {... } + for i=1,#paths do + local path=cleanpath(paths[i]) + if not clibhash[path] then + if trace_libraries then + report_libraries("! extra lib path '%s'",path) + end + clibextras[#clibextras+1]=path + clibpaths[#clibpaths+1]=path end + end end - if not searchers[-2] then - -- use package-path and package-cpath - searchers[-2] = searchers[2] + searchers[-2]=searchers[2] end - local function loadedaslib(resolved,rawname) - return package.loadlib(resolved,"luaopen_" .. gsub(rawname,"%.","_")) + return package.loadlib(resolved,"luaopen_"..gsub(rawname,"%.","_")) end - local function loadedbylua(name) - if trace_libraries then - report_libraries("! locating %q using normal loader",name) - end - local resolved = searchers[-2](name) + if trace_libraries then + report_libraries("! locating %q using normal loader",name) + end + local resolved=searchers[-2](name) end - local function loadedbyformat(name,rawname,suffixes,islib) + if trace_libraries then + report_libraries("! locating %q as %q using formats %q",rawname,name,concat(suffixes)) + end + for i=1,#suffixes do + local format=suffixes[i] + local resolved=resolvers.findfile(name,format) or "" if trace_libraries then - report_libraries("! locating %q as %q using formats %q",rawname,name,concat(suffixes)) + report_libraries("! checking for %q' using format %q",name,format) end - for i=1,#suffixes do -- so we use findfile and not a lookup loop - local format = suffixes[i] - local resolved = resolvers.findfile(name,format) or "" - if trace_libraries then - report_libraries("! checking for %q' using format %q",name,format) - end - if resolved ~= "" then - if trace_libraries then - report_libraries("! lib %q located on %q",name,resolved) - end - if islib then - return loadedaslib(resolved,rawname) - else - return loadfile(resolved) - end - end + if resolved~="" then + if trace_libraries then + report_libraries("! lib %q located on %q",name,resolved) + end + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) + end end + end end - local function loadedbypath(name,rawname,paths,islib,what) - if trace_libraries then - report_libraries("! locating %q as %q on %q paths",rawname,name,what) - end - for p=1,#paths do - local path = paths[p] - local resolved = file.join(path,name) - if trace_libraries then -- mode detail - report_libraries("! checking for %q using %q path %q",name,what,path) - end - if is_readable(resolved) then - if trace_libraries then - report_libraries("! lib %q located on %q",name,resolved) - end - if islib then - return loadedaslib(resolved,rawname) - else - return loadfile(resolved) - end - end + if trace_libraries then + report_libraries("! locating %q as %q on %q paths",rawname,name,what) + end + for p=1,#paths do + local path=paths[p] + local resolved=file.join(path,name) + if trace_libraries then + report_libraries("! checking for %q using %q path %q",name,what,path) + end + if is_readable(resolved) then + if trace_libraries then + report_libraries("! lib %q located on %q",name,resolved) + end + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) + end end + end end - local function notloaded(name) - if trace_libraries then - report_libraries("? unable to locate library %q",name) - end -end - -searchers[2] = function(name) - local thename = gsub(name,"%.","/") - local luaname = file.addsuffix(thename,"lua") - local libname = file.addsuffix(thename,os.libsuffix) - return - loadedbyformat(luaname,name,libsuffixes, false) - or loadedbyformat(libname,name,clibsuffixes, true) - or loadedbypath (luaname,name,getlibpaths (),false,"lua") - or loadedbypath (luaname,name,getclibpaths(),false,"lua") - or loadedbypath (libname,name,getclibpaths(),true, "lib") - or loadedbylua (name) - or notloaded (name) -end - --- searchers[3] = nil --- searchers[4] = nil - -resolvers.loadlualib = require + if trace_libraries then + report_libraries("? unable to locate library %q",name) + end +end +searchers[2]=function(name) + local thename=gsub(name,"%.","/") + local luaname=file.addsuffix(thename,"lua") + local libname=file.addsuffix(thename,os.libsuffix) + return + loadedbyformat(luaname,name,libsuffixes,false) + or loadedbyformat(libname,name,clibsuffixes,true) + or loadedbypath (luaname,name,getlibpaths (),false,"lua") + or loadedbypath (luaname,name,getclibpaths(),false,"lua") + or loadedbypath (libname,name,getclibpaths(),true,"lib") + or loadedbylua (name) + or notloaded (name) +end +resolvers.loadlualib=require end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-aux'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local find = string.find -local type, next = type, next - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local resolvers = resolvers - -local report_scripts = logs.reporter("resolvers","scripts") +-- original size: 2394, stripped down to: 2005 -function resolvers.updatescript(oldname,newname) -- oldname -> own.name, not per se a suffix - local scriptpath = "scripts/context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = resolvers.cleanpath(oldname) +if not modules then modules={} end modules ['data-aux']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find=string.find +local type,next=type,next +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local resolvers=resolvers +local report_scripts=logs.reporter("resolvers","scripts") +function resolvers.updatescript(oldname,newname) + local scriptpath="scripts/context/lua" + newname=file.addsuffix(newname,"lua") + local oldscript=resolvers.cleanpath(oldname) + if trace_locating then + report_scripts("to be replaced old script %s",oldscript) + end + local newscripts=resolvers.findfiles(newname) or {} + if #newscripts==0 then if trace_locating then - report_scripts("to be replaced old script %s", oldscript) + report_scripts("unable to locate new script") end - local newscripts = resolvers.findfiles(newname) or { } - if #newscripts == 0 then + else + for i=1,#newscripts do + local newscript=resolvers.cleanpath(newscripts[i]) + if trace_locating then + report_scripts("checking new script %s",newscript) + end + if oldscript==newscript then if trace_locating then - report_scripts("unable to locate new script") + report_scripts("old and new script are the same") end - else - for i=1,#newscripts do - local newscript = resolvers.cleanpath(newscripts[i]) - if trace_locating then - report_scripts("checking new script %s", newscript) - end - if oldscript == newscript then - if trace_locating then - report_scripts("old and new script are the same") - end - elseif not find(newscript,scriptpath) then - if trace_locating then - report_scripts("new script should come from %s",scriptpath) - end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_locating then - report_scripts("invalid new script name") - end - else - local newdata = io.loaddata(newscript) - if newdata then - if trace_locating then - report_scripts("old script content replaced by new content") - end - io.savedata(oldscript,newdata) - break - elseif trace_locating then - report_scripts("unable to load new script") - end - end + elseif not find(newscript,scriptpath) then + if trace_locating then + report_scripts("new script should come from %s",scriptpath) + end + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_locating then + report_scripts("invalid new script name") + end + else + local newdata=io.loaddata(newscript) + if newdata then + if trace_locating then + report_scripts("old script content replaced by new content") + end + io.savedata(oldscript,newdata) + break + elseif trace_locating then + report_scripts("unable to load new script") end + end end + end end @@ -18153,78 +14211,53 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tmf'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local resolvers = resolvers - -local report_tds = logs.reporter("resolvers","tds") - --- = << --- ? ?? --- < += --- > =+ +-- original size: 2610, stripped down to: 1637 +if not modules then modules={} end modules ['data-tmf']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers=resolvers +local report_tds=logs.reporter("resolvers","tds") function resolvers.load_tree(tree,resolve) - if type(tree) == "string" and tree ~= "" then - - local getenv, setenv = resolvers.getenv, resolvers.setenv - - -- later might listen to the raw osenv var as well - local texos = "texmf-" .. os.platform - - local oldroot = environment.texroot - local newroot = file.collapsepath(tree) - - local newtree = file.join(newroot,texos) - local newpath = file.join(newtree,"bin") - - if not lfs.isdir(newtree) then - report_tds("no '%s' under tree %s",texos,tree) - os.exit() - end - if not lfs.isdir(newpath) then - report_tds("no '%s/bin' under tree %s",texos,tree) - os.exit() - end - - local texmfos = newtree - - environment.texroot = newroot - environment.texos = texos - environment.texmfos = texmfos - - -- Beware, we need to obey the relocatable autoparent so we - -- set TEXMFCNF to its raw value. This is somewhat tricky when - -- we run a mkii job from within. Therefore, in mtxrun, there - -- is a resolve applied when we're in mkii/kpse mode or when - -- --resolve is passed to mtxrun. Maybe we should also set the - -- local AUTOPARENT etc. although these are alwasy set new. - - if resolve then - -- resolvers.luacnfspec = resolvers.joinpath(resolvers.resolve(resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)))) - resolvers.luacnfspec = resolvers.resolve(resolvers.luacnfspec) - end - - setenv('SELFAUTOPARENT', newroot) - setenv('SELFAUTODIR', newtree) - setenv('SELFAUTOLOC', newpath) - setenv('TEXROOT', newroot) - setenv('TEXOS', texos) - setenv('TEXMFOS', texmfos) - setenv('TEXMFCNF', resolvers.luacnfspec,true) -- already resolved - setenv('PATH', newpath .. io.pathseparator .. getenv('PATH')) - - report_tds("changing from root '%s' to '%s'",oldroot,newroot) - report_tds("prepending '%s' to PATH",newpath) - report_tds("setting TEXMFCNF to '%s'",resolvers.luacnfspec) - report_tds() + if type(tree)=="string" and tree~="" then + local getenv,setenv=resolvers.getenv,resolvers.setenv + local texos="texmf-"..os.platform + local oldroot=environment.texroot + local newroot=file.collapsepath(tree) + local newtree=file.join(newroot,texos) + local newpath=file.join(newtree,"bin") + if not lfs.isdir(newtree) then + report_tds("no '%s' under tree %s",texos,tree) + os.exit() + end + if not lfs.isdir(newpath) then + report_tds("no '%s/bin' under tree %s",texos,tree) + os.exit() + end + local texmfos=newtree + environment.texroot=newroot + environment.texos=texos + environment.texmfos=texmfos + if resolve then + resolvers.luacnfspec=resolvers.resolve(resolvers.luacnfspec) end + setenv('SELFAUTOPARENT',newroot) + setenv('SELFAUTODIR',newtree) + setenv('SELFAUTOLOC',newpath) + setenv('TEXROOT',newroot) + setenv('TEXOS',texos) + setenv('TEXMFOS',texmfos) + setenv('TEXMFCNF',resolvers.luacnfspec,true) + setenv('PATH',newpath..io.pathseparator..getenv('PATH')) + report_tds("changing from root '%s' to '%s'",oldroot,newroot) + report_tds("prepending '%s' to PATH",newpath) + report_tds("setting TEXMFCNF to '%s'",resolvers.luacnfspec) + report_tds() + end end @@ -18232,81 +14265,74 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-lst'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- used in mtxrun, can be loaded later .. todo - -local find, concat, upper, format = string.find, table.concat, string.upper, string.format -local fastcopy, sortedpairs = table.fastcopy, table.sortedpairs - -resolvers.listers = resolvers.listers or { } - -local resolvers = resolvers - -local report_lists = logs.reporter("resolvers","lists") +-- original size: 2632, stripped down to: 2278 +if not modules then modules={} end modules ['data-lst']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,concat,upper,format=string.find,table.concat,string.upper,string.format +local fastcopy,sortedpairs=table.fastcopy,table.sortedpairs +resolvers.listers=resolvers.listers or {} +local resolvers=resolvers +local report_lists=logs.reporter("resolvers","lists") local function tabstr(str) - if type(str) == 'table' then - return concat(str," | ") - else - return str - end + if type(str)=='table' then + return concat(str," | ") + else + return str + end end - function resolvers.listers.variables(pattern) - local instance = resolvers.instance - local environment = instance.environment - local variables = instance.variables - local expansions = instance.expansions - local pattern = upper(pattern or "") - local configured = { } - local order = instance.order - for i=1,#order do - for k, v in next, order[i] do - if v ~= nil and configured[k] == nil then - configured[k] = v - end - end - end - local env = fastcopy(environment) - local var = fastcopy(variables) - local exp = fastcopy(expansions) - for key, value in sortedpairs(configured) do - if key ~= "" and (pattern == "" or find(upper(key),pattern)) then - report_lists(key) - report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") - report_lists(" var: %s",tabstr(configured[key]) or "unset") - report_lists(" exp: %s",tabstr(expansions[key]) or "unset") - report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") - end + local instance=resolvers.instance + local environment=instance.environment + local variables=instance.variables + local expansions=instance.expansions + local pattern=upper(pattern or "") + local configured={} + local order=instance.order + for i=1,#order do + for k,v in next,order[i] do + if v~=nil and configured[k]==nil then + configured[k]=v + end end - instance.environment = fastcopy(env) - instance.variables = fastcopy(var) - instance.expansions = fastcopy(exp) + end + local env=fastcopy(environment) + local var=fastcopy(variables) + local exp=fastcopy(expansions) + for key,value in sortedpairs(configured) do + if key~="" and (pattern=="" or find(upper(key),pattern)) then + report_lists(key) + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") + end + end + instance.environment=fastcopy(env) + instance.variables=fastcopy(var) + instance.expansions=fastcopy(exp) end - function resolvers.listers.configurations(report) - local configurations = resolvers.instance.specification - local report = report or texio.write_nl - for i=1,#configurations do - report(format("file : %s",resolvers.resolve(configurations[i]))) - end - report("") - local list = resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) - for i=1,#list do - local li = resolvers.resolve(list[i]) - if lfs.isdir(li) then - report(format("path - %s",li)) - else - report(format("path + %s",li)) - end + local configurations=resolvers.instance.specification + local report=report or texio.write_nl + for i=1,#configurations do + report(format("file : %s",resolvers.resolve(configurations[i]))) + end + report("") + local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + for i=1,#list do + local li=resolvers.resolve(list[i]) + if lfs.isdir(li) then + report(format("path - %s",li)) + else + report(format("path + %s",li)) end + end end @@ -18314,264 +14340,234 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-sta'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this code is used in the updater - -local gmatch, match = string.gmatch, string.match -local type = type - -states = states or { } -local states = states - -states.data = states.data or { } -local data = states.data - -states.hash = states.hash or { } -local hash = states.hash - -states.tag = states.tag or "" -states.filename = states.filename or "" +-- original size: 5703, stripped down to: 2507 +if not modules then modules={} end modules ['luat-sta']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gmatch,match=string.gmatch,string.match +local type=type +states=states or {} +local states=states +states.data=states.data or {} +local data=states.data +states.hash=states.hash or {} +local hash=states.hash +states.tag=states.tag or "" +states.filename=states.filename or "" function states.save(filename,tag) - tag = tag or states.tag - filename = file.addsuffix(filename or states.filename,'lus') - io.savedata(filename, - "-- generator : luat-sta.lua\n" .. - "-- state tag : " .. tag .. "\n\n" .. - table.serialize(data[tag or states.tag] or {},true) - ) + tag=tag or states.tag + filename=file.addsuffix(filename or states.filename,'lus') + io.savedata(filename, + "-- generator : luat-sta.lua\n".."-- state tag : "..tag.."\n\n"..table.serialize(data[tag or states.tag] or {},true) + ) end - function states.load(filename,tag) - states.filename = filename - states.tag = tag or "whatever" - states.filename = file.addsuffix(states.filename,'lus') - data[states.tag], hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } + states.filename=filename + states.tag=tag or "whatever" + states.filename=file.addsuffix(states.filename,'lus') + data[states.tag],hash[states.tag]=(io.exists(filename) and dofile(filename)) or {},{} end - local function set_by_tag(tag,key,value,default,persistent) - local d, h = data[tag], hash[tag] - if d then - if type(d) == "table" then - local dkey, hkey = key, key - local pre, post = match(key,"(.+)%.([^%.]+)$") - if pre and post then - for k in gmatch(pre,"[^%.]+") do - local dk = d[k] - if not dk then - dk = { } - d[k] = dk - elseif type(dk) == "string" then - -- invalid table, unable to upgrade structure - -- hope for the best or delete the state file - break - end - d = dk - end - dkey, hkey = post, key - end - if value == nil then - value = default - elseif value == false then - -- special case - elseif persistent then - value = value or d[dkey] or default - else - value = value or default - end - d[dkey], h[hkey] = value, value - elseif type(d) == "string" then - -- weird - data[tag], hash[tag] = value, value + local d,h=data[tag],hash[tag] + if d then + if type(d)=="table" then + local dkey,hkey=key,key + local pre,post=match(key,"(.+)%.([^%.]+)$") + if pre and post then + for k in gmatch(pre,"[^%.]+") do + local dk=d[k] + if not dk then + dk={} + d[k]=dk + elseif type(dk)=="string" then + break + end + d=dk end + dkey,hkey=post,key + end + if value==nil then + value=default + elseif value==false then + elseif persistent then + value=value or d[dkey] or default + else + value=value or default + end + d[dkey],h[hkey]=value,value + elseif type(d)=="string" then + data[tag],hash[tag]=value,value end + end end - local function get_by_tag(tag,key,default) - local h = hash[tag] - if h and h[key] then - return h[key] - else - local d = data[tag] - if d then - for k in gmatch(key,"[^%.]+") do - local dk = d[k] - if dk ~= nil then - d = dk - else - return default - end - end - if d == false then - return false - else - return d or default - end + local h=hash[tag] + if h and h[key] then + return h[key] + else + local d=data[tag] + if d then + for k in gmatch(key,"[^%.]+") do + local dk=d[k] + if dk~=nil then + d=dk + else + return default end + end + if d==false then + return false + else + return d or default + end end + end end - -states.set_by_tag = set_by_tag -states.get_by_tag = get_by_tag - +states.set_by_tag=set_by_tag +states.get_by_tag=get_by_tag function states.set(key,value,default,persistent) - set_by_tag(states.tag,key,value,default,persistent) + set_by_tag(states.tag,key,value,default,persistent) end - function states.get(key,default) - return get_by_tag(states.tag,key,default) + return get_by_tag(states.tag,key,default) end - - - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-fmt'] = { - version = 1.001, - comment = "companion to mtxrun", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - - -local format = string.format -local quoted = string.quoted -local luasuffixes = utilities.lua.suffixes - -local report_format = logs.reporter("resolvers","formats") +-- original size: 5954, stripped down to: 4923 -local function primaryflags() -- not yet ok - local trackers = environment.argument("trackers") - local directives = environment.argument("directives") - local flags = "" - if trackers and trackers ~= "" then - flags = flags .. "--trackers=" .. quoted(trackers) - end - if directives and directives ~= "" then - flags = flags .. "--directives=" .. quoted(directives) - end - return flags +if not modules then modules={} end modules ['luat-fmt']={ + version=1.001, + comment="companion to mtxrun", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format=string.format +local concat=table.concat +local quoted=string.quoted +local luasuffixes=utilities.lua.suffixes +local report_format=logs.reporter("resolvers","formats") +local function primaryflags() + local trackers=environment.argument("trackers") + local directives=environment.argument("directives") + local flags={} + if trackers and trackers~="" then + flags={ "--trackers="..quoted(trackers) } + end + if directives and directives~="" then + flags={ "--directives="..quoted(directives) } + end + if environment.argument("jit") then + flags={ "--jiton" } + end + return concat(flags," ") end - function environment.make_format(name) - local engine = environment.ownmain or "luatex" - -- change to format path (early as we need expanded paths) - local olddir = dir.current() - local path = caches.getwritablepath("formats",engine) or "" -- maybe platform - if path ~= "" then - lfs.chdir(path) - end - report_format("format path: %s",dir.current()) - -- check source file - local texsourcename = file.addsuffix(name,"mkiv") - local fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" - if fulltexsourcename == "" then - texsourcename = file.addsuffix(name,"tex") - fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" - end - if fulltexsourcename == "" then - report_format("no tex source file with name: %s (mkiv or tex)",name) - lfs.chdir(olddir) - return - else - report_format("using tex source file: %s",fulltexsourcename) - end - local texsourcepath = dir.expandname(file.dirname(fulltexsourcename)) -- really needed - -- check specification - local specificationname = file.replacesuffix(fulltexsourcename,"lus") - local fullspecificationname = resolvers.findfile(specificationname,"tex") or "" - if fullspecificationname == "" then - specificationname = file.join(texsourcepath,"context.lus") - fullspecificationname = resolvers.findfile(specificationname,"tex") or "" - end - if fullspecificationname == "" then - report_format("unknown stub specification: %s",specificationname) - lfs.chdir(olddir) - return - end - local specificationpath = file.dirname(fullspecificationname) - -- load specification - local usedluastub = nil - local usedlualibs = dofile(fullspecificationname) - if type(usedlualibs) == "string" then - usedluastub = file.join(file.dirname(fullspecificationname),usedlualibs) - elseif type(usedlualibs) == "table" then - report_format("using stub specification: %s",fullspecificationname) - local texbasename = file.basename(name) - local luastubname = file.addsuffix(texbasename,luasuffixes.lua) - local lucstubname = file.addsuffix(texbasename,luasuffixes.luc) - -- pack libraries in stub - report_format("creating initialization file: %s",luastubname) - utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) - -- compile stub file (does not save that much as we don't use this stub at startup any more) - if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then - report_format("using compiled initialization file: %s",lucstubname) - usedluastub = lucstubname - else - report_format("using uncompiled initialization file: %s",luastubname) - usedluastub = luastubname - end - else - report_format("invalid stub specification: %s",fullspecificationname) - lfs.chdir(olddir) - return - end - -- generate format - local command = format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform == "unix" and "\\\\" or "\\") - report_format("running command: %s\n",command) - os.spawn(command) - -- remove related mem files - local pattern = file.removesuffix(file.basename(usedluastub)).."-*.mem" - -- report_format("removing related mplib format with pattern '%s'", pattern) - local mp = dir.glob(pattern) - if mp then - for i=1,#mp do - local name = mp[i] - report_format("removing related mplib format %s", file.basename(name)) - os.remove(name) - end - end + local engine=environment.ownmain or "luatex" + local olddir=dir.current() + local path=caches.getwritablepath("formats",engine) or "" + if path~="" then + lfs.chdir(path) + end + report_format("format path: %s",dir.current()) + local texsourcename=file.addsuffix(name,"mkiv") + local fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + if fulltexsourcename=="" then + texsourcename=file.addsuffix(name,"tex") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename=="" then + report_format("no tex source file with name: %s (mkiv or tex)",name) + lfs.chdir(olddir) + return + else + report_format("using tex source file: %s",fulltexsourcename) + end + local texsourcepath=dir.expandname(file.dirname(fulltexsourcename)) + local specificationname=file.replacesuffix(fulltexsourcename,"lus") + local fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + if fullspecificationname=="" then + specificationname=file.join(texsourcepath,"context.lus") + fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + end + if fullspecificationname=="" then + report_format("unknown stub specification: %s",specificationname) + lfs.chdir(olddir) + return + end + local specificationpath=file.dirname(fullspecificationname) + local usedluastub=nil + local usedlualibs=dofile(fullspecificationname) + if type(usedlualibs)=="string" then + usedluastub=file.join(file.dirname(fullspecificationname),usedlualibs) + elseif type(usedlualibs)=="table" then + report_format("using stub specification: %s",fullspecificationname) + local texbasename=file.basename(name) + local luastubname=file.addsuffix(texbasename,luasuffixes.lua) + local lucstubname=file.addsuffix(texbasename,luasuffixes.luc) + report_format("creating initialization file: %s",luastubname) + utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) + if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then + report_format("using compiled initialization file: %s",lucstubname) + usedluastub=lucstubname + else + report_format("using uncompiled initialization file: %s",luastubname) + usedluastub=luastubname + end + else + report_format("invalid stub specification: %s",fullspecificationname) lfs.chdir(olddir) + return + end + local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") + report_format("running command: %s\n",command) + os.spawn(command) + local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" + local mp=dir.glob(pattern) + if mp then + for i=1,#mp do + local name=mp[i] + report_format("removing related mplib format %s",file.basename(name)) + os.remove(name) + end + end + lfs.chdir(olddir) end - function environment.run_format(name,data,more) - if name and name ~= "" then - local engine = environment.ownmain or "luatex" - local barename = file.removesuffix(name) - local fmtname = caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) - if fmtname == "" then - fmtname = resolvers.findfile(file.addsuffix(barename,"fmt")) or "" - end - fmtname = resolvers.cleanpath(fmtname) - if fmtname == "" then - report_format("no format with name: %s",name) - else - local barename = file.removesuffix(name) -- expanded name - local luaname = file.addsuffix(barename,"luc") - if not lfs.isfile(luaname) then - luaname = file.addsuffix(barename,"lua") - end - if not lfs.isfile(luaname) then - report_format("using format name: %s",fmtname) - report_format("no luc/lua with name: %s",barename) - else - local command = format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more ~= "" and quoted(more) or "") - report_format("running command: %s",command) - os.spawn(command) - end - end + if name and name~="" then + local engine=environment.ownmain or "luatex" + local barename=file.removesuffix(name) + local fmtname=caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) + if fmtname=="" then + fmtname=resolvers.findfile(file.addsuffix(barename,"fmt")) or "" + end + fmtname=resolvers.cleanpath(fmtname) + if fmtname=="" then + report_format("no format with name: %s",name) + else + local barename=file.removesuffix(name) + local luaname=file.addsuffix(barename,"luc") + if not lfs.isfile(luaname) then + luaname=file.addsuffix(barename,"lua") + end + if not lfs.isfile(luaname) then + report_format("using format name: %s",fmtname) + report_format("no luc/lua with name: %s",barename) + else + local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") + report_format("running command: %s",command) + os.spawn(command) + end end + end end @@ -18579,128 +14575,106 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-tpl'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This is experimental code. Coming from dos and windows, I've always used %whatever% --- as template variables so let's stick to it. After all, it's easy to parse and stands --- out well. A double %% is turned into a regular %. - -utilities.templates = utilities.templates or { } -local templates = utilities.templates - -local trace_template = false trackers.register("templates.trace",function(v) trace_template = v end) -local report_template = logs.reporter("template") - -local format = string.format -local P, C, Cs, Carg, lpegmatch = lpeg.P, lpeg.C, lpeg.Cs, lpeg.Carg, lpeg.match - --- todo: make installable template.new +-- original size: 3570, stripped down to: 2441 +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local format=string.format +local P,C,Cs,Carg,lpegmatch=lpeg.P,lpeg.C,lpeg.Cs,lpeg.Carg,lpeg.match local replacer - local function replacekey(k,t,recursive) - local v = t[k] - if not v then - if trace_template then - report_template("unknown key %q",k) - end - return "" + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %q",k) + end + return "" + else + if trace_template then + report_template("setting key %q to value %q",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t) else - if trace_template then - report_template("setting key %q to value %q",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t) - else - return v - end + return v end + end end - -local sqlescape = lpeg.replacer { - { "'", "''" }, - { "\\", "\\\\" }, - { "\r\n", "\\n" }, - { "\r", "\\n" }, - -- { "\t", "\\t" }, +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, } - -local escapers = { - lua = function(s) - return format("%q",s) - end, - sql = function(s) - return lpegmatch(sqlescape,s) - end, +local escapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, } - -lpeg.patterns.sqlescape = sqlescape - -local function replacekeyunquoted(s,t,how,recurse) -- ".. \" " - local escaper = how and escapers[how] or escapers.lua - return escaper(replacekey(s,t,recurse)) -end - -local single = P("%") -- test %test% test : resolves test -local double = P("%%") -- test 10%% test : %% becomes % -local lquoted = P("%[") -- test %[test]" test : resolves test with escaped "'s -local rquoted = P("]%") -- - -local escape = double / '%%' -local nosingle = single / '' -local nodouble = double / '' -local nolquoted = lquoted / '' -local norquoted = rquoted / '' - -local key = nosingle * (C((1-nosingle)^1 * Carg(1) * Carg(2) * Carg(3))/replacekey) * nosingle -local unquoted = nolquoted * ((C((1 - norquoted)^1) * Carg(1) * Carg(2) * Carg(3))/replacekeyunquoted) * norquoted -local any = P(1) - - replacer = Cs((unquoted + escape + key + any)^0) - +lpeg.patterns.sqlescape=sqlescape +local function replacekeyunquoted(s,t,how,recurse) + local escaper=how and escapers[how] or escapers.lua + return escaper(replacekey(s,t,recurse)) +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local key=nosingle*(C((1-nosingle)^1*Carg(1)*Carg(2)*Carg(3))/replacekey)*nosingle +local unquoted=nolquoted*((C((1-norquoted)^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyunquoted)*norquoted +local any=P(1) + replacer=Cs((unquoted+escape+key+any)^0) local function replace(str,mapping,how,recurse) - if mapping then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end + if mapping then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end end - --- print(replace("test '%[x]%' test",{ x = [[a 'x'  a]] })) --- print(replace("test '%[x]%' test",{ x = [[a 'x'  a]] },'sql')) - -templates.replace = replace - +templates.replace=replace function templates.load(filename,mapping,how,recurse) - local data = io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end end - function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping = t - end - for k, v in next, t do - t[k] = replace(v,mapping,how,recurse) - end - return t + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t end --- inspect(utilities.templates.replace("test %one% test", { one = "%two%", two = "two" })) --- inspect(utilities.templates.resolve({ one = "%two%", two = "two", three = "%three%" })) - end -- of closure + +-- used libraries : l-lua.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-tab.lua util-sto.lua util-str.lua util-mrg.lua util-lua.lua util-prs.lua util-fmt.lua util-deb.lua trac-inf.lua trac-set.lua trac-log.lua trac-pro.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua luat-sta.lua luat-fmt.lua util-tpl.lua +-- skipped libraries : - +-- original bytes : 589196 +-- stripped bytes : 198197 + -- end library merge own = { } -- not local, might change diff --git a/tex/context/base/bibl-tra.mkiv b/tex/context/base/bibl-tra.mkiv index 08d8eb6fe..914470fbe 100644 --- a/tex/context/base/bibl-tra.mkiv +++ b/tex/context/base/bibl-tra.mkiv @@ -339,19 +339,32 @@ %D because similar numbers can be confusing. So, for the moment this is not %D supported in \MKIV. (So no: see reference [3-5,9] in "some other document") -\def\usepublications[#1]% +\unexpanded\def\usepublications[#1]% {\processcommalist[#1]\dousepublications} \def\dousepublications#1% {\doonlyonce{#1.\f!bibextension}{\dodousepublications{#1}}} +% \def\dodousepublications#1% brr, this par stuff +% {\let\@@savedpar\par +% \let\par\ignorespaces +% \ifhmode\kern\zeropoint\fi +% \readfile{#1.\f!bibextension} +% {\showmessage\m!publications{4}{#1.\f!bibextension}} +% {\showmessage\m!publications{2}{#1.\f!bibextension}}% +% \ifhmode\removeunwantedspaces\fi +% \let\par\@@savedpar} + \def\dodousepublications#1% brr, this par stuff {\let\@@savedpar\par \let\par\ignorespaces \ifhmode\kern\zeropoint\fi + \pushcatcodetable + \setcatcodetable\ctxcatcodes \readfile{#1.\f!bibextension} {\showmessage\m!publications{4}{#1.\f!bibextension}} {\showmessage\m!publications{2}{#1.\f!bibextension}}% + \popcatcodetable \ifhmode\removeunwantedspaces\fi \let\par\@@savedpar} diff --git a/tex/context/base/char-def.lua b/tex/context/base/char-def.lua index 71110ab00..118fbc701 100644 --- a/tex/context/base/char-def.lua +++ b/tex/context/base/char-def.lua @@ -417,6 +417,7 @@ characters.data={ direction="es", linebreak="hy", mathsymbol=0x2212, + mathfiller="relfill", mathextensible='h', unicodeslot=0x002D, }, @@ -601,6 +602,7 @@ characters.data={ name="Relbar", }, }, + mathfiller="equalfill", mathextensible='h', unicodeslot=0x003D, }, @@ -59502,6 +59504,7 @@ characters.data={ description="LEFTWARDS TWO HEADED ARROW", direction="on", linebreak="al", + mathfiller="twoheadleftarrowfill", mathextensible="l", mathclass="relation", mathname="twoheadleftarrow", @@ -59522,6 +59525,7 @@ characters.data={ description="RIGHTWARDS TWO HEADED ARROW", direction="on", linebreak="al", + mathfiller="twoheadrightarrowfill", mathextensible="r", mathclass="relation", mathname="twoheadrightarrow", @@ -59582,6 +59586,7 @@ characters.data={ description="RIGHTWARDS ARROW FROM BAR", direction="on", linebreak="al", + mathfiller="mapstofill", mathextensible="r", mathclass="relation", mathname="mapsto", @@ -59614,6 +59619,7 @@ characters.data={ direction="on", linebreak="al", mathextensible="l", + mathfiller="hookleftarrowfill", mathclass="relation", mathname="hookleftarrow", unicodeslot=0x21A9, @@ -59623,6 +59629,7 @@ characters.data={ description="RIGHTWARDS ARROW WITH HOOK", direction="on", linebreak="al", + mathfiller="hookrightarrowfill", mathextensible="r", mathclass="relation", mathname="hookrightarrow", @@ -59820,6 +59827,8 @@ characters.data={ description="LEFTWARDS HARPOON WITH BARB UPWARDS", direction="on", linebreak="al", + mathfiller="leftharpoonupfill", + mathextensible="l", mathclass="relation", mathname="leftharpoonup", unicodeslot=0x21BC, @@ -59829,6 +59838,8 @@ characters.data={ description="LEFTWARDS HARPOON WITH BARB DOWNWARDS", direction="on", linebreak="al", + mathfiller="leftharpoondownfill", + mathextensible="l", mathclass="relation", mathname="leftharpoondown", unicodeslot=0x21BD, @@ -59865,6 +59876,8 @@ characters.data={ description="RIGHTWARDS HARPOON WITH BARB UPWARDS", direction="on", linebreak="al", + mathfiller="rightharpoonupfill", + mathextensible="r", mathclass="relation", mathname="rightharpoonup", unicodeslot=0x21C0, @@ -59874,6 +59887,9 @@ characters.data={ description="RIGHTWARDS HARPOON WITH BARB DOWNWARDS", direction="on", linebreak="al", + mathfiller="rightharpoondownfill", + mathextensible="r", + mathclass="relation", mathclass="relation", mathname="rightharpoondown", unicodeslot=0x21C1, @@ -59902,6 +59918,7 @@ characters.data={ description="RIGHTWARDS ARROW OVER LEFTWARDS ARROW", direction="on", linebreak="al", + mathfiller="rightoverleftarrowfill", mathextensible="h", mathclass="relation", mathname="rightleftarrows", @@ -59924,6 +59941,7 @@ characters.data={ description="LEFTWARDS ARROW OVER RIGHTWARDS ARROW", direction="on", linebreak="al", + mathfiller="lefgtoverrightarrowfill", mathextensible="h", mathclass="relation", mathname="leftrightarrows", @@ -59974,6 +59992,8 @@ characters.data={ description="LEFTWARDS HARPOON OVER RIGHTWARDS HARPOON", direction="on", linebreak="al", + mathfiller="leftrightharpoonsfill", + mathextensible="h", mathclass="relation", mathname="leftrightharpoons", unicodeslot=0x21CB, @@ -59983,6 +60003,8 @@ characters.data={ description="RIGHTWARDS HARPOON OVER LEFTWARDS HARPOON", direction="on", linebreak="al", + mathfiller="rightleftharpoonsfill", + mathextensible="h", mathclass="relation", mathname="rightleftharpoons", unicodeslot=0x21CC, @@ -61614,8 +61636,9 @@ characters.data={ direction="on", linebreak="al", mathclass="relation", - mathname="nequiv", mathextensible='h', + mathfiller="triplerelfill", + mathname="nequiv", specials={ "char", 0x2261, 0x0338 }, unicodeslot=0x2262, }, @@ -72851,6 +72874,7 @@ characters.data={ direction="on", linebreak="al", mathextensible="h", + mathfiller="leftrightarrowfill", mathclass="relation", mathname="longleftrightarrow", unicodeslot=0x27F7, @@ -72861,6 +72885,7 @@ characters.data={ direction="on", linebreak="al", mathextensible="l", + mathfiller="Leftarrowfill", mathclass="relation", mathname="Longleftarrow", unicodeslot=0x27F8, @@ -72871,6 +72896,7 @@ characters.data={ direction="on", linebreak="al", mathextensible="r", + mathfiller="Rightarrowfill", mathclass="relation", mathname="Longrightarrow", unicodeslot=0x27F9, @@ -72881,6 +72907,7 @@ characters.data={ direction="on", linebreak="al", mathextensible="h", + mathfiller="Leftrightarrowfill", mathclass="relation", mathname="Longleftrightarrow", unicodeslot=0x27FA, diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii index 0ced751ef..3ce67992e 100644 --- a/tex/context/base/cont-new.mkii +++ b/tex/context/base/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2013.01.10 01:04} +\newcontextversion{2013.01.13 23:10} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv index f6097bdf4..18153995f 100644 --- a/tex/context/base/cont-new.mkiv +++ b/tex/context/base/cont-new.mkiv @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2013.01.10 01:04} +\newcontextversion{2013.01.13 23:10} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf new file mode 100644 index 000000000..43f4790bd Binary files /dev/null and b/tex/context/base/context-version.pdf differ diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png index cc76b02df..3157fb849 100644 Binary files a/tex/context/base/context-version.png and b/tex/context/base/context-version.png differ diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii index b2e6fa6ed..6abbe0653 100644 --- a/tex/context/base/context.mkii +++ b/tex/context/base/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2013.01.10 01:04} +\edef\contextversion{2013.01.13 23:10} %D For those who want to use this: diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv index 97b71f5a4..ded53e1cf 100644 --- a/tex/context/base/context.mkiv +++ b/tex/context/base/context.mkiv @@ -25,7 +25,7 @@ %D up and the dependencies are more consistent. \edef\contextformat {\jobname} -\edef\contextversion{2013.01.10 01:04} +\edef\contextversion{2013.01.13 23:10} %D For those who want to use this: @@ -233,6 +233,7 @@ \loadmarkfile{spac-ali} \loadmarkfile{spac-hor} +\loadmarkfile{spac-flr} \loadmarkfile{spac-ver} \loadmarkfile{spac-lin} \loadmarkfile{spac-pag} @@ -415,7 +416,8 @@ \loadmarkfile{math-for} \loadmarkfile{math-def} \loadmarkfile{math-ali} -\loadmarkfile{math-arr} +%loadmarkfile{math-arr} +\loadmkvifile{math-stc} \loadmarkfile{math-frc} \loadmarkfile{math-scr} \loadmarkfile{math-int} diff --git a/tex/context/base/data-ini.lua b/tex/context/base/data-ini.lua index 37b4f62ca..cad3eac14 100644 --- a/tex/context/base/data-ini.lua +++ b/tex/context/base/data-ini.lua @@ -32,7 +32,7 @@ local resolvers = resolvers texconfig.kpse_init = false texconfig.shell_escape = 't' -if kpse and kpse.default_texmfcnf then +if not (environment and environment.default_texmfcnf) and kpse and kpse.default_texmfcnf then local default_texmfcnf = kpse.default_texmfcnf() -- looks more like context: default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") diff --git a/tex/context/base/data-lua.lua b/tex/context/base/data-lua.lua index de20f4820..49033461e 100644 --- a/tex/context/base/data-lua.lua +++ b/tex/context/base/data-lua.lua @@ -46,7 +46,7 @@ local clibextras = { } local pattern = Cs(P("!")^0 / "" * (P("/") * P(-1) / "/" + P("/")^1 / "/" + 1)^0) -local function cleanpath(path) --hm, don't we have a helper for this? +local function cleanpath(path) -- hm, don't we have a helper for this? return resolvers.resolve(lpegmatch(pattern,path)) end diff --git a/tex/context/base/data-res.lua b/tex/context/base/data-res.lua index 41a590228..02ef35f5f 100644 --- a/tex/context/base/data-res.lua +++ b/tex/context/base/data-res.lua @@ -863,7 +863,7 @@ local function collect_files(names) if dname == "" or find(dname,"^%.") then dname = false else - dname = gsub(dname,"*","%.*") + dname = gsub(dname,"%*",".*") dname = "/" .. dname .. "$" end local hashes = instance.hashes diff --git a/tex/context/base/file-job.lua b/tex/context/base/file-job.lua index 572697443..533103ec6 100644 --- a/tex/context/base/file-job.lua +++ b/tex/context/base/file-job.lua @@ -913,7 +913,7 @@ function commands.logoptions() end function commands.doifelsecontinuewithfile(inpname) - local continue = addsuffix(inpname,"tex") == addsuffix(environment.jobname,"tex") + local continue = addsuffix(inpname,"tex") == addsuffix(environment.inputfilename,"tex") if continue then report_system("continuing input file %q",inpname) end diff --git a/tex/context/base/font-con.lua b/tex/context/base/font-con.lua index 516dffe98..58af5a04f 100644 --- a/tex/context/base/font-con.lua +++ b/tex/context/base/font-con.lua @@ -419,6 +419,8 @@ function constructors.scale(tfmdata,specification) target.psname = psname target.name = name -- + -- inspect(properties) + -- properties.fontname = fontname properties.fullname = fullname properties.filename = filename diff --git a/tex/context/base/font-otn.lua b/tex/context/base/font-otn.lua index 5dba663e4..2c6016427 100644 --- a/tex/context/base/font-otn.lua +++ b/tex/context/base/font-otn.lua @@ -250,9 +250,6 @@ local handlers = { } local rlmode = 0 local featurevalue = false --- we cannot optimize with "start = first_glyph(head)" because then we don't --- know which rlmode we're in which messes up cursive handling later on --- -- head is always a whatsit so we can safely assume that head is not changed -- we use this for special testing and documentation @@ -2206,7 +2203,7 @@ local function featuresprocessor(head,font,attr) -- font interactions and then we do need the full sweeps. -- Keeping track of the headnode is needed for devanagari (I generalized it a bit - -- so that multiple cases are also covered. + -- so that multiple cases are also covered.) for s=1,#sequences do local dataset = datasets[s] diff --git a/tex/context/base/l-file.lua b/tex/context/base/l-file.lua index f34bed5fd..2a3d6991e 100644 --- a/tex/context/base/l-file.lua +++ b/tex/context/base/l-file.lua @@ -33,7 +33,8 @@ local noslashes = 1-slashes local name = noperiod^1 local suffix = period/"" * (1-period-slashes)^1 * -1 -local pattern = C((noslashes^0 * slashes^1)^1) +----- pattern = C((noslashes^0 * slashes^1)^1) +local pattern = C((1 - (slashes^1 * noslashes^1 * -1))^1) * P(1) -- there must be a more efficient way local function pathpart(name,default) return name and lpegmatch(pattern,name) or default or "" @@ -45,6 +46,13 @@ local function basename(name) return name and lpegmatch(pattern,name) or name end +-- print(pathpart("file")) +-- print(pathpart("dir/file")) +-- print(pathpart("/dir/file")) +-- print(basename("file")) +-- print(basename("dir/file")) +-- print(basename("/dir/file")) + local pattern = (noslashes^0 * slashes^1)^0 * Cs((1-suffix)^1) * suffix^0 local function nameonly(name) @@ -69,7 +77,7 @@ file.extname = suffixonly -- obsolete -- actually these are schemes local drive = C(R("az","AZ")) * colon -local path = C(((1-slashes)^0 * slashes)^0) +local path = C((noslashes^0 * slashes)^0) local suffix = period * C(P(1-period)^0 * P(-1)) local base = C((1-suffix)^0) local rest = C(P(1)^0) @@ -98,9 +106,14 @@ function file.splitbase(str) return str and lpegmatch(pattern_d,str) -- returns path, base+suffix end -function file.nametotable(str,splitdrive) -- returns table +---- stripslash = C((1 - P("/")^1*P(-1))^0) + +function file.nametotable(str,splitdrive) if str then local path, drive, subpath, name, base, suffix = lpegmatch(pattern_c,str) + -- if path ~= "" then + -- path = lpegmatch(stripslash,path) -- unfortunate hack, maybe this becomes default + -- end if splitdrive then return { path = path, @@ -121,6 +134,20 @@ function file.nametotable(str,splitdrive) -- returns table end end +-- print(file.splitname("file")) +-- print(file.splitname("dir/file")) +-- print(file.splitname("/dir/file")) +-- print(file.splitname("file")) +-- print(file.splitname("dir/file")) +-- print(file.splitname("/dir/file")) + +-- inspect(file.nametotable("file.ext")) +-- inspect(file.nametotable("dir/file.ext")) +-- inspect(file.nametotable("/dir/file.ext")) +-- inspect(file.nametotable("file.ext")) +-- inspect(file.nametotable("dir/file.ext")) +-- inspect(file.nametotable("/dir/file.ext")) + local pattern = Cs(((period * noperiod^1 * -1)/"" + 1)^1) function file.removesuffix(name) diff --git a/tex/context/base/l-lpeg.lua b/tex/context/base/l-lpeg.lua index 407532ebf..8b5389aa1 100644 --- a/tex/context/base/l-lpeg.lua +++ b/tex/context/base/l-lpeg.lua @@ -53,8 +53,10 @@ local report = texio and texio.write_nl or print -- function lpeg.Cmt (l) local p = lpcmt (l) report("LPEG Cmt =") lpprint(l) return p end -- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end -local type, next = type, next +local type, next, tostring = type, next, tostring local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format +----- mod, div = math.mod, math.div +local floor = math.floor -- Beware, we predefine a bunch of patterns here and one reason for doing so -- is that we get consistent behaviour in some of the visualizers. @@ -778,3 +780,44 @@ end -- faster than find(str,"[\n\r]") when match and # > 7 and always faster when # > 3 patterns.containseol = lpeg.finder(eol) -- (1-eol)^0 * eol + +-- The next pattern^n variant is based on an approach suggested +-- by Roberto: constructing a big repetition in chunks. +-- +-- Being sparse is not needed, and only complicate matters and +-- the number of redundant entries is not that large. + +local function nextstep(n,step,result) + local m = n % step -- mod(n,step) + local d = floor(n/step) -- div(n,step) + if d > 0 then + local v = V(tostring(step)) + local s = result.start + for i=1,d do + if s then + s = v * s + else + s = v + end + end + result.start = s + end + if step > 1 and result.start then + local v = V(tostring(step/2)) + result[tostring(step)] = v * v + end + if step > 0 then + return nextstep(m,step/2,result) + else + return result + end +end + +function lpeg.times(pattern,n) + return P(nextstep(n,2^16,{ "start", ["1"] = pattern })) +end + +-- local p = lpeg.Cs((1 - lpeg.times(lpeg.P("AB"),25))^1) +-- local s = "12" .. string.rep("AB",20) .. "34" .. string.rep("AB",30) .. "56" +-- inspect(p) +-- print(lpeg.match(p,s)) diff --git a/tex/context/base/l-string.lua b/tex/context/base/l-string.lua index 7a919d189..828285ca1 100644 --- a/tex/context/base/l-string.lua +++ b/tex/context/base/l-string.lua @@ -49,7 +49,7 @@ end -- print(string.unquoted('"test"')) function string.quoted(str) - return format("%q",str) -- always " + return format("%q",str) -- always double quote end function string.count(str,pattern) -- variant 3 diff --git a/tex/context/base/luat-env.lua b/tex/context/base/luat-env.lua index 7bce0052d..a00acdc63 100644 --- a/tex/context/base/luat-env.lua +++ b/tex/context/base/luat-env.lua @@ -335,8 +335,8 @@ function environment.loadluafile(filename, version) local lucname, luaname, chunk local basename = file.removesuffix(filename) if basename == filename then - luaname = fiule.addsuffix(basename,luasuffixes.lua) - lucname = fiule.addsuffix(basename,luasuffixes.luc) + luaname = file.addsuffix(basename,luasuffixes.lua) + lucname = file.addsuffix(basename,luasuffixes.luc) else luaname = basename -- forced suffix lucname = nil diff --git a/tex/context/base/luat-fmt.lua b/tex/context/base/luat-fmt.lua index 2d2614ecb..af34fe8ce 100644 --- a/tex/context/base/luat-fmt.lua +++ b/tex/context/base/luat-fmt.lua @@ -6,9 +6,9 @@ if not modules then modules = { } end modules ['luat-fmt'] = { license = "see context related readme files" } - local format = string.format -local quoted = string.quoted +local concat = table.concat +local quoted = string.quoted local luasuffixes = utilities.lua.suffixes local report_format = logs.reporter("resolvers","formats") @@ -16,14 +16,17 @@ local report_format = logs.reporter("resolvers","formats") local function primaryflags() -- not yet ok local trackers = environment.argument("trackers") local directives = environment.argument("directives") - local flags = "" + local flags = { } if trackers and trackers ~= "" then - flags = flags .. "--trackers=" .. quoted(trackers) + flags = { "--trackers=" .. quoted(trackers) } end if directives and directives ~= "" then - flags = flags .. "--directives=" .. quoted(directives) + flags = { "--directives=" .. quoted(directives) } + end + if environment.argument("jit") then + flags = { "--jiton" } end - return flags + return concat(flags," ") end function environment.make_format(name) diff --git a/tex/context/base/math-arr.mkiv b/tex/context/base/math-arr.mkiv index e0ef6095e..6824c362e 100644 --- a/tex/context/base/math-arr.mkiv +++ b/tex/context/base/math-arr.mkiv @@ -11,6 +11,10 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. +%D We keep this file around as reference of his things were done in the +%D past. You can still load this module but it has been replaced by more +%D modern code. + \writestatus{loading}{ConTeXt Math Macros / Arrows} \unprotect diff --git a/tex/context/base/math-def.mkiv b/tex/context/base/math-def.mkiv index 61153cfdd..0d80b3de0 100644 --- a/tex/context/base/math-def.mkiv +++ b/tex/context/base/math-def.mkiv @@ -108,7 +108,7 @@ \installcorenamespace{mathbig} -\def\choosemathbig#1#2% so we accent \big{||} as well +\unexpanded\def\choosemathbig#1#2% so we accent \big{||} as well {{\hbox{$% \ifcase\bigmathdelimitermethod \doleftbigmath#2\relax diff --git a/tex/context/base/meta-ini.mkiv b/tex/context/base/meta-ini.mkiv index 5ac7908d5..aea43ee87 100644 --- a/tex/context/base/meta-ini.mkiv +++ b/tex/context/base/meta-ini.mkiv @@ -849,7 +849,7 @@ \fi} \def\meta_start_code_instance#1#2\stopMPcode - {\meta_begin_graphic_group{#1::\s!dummy}% name does not matter + {\meta_begin_graphic_group{#1}% \meta_enable_include \meta_process_graphic{#2}% \meta_end_graphic_group} @@ -861,22 +861,25 @@ \let\stopMPcode\relax \unexpanded\def\MPcode - {\dosinglegroupempty\meta_code} + {\dodoublegroupempty\meta_code} \def\meta_code - {\iffirstargument + {\ifsecondargument \expandafter\meta_code_instance \else \expandafter\meta_code_standard \fi} \def\meta_code_instance#1#2% - {\meta_begin_graphic_group{#1::\s!dummy}% name does not matter + {\meta_begin_graphic_group{#1}% + \meta_enable_include \meta_process_graphic{#2}% \meta_end_graphic_group} -\def\meta_code_standard#1% #2 - {\meta_process_graphic} +\def\meta_code_standard#1#2% + {\let\currentMPinstance\defaultMPinstance + \meta_enable_include + \meta_process_graphic{#1}} % a bit nasty (also needed for compatibility: diff --git a/tex/context/base/mlib-pdf.mkiv b/tex/context/base/mlib-pdf.mkiv index 044c416a2..4deb26203 100644 --- a/tex/context/base/mlib-pdf.mkiv +++ b/tex/context/base/mlib-pdf.mkiv @@ -140,15 +140,17 @@ \let\stopMPLIBtoPDF \directstopMPLIBtoPDF \meta_start_current_graphic \forgetall - \normalexpanded{\noexpand\ctxlua{metapost.graphic( - "\currentMPinstance", - "\currentMPformat", - \!!bs#2;\!!es, - \!!bs\meta_flush_current_initializations;\!!es, - \!!bs\meta_flush_current_preamble;\!!es, - \!!bs\meta_flush_current_instance\!!es, - "all" - )}}% + \normalexpanded{\noexpand\ctxlua{metapost.graphic { + instance = "\currentMPinstance", + format = "\currentMPformat", + data = \!!bs#2;\!!es, + initializations = \!!bs\meta_flush_current_initializations\!!es, + extensions = \!!bs\meta_flush_current_extensions\!!es, + inclusions = \!!bs\meta_flush_current_userinclusions\!!es, + definitions = \!!bs\meta_flush_current_definitions\!!es, + figure = "all", + method = "\MPinstanceparameter\c!method", + }}}% \meta_stop_current_graphic \meta_end_graphic_group} diff --git a/tex/context/base/mlib-pps.lua b/tex/context/base/mlib-pps.lua index 4756690be..43a548c65 100644 --- a/tex/context/base/mlib-pps.lua +++ b/tex/context/base/mlib-pps.lua @@ -6,7 +6,8 @@ if not modules then modules = { } end modules ['mlib-pps'] = { license = "see context related readme files", } --- todo: report max textexts +-- todo: make a hashed textext variant where we only process the text once (normally +-- we cannot assume that no macros are involved which influence a next textext local format, gmatch, match, split = string.format, string.gmatch, string.match, string.split local tonumber, type = tonumber, type @@ -215,10 +216,14 @@ local current_format, current_graphic, current_initializations metapost.multipass = false -local textexts = { } +local textexts = { } -- all boxes, optionally with a different color +local texslots = { } -- references to textexts in order or usage +local texorder = { } -- references to textexts by mp index +local textrial = 0 +local texfinal = 0 local scratchbox = 0 -local function freeboxes() -- todo: mp direct list ipv box +local function freeboxes() for n, box in next, textexts do local tn = textexts[n] if tn then @@ -231,6 +236,10 @@ local function freeboxes() -- todo: mp direct list ipv box end end textexts = { } + texslots = { } + texorder = { } + textrial = 0 + texfinal = 0 end metapost.resettextexts = freeboxes @@ -480,7 +489,8 @@ local f_text_data = formatters["mfun_tt_w[%i] := %f ; mfun_tt_h[%i] := %f ; mfu function metapost.textextsdata() local t, nt, n = { }, 0, 0 - for n, box in next, textexts do + for n=1,#texorder do + local box = textexts[texorder[n]] if box then local wd, ht, dp = box.width/factor, box.height/factor, box.depth/factor if trace_textexts then @@ -492,6 +502,7 @@ function metapost.textextsdata() break end end +-- inspect(t) return t end @@ -765,13 +776,17 @@ local basepoints = number.dimenfactors["bp"] local function cm(object) local op = object.path - local first, second, fourth = op[1], op[2], op[4] - local tx, ty = first.x_coord , first.y_coord - local sx, sy = second.x_coord - tx, fourth.y_coord - ty - local rx, ry = second.y_coord - ty, fourth.x_coord - tx - if sx == 0 then sx = 0.00001 end - if sy == 0 then sy = 0.00001 end - return sx,rx,ry,sy,tx,ty + if op then + local first, second, fourth = op[1], op[2], op[4] + local tx, ty = first.x_coord , first.y_coord + local sx, sy = second.x_coord - tx, fourth.y_coord - ty + local rx, ry = second.y_coord - ty, fourth.x_coord - tx + if sx == 0 then sx = 0.00001 end + if sy == 0 then sy = 0.00001 end + return sx, rx, ry, sy, tx, ty + else + return 1, 0, 0, 1, 0, 0 -- weird case + end end -- color @@ -782,25 +797,115 @@ end -- text -local tx_done = { } +-- local tx_done = { } +-- +-- local function tx_reset() +-- tx_done = { } +-- end +-- +-- local function tx_analyze(object,prescript) -- todo: hash content and reuse them +-- local tx_stage = prescript.tx_stage +-- if tx_stage then +-- local tx_number = tonumber(prescript.tx_number) +-- if not tx_done[tx_number] then +-- tx_done[tx_number] = true +-- if trace_textexts then +-- report_textexts("setting %s %s (first pass)",tx_stage,tx_number) +-- end +-- local s = object.postscript or "" +-- local c = object.color -- only simple ones, no transparency +-- local a = prescript.tr_alternative +-- local t = prescript.tr_transparency +-- if not c then +-- -- no color +-- elseif #c == 1 then +-- if a and t then +-- s = format("\\directcolored[s=%f,a=%f,t=%f]%s",c[1],a,t,s) +-- else +-- s = format("\\directcolored[s=%f]%s",c[1],s) +-- end +-- elseif #c == 3 then +-- if a and t then +-- s = format("\\directcolored[r=%f,g=%f,b=%f,a=%f,t=%f]%s",c[1],c[2],c[3],a,t,s) +-- else +-- s = format("\\directcolored[r=%f,g=%f,b=%f]%s",c[1],c[2],c[3],s) +-- end +-- elseif #c == 4 then +-- if a and t then +-- s = format("\\directcolored[c=%f,m=%f,y=%f,k=%f,a=%f,t=%f]%s",c[1],c[2],c[3],c[4],a,t,s) +-- else +-- s = format("\\directcolored[c=%f,m=%f,y=%f,k=%f]%s",c[1],c[2],c[3],c[4],s) +-- end +-- end +-- context.MPLIBsettext(tx_number,s) -- combine colored in here, saves call +-- metapost.multipass = true +-- end +-- end +-- end +-- +-- local function tx_process(object,prescript,before,after) +-- local tx_number = prescript.tx_number +-- if tx_number then +-- tx_number = tonumber(tx_number) +-- local tx_stage = prescript.tx_stage +-- if tx_stage == "final" then -- redundant test +-- if trace_textexts then +-- report_textexts("processing %s (second pass)",tx_number) +-- end +-- local sx,rx,ry,sy,tx,ty = cm(object) -- outside function ! +-- before[#before+1] = function() +-- -- flush always happens, we can have a special flush function injected before +-- local box = textexts[tx_number] +-- if box then +-- context.MPLIBgettextscaledcm(tx_number, +-- format("%f",sx), -- bah ... %s no longer checks +-- format("%f",rx), -- bah ... %s no longer checks +-- format("%f",ry), -- bah ... %s no longer checks +-- format("%f",sy), -- bah ... %s no longer checks +-- format("%f",tx), -- bah ... %s no longer checks +-- format("%f",ty), -- bah ... %s no longer checks +-- sxsy(box.width,box.height,box.depth)) +-- else +-- report_textexts("unknown %s",tx_number) +-- end +-- end +-- if not trace_textexts then +-- object.path = false -- else: keep it +-- end +-- object.color = false +-- object.grouped = true +-- end +-- end +-- end + +-- experiment + +local tx_hash = { } +local tx_last = 0 local function tx_reset() - tx_done = { } + tx_hash = { } + tx_last = 0 end +local fmt = formatters["%s %s %s % t"] + local function tx_analyze(object,prescript) -- todo: hash content and reuse them local tx_stage = prescript.tx_stage - if tx_stage then + if tx_stage == "trial" then + textrial = textrial + 1 local tx_number = tonumber(prescript.tx_number) - if not tx_done[tx_number] then - tx_done[tx_number] = true - if trace_textexts then - report_textexts("setting %s %s (first pass)",tx_stage,tx_number) - end - local s = object.postscript or "" - local c = object.color -- only simple ones, no transparency - local a = prescript.tr_alternative - local t = prescript.tr_transparency + local s = object.postscript or "" + local c = object.color -- only simple ones, no transparency + local a = prescript.tr_alternative + local t = prescript.tr_transparency + local h = fmt(tx_number,a or "?",t or "?",c) + local n = tx_hash[h] -- todo: hashed variant with s (nicer for similar labels) + if not n then + tx_last = tx_last + 1 + -- if trace_textexts then + -- report_textexts("setting %s %s (first pass)",tx_stage,tx_number) + -- end if not c then -- no color elseif #c == 1 then @@ -822,8 +927,36 @@ local function tx_analyze(object,prescript) -- todo: hash content and reuse them s = format("\\directcolored[c=%f,m=%f,y=%f,k=%f]%s",c[1],c[2],c[3],c[4],s) end end - context.MPLIBsettext(tx_number,s) -- combine colored in here, saves call + context.MPLIBsettext(tx_last,s) + metapost.multipass = true + tx_hash[h] = tx_last + texslots[textrial] = tx_last + texorder[tx_number] = tx_last + if trace_textexts then + report_textexts("stage: %s, usage: %s, number: %s, new: %s, hash: %s",tx_stage,textrial,tx_number,tx_last,h) + end + else + texslots[textrial] = n + if trace_textexts then + report_textexts("stage: %s, usage: %s, number: %s, old: %s, hash: %s",tx_stage,textrial,tx_number,n,h) + end + end + elseif tx_stage == "extra" then + textrial = textrial + 1 + local tx_number = tonumber(prescript.tx_number) + if not texorder[tx_number] then + local s = object.postscript or "" + tx_last = tx_last + 1 + -- if trace_textexts then + -- report_textexts("setting %s %s (first pass)",tx_stage,tx_number) + -- end + context.MPLIBsettext(tx_last,s) metapost.multipass = true + texslots[textrial] = tx_last + texorder[tx_number] = tx_last + if trace_textexts then + report_textexts("stage: %s, usage: %s, number: %s, extra: %s",tx_stage,textrial,tx_number,tx_last) + end end end end @@ -833,18 +966,21 @@ local function tx_process(object,prescript,before,after) if tx_number then tx_number = tonumber(tx_number) local tx_stage = prescript.tx_stage - if tx_stage == "final" then -- redundant test + if tx_stage == "final" then + texfinal = texfinal + 1 + local n = texslots[texfinal] + -- if trace_textexts then + -- report_textexts("processing %s (second pass)",tx_number) + -- end if trace_textexts then - report_textexts("processing %s (second pass)",tx_number) + report_textexts("stage: %s, usage: %s, number: %s, use: %s",tx_stage,texfinal,tx_number,n) end - -- before[#before+1] = f_cm(cm(object)) - local sx,rx,ry,sy,tx,ty = cm(object) - before[#before+1] = function() - -- flush always happens, we can have a special flush function injected before - local box = textexts[tx_number] - if box then - -- context.MPLIBgettextscaled(tx_number,sxsy(box.width,box.height,box.depth)) - context.MPLIBgettextscaledcm(tx_number, + local sx, rx, ry, sy, tx, ty = cm(object) -- needs to be frozen outside the function + local box = textexts[n] + if box then + before[#before+1] = function() + -- flush always happens, we can have a special flush function injected before + context.MPLIBgettextscaledcm(n, format("%f",sx), -- bah ... %s no longer checks format("%f",rx), -- bah ... %s no longer checks format("%f",ry), -- bah ... %s no longer checks @@ -852,11 +988,12 @@ local function tx_process(object,prescript,before,after) format("%f",tx), -- bah ... %s no longer checks format("%f",ty), -- bah ... %s no longer checks sxsy(box.width,box.height,box.depth)) - else + end + else + before[#before+1] = function() report_textexts("unknown %s",tx_number) end end - -- before[#before+1] = "Q" if not trace_textexts then object.path = false -- else: keep it end @@ -958,7 +1095,7 @@ local function sh_process(object,prescript,before,after) end before[#before+1], after[#after+1] = "q /Pattern cs", format("W n /%s sh Q",name) -- false, not nil, else mt triggered - object.colored = false + object.colored = false -- hm, not object.color ? object.type = false object.grouped = true end diff --git a/tex/context/base/node-aux.lua b/tex/context/base/node-aux.lua index 466b7991d..ebfd3976d 100644 --- a/tex/context/base/node-aux.lua +++ b/tex/context/base/node-aux.lua @@ -204,23 +204,23 @@ function nodes.firstcharinbox(n) return 0 end ---~ local function firstline(n) ---~ while n do ---~ local id = n.id ---~ if id == hlist_code then ---~ if n.subtype == line_code then ---~ return n ---~ else ---~ return firstline(n.list) ---~ end ---~ elseif id == vlist_code then ---~ return firstline(n.list) ---~ end ---~ n = n.next ---~ end ---~ end - ---~ nodes.firstline = firstline +-- local function firstline(n) +-- while n do +-- local id = n.id +-- if id == hlist_code then +-- if n.subtype == line_code then +-- return n +-- else +-- return firstline(n.list) +-- end +-- elseif id == vlist_code then +-- return firstline(n.list) +-- end +-- n = n.next +-- end +-- end + +-- nodes.firstline = firstline -- this depends on fonts, so we have a funny dependency ... will be -- sorted out .. we could make tonodes a plugin into this diff --git a/tex/context/base/node-fnt.lua b/tex/context/base/node-fnt.lua index 543f64acb..49e1029e7 100644 --- a/tex/context/base/node-fnt.lua +++ b/tex/context/base/node-fnt.lua @@ -41,7 +41,7 @@ local setmetatableindex = table.setmetatableindex -- potential speedup: check for subtype < 256 so that we can remove that test -- elsewhere, danger: injected nodes will not be dealt with but that does not --- happen often; we could consider processing sublists but that might need mor +-- happen often; we could consider processing sublists but that might need more -- checking later on; the current approach also permits variants local run = 0 @@ -96,7 +96,7 @@ function handlers.characters(head) end end for n in traverse_id(glyph_code,head) do --- if n.subtype<256 then +-- if n.subtype<256 then -- all are 1 local font = n.font local attr = has_attribute(n,0) or 0 -- zero attribute is reserved for fonts in context if font ~= prevfont or attr ~= prevattr then diff --git a/tex/context/base/node-pro.lua b/tex/context/base/node-pro.lua index 42c6a8543..adca502db 100644 --- a/tex/context/base/node-pro.lua +++ b/tex/context/base/node-pro.lua @@ -67,7 +67,7 @@ processors.tracer = tracer processors.enabled = true -- this will become a proper state (like trackers) function processors.pre_linebreak_filter(head,groupcode,size,packtype,direction) - local first, found = first_glyph(head) + local first, found = first_glyph(head) -- they really need to be glyphs if found then if trace_callbacks then local before = nodes.count(head,true) @@ -78,10 +78,10 @@ function processors.pre_linebreak_filter(head,groupcode,size,packtype,direction) else tracer("pre_linebreak","unchanged",head,groupcode,before,after,true) end - return (done and head) or true + return done and head or true else local head, done = actions(head,groupcode,size,packtype,direction) -- todo : pass first - return (done and head) or true + return done and head or true end elseif trace_callbacks then local n = nodes.count(head,false) @@ -94,7 +94,7 @@ local enabled = true function processors.hpack_filter(head,groupcode,size,packtype,direction) if enabled then - local first, found = first_glyph(head) + local first, found = first_glyph(head) -- they really need to be glyphs if found then if trace_callbacks then local before = nodes.count(head,true) @@ -127,8 +127,8 @@ function nodes.fasthpack(...) -- todo: pass explicit arguments return hp, b end -callbacks.register('pre_linebreak_filter', processors.pre_linebreak_filter,"all kind of horizontal manipulations (before par break)") -callbacks.register('hpack_filter' , processors.hpack_filter,"all kind of horizontal manipulations") +callbacks.register('pre_linebreak_filter', processors.pre_linebreak_filter, "all kind of horizontal manipulations (before par break)") +callbacks.register('hpack_filter' , processors.hpack_filter, "all kind of horizontal manipulations (before hbox creation)") local actions = tasks.actions("finalizers") -- head, where @@ -140,27 +140,20 @@ local actions = tasks.actions("finalizers") -- head, where -- something weird here .. group mvl when making a vbox function processors.post_linebreak_filter(head,groupcode) ---~ local first, found = first_glyph(head) ---~ if found then - if trace_callbacks then - local before = nodes.count(head,true) - local head, done = actions(head,groupcode) - local after = nodes.count(head,true) - if done then - tracer("post_linebreak","changed",head,groupcode,before,after,true) - else - tracer("post_linebreak","unchanged",head,groupcode,before,after,true) - end - return (done and head) or true + if trace_callbacks then + local before = nodes.count(head,true) + local head, done = actions(head,groupcode) + local after = nodes.count(head,true) + if done then + tracer("post_linebreak","changed",head,groupcode,before,after,true) else - local head, done = actions(head,groupcode) - return (done and head) or true + tracer("post_linebreak","unchanged",head,groupcode,before,after,true) end ---~ elseif trace_callbacks then ---~ local n = nodes.count(head,false) ---~ tracer("post_linebreak","no chars",head,groupcode,n,n) ---~ end ---~ return true + return (done and head) or true + else + local head, done = actions(head,groupcode) + return (done and head) or true + end end callbacks.register('post_linebreak_filter', processors.post_linebreak_filter,"all kind of horizontal manipulations (after par break)") diff --git a/tex/context/base/page-mix.mkiv b/tex/context/base/page-mix.mkiv index 97774681c..3335c87e2 100644 --- a/tex/context/base/page-mix.mkiv +++ b/tex/context/base/page-mix.mkiv @@ -414,6 +414,13 @@ \let\strc_itemgroups_stop_columns\page_mix_fast_columns_stop +\setupmixedcolumns + [\s!itemgroupcolumns] + [\c!grid=\itemgroupparameter\c!grid] + +\setupitemgroups + [\c!grid=\v!tolerant] + %D The common initialization: \def\page_mix_initialize_columns diff --git a/tex/context/base/scrp-ini.lua b/tex/context/base/scrp-ini.lua index 0b8c49370..16d33d5d7 100644 --- a/tex/context/base/scrp-ini.lua +++ b/tex/context/base/scrp-ini.lua @@ -378,8 +378,8 @@ end -- we can have a fonts.hashes.originals -function scripts.preprocess(head) -- we could probably pass the first glyph (as it's already known) - local start = first_glyph(head) +function scripts.preprocess(head) + local start = first_glyph(head) -- we already have glyphs here (subtype 1) if not start then return head, false else diff --git a/tex/context/base/spac-flr.mkiv b/tex/context/base/spac-flr.mkiv new file mode 100644 index 000000000..0e685b644 --- /dev/null +++ b/tex/context/base/spac-flr.mkiv @@ -0,0 +1,112 @@ +%D \module +%D [ file=spac-fil, +%D version=2013.01.13, +%D title=\CONTEXT\ Spacing Macros, +%D subtitle=Fillers, +%D author={Hans Hagen and Wolfgang Schuster}, +%D date=\currentdate, +%D copyright={PRAGMA ADE \& \CONTEXT\ Development Team}] +%C +%C This module is part of the \CONTEXT\ macro||package and is +%C therefore copyrighted by \PRAGMA. See mreadme.pdf for +%C details. + +\writestatus{loading}{ConTeXt Spacing Macros / Fillers} + +\unprotect + +%D A bit more extensive variant of a prototype posted by Wolfgang to the +%D development list. Instead of dealing with a few leader types it makes +%D sense to support them all as well as simple rule fillers. Eventually we +%D can also use in for the mathfillers. We distinguish between alternatives +%D and with them methods, and a checker is provided for use in applying +%D e.g.\ fillers in lists. + +\installcorenamespace{filler} +\installcorenamespace{filleralternative} +\installcorenamespace{fillerleadermethod} + +\installcommandhandler \??filler {filler} \??filler + +\let\setupfillers\setupfiller + +\unexpanded\def\filler + {\dosingleempty\spac_fillers_indeed} + +\unexpanded\def\checkedfiller#1% + {\ifcsname\namedfillerhash{#1}\s!parent\endcsname + \spac_fillers_indeed[#1]% + \expandafter\gobbleoneargument + \else + \expandafter\firstofoneargument + \fi{#1}} + +\def\spac_fillers_indeed[#1]% + {\removeunwantedspaces + \begingroup + \edef\currentfiller{#1}% + \scratchdimen\fillerparameter\c!leftmargin\relax + \ifdim\scratchdimen=\zeropoint\else + \hskip\scratchdimen + \fi + \fillerparameter\c!left\relax + \expandcheckedcsname\??filleralternative{\fillerparameter\c!alternative}\s!unknown\relax + \fillerparameter\c!right\relax + \scratchdimen\fillerparameter\c!rightmargin\relax + \ifdim\scratchdimen=\zeropoint\else + \hskip\scratchdimen + \fi + \endgroup + \ignorespaces} + +\setvalue{\??filleralternative\s!unknown}% + {} + +\setvalue{\??filleralternative\v!symbol}% + {\expandcheckedcsname\??fillerleadermethod{\fillerparameter\c!method}\v!local + \simplealignedbox + {\fillerparameter\c!width}% + {\fillerparameter\c!align}% + {\fillerparameter\c!symbol}% + \hfill} + +\setvalue{\??filleralternative\c!stretch}% + {\hfill} + +\setvalue{\??filleralternative\c!rule}% + {\expandcheckedcsname\??fillerleadermethod{\fillerparameter\c!method}\v!local + \hrule + \!!height\fillerparameter\c!height + \!!depth \fillerparameter\c!depth + \hfill} + +\letvalue{\??fillerleadermethod\s!local }\normalleaders % overflow ends up inbetween (current box) +\letvalue{\??fillerleadermethod\v!global}\normalgleaders % overflow ends up inbetween (outermost box) +\letvalue{\??fillerleadermethod\v!middle}\normalcleaders % overflow ends up before, after (current box) +\letvalue{\??fillerleadermethod\v!broad }\normalxleaders % overflow ends up before, inbetween, after (current box) + +\setupfillers + [\c!width=\emwidth, + \c!symbol=., + \c!align=\v!middle, + \c!height=.1\exheight, + \c!depth=\zeropoint, + \c!leftmargin=\zeropoint, + \c!rightmargin=\zeropoint, + \c!alternative=\v!symbol, + \c!method=\s!local] + +% maybe box -> symbol + +\protect \endinput + +% \definefiller[test-a][alternative=stretch] +% \definefiller[test-b][alternative=symbol,symbol=!] +% \definefiller[test-c][alternative=rule,height=.1ex,leftmargin=.5em,rightmargin=.25em] + +% \starttext +% text\filler[test-a]text \par +% text\filler[test-b]text \par +% text\filler[test-c]text \par +% text\checkedfiller{}text \par +% \stoptext diff --git a/tex/context/base/spac-ver.lua b/tex/context/base/spac-ver.lua index 696635bfc..68554acb9 100644 --- a/tex/context/base/spac-ver.lua +++ b/tex/context/base/spac-ver.lua @@ -101,12 +101,14 @@ local whatsit_code = nodecodes.whatsit local userskip_code = skipcodes.userskip -builders.vspacing = builders.vspacing or { } -local vspacing = builders.vspacing -vspacing.data = vspacing.data or { } +local vspacing = builders.vspacing or { } +builders.vspacing = vspacing -vspacing.data.snapmethods = vspacing.data.snapmethods or { } -local snapmethods = vspacing.data.snapmethods --maybe some older code can go +local vspacingdata = vspacing.data or { } +vspacing.data = vspacingdata + +vspacingdata.snapmethods = vspacingdata.snapmethods or { } +local snapmethods = vspacingdata.snapmethods --maybe some older code can go storage.register("builders/vspacing/data/snapmethods", snapmethods, "builders.vspacing.data.snapmethods") @@ -554,18 +556,18 @@ function vspacing.tocategory(str) end end -vspacing.data.map = vspacing.data.map or { } -- allocate ? -vspacing.data.skip = vspacing.data.skip or { } -- allocate ? +vspacingdata.map = vspacingdata.map or { } -- allocate ? +vspacingdata.skip = vspacingdata.skip or { } -- allocate ? -storage.register("builders/vspacing/data/map", vspacing.data.map, "builders.vspacing.data.map") -storage.register("builders/vspacing/data/skip", vspacing.data.skip, "builders.vspacing.data.skip") +storage.register("builders/vspacing/data/map", vspacingdata.map, "builders.vspacing.data.map") +storage.register("builders/vspacing/data/skip", vspacingdata.skip, "builders.vspacing.data.skip") do -- todo: interface.variables vspacing.fixed = false - local map = vspacing.data.map - local skip = vspacing.data.skip + local map = vspacingdata.map + local skip = vspacingdata.skip local multiplier = C(S("+-")^0 * R("09")^1) * P("*") local category = P(":") * C(P(1)^1) @@ -1244,23 +1246,21 @@ end function vspacing.pagehandler(newhead,where) -- local newhead = texlists.contrib_head if newhead then - local newtail = find_node_tail(newhead) + local newtail = find_node_tail(newhead) -- best pass that tail, known anyway local flush = false stackhack = true -- todo: only when grid snapping once enabled for n in traverse_nodes(newhead) do -- we could just look for glue nodes local id = n.id - if id == glue_code then - if n.subtype == userskip_code then - if has_attribute(n,a_skipcategory) then - stackhack = true - else - flush = true - end + if id ~= glue_code then + flush = true + elseif n.subtype == userskip_code then + if has_attribute(n,a_skipcategory) then + stackhack = true else - -- tricky + flush = true end else - flush = true + -- tricky end end if flush then diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf index 29bcc4b27..05e831c58 100644 Binary files a/tex/context/base/status-files.pdf and b/tex/context/base/status-files.pdf differ diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf index de709d451..e045bc941 100644 Binary files a/tex/context/base/status-lua.pdf and b/tex/context/base/status-lua.pdf differ diff --git a/tex/context/base/trac-inf.lua b/tex/context/base/trac-inf.lua index 699e2b668..5575639eb 100644 --- a/tex/context/base/trac-inf.lua +++ b/tex/context/base/trac-inf.lua @@ -12,6 +12,7 @@ if not modules then modules = { } end modules ['trac-inf'] = { -- and rawget. local format, lower = string.format, string.lower +local concat = table.concat local clock = os.gettimeofday or os.clock -- should go in environment local write_nl = texio and texio.write_nl or print @@ -121,6 +122,14 @@ function statistics.show(reporter) local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 return format("%s direct, %s indirect, %s total", total-indirect, indirect, total) end) + if jit then + local status = { jit.status() } + if status[1] then + register("luajit status", function() + return concat(status," ",2) + end) + end + end collectgarbage("collect") register("current memory usage", statistics.memused) register("runtime",statistics.runtime) diff --git a/tex/context/base/type-ini.mkvi b/tex/context/base/type-ini.mkvi index 8345a97f9..ddf7cad8f 100644 --- a/tex/context/base/type-ini.mkvi +++ b/tex/context/base/type-ini.mkvi @@ -460,10 +460,6 @@ \def\font_typefaces_define_b[#name][#style][#fontshape][#fontname][#dummya][#dummyb]% {\font_typefaces_define_a[#name][#style][#fontshape][#fontname][\s!default][#dummyb]} -% \def\font_typefaces_define_c[#name][#style][#settings][#dummya][#dummyb][#dummyc]% misuse for settings -% {\font_typefaces_define_indeed[#name][#style]% -% \getparameters[\??tf#name#style][#settings]} % not used - \def\font_typefaces_define_c[#name][#style][#dummya][#dummyb][#dummyc][#dummyd]% {\font_typefaces_define_indeed[#name][#style]} @@ -561,7 +557,7 @@ %D \inherittypeface[palatino] % == [rm,ss,tt,mm] %D \stoptyping -\def\inherittypeface +\unexpanded\def\inherittypeface {\dotripleempty\font_typescripts_inherit_indeed} \def\font_typescripts_inherit_indeed[#name][#styles][#parentclass]% diff --git a/tex/context/base/util-mrg.lua b/tex/context/base/util-mrg.lua index 8d6c5dd31..fad7ef0a1 100644 --- a/tex/context/base/util-mrg.lua +++ b/tex/context/base/util-mrg.lua @@ -12,7 +12,7 @@ local gsub, format = string.gsub, string.format local concat = table.concat local type, next = type, next -utilities = utilities or {} +utilities = utilities or { } local merger = utilities.merger or { } utilities.merger = merger utilities.report = logs and logs.reporter("system") or print @@ -41,6 +41,13 @@ local m_faked = "-- " .. m_begin_merge .. "\n\n" .. "-- " .. m_end_merge .. "\n\n" +local m_report = [[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] + local function self_fake() return m_faked end @@ -59,18 +66,79 @@ local function self_load(name) return data or "" end +-- -- saves some 20K .. scite comments +-- data = gsub(data,"%-%-~[^\n\r]*[\r\n]","") +-- -- saves some 20K .. ldx comments +-- data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-ldx%]%]%-%-","") + +local P, R, S, V, Ct, C, Cs, Cc, Cp, Cmt, Cb, Cg = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Cp, lpeg.Cmt, lpeg.Cb, lpeg.Cg +local lpegmatch, patterns = lpeg.match, lpeg.patterns + +local equals = P("=")^0 +local open = P("[") * Cg(equals,"init") * P("[") * P("\n")^-1 +local close = P("]") * C(equals) * P("]") +local closeeq = Cmt(close * Cb("init"), function(s,i,a,b) return a == b end) +local longstring = open * (1 - closeeq)^0 * close + +local space = patterns.space +local eol = patterns.newline +local quoted = patterns.quoted +local emptyline = space^0 * eol +local operator1 = P("<=") + P(">=") + P("~=") + P("..") + S("/^<>=*+%%") +local operator2 = S("*+/") +local operator3 = S("-") +local separator = S(",;") + +local ignore = (P("]") * space^1 * P("=") * space^1 * P("]")) / "]=[" + + (P("=") * space^1 * P("{")) / "={" + + (P("(") * space^1) / "(" + + (P("{") * (space+eol)^1 * P("}")) / "{}" +local strings = quoted -- / function (s) print("<<"..s..">>") return s end +local longcmt = (emptyline^0 * P("--") * longstring * emptyline^0) / "" +local longstr = longstring +local comment = emptyline^0 * P("--") * P("-")^0 * (1-eol)^0 * emptyline^1 / "\n" +local pack = ((eol+space)^0 / "") * operator1 * ((eol+space)^0 / "") + + ((eol+space)^0 / "") * operator2 * ((space)^0 / "") + + ((eol+space)^1 / "") * operator3 * ((space)^1 / "") + + ((space)^0 / "") * separator * ((space)^0 / "") +local lines = emptyline^2 / "\n" +local spaces = (space * space) / " " +----- spaces = ((space+eol)^1 ) / " " + +local compact = Cs ( ( + ignore + + strings + + longcmt + + longstr + + comment + + pack + + lines + + spaces + + 1 +)^1 ) + +local strip = Cs((emptyline^2/"\n" + 1)^0) + +local function self_compact(data) + if merger.strip_comment then + local before = #data + data = lpeg.match(compact,data) + data = lpeg.match(strip,data) + -- data = string.strip(data) + local after = #data + local delta = before - after + utilities.report("merge: %s bytes compacted to %s (%s bytes stripped)",before,after,delta) + data = format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + return data, delta + else + return data, 0 + end +end + local function self_save(name, data) if data ~= "" then - if merger.strip_comment then - local n = #data - -- saves some 20K .. scite comments - data = gsub(data,"%-%-~[^\n\r]*[\r\n]","") - -- saves some 20K .. ldx comments - data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-ldx%]%]%-%-","") - utilities.report("merge: %s bytes of comment stripped, %s bytes of code left",n-#data,#data) - end io.savedata(name,data) - utilities.report("merge: saving %s",name) + utilities.report("merge: saving %s bytes in %s",#data,name) end end @@ -97,27 +165,32 @@ local function self_libs(libs,list) end if foundpath then utilities.report("merge: using library path %s",foundpath) - local right, wrong = { }, { } + local right, wrong, original, stripped = { }, { }, 0, 0 for i=1,#libs do local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then utilities.report("merge: using library %s",fullname) + local data = io.loaddata(fullname,true) + original = original + #data + local data, delta = self_compact(data) right[#right+1] = lib result[#result+1] = m_begin_closure - result[#result+1] = io.loaddata(fullname,true) + result[#result+1] = data result[#result+1] = m_end_closure + stripped = stripped + delta else utilities.report("merge: skipping library %s",fullname) wrong[#wrong+1] = lib end end - if #right > 0 then - utilities.report("merge: used libraries: %s",concat(right," ")) - end - if #wrong > 0 then - utilities.report("merge: skipped libraries: %s",concat(wrong," ")) - end + right = #right > 0 and concat(right," ") or "-" + wrong = #wrong > 0 and concat(wrong," ") or "-" + utilities.report("merge: used libraries: %s",right) + utilities.report("merge: skipped libraries: %s",wrong) + utilities.report("merge: original bytes: %s",original) + utilities.report("merge: stripped bytes: %s",stripped) + result[#result+1] = format(m_report,right,wrong,original,stripped) else utilities.report("merge: no valid library path found") end diff --git a/tex/context/base/util-seq.lua b/tex/context/base/util-seq.lua index 711424a2c..0bf056365 100644 --- a/tex/context/base/util-seq.lua +++ b/tex/context/base/util-seq.lua @@ -235,6 +235,7 @@ local function construct(t) t.compiled = format("%s\nreturn function(%s)\n%s\nend",variables,arguments,calls) end end +-- print(t.compiled) return t.compiled -- also stored so that we can trace end @@ -273,7 +274,7 @@ sequencers.compile = compile -- todo: use sequencer (can have arguments and returnvalues etc now) -local template = [[ +local template_yes = [[ %s return function(head%s) local ok, done = false, false @@ -281,6 +282,11 @@ return function(head%s) return head, done end]] +local template_nop = [[ +return function() + return false, false +end]] + function sequencers.nodeprocessor(t,nofarguments) -- todo: handle 'kind' in plug into tostring local list, order, kind, gskip, askip = t.list, t.order, t.kind, t.gskip, t.askip local vars, calls, args, n = { }, { }, nil, 0 @@ -319,7 +325,6 @@ function sequencers.nodeprocessor(t,nofarguments) -- todo: handle 'kind' in plug end end end - local processor = format(template,concat(vars,"\n"),args,concat(calls,"\n")) - -- print(processor) + local processor = #calls > 0 and format(template_yes,concat(vars,"\n"),args,concat(calls,"\n")) or template_nop return processor end diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index 828169554..60e2a7942 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua --- merge date : 01/10/13 01:04:42 +-- merge date : 01/13/13 23:10:29 do -- begin closure to overcome local limits and interference @@ -59,8 +59,10 @@ local report = texio and texio.write_nl or print -- function lpeg.Cmt (l) local p = lpcmt (l) report("LPEG Cmt =") lpprint(l) return p end -- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end -local type, next = type, next +local type, next, tostring = type, next, tostring local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format +----- mod, div = math.mod, math.div +local floor = math.floor -- Beware, we predefine a bunch of patterns here and one reason for doing so -- is that we get consistent behaviour in some of the visualizers. @@ -785,6 +787,47 @@ end patterns.containseol = lpeg.finder(eol) -- (1-eol)^0 * eol +-- The next pattern^n variant is based on an approach suggested +-- by Roberto: constructing a big repetition in chunks. +-- +-- Being sparse is not needed, and only complicate matters and +-- the number of redundant entries is not that large. + +local function nextstep(n,step,result) + local m = n % step -- mod(n,step) + local d = floor(n/step) -- div(n,step) + if d > 0 then + local v = V(tostring(step)) + local s = result.start + for i=1,d do + if s then + s = v * s + else + s = v + end + end + result.start = s + end + if step > 1 and result.start then + local v = V(tostring(step/2)) + result[tostring(step)] = v * v + end + if step > 0 then + return nextstep(m,step/2,result) + else + return result + end +end + +function lpeg.times(pattern,n) + return P(nextstep(n,2^16,{ "start", ["1"] = pattern })) +end + +-- local p = lpeg.Cs((1 - lpeg.times(lpeg.P("AB"),25))^1) +-- local s = "12" .. string.rep("AB",20) .. "34" .. string.rep("AB",30) .. "56" +-- inspect(p) +-- print(lpeg.match(p,s)) + end -- closure do -- begin closure to overcome local limits and interference @@ -856,7 +899,7 @@ end -- print(string.unquoted('"test"')) function string.quoted(str) - return format("%q",str) -- always " + return format("%q",str) -- always double quote end function string.count(str,pattern) -- variant 3 @@ -2193,7 +2236,8 @@ local noslashes = 1-slashes local name = noperiod^1 local suffix = period/"" * (1-period-slashes)^1 * -1 -local pattern = C((noslashes^0 * slashes^1)^1) +----- pattern = C((noslashes^0 * slashes^1)^1) +local pattern = C((1 - (slashes^1 * noslashes^1 * -1))^1) * P(1) -- there must be a more efficient way local function pathpart(name,default) return name and lpegmatch(pattern,name) or default or "" @@ -2205,6 +2249,13 @@ local function basename(name) return name and lpegmatch(pattern,name) or name end +-- print(pathpart("file")) +-- print(pathpart("dir/file")) +-- print(pathpart("/dir/file")) +-- print(basename("file")) +-- print(basename("dir/file")) +-- print(basename("/dir/file")) + local pattern = (noslashes^0 * slashes^1)^0 * Cs((1-suffix)^1) * suffix^0 local function nameonly(name) @@ -2229,7 +2280,7 @@ file.extname = suffixonly -- obsolete -- actually these are schemes local drive = C(R("az","AZ")) * colon -local path = C(((1-slashes)^0 * slashes)^0) +local path = C((noslashes^0 * slashes)^0) local suffix = period * C(P(1-period)^0 * P(-1)) local base = C((1-suffix)^0) local rest = C(P(1)^0) @@ -2258,9 +2309,14 @@ function file.splitbase(str) return str and lpegmatch(pattern_d,str) -- returns path, base+suffix end -function file.nametotable(str,splitdrive) -- returns table +---- stripslash = C((1 - P("/")^1*P(-1))^0) + +function file.nametotable(str,splitdrive) if str then local path, drive, subpath, name, base, suffix = lpegmatch(pattern_c,str) + -- if path ~= "" then + -- path = lpegmatch(stripslash,path) -- unfortunate hack, maybe this becomes default + -- end if splitdrive then return { path = path, @@ -2281,6 +2337,20 @@ function file.nametotable(str,splitdrive) -- returns table end end +-- print(file.splitname("file")) +-- print(file.splitname("dir/file")) +-- print(file.splitname("/dir/file")) +-- print(file.splitname("file")) +-- print(file.splitname("dir/file")) +-- print(file.splitname("/dir/file")) + +-- inspect(file.nametotable("file.ext")) +-- inspect(file.nametotable("dir/file.ext")) +-- inspect(file.nametotable("/dir/file.ext")) +-- inspect(file.nametotable("file.ext")) +-- inspect(file.nametotable("dir/file.ext")) +-- inspect(file.nametotable("/dir/file.ext")) + local pattern = Cs(((period * noperiod^1 * -1)/"" + 1)^1) function file.removesuffix(name) @@ -3993,6 +4063,8 @@ function constructors.scale(tfmdata,specification) target.psname = psname target.name = name -- + -- inspect(properties) + -- properties.fontname = fontname properties.fullname = fullname properties.filename = filename @@ -9508,9 +9580,6 @@ local handlers = { } local rlmode = 0 local featurevalue = false --- we cannot optimize with "start = first_glyph(head)" because then we don't --- know which rlmode we're in which messes up cursive handling later on --- -- head is always a whatsit so we can safely assume that head is not changed -- we use this for special testing and documentation @@ -11464,7 +11533,7 @@ local function featuresprocessor(head,font,attr) -- font interactions and then we do need the full sweeps. -- Keeping track of the headnode is needed for devanagari (I generalized it a bit - -- so that multiple cases are also covered. + -- so that multiple cases are also covered.) for s=1,#sequences do local dataset = datasets[s] -- cgit v1.2.3