diff options
126 files changed, 5201 insertions, 4405 deletions
diff --git a/doc/context/documents/general/qrcs/setup-cs.pdf b/doc/context/documents/general/qrcs/setup-cs.pdf Binary files differindex af330a3a9..c2bc58a10 100644 --- a/doc/context/documents/general/qrcs/setup-cs.pdf +++ b/doc/context/documents/general/qrcs/setup-cs.pdf diff --git a/doc/context/documents/general/qrcs/setup-de.pdf b/doc/context/documents/general/qrcs/setup-de.pdf Binary files differindex 56c1b6c25..ccce3d287 100644 --- a/doc/context/documents/general/qrcs/setup-de.pdf +++ b/doc/context/documents/general/qrcs/setup-de.pdf diff --git a/doc/context/documents/general/qrcs/setup-en.pdf b/doc/context/documents/general/qrcs/setup-en.pdf Binary files differindex 94d34d7b3..d57e3f0e4 100644 --- a/doc/context/documents/general/qrcs/setup-en.pdf +++ b/doc/context/documents/general/qrcs/setup-en.pdf diff --git a/doc/context/documents/general/qrcs/setup-fr.pdf b/doc/context/documents/general/qrcs/setup-fr.pdf Binary files differindex fa1361021..330b61558 100644 --- a/doc/context/documents/general/qrcs/setup-fr.pdf +++ b/doc/context/documents/general/qrcs/setup-fr.pdf diff --git a/doc/context/documents/general/qrcs/setup-it.pdf b/doc/context/documents/general/qrcs/setup-it.pdf Binary files differindex 37cef3f91..157475def 100644 --- a/doc/context/documents/general/qrcs/setup-it.pdf +++ b/doc/context/documents/general/qrcs/setup-it.pdf diff --git a/doc/context/documents/general/qrcs/setup-mapping-cs.pdf b/doc/context/documents/general/qrcs/setup-mapping-cs.pdf Binary files differindex 5c68bf5e7..03d835033 100644 --- a/doc/context/documents/general/qrcs/setup-mapping-cs.pdf +++ b/doc/context/documents/general/qrcs/setup-mapping-cs.pdf diff --git a/doc/context/documents/general/qrcs/setup-mapping-de.pdf b/doc/context/documents/general/qrcs/setup-mapping-de.pdf Binary files differindex e12672210..257bf9547 100644 --- a/doc/context/documents/general/qrcs/setup-mapping-de.pdf +++ b/doc/context/documents/general/qrcs/setup-mapping-de.pdf diff --git a/doc/context/documents/general/qrcs/setup-mapping-en.pdf b/doc/context/documents/general/qrcs/setup-mapping-en.pdf Binary files differindex 4eb4377ba..33314b29f 100644 --- a/doc/context/documents/general/qrcs/setup-mapping-en.pdf +++ b/doc/context/documents/general/qrcs/setup-mapping-en.pdf diff --git a/doc/context/documents/general/qrcs/setup-mapping-fr.pdf b/doc/context/documents/general/qrcs/setup-mapping-fr.pdf Binary files differindex 1160f077e..224e69e38 100644 --- a/doc/context/documents/general/qrcs/setup-mapping-fr.pdf +++ b/doc/context/documents/general/qrcs/setup-mapping-fr.pdf diff --git a/doc/context/documents/general/qrcs/setup-mapping-it.pdf b/doc/context/documents/general/qrcs/setup-mapping-it.pdf Binary files differindex 0ec44b70f..b0c3d05d2 100644 --- a/doc/context/documents/general/qrcs/setup-mapping-it.pdf +++ b/doc/context/documents/general/qrcs/setup-mapping-it.pdf diff --git a/doc/context/documents/general/qrcs/setup-mapping-nl.pdf b/doc/context/documents/general/qrcs/setup-mapping-nl.pdf Binary files differindex 972fbf9bf..f60727ee1 100644 --- a/doc/context/documents/general/qrcs/setup-mapping-nl.pdf +++ b/doc/context/documents/general/qrcs/setup-mapping-nl.pdf diff --git a/doc/context/documents/general/qrcs/setup-mapping-ro.pdf b/doc/context/documents/general/qrcs/setup-mapping-ro.pdf Binary files differindex 8914a305f..a847ca390 100644 --- a/doc/context/documents/general/qrcs/setup-mapping-ro.pdf +++ b/doc/context/documents/general/qrcs/setup-mapping-ro.pdf diff --git a/doc/context/documents/general/qrcs/setup-nl.pdf b/doc/context/documents/general/qrcs/setup-nl.pdf Binary files differindex f343e38b7..050f05eba 100644 --- a/doc/context/documents/general/qrcs/setup-nl.pdf +++ b/doc/context/documents/general/qrcs/setup-nl.pdf diff --git a/doc/context/documents/general/qrcs/setup-ro.pdf b/doc/context/documents/general/qrcs/setup-ro.pdf Binary files differindex 3c831ce25..8e4a2831d 100644 --- a/doc/context/documents/general/qrcs/setup-ro.pdf +++ b/doc/context/documents/general/qrcs/setup-ro.pdf diff --git a/doc/context/scripts/mkiv/context.html b/doc/context/scripts/mkiv/context.html index 9a8a18609..3262b91d1 100644 --- a/doc/context/scripts/mkiv/context.html +++ b/doc/context/scripts/mkiv/context.html @@ -62,7 +62,8 @@ <tr><th>--directives</th><td>list</td><td>set directive variables (show list with --showdirectives)</td></tr> <tr><th>--silent</th><td>list</td><td>disable logcatgories (show list with --showlogcategories)</td></tr> <tr><th>--strip</th><td></td><td>strip Lua code (only meant for production where no errors are expected)</td></tr> - <tr><th>--errors</th><td>list</td><td>show errors at the end of a run, quit when in list (also when ----silent)</td></tr> + <tr><th>--errors</th><td>list</td><td>show errors at the end of a run, quit when in list (also when --silent)</td></tr> + <tr><th>--htmlerrorpage</th><td></td><td>generate html error page instead (optional: =scite)</td></tr> <tr><th>--noconsole</th><td></td><td>disable logging to the console (logfile only)</td></tr> <tr><th>--purgeresult</th><td></td><td>purge result file before run</td></tr> <tr><th/><td/><td/></tr> @@ -83,8 +84,10 @@ <tr><th/><td/><td/></tr> <tr><th>--batchmode</th><td></td><td>run without stopping and do not show messages on the console</td></tr> <tr><th>--nonstopmode</th><td></td><td>run without stopping</td></tr> + <tr><th/><td/><td/></tr> <tr><th>--nosynctex</th><td></td><td>never initializes synctex (for production runs)</td></tr> <tr><th>--synctex</th><td></td><td>run with synctex enabled (better use \setupsynctex[state=start]</td></tr> + <tr><th/><td/><td/></tr> <tr><th>--nodates</th><td></td><td>omit runtime dates in pdf file (optional value: a number (this 1970 offset time) or string "YYYY-MM-DD HH:MM")</td></tr> <tr><th>--nocompression</th><td></td><td>forcefully turns off compression in the backend</td></tr> <tr><th>--trailerid</th><td></td><td>alternative trailer id (or constant one)</td></tr> diff --git a/doc/context/scripts/mkiv/context.man b/doc/context/scripts/mkiv/context.man index 5f28efd81..04e3fdc1a 100644 --- a/doc/context/scripts/mkiv/context.man +++ b/doc/context/scripts/mkiv/context.man @@ -69,7 +69,10 @@ disable logcatgories (show list with --showlogcategories) strip Lua code (only meant for production where no errors are expected) .TP .B --errors=list -show errors at the end of a run, quit when in list (also when ----silent) +show errors at the end of a run, quit when in list (also when --silent) +.TP +.B --htmlerrorpage +generate html error page instead (optional: =scite) .TP .B --noconsole disable logging to the console (logfile only) diff --git a/doc/context/scripts/mkiv/context.xml b/doc/context/scripts/mkiv/context.xml index b0566517d..0c7038d26 100644 --- a/doc/context/scripts/mkiv/context.xml +++ b/doc/context/scripts/mkiv/context.xml @@ -74,7 +74,10 @@ <short>strip Lua code (only meant for production where no errors are expected)</short> </flag> <flag name="errors" value="list"> - <short>show errors at the end of a run, quit when in list (also when <ref name="--silent"/>)</short> + <short>show errors at the end of a run, quit when in list (also when <ref name="silent"/>)</short> + </flag> + <flag name="htmlerrorpage"> + <short>generate html error page instead (optional: =scite)</short> </flag> <flag name="noconsole"> <short>disable logging to the console (logfile only)</short> @@ -131,12 +134,16 @@ <flag name="nonstopmode"> <short>run without stopping</short> </flag> + </subcategory> + <subcategory> <flag name="nosynctex"> <short>never initializes synctex (for production runs)</short> </flag> <flag name="synctex"> <short>run with synctex enabled (better use \setupsynctex[state=start]</short> </flag> + </subcategory> + <subcategory> <flag name="nodates"> <short>omit runtime dates in pdf file (optional value: a number (this 1970 offset time) or string "YYYY-MM-DD HH:MM")</short> </flag> @@ -146,7 +153,7 @@ <flag name="trailerid"> <short>alternative trailer id (or constant one)</short> </flag> - </subcategory> + </subcategory> <subcategory> <flag name="generate"> <short>generate file database etc. (as luatools does)</short> diff --git a/doc/context/scripts/mkiv/mtx-context.html b/doc/context/scripts/mkiv/mtx-context.html index 9a8a18609..3262b91d1 100644 --- a/doc/context/scripts/mkiv/mtx-context.html +++ b/doc/context/scripts/mkiv/mtx-context.html @@ -62,7 +62,8 @@ <tr><th>--directives</th><td>list</td><td>set directive variables (show list with --showdirectives)</td></tr> <tr><th>--silent</th><td>list</td><td>disable logcatgories (show list with --showlogcategories)</td></tr> <tr><th>--strip</th><td></td><td>strip Lua code (only meant for production where no errors are expected)</td></tr> - <tr><th>--errors</th><td>list</td><td>show errors at the end of a run, quit when in list (also when ----silent)</td></tr> + <tr><th>--errors</th><td>list</td><td>show errors at the end of a run, quit when in list (also when --silent)</td></tr> + <tr><th>--htmlerrorpage</th><td></td><td>generate html error page instead (optional: =scite)</td></tr> <tr><th>--noconsole</th><td></td><td>disable logging to the console (logfile only)</td></tr> <tr><th>--purgeresult</th><td></td><td>purge result file before run</td></tr> <tr><th/><td/><td/></tr> @@ -83,8 +84,10 @@ <tr><th/><td/><td/></tr> <tr><th>--batchmode</th><td></td><td>run without stopping and do not show messages on the console</td></tr> <tr><th>--nonstopmode</th><td></td><td>run without stopping</td></tr> + <tr><th/><td/><td/></tr> <tr><th>--nosynctex</th><td></td><td>never initializes synctex (for production runs)</td></tr> <tr><th>--synctex</th><td></td><td>run with synctex enabled (better use \setupsynctex[state=start]</td></tr> + <tr><th/><td/><td/></tr> <tr><th>--nodates</th><td></td><td>omit runtime dates in pdf file (optional value: a number (this 1970 offset time) or string "YYYY-MM-DD HH:MM")</td></tr> <tr><th>--nocompression</th><td></td><td>forcefully turns off compression in the backend</td></tr> <tr><th>--trailerid</th><td></td><td>alternative trailer id (or constant one)</td></tr> diff --git a/doc/context/scripts/mkiv/mtx-context.man b/doc/context/scripts/mkiv/mtx-context.man index 5f28efd81..04e3fdc1a 100644 --- a/doc/context/scripts/mkiv/mtx-context.man +++ b/doc/context/scripts/mkiv/mtx-context.man @@ -69,7 +69,10 @@ disable logcatgories (show list with --showlogcategories) strip Lua code (only meant for production where no errors are expected) .TP .B --errors=list -show errors at the end of a run, quit when in list (also when ----silent) +show errors at the end of a run, quit when in list (also when --silent) +.TP +.B --htmlerrorpage +generate html error page instead (optional: =scite) .TP .B --noconsole disable logging to the console (logfile only) diff --git a/doc/context/scripts/mkiv/mtx-context.xml b/doc/context/scripts/mkiv/mtx-context.xml index b0566517d..0c7038d26 100644 --- a/doc/context/scripts/mkiv/mtx-context.xml +++ b/doc/context/scripts/mkiv/mtx-context.xml @@ -74,7 +74,10 @@ <short>strip Lua code (only meant for production where no errors are expected)</short> </flag> <flag name="errors" value="list"> - <short>show errors at the end of a run, quit when in list (also when <ref name="--silent"/>)</short> + <short>show errors at the end of a run, quit when in list (also when <ref name="silent"/>)</short> + </flag> + <flag name="htmlerrorpage"> + <short>generate html error page instead (optional: =scite)</short> </flag> <flag name="noconsole"> <short>disable logging to the console (logfile only)</short> @@ -131,12 +134,16 @@ <flag name="nonstopmode"> <short>run without stopping</short> </flag> + </subcategory> + <subcategory> <flag name="nosynctex"> <short>never initializes synctex (for production runs)</short> </flag> <flag name="synctex"> <short>run with synctex enabled (better use \setupsynctex[state=start]</short> </flag> + </subcategory> + <subcategory> <flag name="nodates"> <short>omit runtime dates in pdf file (optional value: a number (this 1970 offset time) or string "YYYY-MM-DD HH:MM")</short> </flag> @@ -146,7 +153,7 @@ <flag name="trailerid"> <short>alternative trailer id (or constant one)</short> </flag> - </subcategory> + </subcategory> <subcategory> <flag name="generate"> <short>generate file database etc. (as luatools does)</short> diff --git a/doc/context/scripts/mkiv/mtx-scite.html b/doc/context/scripts/mkiv/mtx-scite.html index 497098f9e..957d7595e 100644 --- a/doc/context/scripts/mkiv/mtx-scite.html +++ b/doc/context/scripts/mkiv/mtx-scite.html @@ -41,7 +41,7 @@ <tr><th/><td/><td/></tr> <tr><th>--words</th><td></td><td>convert spell-*.txt into spell-*.lua</td></tr> <tr><th>--tree</th><td></td><td>converts a tree into an html tree (--source --target --numbers)</td></tr> - <tr><th>--file</th><td></td><td>converts a file into an html tree (--source --target --numbers --lexer)</td></tr> + <tr><th>--file</th><td></td><td>converts a file into an html file (--source --target --numbers --lexer)</td></tr> </table> <br/> </div> diff --git a/doc/context/scripts/mkiv/mtx-scite.man b/doc/context/scripts/mkiv/mtx-scite.man index 60450c516..cdae4ff23 100644 --- a/doc/context/scripts/mkiv/mtx-scite.man +++ b/doc/context/scripts/mkiv/mtx-scite.man @@ -18,7 +18,7 @@ convert spell-*.txt into spell-*.lua converts a tree into an html tree (--source --target --numbers) .TP .B --file -converts a file into an html tree (--source --target --numbers --lexer) +converts a file into an html file (--source --target --numbers --lexer) .SH AUTHOR More information about ConTeXt and the tools that come with it can be found at: diff --git a/doc/context/scripts/mkiv/mtx-scite.xml b/doc/context/scripts/mkiv/mtx-scite.xml index 65ad8736a..f49aa8116 100644 --- a/doc/context/scripts/mkiv/mtx-scite.xml +++ b/doc/context/scripts/mkiv/mtx-scite.xml @@ -10,7 +10,7 @@ <subcategory> <flag name="words"><short>convert spell-*.txt into spell-*.lua</short></flag> <flag name="tree"><short>converts a tree into an html tree (--source --target --numbers)</short></flag> - <flag name="file"><short>converts a file into an html tree (--source --target --numbers --lexer)</short></flag> + <flag name="file"><short>converts a file into an html file (--source --target --numbers --lexer)</short></flag> </subcategory> </category> </flags> diff --git a/metapost/context/base/mpiv/mp-lmtx.mpxl b/metapost/context/base/mpiv/mp-lmtx.mpxl index 4a6fde70d..90095d456 100644 --- a/metapost/context/base/mpiv/mp-lmtx.mpxl +++ b/metapost/context/base/mpiv/mp-lmtx.mpxl @@ -19,15 +19,18 @@ if known context_lmtx : endinput ; fi ; boolean context_lmtx ; context_lmtx := true ; presetparameters "text" [ - offset = 0, - strut = "auto", - style = "", - color = "", - text = "", - anchor = "", - format = "", - position = origin, - trace = false, + offset = 0, + strut = "auto", + style = "", + color = "", + text = "", + anchor = "", + format = "", + position = origin, + trace = false, + + background = "color", + backgroundcolor = "gray", ] ; def lmt_text = applyparameters "text" "lmt_do_text" enddef ; @@ -36,13 +39,14 @@ vardef lmt_do_text = image ( pushparameters "text" ; save style, anchor, txt, fmt, strt ; - string style, anchor, txt, fmt, strt ; + string style, anchor, txt, fmt, strt, bgr ; interim textextoffset := getparameter "offset" ; style := getparameter "style" ; anchor := getparameter "anchor" ; strt := getparameter "strut" ; fmt := getparameter "format" ; txt := getparameter "text" ; + bgr := getparameter "background" ; if fmt <> "" : txt := "\formatone{" & fmt & "}{" & txt & "}" fi ; @@ -63,6 +67,9 @@ vardef lmt_do_text = getparameter "position" ) withcolor getparameter "color" ; + if bgr = "color" : + addbackground withcolor getparameter "backgroundcolor" ; + fi ; popparameters ; ) enddef ; diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index 0a72522e5..4e8c364d5 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -1596,6 +1596,17 @@ end do + local htmlerrorpage = getargument("htmlerrorpage") + if htmlerrorpage == "scite" then + directives.enable("system.showerror=scite") + elseif htmlerrorpage then + directives.enable("system.showerror") + end + +end + +do + local silent = getargument("silent") if type(silent) == "string" then directives.enable(format("logs.blocked={%s}",silent)) diff --git a/scripts/context/lua/mtx-context.xml b/scripts/context/lua/mtx-context.xml index b0566517d..0c7038d26 100644 --- a/scripts/context/lua/mtx-context.xml +++ b/scripts/context/lua/mtx-context.xml @@ -74,7 +74,10 @@ <short>strip Lua code (only meant for production where no errors are expected)</short> </flag> <flag name="errors" value="list"> - <short>show errors at the end of a run, quit when in list (also when <ref name="--silent"/>)</short> + <short>show errors at the end of a run, quit when in list (also when <ref name="silent"/>)</short> + </flag> + <flag name="htmlerrorpage"> + <short>generate html error page instead (optional: =scite)</short> </flag> <flag name="noconsole"> <short>disable logging to the console (logfile only)</short> @@ -131,12 +134,16 @@ <flag name="nonstopmode"> <short>run without stopping</short> </flag> + </subcategory> + <subcategory> <flag name="nosynctex"> <short>never initializes synctex (for production runs)</short> </flag> <flag name="synctex"> <short>run with synctex enabled (better use \setupsynctex[state=start]</short> </flag> + </subcategory> + <subcategory> <flag name="nodates"> <short>omit runtime dates in pdf file (optional value: a number (this 1970 offset time) or string "YYYY-MM-DD HH:MM")</short> </flag> @@ -146,7 +153,7 @@ <flag name="trailerid"> <short>alternative trailer id (or constant one)</short> </flag> - </subcategory> + </subcategory> <subcategory> <flag name="generate"> <short>generate file database etc. (as luatools does)</short> diff --git a/scripts/context/lua/mtx-scite.lua b/scripts/context/lua/mtx-scite.lua index ae8c67387..96020de3d 100644 --- a/scripts/context/lua/mtx-scite.lua +++ b/scripts/context/lua/mtx-scite.lua @@ -25,7 +25,7 @@ local helpinfo = [[ <subcategory> <flag name="words"><short>convert spell-*.txt into spell-*.lua</short></flag> <flag name="tree"><short>converts a tree into an html tree (--source --target --numbers)</short></flag> - <flag name="file"><short>converts a file into an html tree (--source --target --numbers --lexer)</short></flag> + <flag name="file"><short>converts a file into an html file (--source --target --numbers --lexer)</short></flag> </subcategory> </category> </flags> diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 36a7bbcd2..bf87526f2 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -324,7 +324,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-macro"] = package.loaded["l-macro"] or true --- original size: 10131, stripped down to: 5991 +-- original size: 10130, stripped down to: 5990 if not modules then modules={} end modules ['l-macros']={ version=1.001, @@ -549,7 +549,7 @@ end macros.loaded=loaded function required(name,trace) local filename=file.addsuffix(name,"lua") - local fullname=resolvers and resolvers.find_file(filename) or filename + local fullname=resolvers and resolvers.findfile(filename) or filename if not fullname or fullname=="" then return false end @@ -12608,7 +12608,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 13348, stripped down to: 8838 +-- original size: 13394, stripped down to: 8882 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -12618,8 +12618,9 @@ if not modules then modules={} end modules ['trac-set']={ license="see context related readme files" } local type,next,tostring,tonumber=type,next,tostring,tonumber +local print=print local concat,sortedhash=table.concat,table.sortedhash -local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local formatters,find,lower,gsub,topattern=string.formatters,string.find,string.lower,string.gsub,string.topattern local is_boolean=string.is_boolean local settings_to_hash=utilities.parsers.settings_to_hash local allocate=utilities.storage.allocate @@ -12826,8 +12827,8 @@ function setters.show(t) end end local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show -function setters.report(setter,...) - print(format("%-15s : %s\n",setter.name,format(...))) +function setters.report(setter,fmt,...) + print(formatters["%-15s : %s\n"](setter.name,formatters[fmt](...))) end local function default(setter,name) local d=setter.data[name] @@ -12950,7 +12951,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 33803, stripped down to: 21818 +-- original size: 32997, stripped down to: 21715 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -13613,12 +13614,6 @@ end local nesting=0 local verbose=false local hasscheme=url.hasscheme -function logs.show_open(name) -end -function logs.show_close(name) -end -function logs.show_load(name) -end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple @@ -20686,7 +20681,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 10987, stripped down to: 7056 +-- original size: 11019, stripped down to: 7086 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -20699,6 +20694,7 @@ local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local sortedpairs=table.sortedpairs local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) @@ -20909,7 +20905,7 @@ if ostype=="unix" then rawset(t,k,v) end local colon=P(":") - for k,v in table.sortedpairs(prefixes) do + for k,v in sortedpairs(prefixes) do if p then p=P(k)+p else @@ -20936,7 +20932,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18154, stripped down to: 10416 +-- original size: 18173, stripped down to: 10432 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -21220,7 +21216,7 @@ local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) scancache[sub(full,1,-2)]=files return files,remap,n,m,r end -function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) +local function scanfiles(path,branch,usecache,onlyonce,tolerant) local realpath=resolveprefix(path) if usecache then local content=fullcache[realpath] @@ -21275,8 +21271,9 @@ function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) statistics.stoptiming(timer) return content end +resolvers.scanfiles=scanfiles function resolvers.simplescanfiles(path,branch,usecache) - return resolvers.scanfiles(path,branch,usecache,true,true) + return scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -21343,7 +21340,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9384, stripped down to: 6333 +-- original size: 9400, stripped down to: 6347 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -21353,7 +21350,7 @@ if not modules then modules={} end modules ['data-env']={ license="see context related readme files", } local lower,gsub=string.lower,string.gsub -local next=next +local next,rawget=next,rawget local resolvers=resolvers local allocate=utilities.storage.allocate local setmetatableindex=table.setmetatableindex @@ -21628,7 +21625,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16387, stripped down to: 11132 +-- original size: 16070, stripped down to: 11353 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -21637,49 +21634,83 @@ if not modules then modules={} end modules ['data-tmp']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat=table.concat -local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile -local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable -local formatters=string.formatters local next,type=next,type +local pcall,loadfile,collectgarbage=pcall,loadfile,collectgarbage +local format,lower,gsub=string.format,string.lower,string.gsub +local concat,serialize,fastserialize,serializetofile=table.concat,table.serialize,table.fastserialize,table.tofile +local mkdirs,expanddirname,isdir,isfile=dir.mkdirs,dir.expandname,lfs.isdir,lfs.isfile +local is_writable,is_readable=file.is_writable,file.is_readable +local collapsepath,joinfile,addsuffix,dirname=file.collapsepath,file.join,file.addsuffix,file.dirname +local savedata=file.savedata +local formatters=string.formatters +local osexit,osdate,osuuid=os.exit,os.date,os.uuid +local removefile=os.remove +local md5hex=md5.hex local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers local cleanpath=resolvers.cleanpath -local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) -local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) -local compile=utilities.lua.compile -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup==nil then cleanup=directive_cleanup end - if strip==nil then strip=directive_strip end - return compile(luafile,lucfile,cleanup,strip) +local resolvepath=resolvers.resolve +local luautilities=utilities.lua +do + local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) + local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) + local compilelua=luautilities.compile + function luautilities.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compilelua(luafile,lucfile,cleanup,strip) + end end caches=caches or {} local caches=caches -local luasuffixes=utilities.lua.suffixes -caches.base=caches.base or "luatex-cache" -caches.more=caches.more or "context" -caches.direct=false -caches.tree=false -caches.force=true -caches.ask=false -caches.relocate=false +local writable=nil +local readables={} +local usedreadables={} +local compilelua=luautilities.compile +local luasuffixes=luautilities.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } -directives.register("system.caches.fast",function(v) caches.fast=true end) -directives.register("system.caches.direct",function(v) caches.direct=true end) -local writable,readables,usedreadables=nil,{},{} +local direct_cache=false +local fast_cache=false +local cache_tree=false +directives.register("system.caches.direct",function(v) direct_cache=true end) +directives.register("system.caches.fast",function(v) fast_cache=true end) +local function configfiles() + return concat(resolvers.configurationfiles(),";") +end +local function hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +local function treehash() + local tree=configfiles() + if not tree or tree=="" then + return false + else + return hashed(tree) + end +end +caches.hashed=hashed +caches.treehash=treehash +caches.configfiles=configfiles local function identify() local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) - cachepath=file.collapsepath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) + cachepath=collapsepath(cachepath) local valid=isdir(cachepath) if valid then if is_readable(cachepath) then @@ -21688,16 +21719,14 @@ local function identify() writable=cachepath end end - elseif not writable and caches.force then - local cacheparent=file.dirname(cachepath) - if is_writable(cacheparent) and true then - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("path %a created",cachepath) - writable=cachepath - readables[#readables+1]=cachepath - end + elseif not writable then + local cacheparent=dirname(cachepath) + if is_writable(cacheparent) then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath end end end @@ -21710,8 +21739,8 @@ local function identify() local cachepath=texmfcaches[k] cachepath=resolvers.expansion(cachepath) if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) local valid=isdir(cachepath) if valid and is_readable(cachepath) then if not writable and is_writable(cachepath) then @@ -21725,23 +21754,25 @@ local function identify() end if not writable then report_caches("fatal error: there is no valid writable cache path defined") - os.exit() + osexit() elseif #readables==0 then report_caches("fatal error: there is no valid readable cache path defined") - os.exit() + osexit() end - writable=dir.expandname(resolvers.cleanpath(writable)) - local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + writable=expanddirname(cleanpath(writable)) + local base=caches.base + local more=caches.more + local tree=cache_tree or treehash() if tree then - caches.tree=tree + cache_tree=tree writable=mkdirs(writable,base,more,tree) for i=1,#readables do - readables[i]=file.join(readables[i],base,more,tree) + readables[i]=joinfile(readables[i],base,more,tree) end else writable=mkdirs(writable,base,more) for i=1,#readables do - readables[i]=file.join(readables[i],base,more) + readables[i]=joinfile(readables[i],base,more) end end if trace_cache then @@ -21778,27 +21809,8 @@ function caches.usedpaths(separator) return writable or "?" end end -function caches.configfiles() - return concat(resolvers.configurationfiles(),";") -end -function caches.hashed(tree) - tree=gsub(tree,"[\\/]+$","") - tree=lower(tree) - local hash=md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %a, hash %a",tree,hash) - end - return hash -end -function caches.treehash() - local tree=caches.configfiles() - if not tree or tree=="" then - return false - else - return caches.hashed(tree) - end -end -local r_cache,w_cache={},{} +local r_cache={} +local w_cache={} local function getreadablepaths(...) local tags={... } local hash=concat(tags,"/") @@ -21808,7 +21820,7 @@ local function getreadablepaths(...) if #tags>0 then done={} for i=1,#readables do - done[i]=file.join(readables[i],...) + done[i]=joinfile(readables[i],...) end else done=readables @@ -21832,17 +21844,25 @@ local function getwritablepath(...) end return done end -caches.getreadablepaths=getreadablepaths -caches.getwritablepath=getwritablepath -function caches.getfirstreadablefile(filename,...) - local fullname,path=caches.setfirstwritablefile(filename,...) +local function setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=joinfile(wr,filename) + return fullname,wr +end +local function setluanames(path,name) + return + format("%s/%s.%s",path,name,luasuffixes.tma), + format("%s/%s.%s",path,name,luasuffixes.tmc) +end +local function getfirstreadablefile(filename,...) + local fullname,path=setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path end local rd=getreadablepaths(...) for i=1,#rd do local path=rd[i] - local fullname=file.join(path,filename) + local fullname=joinfile(path,filename) if is_readable(fullname) then usedreadables[i]=true return fullname,path @@ -21850,19 +21870,11 @@ function caches.getfirstreadablefile(filename,...) end return fullname,path end -function caches.setfirstwritablefile(filename,...) - local wr=getwritablepath(...) - local fullname=file.join(wr,filename) - return fullname,wr -end -function caches.define(category,subcategory) - return function() - return getwritablepath(category,subcategory) - end -end -function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) -end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +caches.setfirstwritablefile=setfirstwritablefile +caches.getfirstreadablefile=getfirstreadablefile +caches.setluanames=setluanames function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } @@ -21871,16 +21883,16 @@ function caches.loaddata(readables,name,writable) local path=readables[i] local loader=false local state=false - local tmaname,tmcname=caches.setluanames(path,name) + local tmaname,tmcname=setluanames(path,name) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end if not loader and isfile(tmaname) then - local tmacrap,tmcname=caches.setluanames(writable,name) + local tmacrap,tmcname=setluanames(writable,name) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end @@ -21897,21 +21909,21 @@ function caches.loaddata(readables,name,writable) return false end function caches.is_writable(filepath,filename) - local tmaname,tmcname=caches.setluanames(filepath,filename) + local tmaname,tmcname=setluanames(filepath,filename) return is_writable(tmaname) end local saveoptions={ compact=true } -function caches.savedata(filepath,filename,data) - local tmaname,tmcname=caches.setluanames(filepath,filename) - data.cache_uuid=os.uuid() - if caches.fast then - file.savedata(tmaname,table.fastserialize(data,true)) - elseif caches.direct then - file.savedata(tmaname,table.serialize(data,true,saveoptions)) +function caches.savedata(filepath,filename,data,fast) + local tmaname,tmcname=setluanames(filepath,filename) + data.cache_uuid=osuuid() + if fast or fast_cache then + savedata(tmaname,fastserialize(data,true)) + elseif direct_cache then + savedata(tmaname,serialize(data,true,saveoptions)) else - table.tofile(tmaname,data,true,saveoptions) + serializetofile(tmaname,data,true,saveoptions) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) end local content_state={} function caches.contentstate() @@ -21919,9 +21931,9 @@ function caches.contentstate() end function caches.loadcontent(cachename,dataname,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.luc)) if not blob then @@ -21959,9 +21971,9 @@ function caches.collapsecontent(content) end function caches.savecontent(cachename,dataname,content,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) @@ -21972,17 +21984,17 @@ function caches.savecontent(cachename,dataname,content,filename) type=dataname, root=cachename, version=resolvers.cacheversion, - date=os.date("%Y-%m-%d"), - time=os.date("%H:%M:%S"), + date=osdate("%Y-%m-%d"), + time=osdate("%H:%M:%S"), content=content, - uuid=os.uuid(), + uuid=osuuid(), } - local ok=io.savedata(luaname,table.serialize(data,true)) + local ok=savedata(luaname,serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - if utilities.lua.compile(luaname,lucname) then + if compilelua(luaname,lucname) then if trace_locating then report_resolvers("%a compiled to %a",dataname,lucname) end @@ -21991,7 +22003,7 @@ function caches.savecontent(cachename,dataname,content,filename) if trace_locating then report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) end - os.remove(lucname) + removefile(lucname) end elseif trace_locating then report_resolvers("unable to save %a in %a (access error)",dataname,luaname) @@ -22005,7 +22017,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5310, stripped down to: 3784 +-- original size: 5518, stripped down to: 3854 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -22014,31 +22026,45 @@ if not modules then modules={} end modules ['data-met']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,format=string.find,string.format -local sequenced=table.sequenced +local type=type +local find=string.find local addurlscheme,urlhashed=url.addscheme,url.hashed +local collapsepath,joinfile=file.collapsepath,file.join +local report_methods=logs.reporter("resolvers","methods") local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) trackers.register("resolvers.methods",function(v) trace_methods=v end) -local report_methods=logs.reporter("resolvers","methods") local allocate=utilities.storage.allocate local resolvers=resolvers local registered={} local function splitmethod(filename) if not filename then - return { scheme="unknown",original=filename } + return { + scheme="unknown", + original=filename, + } end if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") + filename=collapsepath(filename,".") if not find(filename,"://",1,true) then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } end - local specification=url.hashed(filename) + local specification=urlhashed(filename) if not specification.scheme or specification.scheme=="" then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } else return specification end @@ -22047,7 +22073,8 @@ resolvers.splitmethod=splitmethod local function methodhandler(what,first,...) local method=registered[what] if method then - local how,namespace=method.how,method.namespace + local how=method.how + local namespace=method.namespace if how=="uri" or how=="url" then local specification=splitmethod(first) local scheme=specification.scheme @@ -22093,7 +22120,10 @@ local function methodhandler(what,first,...) end resolvers.methodhandler=methodhandler function resolvers.registermethod(name,namespace,how) - registered[name]={ how=how or "tag",namespace=namespace } + registered[name]={ + how=how or "tag", + namespace=namespace + } namespace["byscheme"]=function(scheme,filename,...) if scheme=="file" then return methodhandler(name,filename,...) @@ -22102,7 +22132,7 @@ function resolvers.registermethod(name,namespace,how) end end end -local concatinators=allocate { notfound=file.join } +local concatinators=allocate { notfound=joinfile } local locators=allocate { notfound=function() end } local hashers=allocate { notfound=function() end } local generators=allocate { notfound=function() end } @@ -22123,7 +22153,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 68209, stripped down to: 43711 +-- original size: 69574, stripped down to: 44470 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -22134,7 +22164,8 @@ if not modules then modules={} end modules ['data-res']={ } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch local concat,insert,remove=table.concat,table.insert,table.remove -local next,type,rawget=next,type,rawget +local next,type,rawget,loadfile=next,type,rawget,loadfile +local mergedtable=table.merged local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns @@ -22150,6 +22181,7 @@ local joinpath=file.joinpath local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local urlhasscheme=url.hasscheme local getcurrentdir=lfs.currentdir local isfile=lfs.isfile local isdir=lfs.isdir @@ -22175,10 +22207,17 @@ local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os. resolvers.cacheversion="1.100" resolvers.configbanner="" resolvers.homedir=environment.homedir -resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" +local criticalvars={ + "SELFAUTOLOC", + "SELFAUTODIR", + "SELFAUTOPARENT", + "TEXMFCNF", + "TEXMF", + "TEXOS", +} if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf else @@ -22198,13 +22237,20 @@ local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } local instance=nil -function resolvers.setenv(key,value,raw) +local variable +local expansion +local setenv +local getenv +local formatofsuffix=resolvers.formatofsuffix +local splitpath=resolvers.splitpath +local splitmethod=resolvers.splitmethod +setenv=function(key,value,raw) if instance then instance.environment[key]=value ossetenv(key,raw and value or resolveprefix(value)) end end -local function getenv(key) +getenv=function(key) local value=rawget(instance.environment,key) if value and value~="" then return value @@ -22214,94 +22260,97 @@ local function getenv(key) end end resolvers.getenv=getenv -resolvers.env=getenv -local function resolvevariable(k) - return instance.expansions[k] -end +resolvers.setenv=setenv local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local somevariable=P("$")/"" -local somekey=C(R("az","AZ","09","__","--")^1) -local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) -local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" -local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolvevariable -local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) -local variableresolver=Cs((variable+P(1))^0) -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end -function resolvers.reset() - if trace_locating then - report_resolving("creating instance") - end - local environment={} - local variables={} - local expansions={} - local order={} - instance={ - environment=environment, - variables=variables, - expansions=expansions, - order=order, - files={}, - setups={}, - found={}, - foundintrees={}, - hashes={}, - hashed={}, - pathlists=false, - specification={}, - lists={}, - data={}, - fakepaths={}, - remember=true, - diskcache=true, - renewcache=false, - renewtree=false, - loaderror=false, - savelists=true, - pattern=nil, - force_suffixes=true, - pathstack={}, - } - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v=order[i][k] +local expandedvariable,resolvedvariable do + local function resolveinstancevariable(k) + return instance.expansions[k] + end + local p_variable=P("$")/"" + local p_key=C(R("az","AZ","09","__","--")^1) + local p_whatever=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) + local variableexpander=Cs((p_variable*(p_key/resolveinstancevariable)+p_whatever)^1 ) + local p_cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" + local variablecleaner=Cs((p_cleaner+P(1))^0) + local p_variable=R("az","AZ","09","__","--")^1/resolveinstancevariable + local p_variable=(P("$")/"")*(p_variable+(P("{")/"")*p_variable*(P("}")/"")) + local variableresolver=Cs((p_variable+P(1))^0) + expandedvariable=function(var) + return lpegmatch(variableexpander,var) or var + end + function resolvers.reset() + if trace_locating then + report_resolving("creating instance") + end + local environment={} + local variables={} + local expansions={} + local order={} + instance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files={}, + setups={}, + found={}, + foundintrees={}, + hashes={}, + hashed={}, + pathlists=false, + specification={}, + lists={}, + data={}, + fakepaths={}, + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + pathstack={}, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + local repath=resolvers.repath + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end if v~=nil then - t[k]=v - return v + v=checkedvariable(v) or "" end - end - if v==nil then - v="" - end - t[k]=v - return v - end) - setmetatableindex(environment,function(t,k) - local v=osgetenv(k) - if v==nil then - v=variables[k] - end - if v~=nil then - v=checkedvariable(v) or "" - end - v=resolvers.repath(v) - t[k]=v - return v - end) - setmetatableindex(expansions,function(t,k) - local v=environment[k] - if type(v)=="string" then - v=lpegmatch(variableresolver,v) - v=lpegmatch(variablecleaner,v) - end - t[k]=v - return v - end) + v=repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + end end function resolvers.initialized() return instance~=nil @@ -22315,31 +22364,33 @@ local function reset_caches() instance.lists={} instance.pathlists=false end -local slash=P("/") -local pathexpressionpattern=Cs ( - Cc("^")*( - Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +local makepathexpression do + local slash=P("/") + local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) - )^1*Cc("$") -) -local cache={} -local function makepathexpression(str) - if str=="." then - return "^%./$" - else - local c=cache[str] - if not c then - c=lpegmatch(pathexpressionpattern,str) - cache[str]=c + )^1*Cc("$") + ) + local cache={} + makepathexpression=function(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c end - return c end end local function reportcriticalvariables(cnfspec) if trace_locating then - for i=1,#resolvers.criticalvars do - local k=resolvers.criticalvars[i] - local v=resolvers.getenv(k) or "unknown" + for i=1,#criticalvars do + local k=criticalvars[i] + local v=getenv(k) or "unknown" report_resolving("variable %a set to %a",k,v) end report_resolving() @@ -22361,7 +22412,7 @@ local function identify_configuration_files() resolvers.luacnfstate="environment" end reportcriticalvariables(cnfspec) - local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local cnfpaths=expandedpathfromlist(splitpath(cnfspec)) local function locatecnf(luacnfname,kind) for i=1,#cnfpaths do local filepath=cnfpaths[i] @@ -22394,6 +22445,8 @@ local function identify_configuration_files() end local function load_configuration_files() local specification=instance.specification + local setups=instance.setups + local order=instance.order if #specification>0 then local luacnfname=resolvers.luacnfname for i=1,#specification do @@ -22403,7 +22456,6 @@ local function load_configuration_files() local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then - local setups=instance.setups local data=blob() local parent=data and data.parent if parent then @@ -22414,7 +22466,7 @@ local function load_configuration_files() local parentdata=blob() if parentdata then report_resolving("loading configuration file %a",filename) - data=table.merged(parentdata,data) + data=mergedtable(parentdata,data) end end end @@ -22446,7 +22498,7 @@ local function load_configuration_files() if trace_locating then report_resolving("reloading configuration due to TEXMF redefinition") end - resolvers.setenv("TEXMFCNF",cnfspec) + setenv("TEXMFCNF",cnfspec) instance.specification={} identify_configuration_files() load_configuration_files() @@ -22464,7 +22516,7 @@ local function load_configuration_files() elseif trace_locating then report_resolving("skipping configuration file %a (no valid format)",filename) end - instance.order[#instance.order+1]=instance.setups[pathname] + order[#order+1]=setups[pathname] if instance.loaderror then break end @@ -22473,6 +22525,8 @@ local function load_configuration_files() report_resolving("warning: no lua configuration files found") end end +local expandedpathlist +local unexpandedpathlist function resolvers.configurationfiles() return instance.specification or {} end @@ -22489,7 +22543,7 @@ local function load_file_databases() end end local function locate_file_databases() - local texmfpaths=resolvers.expandedpathlist("TEXMF") + local texmfpaths=expandedpathlist("TEXMF") if #texmfpaths>0 then for i=1,#texmfpaths do local path=collapsepath(texmfpaths[i]) @@ -22498,7 +22552,7 @@ local function locate_file_databases() if stripped~="" then local runtime=stripped==path path=cleanpath(path) - local spec=resolvers.splitmethod(stripped) + local spec=splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped elseif spec.scheme=="cache" or spec.scheme=="file" then @@ -22532,11 +22586,13 @@ local function generate_file_databases() end end local function save_file_databases() - for i=1,#instance.hashes do - local hash=instance.hashes[i] + local hashes=instance.hashes + local files=instance.files + for i=1,#hashes do + local hash=hashes[i] local cachename=hash.name if hash.cache then - local content=instance.files[cachename] + local content=files[cachename] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",cachename) @@ -22548,8 +22604,9 @@ local function save_file_databases() end end function resolvers.renew(hashname) + local files=instance.files if hashname and hashname~="" then - local expanded=resolvers.expansion(hashname) or "" + local expanded=expansion(hashname) or "" if expanded~="" then if trace_locating then report_resolving("identifying tree %a from %a",expanded,hashname) @@ -22566,7 +22623,7 @@ function resolvers.renew(hashname) report_resolving("using path %a",realpath) end methodhandler('generators',hashname) - local content=instance.files[hashname] + local content=files[hashname] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",hashname) @@ -22593,38 +22650,46 @@ local function load_databases() end end function resolvers.appendhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a appended",name) end - insert(instance.hashes,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.prependhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a prepended",name) end - insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,1,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.extendtexmfvariable(specification) - local t=resolvers.splitpath(getenv("TEXMF")) - insert(t,1,specification) - local newspec=concat(t,",") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"]=newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"]=newspec + local environment=instance.environment + local variables=instance.variables + local texmftrees=splitpath(getenv("TEXMF")) + insert(texmftrees,1,specification) + texmftrees=concat(texmftrees,",") + if environment["TEXMF"] then + environment["TEXMF"]=texmftrees + elseif variables["TEXMF"] then + variables["TEXMF"]=texmftrees else end reset_hashes() end function resolvers.splitexpansions() - local ie=instance.expansions - for k,v in next,ie do + local expansions=instance.expansions + for k,v in next,expansions do local t,tn,h,p={},0,{},splitconfigurationpath(v) for kk=1,#p do local vv=p[kk] @@ -22634,33 +22699,37 @@ function resolvers.splitexpansions() h[vv]=true end end - if #t>1 then - ie[k]=t + if tn>1 then + expansions[k]=t else - ie[k]=t[1] + expansions[k]=t[1] end end end function resolvers.datastate() return caches.contentstate() end -function resolvers.variable(name) +variable=function(name) + local variables=instance.variables local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.variables[name] + local result=name and variables[name] return result~=nil and result or "" end -function resolvers.expansion(name) +expansion=function(name) + local expansions=instance.expansions local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.expansions[name] + local result=name and expansions[name] return result~=nil and result or "" end -function resolvers.unexpandedpathlist(str) - local pth=resolvers.variable(str) - local lst=resolvers.splitpath(pth) +resolvers.variable=variable +resolvers.expansion=expansion +unexpandedpathlist=function(str) + local pth=variable(str) + local lst=splitpath(pth) return expandedpathfromlist(lst) end function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(unexpandedpathlist(str)) end function resolvers.pushpath(name) local pathstack=instance.pathstack @@ -22690,8 +22759,8 @@ function resolvers.stackpath() end local done={} function resolvers.resetextrapaths() - local ep=instance.extra_paths - if not ep then + local extra_paths=instance.extra_paths + if not extra_paths then done={} instance.extra_paths={} elseif #ep>0 then @@ -22712,8 +22781,8 @@ function resolvers.registerextrapath(paths,subpaths) end local paths=settings_to_array(paths) local subpaths=settings_to_array(subpaths) - local ep=instance.extra_paths or {} - local oldn=#ep + local extra_paths=instance.extra_paths or {} + local oldn=#extra_paths local newn=oldn local nofpaths=#paths local nofsubpaths=#subpaths @@ -22726,7 +22795,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end @@ -22736,7 +22805,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=cleanpath(p) + extra_paths[newn]=cleanpath(p) done[p]=true end end @@ -22745,17 +22814,17 @@ function resolvers.registerextrapath(paths,subpaths) for i=1,oldn do for j=1,nofsubpaths do local s=subpaths[j] - local ps=ep[i].."/"..s + local ps=extra_paths[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end end end if newn>0 then - instance.extra_paths=ep + instance.extra_paths=extra_paths end if newn~=oldn then reset_caches() @@ -22763,17 +22832,19 @@ function resolvers.registerextrapath(paths,subpaths) end function resolvers.pushextrapath(path) local paths=settings_to_array(path) - if instance.extra_stack then - insert(instance.extra_stack,1,paths) + local extra_stack=instance.extra_stack + if extra_stack then + insert(extra_stack,1,paths) else instance.extra_stack={ paths } end reset_caches() end function resolvers.popextrapath() - if instance.extra_stack then + local extra_stack=instance.extra_stack + if extra_stack then reset_caches() - return remove(instance.extra_stack,1) + return remove(extra_stack,1) end end local function made_list(instance,list,extra_too) @@ -22802,33 +22873,21 @@ local function made_list(instance,list,extra_too) end end if extra_too then - local es=instance.extra_stack - if es and #es>0 then - for k=1,#es do - add(es[k]) + local extra_stack=instance.extra_stack + local extra_paths=instance.extra_paths + if extra_stack and #extra_stack>0 then + for k=1,#extra_stack do + add(extra_stack[k]) end end - local ep=instance.extra_paths - if ep and #ep>0 then - add(ep) + if extra_paths and #extra_paths>0 then + add(extra_paths) end end add(list) return new end -function resolvers.cleanpathlist(str) - local t=resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i]=collapsepath(cleanpath(t[i])) - end - end - return t -end -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) -end -function resolvers.expandedpathlist(str,extra_too) +expandedpathlist=function(str,extra_too) if not str then return {} elseif instance.savelists then @@ -22836,40 +22895,56 @@ function resolvers.expandedpathlist(str,extra_too) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + local l=made_list(instance,splitpath(expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else - local lst=resolvers.splitpath(resolvers.expansion(str)) + local lst=splitpath(expansion(str)) return made_list(instance,expandedpathfromlist(lst),extra_too) end end -function resolvers.expandedpathlistfromvariable(str) +resolvers.expandedpathlist=expandedpathlist +resolvers.unexpandedpathlist=unexpandedpathlist +function resolvers.cleanpathlist(str) + local t=expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(cleanpath(t[i])) + end + end + return t +end +function resolvers.expandpath(str) + return joinpath(expandedpathlist(str)) +end +local function expandedpathlistfromvariable(str) str=lpegmatch(dollarstripper,str) local tmp=resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp~="" and tmp or str) + return expandedpathlist(tmp~="" and tmp or str) end function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(expandedpathlistfromvariable(str)) end +resolvers.expandedpathlistfromvariable=expandedpathlistfromvariable function resolvers.cleanedpathlist(v) - local t=resolvers.expandedpathlist(v) + local t=expandedpathlist(v) for i=1,#t do - t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + t[i]=resolveprefix(cleanpath(t[i])) end return t end function resolvers.expandbraces(str) - local pth=expandedpathfromlist(resolvers.splitpath(str)) + local pth=expandedpathfromlist(splitpath(str)) return joinpath(pth) end function resolvers.registerfilehash(name,content,someerror) + local files=instance.files if content then - instance.files[name]=content + files[name]=content else - instance.files[name]={} + files[name]={} if somerror==true then instance.loaderror=someerror end @@ -22926,10 +23001,11 @@ local function collect_files(names) pathname="/"..pathname.."$" end local hashes=instance.hashes + local files=instance.files for h=1,#hashes do local hash=hashes[h] local hashname=hash.name - local content=hashname and instance.files[hashname] + local content=hashname and files[hashname] if content then if trace_details then report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) @@ -22999,7 +23075,6 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if filesuffix=="" or not suffixmap[filesuffix] then local defaultsuffixes=resolvers.defaultsuffixes - local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname @@ -23009,7 +23084,7 @@ local function find_analyze(filename,askedformat,allresults) end end else - filetype=resolvers.formatofsuffix(filename) + filetype=formatofsuffix(filename) if trace_locating then report_resolving("using suffix based filetype %a",filetype) end @@ -23070,11 +23145,11 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] if format_suffixes then for i=1,#format_suffixes do - local s=format_suffixes[i] - forcedname=filename.."."..s + local suffix=format_suffixes[i] + forcedname=filename.."."..suffix if isreadable(forcedname) then if trace_locating then - report_resolving("no suffix, forcing format filetype %a",s) + report_resolving("no suffix, forcing format filetype %a",suffix) end return "qualified",{ forcedname } end @@ -23087,7 +23162,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local savedformat=askedformat local format=savedformat or "" if format=="" then - askedformat=resolvers.formatofsuffix(suffix) + askedformat=formatofsuffix(suffix) end if not format then askedformat="othertextfiles" @@ -23126,7 +23201,7 @@ local function check_subpath(fname) end local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local pathlist=expandedpathlist(typespec,filetype and usertypes[filetype]) local entry={} if pathlist and #pathlist>0 then for k=1,#pathlist do @@ -23137,7 +23212,7 @@ local function makepathlist(list,filetype) local expression=makepathexpression(pathname) local barename=gsub(pathname,"/+$","") barename=resolveprefix(barename) - local scheme=url.hasscheme(barename) + local scheme=urlhasscheme(barename) local schemename=gsub(barename,"%.%*$",'') entry[k]={ path=path, @@ -23342,7 +23417,9 @@ collect_instance_files=function(filename,askedformat,allresults) { find_onpath (filename,filetype,wantedfiles,true) }, { find_otherwise(filename,filetype,wantedfiles,true) }, } - local result,status,done={},{},{} + local result={} + local status={} + local done={} for k,r in next,results do local method,list=r[1],r[2] if method and list then @@ -23413,6 +23490,9 @@ local function findfiles(filename,filetype,allresults) if not filename or filename=="" then return {} end + if allresults==nil then + allresults=true + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -23422,27 +23502,25 @@ local function findfiles(filename,filetype,allresults) end return result or {},status end -function resolvers.findfiles(filename,filetype) - if not filename or filename=="" then - return "" - else - return findfiles(filename,filetype,true) - end -end -function resolvers.findfile(filename,filetype) +local function findfile(filename,filetype) if not filename or filename=="" then return "" else return findfiles(filename,filetype,false)[1] or "" end end +resolvers.findfiles=findfiles +resolvers.findfile=findfile +resolvers.find_file=findfile +resolvers.find_files=findfiles function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) + local hashes=instance.hashes + local files=instance.files local base=filebasename(filename) local result={} - local hashes=instance.hashes local function okay(hash,path,name) local found=methodhandler('concatinators',hash.type,hash.name,path,name) if found and found~="" then @@ -23452,7 +23530,7 @@ local function findgivenfiles(filename,allresults) end for k=1,#hashes do local hash=hashes[k] - local content=instance.files[hash.name] + local content=files[hash.name] if content then local path,name=lookup(content,base) if not path then @@ -23484,14 +23562,14 @@ function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end local function findwildcardfiles(filename,allresults,result) + local files=instance.files + local hashes=instance.hashes local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files=instance.files if find(name,"*",1,true) then - local hashes=instance.hashes local function okay(found,path,base,hashname,hashtype) if find(found,path) then local full=methodhandler('concatinators',hashtype,hashname,found,base) @@ -23531,7 +23609,6 @@ local function findwildcardfiles(filename,allresults,result) end end end - local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] local hashname=hash.name @@ -23561,13 +23638,21 @@ end function resolvers.findwildcardfile(filename) return findwildcardfiles(filename,false)[1] or "" end -function resolvers.automount() -end -function resolvers.starttiming() - statistics.starttiming(instance) +do + local starttiming=statistics.starttiming + local stoptiming=statistics.stoptiming + local elapsedtime=statistics.elapsedtime + function resolvers.starttiming() + starttiming(instance) + end + function resolvers.stoptiming() + stoptiming(instance) + end + function resolvers.loadtime() + return elapsedtime(instance) + end end -function resolvers.stoptiming() - statistics.stoptiming(instance) +function resolvers.automount() end function resolvers.load(option) resolvers.starttiming() @@ -23581,9 +23666,6 @@ function resolvers.load(option) local files=instance.files return files and next(files) and true end -function resolvers.loadtime() - return statistics.elapsedtime(instance) -end local function report(str) if trace_locating then report_resolving(str) @@ -23613,7 +23695,7 @@ function resolvers.dowithfilesandreport(command,files,...) end end function resolvers.showpath(str) - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) + return joinpath(expandedpathlist(resolvers.formatofvariable(str))) end function resolvers.registerfile(files,name,path) if files[name] then @@ -23627,7 +23709,7 @@ function resolvers.registerfile(files,name,path) end end function resolvers.dowithpath(name,func) - local pathlist=resolvers.expandedpathlist(name) + local pathlist=expandedpathlist(name) for i=1,#pathlist do func("^"..cleanpath(pathlist[i])) end @@ -23641,7 +23723,7 @@ function resolvers.locateformat(name) local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then - fmtname=resolvers.findfile(fullname) + fmtname=findfile(fullname) fmtname=cleanpath(fmtname) end if fmtname~="" then @@ -23660,7 +23742,7 @@ function resolvers.locateformat(name) return nil,nil end function resolvers.booleanvariable(str,default) - local b=resolvers.expansion(str) + local b=expansion(str) if b=="" then return default else @@ -23670,6 +23752,7 @@ function resolvers.booleanvariable(str,default) end function resolvers.dowithfilesintree(pattern,handle,before,after) local hashes=instance.hashes + local files=instance.files for i=1,#hashes do local hash=hashes[i] local blobtype=hash.type @@ -23681,7 +23764,7 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) if before then before(blobtype,blobpath,pattern) end - for path,name in filtered(instance.files[blobpath],pattern) do + for path,name in filtered(files[blobpath],pattern) do if type(path)=="string" then checked=checked+1 if handle(blobtype,blobpath,path,name) then @@ -23702,10 +23785,6 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -local obsolete=resolvers.obsolete or {} -resolvers.obsolete=obsolete -resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile -resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles function resolvers.knownvariables(pattern) if instance then local environment=instance.environment @@ -23739,7 +23818,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4854, stripped down to: 2889 +-- original size: 5088, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -23748,6 +23827,7 @@ if not modules then modules={} end modules ['data-pre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local insert,remove=table.insert,table.remove local resolvers=resolvers local prefixes=resolvers.prefixes local cleanpath=resolvers.cleanpath @@ -23825,8 +23905,9 @@ prefixes.kpse=prefixes.locate prefixes.full=prefixes.locate prefixes.file=prefixes.filename prefixes.path=prefixes.pathname +local inputstack={} +local stackpath=resolvers.stackpath local function toppath() - local inputstack=resolvers.inputstack if not inputstack then return "." end @@ -23838,15 +23919,23 @@ local function toppath() end end local function jobpath() - local path=resolvers.stackpath() + local path=stackpath() if not path or path=="" then return "." else return path end end +local function pushinputname(name) + insert(inputstack,name) +end +local function popinputname(name) + return remove(inputstack) +end resolvers.toppath=toppath resolvers.jobpath=jobpath +resolvers.pushinputname=pushinputname +resolvers.popinputname=popinputname prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end resolvers.setdynamic("toppath") @@ -23889,7 +23978,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 470 +-- original size: 551, stripped down to: 470 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -23912,7 +24001,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3170 +-- original size: 4365, stripped down to: 3588 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -23921,39 +24010,46 @@ if not modules then modules={} end modules ['data-fil']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local ioopen=io.open +local isdir=lfs.isdir local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers local resolveprefix=resolvers.resolve -local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers -local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local findfile=resolvers.findfile +local scanfiles=resolvers.scanfiles +local registerfilehash=resolvers.registerfilehash +local appendhash=resolvers.appendhash +local loadcachecontent=caches.loadcontent local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check -function locators.file(specification) +function resolvers.locators.file(specification) local filename=specification.filename local realname=resolveprefix(filename) - if realname and realname~='' and lfs.isdir(realname) then + if realname and realname~='' and isdir(realname) then if trace_locating then report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',filename,true) + appendhash('file',filename,true) elseif trace_locating then report_files("file locator %a not found",filename) end end -function hashers.file(specification) +function resolvers.hashers.file(specification) local pathname=specification.filename - local content=caches.loadcontent(pathname,'files') - resolvers.registerfilehash(pathname,content,content==nil) + local content=loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) end -function generators.file(specification) +function resolvers.generators.file(specification) local pathname=specification.filename - local content=resolvers.scanfiles(pathname,false,true) - resolvers.registerfilehash(pathname,content,true) + local content=scanfiles(pathname,false,true) + registerfilehash(pathname,content,true) end -concatinators.file=file.join +resolvers.concatinators.file=file.join +local finders=resolvers.finders +local notfound=finders.notfound function finders.file(specification,filetype) local filename=specification.filename - local foundname=resolvers.findfile(filename,filetype) + local foundname=findfile(filename,filetype) if foundname and foundname~="" then if trace_locating then report_files("file finder: %a found",filename) @@ -23963,37 +24059,55 @@ function finders.file(specification,filetype) if trace_locating then report_files("file finder: %a not found",filename) end - return finders.notfound() + return notfound() end end -function openers.helpers.textopener(tag,filename,f) +local openers=resolvers.openers +local notfound=openers.notfound +local overloaded=false +local function textopener(tag,filename,f) return { - reader=function() return f:read () end, - close=function() logs.show_close(filename) return f:close() end, + reader=function() return f:read () end, + close=function() return f:close() end, } end +function openers.helpers.textopener(...) + return textopener(...) +end +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded=true + textopener=opener + end +end function openers.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"r") + local f=ioopen(filename,"r") if f then if trace_locating then report_files("file opener: %a opened",filename) end - return openers.helpers.textopener("file",filename,f) + return textopener("file",filename,f) end end if trace_locating then report_files("file opener: %a not found",filename) end - return openers.notfound() + return notfound() end +local loaders=resolvers.loaders +local notfound=loaders.notfound function loaders.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"rb") + local f=ioopen(filename,"rb") if f then - logs.show_load(filename) if trace_locating then report_files("file loader: %a loaded",filename) end @@ -24010,7 +24124,7 @@ function loaders.file(specification,filetype) if trace_locating then report_files("file loader: %a not found",filename) end - return loaders.notfound() + return notfound() end @@ -24020,7 +24134,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5029, stripped down to: 3432 +-- original size: 5388, stripped down to: 3685 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -24029,6 +24143,7 @@ if not modules then modules={} end modules ['data-con']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local setmetatable=setmetatable local format,lower,gsub=string.format,string.lower,string.gsub local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) @@ -24036,16 +24151,21 @@ local trace_storage=false trackers.register("resolvers.storage",function(v) tra containers=containers or {} local containers=containers containers.usecache=true +local getwritablepath=caches.getwritablepath +local getreadablepaths=caches.getreadablepaths +local cacheiswritable=caches.is_writable +local loaddatafromcache=caches.loaddata +local savedataincache=caches.savedata local report_containers=logs.reporter("resolvers","containers") local allocated={} local mt={ __index=function(t,k) if k=="writable" then - local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable=getwritablepath(t.category,t.subcategory) or { "." } t.writable=writable return writable elseif k=="readables" then - local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables=getreadablepaths(t.category,t.subcategory) or { "." } t.readables=readables return readables end @@ -24076,7 +24196,7 @@ function containers.define(category,subcategory,version,enabled) end end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable,name) + return container.enabled and caches and cacheiswritable(container.writable,name) end function containers.is_valid(container,name) if name and name~="" then @@ -24090,7 +24210,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name,container.writable) + stored=loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -24106,17 +24226,20 @@ function containers.read(container,name) end return stored end -function containers.write(container,name,data) +function containers.write(container,name,data,fast) if data then data.cache_version=container.version if container.enabled and caches then - local unique,shared=data.unique,data.shared - data.unique,data.shared=nil,nil - caches.savedata(container.writable,name,data) + local unique=data.unique + local shared=data.shared + data.unique=nil + data.shared=nil + savedataincache(container.writable,name,data,fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique,data.shared=unique,shared + data.unique=unique + data.shared=shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) @@ -24139,7 +24262,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 5203, stripped down to: 3765 +-- original size: 5790, stripped down to: 2910 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -24148,40 +24271,11 @@ if not modules then modules={} end modules ['data-use']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local format=string.format local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_mounts=logs.reporter("resolvers","mounts") local resolvers=resolvers -resolvers.automounted=resolvers.automounted or {} -function resolvers.automount(usecache) - local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths==0) and usecache then - mountpaths=caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths>0 then - resolvers.starttiming() - for k=1,#mountpaths do - local root=mountpaths[k] - local f=io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %a",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() - end - end - resolvers.stoptiming() - end -end +local findfile=resolvers.findfile statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) @@ -24191,7 +24285,7 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local luvdata={ enginebanner=enginebanner, formatbanner=formatbanner, - sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcehash=md5.hex(io.loaddata(findfile(sourcefile)) or "unknown"), sourcefile=sourcefile, luaversion=LUAVERSION, formatid=LUATEXFORMATID, @@ -24215,7 +24309,7 @@ function statistics.checkfmtstatus(texname) if lfs.isfile(luvname) then local luv=dofile(luvname) if luv and luv.sourcefile then - local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local sourcehash=md5.hex(io.loaddata(findfile(luv.sourcefile)) or "unknown") local luvbanner=luv.enginebanner or "?" if luvbanner~=enginebanner then return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) @@ -24256,7 +24350,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 10263, stripped down to: 7556 +-- original size: 10725, stripped down to: 7949 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -24269,6 +24363,14 @@ local format,find,match=string.format,string.find,string.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_zip=logs.reporter("resolvers","zip") local resolvers=resolvers +local findfile=resolvers.findfile +local registerfile=resolvers.registerfile +local splitmethod=resolvers.splitmethod +local prependhash=resolvers.prependhash +local starttiming=resolvers.starttiming +local extendtexmf=resolvers.extendtexmfvariable +local stoptiming=resolvers.stoptiming +local urlquery=url.query zip=zip or {} local zip=zip local archives=zip.archives or {} @@ -24283,8 +24385,9 @@ if zipfiles then closezip=zipfiles.close validfile=zipfiles.found wholefile=zipfiles.unzip + local listzip=zipfiles.list traversezip=function(zfile) - return ipairs(zipfiles.list(zfile)) + return ipairs(listzip(zfile)) end local streams=utilities.streams local openstream=streams.open @@ -24341,28 +24444,30 @@ local function validzip(str) return str end end -function zip.openarchive(name) +local function openarchive(name) if not name or name=="" then return nil else local arch=archives[name] if not arch then - local full=resolvers.findfile(name) or "" + local full=findfile(name) or "" arch=full~="" and openzip(full) or false archives[name]=arch end return arch end end -function zip.closearchive(name) +local function closearchive(name) if not name or (name=="" and archives[name]) then closezip(archives[name]) archives[name]=nil end end +zip.openarchive=openarchive +zip.closearchive=closearchive function resolvers.locators.zip(specification) local archive=specification.filename - local zipfile=archive and archive~="" and zip.openarchive(archive) + local zipfile=archive and archive~="" and openarchive(archive) if trace_locating then if zipfile then report_zip("locator: archive %a found",archive) @@ -24371,13 +24476,6 @@ function resolvers.locators.zip(specification) end end end -function resolvers.hashers.zip(specification) - local archive=specification.filename - if trace_locating then - report_zip("loading file %a",archive) - end - resolvers.usezipfile(specification.original) -end function resolvers.concatinators.zip(zipfile,path,name) if not path or path=="" then return format('%s?name=%s',zipfile,name) @@ -24385,14 +24483,16 @@ function resolvers.concatinators.zip(zipfile,path,name) return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.finders.zip(specification) +local finders=resolvers.finders +local notfound=finders.notfound +function finders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("finder: archive %a found",archive) @@ -24413,16 +24513,19 @@ function resolvers.finders.zip(specification) if trace_locating then report_zip("finder: %a not found",original) end - return resolvers.finders.notfound() + return notfound() end -function resolvers.openers.zip(specification) +local openers=resolvers.openers +local notfound=openers.notfound +local textopener=openers.helpers.textopener +function openers.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("opener; archive %a opened",archive) @@ -24432,7 +24535,7 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: file %a found",queryname) end - return resolvers.openers.helpers.textopener('zip',original,handle) + return textopener('zip',original,handle) elseif trace_locating then report_zip("opener: file %a not found",queryname) end @@ -24444,23 +24547,24 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -function resolvers.loaders.zip(specification) +local loaders=resolvers.loaders +local notfound=loaders.notfound +function loaders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("loader: archive %a opened",archive) end local data=wholefile(zfile,queryname) if data then - logs.show_load(original) if trace_locating then report_zip("loader; file %a loaded",original) end @@ -24476,47 +24580,24 @@ function resolvers.loaders.zip(specification) if trace_locating then report_zip("loader: %a not found",original) end - return resolvers.openers.notfound() -end -function resolvers.usezipfile(archive) - local specification=resolvers.splitmethod(archive) - local archive=specification.filename - if archive and not registeredfiles[archive] then - local z=zip.openarchive(archive) - if z then - local tree=url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering: archive %a",archive) - end - resolvers.starttiming() - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) - registeredfiles[archive]=z - resolvers.registerfilehash(archive,resolvers.registerzipfile(z,tree)) - resolvers.stoptiming() - elseif trace_locating then - report_zip("registering: unknown archive %a",archive) - end - elseif trace_locating then - report_zip("registering: archive %a not found",archive) - end + return notfound() end -function resolvers.registerzipfile(z,tree) +local function registerzipfile(z,tree) local names={} local files={} local remap={} local n=0 local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) - local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end + starttiming() for i in traversezip(z) do local filename=i.filename local path,name=match(filename,filter) if not path then n=n+1 - register(names,filename,"") + registerfile(names,filename,"") local usedname=lower(filename) files[usedname]="" if usedname~=filename then @@ -24533,12 +24614,43 @@ function resolvers.registerzipfile(z,tree) else end end + stoptiming() report_zip("registering: %s files registered",n) return { files=files, remap=remap, } end +local function usezipfile(archive) + local specification=splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=openarchive(archive) + if z then + local tree=urlquery(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + prependhash('zip',archive) + extendtexmf(archive) + registeredfiles[archive]=z + registerfilehash(archive,registerzipfile(z,tree)) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +resolvers.usezipfile=usezipfile +resolvers.registerzipfile=registerzipfile +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + usezipfile(specification.original) +end end -- of closure @@ -24547,7 +24659,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8478, stripped down to: 5223 +-- original size: 9444, stripped down to: 5650 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -24563,173 +24675,204 @@ local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local resolveprefix=resolvers.resolve -local notfound=resolvers.finders.notfound -local lookup=resolvers.get_from_content -local collectors={} -local found={} -function resolvers.finders.tree(specification) - local spec=specification.filename - local okay=found[spec] - if okay==nil then - if spec~="" then - local path=dirname(spec) - local name=basename(spec) - if path=="" then - path="." - end - local names=collectors[path] - if not names then - local pattern=find(path,"/%*+$") and path or (path.."/*") - names=globdir(pattern) - collectors[path]=names - end - local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for i=1,#names do - local fullname=names[i] - if find(fullname,pattern) then - found[spec]=fullname - return fullname - end - end - local pattern=lower(pattern) - for i=1,#names do - local fullname=lower(names[i]) - if find(fullname,pattern) then - if isfile(fullname) then +local finders=resolvers.finders +local openers=resolvers.openers +local loaders=resolvers.loaders +local locators=resolvers.locators +local hashers=resolvers.hashers +local generators=resolvers.generators +do + local collectors={} + local found={} + local notfound=finders.notfound + function finders.tree(specification) + local spec=specification.filename + local okay=found[spec] + if okay==nil then + if spec~="" then + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then found[spec]=fullname return fullname - else - break + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end + okay=notfound() + found[spec]=okay end - okay=notfound() - found[spec]=okay + return okay end - return okay end -function resolvers.locators.tree(specification) - local name=specification.filename - local realname=resolveprefix(name) - if realname and realname~='' and isdir(realname) then - if trace_locating then - report_trees("locator %a found",realname) +do + local resolveprefix=resolvers.resolve + local appendhash=resolvers.appendhash + local function dolocate(specification) + local name=specification.filename + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) end - resolvers.appendhash('tree',name,false) - elseif trace_locating then - report_trees("locator %a not found",name) end + locators.tree=dolocate + locators.dirlist=dolocate + locators.dirfile=dolocate end -function resolvers.hashers.tree(specification) - local name=specification.filename - if trace_locating then - report_trees("analyzing %a",name) - end - resolvers.methodhandler("hashers",name) - resolvers.generators.file(specification) -end -local collectors={} -local splitter=lpeg.splitat("/**/") -local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } -table.setmetatableindex(collectors,function(t,k) - local rootname=lpegmatch(stripper,k) - local dataname=joinname(rootname,"dirlist") - local content=caches.loadcontent(dataname,"files",dataname) - if not content then - content=resolvers.scanfiles(rootname,nil,nil,false,true) - caches.savecontent(dataname,"files",content,dataname) - end - t[k]=content - return content -end) -local function checked(root,p,n) - if p then - if type(p)=="table" then - for i=1,#p do - local fullname=joinname(root,p[i],n) +do + local filegenerator=generators.file + generators.dirlist=filegenerator + generators.dirfile=filegenerator +end +do + local filegenerator=generators.file + local methodhandler=resolvers.methodhandler + local function dohash(specification) + local name=specification.filename + if trace_locating then + report_trees("analyzing %a",name) + end + methodhandler("hashers",name) + filegenerator(specification) + end + hashers.tree=dohash + hashers.dirlist=dohash + hashers.dirfile=dohash +end +local resolve do + local collectors={} + local splitter=lpeg.splitat("/**/") + local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } + local loadcontent=caches.loadcontent + local savecontent=caches.savecontent + local notfound=finders.notfound + local scanfiles=resolvers.scanfiles + local lookup=resolvers.get_from_content + table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=loadcontent(dataname,"files",dataname) + if not content then + content=scanfiles(rootname,nil,nil,false,true) + savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content + end) + local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) if isfile(fullname) then return fullname end end - else - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname - end end + return notfound() end - return notfound() -end -local function resolve(specification) - local filename=specification.filename - if filename~="" then - local root,rest=lpegmatch(splitter,filename) - if root and rest then - local path,name=dirname(rest),basename(rest) - if name~=rest then - local content=collectors[root] - local p,n=lookup(content,name) - if not p then - return notfound() - end - local pattern=".*/"..path.."$" - local istable=type(p)=="table" - if istable then - for i=1,#p do - local pi=p[i] - if pi==path or find(pi,pattern) then - local fullname=joinname(root,pi,n) - if isfile(fullname) then - return fullname + resolve=function(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end end end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end end - elseif p==path or find(p,pattern) then - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() end end - local queries=specification.queries - if queries and queries.option=="fileonly" then - return checked(root,p,n) - else - return notfound() - end + end + local path=dirname(filename) + local name=basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) end end - local path,name=dirname(filename),basename(filename) - local root=lpegmatch(stripper,path) - local content=collectors[path] - local p,n=lookup(content,name) - if p then - return checked(root,p,n) + return notfound() + end + finders.dirlist=resolve + function finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } end + return resolve(specification) end - return notfound() end -resolvers.finders .dirlist=resolve -resolvers.locators .dirlist=resolvers.locators .tree -resolvers.hashers .dirlist=resolvers.hashers .tree -resolvers.generators.dirlist=resolvers.generators.file -resolvers.openers .dirlist=resolvers.openers .file -resolvers.loaders .dirlist=resolvers.loaders .file -function resolvers.finders.dirfile(specification) - local queries=specification.queries - if queries then - queries.option="fileonly" - else - specification.queries={ option="fileonly" } - end - return resolve(specification) +do + local fileopener=openers.file + local fileloader=loaders.file + openers.dirlist=fileopener + loaders.dirlist=fileloader + openers.dirfile=fileopener + loaders.dirfile=fileloader end -resolvers.locators .dirfile=resolvers.locators .dirlist -resolvers.hashers .dirfile=resolvers.hashers .dirlist -resolvers.generators.dirfile=resolvers.generators.dirlist -resolvers.openers .dirfile=resolvers.openers .dirlist -resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -24738,7 +24881,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6757, stripped down to: 5272 +-- original size: 6945, stripped down to: 5408 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -24748,8 +24891,11 @@ if not modules then modules={} end modules ['data-sch']={ license="see context related readme files" } local load,tonumber=load,tonumber -local gsub,concat,format=string.gsub,table.concat,string.format +local gsub,format=string.gsub,string.format +local sortedhash,concat=table.sortedhash,table.concat local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local addsuffix,suffix,splitbase=file.addsuffix,file.suffix,file.splitbase +local md5hex=md5.hex local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) local report_schemes=logs.reporter("resolvers","schemes") local http=require("socket.http") @@ -24766,7 +24912,7 @@ function cleaners.none(specification) return specification.original end function cleaners.strip(specification) - local path,name=file.splitbase(specification.original) + local path,name=splitbase(specification.original) if path=="" then return (gsub(name,"[^%a%d%.]+","-")) else @@ -24774,7 +24920,7 @@ function cleaners.strip(specification) end end function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return addsuffix(md5hex(specification.original),suffix(specification.path)) end local cleaner=cleaners.strip directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) @@ -24865,10 +25011,10 @@ end schemes.install=install local function http_handler(specification,cachename) local tempname=cachename..".tmp" - local f=io.open(tempname,"wb") + local handle=io.open(tempname,"wb") local status,message=http.request { url=specification.original, - sink=ltn12.sink.file(f) + sink=ltn12.sink.file(handle) } if not status then os.remove(tempname) @@ -24883,13 +25029,13 @@ install('https') install('ftp') statistics.register("scheme handling time",function() local l,r,nl,nr={},{},0,0 - for k,v in table.sortedhash(loaded) do + for k,v in sortedhash(loaded) do if v>0 then nl=nl+1 l[nl]=k..":"..v end end - for k,v in table.sortedhash(reused) do + for k,v in sortedhash(reused) do if v>0 then nr=nr+1 r[nr]=k..":"..v @@ -24897,10 +25043,10 @@ statistics.register("scheme handling time",function() end local n=nl+nr if n>0 then - l=nl>0 and concat(l) or "none" - r=nr>0 and concat(r) or "none" + if nl==0 then l={ "none" } end + if nr==0 then r={ "none" } end return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes),n,threshold,l,r) + statistics.elapsedtime(schemes),n,threshold,concat(l," "),concat(l," ")) else return nil end @@ -24933,7 +25079,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4207, stripped down to: 3041 +-- original size: 4227, stripped down to: 3049 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -24943,8 +25089,7 @@ if not modules then modules={} end modules ['data-lua']={ license="see context related readme files" } local package,lpeg=package,lpeg -local gsub=string.gsub -local concat=table.concat +local loadfile=loadfile local addsuffix=file.addsuffix local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match local luasuffixes={ 'tex','lua' } @@ -24955,6 +25100,8 @@ local helpers=package.helpers or {} local methods=helpers.methods or {} local resolvers=resolvers local resolveprefix=resolvers.resolve +local expandedpaths=resolvers.expandedpathlistfromvariable +local findfile=resolvers.findfile helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) @@ -24983,7 +25130,7 @@ local function getluaformatpaths() if not luaformatpaths then luaformatpaths={} for i=1,#luaformats do - registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + registerpath("lua format","lua",luaformatpaths,expandedpaths(luaformats[i])) end end return luaformatpaths @@ -24992,7 +25139,7 @@ local function getlibformatpaths() if not libformatpaths then libformatpaths={} for i=1,#libformats do - registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + registerpath("lib format","lib",libformatpaths,expandedpaths(libformats[i])) end end return libformatpaths @@ -25002,7 +25149,7 @@ local function loadedbyformat(name,rawname,suffixes,islib,what) local report=helpers.report for i=1,#suffixes do local format=suffixes[i] - local resolved=resolvers.findfile(name,format) or "" + local resolved=findfile(name,format) or "" if trace then report("%s format, identifying %a using format %a",what,name,format) end @@ -25040,7 +25187,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2452, stripped down to: 1877 +-- original size: 2610, stripped down to: 2019 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -25051,24 +25198,28 @@ if not modules then modules={} end modules ['data-aux']={ } local find=string.find local type,next=type,next +local addsuffix,removesuffix=file.addsuffix,file.removesuffix +local loaddata,savedata=io.loaddata,io.savedata local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers +local cleanpath=resolvers.cleanpath +local findfiles=resolvers.findfiles local report_scripts=logs.reporter("resolvers","scripts") function resolvers.updatescript(oldname,newname) local scriptpath="context/lua" - newname=file.addsuffix(newname,"lua") - local oldscript=resolvers.cleanpath(oldname) + local oldscript=cleanpath(oldname) + local newname=addsuffix(newname,"lua") + local newscripts=findfiles(newname) or {} if trace_locating then report_scripts("to be replaced old script %a",oldscript) end - local newscripts=resolvers.findfiles(newname) or {} if #newscripts==0 then if trace_locating then report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript=resolvers.cleanpath(newscripts[i]) + local newscript=cleanpath(newscripts[i]) if trace_locating then report_scripts("checking new script %a",newscript) end @@ -25080,17 +25231,17 @@ function resolvers.updatescript(oldname,newname) if trace_locating then report_scripts("new script should come from %a",scriptpath) end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + elseif not (find(oldscript,removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then report_scripts("invalid new script name") end else - local newdata=io.loaddata(newscript) + local newdata=loaddata(newscript) if newdata then if trace_locating then report_scripts("old script content replaced by new content: %s",oldscript) end - io.savedata(oldscript,newdata) + savedata(oldscript,newdata) break elseif trace_locating then report_scripts("unable to load new script") @@ -25163,7 +25314,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 1823, stripped down to: 1542 +-- original size: 1912, stripped down to: 1591 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -25173,40 +25324,39 @@ if not modules then modules={} end modules ['data-lst']={ license="see context related readme files" } local type=type -local concat,sortedhash=table.concat,table.sortedhash +local sortedhash=table.sortedhash +local isdir=lfs.isdir local resolvers=resolvers local listers=resolvers.listers or {} resolvers.listers=listers local resolveprefix=resolvers.resolve +local configurationfiles=resolvers.configurationfiles +local expandedpathfromlist=resolvers.expandedpathfromlist +local splitpath=resolvers.splitpath +local knownvariables=resolvers.knownvariables local report_lists=logs.reporter("resolvers","lists") local report_resolved=logs.reporter("system","resolved") -local function tabstr(str) - if type(str)=='table' then - return concat(str," | ") - else - return str - end -end function listers.variables(pattern) local result=resolvers.knownvariables(pattern) + local unset={ "unset" } for key,value in sortedhash(result) do report_lists(key) - report_lists(" env: %s",tabstr(value.environment or "unset")) - report_lists(" var: %s",tabstr(value.variable or "unset")) - report_lists(" exp: %s",tabstr(value.expansion or "unset")) - report_lists(" res: %s",tabstr(value.resolved or "unset")) + report_lists(" env: % | t",value.environment or unset) + report_lists(" var: % | t",value.variable or unset) + report_lists(" exp: % | t",value.expansion or unset) + report_lists(" res: % | t",value.resolved or unset) end end function listers.configurations() - local configurations=resolvers.configurationfiles() + local configurations=configurationfiles() for i=1,#configurations do report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") - local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + local list=expandedpathfromlist(splitpath(resolvers.luacnfspec)) for i=1,#list do local li=resolveprefix(list[i]) - if lfs.isdir(li) then + if isdir(li) then report_resolved("path - %s",li) else report_resolved("path + %s",li) @@ -25978,8 +26128,8 @@ end -- of closure -- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 1036669 --- stripped bytes : 410250 +-- original bytes : 1040817 +-- stripped bytes : 412122 -- end library merge diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua index 36a7bbcd2..bf87526f2 100644 --- a/scripts/context/stubs/mswin/mtxrun.lua +++ b/scripts/context/stubs/mswin/mtxrun.lua @@ -324,7 +324,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-macro"] = package.loaded["l-macro"] or true --- original size: 10131, stripped down to: 5991 +-- original size: 10130, stripped down to: 5990 if not modules then modules={} end modules ['l-macros']={ version=1.001, @@ -549,7 +549,7 @@ end macros.loaded=loaded function required(name,trace) local filename=file.addsuffix(name,"lua") - local fullname=resolvers and resolvers.find_file(filename) or filename + local fullname=resolvers and resolvers.findfile(filename) or filename if not fullname or fullname=="" then return false end @@ -12608,7 +12608,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 13348, stripped down to: 8838 +-- original size: 13394, stripped down to: 8882 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -12618,8 +12618,9 @@ if not modules then modules={} end modules ['trac-set']={ license="see context related readme files" } local type,next,tostring,tonumber=type,next,tostring,tonumber +local print=print local concat,sortedhash=table.concat,table.sortedhash -local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local formatters,find,lower,gsub,topattern=string.formatters,string.find,string.lower,string.gsub,string.topattern local is_boolean=string.is_boolean local settings_to_hash=utilities.parsers.settings_to_hash local allocate=utilities.storage.allocate @@ -12826,8 +12827,8 @@ function setters.show(t) end end local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show -function setters.report(setter,...) - print(format("%-15s : %s\n",setter.name,format(...))) +function setters.report(setter,fmt,...) + print(formatters["%-15s : %s\n"](setter.name,formatters[fmt](...))) end local function default(setter,name) local d=setter.data[name] @@ -12950,7 +12951,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 33803, stripped down to: 21818 +-- original size: 32997, stripped down to: 21715 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -13613,12 +13614,6 @@ end local nesting=0 local verbose=false local hasscheme=url.hasscheme -function logs.show_open(name) -end -function logs.show_close(name) -end -function logs.show_load(name) -end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple @@ -20686,7 +20681,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 10987, stripped down to: 7056 +-- original size: 11019, stripped down to: 7086 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -20699,6 +20694,7 @@ local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local sortedpairs=table.sortedpairs local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) @@ -20909,7 +20905,7 @@ if ostype=="unix" then rawset(t,k,v) end local colon=P(":") - for k,v in table.sortedpairs(prefixes) do + for k,v in sortedpairs(prefixes) do if p then p=P(k)+p else @@ -20936,7 +20932,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18154, stripped down to: 10416 +-- original size: 18173, stripped down to: 10432 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -21220,7 +21216,7 @@ local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) scancache[sub(full,1,-2)]=files return files,remap,n,m,r end -function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) +local function scanfiles(path,branch,usecache,onlyonce,tolerant) local realpath=resolveprefix(path) if usecache then local content=fullcache[realpath] @@ -21275,8 +21271,9 @@ function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) statistics.stoptiming(timer) return content end +resolvers.scanfiles=scanfiles function resolvers.simplescanfiles(path,branch,usecache) - return resolvers.scanfiles(path,branch,usecache,true,true) + return scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -21343,7 +21340,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9384, stripped down to: 6333 +-- original size: 9400, stripped down to: 6347 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -21353,7 +21350,7 @@ if not modules then modules={} end modules ['data-env']={ license="see context related readme files", } local lower,gsub=string.lower,string.gsub -local next=next +local next,rawget=next,rawget local resolvers=resolvers local allocate=utilities.storage.allocate local setmetatableindex=table.setmetatableindex @@ -21628,7 +21625,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16387, stripped down to: 11132 +-- original size: 16070, stripped down to: 11353 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -21637,49 +21634,83 @@ if not modules then modules={} end modules ['data-tmp']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat=table.concat -local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile -local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable -local formatters=string.formatters local next,type=next,type +local pcall,loadfile,collectgarbage=pcall,loadfile,collectgarbage +local format,lower,gsub=string.format,string.lower,string.gsub +local concat,serialize,fastserialize,serializetofile=table.concat,table.serialize,table.fastserialize,table.tofile +local mkdirs,expanddirname,isdir,isfile=dir.mkdirs,dir.expandname,lfs.isdir,lfs.isfile +local is_writable,is_readable=file.is_writable,file.is_readable +local collapsepath,joinfile,addsuffix,dirname=file.collapsepath,file.join,file.addsuffix,file.dirname +local savedata=file.savedata +local formatters=string.formatters +local osexit,osdate,osuuid=os.exit,os.date,os.uuid +local removefile=os.remove +local md5hex=md5.hex local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers local cleanpath=resolvers.cleanpath -local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) -local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) -local compile=utilities.lua.compile -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup==nil then cleanup=directive_cleanup end - if strip==nil then strip=directive_strip end - return compile(luafile,lucfile,cleanup,strip) +local resolvepath=resolvers.resolve +local luautilities=utilities.lua +do + local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) + local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) + local compilelua=luautilities.compile + function luautilities.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compilelua(luafile,lucfile,cleanup,strip) + end end caches=caches or {} local caches=caches -local luasuffixes=utilities.lua.suffixes -caches.base=caches.base or "luatex-cache" -caches.more=caches.more or "context" -caches.direct=false -caches.tree=false -caches.force=true -caches.ask=false -caches.relocate=false +local writable=nil +local readables={} +local usedreadables={} +local compilelua=luautilities.compile +local luasuffixes=luautilities.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } -directives.register("system.caches.fast",function(v) caches.fast=true end) -directives.register("system.caches.direct",function(v) caches.direct=true end) -local writable,readables,usedreadables=nil,{},{} +local direct_cache=false +local fast_cache=false +local cache_tree=false +directives.register("system.caches.direct",function(v) direct_cache=true end) +directives.register("system.caches.fast",function(v) fast_cache=true end) +local function configfiles() + return concat(resolvers.configurationfiles(),";") +end +local function hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +local function treehash() + local tree=configfiles() + if not tree or tree=="" then + return false + else + return hashed(tree) + end +end +caches.hashed=hashed +caches.treehash=treehash +caches.configfiles=configfiles local function identify() local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) - cachepath=file.collapsepath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) + cachepath=collapsepath(cachepath) local valid=isdir(cachepath) if valid then if is_readable(cachepath) then @@ -21688,16 +21719,14 @@ local function identify() writable=cachepath end end - elseif not writable and caches.force then - local cacheparent=file.dirname(cachepath) - if is_writable(cacheparent) and true then - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("path %a created",cachepath) - writable=cachepath - readables[#readables+1]=cachepath - end + elseif not writable then + local cacheparent=dirname(cachepath) + if is_writable(cacheparent) then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath end end end @@ -21710,8 +21739,8 @@ local function identify() local cachepath=texmfcaches[k] cachepath=resolvers.expansion(cachepath) if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) local valid=isdir(cachepath) if valid and is_readable(cachepath) then if not writable and is_writable(cachepath) then @@ -21725,23 +21754,25 @@ local function identify() end if not writable then report_caches("fatal error: there is no valid writable cache path defined") - os.exit() + osexit() elseif #readables==0 then report_caches("fatal error: there is no valid readable cache path defined") - os.exit() + osexit() end - writable=dir.expandname(resolvers.cleanpath(writable)) - local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + writable=expanddirname(cleanpath(writable)) + local base=caches.base + local more=caches.more + local tree=cache_tree or treehash() if tree then - caches.tree=tree + cache_tree=tree writable=mkdirs(writable,base,more,tree) for i=1,#readables do - readables[i]=file.join(readables[i],base,more,tree) + readables[i]=joinfile(readables[i],base,more,tree) end else writable=mkdirs(writable,base,more) for i=1,#readables do - readables[i]=file.join(readables[i],base,more) + readables[i]=joinfile(readables[i],base,more) end end if trace_cache then @@ -21778,27 +21809,8 @@ function caches.usedpaths(separator) return writable or "?" end end -function caches.configfiles() - return concat(resolvers.configurationfiles(),";") -end -function caches.hashed(tree) - tree=gsub(tree,"[\\/]+$","") - tree=lower(tree) - local hash=md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %a, hash %a",tree,hash) - end - return hash -end -function caches.treehash() - local tree=caches.configfiles() - if not tree or tree=="" then - return false - else - return caches.hashed(tree) - end -end -local r_cache,w_cache={},{} +local r_cache={} +local w_cache={} local function getreadablepaths(...) local tags={... } local hash=concat(tags,"/") @@ -21808,7 +21820,7 @@ local function getreadablepaths(...) if #tags>0 then done={} for i=1,#readables do - done[i]=file.join(readables[i],...) + done[i]=joinfile(readables[i],...) end else done=readables @@ -21832,17 +21844,25 @@ local function getwritablepath(...) end return done end -caches.getreadablepaths=getreadablepaths -caches.getwritablepath=getwritablepath -function caches.getfirstreadablefile(filename,...) - local fullname,path=caches.setfirstwritablefile(filename,...) +local function setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=joinfile(wr,filename) + return fullname,wr +end +local function setluanames(path,name) + return + format("%s/%s.%s",path,name,luasuffixes.tma), + format("%s/%s.%s",path,name,luasuffixes.tmc) +end +local function getfirstreadablefile(filename,...) + local fullname,path=setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path end local rd=getreadablepaths(...) for i=1,#rd do local path=rd[i] - local fullname=file.join(path,filename) + local fullname=joinfile(path,filename) if is_readable(fullname) then usedreadables[i]=true return fullname,path @@ -21850,19 +21870,11 @@ function caches.getfirstreadablefile(filename,...) end return fullname,path end -function caches.setfirstwritablefile(filename,...) - local wr=getwritablepath(...) - local fullname=file.join(wr,filename) - return fullname,wr -end -function caches.define(category,subcategory) - return function() - return getwritablepath(category,subcategory) - end -end -function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) -end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +caches.setfirstwritablefile=setfirstwritablefile +caches.getfirstreadablefile=getfirstreadablefile +caches.setluanames=setluanames function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } @@ -21871,16 +21883,16 @@ function caches.loaddata(readables,name,writable) local path=readables[i] local loader=false local state=false - local tmaname,tmcname=caches.setluanames(path,name) + local tmaname,tmcname=setluanames(path,name) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end if not loader and isfile(tmaname) then - local tmacrap,tmcname=caches.setluanames(writable,name) + local tmacrap,tmcname=setluanames(writable,name) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end @@ -21897,21 +21909,21 @@ function caches.loaddata(readables,name,writable) return false end function caches.is_writable(filepath,filename) - local tmaname,tmcname=caches.setluanames(filepath,filename) + local tmaname,tmcname=setluanames(filepath,filename) return is_writable(tmaname) end local saveoptions={ compact=true } -function caches.savedata(filepath,filename,data) - local tmaname,tmcname=caches.setluanames(filepath,filename) - data.cache_uuid=os.uuid() - if caches.fast then - file.savedata(tmaname,table.fastserialize(data,true)) - elseif caches.direct then - file.savedata(tmaname,table.serialize(data,true,saveoptions)) +function caches.savedata(filepath,filename,data,fast) + local tmaname,tmcname=setluanames(filepath,filename) + data.cache_uuid=osuuid() + if fast or fast_cache then + savedata(tmaname,fastserialize(data,true)) + elseif direct_cache then + savedata(tmaname,serialize(data,true,saveoptions)) else - table.tofile(tmaname,data,true,saveoptions) + serializetofile(tmaname,data,true,saveoptions) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) end local content_state={} function caches.contentstate() @@ -21919,9 +21931,9 @@ function caches.contentstate() end function caches.loadcontent(cachename,dataname,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.luc)) if not blob then @@ -21959,9 +21971,9 @@ function caches.collapsecontent(content) end function caches.savecontent(cachename,dataname,content,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) @@ -21972,17 +21984,17 @@ function caches.savecontent(cachename,dataname,content,filename) type=dataname, root=cachename, version=resolvers.cacheversion, - date=os.date("%Y-%m-%d"), - time=os.date("%H:%M:%S"), + date=osdate("%Y-%m-%d"), + time=osdate("%H:%M:%S"), content=content, - uuid=os.uuid(), + uuid=osuuid(), } - local ok=io.savedata(luaname,table.serialize(data,true)) + local ok=savedata(luaname,serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - if utilities.lua.compile(luaname,lucname) then + if compilelua(luaname,lucname) then if trace_locating then report_resolvers("%a compiled to %a",dataname,lucname) end @@ -21991,7 +22003,7 @@ function caches.savecontent(cachename,dataname,content,filename) if trace_locating then report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) end - os.remove(lucname) + removefile(lucname) end elseif trace_locating then report_resolvers("unable to save %a in %a (access error)",dataname,luaname) @@ -22005,7 +22017,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5310, stripped down to: 3784 +-- original size: 5518, stripped down to: 3854 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -22014,31 +22026,45 @@ if not modules then modules={} end modules ['data-met']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,format=string.find,string.format -local sequenced=table.sequenced +local type=type +local find=string.find local addurlscheme,urlhashed=url.addscheme,url.hashed +local collapsepath,joinfile=file.collapsepath,file.join +local report_methods=logs.reporter("resolvers","methods") local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) trackers.register("resolvers.methods",function(v) trace_methods=v end) -local report_methods=logs.reporter("resolvers","methods") local allocate=utilities.storage.allocate local resolvers=resolvers local registered={} local function splitmethod(filename) if not filename then - return { scheme="unknown",original=filename } + return { + scheme="unknown", + original=filename, + } end if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") + filename=collapsepath(filename,".") if not find(filename,"://",1,true) then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } end - local specification=url.hashed(filename) + local specification=urlhashed(filename) if not specification.scheme or specification.scheme=="" then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } else return specification end @@ -22047,7 +22073,8 @@ resolvers.splitmethod=splitmethod local function methodhandler(what,first,...) local method=registered[what] if method then - local how,namespace=method.how,method.namespace + local how=method.how + local namespace=method.namespace if how=="uri" or how=="url" then local specification=splitmethod(first) local scheme=specification.scheme @@ -22093,7 +22120,10 @@ local function methodhandler(what,first,...) end resolvers.methodhandler=methodhandler function resolvers.registermethod(name,namespace,how) - registered[name]={ how=how or "tag",namespace=namespace } + registered[name]={ + how=how or "tag", + namespace=namespace + } namespace["byscheme"]=function(scheme,filename,...) if scheme=="file" then return methodhandler(name,filename,...) @@ -22102,7 +22132,7 @@ function resolvers.registermethod(name,namespace,how) end end end -local concatinators=allocate { notfound=file.join } +local concatinators=allocate { notfound=joinfile } local locators=allocate { notfound=function() end } local hashers=allocate { notfound=function() end } local generators=allocate { notfound=function() end } @@ -22123,7 +22153,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 68209, stripped down to: 43711 +-- original size: 69574, stripped down to: 44470 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -22134,7 +22164,8 @@ if not modules then modules={} end modules ['data-res']={ } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch local concat,insert,remove=table.concat,table.insert,table.remove -local next,type,rawget=next,type,rawget +local next,type,rawget,loadfile=next,type,rawget,loadfile +local mergedtable=table.merged local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns @@ -22150,6 +22181,7 @@ local joinpath=file.joinpath local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local urlhasscheme=url.hasscheme local getcurrentdir=lfs.currentdir local isfile=lfs.isfile local isdir=lfs.isdir @@ -22175,10 +22207,17 @@ local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os. resolvers.cacheversion="1.100" resolvers.configbanner="" resolvers.homedir=environment.homedir -resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" +local criticalvars={ + "SELFAUTOLOC", + "SELFAUTODIR", + "SELFAUTOPARENT", + "TEXMFCNF", + "TEXMF", + "TEXOS", +} if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf else @@ -22198,13 +22237,20 @@ local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } local instance=nil -function resolvers.setenv(key,value,raw) +local variable +local expansion +local setenv +local getenv +local formatofsuffix=resolvers.formatofsuffix +local splitpath=resolvers.splitpath +local splitmethod=resolvers.splitmethod +setenv=function(key,value,raw) if instance then instance.environment[key]=value ossetenv(key,raw and value or resolveprefix(value)) end end -local function getenv(key) +getenv=function(key) local value=rawget(instance.environment,key) if value and value~="" then return value @@ -22214,94 +22260,97 @@ local function getenv(key) end end resolvers.getenv=getenv -resolvers.env=getenv -local function resolvevariable(k) - return instance.expansions[k] -end +resolvers.setenv=setenv local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local somevariable=P("$")/"" -local somekey=C(R("az","AZ","09","__","--")^1) -local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) -local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" -local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolvevariable -local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) -local variableresolver=Cs((variable+P(1))^0) -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end -function resolvers.reset() - if trace_locating then - report_resolving("creating instance") - end - local environment={} - local variables={} - local expansions={} - local order={} - instance={ - environment=environment, - variables=variables, - expansions=expansions, - order=order, - files={}, - setups={}, - found={}, - foundintrees={}, - hashes={}, - hashed={}, - pathlists=false, - specification={}, - lists={}, - data={}, - fakepaths={}, - remember=true, - diskcache=true, - renewcache=false, - renewtree=false, - loaderror=false, - savelists=true, - pattern=nil, - force_suffixes=true, - pathstack={}, - } - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v=order[i][k] +local expandedvariable,resolvedvariable do + local function resolveinstancevariable(k) + return instance.expansions[k] + end + local p_variable=P("$")/"" + local p_key=C(R("az","AZ","09","__","--")^1) + local p_whatever=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) + local variableexpander=Cs((p_variable*(p_key/resolveinstancevariable)+p_whatever)^1 ) + local p_cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" + local variablecleaner=Cs((p_cleaner+P(1))^0) + local p_variable=R("az","AZ","09","__","--")^1/resolveinstancevariable + local p_variable=(P("$")/"")*(p_variable+(P("{")/"")*p_variable*(P("}")/"")) + local variableresolver=Cs((p_variable+P(1))^0) + expandedvariable=function(var) + return lpegmatch(variableexpander,var) or var + end + function resolvers.reset() + if trace_locating then + report_resolving("creating instance") + end + local environment={} + local variables={} + local expansions={} + local order={} + instance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files={}, + setups={}, + found={}, + foundintrees={}, + hashes={}, + hashed={}, + pathlists=false, + specification={}, + lists={}, + data={}, + fakepaths={}, + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + pathstack={}, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + local repath=resolvers.repath + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end if v~=nil then - t[k]=v - return v + v=checkedvariable(v) or "" end - end - if v==nil then - v="" - end - t[k]=v - return v - end) - setmetatableindex(environment,function(t,k) - local v=osgetenv(k) - if v==nil then - v=variables[k] - end - if v~=nil then - v=checkedvariable(v) or "" - end - v=resolvers.repath(v) - t[k]=v - return v - end) - setmetatableindex(expansions,function(t,k) - local v=environment[k] - if type(v)=="string" then - v=lpegmatch(variableresolver,v) - v=lpegmatch(variablecleaner,v) - end - t[k]=v - return v - end) + v=repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + end end function resolvers.initialized() return instance~=nil @@ -22315,31 +22364,33 @@ local function reset_caches() instance.lists={} instance.pathlists=false end -local slash=P("/") -local pathexpressionpattern=Cs ( - Cc("^")*( - Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +local makepathexpression do + local slash=P("/") + local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) - )^1*Cc("$") -) -local cache={} -local function makepathexpression(str) - if str=="." then - return "^%./$" - else - local c=cache[str] - if not c then - c=lpegmatch(pathexpressionpattern,str) - cache[str]=c + )^1*Cc("$") + ) + local cache={} + makepathexpression=function(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c end - return c end end local function reportcriticalvariables(cnfspec) if trace_locating then - for i=1,#resolvers.criticalvars do - local k=resolvers.criticalvars[i] - local v=resolvers.getenv(k) or "unknown" + for i=1,#criticalvars do + local k=criticalvars[i] + local v=getenv(k) or "unknown" report_resolving("variable %a set to %a",k,v) end report_resolving() @@ -22361,7 +22412,7 @@ local function identify_configuration_files() resolvers.luacnfstate="environment" end reportcriticalvariables(cnfspec) - local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local cnfpaths=expandedpathfromlist(splitpath(cnfspec)) local function locatecnf(luacnfname,kind) for i=1,#cnfpaths do local filepath=cnfpaths[i] @@ -22394,6 +22445,8 @@ local function identify_configuration_files() end local function load_configuration_files() local specification=instance.specification + local setups=instance.setups + local order=instance.order if #specification>0 then local luacnfname=resolvers.luacnfname for i=1,#specification do @@ -22403,7 +22456,6 @@ local function load_configuration_files() local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then - local setups=instance.setups local data=blob() local parent=data and data.parent if parent then @@ -22414,7 +22466,7 @@ local function load_configuration_files() local parentdata=blob() if parentdata then report_resolving("loading configuration file %a",filename) - data=table.merged(parentdata,data) + data=mergedtable(parentdata,data) end end end @@ -22446,7 +22498,7 @@ local function load_configuration_files() if trace_locating then report_resolving("reloading configuration due to TEXMF redefinition") end - resolvers.setenv("TEXMFCNF",cnfspec) + setenv("TEXMFCNF",cnfspec) instance.specification={} identify_configuration_files() load_configuration_files() @@ -22464,7 +22516,7 @@ local function load_configuration_files() elseif trace_locating then report_resolving("skipping configuration file %a (no valid format)",filename) end - instance.order[#instance.order+1]=instance.setups[pathname] + order[#order+1]=setups[pathname] if instance.loaderror then break end @@ -22473,6 +22525,8 @@ local function load_configuration_files() report_resolving("warning: no lua configuration files found") end end +local expandedpathlist +local unexpandedpathlist function resolvers.configurationfiles() return instance.specification or {} end @@ -22489,7 +22543,7 @@ local function load_file_databases() end end local function locate_file_databases() - local texmfpaths=resolvers.expandedpathlist("TEXMF") + local texmfpaths=expandedpathlist("TEXMF") if #texmfpaths>0 then for i=1,#texmfpaths do local path=collapsepath(texmfpaths[i]) @@ -22498,7 +22552,7 @@ local function locate_file_databases() if stripped~="" then local runtime=stripped==path path=cleanpath(path) - local spec=resolvers.splitmethod(stripped) + local spec=splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped elseif spec.scheme=="cache" or spec.scheme=="file" then @@ -22532,11 +22586,13 @@ local function generate_file_databases() end end local function save_file_databases() - for i=1,#instance.hashes do - local hash=instance.hashes[i] + local hashes=instance.hashes + local files=instance.files + for i=1,#hashes do + local hash=hashes[i] local cachename=hash.name if hash.cache then - local content=instance.files[cachename] + local content=files[cachename] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",cachename) @@ -22548,8 +22604,9 @@ local function save_file_databases() end end function resolvers.renew(hashname) + local files=instance.files if hashname and hashname~="" then - local expanded=resolvers.expansion(hashname) or "" + local expanded=expansion(hashname) or "" if expanded~="" then if trace_locating then report_resolving("identifying tree %a from %a",expanded,hashname) @@ -22566,7 +22623,7 @@ function resolvers.renew(hashname) report_resolving("using path %a",realpath) end methodhandler('generators',hashname) - local content=instance.files[hashname] + local content=files[hashname] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",hashname) @@ -22593,38 +22650,46 @@ local function load_databases() end end function resolvers.appendhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a appended",name) end - insert(instance.hashes,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.prependhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a prepended",name) end - insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,1,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.extendtexmfvariable(specification) - local t=resolvers.splitpath(getenv("TEXMF")) - insert(t,1,specification) - local newspec=concat(t,",") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"]=newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"]=newspec + local environment=instance.environment + local variables=instance.variables + local texmftrees=splitpath(getenv("TEXMF")) + insert(texmftrees,1,specification) + texmftrees=concat(texmftrees,",") + if environment["TEXMF"] then + environment["TEXMF"]=texmftrees + elseif variables["TEXMF"] then + variables["TEXMF"]=texmftrees else end reset_hashes() end function resolvers.splitexpansions() - local ie=instance.expansions - for k,v in next,ie do + local expansions=instance.expansions + for k,v in next,expansions do local t,tn,h,p={},0,{},splitconfigurationpath(v) for kk=1,#p do local vv=p[kk] @@ -22634,33 +22699,37 @@ function resolvers.splitexpansions() h[vv]=true end end - if #t>1 then - ie[k]=t + if tn>1 then + expansions[k]=t else - ie[k]=t[1] + expansions[k]=t[1] end end end function resolvers.datastate() return caches.contentstate() end -function resolvers.variable(name) +variable=function(name) + local variables=instance.variables local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.variables[name] + local result=name and variables[name] return result~=nil and result or "" end -function resolvers.expansion(name) +expansion=function(name) + local expansions=instance.expansions local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.expansions[name] + local result=name and expansions[name] return result~=nil and result or "" end -function resolvers.unexpandedpathlist(str) - local pth=resolvers.variable(str) - local lst=resolvers.splitpath(pth) +resolvers.variable=variable +resolvers.expansion=expansion +unexpandedpathlist=function(str) + local pth=variable(str) + local lst=splitpath(pth) return expandedpathfromlist(lst) end function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(unexpandedpathlist(str)) end function resolvers.pushpath(name) local pathstack=instance.pathstack @@ -22690,8 +22759,8 @@ function resolvers.stackpath() end local done={} function resolvers.resetextrapaths() - local ep=instance.extra_paths - if not ep then + local extra_paths=instance.extra_paths + if not extra_paths then done={} instance.extra_paths={} elseif #ep>0 then @@ -22712,8 +22781,8 @@ function resolvers.registerextrapath(paths,subpaths) end local paths=settings_to_array(paths) local subpaths=settings_to_array(subpaths) - local ep=instance.extra_paths or {} - local oldn=#ep + local extra_paths=instance.extra_paths or {} + local oldn=#extra_paths local newn=oldn local nofpaths=#paths local nofsubpaths=#subpaths @@ -22726,7 +22795,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end @@ -22736,7 +22805,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=cleanpath(p) + extra_paths[newn]=cleanpath(p) done[p]=true end end @@ -22745,17 +22814,17 @@ function resolvers.registerextrapath(paths,subpaths) for i=1,oldn do for j=1,nofsubpaths do local s=subpaths[j] - local ps=ep[i].."/"..s + local ps=extra_paths[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end end end if newn>0 then - instance.extra_paths=ep + instance.extra_paths=extra_paths end if newn~=oldn then reset_caches() @@ -22763,17 +22832,19 @@ function resolvers.registerextrapath(paths,subpaths) end function resolvers.pushextrapath(path) local paths=settings_to_array(path) - if instance.extra_stack then - insert(instance.extra_stack,1,paths) + local extra_stack=instance.extra_stack + if extra_stack then + insert(extra_stack,1,paths) else instance.extra_stack={ paths } end reset_caches() end function resolvers.popextrapath() - if instance.extra_stack then + local extra_stack=instance.extra_stack + if extra_stack then reset_caches() - return remove(instance.extra_stack,1) + return remove(extra_stack,1) end end local function made_list(instance,list,extra_too) @@ -22802,33 +22873,21 @@ local function made_list(instance,list,extra_too) end end if extra_too then - local es=instance.extra_stack - if es and #es>0 then - for k=1,#es do - add(es[k]) + local extra_stack=instance.extra_stack + local extra_paths=instance.extra_paths + if extra_stack and #extra_stack>0 then + for k=1,#extra_stack do + add(extra_stack[k]) end end - local ep=instance.extra_paths - if ep and #ep>0 then - add(ep) + if extra_paths and #extra_paths>0 then + add(extra_paths) end end add(list) return new end -function resolvers.cleanpathlist(str) - local t=resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i]=collapsepath(cleanpath(t[i])) - end - end - return t -end -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) -end -function resolvers.expandedpathlist(str,extra_too) +expandedpathlist=function(str,extra_too) if not str then return {} elseif instance.savelists then @@ -22836,40 +22895,56 @@ function resolvers.expandedpathlist(str,extra_too) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + local l=made_list(instance,splitpath(expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else - local lst=resolvers.splitpath(resolvers.expansion(str)) + local lst=splitpath(expansion(str)) return made_list(instance,expandedpathfromlist(lst),extra_too) end end -function resolvers.expandedpathlistfromvariable(str) +resolvers.expandedpathlist=expandedpathlist +resolvers.unexpandedpathlist=unexpandedpathlist +function resolvers.cleanpathlist(str) + local t=expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(cleanpath(t[i])) + end + end + return t +end +function resolvers.expandpath(str) + return joinpath(expandedpathlist(str)) +end +local function expandedpathlistfromvariable(str) str=lpegmatch(dollarstripper,str) local tmp=resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp~="" and tmp or str) + return expandedpathlist(tmp~="" and tmp or str) end function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(expandedpathlistfromvariable(str)) end +resolvers.expandedpathlistfromvariable=expandedpathlistfromvariable function resolvers.cleanedpathlist(v) - local t=resolvers.expandedpathlist(v) + local t=expandedpathlist(v) for i=1,#t do - t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + t[i]=resolveprefix(cleanpath(t[i])) end return t end function resolvers.expandbraces(str) - local pth=expandedpathfromlist(resolvers.splitpath(str)) + local pth=expandedpathfromlist(splitpath(str)) return joinpath(pth) end function resolvers.registerfilehash(name,content,someerror) + local files=instance.files if content then - instance.files[name]=content + files[name]=content else - instance.files[name]={} + files[name]={} if somerror==true then instance.loaderror=someerror end @@ -22926,10 +23001,11 @@ local function collect_files(names) pathname="/"..pathname.."$" end local hashes=instance.hashes + local files=instance.files for h=1,#hashes do local hash=hashes[h] local hashname=hash.name - local content=hashname and instance.files[hashname] + local content=hashname and files[hashname] if content then if trace_details then report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) @@ -22999,7 +23075,6 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if filesuffix=="" or not suffixmap[filesuffix] then local defaultsuffixes=resolvers.defaultsuffixes - local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname @@ -23009,7 +23084,7 @@ local function find_analyze(filename,askedformat,allresults) end end else - filetype=resolvers.formatofsuffix(filename) + filetype=formatofsuffix(filename) if trace_locating then report_resolving("using suffix based filetype %a",filetype) end @@ -23070,11 +23145,11 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] if format_suffixes then for i=1,#format_suffixes do - local s=format_suffixes[i] - forcedname=filename.."."..s + local suffix=format_suffixes[i] + forcedname=filename.."."..suffix if isreadable(forcedname) then if trace_locating then - report_resolving("no suffix, forcing format filetype %a",s) + report_resolving("no suffix, forcing format filetype %a",suffix) end return "qualified",{ forcedname } end @@ -23087,7 +23162,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local savedformat=askedformat local format=savedformat or "" if format=="" then - askedformat=resolvers.formatofsuffix(suffix) + askedformat=formatofsuffix(suffix) end if not format then askedformat="othertextfiles" @@ -23126,7 +23201,7 @@ local function check_subpath(fname) end local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local pathlist=expandedpathlist(typespec,filetype and usertypes[filetype]) local entry={} if pathlist and #pathlist>0 then for k=1,#pathlist do @@ -23137,7 +23212,7 @@ local function makepathlist(list,filetype) local expression=makepathexpression(pathname) local barename=gsub(pathname,"/+$","") barename=resolveprefix(barename) - local scheme=url.hasscheme(barename) + local scheme=urlhasscheme(barename) local schemename=gsub(barename,"%.%*$",'') entry[k]={ path=path, @@ -23342,7 +23417,9 @@ collect_instance_files=function(filename,askedformat,allresults) { find_onpath (filename,filetype,wantedfiles,true) }, { find_otherwise(filename,filetype,wantedfiles,true) }, } - local result,status,done={},{},{} + local result={} + local status={} + local done={} for k,r in next,results do local method,list=r[1],r[2] if method and list then @@ -23413,6 +23490,9 @@ local function findfiles(filename,filetype,allresults) if not filename or filename=="" then return {} end + if allresults==nil then + allresults=true + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -23422,27 +23502,25 @@ local function findfiles(filename,filetype,allresults) end return result or {},status end -function resolvers.findfiles(filename,filetype) - if not filename or filename=="" then - return "" - else - return findfiles(filename,filetype,true) - end -end -function resolvers.findfile(filename,filetype) +local function findfile(filename,filetype) if not filename or filename=="" then return "" else return findfiles(filename,filetype,false)[1] or "" end end +resolvers.findfiles=findfiles +resolvers.findfile=findfile +resolvers.find_file=findfile +resolvers.find_files=findfiles function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) + local hashes=instance.hashes + local files=instance.files local base=filebasename(filename) local result={} - local hashes=instance.hashes local function okay(hash,path,name) local found=methodhandler('concatinators',hash.type,hash.name,path,name) if found and found~="" then @@ -23452,7 +23530,7 @@ local function findgivenfiles(filename,allresults) end for k=1,#hashes do local hash=hashes[k] - local content=instance.files[hash.name] + local content=files[hash.name] if content then local path,name=lookup(content,base) if not path then @@ -23484,14 +23562,14 @@ function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end local function findwildcardfiles(filename,allresults,result) + local files=instance.files + local hashes=instance.hashes local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files=instance.files if find(name,"*",1,true) then - local hashes=instance.hashes local function okay(found,path,base,hashname,hashtype) if find(found,path) then local full=methodhandler('concatinators',hashtype,hashname,found,base) @@ -23531,7 +23609,6 @@ local function findwildcardfiles(filename,allresults,result) end end end - local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] local hashname=hash.name @@ -23561,13 +23638,21 @@ end function resolvers.findwildcardfile(filename) return findwildcardfiles(filename,false)[1] or "" end -function resolvers.automount() -end -function resolvers.starttiming() - statistics.starttiming(instance) +do + local starttiming=statistics.starttiming + local stoptiming=statistics.stoptiming + local elapsedtime=statistics.elapsedtime + function resolvers.starttiming() + starttiming(instance) + end + function resolvers.stoptiming() + stoptiming(instance) + end + function resolvers.loadtime() + return elapsedtime(instance) + end end -function resolvers.stoptiming() - statistics.stoptiming(instance) +function resolvers.automount() end function resolvers.load(option) resolvers.starttiming() @@ -23581,9 +23666,6 @@ function resolvers.load(option) local files=instance.files return files and next(files) and true end -function resolvers.loadtime() - return statistics.elapsedtime(instance) -end local function report(str) if trace_locating then report_resolving(str) @@ -23613,7 +23695,7 @@ function resolvers.dowithfilesandreport(command,files,...) end end function resolvers.showpath(str) - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) + return joinpath(expandedpathlist(resolvers.formatofvariable(str))) end function resolvers.registerfile(files,name,path) if files[name] then @@ -23627,7 +23709,7 @@ function resolvers.registerfile(files,name,path) end end function resolvers.dowithpath(name,func) - local pathlist=resolvers.expandedpathlist(name) + local pathlist=expandedpathlist(name) for i=1,#pathlist do func("^"..cleanpath(pathlist[i])) end @@ -23641,7 +23723,7 @@ function resolvers.locateformat(name) local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then - fmtname=resolvers.findfile(fullname) + fmtname=findfile(fullname) fmtname=cleanpath(fmtname) end if fmtname~="" then @@ -23660,7 +23742,7 @@ function resolvers.locateformat(name) return nil,nil end function resolvers.booleanvariable(str,default) - local b=resolvers.expansion(str) + local b=expansion(str) if b=="" then return default else @@ -23670,6 +23752,7 @@ function resolvers.booleanvariable(str,default) end function resolvers.dowithfilesintree(pattern,handle,before,after) local hashes=instance.hashes + local files=instance.files for i=1,#hashes do local hash=hashes[i] local blobtype=hash.type @@ -23681,7 +23764,7 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) if before then before(blobtype,blobpath,pattern) end - for path,name in filtered(instance.files[blobpath],pattern) do + for path,name in filtered(files[blobpath],pattern) do if type(path)=="string" then checked=checked+1 if handle(blobtype,blobpath,path,name) then @@ -23702,10 +23785,6 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -local obsolete=resolvers.obsolete or {} -resolvers.obsolete=obsolete -resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile -resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles function resolvers.knownvariables(pattern) if instance then local environment=instance.environment @@ -23739,7 +23818,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4854, stripped down to: 2889 +-- original size: 5088, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -23748,6 +23827,7 @@ if not modules then modules={} end modules ['data-pre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local insert,remove=table.insert,table.remove local resolvers=resolvers local prefixes=resolvers.prefixes local cleanpath=resolvers.cleanpath @@ -23825,8 +23905,9 @@ prefixes.kpse=prefixes.locate prefixes.full=prefixes.locate prefixes.file=prefixes.filename prefixes.path=prefixes.pathname +local inputstack={} +local stackpath=resolvers.stackpath local function toppath() - local inputstack=resolvers.inputstack if not inputstack then return "." end @@ -23838,15 +23919,23 @@ local function toppath() end end local function jobpath() - local path=resolvers.stackpath() + local path=stackpath() if not path or path=="" then return "." else return path end end +local function pushinputname(name) + insert(inputstack,name) +end +local function popinputname(name) + return remove(inputstack) +end resolvers.toppath=toppath resolvers.jobpath=jobpath +resolvers.pushinputname=pushinputname +resolvers.popinputname=popinputname prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end resolvers.setdynamic("toppath") @@ -23889,7 +23978,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 470 +-- original size: 551, stripped down to: 470 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -23912,7 +24001,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3170 +-- original size: 4365, stripped down to: 3588 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -23921,39 +24010,46 @@ if not modules then modules={} end modules ['data-fil']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local ioopen=io.open +local isdir=lfs.isdir local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers local resolveprefix=resolvers.resolve -local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers -local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local findfile=resolvers.findfile +local scanfiles=resolvers.scanfiles +local registerfilehash=resolvers.registerfilehash +local appendhash=resolvers.appendhash +local loadcachecontent=caches.loadcontent local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check -function locators.file(specification) +function resolvers.locators.file(specification) local filename=specification.filename local realname=resolveprefix(filename) - if realname and realname~='' and lfs.isdir(realname) then + if realname and realname~='' and isdir(realname) then if trace_locating then report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',filename,true) + appendhash('file',filename,true) elseif trace_locating then report_files("file locator %a not found",filename) end end -function hashers.file(specification) +function resolvers.hashers.file(specification) local pathname=specification.filename - local content=caches.loadcontent(pathname,'files') - resolvers.registerfilehash(pathname,content,content==nil) + local content=loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) end -function generators.file(specification) +function resolvers.generators.file(specification) local pathname=specification.filename - local content=resolvers.scanfiles(pathname,false,true) - resolvers.registerfilehash(pathname,content,true) + local content=scanfiles(pathname,false,true) + registerfilehash(pathname,content,true) end -concatinators.file=file.join +resolvers.concatinators.file=file.join +local finders=resolvers.finders +local notfound=finders.notfound function finders.file(specification,filetype) local filename=specification.filename - local foundname=resolvers.findfile(filename,filetype) + local foundname=findfile(filename,filetype) if foundname and foundname~="" then if trace_locating then report_files("file finder: %a found",filename) @@ -23963,37 +24059,55 @@ function finders.file(specification,filetype) if trace_locating then report_files("file finder: %a not found",filename) end - return finders.notfound() + return notfound() end end -function openers.helpers.textopener(tag,filename,f) +local openers=resolvers.openers +local notfound=openers.notfound +local overloaded=false +local function textopener(tag,filename,f) return { - reader=function() return f:read () end, - close=function() logs.show_close(filename) return f:close() end, + reader=function() return f:read () end, + close=function() return f:close() end, } end +function openers.helpers.textopener(...) + return textopener(...) +end +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded=true + textopener=opener + end +end function openers.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"r") + local f=ioopen(filename,"r") if f then if trace_locating then report_files("file opener: %a opened",filename) end - return openers.helpers.textopener("file",filename,f) + return textopener("file",filename,f) end end if trace_locating then report_files("file opener: %a not found",filename) end - return openers.notfound() + return notfound() end +local loaders=resolvers.loaders +local notfound=loaders.notfound function loaders.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"rb") + local f=ioopen(filename,"rb") if f then - logs.show_load(filename) if trace_locating then report_files("file loader: %a loaded",filename) end @@ -24010,7 +24124,7 @@ function loaders.file(specification,filetype) if trace_locating then report_files("file loader: %a not found",filename) end - return loaders.notfound() + return notfound() end @@ -24020,7 +24134,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5029, stripped down to: 3432 +-- original size: 5388, stripped down to: 3685 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -24029,6 +24143,7 @@ if not modules then modules={} end modules ['data-con']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local setmetatable=setmetatable local format,lower,gsub=string.format,string.lower,string.gsub local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) @@ -24036,16 +24151,21 @@ local trace_storage=false trackers.register("resolvers.storage",function(v) tra containers=containers or {} local containers=containers containers.usecache=true +local getwritablepath=caches.getwritablepath +local getreadablepaths=caches.getreadablepaths +local cacheiswritable=caches.is_writable +local loaddatafromcache=caches.loaddata +local savedataincache=caches.savedata local report_containers=logs.reporter("resolvers","containers") local allocated={} local mt={ __index=function(t,k) if k=="writable" then - local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable=getwritablepath(t.category,t.subcategory) or { "." } t.writable=writable return writable elseif k=="readables" then - local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables=getreadablepaths(t.category,t.subcategory) or { "." } t.readables=readables return readables end @@ -24076,7 +24196,7 @@ function containers.define(category,subcategory,version,enabled) end end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable,name) + return container.enabled and caches and cacheiswritable(container.writable,name) end function containers.is_valid(container,name) if name and name~="" then @@ -24090,7 +24210,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name,container.writable) + stored=loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -24106,17 +24226,20 @@ function containers.read(container,name) end return stored end -function containers.write(container,name,data) +function containers.write(container,name,data,fast) if data then data.cache_version=container.version if container.enabled and caches then - local unique,shared=data.unique,data.shared - data.unique,data.shared=nil,nil - caches.savedata(container.writable,name,data) + local unique=data.unique + local shared=data.shared + data.unique=nil + data.shared=nil + savedataincache(container.writable,name,data,fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique,data.shared=unique,shared + data.unique=unique + data.shared=shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) @@ -24139,7 +24262,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 5203, stripped down to: 3765 +-- original size: 5790, stripped down to: 2910 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -24148,40 +24271,11 @@ if not modules then modules={} end modules ['data-use']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local format=string.format local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_mounts=logs.reporter("resolvers","mounts") local resolvers=resolvers -resolvers.automounted=resolvers.automounted or {} -function resolvers.automount(usecache) - local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths==0) and usecache then - mountpaths=caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths>0 then - resolvers.starttiming() - for k=1,#mountpaths do - local root=mountpaths[k] - local f=io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %a",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() - end - end - resolvers.stoptiming() - end -end +local findfile=resolvers.findfile statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) @@ -24191,7 +24285,7 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local luvdata={ enginebanner=enginebanner, formatbanner=formatbanner, - sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcehash=md5.hex(io.loaddata(findfile(sourcefile)) or "unknown"), sourcefile=sourcefile, luaversion=LUAVERSION, formatid=LUATEXFORMATID, @@ -24215,7 +24309,7 @@ function statistics.checkfmtstatus(texname) if lfs.isfile(luvname) then local luv=dofile(luvname) if luv and luv.sourcefile then - local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local sourcehash=md5.hex(io.loaddata(findfile(luv.sourcefile)) or "unknown") local luvbanner=luv.enginebanner or "?" if luvbanner~=enginebanner then return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) @@ -24256,7 +24350,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 10263, stripped down to: 7556 +-- original size: 10725, stripped down to: 7949 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -24269,6 +24363,14 @@ local format,find,match=string.format,string.find,string.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_zip=logs.reporter("resolvers","zip") local resolvers=resolvers +local findfile=resolvers.findfile +local registerfile=resolvers.registerfile +local splitmethod=resolvers.splitmethod +local prependhash=resolvers.prependhash +local starttiming=resolvers.starttiming +local extendtexmf=resolvers.extendtexmfvariable +local stoptiming=resolvers.stoptiming +local urlquery=url.query zip=zip or {} local zip=zip local archives=zip.archives or {} @@ -24283,8 +24385,9 @@ if zipfiles then closezip=zipfiles.close validfile=zipfiles.found wholefile=zipfiles.unzip + local listzip=zipfiles.list traversezip=function(zfile) - return ipairs(zipfiles.list(zfile)) + return ipairs(listzip(zfile)) end local streams=utilities.streams local openstream=streams.open @@ -24341,28 +24444,30 @@ local function validzip(str) return str end end -function zip.openarchive(name) +local function openarchive(name) if not name or name=="" then return nil else local arch=archives[name] if not arch then - local full=resolvers.findfile(name) or "" + local full=findfile(name) or "" arch=full~="" and openzip(full) or false archives[name]=arch end return arch end end -function zip.closearchive(name) +local function closearchive(name) if not name or (name=="" and archives[name]) then closezip(archives[name]) archives[name]=nil end end +zip.openarchive=openarchive +zip.closearchive=closearchive function resolvers.locators.zip(specification) local archive=specification.filename - local zipfile=archive and archive~="" and zip.openarchive(archive) + local zipfile=archive and archive~="" and openarchive(archive) if trace_locating then if zipfile then report_zip("locator: archive %a found",archive) @@ -24371,13 +24476,6 @@ function resolvers.locators.zip(specification) end end end -function resolvers.hashers.zip(specification) - local archive=specification.filename - if trace_locating then - report_zip("loading file %a",archive) - end - resolvers.usezipfile(specification.original) -end function resolvers.concatinators.zip(zipfile,path,name) if not path or path=="" then return format('%s?name=%s',zipfile,name) @@ -24385,14 +24483,16 @@ function resolvers.concatinators.zip(zipfile,path,name) return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.finders.zip(specification) +local finders=resolvers.finders +local notfound=finders.notfound +function finders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("finder: archive %a found",archive) @@ -24413,16 +24513,19 @@ function resolvers.finders.zip(specification) if trace_locating then report_zip("finder: %a not found",original) end - return resolvers.finders.notfound() + return notfound() end -function resolvers.openers.zip(specification) +local openers=resolvers.openers +local notfound=openers.notfound +local textopener=openers.helpers.textopener +function openers.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("opener; archive %a opened",archive) @@ -24432,7 +24535,7 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: file %a found",queryname) end - return resolvers.openers.helpers.textopener('zip',original,handle) + return textopener('zip',original,handle) elseif trace_locating then report_zip("opener: file %a not found",queryname) end @@ -24444,23 +24547,24 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -function resolvers.loaders.zip(specification) +local loaders=resolvers.loaders +local notfound=loaders.notfound +function loaders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("loader: archive %a opened",archive) end local data=wholefile(zfile,queryname) if data then - logs.show_load(original) if trace_locating then report_zip("loader; file %a loaded",original) end @@ -24476,47 +24580,24 @@ function resolvers.loaders.zip(specification) if trace_locating then report_zip("loader: %a not found",original) end - return resolvers.openers.notfound() -end -function resolvers.usezipfile(archive) - local specification=resolvers.splitmethod(archive) - local archive=specification.filename - if archive and not registeredfiles[archive] then - local z=zip.openarchive(archive) - if z then - local tree=url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering: archive %a",archive) - end - resolvers.starttiming() - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) - registeredfiles[archive]=z - resolvers.registerfilehash(archive,resolvers.registerzipfile(z,tree)) - resolvers.stoptiming() - elseif trace_locating then - report_zip("registering: unknown archive %a",archive) - end - elseif trace_locating then - report_zip("registering: archive %a not found",archive) - end + return notfound() end -function resolvers.registerzipfile(z,tree) +local function registerzipfile(z,tree) local names={} local files={} local remap={} local n=0 local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) - local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end + starttiming() for i in traversezip(z) do local filename=i.filename local path,name=match(filename,filter) if not path then n=n+1 - register(names,filename,"") + registerfile(names,filename,"") local usedname=lower(filename) files[usedname]="" if usedname~=filename then @@ -24533,12 +24614,43 @@ function resolvers.registerzipfile(z,tree) else end end + stoptiming() report_zip("registering: %s files registered",n) return { files=files, remap=remap, } end +local function usezipfile(archive) + local specification=splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=openarchive(archive) + if z then + local tree=urlquery(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + prependhash('zip',archive) + extendtexmf(archive) + registeredfiles[archive]=z + registerfilehash(archive,registerzipfile(z,tree)) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +resolvers.usezipfile=usezipfile +resolvers.registerzipfile=registerzipfile +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + usezipfile(specification.original) +end end -- of closure @@ -24547,7 +24659,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8478, stripped down to: 5223 +-- original size: 9444, stripped down to: 5650 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -24563,173 +24675,204 @@ local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local resolveprefix=resolvers.resolve -local notfound=resolvers.finders.notfound -local lookup=resolvers.get_from_content -local collectors={} -local found={} -function resolvers.finders.tree(specification) - local spec=specification.filename - local okay=found[spec] - if okay==nil then - if spec~="" then - local path=dirname(spec) - local name=basename(spec) - if path=="" then - path="." - end - local names=collectors[path] - if not names then - local pattern=find(path,"/%*+$") and path or (path.."/*") - names=globdir(pattern) - collectors[path]=names - end - local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for i=1,#names do - local fullname=names[i] - if find(fullname,pattern) then - found[spec]=fullname - return fullname - end - end - local pattern=lower(pattern) - for i=1,#names do - local fullname=lower(names[i]) - if find(fullname,pattern) then - if isfile(fullname) then +local finders=resolvers.finders +local openers=resolvers.openers +local loaders=resolvers.loaders +local locators=resolvers.locators +local hashers=resolvers.hashers +local generators=resolvers.generators +do + local collectors={} + local found={} + local notfound=finders.notfound + function finders.tree(specification) + local spec=specification.filename + local okay=found[spec] + if okay==nil then + if spec~="" then + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then found[spec]=fullname return fullname - else - break + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end + okay=notfound() + found[spec]=okay end - okay=notfound() - found[spec]=okay + return okay end - return okay end -function resolvers.locators.tree(specification) - local name=specification.filename - local realname=resolveprefix(name) - if realname and realname~='' and isdir(realname) then - if trace_locating then - report_trees("locator %a found",realname) +do + local resolveprefix=resolvers.resolve + local appendhash=resolvers.appendhash + local function dolocate(specification) + local name=specification.filename + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) end - resolvers.appendhash('tree',name,false) - elseif trace_locating then - report_trees("locator %a not found",name) end + locators.tree=dolocate + locators.dirlist=dolocate + locators.dirfile=dolocate end -function resolvers.hashers.tree(specification) - local name=specification.filename - if trace_locating then - report_trees("analyzing %a",name) - end - resolvers.methodhandler("hashers",name) - resolvers.generators.file(specification) -end -local collectors={} -local splitter=lpeg.splitat("/**/") -local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } -table.setmetatableindex(collectors,function(t,k) - local rootname=lpegmatch(stripper,k) - local dataname=joinname(rootname,"dirlist") - local content=caches.loadcontent(dataname,"files",dataname) - if not content then - content=resolvers.scanfiles(rootname,nil,nil,false,true) - caches.savecontent(dataname,"files",content,dataname) - end - t[k]=content - return content -end) -local function checked(root,p,n) - if p then - if type(p)=="table" then - for i=1,#p do - local fullname=joinname(root,p[i],n) +do + local filegenerator=generators.file + generators.dirlist=filegenerator + generators.dirfile=filegenerator +end +do + local filegenerator=generators.file + local methodhandler=resolvers.methodhandler + local function dohash(specification) + local name=specification.filename + if trace_locating then + report_trees("analyzing %a",name) + end + methodhandler("hashers",name) + filegenerator(specification) + end + hashers.tree=dohash + hashers.dirlist=dohash + hashers.dirfile=dohash +end +local resolve do + local collectors={} + local splitter=lpeg.splitat("/**/") + local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } + local loadcontent=caches.loadcontent + local savecontent=caches.savecontent + local notfound=finders.notfound + local scanfiles=resolvers.scanfiles + local lookup=resolvers.get_from_content + table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=loadcontent(dataname,"files",dataname) + if not content then + content=scanfiles(rootname,nil,nil,false,true) + savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content + end) + local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) if isfile(fullname) then return fullname end end - else - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname - end end + return notfound() end - return notfound() -end -local function resolve(specification) - local filename=specification.filename - if filename~="" then - local root,rest=lpegmatch(splitter,filename) - if root and rest then - local path,name=dirname(rest),basename(rest) - if name~=rest then - local content=collectors[root] - local p,n=lookup(content,name) - if not p then - return notfound() - end - local pattern=".*/"..path.."$" - local istable=type(p)=="table" - if istable then - for i=1,#p do - local pi=p[i] - if pi==path or find(pi,pattern) then - local fullname=joinname(root,pi,n) - if isfile(fullname) then - return fullname + resolve=function(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end end end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end end - elseif p==path or find(p,pattern) then - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() end end - local queries=specification.queries - if queries and queries.option=="fileonly" then - return checked(root,p,n) - else - return notfound() - end + end + local path=dirname(filename) + local name=basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) end end - local path,name=dirname(filename),basename(filename) - local root=lpegmatch(stripper,path) - local content=collectors[path] - local p,n=lookup(content,name) - if p then - return checked(root,p,n) + return notfound() + end + finders.dirlist=resolve + function finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } end + return resolve(specification) end - return notfound() end -resolvers.finders .dirlist=resolve -resolvers.locators .dirlist=resolvers.locators .tree -resolvers.hashers .dirlist=resolvers.hashers .tree -resolvers.generators.dirlist=resolvers.generators.file -resolvers.openers .dirlist=resolvers.openers .file -resolvers.loaders .dirlist=resolvers.loaders .file -function resolvers.finders.dirfile(specification) - local queries=specification.queries - if queries then - queries.option="fileonly" - else - specification.queries={ option="fileonly" } - end - return resolve(specification) +do + local fileopener=openers.file + local fileloader=loaders.file + openers.dirlist=fileopener + loaders.dirlist=fileloader + openers.dirfile=fileopener + loaders.dirfile=fileloader end -resolvers.locators .dirfile=resolvers.locators .dirlist -resolvers.hashers .dirfile=resolvers.hashers .dirlist -resolvers.generators.dirfile=resolvers.generators.dirlist -resolvers.openers .dirfile=resolvers.openers .dirlist -resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -24738,7 +24881,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6757, stripped down to: 5272 +-- original size: 6945, stripped down to: 5408 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -24748,8 +24891,11 @@ if not modules then modules={} end modules ['data-sch']={ license="see context related readme files" } local load,tonumber=load,tonumber -local gsub,concat,format=string.gsub,table.concat,string.format +local gsub,format=string.gsub,string.format +local sortedhash,concat=table.sortedhash,table.concat local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local addsuffix,suffix,splitbase=file.addsuffix,file.suffix,file.splitbase +local md5hex=md5.hex local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) local report_schemes=logs.reporter("resolvers","schemes") local http=require("socket.http") @@ -24766,7 +24912,7 @@ function cleaners.none(specification) return specification.original end function cleaners.strip(specification) - local path,name=file.splitbase(specification.original) + local path,name=splitbase(specification.original) if path=="" then return (gsub(name,"[^%a%d%.]+","-")) else @@ -24774,7 +24920,7 @@ function cleaners.strip(specification) end end function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return addsuffix(md5hex(specification.original),suffix(specification.path)) end local cleaner=cleaners.strip directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) @@ -24865,10 +25011,10 @@ end schemes.install=install local function http_handler(specification,cachename) local tempname=cachename..".tmp" - local f=io.open(tempname,"wb") + local handle=io.open(tempname,"wb") local status,message=http.request { url=specification.original, - sink=ltn12.sink.file(f) + sink=ltn12.sink.file(handle) } if not status then os.remove(tempname) @@ -24883,13 +25029,13 @@ install('https') install('ftp') statistics.register("scheme handling time",function() local l,r,nl,nr={},{},0,0 - for k,v in table.sortedhash(loaded) do + for k,v in sortedhash(loaded) do if v>0 then nl=nl+1 l[nl]=k..":"..v end end - for k,v in table.sortedhash(reused) do + for k,v in sortedhash(reused) do if v>0 then nr=nr+1 r[nr]=k..":"..v @@ -24897,10 +25043,10 @@ statistics.register("scheme handling time",function() end local n=nl+nr if n>0 then - l=nl>0 and concat(l) or "none" - r=nr>0 and concat(r) or "none" + if nl==0 then l={ "none" } end + if nr==0 then r={ "none" } end return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes),n,threshold,l,r) + statistics.elapsedtime(schemes),n,threshold,concat(l," "),concat(l," ")) else return nil end @@ -24933,7 +25079,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4207, stripped down to: 3041 +-- original size: 4227, stripped down to: 3049 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -24943,8 +25089,7 @@ if not modules then modules={} end modules ['data-lua']={ license="see context related readme files" } local package,lpeg=package,lpeg -local gsub=string.gsub -local concat=table.concat +local loadfile=loadfile local addsuffix=file.addsuffix local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match local luasuffixes={ 'tex','lua' } @@ -24955,6 +25100,8 @@ local helpers=package.helpers or {} local methods=helpers.methods or {} local resolvers=resolvers local resolveprefix=resolvers.resolve +local expandedpaths=resolvers.expandedpathlistfromvariable +local findfile=resolvers.findfile helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) @@ -24983,7 +25130,7 @@ local function getluaformatpaths() if not luaformatpaths then luaformatpaths={} for i=1,#luaformats do - registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + registerpath("lua format","lua",luaformatpaths,expandedpaths(luaformats[i])) end end return luaformatpaths @@ -24992,7 +25139,7 @@ local function getlibformatpaths() if not libformatpaths then libformatpaths={} for i=1,#libformats do - registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + registerpath("lib format","lib",libformatpaths,expandedpaths(libformats[i])) end end return libformatpaths @@ -25002,7 +25149,7 @@ local function loadedbyformat(name,rawname,suffixes,islib,what) local report=helpers.report for i=1,#suffixes do local format=suffixes[i] - local resolved=resolvers.findfile(name,format) or "" + local resolved=findfile(name,format) or "" if trace then report("%s format, identifying %a using format %a",what,name,format) end @@ -25040,7 +25187,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2452, stripped down to: 1877 +-- original size: 2610, stripped down to: 2019 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -25051,24 +25198,28 @@ if not modules then modules={} end modules ['data-aux']={ } local find=string.find local type,next=type,next +local addsuffix,removesuffix=file.addsuffix,file.removesuffix +local loaddata,savedata=io.loaddata,io.savedata local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers +local cleanpath=resolvers.cleanpath +local findfiles=resolvers.findfiles local report_scripts=logs.reporter("resolvers","scripts") function resolvers.updatescript(oldname,newname) local scriptpath="context/lua" - newname=file.addsuffix(newname,"lua") - local oldscript=resolvers.cleanpath(oldname) + local oldscript=cleanpath(oldname) + local newname=addsuffix(newname,"lua") + local newscripts=findfiles(newname) or {} if trace_locating then report_scripts("to be replaced old script %a",oldscript) end - local newscripts=resolvers.findfiles(newname) or {} if #newscripts==0 then if trace_locating then report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript=resolvers.cleanpath(newscripts[i]) + local newscript=cleanpath(newscripts[i]) if trace_locating then report_scripts("checking new script %a",newscript) end @@ -25080,17 +25231,17 @@ function resolvers.updatescript(oldname,newname) if trace_locating then report_scripts("new script should come from %a",scriptpath) end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + elseif not (find(oldscript,removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then report_scripts("invalid new script name") end else - local newdata=io.loaddata(newscript) + local newdata=loaddata(newscript) if newdata then if trace_locating then report_scripts("old script content replaced by new content: %s",oldscript) end - io.savedata(oldscript,newdata) + savedata(oldscript,newdata) break elseif trace_locating then report_scripts("unable to load new script") @@ -25163,7 +25314,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 1823, stripped down to: 1542 +-- original size: 1912, stripped down to: 1591 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -25173,40 +25324,39 @@ if not modules then modules={} end modules ['data-lst']={ license="see context related readme files" } local type=type -local concat,sortedhash=table.concat,table.sortedhash +local sortedhash=table.sortedhash +local isdir=lfs.isdir local resolvers=resolvers local listers=resolvers.listers or {} resolvers.listers=listers local resolveprefix=resolvers.resolve +local configurationfiles=resolvers.configurationfiles +local expandedpathfromlist=resolvers.expandedpathfromlist +local splitpath=resolvers.splitpath +local knownvariables=resolvers.knownvariables local report_lists=logs.reporter("resolvers","lists") local report_resolved=logs.reporter("system","resolved") -local function tabstr(str) - if type(str)=='table' then - return concat(str," | ") - else - return str - end -end function listers.variables(pattern) local result=resolvers.knownvariables(pattern) + local unset={ "unset" } for key,value in sortedhash(result) do report_lists(key) - report_lists(" env: %s",tabstr(value.environment or "unset")) - report_lists(" var: %s",tabstr(value.variable or "unset")) - report_lists(" exp: %s",tabstr(value.expansion or "unset")) - report_lists(" res: %s",tabstr(value.resolved or "unset")) + report_lists(" env: % | t",value.environment or unset) + report_lists(" var: % | t",value.variable or unset) + report_lists(" exp: % | t",value.expansion or unset) + report_lists(" res: % | t",value.resolved or unset) end end function listers.configurations() - local configurations=resolvers.configurationfiles() + local configurations=configurationfiles() for i=1,#configurations do report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") - local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + local list=expandedpathfromlist(splitpath(resolvers.luacnfspec)) for i=1,#list do local li=resolveprefix(list[i]) - if lfs.isdir(li) then + if isdir(li) then report_resolved("path - %s",li) else report_resolved("path + %s",li) @@ -25978,8 +26128,8 @@ end -- of closure -- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 1036669 --- stripped bytes : 410250 +-- original bytes : 1040817 +-- stripped bytes : 412122 -- end library merge diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun index 36a7bbcd2..bf87526f2 100644 --- a/scripts/context/stubs/unix/mtxrun +++ b/scripts/context/stubs/unix/mtxrun @@ -324,7 +324,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-macro"] = package.loaded["l-macro"] or true --- original size: 10131, stripped down to: 5991 +-- original size: 10130, stripped down to: 5990 if not modules then modules={} end modules ['l-macros']={ version=1.001, @@ -549,7 +549,7 @@ end macros.loaded=loaded function required(name,trace) local filename=file.addsuffix(name,"lua") - local fullname=resolvers and resolvers.find_file(filename) or filename + local fullname=resolvers and resolvers.findfile(filename) or filename if not fullname or fullname=="" then return false end @@ -12608,7 +12608,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 13348, stripped down to: 8838 +-- original size: 13394, stripped down to: 8882 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -12618,8 +12618,9 @@ if not modules then modules={} end modules ['trac-set']={ license="see context related readme files" } local type,next,tostring,tonumber=type,next,tostring,tonumber +local print=print local concat,sortedhash=table.concat,table.sortedhash -local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local formatters,find,lower,gsub,topattern=string.formatters,string.find,string.lower,string.gsub,string.topattern local is_boolean=string.is_boolean local settings_to_hash=utilities.parsers.settings_to_hash local allocate=utilities.storage.allocate @@ -12826,8 +12827,8 @@ function setters.show(t) end end local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show -function setters.report(setter,...) - print(format("%-15s : %s\n",setter.name,format(...))) +function setters.report(setter,fmt,...) + print(formatters["%-15s : %s\n"](setter.name,formatters[fmt](...))) end local function default(setter,name) local d=setter.data[name] @@ -12950,7 +12951,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 33803, stripped down to: 21818 +-- original size: 32997, stripped down to: 21715 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -13613,12 +13614,6 @@ end local nesting=0 local verbose=false local hasscheme=url.hasscheme -function logs.show_open(name) -end -function logs.show_close(name) -end -function logs.show_load(name) -end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple @@ -20686,7 +20681,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 10987, stripped down to: 7056 +-- original size: 11019, stripped down to: 7086 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -20699,6 +20694,7 @@ local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local sortedpairs=table.sortedpairs local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) @@ -20909,7 +20905,7 @@ if ostype=="unix" then rawset(t,k,v) end local colon=P(":") - for k,v in table.sortedpairs(prefixes) do + for k,v in sortedpairs(prefixes) do if p then p=P(k)+p else @@ -20936,7 +20932,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18154, stripped down to: 10416 +-- original size: 18173, stripped down to: 10432 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -21220,7 +21216,7 @@ local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) scancache[sub(full,1,-2)]=files return files,remap,n,m,r end -function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) +local function scanfiles(path,branch,usecache,onlyonce,tolerant) local realpath=resolveprefix(path) if usecache then local content=fullcache[realpath] @@ -21275,8 +21271,9 @@ function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) statistics.stoptiming(timer) return content end +resolvers.scanfiles=scanfiles function resolvers.simplescanfiles(path,branch,usecache) - return resolvers.scanfiles(path,branch,usecache,true,true) + return scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -21343,7 +21340,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9384, stripped down to: 6333 +-- original size: 9400, stripped down to: 6347 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -21353,7 +21350,7 @@ if not modules then modules={} end modules ['data-env']={ license="see context related readme files", } local lower,gsub=string.lower,string.gsub -local next=next +local next,rawget=next,rawget local resolvers=resolvers local allocate=utilities.storage.allocate local setmetatableindex=table.setmetatableindex @@ -21628,7 +21625,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16387, stripped down to: 11132 +-- original size: 16070, stripped down to: 11353 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -21637,49 +21634,83 @@ if not modules then modules={} end modules ['data-tmp']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat=table.concat -local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile -local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable -local formatters=string.formatters local next,type=next,type +local pcall,loadfile,collectgarbage=pcall,loadfile,collectgarbage +local format,lower,gsub=string.format,string.lower,string.gsub +local concat,serialize,fastserialize,serializetofile=table.concat,table.serialize,table.fastserialize,table.tofile +local mkdirs,expanddirname,isdir,isfile=dir.mkdirs,dir.expandname,lfs.isdir,lfs.isfile +local is_writable,is_readable=file.is_writable,file.is_readable +local collapsepath,joinfile,addsuffix,dirname=file.collapsepath,file.join,file.addsuffix,file.dirname +local savedata=file.savedata +local formatters=string.formatters +local osexit,osdate,osuuid=os.exit,os.date,os.uuid +local removefile=os.remove +local md5hex=md5.hex local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers local cleanpath=resolvers.cleanpath -local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) -local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) -local compile=utilities.lua.compile -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup==nil then cleanup=directive_cleanup end - if strip==nil then strip=directive_strip end - return compile(luafile,lucfile,cleanup,strip) +local resolvepath=resolvers.resolve +local luautilities=utilities.lua +do + local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) + local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) + local compilelua=luautilities.compile + function luautilities.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compilelua(luafile,lucfile,cleanup,strip) + end end caches=caches or {} local caches=caches -local luasuffixes=utilities.lua.suffixes -caches.base=caches.base or "luatex-cache" -caches.more=caches.more or "context" -caches.direct=false -caches.tree=false -caches.force=true -caches.ask=false -caches.relocate=false +local writable=nil +local readables={} +local usedreadables={} +local compilelua=luautilities.compile +local luasuffixes=luautilities.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } -directives.register("system.caches.fast",function(v) caches.fast=true end) -directives.register("system.caches.direct",function(v) caches.direct=true end) -local writable,readables,usedreadables=nil,{},{} +local direct_cache=false +local fast_cache=false +local cache_tree=false +directives.register("system.caches.direct",function(v) direct_cache=true end) +directives.register("system.caches.fast",function(v) fast_cache=true end) +local function configfiles() + return concat(resolvers.configurationfiles(),";") +end +local function hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +local function treehash() + local tree=configfiles() + if not tree or tree=="" then + return false + else + return hashed(tree) + end +end +caches.hashed=hashed +caches.treehash=treehash +caches.configfiles=configfiles local function identify() local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) - cachepath=file.collapsepath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) + cachepath=collapsepath(cachepath) local valid=isdir(cachepath) if valid then if is_readable(cachepath) then @@ -21688,16 +21719,14 @@ local function identify() writable=cachepath end end - elseif not writable and caches.force then - local cacheparent=file.dirname(cachepath) - if is_writable(cacheparent) and true then - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("path %a created",cachepath) - writable=cachepath - readables[#readables+1]=cachepath - end + elseif not writable then + local cacheparent=dirname(cachepath) + if is_writable(cacheparent) then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath end end end @@ -21710,8 +21739,8 @@ local function identify() local cachepath=texmfcaches[k] cachepath=resolvers.expansion(cachepath) if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) local valid=isdir(cachepath) if valid and is_readable(cachepath) then if not writable and is_writable(cachepath) then @@ -21725,23 +21754,25 @@ local function identify() end if not writable then report_caches("fatal error: there is no valid writable cache path defined") - os.exit() + osexit() elseif #readables==0 then report_caches("fatal error: there is no valid readable cache path defined") - os.exit() + osexit() end - writable=dir.expandname(resolvers.cleanpath(writable)) - local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + writable=expanddirname(cleanpath(writable)) + local base=caches.base + local more=caches.more + local tree=cache_tree or treehash() if tree then - caches.tree=tree + cache_tree=tree writable=mkdirs(writable,base,more,tree) for i=1,#readables do - readables[i]=file.join(readables[i],base,more,tree) + readables[i]=joinfile(readables[i],base,more,tree) end else writable=mkdirs(writable,base,more) for i=1,#readables do - readables[i]=file.join(readables[i],base,more) + readables[i]=joinfile(readables[i],base,more) end end if trace_cache then @@ -21778,27 +21809,8 @@ function caches.usedpaths(separator) return writable or "?" end end -function caches.configfiles() - return concat(resolvers.configurationfiles(),";") -end -function caches.hashed(tree) - tree=gsub(tree,"[\\/]+$","") - tree=lower(tree) - local hash=md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %a, hash %a",tree,hash) - end - return hash -end -function caches.treehash() - local tree=caches.configfiles() - if not tree or tree=="" then - return false - else - return caches.hashed(tree) - end -end -local r_cache,w_cache={},{} +local r_cache={} +local w_cache={} local function getreadablepaths(...) local tags={... } local hash=concat(tags,"/") @@ -21808,7 +21820,7 @@ local function getreadablepaths(...) if #tags>0 then done={} for i=1,#readables do - done[i]=file.join(readables[i],...) + done[i]=joinfile(readables[i],...) end else done=readables @@ -21832,17 +21844,25 @@ local function getwritablepath(...) end return done end -caches.getreadablepaths=getreadablepaths -caches.getwritablepath=getwritablepath -function caches.getfirstreadablefile(filename,...) - local fullname,path=caches.setfirstwritablefile(filename,...) +local function setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=joinfile(wr,filename) + return fullname,wr +end +local function setluanames(path,name) + return + format("%s/%s.%s",path,name,luasuffixes.tma), + format("%s/%s.%s",path,name,luasuffixes.tmc) +end +local function getfirstreadablefile(filename,...) + local fullname,path=setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path end local rd=getreadablepaths(...) for i=1,#rd do local path=rd[i] - local fullname=file.join(path,filename) + local fullname=joinfile(path,filename) if is_readable(fullname) then usedreadables[i]=true return fullname,path @@ -21850,19 +21870,11 @@ function caches.getfirstreadablefile(filename,...) end return fullname,path end -function caches.setfirstwritablefile(filename,...) - local wr=getwritablepath(...) - local fullname=file.join(wr,filename) - return fullname,wr -end -function caches.define(category,subcategory) - return function() - return getwritablepath(category,subcategory) - end -end -function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) -end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +caches.setfirstwritablefile=setfirstwritablefile +caches.getfirstreadablefile=getfirstreadablefile +caches.setluanames=setluanames function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } @@ -21871,16 +21883,16 @@ function caches.loaddata(readables,name,writable) local path=readables[i] local loader=false local state=false - local tmaname,tmcname=caches.setluanames(path,name) + local tmaname,tmcname=setluanames(path,name) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end if not loader and isfile(tmaname) then - local tmacrap,tmcname=caches.setluanames(writable,name) + local tmacrap,tmcname=setluanames(writable,name) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end @@ -21897,21 +21909,21 @@ function caches.loaddata(readables,name,writable) return false end function caches.is_writable(filepath,filename) - local tmaname,tmcname=caches.setluanames(filepath,filename) + local tmaname,tmcname=setluanames(filepath,filename) return is_writable(tmaname) end local saveoptions={ compact=true } -function caches.savedata(filepath,filename,data) - local tmaname,tmcname=caches.setluanames(filepath,filename) - data.cache_uuid=os.uuid() - if caches.fast then - file.savedata(tmaname,table.fastserialize(data,true)) - elseif caches.direct then - file.savedata(tmaname,table.serialize(data,true,saveoptions)) +function caches.savedata(filepath,filename,data,fast) + local tmaname,tmcname=setluanames(filepath,filename) + data.cache_uuid=osuuid() + if fast or fast_cache then + savedata(tmaname,fastserialize(data,true)) + elseif direct_cache then + savedata(tmaname,serialize(data,true,saveoptions)) else - table.tofile(tmaname,data,true,saveoptions) + serializetofile(tmaname,data,true,saveoptions) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) end local content_state={} function caches.contentstate() @@ -21919,9 +21931,9 @@ function caches.contentstate() end function caches.loadcontent(cachename,dataname,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.luc)) if not blob then @@ -21959,9 +21971,9 @@ function caches.collapsecontent(content) end function caches.savecontent(cachename,dataname,content,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) @@ -21972,17 +21984,17 @@ function caches.savecontent(cachename,dataname,content,filename) type=dataname, root=cachename, version=resolvers.cacheversion, - date=os.date("%Y-%m-%d"), - time=os.date("%H:%M:%S"), + date=osdate("%Y-%m-%d"), + time=osdate("%H:%M:%S"), content=content, - uuid=os.uuid(), + uuid=osuuid(), } - local ok=io.savedata(luaname,table.serialize(data,true)) + local ok=savedata(luaname,serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - if utilities.lua.compile(luaname,lucname) then + if compilelua(luaname,lucname) then if trace_locating then report_resolvers("%a compiled to %a",dataname,lucname) end @@ -21991,7 +22003,7 @@ function caches.savecontent(cachename,dataname,content,filename) if trace_locating then report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) end - os.remove(lucname) + removefile(lucname) end elseif trace_locating then report_resolvers("unable to save %a in %a (access error)",dataname,luaname) @@ -22005,7 +22017,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5310, stripped down to: 3784 +-- original size: 5518, stripped down to: 3854 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -22014,31 +22026,45 @@ if not modules then modules={} end modules ['data-met']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,format=string.find,string.format -local sequenced=table.sequenced +local type=type +local find=string.find local addurlscheme,urlhashed=url.addscheme,url.hashed +local collapsepath,joinfile=file.collapsepath,file.join +local report_methods=logs.reporter("resolvers","methods") local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) trackers.register("resolvers.methods",function(v) trace_methods=v end) -local report_methods=logs.reporter("resolvers","methods") local allocate=utilities.storage.allocate local resolvers=resolvers local registered={} local function splitmethod(filename) if not filename then - return { scheme="unknown",original=filename } + return { + scheme="unknown", + original=filename, + } end if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") + filename=collapsepath(filename,".") if not find(filename,"://",1,true) then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } end - local specification=url.hashed(filename) + local specification=urlhashed(filename) if not specification.scheme or specification.scheme=="" then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } else return specification end @@ -22047,7 +22073,8 @@ resolvers.splitmethod=splitmethod local function methodhandler(what,first,...) local method=registered[what] if method then - local how,namespace=method.how,method.namespace + local how=method.how + local namespace=method.namespace if how=="uri" or how=="url" then local specification=splitmethod(first) local scheme=specification.scheme @@ -22093,7 +22120,10 @@ local function methodhandler(what,first,...) end resolvers.methodhandler=methodhandler function resolvers.registermethod(name,namespace,how) - registered[name]={ how=how or "tag",namespace=namespace } + registered[name]={ + how=how or "tag", + namespace=namespace + } namespace["byscheme"]=function(scheme,filename,...) if scheme=="file" then return methodhandler(name,filename,...) @@ -22102,7 +22132,7 @@ function resolvers.registermethod(name,namespace,how) end end end -local concatinators=allocate { notfound=file.join } +local concatinators=allocate { notfound=joinfile } local locators=allocate { notfound=function() end } local hashers=allocate { notfound=function() end } local generators=allocate { notfound=function() end } @@ -22123,7 +22153,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 68209, stripped down to: 43711 +-- original size: 69574, stripped down to: 44470 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -22134,7 +22164,8 @@ if not modules then modules={} end modules ['data-res']={ } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch local concat,insert,remove=table.concat,table.insert,table.remove -local next,type,rawget=next,type,rawget +local next,type,rawget,loadfile=next,type,rawget,loadfile +local mergedtable=table.merged local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns @@ -22150,6 +22181,7 @@ local joinpath=file.joinpath local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local urlhasscheme=url.hasscheme local getcurrentdir=lfs.currentdir local isfile=lfs.isfile local isdir=lfs.isdir @@ -22175,10 +22207,17 @@ local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os. resolvers.cacheversion="1.100" resolvers.configbanner="" resolvers.homedir=environment.homedir -resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" +local criticalvars={ + "SELFAUTOLOC", + "SELFAUTODIR", + "SELFAUTOPARENT", + "TEXMFCNF", + "TEXMF", + "TEXOS", +} if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf else @@ -22198,13 +22237,20 @@ local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } local instance=nil -function resolvers.setenv(key,value,raw) +local variable +local expansion +local setenv +local getenv +local formatofsuffix=resolvers.formatofsuffix +local splitpath=resolvers.splitpath +local splitmethod=resolvers.splitmethod +setenv=function(key,value,raw) if instance then instance.environment[key]=value ossetenv(key,raw and value or resolveprefix(value)) end end -local function getenv(key) +getenv=function(key) local value=rawget(instance.environment,key) if value and value~="" then return value @@ -22214,94 +22260,97 @@ local function getenv(key) end end resolvers.getenv=getenv -resolvers.env=getenv -local function resolvevariable(k) - return instance.expansions[k] -end +resolvers.setenv=setenv local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local somevariable=P("$")/"" -local somekey=C(R("az","AZ","09","__","--")^1) -local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) -local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" -local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolvevariable -local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) -local variableresolver=Cs((variable+P(1))^0) -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end -function resolvers.reset() - if trace_locating then - report_resolving("creating instance") - end - local environment={} - local variables={} - local expansions={} - local order={} - instance={ - environment=environment, - variables=variables, - expansions=expansions, - order=order, - files={}, - setups={}, - found={}, - foundintrees={}, - hashes={}, - hashed={}, - pathlists=false, - specification={}, - lists={}, - data={}, - fakepaths={}, - remember=true, - diskcache=true, - renewcache=false, - renewtree=false, - loaderror=false, - savelists=true, - pattern=nil, - force_suffixes=true, - pathstack={}, - } - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v=order[i][k] +local expandedvariable,resolvedvariable do + local function resolveinstancevariable(k) + return instance.expansions[k] + end + local p_variable=P("$")/"" + local p_key=C(R("az","AZ","09","__","--")^1) + local p_whatever=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) + local variableexpander=Cs((p_variable*(p_key/resolveinstancevariable)+p_whatever)^1 ) + local p_cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" + local variablecleaner=Cs((p_cleaner+P(1))^0) + local p_variable=R("az","AZ","09","__","--")^1/resolveinstancevariable + local p_variable=(P("$")/"")*(p_variable+(P("{")/"")*p_variable*(P("}")/"")) + local variableresolver=Cs((p_variable+P(1))^0) + expandedvariable=function(var) + return lpegmatch(variableexpander,var) or var + end + function resolvers.reset() + if trace_locating then + report_resolving("creating instance") + end + local environment={} + local variables={} + local expansions={} + local order={} + instance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files={}, + setups={}, + found={}, + foundintrees={}, + hashes={}, + hashed={}, + pathlists=false, + specification={}, + lists={}, + data={}, + fakepaths={}, + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + pathstack={}, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + local repath=resolvers.repath + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end if v~=nil then - t[k]=v - return v + v=checkedvariable(v) or "" end - end - if v==nil then - v="" - end - t[k]=v - return v - end) - setmetatableindex(environment,function(t,k) - local v=osgetenv(k) - if v==nil then - v=variables[k] - end - if v~=nil then - v=checkedvariable(v) or "" - end - v=resolvers.repath(v) - t[k]=v - return v - end) - setmetatableindex(expansions,function(t,k) - local v=environment[k] - if type(v)=="string" then - v=lpegmatch(variableresolver,v) - v=lpegmatch(variablecleaner,v) - end - t[k]=v - return v - end) + v=repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + end end function resolvers.initialized() return instance~=nil @@ -22315,31 +22364,33 @@ local function reset_caches() instance.lists={} instance.pathlists=false end -local slash=P("/") -local pathexpressionpattern=Cs ( - Cc("^")*( - Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +local makepathexpression do + local slash=P("/") + local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) - )^1*Cc("$") -) -local cache={} -local function makepathexpression(str) - if str=="." then - return "^%./$" - else - local c=cache[str] - if not c then - c=lpegmatch(pathexpressionpattern,str) - cache[str]=c + )^1*Cc("$") + ) + local cache={} + makepathexpression=function(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c end - return c end end local function reportcriticalvariables(cnfspec) if trace_locating then - for i=1,#resolvers.criticalvars do - local k=resolvers.criticalvars[i] - local v=resolvers.getenv(k) or "unknown" + for i=1,#criticalvars do + local k=criticalvars[i] + local v=getenv(k) or "unknown" report_resolving("variable %a set to %a",k,v) end report_resolving() @@ -22361,7 +22412,7 @@ local function identify_configuration_files() resolvers.luacnfstate="environment" end reportcriticalvariables(cnfspec) - local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local cnfpaths=expandedpathfromlist(splitpath(cnfspec)) local function locatecnf(luacnfname,kind) for i=1,#cnfpaths do local filepath=cnfpaths[i] @@ -22394,6 +22445,8 @@ local function identify_configuration_files() end local function load_configuration_files() local specification=instance.specification + local setups=instance.setups + local order=instance.order if #specification>0 then local luacnfname=resolvers.luacnfname for i=1,#specification do @@ -22403,7 +22456,6 @@ local function load_configuration_files() local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then - local setups=instance.setups local data=blob() local parent=data and data.parent if parent then @@ -22414,7 +22466,7 @@ local function load_configuration_files() local parentdata=blob() if parentdata then report_resolving("loading configuration file %a",filename) - data=table.merged(parentdata,data) + data=mergedtable(parentdata,data) end end end @@ -22446,7 +22498,7 @@ local function load_configuration_files() if trace_locating then report_resolving("reloading configuration due to TEXMF redefinition") end - resolvers.setenv("TEXMFCNF",cnfspec) + setenv("TEXMFCNF",cnfspec) instance.specification={} identify_configuration_files() load_configuration_files() @@ -22464,7 +22516,7 @@ local function load_configuration_files() elseif trace_locating then report_resolving("skipping configuration file %a (no valid format)",filename) end - instance.order[#instance.order+1]=instance.setups[pathname] + order[#order+1]=setups[pathname] if instance.loaderror then break end @@ -22473,6 +22525,8 @@ local function load_configuration_files() report_resolving("warning: no lua configuration files found") end end +local expandedpathlist +local unexpandedpathlist function resolvers.configurationfiles() return instance.specification or {} end @@ -22489,7 +22543,7 @@ local function load_file_databases() end end local function locate_file_databases() - local texmfpaths=resolvers.expandedpathlist("TEXMF") + local texmfpaths=expandedpathlist("TEXMF") if #texmfpaths>0 then for i=1,#texmfpaths do local path=collapsepath(texmfpaths[i]) @@ -22498,7 +22552,7 @@ local function locate_file_databases() if stripped~="" then local runtime=stripped==path path=cleanpath(path) - local spec=resolvers.splitmethod(stripped) + local spec=splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped elseif spec.scheme=="cache" or spec.scheme=="file" then @@ -22532,11 +22586,13 @@ local function generate_file_databases() end end local function save_file_databases() - for i=1,#instance.hashes do - local hash=instance.hashes[i] + local hashes=instance.hashes + local files=instance.files + for i=1,#hashes do + local hash=hashes[i] local cachename=hash.name if hash.cache then - local content=instance.files[cachename] + local content=files[cachename] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",cachename) @@ -22548,8 +22604,9 @@ local function save_file_databases() end end function resolvers.renew(hashname) + local files=instance.files if hashname and hashname~="" then - local expanded=resolvers.expansion(hashname) or "" + local expanded=expansion(hashname) or "" if expanded~="" then if trace_locating then report_resolving("identifying tree %a from %a",expanded,hashname) @@ -22566,7 +22623,7 @@ function resolvers.renew(hashname) report_resolving("using path %a",realpath) end methodhandler('generators',hashname) - local content=instance.files[hashname] + local content=files[hashname] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",hashname) @@ -22593,38 +22650,46 @@ local function load_databases() end end function resolvers.appendhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a appended",name) end - insert(instance.hashes,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.prependhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a prepended",name) end - insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,1,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.extendtexmfvariable(specification) - local t=resolvers.splitpath(getenv("TEXMF")) - insert(t,1,specification) - local newspec=concat(t,",") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"]=newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"]=newspec + local environment=instance.environment + local variables=instance.variables + local texmftrees=splitpath(getenv("TEXMF")) + insert(texmftrees,1,specification) + texmftrees=concat(texmftrees,",") + if environment["TEXMF"] then + environment["TEXMF"]=texmftrees + elseif variables["TEXMF"] then + variables["TEXMF"]=texmftrees else end reset_hashes() end function resolvers.splitexpansions() - local ie=instance.expansions - for k,v in next,ie do + local expansions=instance.expansions + for k,v in next,expansions do local t,tn,h,p={},0,{},splitconfigurationpath(v) for kk=1,#p do local vv=p[kk] @@ -22634,33 +22699,37 @@ function resolvers.splitexpansions() h[vv]=true end end - if #t>1 then - ie[k]=t + if tn>1 then + expansions[k]=t else - ie[k]=t[1] + expansions[k]=t[1] end end end function resolvers.datastate() return caches.contentstate() end -function resolvers.variable(name) +variable=function(name) + local variables=instance.variables local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.variables[name] + local result=name and variables[name] return result~=nil and result or "" end -function resolvers.expansion(name) +expansion=function(name) + local expansions=instance.expansions local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.expansions[name] + local result=name and expansions[name] return result~=nil and result or "" end -function resolvers.unexpandedpathlist(str) - local pth=resolvers.variable(str) - local lst=resolvers.splitpath(pth) +resolvers.variable=variable +resolvers.expansion=expansion +unexpandedpathlist=function(str) + local pth=variable(str) + local lst=splitpath(pth) return expandedpathfromlist(lst) end function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(unexpandedpathlist(str)) end function resolvers.pushpath(name) local pathstack=instance.pathstack @@ -22690,8 +22759,8 @@ function resolvers.stackpath() end local done={} function resolvers.resetextrapaths() - local ep=instance.extra_paths - if not ep then + local extra_paths=instance.extra_paths + if not extra_paths then done={} instance.extra_paths={} elseif #ep>0 then @@ -22712,8 +22781,8 @@ function resolvers.registerextrapath(paths,subpaths) end local paths=settings_to_array(paths) local subpaths=settings_to_array(subpaths) - local ep=instance.extra_paths or {} - local oldn=#ep + local extra_paths=instance.extra_paths or {} + local oldn=#extra_paths local newn=oldn local nofpaths=#paths local nofsubpaths=#subpaths @@ -22726,7 +22795,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end @@ -22736,7 +22805,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=cleanpath(p) + extra_paths[newn]=cleanpath(p) done[p]=true end end @@ -22745,17 +22814,17 @@ function resolvers.registerextrapath(paths,subpaths) for i=1,oldn do for j=1,nofsubpaths do local s=subpaths[j] - local ps=ep[i].."/"..s + local ps=extra_paths[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end end end if newn>0 then - instance.extra_paths=ep + instance.extra_paths=extra_paths end if newn~=oldn then reset_caches() @@ -22763,17 +22832,19 @@ function resolvers.registerextrapath(paths,subpaths) end function resolvers.pushextrapath(path) local paths=settings_to_array(path) - if instance.extra_stack then - insert(instance.extra_stack,1,paths) + local extra_stack=instance.extra_stack + if extra_stack then + insert(extra_stack,1,paths) else instance.extra_stack={ paths } end reset_caches() end function resolvers.popextrapath() - if instance.extra_stack then + local extra_stack=instance.extra_stack + if extra_stack then reset_caches() - return remove(instance.extra_stack,1) + return remove(extra_stack,1) end end local function made_list(instance,list,extra_too) @@ -22802,33 +22873,21 @@ local function made_list(instance,list,extra_too) end end if extra_too then - local es=instance.extra_stack - if es and #es>0 then - for k=1,#es do - add(es[k]) + local extra_stack=instance.extra_stack + local extra_paths=instance.extra_paths + if extra_stack and #extra_stack>0 then + for k=1,#extra_stack do + add(extra_stack[k]) end end - local ep=instance.extra_paths - if ep and #ep>0 then - add(ep) + if extra_paths and #extra_paths>0 then + add(extra_paths) end end add(list) return new end -function resolvers.cleanpathlist(str) - local t=resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i]=collapsepath(cleanpath(t[i])) - end - end - return t -end -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) -end -function resolvers.expandedpathlist(str,extra_too) +expandedpathlist=function(str,extra_too) if not str then return {} elseif instance.savelists then @@ -22836,40 +22895,56 @@ function resolvers.expandedpathlist(str,extra_too) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + local l=made_list(instance,splitpath(expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else - local lst=resolvers.splitpath(resolvers.expansion(str)) + local lst=splitpath(expansion(str)) return made_list(instance,expandedpathfromlist(lst),extra_too) end end -function resolvers.expandedpathlistfromvariable(str) +resolvers.expandedpathlist=expandedpathlist +resolvers.unexpandedpathlist=unexpandedpathlist +function resolvers.cleanpathlist(str) + local t=expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(cleanpath(t[i])) + end + end + return t +end +function resolvers.expandpath(str) + return joinpath(expandedpathlist(str)) +end +local function expandedpathlistfromvariable(str) str=lpegmatch(dollarstripper,str) local tmp=resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp~="" and tmp or str) + return expandedpathlist(tmp~="" and tmp or str) end function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(expandedpathlistfromvariable(str)) end +resolvers.expandedpathlistfromvariable=expandedpathlistfromvariable function resolvers.cleanedpathlist(v) - local t=resolvers.expandedpathlist(v) + local t=expandedpathlist(v) for i=1,#t do - t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + t[i]=resolveprefix(cleanpath(t[i])) end return t end function resolvers.expandbraces(str) - local pth=expandedpathfromlist(resolvers.splitpath(str)) + local pth=expandedpathfromlist(splitpath(str)) return joinpath(pth) end function resolvers.registerfilehash(name,content,someerror) + local files=instance.files if content then - instance.files[name]=content + files[name]=content else - instance.files[name]={} + files[name]={} if somerror==true then instance.loaderror=someerror end @@ -22926,10 +23001,11 @@ local function collect_files(names) pathname="/"..pathname.."$" end local hashes=instance.hashes + local files=instance.files for h=1,#hashes do local hash=hashes[h] local hashname=hash.name - local content=hashname and instance.files[hashname] + local content=hashname and files[hashname] if content then if trace_details then report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) @@ -22999,7 +23075,6 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if filesuffix=="" or not suffixmap[filesuffix] then local defaultsuffixes=resolvers.defaultsuffixes - local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname @@ -23009,7 +23084,7 @@ local function find_analyze(filename,askedformat,allresults) end end else - filetype=resolvers.formatofsuffix(filename) + filetype=formatofsuffix(filename) if trace_locating then report_resolving("using suffix based filetype %a",filetype) end @@ -23070,11 +23145,11 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] if format_suffixes then for i=1,#format_suffixes do - local s=format_suffixes[i] - forcedname=filename.."."..s + local suffix=format_suffixes[i] + forcedname=filename.."."..suffix if isreadable(forcedname) then if trace_locating then - report_resolving("no suffix, forcing format filetype %a",s) + report_resolving("no suffix, forcing format filetype %a",suffix) end return "qualified",{ forcedname } end @@ -23087,7 +23162,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local savedformat=askedformat local format=savedformat or "" if format=="" then - askedformat=resolvers.formatofsuffix(suffix) + askedformat=formatofsuffix(suffix) end if not format then askedformat="othertextfiles" @@ -23126,7 +23201,7 @@ local function check_subpath(fname) end local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local pathlist=expandedpathlist(typespec,filetype and usertypes[filetype]) local entry={} if pathlist and #pathlist>0 then for k=1,#pathlist do @@ -23137,7 +23212,7 @@ local function makepathlist(list,filetype) local expression=makepathexpression(pathname) local barename=gsub(pathname,"/+$","") barename=resolveprefix(barename) - local scheme=url.hasscheme(barename) + local scheme=urlhasscheme(barename) local schemename=gsub(barename,"%.%*$",'') entry[k]={ path=path, @@ -23342,7 +23417,9 @@ collect_instance_files=function(filename,askedformat,allresults) { find_onpath (filename,filetype,wantedfiles,true) }, { find_otherwise(filename,filetype,wantedfiles,true) }, } - local result,status,done={},{},{} + local result={} + local status={} + local done={} for k,r in next,results do local method,list=r[1],r[2] if method and list then @@ -23413,6 +23490,9 @@ local function findfiles(filename,filetype,allresults) if not filename or filename=="" then return {} end + if allresults==nil then + allresults=true + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -23422,27 +23502,25 @@ local function findfiles(filename,filetype,allresults) end return result or {},status end -function resolvers.findfiles(filename,filetype) - if not filename or filename=="" then - return "" - else - return findfiles(filename,filetype,true) - end -end -function resolvers.findfile(filename,filetype) +local function findfile(filename,filetype) if not filename or filename=="" then return "" else return findfiles(filename,filetype,false)[1] or "" end end +resolvers.findfiles=findfiles +resolvers.findfile=findfile +resolvers.find_file=findfile +resolvers.find_files=findfiles function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) + local hashes=instance.hashes + local files=instance.files local base=filebasename(filename) local result={} - local hashes=instance.hashes local function okay(hash,path,name) local found=methodhandler('concatinators',hash.type,hash.name,path,name) if found and found~="" then @@ -23452,7 +23530,7 @@ local function findgivenfiles(filename,allresults) end for k=1,#hashes do local hash=hashes[k] - local content=instance.files[hash.name] + local content=files[hash.name] if content then local path,name=lookup(content,base) if not path then @@ -23484,14 +23562,14 @@ function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end local function findwildcardfiles(filename,allresults,result) + local files=instance.files + local hashes=instance.hashes local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files=instance.files if find(name,"*",1,true) then - local hashes=instance.hashes local function okay(found,path,base,hashname,hashtype) if find(found,path) then local full=methodhandler('concatinators',hashtype,hashname,found,base) @@ -23531,7 +23609,6 @@ local function findwildcardfiles(filename,allresults,result) end end end - local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] local hashname=hash.name @@ -23561,13 +23638,21 @@ end function resolvers.findwildcardfile(filename) return findwildcardfiles(filename,false)[1] or "" end -function resolvers.automount() -end -function resolvers.starttiming() - statistics.starttiming(instance) +do + local starttiming=statistics.starttiming + local stoptiming=statistics.stoptiming + local elapsedtime=statistics.elapsedtime + function resolvers.starttiming() + starttiming(instance) + end + function resolvers.stoptiming() + stoptiming(instance) + end + function resolvers.loadtime() + return elapsedtime(instance) + end end -function resolvers.stoptiming() - statistics.stoptiming(instance) +function resolvers.automount() end function resolvers.load(option) resolvers.starttiming() @@ -23581,9 +23666,6 @@ function resolvers.load(option) local files=instance.files return files and next(files) and true end -function resolvers.loadtime() - return statistics.elapsedtime(instance) -end local function report(str) if trace_locating then report_resolving(str) @@ -23613,7 +23695,7 @@ function resolvers.dowithfilesandreport(command,files,...) end end function resolvers.showpath(str) - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) + return joinpath(expandedpathlist(resolvers.formatofvariable(str))) end function resolvers.registerfile(files,name,path) if files[name] then @@ -23627,7 +23709,7 @@ function resolvers.registerfile(files,name,path) end end function resolvers.dowithpath(name,func) - local pathlist=resolvers.expandedpathlist(name) + local pathlist=expandedpathlist(name) for i=1,#pathlist do func("^"..cleanpath(pathlist[i])) end @@ -23641,7 +23723,7 @@ function resolvers.locateformat(name) local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then - fmtname=resolvers.findfile(fullname) + fmtname=findfile(fullname) fmtname=cleanpath(fmtname) end if fmtname~="" then @@ -23660,7 +23742,7 @@ function resolvers.locateformat(name) return nil,nil end function resolvers.booleanvariable(str,default) - local b=resolvers.expansion(str) + local b=expansion(str) if b=="" then return default else @@ -23670,6 +23752,7 @@ function resolvers.booleanvariable(str,default) end function resolvers.dowithfilesintree(pattern,handle,before,after) local hashes=instance.hashes + local files=instance.files for i=1,#hashes do local hash=hashes[i] local blobtype=hash.type @@ -23681,7 +23764,7 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) if before then before(blobtype,blobpath,pattern) end - for path,name in filtered(instance.files[blobpath],pattern) do + for path,name in filtered(files[blobpath],pattern) do if type(path)=="string" then checked=checked+1 if handle(blobtype,blobpath,path,name) then @@ -23702,10 +23785,6 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -local obsolete=resolvers.obsolete or {} -resolvers.obsolete=obsolete -resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile -resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles function resolvers.knownvariables(pattern) if instance then local environment=instance.environment @@ -23739,7 +23818,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4854, stripped down to: 2889 +-- original size: 5088, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -23748,6 +23827,7 @@ if not modules then modules={} end modules ['data-pre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local insert,remove=table.insert,table.remove local resolvers=resolvers local prefixes=resolvers.prefixes local cleanpath=resolvers.cleanpath @@ -23825,8 +23905,9 @@ prefixes.kpse=prefixes.locate prefixes.full=prefixes.locate prefixes.file=prefixes.filename prefixes.path=prefixes.pathname +local inputstack={} +local stackpath=resolvers.stackpath local function toppath() - local inputstack=resolvers.inputstack if not inputstack then return "." end @@ -23838,15 +23919,23 @@ local function toppath() end end local function jobpath() - local path=resolvers.stackpath() + local path=stackpath() if not path or path=="" then return "." else return path end end +local function pushinputname(name) + insert(inputstack,name) +end +local function popinputname(name) + return remove(inputstack) +end resolvers.toppath=toppath resolvers.jobpath=jobpath +resolvers.pushinputname=pushinputname +resolvers.popinputname=popinputname prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end resolvers.setdynamic("toppath") @@ -23889,7 +23978,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 470 +-- original size: 551, stripped down to: 470 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -23912,7 +24001,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3170 +-- original size: 4365, stripped down to: 3588 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -23921,39 +24010,46 @@ if not modules then modules={} end modules ['data-fil']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local ioopen=io.open +local isdir=lfs.isdir local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers local resolveprefix=resolvers.resolve -local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers -local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local findfile=resolvers.findfile +local scanfiles=resolvers.scanfiles +local registerfilehash=resolvers.registerfilehash +local appendhash=resolvers.appendhash +local loadcachecontent=caches.loadcontent local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check -function locators.file(specification) +function resolvers.locators.file(specification) local filename=specification.filename local realname=resolveprefix(filename) - if realname and realname~='' and lfs.isdir(realname) then + if realname and realname~='' and isdir(realname) then if trace_locating then report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',filename,true) + appendhash('file',filename,true) elseif trace_locating then report_files("file locator %a not found",filename) end end -function hashers.file(specification) +function resolvers.hashers.file(specification) local pathname=specification.filename - local content=caches.loadcontent(pathname,'files') - resolvers.registerfilehash(pathname,content,content==nil) + local content=loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) end -function generators.file(specification) +function resolvers.generators.file(specification) local pathname=specification.filename - local content=resolvers.scanfiles(pathname,false,true) - resolvers.registerfilehash(pathname,content,true) + local content=scanfiles(pathname,false,true) + registerfilehash(pathname,content,true) end -concatinators.file=file.join +resolvers.concatinators.file=file.join +local finders=resolvers.finders +local notfound=finders.notfound function finders.file(specification,filetype) local filename=specification.filename - local foundname=resolvers.findfile(filename,filetype) + local foundname=findfile(filename,filetype) if foundname and foundname~="" then if trace_locating then report_files("file finder: %a found",filename) @@ -23963,37 +24059,55 @@ function finders.file(specification,filetype) if trace_locating then report_files("file finder: %a not found",filename) end - return finders.notfound() + return notfound() end end -function openers.helpers.textopener(tag,filename,f) +local openers=resolvers.openers +local notfound=openers.notfound +local overloaded=false +local function textopener(tag,filename,f) return { - reader=function() return f:read () end, - close=function() logs.show_close(filename) return f:close() end, + reader=function() return f:read () end, + close=function() return f:close() end, } end +function openers.helpers.textopener(...) + return textopener(...) +end +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded=true + textopener=opener + end +end function openers.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"r") + local f=ioopen(filename,"r") if f then if trace_locating then report_files("file opener: %a opened",filename) end - return openers.helpers.textopener("file",filename,f) + return textopener("file",filename,f) end end if trace_locating then report_files("file opener: %a not found",filename) end - return openers.notfound() + return notfound() end +local loaders=resolvers.loaders +local notfound=loaders.notfound function loaders.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"rb") + local f=ioopen(filename,"rb") if f then - logs.show_load(filename) if trace_locating then report_files("file loader: %a loaded",filename) end @@ -24010,7 +24124,7 @@ function loaders.file(specification,filetype) if trace_locating then report_files("file loader: %a not found",filename) end - return loaders.notfound() + return notfound() end @@ -24020,7 +24134,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5029, stripped down to: 3432 +-- original size: 5388, stripped down to: 3685 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -24029,6 +24143,7 @@ if not modules then modules={} end modules ['data-con']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local setmetatable=setmetatable local format,lower,gsub=string.format,string.lower,string.gsub local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) @@ -24036,16 +24151,21 @@ local trace_storage=false trackers.register("resolvers.storage",function(v) tra containers=containers or {} local containers=containers containers.usecache=true +local getwritablepath=caches.getwritablepath +local getreadablepaths=caches.getreadablepaths +local cacheiswritable=caches.is_writable +local loaddatafromcache=caches.loaddata +local savedataincache=caches.savedata local report_containers=logs.reporter("resolvers","containers") local allocated={} local mt={ __index=function(t,k) if k=="writable" then - local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable=getwritablepath(t.category,t.subcategory) or { "." } t.writable=writable return writable elseif k=="readables" then - local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables=getreadablepaths(t.category,t.subcategory) or { "." } t.readables=readables return readables end @@ -24076,7 +24196,7 @@ function containers.define(category,subcategory,version,enabled) end end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable,name) + return container.enabled and caches and cacheiswritable(container.writable,name) end function containers.is_valid(container,name) if name and name~="" then @@ -24090,7 +24210,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name,container.writable) + stored=loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -24106,17 +24226,20 @@ function containers.read(container,name) end return stored end -function containers.write(container,name,data) +function containers.write(container,name,data,fast) if data then data.cache_version=container.version if container.enabled and caches then - local unique,shared=data.unique,data.shared - data.unique,data.shared=nil,nil - caches.savedata(container.writable,name,data) + local unique=data.unique + local shared=data.shared + data.unique=nil + data.shared=nil + savedataincache(container.writable,name,data,fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique,data.shared=unique,shared + data.unique=unique + data.shared=shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) @@ -24139,7 +24262,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 5203, stripped down to: 3765 +-- original size: 5790, stripped down to: 2910 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -24148,40 +24271,11 @@ if not modules then modules={} end modules ['data-use']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local format=string.format local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_mounts=logs.reporter("resolvers","mounts") local resolvers=resolvers -resolvers.automounted=resolvers.automounted or {} -function resolvers.automount(usecache) - local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths==0) and usecache then - mountpaths=caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths>0 then - resolvers.starttiming() - for k=1,#mountpaths do - local root=mountpaths[k] - local f=io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %a",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() - end - end - resolvers.stoptiming() - end -end +local findfile=resolvers.findfile statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) @@ -24191,7 +24285,7 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local luvdata={ enginebanner=enginebanner, formatbanner=formatbanner, - sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcehash=md5.hex(io.loaddata(findfile(sourcefile)) or "unknown"), sourcefile=sourcefile, luaversion=LUAVERSION, formatid=LUATEXFORMATID, @@ -24215,7 +24309,7 @@ function statistics.checkfmtstatus(texname) if lfs.isfile(luvname) then local luv=dofile(luvname) if luv and luv.sourcefile then - local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local sourcehash=md5.hex(io.loaddata(findfile(luv.sourcefile)) or "unknown") local luvbanner=luv.enginebanner or "?" if luvbanner~=enginebanner then return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) @@ -24256,7 +24350,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 10263, stripped down to: 7556 +-- original size: 10725, stripped down to: 7949 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -24269,6 +24363,14 @@ local format,find,match=string.format,string.find,string.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_zip=logs.reporter("resolvers","zip") local resolvers=resolvers +local findfile=resolvers.findfile +local registerfile=resolvers.registerfile +local splitmethod=resolvers.splitmethod +local prependhash=resolvers.prependhash +local starttiming=resolvers.starttiming +local extendtexmf=resolvers.extendtexmfvariable +local stoptiming=resolvers.stoptiming +local urlquery=url.query zip=zip or {} local zip=zip local archives=zip.archives or {} @@ -24283,8 +24385,9 @@ if zipfiles then closezip=zipfiles.close validfile=zipfiles.found wholefile=zipfiles.unzip + local listzip=zipfiles.list traversezip=function(zfile) - return ipairs(zipfiles.list(zfile)) + return ipairs(listzip(zfile)) end local streams=utilities.streams local openstream=streams.open @@ -24341,28 +24444,30 @@ local function validzip(str) return str end end -function zip.openarchive(name) +local function openarchive(name) if not name or name=="" then return nil else local arch=archives[name] if not arch then - local full=resolvers.findfile(name) or "" + local full=findfile(name) or "" arch=full~="" and openzip(full) or false archives[name]=arch end return arch end end -function zip.closearchive(name) +local function closearchive(name) if not name or (name=="" and archives[name]) then closezip(archives[name]) archives[name]=nil end end +zip.openarchive=openarchive +zip.closearchive=closearchive function resolvers.locators.zip(specification) local archive=specification.filename - local zipfile=archive and archive~="" and zip.openarchive(archive) + local zipfile=archive and archive~="" and openarchive(archive) if trace_locating then if zipfile then report_zip("locator: archive %a found",archive) @@ -24371,13 +24476,6 @@ function resolvers.locators.zip(specification) end end end -function resolvers.hashers.zip(specification) - local archive=specification.filename - if trace_locating then - report_zip("loading file %a",archive) - end - resolvers.usezipfile(specification.original) -end function resolvers.concatinators.zip(zipfile,path,name) if not path or path=="" then return format('%s?name=%s',zipfile,name) @@ -24385,14 +24483,16 @@ function resolvers.concatinators.zip(zipfile,path,name) return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.finders.zip(specification) +local finders=resolvers.finders +local notfound=finders.notfound +function finders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("finder: archive %a found",archive) @@ -24413,16 +24513,19 @@ function resolvers.finders.zip(specification) if trace_locating then report_zip("finder: %a not found",original) end - return resolvers.finders.notfound() + return notfound() end -function resolvers.openers.zip(specification) +local openers=resolvers.openers +local notfound=openers.notfound +local textopener=openers.helpers.textopener +function openers.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("opener; archive %a opened",archive) @@ -24432,7 +24535,7 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: file %a found",queryname) end - return resolvers.openers.helpers.textopener('zip',original,handle) + return textopener('zip',original,handle) elseif trace_locating then report_zip("opener: file %a not found",queryname) end @@ -24444,23 +24547,24 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -function resolvers.loaders.zip(specification) +local loaders=resolvers.loaders +local notfound=loaders.notfound +function loaders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("loader: archive %a opened",archive) end local data=wholefile(zfile,queryname) if data then - logs.show_load(original) if trace_locating then report_zip("loader; file %a loaded",original) end @@ -24476,47 +24580,24 @@ function resolvers.loaders.zip(specification) if trace_locating then report_zip("loader: %a not found",original) end - return resolvers.openers.notfound() -end -function resolvers.usezipfile(archive) - local specification=resolvers.splitmethod(archive) - local archive=specification.filename - if archive and not registeredfiles[archive] then - local z=zip.openarchive(archive) - if z then - local tree=url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering: archive %a",archive) - end - resolvers.starttiming() - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) - registeredfiles[archive]=z - resolvers.registerfilehash(archive,resolvers.registerzipfile(z,tree)) - resolvers.stoptiming() - elseif trace_locating then - report_zip("registering: unknown archive %a",archive) - end - elseif trace_locating then - report_zip("registering: archive %a not found",archive) - end + return notfound() end -function resolvers.registerzipfile(z,tree) +local function registerzipfile(z,tree) local names={} local files={} local remap={} local n=0 local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) - local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end + starttiming() for i in traversezip(z) do local filename=i.filename local path,name=match(filename,filter) if not path then n=n+1 - register(names,filename,"") + registerfile(names,filename,"") local usedname=lower(filename) files[usedname]="" if usedname~=filename then @@ -24533,12 +24614,43 @@ function resolvers.registerzipfile(z,tree) else end end + stoptiming() report_zip("registering: %s files registered",n) return { files=files, remap=remap, } end +local function usezipfile(archive) + local specification=splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=openarchive(archive) + if z then + local tree=urlquery(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + prependhash('zip',archive) + extendtexmf(archive) + registeredfiles[archive]=z + registerfilehash(archive,registerzipfile(z,tree)) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +resolvers.usezipfile=usezipfile +resolvers.registerzipfile=registerzipfile +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + usezipfile(specification.original) +end end -- of closure @@ -24547,7 +24659,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8478, stripped down to: 5223 +-- original size: 9444, stripped down to: 5650 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -24563,173 +24675,204 @@ local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local resolveprefix=resolvers.resolve -local notfound=resolvers.finders.notfound -local lookup=resolvers.get_from_content -local collectors={} -local found={} -function resolvers.finders.tree(specification) - local spec=specification.filename - local okay=found[spec] - if okay==nil then - if spec~="" then - local path=dirname(spec) - local name=basename(spec) - if path=="" then - path="." - end - local names=collectors[path] - if not names then - local pattern=find(path,"/%*+$") and path or (path.."/*") - names=globdir(pattern) - collectors[path]=names - end - local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for i=1,#names do - local fullname=names[i] - if find(fullname,pattern) then - found[spec]=fullname - return fullname - end - end - local pattern=lower(pattern) - for i=1,#names do - local fullname=lower(names[i]) - if find(fullname,pattern) then - if isfile(fullname) then +local finders=resolvers.finders +local openers=resolvers.openers +local loaders=resolvers.loaders +local locators=resolvers.locators +local hashers=resolvers.hashers +local generators=resolvers.generators +do + local collectors={} + local found={} + local notfound=finders.notfound + function finders.tree(specification) + local spec=specification.filename + local okay=found[spec] + if okay==nil then + if spec~="" then + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then found[spec]=fullname return fullname - else - break + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end + okay=notfound() + found[spec]=okay end - okay=notfound() - found[spec]=okay + return okay end - return okay end -function resolvers.locators.tree(specification) - local name=specification.filename - local realname=resolveprefix(name) - if realname and realname~='' and isdir(realname) then - if trace_locating then - report_trees("locator %a found",realname) +do + local resolveprefix=resolvers.resolve + local appendhash=resolvers.appendhash + local function dolocate(specification) + local name=specification.filename + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) end - resolvers.appendhash('tree',name,false) - elseif trace_locating then - report_trees("locator %a not found",name) end + locators.tree=dolocate + locators.dirlist=dolocate + locators.dirfile=dolocate end -function resolvers.hashers.tree(specification) - local name=specification.filename - if trace_locating then - report_trees("analyzing %a",name) - end - resolvers.methodhandler("hashers",name) - resolvers.generators.file(specification) -end -local collectors={} -local splitter=lpeg.splitat("/**/") -local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } -table.setmetatableindex(collectors,function(t,k) - local rootname=lpegmatch(stripper,k) - local dataname=joinname(rootname,"dirlist") - local content=caches.loadcontent(dataname,"files",dataname) - if not content then - content=resolvers.scanfiles(rootname,nil,nil,false,true) - caches.savecontent(dataname,"files",content,dataname) - end - t[k]=content - return content -end) -local function checked(root,p,n) - if p then - if type(p)=="table" then - for i=1,#p do - local fullname=joinname(root,p[i],n) +do + local filegenerator=generators.file + generators.dirlist=filegenerator + generators.dirfile=filegenerator +end +do + local filegenerator=generators.file + local methodhandler=resolvers.methodhandler + local function dohash(specification) + local name=specification.filename + if trace_locating then + report_trees("analyzing %a",name) + end + methodhandler("hashers",name) + filegenerator(specification) + end + hashers.tree=dohash + hashers.dirlist=dohash + hashers.dirfile=dohash +end +local resolve do + local collectors={} + local splitter=lpeg.splitat("/**/") + local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } + local loadcontent=caches.loadcontent + local savecontent=caches.savecontent + local notfound=finders.notfound + local scanfiles=resolvers.scanfiles + local lookup=resolvers.get_from_content + table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=loadcontent(dataname,"files",dataname) + if not content then + content=scanfiles(rootname,nil,nil,false,true) + savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content + end) + local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) if isfile(fullname) then return fullname end end - else - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname - end end + return notfound() end - return notfound() -end -local function resolve(specification) - local filename=specification.filename - if filename~="" then - local root,rest=lpegmatch(splitter,filename) - if root and rest then - local path,name=dirname(rest),basename(rest) - if name~=rest then - local content=collectors[root] - local p,n=lookup(content,name) - if not p then - return notfound() - end - local pattern=".*/"..path.."$" - local istable=type(p)=="table" - if istable then - for i=1,#p do - local pi=p[i] - if pi==path or find(pi,pattern) then - local fullname=joinname(root,pi,n) - if isfile(fullname) then - return fullname + resolve=function(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end end end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end end - elseif p==path or find(p,pattern) then - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() end end - local queries=specification.queries - if queries and queries.option=="fileonly" then - return checked(root,p,n) - else - return notfound() - end + end + local path=dirname(filename) + local name=basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) end end - local path,name=dirname(filename),basename(filename) - local root=lpegmatch(stripper,path) - local content=collectors[path] - local p,n=lookup(content,name) - if p then - return checked(root,p,n) + return notfound() + end + finders.dirlist=resolve + function finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } end + return resolve(specification) end - return notfound() end -resolvers.finders .dirlist=resolve -resolvers.locators .dirlist=resolvers.locators .tree -resolvers.hashers .dirlist=resolvers.hashers .tree -resolvers.generators.dirlist=resolvers.generators.file -resolvers.openers .dirlist=resolvers.openers .file -resolvers.loaders .dirlist=resolvers.loaders .file -function resolvers.finders.dirfile(specification) - local queries=specification.queries - if queries then - queries.option="fileonly" - else - specification.queries={ option="fileonly" } - end - return resolve(specification) +do + local fileopener=openers.file + local fileloader=loaders.file + openers.dirlist=fileopener + loaders.dirlist=fileloader + openers.dirfile=fileopener + loaders.dirfile=fileloader end -resolvers.locators .dirfile=resolvers.locators .dirlist -resolvers.hashers .dirfile=resolvers.hashers .dirlist -resolvers.generators.dirfile=resolvers.generators.dirlist -resolvers.openers .dirfile=resolvers.openers .dirlist -resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -24738,7 +24881,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6757, stripped down to: 5272 +-- original size: 6945, stripped down to: 5408 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -24748,8 +24891,11 @@ if not modules then modules={} end modules ['data-sch']={ license="see context related readme files" } local load,tonumber=load,tonumber -local gsub,concat,format=string.gsub,table.concat,string.format +local gsub,format=string.gsub,string.format +local sortedhash,concat=table.sortedhash,table.concat local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local addsuffix,suffix,splitbase=file.addsuffix,file.suffix,file.splitbase +local md5hex=md5.hex local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) local report_schemes=logs.reporter("resolvers","schemes") local http=require("socket.http") @@ -24766,7 +24912,7 @@ function cleaners.none(specification) return specification.original end function cleaners.strip(specification) - local path,name=file.splitbase(specification.original) + local path,name=splitbase(specification.original) if path=="" then return (gsub(name,"[^%a%d%.]+","-")) else @@ -24774,7 +24920,7 @@ function cleaners.strip(specification) end end function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return addsuffix(md5hex(specification.original),suffix(specification.path)) end local cleaner=cleaners.strip directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) @@ -24865,10 +25011,10 @@ end schemes.install=install local function http_handler(specification,cachename) local tempname=cachename..".tmp" - local f=io.open(tempname,"wb") + local handle=io.open(tempname,"wb") local status,message=http.request { url=specification.original, - sink=ltn12.sink.file(f) + sink=ltn12.sink.file(handle) } if not status then os.remove(tempname) @@ -24883,13 +25029,13 @@ install('https') install('ftp') statistics.register("scheme handling time",function() local l,r,nl,nr={},{},0,0 - for k,v in table.sortedhash(loaded) do + for k,v in sortedhash(loaded) do if v>0 then nl=nl+1 l[nl]=k..":"..v end end - for k,v in table.sortedhash(reused) do + for k,v in sortedhash(reused) do if v>0 then nr=nr+1 r[nr]=k..":"..v @@ -24897,10 +25043,10 @@ statistics.register("scheme handling time",function() end local n=nl+nr if n>0 then - l=nl>0 and concat(l) or "none" - r=nr>0 and concat(r) or "none" + if nl==0 then l={ "none" } end + if nr==0 then r={ "none" } end return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes),n,threshold,l,r) + statistics.elapsedtime(schemes),n,threshold,concat(l," "),concat(l," ")) else return nil end @@ -24933,7 +25079,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4207, stripped down to: 3041 +-- original size: 4227, stripped down to: 3049 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -24943,8 +25089,7 @@ if not modules then modules={} end modules ['data-lua']={ license="see context related readme files" } local package,lpeg=package,lpeg -local gsub=string.gsub -local concat=table.concat +local loadfile=loadfile local addsuffix=file.addsuffix local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match local luasuffixes={ 'tex','lua' } @@ -24955,6 +25100,8 @@ local helpers=package.helpers or {} local methods=helpers.methods or {} local resolvers=resolvers local resolveprefix=resolvers.resolve +local expandedpaths=resolvers.expandedpathlistfromvariable +local findfile=resolvers.findfile helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) @@ -24983,7 +25130,7 @@ local function getluaformatpaths() if not luaformatpaths then luaformatpaths={} for i=1,#luaformats do - registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + registerpath("lua format","lua",luaformatpaths,expandedpaths(luaformats[i])) end end return luaformatpaths @@ -24992,7 +25139,7 @@ local function getlibformatpaths() if not libformatpaths then libformatpaths={} for i=1,#libformats do - registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + registerpath("lib format","lib",libformatpaths,expandedpaths(libformats[i])) end end return libformatpaths @@ -25002,7 +25149,7 @@ local function loadedbyformat(name,rawname,suffixes,islib,what) local report=helpers.report for i=1,#suffixes do local format=suffixes[i] - local resolved=resolvers.findfile(name,format) or "" + local resolved=findfile(name,format) or "" if trace then report("%s format, identifying %a using format %a",what,name,format) end @@ -25040,7 +25187,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2452, stripped down to: 1877 +-- original size: 2610, stripped down to: 2019 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -25051,24 +25198,28 @@ if not modules then modules={} end modules ['data-aux']={ } local find=string.find local type,next=type,next +local addsuffix,removesuffix=file.addsuffix,file.removesuffix +local loaddata,savedata=io.loaddata,io.savedata local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers +local cleanpath=resolvers.cleanpath +local findfiles=resolvers.findfiles local report_scripts=logs.reporter("resolvers","scripts") function resolvers.updatescript(oldname,newname) local scriptpath="context/lua" - newname=file.addsuffix(newname,"lua") - local oldscript=resolvers.cleanpath(oldname) + local oldscript=cleanpath(oldname) + local newname=addsuffix(newname,"lua") + local newscripts=findfiles(newname) or {} if trace_locating then report_scripts("to be replaced old script %a",oldscript) end - local newscripts=resolvers.findfiles(newname) or {} if #newscripts==0 then if trace_locating then report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript=resolvers.cleanpath(newscripts[i]) + local newscript=cleanpath(newscripts[i]) if trace_locating then report_scripts("checking new script %a",newscript) end @@ -25080,17 +25231,17 @@ function resolvers.updatescript(oldname,newname) if trace_locating then report_scripts("new script should come from %a",scriptpath) end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + elseif not (find(oldscript,removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then report_scripts("invalid new script name") end else - local newdata=io.loaddata(newscript) + local newdata=loaddata(newscript) if newdata then if trace_locating then report_scripts("old script content replaced by new content: %s",oldscript) end - io.savedata(oldscript,newdata) + savedata(oldscript,newdata) break elseif trace_locating then report_scripts("unable to load new script") @@ -25163,7 +25314,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 1823, stripped down to: 1542 +-- original size: 1912, stripped down to: 1591 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -25173,40 +25324,39 @@ if not modules then modules={} end modules ['data-lst']={ license="see context related readme files" } local type=type -local concat,sortedhash=table.concat,table.sortedhash +local sortedhash=table.sortedhash +local isdir=lfs.isdir local resolvers=resolvers local listers=resolvers.listers or {} resolvers.listers=listers local resolveprefix=resolvers.resolve +local configurationfiles=resolvers.configurationfiles +local expandedpathfromlist=resolvers.expandedpathfromlist +local splitpath=resolvers.splitpath +local knownvariables=resolvers.knownvariables local report_lists=logs.reporter("resolvers","lists") local report_resolved=logs.reporter("system","resolved") -local function tabstr(str) - if type(str)=='table' then - return concat(str," | ") - else - return str - end -end function listers.variables(pattern) local result=resolvers.knownvariables(pattern) + local unset={ "unset" } for key,value in sortedhash(result) do report_lists(key) - report_lists(" env: %s",tabstr(value.environment or "unset")) - report_lists(" var: %s",tabstr(value.variable or "unset")) - report_lists(" exp: %s",tabstr(value.expansion or "unset")) - report_lists(" res: %s",tabstr(value.resolved or "unset")) + report_lists(" env: % | t",value.environment or unset) + report_lists(" var: % | t",value.variable or unset) + report_lists(" exp: % | t",value.expansion or unset) + report_lists(" res: % | t",value.resolved or unset) end end function listers.configurations() - local configurations=resolvers.configurationfiles() + local configurations=configurationfiles() for i=1,#configurations do report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") - local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + local list=expandedpathfromlist(splitpath(resolvers.luacnfspec)) for i=1,#list do local li=resolveprefix(list[i]) - if lfs.isdir(li) then + if isdir(li) then report_resolved("path - %s",li) else report_resolved("path + %s",li) @@ -25978,8 +26128,8 @@ end -- of closure -- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 1036669 --- stripped bytes : 410250 +-- original bytes : 1040817 +-- stripped bytes : 412122 -- end library merge diff --git a/scripts/context/stubs/win64/mtxrun.lua b/scripts/context/stubs/win64/mtxrun.lua index 36a7bbcd2..bf87526f2 100644 --- a/scripts/context/stubs/win64/mtxrun.lua +++ b/scripts/context/stubs/win64/mtxrun.lua @@ -324,7 +324,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-macro"] = package.loaded["l-macro"] or true --- original size: 10131, stripped down to: 5991 +-- original size: 10130, stripped down to: 5990 if not modules then modules={} end modules ['l-macros']={ version=1.001, @@ -549,7 +549,7 @@ end macros.loaded=loaded function required(name,trace) local filename=file.addsuffix(name,"lua") - local fullname=resolvers and resolvers.find_file(filename) or filename + local fullname=resolvers and resolvers.findfile(filename) or filename if not fullname or fullname=="" then return false end @@ -12608,7 +12608,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 13348, stripped down to: 8838 +-- original size: 13394, stripped down to: 8882 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -12618,8 +12618,9 @@ if not modules then modules={} end modules ['trac-set']={ license="see context related readme files" } local type,next,tostring,tonumber=type,next,tostring,tonumber +local print=print local concat,sortedhash=table.concat,table.sortedhash -local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local formatters,find,lower,gsub,topattern=string.formatters,string.find,string.lower,string.gsub,string.topattern local is_boolean=string.is_boolean local settings_to_hash=utilities.parsers.settings_to_hash local allocate=utilities.storage.allocate @@ -12826,8 +12827,8 @@ function setters.show(t) end end local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show -function setters.report(setter,...) - print(format("%-15s : %s\n",setter.name,format(...))) +function setters.report(setter,fmt,...) + print(formatters["%-15s : %s\n"](setter.name,formatters[fmt](...))) end local function default(setter,name) local d=setter.data[name] @@ -12950,7 +12951,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 33803, stripped down to: 21818 +-- original size: 32997, stripped down to: 21715 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -13613,12 +13614,6 @@ end local nesting=0 local verbose=false local hasscheme=url.hasscheme -function logs.show_open(name) -end -function logs.show_close(name) -end -function logs.show_load(name) -end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple @@ -20686,7 +20681,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 10987, stripped down to: 7056 +-- original size: 11019, stripped down to: 7086 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -20699,6 +20694,7 @@ local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local sortedpairs=table.sortedpairs local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) @@ -20909,7 +20905,7 @@ if ostype=="unix" then rawset(t,k,v) end local colon=P(":") - for k,v in table.sortedpairs(prefixes) do + for k,v in sortedpairs(prefixes) do if p then p=P(k)+p else @@ -20936,7 +20932,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18154, stripped down to: 10416 +-- original size: 18173, stripped down to: 10432 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -21220,7 +21216,7 @@ local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) scancache[sub(full,1,-2)]=files return files,remap,n,m,r end -function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) +local function scanfiles(path,branch,usecache,onlyonce,tolerant) local realpath=resolveprefix(path) if usecache then local content=fullcache[realpath] @@ -21275,8 +21271,9 @@ function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) statistics.stoptiming(timer) return content end +resolvers.scanfiles=scanfiles function resolvers.simplescanfiles(path,branch,usecache) - return resolvers.scanfiles(path,branch,usecache,true,true) + return scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -21343,7 +21340,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9384, stripped down to: 6333 +-- original size: 9400, stripped down to: 6347 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -21353,7 +21350,7 @@ if not modules then modules={} end modules ['data-env']={ license="see context related readme files", } local lower,gsub=string.lower,string.gsub -local next=next +local next,rawget=next,rawget local resolvers=resolvers local allocate=utilities.storage.allocate local setmetatableindex=table.setmetatableindex @@ -21628,7 +21625,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16387, stripped down to: 11132 +-- original size: 16070, stripped down to: 11353 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -21637,49 +21634,83 @@ if not modules then modules={} end modules ['data-tmp']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat=table.concat -local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile -local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable -local formatters=string.formatters local next,type=next,type +local pcall,loadfile,collectgarbage=pcall,loadfile,collectgarbage +local format,lower,gsub=string.format,string.lower,string.gsub +local concat,serialize,fastserialize,serializetofile=table.concat,table.serialize,table.fastserialize,table.tofile +local mkdirs,expanddirname,isdir,isfile=dir.mkdirs,dir.expandname,lfs.isdir,lfs.isfile +local is_writable,is_readable=file.is_writable,file.is_readable +local collapsepath,joinfile,addsuffix,dirname=file.collapsepath,file.join,file.addsuffix,file.dirname +local savedata=file.savedata +local formatters=string.formatters +local osexit,osdate,osuuid=os.exit,os.date,os.uuid +local removefile=os.remove +local md5hex=md5.hex local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers local cleanpath=resolvers.cleanpath -local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) -local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) -local compile=utilities.lua.compile -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup==nil then cleanup=directive_cleanup end - if strip==nil then strip=directive_strip end - return compile(luafile,lucfile,cleanup,strip) +local resolvepath=resolvers.resolve +local luautilities=utilities.lua +do + local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) + local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) + local compilelua=luautilities.compile + function luautilities.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compilelua(luafile,lucfile,cleanup,strip) + end end caches=caches or {} local caches=caches -local luasuffixes=utilities.lua.suffixes -caches.base=caches.base or "luatex-cache" -caches.more=caches.more or "context" -caches.direct=false -caches.tree=false -caches.force=true -caches.ask=false -caches.relocate=false +local writable=nil +local readables={} +local usedreadables={} +local compilelua=luautilities.compile +local luasuffixes=luautilities.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } -directives.register("system.caches.fast",function(v) caches.fast=true end) -directives.register("system.caches.direct",function(v) caches.direct=true end) -local writable,readables,usedreadables=nil,{},{} +local direct_cache=false +local fast_cache=false +local cache_tree=false +directives.register("system.caches.direct",function(v) direct_cache=true end) +directives.register("system.caches.fast",function(v) fast_cache=true end) +local function configfiles() + return concat(resolvers.configurationfiles(),";") +end +local function hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +local function treehash() + local tree=configfiles() + if not tree or tree=="" then + return false + else + return hashed(tree) + end +end +caches.hashed=hashed +caches.treehash=treehash +caches.configfiles=configfiles local function identify() local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) - cachepath=file.collapsepath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) + cachepath=collapsepath(cachepath) local valid=isdir(cachepath) if valid then if is_readable(cachepath) then @@ -21688,16 +21719,14 @@ local function identify() writable=cachepath end end - elseif not writable and caches.force then - local cacheparent=file.dirname(cachepath) - if is_writable(cacheparent) and true then - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("path %a created",cachepath) - writable=cachepath - readables[#readables+1]=cachepath - end + elseif not writable then + local cacheparent=dirname(cachepath) + if is_writable(cacheparent) then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath end end end @@ -21710,8 +21739,8 @@ local function identify() local cachepath=texmfcaches[k] cachepath=resolvers.expansion(cachepath) if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) local valid=isdir(cachepath) if valid and is_readable(cachepath) then if not writable and is_writable(cachepath) then @@ -21725,23 +21754,25 @@ local function identify() end if not writable then report_caches("fatal error: there is no valid writable cache path defined") - os.exit() + osexit() elseif #readables==0 then report_caches("fatal error: there is no valid readable cache path defined") - os.exit() + osexit() end - writable=dir.expandname(resolvers.cleanpath(writable)) - local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + writable=expanddirname(cleanpath(writable)) + local base=caches.base + local more=caches.more + local tree=cache_tree or treehash() if tree then - caches.tree=tree + cache_tree=tree writable=mkdirs(writable,base,more,tree) for i=1,#readables do - readables[i]=file.join(readables[i],base,more,tree) + readables[i]=joinfile(readables[i],base,more,tree) end else writable=mkdirs(writable,base,more) for i=1,#readables do - readables[i]=file.join(readables[i],base,more) + readables[i]=joinfile(readables[i],base,more) end end if trace_cache then @@ -21778,27 +21809,8 @@ function caches.usedpaths(separator) return writable or "?" end end -function caches.configfiles() - return concat(resolvers.configurationfiles(),";") -end -function caches.hashed(tree) - tree=gsub(tree,"[\\/]+$","") - tree=lower(tree) - local hash=md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %a, hash %a",tree,hash) - end - return hash -end -function caches.treehash() - local tree=caches.configfiles() - if not tree or tree=="" then - return false - else - return caches.hashed(tree) - end -end -local r_cache,w_cache={},{} +local r_cache={} +local w_cache={} local function getreadablepaths(...) local tags={... } local hash=concat(tags,"/") @@ -21808,7 +21820,7 @@ local function getreadablepaths(...) if #tags>0 then done={} for i=1,#readables do - done[i]=file.join(readables[i],...) + done[i]=joinfile(readables[i],...) end else done=readables @@ -21832,17 +21844,25 @@ local function getwritablepath(...) end return done end -caches.getreadablepaths=getreadablepaths -caches.getwritablepath=getwritablepath -function caches.getfirstreadablefile(filename,...) - local fullname,path=caches.setfirstwritablefile(filename,...) +local function setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=joinfile(wr,filename) + return fullname,wr +end +local function setluanames(path,name) + return + format("%s/%s.%s",path,name,luasuffixes.tma), + format("%s/%s.%s",path,name,luasuffixes.tmc) +end +local function getfirstreadablefile(filename,...) + local fullname,path=setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path end local rd=getreadablepaths(...) for i=1,#rd do local path=rd[i] - local fullname=file.join(path,filename) + local fullname=joinfile(path,filename) if is_readable(fullname) then usedreadables[i]=true return fullname,path @@ -21850,19 +21870,11 @@ function caches.getfirstreadablefile(filename,...) end return fullname,path end -function caches.setfirstwritablefile(filename,...) - local wr=getwritablepath(...) - local fullname=file.join(wr,filename) - return fullname,wr -end -function caches.define(category,subcategory) - return function() - return getwritablepath(category,subcategory) - end -end -function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) -end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +caches.setfirstwritablefile=setfirstwritablefile +caches.getfirstreadablefile=getfirstreadablefile +caches.setluanames=setluanames function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } @@ -21871,16 +21883,16 @@ function caches.loaddata(readables,name,writable) local path=readables[i] local loader=false local state=false - local tmaname,tmcname=caches.setluanames(path,name) + local tmaname,tmcname=setluanames(path,name) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end if not loader and isfile(tmaname) then - local tmacrap,tmcname=caches.setluanames(writable,name) + local tmacrap,tmcname=setluanames(writable,name) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) if isfile(tmcname) then state,loader=pcall(loadfile,tmcname) end @@ -21897,21 +21909,21 @@ function caches.loaddata(readables,name,writable) return false end function caches.is_writable(filepath,filename) - local tmaname,tmcname=caches.setluanames(filepath,filename) + local tmaname,tmcname=setluanames(filepath,filename) return is_writable(tmaname) end local saveoptions={ compact=true } -function caches.savedata(filepath,filename,data) - local tmaname,tmcname=caches.setluanames(filepath,filename) - data.cache_uuid=os.uuid() - if caches.fast then - file.savedata(tmaname,table.fastserialize(data,true)) - elseif caches.direct then - file.savedata(tmaname,table.serialize(data,true,saveoptions)) +function caches.savedata(filepath,filename,data,fast) + local tmaname,tmcname=setluanames(filepath,filename) + data.cache_uuid=osuuid() + if fast or fast_cache then + savedata(tmaname,fastserialize(data,true)) + elseif direct_cache then + savedata(tmaname,serialize(data,true,saveoptions)) else - table.tofile(tmaname,data,true,saveoptions) + serializetofile(tmaname,data,true,saveoptions) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) end local content_state={} function caches.contentstate() @@ -21919,9 +21931,9 @@ function caches.contentstate() end function caches.loadcontent(cachename,dataname,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.luc)) if not blob then @@ -21959,9 +21971,9 @@ function caches.collapsecontent(content) end function caches.savecontent(cachename,dataname,content,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) @@ -21972,17 +21984,17 @@ function caches.savecontent(cachename,dataname,content,filename) type=dataname, root=cachename, version=resolvers.cacheversion, - date=os.date("%Y-%m-%d"), - time=os.date("%H:%M:%S"), + date=osdate("%Y-%m-%d"), + time=osdate("%H:%M:%S"), content=content, - uuid=os.uuid(), + uuid=osuuid(), } - local ok=io.savedata(luaname,table.serialize(data,true)) + local ok=savedata(luaname,serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - if utilities.lua.compile(luaname,lucname) then + if compilelua(luaname,lucname) then if trace_locating then report_resolvers("%a compiled to %a",dataname,lucname) end @@ -21991,7 +22003,7 @@ function caches.savecontent(cachename,dataname,content,filename) if trace_locating then report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) end - os.remove(lucname) + removefile(lucname) end elseif trace_locating then report_resolvers("unable to save %a in %a (access error)",dataname,luaname) @@ -22005,7 +22017,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5310, stripped down to: 3784 +-- original size: 5518, stripped down to: 3854 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -22014,31 +22026,45 @@ if not modules then modules={} end modules ['data-met']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,format=string.find,string.format -local sequenced=table.sequenced +local type=type +local find=string.find local addurlscheme,urlhashed=url.addscheme,url.hashed +local collapsepath,joinfile=file.collapsepath,file.join +local report_methods=logs.reporter("resolvers","methods") local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) trackers.register("resolvers.methods",function(v) trace_methods=v end) -local report_methods=logs.reporter("resolvers","methods") local allocate=utilities.storage.allocate local resolvers=resolvers local registered={} local function splitmethod(filename) if not filename then - return { scheme="unknown",original=filename } + return { + scheme="unknown", + original=filename, + } end if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") + filename=collapsepath(filename,".") if not find(filename,"://",1,true) then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } end - local specification=url.hashed(filename) + local specification=urlhashed(filename) if not specification.scheme or specification.scheme=="" then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } else return specification end @@ -22047,7 +22073,8 @@ resolvers.splitmethod=splitmethod local function methodhandler(what,first,...) local method=registered[what] if method then - local how,namespace=method.how,method.namespace + local how=method.how + local namespace=method.namespace if how=="uri" or how=="url" then local specification=splitmethod(first) local scheme=specification.scheme @@ -22093,7 +22120,10 @@ local function methodhandler(what,first,...) end resolvers.methodhandler=methodhandler function resolvers.registermethod(name,namespace,how) - registered[name]={ how=how or "tag",namespace=namespace } + registered[name]={ + how=how or "tag", + namespace=namespace + } namespace["byscheme"]=function(scheme,filename,...) if scheme=="file" then return methodhandler(name,filename,...) @@ -22102,7 +22132,7 @@ function resolvers.registermethod(name,namespace,how) end end end -local concatinators=allocate { notfound=file.join } +local concatinators=allocate { notfound=joinfile } local locators=allocate { notfound=function() end } local hashers=allocate { notfound=function() end } local generators=allocate { notfound=function() end } @@ -22123,7 +22153,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 68209, stripped down to: 43711 +-- original size: 69574, stripped down to: 44470 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -22134,7 +22164,8 @@ if not modules then modules={} end modules ['data-res']={ } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch local concat,insert,remove=table.concat,table.insert,table.remove -local next,type,rawget=next,type,rawget +local next,type,rawget,loadfile=next,type,rawget,loadfile +local mergedtable=table.merged local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns @@ -22150,6 +22181,7 @@ local joinpath=file.joinpath local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local urlhasscheme=url.hasscheme local getcurrentdir=lfs.currentdir local isfile=lfs.isfile local isdir=lfs.isdir @@ -22175,10 +22207,17 @@ local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os. resolvers.cacheversion="1.100" resolvers.configbanner="" resolvers.homedir=environment.homedir -resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" +local criticalvars={ + "SELFAUTOLOC", + "SELFAUTODIR", + "SELFAUTOPARENT", + "TEXMFCNF", + "TEXMF", + "TEXOS", +} if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf else @@ -22198,13 +22237,20 @@ local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } local instance=nil -function resolvers.setenv(key,value,raw) +local variable +local expansion +local setenv +local getenv +local formatofsuffix=resolvers.formatofsuffix +local splitpath=resolvers.splitpath +local splitmethod=resolvers.splitmethod +setenv=function(key,value,raw) if instance then instance.environment[key]=value ossetenv(key,raw and value or resolveprefix(value)) end end -local function getenv(key) +getenv=function(key) local value=rawget(instance.environment,key) if value and value~="" then return value @@ -22214,94 +22260,97 @@ local function getenv(key) end end resolvers.getenv=getenv -resolvers.env=getenv -local function resolvevariable(k) - return instance.expansions[k] -end +resolvers.setenv=setenv local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local somevariable=P("$")/"" -local somekey=C(R("az","AZ","09","__","--")^1) -local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) -local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" -local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolvevariable -local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) -local variableresolver=Cs((variable+P(1))^0) -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end -function resolvers.reset() - if trace_locating then - report_resolving("creating instance") - end - local environment={} - local variables={} - local expansions={} - local order={} - instance={ - environment=environment, - variables=variables, - expansions=expansions, - order=order, - files={}, - setups={}, - found={}, - foundintrees={}, - hashes={}, - hashed={}, - pathlists=false, - specification={}, - lists={}, - data={}, - fakepaths={}, - remember=true, - diskcache=true, - renewcache=false, - renewtree=false, - loaderror=false, - savelists=true, - pattern=nil, - force_suffixes=true, - pathstack={}, - } - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v=order[i][k] +local expandedvariable,resolvedvariable do + local function resolveinstancevariable(k) + return instance.expansions[k] + end + local p_variable=P("$")/"" + local p_key=C(R("az","AZ","09","__","--")^1) + local p_whatever=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) + local variableexpander=Cs((p_variable*(p_key/resolveinstancevariable)+p_whatever)^1 ) + local p_cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" + local variablecleaner=Cs((p_cleaner+P(1))^0) + local p_variable=R("az","AZ","09","__","--")^1/resolveinstancevariable + local p_variable=(P("$")/"")*(p_variable+(P("{")/"")*p_variable*(P("}")/"")) + local variableresolver=Cs((p_variable+P(1))^0) + expandedvariable=function(var) + return lpegmatch(variableexpander,var) or var + end + function resolvers.reset() + if trace_locating then + report_resolving("creating instance") + end + local environment={} + local variables={} + local expansions={} + local order={} + instance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files={}, + setups={}, + found={}, + foundintrees={}, + hashes={}, + hashed={}, + pathlists=false, + specification={}, + lists={}, + data={}, + fakepaths={}, + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + pathstack={}, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + local repath=resolvers.repath + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end if v~=nil then - t[k]=v - return v + v=checkedvariable(v) or "" end - end - if v==nil then - v="" - end - t[k]=v - return v - end) - setmetatableindex(environment,function(t,k) - local v=osgetenv(k) - if v==nil then - v=variables[k] - end - if v~=nil then - v=checkedvariable(v) or "" - end - v=resolvers.repath(v) - t[k]=v - return v - end) - setmetatableindex(expansions,function(t,k) - local v=environment[k] - if type(v)=="string" then - v=lpegmatch(variableresolver,v) - v=lpegmatch(variablecleaner,v) - end - t[k]=v - return v - end) + v=repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + end end function resolvers.initialized() return instance~=nil @@ -22315,31 +22364,33 @@ local function reset_caches() instance.lists={} instance.pathlists=false end -local slash=P("/") -local pathexpressionpattern=Cs ( - Cc("^")*( - Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +local makepathexpression do + local slash=P("/") + local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) - )^1*Cc("$") -) -local cache={} -local function makepathexpression(str) - if str=="." then - return "^%./$" - else - local c=cache[str] - if not c then - c=lpegmatch(pathexpressionpattern,str) - cache[str]=c + )^1*Cc("$") + ) + local cache={} + makepathexpression=function(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c end - return c end end local function reportcriticalvariables(cnfspec) if trace_locating then - for i=1,#resolvers.criticalvars do - local k=resolvers.criticalvars[i] - local v=resolvers.getenv(k) or "unknown" + for i=1,#criticalvars do + local k=criticalvars[i] + local v=getenv(k) or "unknown" report_resolving("variable %a set to %a",k,v) end report_resolving() @@ -22361,7 +22412,7 @@ local function identify_configuration_files() resolvers.luacnfstate="environment" end reportcriticalvariables(cnfspec) - local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local cnfpaths=expandedpathfromlist(splitpath(cnfspec)) local function locatecnf(luacnfname,kind) for i=1,#cnfpaths do local filepath=cnfpaths[i] @@ -22394,6 +22445,8 @@ local function identify_configuration_files() end local function load_configuration_files() local specification=instance.specification + local setups=instance.setups + local order=instance.order if #specification>0 then local luacnfname=resolvers.luacnfname for i=1,#specification do @@ -22403,7 +22456,6 @@ local function load_configuration_files() local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then - local setups=instance.setups local data=blob() local parent=data and data.parent if parent then @@ -22414,7 +22466,7 @@ local function load_configuration_files() local parentdata=blob() if parentdata then report_resolving("loading configuration file %a",filename) - data=table.merged(parentdata,data) + data=mergedtable(parentdata,data) end end end @@ -22446,7 +22498,7 @@ local function load_configuration_files() if trace_locating then report_resolving("reloading configuration due to TEXMF redefinition") end - resolvers.setenv("TEXMFCNF",cnfspec) + setenv("TEXMFCNF",cnfspec) instance.specification={} identify_configuration_files() load_configuration_files() @@ -22464,7 +22516,7 @@ local function load_configuration_files() elseif trace_locating then report_resolving("skipping configuration file %a (no valid format)",filename) end - instance.order[#instance.order+1]=instance.setups[pathname] + order[#order+1]=setups[pathname] if instance.loaderror then break end @@ -22473,6 +22525,8 @@ local function load_configuration_files() report_resolving("warning: no lua configuration files found") end end +local expandedpathlist +local unexpandedpathlist function resolvers.configurationfiles() return instance.specification or {} end @@ -22489,7 +22543,7 @@ local function load_file_databases() end end local function locate_file_databases() - local texmfpaths=resolvers.expandedpathlist("TEXMF") + local texmfpaths=expandedpathlist("TEXMF") if #texmfpaths>0 then for i=1,#texmfpaths do local path=collapsepath(texmfpaths[i]) @@ -22498,7 +22552,7 @@ local function locate_file_databases() if stripped~="" then local runtime=stripped==path path=cleanpath(path) - local spec=resolvers.splitmethod(stripped) + local spec=splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped elseif spec.scheme=="cache" or spec.scheme=="file" then @@ -22532,11 +22586,13 @@ local function generate_file_databases() end end local function save_file_databases() - for i=1,#instance.hashes do - local hash=instance.hashes[i] + local hashes=instance.hashes + local files=instance.files + for i=1,#hashes do + local hash=hashes[i] local cachename=hash.name if hash.cache then - local content=instance.files[cachename] + local content=files[cachename] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",cachename) @@ -22548,8 +22604,9 @@ local function save_file_databases() end end function resolvers.renew(hashname) + local files=instance.files if hashname and hashname~="" then - local expanded=resolvers.expansion(hashname) or "" + local expanded=expansion(hashname) or "" if expanded~="" then if trace_locating then report_resolving("identifying tree %a from %a",expanded,hashname) @@ -22566,7 +22623,7 @@ function resolvers.renew(hashname) report_resolving("using path %a",realpath) end methodhandler('generators',hashname) - local content=instance.files[hashname] + local content=files[hashname] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",hashname) @@ -22593,38 +22650,46 @@ local function load_databases() end end function resolvers.appendhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a appended",name) end - insert(instance.hashes,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.prependhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a prepended",name) end - insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,1,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.extendtexmfvariable(specification) - local t=resolvers.splitpath(getenv("TEXMF")) - insert(t,1,specification) - local newspec=concat(t,",") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"]=newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"]=newspec + local environment=instance.environment + local variables=instance.variables + local texmftrees=splitpath(getenv("TEXMF")) + insert(texmftrees,1,specification) + texmftrees=concat(texmftrees,",") + if environment["TEXMF"] then + environment["TEXMF"]=texmftrees + elseif variables["TEXMF"] then + variables["TEXMF"]=texmftrees else end reset_hashes() end function resolvers.splitexpansions() - local ie=instance.expansions - for k,v in next,ie do + local expansions=instance.expansions + for k,v in next,expansions do local t,tn,h,p={},0,{},splitconfigurationpath(v) for kk=1,#p do local vv=p[kk] @@ -22634,33 +22699,37 @@ function resolvers.splitexpansions() h[vv]=true end end - if #t>1 then - ie[k]=t + if tn>1 then + expansions[k]=t else - ie[k]=t[1] + expansions[k]=t[1] end end end function resolvers.datastate() return caches.contentstate() end -function resolvers.variable(name) +variable=function(name) + local variables=instance.variables local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.variables[name] + local result=name and variables[name] return result~=nil and result or "" end -function resolvers.expansion(name) +expansion=function(name) + local expansions=instance.expansions local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.expansions[name] + local result=name and expansions[name] return result~=nil and result or "" end -function resolvers.unexpandedpathlist(str) - local pth=resolvers.variable(str) - local lst=resolvers.splitpath(pth) +resolvers.variable=variable +resolvers.expansion=expansion +unexpandedpathlist=function(str) + local pth=variable(str) + local lst=splitpath(pth) return expandedpathfromlist(lst) end function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(unexpandedpathlist(str)) end function resolvers.pushpath(name) local pathstack=instance.pathstack @@ -22690,8 +22759,8 @@ function resolvers.stackpath() end local done={} function resolvers.resetextrapaths() - local ep=instance.extra_paths - if not ep then + local extra_paths=instance.extra_paths + if not extra_paths then done={} instance.extra_paths={} elseif #ep>0 then @@ -22712,8 +22781,8 @@ function resolvers.registerextrapath(paths,subpaths) end local paths=settings_to_array(paths) local subpaths=settings_to_array(subpaths) - local ep=instance.extra_paths or {} - local oldn=#ep + local extra_paths=instance.extra_paths or {} + local oldn=#extra_paths local newn=oldn local nofpaths=#paths local nofsubpaths=#subpaths @@ -22726,7 +22795,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end @@ -22736,7 +22805,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=cleanpath(p) + extra_paths[newn]=cleanpath(p) done[p]=true end end @@ -22745,17 +22814,17 @@ function resolvers.registerextrapath(paths,subpaths) for i=1,oldn do for j=1,nofsubpaths do local s=subpaths[j] - local ps=ep[i].."/"..s + local ps=extra_paths[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end end end if newn>0 then - instance.extra_paths=ep + instance.extra_paths=extra_paths end if newn~=oldn then reset_caches() @@ -22763,17 +22832,19 @@ function resolvers.registerextrapath(paths,subpaths) end function resolvers.pushextrapath(path) local paths=settings_to_array(path) - if instance.extra_stack then - insert(instance.extra_stack,1,paths) + local extra_stack=instance.extra_stack + if extra_stack then + insert(extra_stack,1,paths) else instance.extra_stack={ paths } end reset_caches() end function resolvers.popextrapath() - if instance.extra_stack then + local extra_stack=instance.extra_stack + if extra_stack then reset_caches() - return remove(instance.extra_stack,1) + return remove(extra_stack,1) end end local function made_list(instance,list,extra_too) @@ -22802,33 +22873,21 @@ local function made_list(instance,list,extra_too) end end if extra_too then - local es=instance.extra_stack - if es and #es>0 then - for k=1,#es do - add(es[k]) + local extra_stack=instance.extra_stack + local extra_paths=instance.extra_paths + if extra_stack and #extra_stack>0 then + for k=1,#extra_stack do + add(extra_stack[k]) end end - local ep=instance.extra_paths - if ep and #ep>0 then - add(ep) + if extra_paths and #extra_paths>0 then + add(extra_paths) end end add(list) return new end -function resolvers.cleanpathlist(str) - local t=resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i]=collapsepath(cleanpath(t[i])) - end - end - return t -end -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) -end -function resolvers.expandedpathlist(str,extra_too) +expandedpathlist=function(str,extra_too) if not str then return {} elseif instance.savelists then @@ -22836,40 +22895,56 @@ function resolvers.expandedpathlist(str,extra_too) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + local l=made_list(instance,splitpath(expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else - local lst=resolvers.splitpath(resolvers.expansion(str)) + local lst=splitpath(expansion(str)) return made_list(instance,expandedpathfromlist(lst),extra_too) end end -function resolvers.expandedpathlistfromvariable(str) +resolvers.expandedpathlist=expandedpathlist +resolvers.unexpandedpathlist=unexpandedpathlist +function resolvers.cleanpathlist(str) + local t=expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(cleanpath(t[i])) + end + end + return t +end +function resolvers.expandpath(str) + return joinpath(expandedpathlist(str)) +end +local function expandedpathlistfromvariable(str) str=lpegmatch(dollarstripper,str) local tmp=resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp~="" and tmp or str) + return expandedpathlist(tmp~="" and tmp or str) end function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(expandedpathlistfromvariable(str)) end +resolvers.expandedpathlistfromvariable=expandedpathlistfromvariable function resolvers.cleanedpathlist(v) - local t=resolvers.expandedpathlist(v) + local t=expandedpathlist(v) for i=1,#t do - t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + t[i]=resolveprefix(cleanpath(t[i])) end return t end function resolvers.expandbraces(str) - local pth=expandedpathfromlist(resolvers.splitpath(str)) + local pth=expandedpathfromlist(splitpath(str)) return joinpath(pth) end function resolvers.registerfilehash(name,content,someerror) + local files=instance.files if content then - instance.files[name]=content + files[name]=content else - instance.files[name]={} + files[name]={} if somerror==true then instance.loaderror=someerror end @@ -22926,10 +23001,11 @@ local function collect_files(names) pathname="/"..pathname.."$" end local hashes=instance.hashes + local files=instance.files for h=1,#hashes do local hash=hashes[h] local hashname=hash.name - local content=hashname and instance.files[hashname] + local content=hashname and files[hashname] if content then if trace_details then report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) @@ -22999,7 +23075,6 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if filesuffix=="" or not suffixmap[filesuffix] then local defaultsuffixes=resolvers.defaultsuffixes - local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname @@ -23009,7 +23084,7 @@ local function find_analyze(filename,askedformat,allresults) end end else - filetype=resolvers.formatofsuffix(filename) + filetype=formatofsuffix(filename) if trace_locating then report_resolving("using suffix based filetype %a",filetype) end @@ -23070,11 +23145,11 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] if format_suffixes then for i=1,#format_suffixes do - local s=format_suffixes[i] - forcedname=filename.."."..s + local suffix=format_suffixes[i] + forcedname=filename.."."..suffix if isreadable(forcedname) then if trace_locating then - report_resolving("no suffix, forcing format filetype %a",s) + report_resolving("no suffix, forcing format filetype %a",suffix) end return "qualified",{ forcedname } end @@ -23087,7 +23162,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local savedformat=askedformat local format=savedformat or "" if format=="" then - askedformat=resolvers.formatofsuffix(suffix) + askedformat=formatofsuffix(suffix) end if not format then askedformat="othertextfiles" @@ -23126,7 +23201,7 @@ local function check_subpath(fname) end local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local pathlist=expandedpathlist(typespec,filetype and usertypes[filetype]) local entry={} if pathlist and #pathlist>0 then for k=1,#pathlist do @@ -23137,7 +23212,7 @@ local function makepathlist(list,filetype) local expression=makepathexpression(pathname) local barename=gsub(pathname,"/+$","") barename=resolveprefix(barename) - local scheme=url.hasscheme(barename) + local scheme=urlhasscheme(barename) local schemename=gsub(barename,"%.%*$",'') entry[k]={ path=path, @@ -23342,7 +23417,9 @@ collect_instance_files=function(filename,askedformat,allresults) { find_onpath (filename,filetype,wantedfiles,true) }, { find_otherwise(filename,filetype,wantedfiles,true) }, } - local result,status,done={},{},{} + local result={} + local status={} + local done={} for k,r in next,results do local method,list=r[1],r[2] if method and list then @@ -23413,6 +23490,9 @@ local function findfiles(filename,filetype,allresults) if not filename or filename=="" then return {} end + if allresults==nil then + allresults=true + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -23422,27 +23502,25 @@ local function findfiles(filename,filetype,allresults) end return result or {},status end -function resolvers.findfiles(filename,filetype) - if not filename or filename=="" then - return "" - else - return findfiles(filename,filetype,true) - end -end -function resolvers.findfile(filename,filetype) +local function findfile(filename,filetype) if not filename or filename=="" then return "" else return findfiles(filename,filetype,false)[1] or "" end end +resolvers.findfiles=findfiles +resolvers.findfile=findfile +resolvers.find_file=findfile +resolvers.find_files=findfiles function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) + local hashes=instance.hashes + local files=instance.files local base=filebasename(filename) local result={} - local hashes=instance.hashes local function okay(hash,path,name) local found=methodhandler('concatinators',hash.type,hash.name,path,name) if found and found~="" then @@ -23452,7 +23530,7 @@ local function findgivenfiles(filename,allresults) end for k=1,#hashes do local hash=hashes[k] - local content=instance.files[hash.name] + local content=files[hash.name] if content then local path,name=lookup(content,base) if not path then @@ -23484,14 +23562,14 @@ function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end local function findwildcardfiles(filename,allresults,result) + local files=instance.files + local hashes=instance.hashes local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files=instance.files if find(name,"*",1,true) then - local hashes=instance.hashes local function okay(found,path,base,hashname,hashtype) if find(found,path) then local full=methodhandler('concatinators',hashtype,hashname,found,base) @@ -23531,7 +23609,6 @@ local function findwildcardfiles(filename,allresults,result) end end end - local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] local hashname=hash.name @@ -23561,13 +23638,21 @@ end function resolvers.findwildcardfile(filename) return findwildcardfiles(filename,false)[1] or "" end -function resolvers.automount() -end -function resolvers.starttiming() - statistics.starttiming(instance) +do + local starttiming=statistics.starttiming + local stoptiming=statistics.stoptiming + local elapsedtime=statistics.elapsedtime + function resolvers.starttiming() + starttiming(instance) + end + function resolvers.stoptiming() + stoptiming(instance) + end + function resolvers.loadtime() + return elapsedtime(instance) + end end -function resolvers.stoptiming() - statistics.stoptiming(instance) +function resolvers.automount() end function resolvers.load(option) resolvers.starttiming() @@ -23581,9 +23666,6 @@ function resolvers.load(option) local files=instance.files return files and next(files) and true end -function resolvers.loadtime() - return statistics.elapsedtime(instance) -end local function report(str) if trace_locating then report_resolving(str) @@ -23613,7 +23695,7 @@ function resolvers.dowithfilesandreport(command,files,...) end end function resolvers.showpath(str) - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) + return joinpath(expandedpathlist(resolvers.formatofvariable(str))) end function resolvers.registerfile(files,name,path) if files[name] then @@ -23627,7 +23709,7 @@ function resolvers.registerfile(files,name,path) end end function resolvers.dowithpath(name,func) - local pathlist=resolvers.expandedpathlist(name) + local pathlist=expandedpathlist(name) for i=1,#pathlist do func("^"..cleanpath(pathlist[i])) end @@ -23641,7 +23723,7 @@ function resolvers.locateformat(name) local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then - fmtname=resolvers.findfile(fullname) + fmtname=findfile(fullname) fmtname=cleanpath(fmtname) end if fmtname~="" then @@ -23660,7 +23742,7 @@ function resolvers.locateformat(name) return nil,nil end function resolvers.booleanvariable(str,default) - local b=resolvers.expansion(str) + local b=expansion(str) if b=="" then return default else @@ -23670,6 +23752,7 @@ function resolvers.booleanvariable(str,default) end function resolvers.dowithfilesintree(pattern,handle,before,after) local hashes=instance.hashes + local files=instance.files for i=1,#hashes do local hash=hashes[i] local blobtype=hash.type @@ -23681,7 +23764,7 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) if before then before(blobtype,blobpath,pattern) end - for path,name in filtered(instance.files[blobpath],pattern) do + for path,name in filtered(files[blobpath],pattern) do if type(path)=="string" then checked=checked+1 if handle(blobtype,blobpath,path,name) then @@ -23702,10 +23785,6 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -local obsolete=resolvers.obsolete or {} -resolvers.obsolete=obsolete -resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile -resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles function resolvers.knownvariables(pattern) if instance then local environment=instance.environment @@ -23739,7 +23818,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4854, stripped down to: 2889 +-- original size: 5088, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -23748,6 +23827,7 @@ if not modules then modules={} end modules ['data-pre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local insert,remove=table.insert,table.remove local resolvers=resolvers local prefixes=resolvers.prefixes local cleanpath=resolvers.cleanpath @@ -23825,8 +23905,9 @@ prefixes.kpse=prefixes.locate prefixes.full=prefixes.locate prefixes.file=prefixes.filename prefixes.path=prefixes.pathname +local inputstack={} +local stackpath=resolvers.stackpath local function toppath() - local inputstack=resolvers.inputstack if not inputstack then return "." end @@ -23838,15 +23919,23 @@ local function toppath() end end local function jobpath() - local path=resolvers.stackpath() + local path=stackpath() if not path or path=="" then return "." else return path end end +local function pushinputname(name) + insert(inputstack,name) +end +local function popinputname(name) + return remove(inputstack) +end resolvers.toppath=toppath resolvers.jobpath=jobpath +resolvers.pushinputname=pushinputname +resolvers.popinputname=popinputname prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end resolvers.setdynamic("toppath") @@ -23889,7 +23978,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 470 +-- original size: 551, stripped down to: 470 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -23912,7 +24001,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3170 +-- original size: 4365, stripped down to: 3588 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -23921,39 +24010,46 @@ if not modules then modules={} end modules ['data-fil']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local ioopen=io.open +local isdir=lfs.isdir local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers local resolveprefix=resolvers.resolve -local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers -local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local findfile=resolvers.findfile +local scanfiles=resolvers.scanfiles +local registerfilehash=resolvers.registerfilehash +local appendhash=resolvers.appendhash +local loadcachecontent=caches.loadcontent local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check -function locators.file(specification) +function resolvers.locators.file(specification) local filename=specification.filename local realname=resolveprefix(filename) - if realname and realname~='' and lfs.isdir(realname) then + if realname and realname~='' and isdir(realname) then if trace_locating then report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',filename,true) + appendhash('file',filename,true) elseif trace_locating then report_files("file locator %a not found",filename) end end -function hashers.file(specification) +function resolvers.hashers.file(specification) local pathname=specification.filename - local content=caches.loadcontent(pathname,'files') - resolvers.registerfilehash(pathname,content,content==nil) + local content=loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) end -function generators.file(specification) +function resolvers.generators.file(specification) local pathname=specification.filename - local content=resolvers.scanfiles(pathname,false,true) - resolvers.registerfilehash(pathname,content,true) + local content=scanfiles(pathname,false,true) + registerfilehash(pathname,content,true) end -concatinators.file=file.join +resolvers.concatinators.file=file.join +local finders=resolvers.finders +local notfound=finders.notfound function finders.file(specification,filetype) local filename=specification.filename - local foundname=resolvers.findfile(filename,filetype) + local foundname=findfile(filename,filetype) if foundname and foundname~="" then if trace_locating then report_files("file finder: %a found",filename) @@ -23963,37 +24059,55 @@ function finders.file(specification,filetype) if trace_locating then report_files("file finder: %a not found",filename) end - return finders.notfound() + return notfound() end end -function openers.helpers.textopener(tag,filename,f) +local openers=resolvers.openers +local notfound=openers.notfound +local overloaded=false +local function textopener(tag,filename,f) return { - reader=function() return f:read () end, - close=function() logs.show_close(filename) return f:close() end, + reader=function() return f:read () end, + close=function() return f:close() end, } end +function openers.helpers.textopener(...) + return textopener(...) +end +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded=true + textopener=opener + end +end function openers.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"r") + local f=ioopen(filename,"r") if f then if trace_locating then report_files("file opener: %a opened",filename) end - return openers.helpers.textopener("file",filename,f) + return textopener("file",filename,f) end end if trace_locating then report_files("file opener: %a not found",filename) end - return openers.notfound() + return notfound() end +local loaders=resolvers.loaders +local notfound=loaders.notfound function loaders.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"rb") + local f=ioopen(filename,"rb") if f then - logs.show_load(filename) if trace_locating then report_files("file loader: %a loaded",filename) end @@ -24010,7 +24124,7 @@ function loaders.file(specification,filetype) if trace_locating then report_files("file loader: %a not found",filename) end - return loaders.notfound() + return notfound() end @@ -24020,7 +24134,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5029, stripped down to: 3432 +-- original size: 5388, stripped down to: 3685 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -24029,6 +24143,7 @@ if not modules then modules={} end modules ['data-con']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local setmetatable=setmetatable local format,lower,gsub=string.format,string.lower,string.gsub local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) @@ -24036,16 +24151,21 @@ local trace_storage=false trackers.register("resolvers.storage",function(v) tra containers=containers or {} local containers=containers containers.usecache=true +local getwritablepath=caches.getwritablepath +local getreadablepaths=caches.getreadablepaths +local cacheiswritable=caches.is_writable +local loaddatafromcache=caches.loaddata +local savedataincache=caches.savedata local report_containers=logs.reporter("resolvers","containers") local allocated={} local mt={ __index=function(t,k) if k=="writable" then - local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable=getwritablepath(t.category,t.subcategory) or { "." } t.writable=writable return writable elseif k=="readables" then - local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables=getreadablepaths(t.category,t.subcategory) or { "." } t.readables=readables return readables end @@ -24076,7 +24196,7 @@ function containers.define(category,subcategory,version,enabled) end end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable,name) + return container.enabled and caches and cacheiswritable(container.writable,name) end function containers.is_valid(container,name) if name and name~="" then @@ -24090,7 +24210,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name,container.writable) + stored=loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -24106,17 +24226,20 @@ function containers.read(container,name) end return stored end -function containers.write(container,name,data) +function containers.write(container,name,data,fast) if data then data.cache_version=container.version if container.enabled and caches then - local unique,shared=data.unique,data.shared - data.unique,data.shared=nil,nil - caches.savedata(container.writable,name,data) + local unique=data.unique + local shared=data.shared + data.unique=nil + data.shared=nil + savedataincache(container.writable,name,data,fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique,data.shared=unique,shared + data.unique=unique + data.shared=shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) @@ -24139,7 +24262,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 5203, stripped down to: 3765 +-- original size: 5790, stripped down to: 2910 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -24148,40 +24271,11 @@ if not modules then modules={} end modules ['data-use']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local format=string.format local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_mounts=logs.reporter("resolvers","mounts") local resolvers=resolvers -resolvers.automounted=resolvers.automounted or {} -function resolvers.automount(usecache) - local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths==0) and usecache then - mountpaths=caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths>0 then - resolvers.starttiming() - for k=1,#mountpaths do - local root=mountpaths[k] - local f=io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %a",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() - end - end - resolvers.stoptiming() - end -end +local findfile=resolvers.findfile statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) @@ -24191,7 +24285,7 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local luvdata={ enginebanner=enginebanner, formatbanner=formatbanner, - sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcehash=md5.hex(io.loaddata(findfile(sourcefile)) or "unknown"), sourcefile=sourcefile, luaversion=LUAVERSION, formatid=LUATEXFORMATID, @@ -24215,7 +24309,7 @@ function statistics.checkfmtstatus(texname) if lfs.isfile(luvname) then local luv=dofile(luvname) if luv and luv.sourcefile then - local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local sourcehash=md5.hex(io.loaddata(findfile(luv.sourcefile)) or "unknown") local luvbanner=luv.enginebanner or "?" if luvbanner~=enginebanner then return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) @@ -24256,7 +24350,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 10263, stripped down to: 7556 +-- original size: 10725, stripped down to: 7949 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -24269,6 +24363,14 @@ local format,find,match=string.format,string.find,string.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_zip=logs.reporter("resolvers","zip") local resolvers=resolvers +local findfile=resolvers.findfile +local registerfile=resolvers.registerfile +local splitmethod=resolvers.splitmethod +local prependhash=resolvers.prependhash +local starttiming=resolvers.starttiming +local extendtexmf=resolvers.extendtexmfvariable +local stoptiming=resolvers.stoptiming +local urlquery=url.query zip=zip or {} local zip=zip local archives=zip.archives or {} @@ -24283,8 +24385,9 @@ if zipfiles then closezip=zipfiles.close validfile=zipfiles.found wholefile=zipfiles.unzip + local listzip=zipfiles.list traversezip=function(zfile) - return ipairs(zipfiles.list(zfile)) + return ipairs(listzip(zfile)) end local streams=utilities.streams local openstream=streams.open @@ -24341,28 +24444,30 @@ local function validzip(str) return str end end -function zip.openarchive(name) +local function openarchive(name) if not name or name=="" then return nil else local arch=archives[name] if not arch then - local full=resolvers.findfile(name) or "" + local full=findfile(name) or "" arch=full~="" and openzip(full) or false archives[name]=arch end return arch end end -function zip.closearchive(name) +local function closearchive(name) if not name or (name=="" and archives[name]) then closezip(archives[name]) archives[name]=nil end end +zip.openarchive=openarchive +zip.closearchive=closearchive function resolvers.locators.zip(specification) local archive=specification.filename - local zipfile=archive and archive~="" and zip.openarchive(archive) + local zipfile=archive and archive~="" and openarchive(archive) if trace_locating then if zipfile then report_zip("locator: archive %a found",archive) @@ -24371,13 +24476,6 @@ function resolvers.locators.zip(specification) end end end -function resolvers.hashers.zip(specification) - local archive=specification.filename - if trace_locating then - report_zip("loading file %a",archive) - end - resolvers.usezipfile(specification.original) -end function resolvers.concatinators.zip(zipfile,path,name) if not path or path=="" then return format('%s?name=%s',zipfile,name) @@ -24385,14 +24483,16 @@ function resolvers.concatinators.zip(zipfile,path,name) return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.finders.zip(specification) +local finders=resolvers.finders +local notfound=finders.notfound +function finders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("finder: archive %a found",archive) @@ -24413,16 +24513,19 @@ function resolvers.finders.zip(specification) if trace_locating then report_zip("finder: %a not found",original) end - return resolvers.finders.notfound() + return notfound() end -function resolvers.openers.zip(specification) +local openers=resolvers.openers +local notfound=openers.notfound +local textopener=openers.helpers.textopener +function openers.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("opener; archive %a opened",archive) @@ -24432,7 +24535,7 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: file %a found",queryname) end - return resolvers.openers.helpers.textopener('zip',original,handle) + return textopener('zip',original,handle) elseif trace_locating then report_zip("opener: file %a not found",queryname) end @@ -24444,23 +24547,24 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -function resolvers.loaders.zip(specification) +local loaders=resolvers.loaders +local notfound=loaders.notfound +function loaders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("loader: archive %a opened",archive) end local data=wholefile(zfile,queryname) if data then - logs.show_load(original) if trace_locating then report_zip("loader; file %a loaded",original) end @@ -24476,47 +24580,24 @@ function resolvers.loaders.zip(specification) if trace_locating then report_zip("loader: %a not found",original) end - return resolvers.openers.notfound() -end -function resolvers.usezipfile(archive) - local specification=resolvers.splitmethod(archive) - local archive=specification.filename - if archive and not registeredfiles[archive] then - local z=zip.openarchive(archive) - if z then - local tree=url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering: archive %a",archive) - end - resolvers.starttiming() - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) - registeredfiles[archive]=z - resolvers.registerfilehash(archive,resolvers.registerzipfile(z,tree)) - resolvers.stoptiming() - elseif trace_locating then - report_zip("registering: unknown archive %a",archive) - end - elseif trace_locating then - report_zip("registering: archive %a not found",archive) - end + return notfound() end -function resolvers.registerzipfile(z,tree) +local function registerzipfile(z,tree) local names={} local files={} local remap={} local n=0 local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) - local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end + starttiming() for i in traversezip(z) do local filename=i.filename local path,name=match(filename,filter) if not path then n=n+1 - register(names,filename,"") + registerfile(names,filename,"") local usedname=lower(filename) files[usedname]="" if usedname~=filename then @@ -24533,12 +24614,43 @@ function resolvers.registerzipfile(z,tree) else end end + stoptiming() report_zip("registering: %s files registered",n) return { files=files, remap=remap, } end +local function usezipfile(archive) + local specification=splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=openarchive(archive) + if z then + local tree=urlquery(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + prependhash('zip',archive) + extendtexmf(archive) + registeredfiles[archive]=z + registerfilehash(archive,registerzipfile(z,tree)) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +resolvers.usezipfile=usezipfile +resolvers.registerzipfile=registerzipfile +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + usezipfile(specification.original) +end end -- of closure @@ -24547,7 +24659,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8478, stripped down to: 5223 +-- original size: 9444, stripped down to: 5650 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -24563,173 +24675,204 @@ local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local resolveprefix=resolvers.resolve -local notfound=resolvers.finders.notfound -local lookup=resolvers.get_from_content -local collectors={} -local found={} -function resolvers.finders.tree(specification) - local spec=specification.filename - local okay=found[spec] - if okay==nil then - if spec~="" then - local path=dirname(spec) - local name=basename(spec) - if path=="" then - path="." - end - local names=collectors[path] - if not names then - local pattern=find(path,"/%*+$") and path or (path.."/*") - names=globdir(pattern) - collectors[path]=names - end - local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for i=1,#names do - local fullname=names[i] - if find(fullname,pattern) then - found[spec]=fullname - return fullname - end - end - local pattern=lower(pattern) - for i=1,#names do - local fullname=lower(names[i]) - if find(fullname,pattern) then - if isfile(fullname) then +local finders=resolvers.finders +local openers=resolvers.openers +local loaders=resolvers.loaders +local locators=resolvers.locators +local hashers=resolvers.hashers +local generators=resolvers.generators +do + local collectors={} + local found={} + local notfound=finders.notfound + function finders.tree(specification) + local spec=specification.filename + local okay=found[spec] + if okay==nil then + if spec~="" then + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then found[spec]=fullname return fullname - else - break + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end + okay=notfound() + found[spec]=okay end - okay=notfound() - found[spec]=okay + return okay end - return okay end -function resolvers.locators.tree(specification) - local name=specification.filename - local realname=resolveprefix(name) - if realname and realname~='' and isdir(realname) then - if trace_locating then - report_trees("locator %a found",realname) +do + local resolveprefix=resolvers.resolve + local appendhash=resolvers.appendhash + local function dolocate(specification) + local name=specification.filename + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) end - resolvers.appendhash('tree',name,false) - elseif trace_locating then - report_trees("locator %a not found",name) end + locators.tree=dolocate + locators.dirlist=dolocate + locators.dirfile=dolocate end -function resolvers.hashers.tree(specification) - local name=specification.filename - if trace_locating then - report_trees("analyzing %a",name) - end - resolvers.methodhandler("hashers",name) - resolvers.generators.file(specification) -end -local collectors={} -local splitter=lpeg.splitat("/**/") -local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } -table.setmetatableindex(collectors,function(t,k) - local rootname=lpegmatch(stripper,k) - local dataname=joinname(rootname,"dirlist") - local content=caches.loadcontent(dataname,"files",dataname) - if not content then - content=resolvers.scanfiles(rootname,nil,nil,false,true) - caches.savecontent(dataname,"files",content,dataname) - end - t[k]=content - return content -end) -local function checked(root,p,n) - if p then - if type(p)=="table" then - for i=1,#p do - local fullname=joinname(root,p[i],n) +do + local filegenerator=generators.file + generators.dirlist=filegenerator + generators.dirfile=filegenerator +end +do + local filegenerator=generators.file + local methodhandler=resolvers.methodhandler + local function dohash(specification) + local name=specification.filename + if trace_locating then + report_trees("analyzing %a",name) + end + methodhandler("hashers",name) + filegenerator(specification) + end + hashers.tree=dohash + hashers.dirlist=dohash + hashers.dirfile=dohash +end +local resolve do + local collectors={} + local splitter=lpeg.splitat("/**/") + local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } + local loadcontent=caches.loadcontent + local savecontent=caches.savecontent + local notfound=finders.notfound + local scanfiles=resolvers.scanfiles + local lookup=resolvers.get_from_content + table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=loadcontent(dataname,"files",dataname) + if not content then + content=scanfiles(rootname,nil,nil,false,true) + savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content + end) + local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) if isfile(fullname) then return fullname end end - else - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname - end end + return notfound() end - return notfound() -end -local function resolve(specification) - local filename=specification.filename - if filename~="" then - local root,rest=lpegmatch(splitter,filename) - if root and rest then - local path,name=dirname(rest),basename(rest) - if name~=rest then - local content=collectors[root] - local p,n=lookup(content,name) - if not p then - return notfound() - end - local pattern=".*/"..path.."$" - local istable=type(p)=="table" - if istable then - for i=1,#p do - local pi=p[i] - if pi==path or find(pi,pattern) then - local fullname=joinname(root,pi,n) - if isfile(fullname) then - return fullname + resolve=function(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end end end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end end - elseif p==path or find(p,pattern) then - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() end end - local queries=specification.queries - if queries and queries.option=="fileonly" then - return checked(root,p,n) - else - return notfound() - end + end + local path=dirname(filename) + local name=basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) end end - local path,name=dirname(filename),basename(filename) - local root=lpegmatch(stripper,path) - local content=collectors[path] - local p,n=lookup(content,name) - if p then - return checked(root,p,n) + return notfound() + end + finders.dirlist=resolve + function finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } end + return resolve(specification) end - return notfound() end -resolvers.finders .dirlist=resolve -resolvers.locators .dirlist=resolvers.locators .tree -resolvers.hashers .dirlist=resolvers.hashers .tree -resolvers.generators.dirlist=resolvers.generators.file -resolvers.openers .dirlist=resolvers.openers .file -resolvers.loaders .dirlist=resolvers.loaders .file -function resolvers.finders.dirfile(specification) - local queries=specification.queries - if queries then - queries.option="fileonly" - else - specification.queries={ option="fileonly" } - end - return resolve(specification) +do + local fileopener=openers.file + local fileloader=loaders.file + openers.dirlist=fileopener + loaders.dirlist=fileloader + openers.dirfile=fileopener + loaders.dirfile=fileloader end -resolvers.locators .dirfile=resolvers.locators .dirlist -resolvers.hashers .dirfile=resolvers.hashers .dirlist -resolvers.generators.dirfile=resolvers.generators.dirlist -resolvers.openers .dirfile=resolvers.openers .dirlist -resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -24738,7 +24881,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6757, stripped down to: 5272 +-- original size: 6945, stripped down to: 5408 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -24748,8 +24891,11 @@ if not modules then modules={} end modules ['data-sch']={ license="see context related readme files" } local load,tonumber=load,tonumber -local gsub,concat,format=string.gsub,table.concat,string.format +local gsub,format=string.gsub,string.format +local sortedhash,concat=table.sortedhash,table.concat local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local addsuffix,suffix,splitbase=file.addsuffix,file.suffix,file.splitbase +local md5hex=md5.hex local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) local report_schemes=logs.reporter("resolvers","schemes") local http=require("socket.http") @@ -24766,7 +24912,7 @@ function cleaners.none(specification) return specification.original end function cleaners.strip(specification) - local path,name=file.splitbase(specification.original) + local path,name=splitbase(specification.original) if path=="" then return (gsub(name,"[^%a%d%.]+","-")) else @@ -24774,7 +24920,7 @@ function cleaners.strip(specification) end end function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return addsuffix(md5hex(specification.original),suffix(specification.path)) end local cleaner=cleaners.strip directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) @@ -24865,10 +25011,10 @@ end schemes.install=install local function http_handler(specification,cachename) local tempname=cachename..".tmp" - local f=io.open(tempname,"wb") + local handle=io.open(tempname,"wb") local status,message=http.request { url=specification.original, - sink=ltn12.sink.file(f) + sink=ltn12.sink.file(handle) } if not status then os.remove(tempname) @@ -24883,13 +25029,13 @@ install('https') install('ftp') statistics.register("scheme handling time",function() local l,r,nl,nr={},{},0,0 - for k,v in table.sortedhash(loaded) do + for k,v in sortedhash(loaded) do if v>0 then nl=nl+1 l[nl]=k..":"..v end end - for k,v in table.sortedhash(reused) do + for k,v in sortedhash(reused) do if v>0 then nr=nr+1 r[nr]=k..":"..v @@ -24897,10 +25043,10 @@ statistics.register("scheme handling time",function() end local n=nl+nr if n>0 then - l=nl>0 and concat(l) or "none" - r=nr>0 and concat(r) or "none" + if nl==0 then l={ "none" } end + if nr==0 then r={ "none" } end return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes),n,threshold,l,r) + statistics.elapsedtime(schemes),n,threshold,concat(l," "),concat(l," ")) else return nil end @@ -24933,7 +25079,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4207, stripped down to: 3041 +-- original size: 4227, stripped down to: 3049 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -24943,8 +25089,7 @@ if not modules then modules={} end modules ['data-lua']={ license="see context related readme files" } local package,lpeg=package,lpeg -local gsub=string.gsub -local concat=table.concat +local loadfile=loadfile local addsuffix=file.addsuffix local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match local luasuffixes={ 'tex','lua' } @@ -24955,6 +25100,8 @@ local helpers=package.helpers or {} local methods=helpers.methods or {} local resolvers=resolvers local resolveprefix=resolvers.resolve +local expandedpaths=resolvers.expandedpathlistfromvariable +local findfile=resolvers.findfile helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) @@ -24983,7 +25130,7 @@ local function getluaformatpaths() if not luaformatpaths then luaformatpaths={} for i=1,#luaformats do - registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + registerpath("lua format","lua",luaformatpaths,expandedpaths(luaformats[i])) end end return luaformatpaths @@ -24992,7 +25139,7 @@ local function getlibformatpaths() if not libformatpaths then libformatpaths={} for i=1,#libformats do - registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + registerpath("lib format","lib",libformatpaths,expandedpaths(libformats[i])) end end return libformatpaths @@ -25002,7 +25149,7 @@ local function loadedbyformat(name,rawname,suffixes,islib,what) local report=helpers.report for i=1,#suffixes do local format=suffixes[i] - local resolved=resolvers.findfile(name,format) or "" + local resolved=findfile(name,format) or "" if trace then report("%s format, identifying %a using format %a",what,name,format) end @@ -25040,7 +25187,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2452, stripped down to: 1877 +-- original size: 2610, stripped down to: 2019 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -25051,24 +25198,28 @@ if not modules then modules={} end modules ['data-aux']={ } local find=string.find local type,next=type,next +local addsuffix,removesuffix=file.addsuffix,file.removesuffix +local loaddata,savedata=io.loaddata,io.savedata local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers +local cleanpath=resolvers.cleanpath +local findfiles=resolvers.findfiles local report_scripts=logs.reporter("resolvers","scripts") function resolvers.updatescript(oldname,newname) local scriptpath="context/lua" - newname=file.addsuffix(newname,"lua") - local oldscript=resolvers.cleanpath(oldname) + local oldscript=cleanpath(oldname) + local newname=addsuffix(newname,"lua") + local newscripts=findfiles(newname) or {} if trace_locating then report_scripts("to be replaced old script %a",oldscript) end - local newscripts=resolvers.findfiles(newname) or {} if #newscripts==0 then if trace_locating then report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript=resolvers.cleanpath(newscripts[i]) + local newscript=cleanpath(newscripts[i]) if trace_locating then report_scripts("checking new script %a",newscript) end @@ -25080,17 +25231,17 @@ function resolvers.updatescript(oldname,newname) if trace_locating then report_scripts("new script should come from %a",scriptpath) end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + elseif not (find(oldscript,removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then report_scripts("invalid new script name") end else - local newdata=io.loaddata(newscript) + local newdata=loaddata(newscript) if newdata then if trace_locating then report_scripts("old script content replaced by new content: %s",oldscript) end - io.savedata(oldscript,newdata) + savedata(oldscript,newdata) break elseif trace_locating then report_scripts("unable to load new script") @@ -25163,7 +25314,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 1823, stripped down to: 1542 +-- original size: 1912, stripped down to: 1591 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -25173,40 +25324,39 @@ if not modules then modules={} end modules ['data-lst']={ license="see context related readme files" } local type=type -local concat,sortedhash=table.concat,table.sortedhash +local sortedhash=table.sortedhash +local isdir=lfs.isdir local resolvers=resolvers local listers=resolvers.listers or {} resolvers.listers=listers local resolveprefix=resolvers.resolve +local configurationfiles=resolvers.configurationfiles +local expandedpathfromlist=resolvers.expandedpathfromlist +local splitpath=resolvers.splitpath +local knownvariables=resolvers.knownvariables local report_lists=logs.reporter("resolvers","lists") local report_resolved=logs.reporter("system","resolved") -local function tabstr(str) - if type(str)=='table' then - return concat(str," | ") - else - return str - end -end function listers.variables(pattern) local result=resolvers.knownvariables(pattern) + local unset={ "unset" } for key,value in sortedhash(result) do report_lists(key) - report_lists(" env: %s",tabstr(value.environment or "unset")) - report_lists(" var: %s",tabstr(value.variable or "unset")) - report_lists(" exp: %s",tabstr(value.expansion or "unset")) - report_lists(" res: %s",tabstr(value.resolved or "unset")) + report_lists(" env: % | t",value.environment or unset) + report_lists(" var: % | t",value.variable or unset) + report_lists(" exp: % | t",value.expansion or unset) + report_lists(" res: % | t",value.resolved or unset) end end function listers.configurations() - local configurations=resolvers.configurationfiles() + local configurations=configurationfiles() for i=1,#configurations do report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") - local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + local list=expandedpathfromlist(splitpath(resolvers.luacnfspec)) for i=1,#list do local li=resolveprefix(list[i]) - if lfs.isdir(li) then + if isdir(li) then report_resolved("path - %s",li) else report_resolved("path + %s",li) @@ -25978,8 +26128,8 @@ end -- of closure -- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 1036669 --- stripped bytes : 410250 +-- original bytes : 1040817 +-- stripped bytes : 412122 -- end library merge diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii index f3f322424..129aa988b 100644 --- a/tex/context/base/mkii/cont-new.mkii +++ b/tex/context/base/mkii/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2019.11.14 17:07} +\newcontextversion{2019.11.25 17:28} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii index 3b4bbfc72..1f1e6a492 100644 --- a/tex/context/base/mkii/context.mkii +++ b/tex/context/base/mkii/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2019.11.14 17:07} +\edef\contextversion{2019.11.25 17:28} %D For those who want to use this: diff --git a/tex/context/base/mkii/mult-cs.mkii b/tex/context/base/mkii/mult-cs.mkii index 374f96c7c..687c2bd4c 100644 --- a/tex/context/base/mkii/mult-cs.mkii +++ b/tex/context/base/mkii/mult-cs.mkii @@ -232,6 +232,7 @@ \setinterfacevariable{handwritten}{rukopisne} \setinterfacevariable{hang}{zaveseni} \setinterfacevariable{hanging}{visici} +\setinterfacevariable{hangingboth}{hangingboth} \setinterfacevariable{head}{hlavicka} \setinterfacevariable{header}{zahlavi} \setinterfacevariable{headintext}{headintext} @@ -1044,6 +1045,7 @@ \setinterfaceconstant{overprint}{overprint} \setinterfaceconstant{ownerpassword}{ownerpassword} \setinterfaceconstant{ownnumber}{vlastnicislo} +\setinterfaceconstant{packcriterium}{packcriterium} \setinterfaceconstant{page}{stranka} \setinterfaceconstant{pageboundaries}{hranicestranky} \setinterfaceconstant{pagecolor}{barvastranky} @@ -1053,6 +1055,7 @@ \setinterfaceconstant{pageconversion}{pageconversion} \setinterfaceconstant{pageconversionset}{pageconversionset} \setinterfaceconstant{pageleft}{pageleft} +\setinterfaceconstant{pagemethod}{pagemethod} \setinterfaceconstant{pagenumber}{cislostranky} \setinterfaceconstant{pageprefix}{pageprefix} \setinterfaceconstant{pageprefixconnector}{pageprefixconnector} diff --git a/tex/context/base/mkii/mult-de.mkii b/tex/context/base/mkii/mult-de.mkii index 005d93672..1690133a5 100644 --- a/tex/context/base/mkii/mult-de.mkii +++ b/tex/context/base/mkii/mult-de.mkii @@ -232,6 +232,7 @@ \setinterfacevariable{handwritten}{handschrift} \setinterfacevariable{hang}{haengend} \setinterfacevariable{hanging}{haengend} +\setinterfacevariable{hangingboth}{hangingboth} \setinterfacevariable{head}{kopf} \setinterfacevariable{header}{kopfzeile} \setinterfacevariable{headintext}{headintext} @@ -1044,6 +1045,7 @@ \setinterfaceconstant{overprint}{overprint} \setinterfaceconstant{ownerpassword}{ownerpassword} \setinterfaceconstant{ownnumber}{eigenenummer} +\setinterfaceconstant{packcriterium}{packcriterium} \setinterfaceconstant{page}{seite} \setinterfaceconstant{pageboundaries}{seitenbegrenzung} \setinterfaceconstant{pagecolor}{seitenfarbe} diff --git a/tex/context/base/mkii/mult-nl.mkii b/tex/context/base/mkii/mult-nl.mkii index ce86f1edd..8b7e22f85 100644 --- a/tex/context/base/mkii/mult-nl.mkii +++ b/tex/context/base/mkii/mult-nl.mkii @@ -232,6 +232,7 @@ \setinterfacevariable{handwritten}{handschrift} \setinterfacevariable{hang}{hang} \setinterfacevariable{hanging}{hangend} +\setinterfacevariable{hangingboth}{beidehangend} \setinterfacevariable{head}{kop} \setinterfacevariable{header}{hoofd} \setinterfacevariable{headintext}{kopintekst} @@ -1044,6 +1045,7 @@ \setinterfaceconstant{overprint}{overprint} \setinterfaceconstant{ownerpassword}{ownerpassword} \setinterfaceconstant{ownnumber}{eigennummer} +\setinterfaceconstant{packcriterium}{opelkaarcriterium} \setinterfaceconstant{page}{pagina} \setinterfaceconstant{pageboundaries}{paginaovergangen} \setinterfaceconstant{pagecolor}{paginakleur} @@ -1053,6 +1055,7 @@ \setinterfaceconstant{pageconversion}{pageconversion} \setinterfaceconstant{pageconversionset}{pageconversionset} \setinterfaceconstant{pageleft}{pageleft} +\setinterfaceconstant{pagemethod}{paginamethode} \setinterfaceconstant{pagenumber}{paginanummer} \setinterfaceconstant{pageprefix}{pageprefix} \setinterfaceconstant{pageprefixconnector}{pageprefixconnector} diff --git a/tex/context/base/mkii/mult-ro.mkii b/tex/context/base/mkii/mult-ro.mkii index ccdce5dcc..219e59dd8 100644 --- a/tex/context/base/mkii/mult-ro.mkii +++ b/tex/context/base/mkii/mult-ro.mkii @@ -232,6 +232,7 @@ \setinterfacevariable{handwritten}{scrismanual} \setinterfacevariable{hang}{suspenda} \setinterfacevariable{hanging}{suspendat} +\setinterfacevariable{hangingboth}{hangingboth} \setinterfacevariable{head}{antet} \setinterfacevariable{header}{antet} \setinterfacevariable{headintext}{headintext} @@ -1044,6 +1045,7 @@ \setinterfaceconstant{overprint}{overprint} \setinterfaceconstant{ownerpassword}{ownerpassword} \setinterfaceconstant{ownnumber}{numarpropriu} +\setinterfaceconstant{packcriterium}{packcriterium} \setinterfaceconstant{page}{pagina} \setinterfaceconstant{pageboundaries}{marginipagina} \setinterfaceconstant{pagecolor}{culoarepagina} @@ -1053,6 +1055,7 @@ \setinterfaceconstant{pageconversion}{pageconversion} \setinterfaceconstant{pageconversionset}{pageconversionset} \setinterfaceconstant{pageleft}{pageleft} +\setinterfaceconstant{pagemethod}{pagemethod} \setinterfaceconstant{pagenumber}{numarpagina} \setinterfaceconstant{pageprefix}{pageprefix} \setinterfaceconstant{pageprefixconnector}{pageprefixconnector} diff --git a/tex/context/base/mkiv/anch-pgr.lua b/tex/context/base/mkiv/anch-pgr.lua index 14afa3967..be9e28c08 100644 --- a/tex/context/base/mkiv/anch-pgr.lua +++ b/tex/context/base/mkiv/anch-pgr.lua @@ -25,6 +25,8 @@ local jobpositions = job.positions local formatters = string.formatters local setmetatableindex = table.setmetatableindex +local enableaction = nodes.tasks.enableaction + local commands = commands local context = context @@ -76,6 +78,7 @@ local getdepth = nuts.getdepth local nodecodes = nodes.nodecodes local localpar_code = nodecodes.localpar +local start_of_par = nuts.start_of_par local insert_before = nuts.insert_before local insert_after = nuts.insert_after @@ -85,6 +88,7 @@ local unsetvalue = attributes.unsetvalue local jobpositions = job.positions local getpos = jobpositions.getpos +local getfree = jobpositions.getfree local data = { } local realpage = 1 @@ -161,7 +165,7 @@ local function flush(head,f,l,a,parent,depth) ln = new_hlist(setlink(new_rule(65536,65536*4,0),new_kern(-65536),ln)) rn = new_hlist(setlink(new_rule(65536,0,65536*4),new_kern(-65536),rn)) end - if getid(f) == localpar_code and getsubtype(f) == 0 then -- we need to clean this mess + if getid(f) == localpar_code and start_of_par(f) then -- we need to clean this mess insert_after(head,f,ln) else head, f = insert_before(head,f,ln) @@ -193,7 +197,7 @@ local function registerbackground(name) } texsetattribute(a_textbackground,n) if not enabled then - nodes.tasks.enableaction("contributers", "nodes.handlers.textbackgrounds") + enableaction("contributers", "nodes.handlers.textbackgrounds") enabled = true end else @@ -1130,7 +1134,7 @@ local function fetchmultipar(n,anchor,page) local k = data.bpos.k if k ~= 3 then -- to be checked: no need in txt mode - freemultipar(pagedata,jobpositions.free[page]) + freemultipar(pagedata,getfree(page)) end local nofmultipars = #pagedata if trace_shapes then diff --git a/tex/context/base/mkiv/anch-pos.lua b/tex/context/base/mkiv/anch-pos.lua index e87409dbe..f1ec59d8b 100644 --- a/tex/context/base/mkiv/anch-pos.lua +++ b/tex/context/base/mkiv/anch-pos.lua @@ -121,18 +121,17 @@ local f_region = formatters["region:%s"] local f_tag_three = formatters["%s:%s:%s"] local f_tag_two = formatters["%s:%s"] -jobpositions.used = false - local nofregular = 0 local nofspecial = 0 local splitter = lpeg.splitat(":",true) +local pagedata = { } +local columndata = setmetatableindex("table") -- per page +local freedata = setmetatableindex("table") -- per page + local function initializer() tobesaved = jobpositions.tobesaved collected = jobpositions.collected - local pagedata = { } - local columndata = setmetatableindex("table") - local freedata = setmetatableindex("table") for tag, data in next, collected do local prefix, rest = lpegmatch(splitter,tag) if prefix == "p" then @@ -174,102 +173,23 @@ local function initializer() end end end - jobpositions.page = pagedata - jobpositions.column = columndata - jobpositions.free = freedata - jobpositions.used = next(collected) + jobpositions.pagedata = pagedata end --- -- we can gain a little when we group positions but then we still have to --- -- deal with regions and cells so we either end up with lots of extra small --- -- tables pointing to them and/or assembling/disassembling so in the end --- -- it makes no sense to do it (now) and still have such a mix --- --- local splitter = lpeg.splitat(":",true) --- --- local function setpos(t,k,v) --- local class, tag = lpegmatch(splitter,k) --- if tag then --- local c = rawget(t,class) --- if c then --- c[tonumber(tag) or tag] = v --- else --- rawset(t,class,{ [tonumber(tag) or tag] = v }) --- end --- else --- t.default[tonumber(k) or k] = v --- end --- end --- --- local function getpos(t,k) --- local class, tag = lpegmatch(splitter,k) --- if tag then --- local c = rawget(t,class) --- if c then --- return c[tonumber(tag) or tag] --- end --- else --- return c.default[tonumber(k) or k] --- end --- end --- --- tobesaved.default = tobesaved.default or { } --- setmetatablenewindex(tobesaved,setpos) --- setmetatableindex (tobesaved,getpos) --- --- local function initializer() --- tobesaved = jobpositions.tobesaved --- collected = jobpositions.collected --- --- tobesaved.default = tobesaved.default or { } --- collected.default = collected.default or { } --- --- setmetatablenewindex(tobesaved,setpos) --- setmetatableindex (collected,getpos) --- setmetatableindex (tobesaved,getpos) --- --- for class, list in next, collected do --- for tag, data in next, list do --- setmetatable(data,default) --- nofregular = nofregular + 1 --- end --- end --- --- local pagedata = collected.page or { } --- local freedata = setmetatableindex("table") --- --- for tag, data in next, collected.free or { } do --- local t = freedata[data.p or 0] --- t[#t+1] = data --- end --- --- local pages = structures.pages.collected --- if pages then --- local last = nil --- for p=1,#pages do --- local data = pagedata[p] --- local free = freedata[p] --- if free then --- sort(free,function(a,b) return b.y < a.y end) -- order matters ! --- end --- if data then --- last = data --- last.free = free --- elseif last then --- local t = setmetatableindex({ free = free, p = p },last) --- if not pagedata[p] then --- pagedata[p] = t --- end --- end --- end --- end --- jobpositions.page = pagedata --- jobpositions.free = freedata --- jobpositions.used = next(collected) --- end +function jobpositions.used() + return next(collected) -- we can safe it +end + +function jobpositions.getfree(page) + return freedata[page] +end + +-- we can gain a little when we group positions but then we still have to +-- deal with regions and cells so we either end up with lots of extra small +-- tables pointing to them and/or assembling/disassembling so in the end +-- it makes no sense to do it (now) and still have such a mix -- --- function jobpositions.getcollected(class,tag) if tag then return collected[class..tag] else return collected[class] end end --- function jobpositions.gettobesaved(class,tag) if tag then return tobesaved[class..tag] else return tobesaved[class] end end +-- proof of concept code removed ... see archive local function finalizer() -- We make the (possible extensive) shape lists sparse working @@ -818,12 +738,22 @@ function jobpositions.replace(id,p,x,y,w,h,d) collected[id] = { p = p, x = x, y = y, w = w, h = h, d = d } -- c g end -function jobpositions.page(id) +local function getpage(id) local jpi = collected[id] return jpi and jpi.p end -function jobpositions.region(id) +local function getcolumn(id) + local jpi = collected[id] + return jpi and jpi.c or false +end + +local function getparagraph(id) + local jpi = collected[id] + return jpi and jpi.n +end + +local function getregion(id) local jpi = collected[id] if jpi then local r = jpi.r @@ -838,20 +768,15 @@ function jobpositions.region(id) return false end -function jobpositions.column(id) - local jpi = collected[id] - return jpi and jpi.c or false -end - -function jobpositions.paragraph(id) - local jpi = collected[id] - return jpi and jpi.n -end +jobpositions.page = getpage +jobpositions.column = getcolumn +jobpositions.paragraph = getparagraph +jobpositions.region = getregion -jobpositions.p = jobpositions.page -jobpositions.r = jobpositions.region -jobpositions.c = jobpositions.column -jobpositions.n = jobpositions.paragraph +jobpositions.p = getpage -- not used, kind of obsolete +jobpositions.c = getcolumn -- idem +jobpositions.n = getparagraph -- idem +jobpositions.r = getregion -- idem function jobpositions.x(id) local jpi = collected[id] @@ -1046,7 +971,7 @@ local function onsamepage(list,page) end local function columnofpos(realpage,xposition) - local p = job.positions.column[realpage] + local p = columndata[realpage] if p then for i=1,#p do local c = p[i] @@ -1318,7 +1243,7 @@ scanners.MPposset = function() -- name (special helper, used in backgrounds) local b = f_b_tag(name) local e = f_e_tag(name) local w = f_w_tag(name) - local p = f_p_tag(jobpositions.n(b)) + local p = f_p_tag(getparagraph(b)) MPpos(b) context(",") MPpos(e) context(",") MPpos(w) context(",") MPpos(p) context(",") MPpardata(p) end diff --git a/tex/context/base/mkiv/anch-snc.lua b/tex/context/base/mkiv/anch-snc.lua index 8006f3bd6..ee13adca3 100644 --- a/tex/context/base/mkiv/anch-snc.lua +++ b/tex/context/base/mkiv/anch-snc.lua @@ -7,18 +7,14 @@ if not modules then modules = { } end modules ['anch-snc'] = { } -function mp.xxOverlayRegion() - local r = tokens.getters.macro("m_overlay_region") - mp.quoted('"'.. r .. '"') --- mp.print('"'.. r .. '"') -end - -- use factors as in mlib-int.lua local tonumber, next, setmetatable = tonumber, next, setmetatable local concat, sort, remove, copy = table.concat, table.sort, table.remove, table.copy local match, find = string.match, string.find -local lpegmatch = lpeg.match +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns + +local P, Cc = lpeg.P, lpeg.Cc local setmetatableindex = table.setmetatableindex @@ -28,11 +24,10 @@ local mpnumeric = mp.numeric local mppoints = mp.points local texgetdimen = tex.getdimen -local p_number = lpeg.patterns.cardinal/tonumber -local p_space = lpeg.patterns.whitespace^0 -local p_tag = lpeg.P("syncpos:") * p_number * lpeg.P(":") * p_number -local p_option = p_number -- for now - * ((lpeg.P(",") * p_space * lpeg.P("reset") * lpeg.Cc(true)) + lpeg.Cc(false)) +local p_number = patterns.cardinal/tonumber +local p_space = patterns.whitespace^0 +local p_tag = P("syncpos:") * p_number * P(":") * p_number +local p_option = p_number * ((P(",") * p_space * P("reset") * Cc(true)) + Cc(false)) -- for now local list = { } local colors = setmetatableindex("table") @@ -269,3 +264,9 @@ function mp.sync_get_y() mpnumeric((list.page.y or 0)*factor) end function mp.sync_get_w() mpnumeric((list.page.w or 0)*factor) end function mp.sync_get_h() mpnumeric((list.page.h or 0)*factor) end function mp.sync_get_d() mpnumeric((list.page.d or 0)*factor) end + +-- function mp.xxOverlayRegion() +-- local r = tokens.getters.macro("m_overlay_region") +-- mp.quoted('"'.. r .. '"') +-- end + diff --git a/tex/context/base/mkiv/attr-ini.lua b/tex/context/base/mkiv/attr-ini.lua index 5d35d6ab5..6145fa08d 100644 --- a/tex/context/base/mkiv/attr-ini.lua +++ b/tex/context/base/mkiv/attr-ini.lua @@ -7,6 +7,7 @@ if not modules then modules = { } end modules ['attr-ini'] = { } local next, type = next, type +local osexit = os.exit --[[ldx-- <p>We start with a registration system for atributes so that we can use the @@ -76,7 +77,7 @@ function attributes.private(name) -- at the lua end (hidden from user) sharedstorage.attributes_last_private = last else report_attribute("no more room for private attributes") - os.exit() + osexit() end number = last numbers[name], names[number], list[number] = number, name, { } @@ -93,7 +94,7 @@ function attributes.public(name) -- at the lua end (hidden from user) sharedstorage.attributes_last_public = last else report_attribute("no more room for public attributes") - os.exit() + osexit() end number = last numbers[name], names[number], list[number] = number, name, { } diff --git a/tex/context/base/mkiv/back-exp.lua b/tex/context/base/mkiv/back-exp.lua index deb11b4d2..ea047db0c 100644 --- a/tex/context/base/mkiv/back-exp.lua +++ b/tex/context/base/mkiv/back-exp.lua @@ -2947,6 +2947,8 @@ local collectresults do -- too many locals otherwise local getkern = nuts.getkern local getwidth = nuts.getwidth + local start_of_par = nuts.start_of_par + local nexthlist = nuts.traversers.hlist local nextnode = nuts.traversers.node @@ -3321,7 +3323,7 @@ local collectresults do -- too many locals otherwise last = nil currentparagraph = nil end - elseif not localparagraph and id == localpar_code and subtype == 0 then + elseif not localparagraph and id == localpar_code and start_of_par(n) then localparagraph = getattr(n,a_taggedpar) end p = n @@ -3962,7 +3964,7 @@ local htmltemplate = [[ end - local examplefilename = resolvers.find_file("export-example.css") + local examplefilename = resolvers.findfile("export-example.css") if examplefilename then local data = io.loaddata(examplefilename) if not data or data == "" then diff --git a/tex/context/base/mkiv/char-ini.lua b/tex/context/base/mkiv/char-ini.lua index e0cbf6bdd..1d202c8d4 100644 --- a/tex/context/base/mkiv/char-ini.lua +++ b/tex/context/base/mkiv/char-ini.lua @@ -1526,7 +1526,7 @@ do )^1) local function load() - local name = resolvers.find_file("char-emj.lua") + local name = resolvers.findfile("char-emj.lua") local data = name and name ~= "" and dofile(name) or { } local hash = { } for d, c in next, data do diff --git a/tex/context/base/mkiv/chem-str.lua b/tex/context/base/mkiv/chem-str.lua index 9f0738fc5..1316769ca 100644 --- a/tex/context/base/mkiv/chem-str.lua +++ b/tex/context/base/mkiv/chem-str.lua @@ -69,7 +69,8 @@ local chemistry = chemistry chemistry.instance = "chemistry" chemistry.format = "metafun" chemistry.method = "double" -chemistry.structures = 0 + +local nofstructures = 0 local common_keys = { b = "line", @@ -739,7 +740,7 @@ function chemistry.start(settings) return end -- - chemistry.structures = chemistry.structures + 1 + nofstructures = nofstructures + 1 -- rotation = tonumber(rotation) or 0 -- @@ -749,7 +750,7 @@ function chemistry.start(settings) report_chemistry("%s scale %a, rotation %a, width %s, height %s, left %s, right %s, top %s, bottom %s","used",scale,rotation,width,height,left,right,top,bottom) end metacode[#metacode+1] = f_start_structure( - chemistry.structures, + nofstructures, left, right, top, bottom, rotation, topoints(unit), bondlength, scale, topoints(offset), tostring(settings.axis == v_on), topoints(rulethickness), axiscolor @@ -788,8 +789,8 @@ function chemistry.component(spec,text,rulethickness,rulecolor) end statistics.register("chemical formulas", function() - if chemistry.structures > 0 then - return format("%s chemical structure formulas",chemistry.structures) -- no timing needed, part of metapost + if nofstructures > 0 then + return format("%s chemical structure formulas",nofstructures) -- no timing needed, part of metapost end end) diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv index 532241803..044fc4ce4 100644 --- a/tex/context/base/mkiv/cont-new.mkiv +++ b/tex/context/base/mkiv/cont-new.mkiv @@ -13,7 +13,7 @@ % \normalend % uncomment this to get the real base runtime -\newcontextversion{2019.11.14 17:07} +\newcontextversion{2019.11.25 17:28} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/mkiv/context-todo.tex b/tex/context/base/mkiv/context-todo.tex index a356d5964..6aa5bca08 100644 --- a/tex/context/base/mkiv/context-todo.tex +++ b/tex/context/base/mkiv/context-todo.tex @@ -70,16 +70,18 @@ draw d withcolor "ColorTwo" withpen pencircle scaled (o - 5) ; draw l withcolor white withpen pencircle scaled (o / 5) ; draw l withcolor "ColorTwo" withpen pencircle scaled (o /10) ; - draw thelabel.ulft(t xsized .5PaperWidth,lrcorner Page shifted - (PaperWidth/20,-PaperWidth/40)) ; + draw thelabel.ulft((t xsized .5PaperWidth),lrcorner Page shifted - (PaperWidth/20,-PaperWidth/40)) ; StopPage ; \stopMPpage - \startsubject[title={Introduction}] -There is not really a long term roadmap for development. One reason is that there is already -a lot available. When we started with \LUATEX, the \CONTEXT\ code was mostly rewritten, -and that process is more of less finished. Of course there is always work left. +There is not really a long term roadmap for development. One reason is that there +is already a lot available. When we started with \LUATEX, the \CONTEXT\ code was +mostly rewritten from \MKII\ to \MKIV, and that process is more of less finished. +Of course there is always work left. We have moved on to \LUAMETATEX\ and \LMTX\ +as follow up on \MKIV. Deep down some things might (and will) change but the user +interface will stay the same (as usual). This file is not a complete overview of our plans but users can at least get an idea of what we're working on and what is coming. Feel free to submit @@ -93,35 +95,32 @@ Hasselt NL \stopsubject -\startsubject[title={On the agenda for \LUATEX}] +\startsubject[title={On the agenda for \LUAMETATEX}] + +The agenda for \LUATEX\ is basically empty as this program is (supposed to be) +frozen, but for \LUAMETATEX\ we can still consider improvements. In fact, there +are quite some changes between the engines already (internally). Items that +concern \LUATEX\ have been removed from the agenda but some improvements in +\LUAMETATEX\ might trickle back. \startitemize \startitem - cleanup passive nodes - \stopitem - \startitem - optimize some callback resolution (more direct) + cleanup passive nodes in the par builder \stopitem \startitem - add \type {--output-filename} for \PDF\ filename + optimize some callback resolution (more direct) but there is not that + much to gain there \stopitem \startitem - more consistent \type {lang_variables} and \type {tex_language} in \type - {texlang.w} and also store the \type {*mins} + remove local par in head of line when done with linebreak and maybe also + ensure leftskip and rightskip \stopitem \startitem - remove local par in head of line when done with linebreak + only return nil in lua helpers when we expect multiple calls in in one + line \stopitem \startitem - check why leftskip doesn't always inherit attributes (maybe dir notes don't have them) - (also check redundant \type {delete_attribute_ref} after \type {new_glue}) - \stopitem - \startitem - only return nil when we expect multiple calls in in one line - \stopitem - \startitem - pdf injection in virtual characters (currently qQ interferes with font switch - flushing) so a pdf page hack is needed + experiment a bit more with the new protrusion code \stopitem \stopitemize @@ -140,7 +139,7 @@ Hasselt NL add flag to font for math engine \stopitem \startitem - get rid of components + get rid of components in glyph nodes \stopitem \startitem play with box attributes @@ -150,15 +149,24 @@ Hasselt NL styles, properties) \stopitem \startitem + freeze actions and tasks (by name or function), maybe a register + function that does that so no one can overload built-in features + \startitem redo some of the spacing (adapt to improvements in engine) \stopitem \startitem - reorganize position data (more subtables) + more node and code injections + \stopitem + \startitem + maybe reorganize position data (more subtables) \stopitem \startitem use \type {\matheqnogapstep}, \type {\Ustack}, \type {\mathscriptsmode}, \type {\mathdisplayskipmode} and other new math primitives \stopitem + \startitem + cleanup some lua helpers (io.exists vs lfs.isfile) + \stopitem \stopitemize \stopsubject diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv index 7812d4789..8c264224b 100644 --- a/tex/context/base/mkiv/context.mkiv +++ b/tex/context/base/mkiv/context.mkiv @@ -45,7 +45,7 @@ %D {YYYY.MM.DD HH:MM} format. \edef\contextformat {\jobname} -\edef\contextversion{2019.11.14 17:07} +\edef\contextversion{2019.11.25 17:28} \edef\contextkind {beta} %D Kind of special: diff --git a/tex/context/base/mkiv/context.mkxl b/tex/context/base/mkiv/context.mkxl index 68c401bd8..95a7f2f8e 100644 --- a/tex/context/base/mkiv/context.mkxl +++ b/tex/context/base/mkiv/context.mkxl @@ -29,7 +29,7 @@ %D {YYYY.MM.DD HH:MM} format. \edef\contextformat {\jobname} -\edef\contextversion{2019.11.14 17:07} +\edef\contextversion{2019.11.25 17:28} \edef\contextkind {beta} %D Kind of special: diff --git a/tex/context/base/mkiv/data-aux.lua b/tex/context/base/mkiv/data-aux.lua index c57f16d2c..028794e51 100644 --- a/tex/context/base/mkiv/data-aux.lua +++ b/tex/context/base/mkiv/data-aux.lua @@ -8,29 +8,33 @@ if not modules then modules = { } end modules ['data-aux'] = { local find = string.find local type, next = type, next +local addsuffix, removesuffix = file.addsuffix, file.removesuffix +local loaddata, savedata = io.loaddata, io.savedata local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) local resolvers = resolvers +local cleanpath = resolvers.cleanpath +local findfiles = resolvers.findfiles local report_scripts = logs.reporter("resolvers","scripts") function resolvers.updatescript(oldname,newname) -- oldname -> own.name, not per se a suffix -- local scriptpath = "scripts/context/lua" local scriptpath = "context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = resolvers.cleanpath(oldname) + local oldscript = cleanpath(oldname) + local newname = addsuffix(newname,"lua") + local newscripts = findfiles(newname) or { } if trace_locating then report_scripts("to be replaced old script %a", oldscript) end - local newscripts = resolvers.findfiles(newname) or { } if #newscripts == 0 then if trace_locating then report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript = resolvers.cleanpath(newscripts[i]) + local newscript = cleanpath(newscripts[i]) if trace_locating then report_scripts("checking new script %a", newscript) end @@ -42,17 +46,17 @@ function resolvers.updatescript(oldname,newname) -- oldname -> own.name, not per if trace_locating then report_scripts("new script should come from %a",scriptpath) end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + elseif not (find(oldscript,removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then report_scripts("invalid new script name") end else - local newdata = io.loaddata(newscript) + local newdata = loaddata(newscript) if newdata then if trace_locating then report_scripts("old script content replaced by new content: %s",oldscript) end - io.savedata(oldscript,newdata) + savedata(oldscript,newdata) break elseif trace_locating then report_scripts("unable to load new script") diff --git a/tex/context/base/mkiv/data-bin.lua b/tex/context/base/mkiv/data-bin.lua index 1d1e8b749..2bac901ea 100644 --- a/tex/context/base/mkiv/data-bin.lua +++ b/tex/context/base/mkiv/data-bin.lua @@ -6,22 +6,25 @@ if not modules then modules = { } end modules ['data-bin'] = { license = "see context related readme files" } -local resolvers = resolvers +local resolvers = resolvers local methodhandler = resolvers.methodhandler +local notfound = resolvers.loaders.notfound function resolvers.findbinfile(filename,filetype) return methodhandler('finders',filename,filetype) end -function resolvers.openbinfile(filename) +local function openbinfile(filename) return methodhandler('loaders',filename) -- a bit weird: load end +resolvers.openbinfile = openbinfile + function resolvers.loadbinfile(filename,filetype) local fname = methodhandler('finders',filename,filetype) if fname and fname ~= "" then - return resolvers.openbinfile(fname) -- a bit weird: open + return openbinfile(fname) -- a bit weird: open else - return resolvers.loaders.notfound() + return notfound() end end diff --git a/tex/context/base/mkiv/data-con.lua b/tex/context/base/mkiv/data-con.lua index c79fca7c5..ec2251b35 100644 --- a/tex/context/base/mkiv/data-con.lua +++ b/tex/context/base/mkiv/data-con.lua @@ -6,6 +6,7 @@ if not modules then modules = { } end modules ['data-con'] = { license = "see context related readme files" } +local setmetatable = setmetatable local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) @@ -13,21 +14,28 @@ local trace_containers = false trackers.register("resolvers.containers", functi local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) --[[ldx-- -<p>Once we found ourselves defining similar cache constructs -several times, containers were introduced. Containers are used -to collect tables in memory and reuse them when possible based -on (unique) hashes (to be provided by the calling function).</p> +<p>Once we found ourselves defining similar cache constructs several times, +containers were introduced. Containers are used to collect tables in memory and +reuse them when possible based on (unique) hashes (to be provided by the calling +function).</p> -<p>Caching to disk is disabled by default. Version numbers are -stored in the saved table which makes it possible to change the -table structures without bothering about the disk cache.</p> +<p>Caching to disk is disabled by default. Version numbers are stored in the +saved table which makes it possible to change the table structures without +bothering about the disk cache.</p> -<p>Examples of usage can be found in the font related code.</p> +<p>Examples of usage can be found in the font related code. This code is not +ideal but we need it in generic too so we compromise.</p> --ldx]]-- -containers = containers or { } -local containers = containers -containers.usecache = true +containers = containers or { } +local containers = containers +containers.usecache = true + +local getwritablepath = caches.getwritablepath +local getreadablepaths = caches.getreadablepaths +local cacheiswritable = caches.is_writable +local loaddatafromcache = caches.loaddata +local savedataincache = caches.savedata local report_containers = logs.reporter("resolvers","containers") @@ -36,11 +44,11 @@ local allocated = { } local mt = { __index = function(t,k) if k == "writable" then - local writable = caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable = getwritablepath(t.category,t.subcategory) or { "." } t.writable = writable return writable elseif k == "readables" then - local readables = caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables = getreadablepaths(t.category,t.subcategory) or { "." } t.readables = readables return readables end @@ -64,8 +72,8 @@ function containers.define(category, subcategory, version, enabled) enabled = enabled, version = version or math.pi, -- after all, this is TeX trace = false, - -- writable = caches.getwritablepath and caches.getwritablepath (category,subcategory) or { "." }, - -- readables = caches.getreadablepaths and caches.getreadablepaths(category,subcategory) or { "." }, + -- writable = getwritablepath and getwritablepath (category,subcategory) or { "." }, + -- readables = getreadablepaths and getreadablepaths(category,subcategory) or { "." }, } setmetatable(s,mt) c[subcategory] = s @@ -75,7 +83,7 @@ function containers.define(category, subcategory, version, enabled) end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable, name) + return container.enabled and caches and cacheiswritable(container.writable, name) end function containers.is_valid(container,name) @@ -91,7 +99,7 @@ function containers.read(container,name) local storage = container.storage local stored = storage[name] if not stored and container.enabled and caches and containers.usecache then - stored = caches.loaddata(container.readables,name,container.writable) + stored = loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version == container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -108,17 +116,20 @@ function containers.read(container,name) return stored end -function containers.write(container, name, data) +function containers.write(container, name, data, fast) if data then data.cache_version = container.version if container.enabled and caches then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.writable, name, data) + local unique = data.unique + local shared = data.shared + data.unique = nil + data.shared = nil + savedataincache(container.writable, name, data, fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique, data.shared = unique, shared + data.unique = unique + data.shared = shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) diff --git a/tex/context/base/mkiv/data-env.lua b/tex/context/base/mkiv/data-env.lua index 5f9b579cd..53f0c27cc 100644 --- a/tex/context/base/mkiv/data-env.lua +++ b/tex/context/base/mkiv/data-env.lua @@ -7,7 +7,7 @@ if not modules then modules = { } end modules ['data-env'] = { } local lower, gsub = string.lower, string.gsub -local next = next +local next, rawget = next, rawget local resolvers = resolvers diff --git a/tex/context/base/mkiv/data-exp.lua b/tex/context/base/mkiv/data-exp.lua index ce7f95222..1ebdeb1e5 100644 --- a/tex/context/base/mkiv/data-exp.lua +++ b/tex/context/base/mkiv/data-exp.lua @@ -423,7 +423,7 @@ local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) return files, remap, n, m, r end -function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) +local function scanfiles(path,branch,usecache,onlyonce,tolerant) local realpath = resolveprefix(path) if usecache then local content = fullcache[realpath] @@ -480,8 +480,10 @@ function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) return content end +resolvers.scanfiles = scanfiles + function resolvers.simplescanfiles(path,branch,usecache) - return resolvers.scanfiles(path,branch,usecache,true,true) -- onlyonce + return scanfiles(path,branch,usecache,true,true) -- onlyonce end function resolvers.scandata() diff --git a/tex/context/base/mkiv/data-fil.lua b/tex/context/base/mkiv/data-fil.lua index b699fc9e3..bbcc954b2 100644 --- a/tex/context/base/mkiv/data-fil.lua +++ b/tex/context/base/mkiv/data-fil.lua @@ -6,48 +6,57 @@ if not modules then modules = { } end modules ['data-fil'] = { license = "see context related readme files" } +local ioopen = io.open +local isdir = lfs.isdir + local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) local report_files = logs.reporter("resolvers","files") -local resolvers = resolvers -local resolveprefix = resolvers.resolve +local resolvers = resolvers +local resolveprefix = resolvers.resolve +local findfile = resolvers.findfile +local scanfiles = resolvers.scanfiles +local registerfilehash = resolvers.registerfilehash +local appendhash = resolvers.appendhash -local finders, openers, loaders, savers = resolvers.finders, resolvers.openers, resolvers.loaders, resolvers.savers -local locators, hashers, generators, concatinators = resolvers.locators, resolvers.hashers, resolvers.generators, resolvers.concatinators +local loadcachecontent = caches.loadcontent -local checkgarbage = utilities.garbagecollector and utilities.garbagecollector.check +local checkgarbage = utilities.garbagecollector and utilities.garbagecollector.check -function locators.file(specification) +function resolvers.locators.file(specification) local filename = specification.filename local realname = resolveprefix(filename) -- no shortcut - if realname and realname ~= '' and lfs.isdir(realname) then + if realname and realname ~= '' and isdir(realname) then if trace_locating then report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',filename,true) -- cache + appendhash('file',filename,true) -- cache elseif trace_locating then report_files("file locator %a not found",filename) end end -function hashers.file(specification) +function resolvers.hashers.file(specification) local pathname = specification.filename - local content = caches.loadcontent(pathname,'files') - resolvers.registerfilehash(pathname,content,content==nil) + local content = loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) end -function generators.file(specification) +function resolvers.generators.file(specification) local pathname = specification.filename - local content = resolvers.scanfiles(pathname,false,true) -- scan once - resolvers.registerfilehash(pathname,content,true) + local content = scanfiles(pathname,false,true) -- scan once + registerfilehash(pathname,content,true) end -concatinators.file = file.join +resolvers.concatinators.file = file.join + +local finders = resolvers.finders +local notfound = finders.notfound function finders.file(specification,filetype) local filename = specification.filename - local foundname = resolvers.findfile(filename,filetype) + local foundname = findfile(filename,filetype) if foundname and foundname ~= "" then if trace_locating then report_files("file finder: %a found",filename) @@ -57,42 +66,64 @@ function finders.file(specification,filetype) if trace_locating then report_files("file finder: %a not found",filename) end - return finders.notfound() + return notfound() end end -- The default textopener will be overloaded later on. -function openers.helpers.textopener(tag,filename,f) +local openers = resolvers.openers +local notfound = openers.notfound +local overloaded = false + +local function textopener(tag,filename,f) return { - reader = function() return f:read () end, - close = function() logs.show_close(filename) return f:close() end, + reader = function() return f:read () end, + close = function() return f:close() end, } end +function openers.helpers.textopener(...) + return textopener(...) +end + +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded = true + textopener = opener + end +end + function openers.file(specification,filetype) local filename = specification.filename if filename and filename ~= "" then - local f = io.open(filename,"r") + local f = ioopen(filename,"r") if f then if trace_locating then report_files("file opener: %a opened",filename) end - return openers.helpers.textopener("file",filename,f) + return textopener("file",filename,f) end end if trace_locating then report_files("file opener: %a not found",filename) end - return openers.notfound() + return notfound() end +local loaders = resolvers.loaders +local notfound = loaders.notfound + function loaders.file(specification,filetype) local filename = specification.filename if filename and filename ~= "" then - local f = io.open(filename,"rb") + local f = ioopen(filename,"rb") if f then - logs.show_load(filename) if trace_locating then report_files("file loader: %a loaded",filename) end @@ -109,5 +140,5 @@ function loaders.file(specification,filetype) if trace_locating then report_files("file loader: %a not found",filename) end - return loaders.notfound() + return notfound() end diff --git a/tex/context/base/mkiv/data-ini.lua b/tex/context/base/mkiv/data-ini.lua index c4e1bf468..0fe70b5f1 100644 --- a/tex/context/base/mkiv/data-ini.lua +++ b/tex/context/base/mkiv/data-ini.lua @@ -10,6 +10,7 @@ local next, type, getmetatable, rawset = next, type, getmetatable, rawset local gsub, find, gmatch, char = string.gsub, string.find, string.gmatch, string.char local filedirname, filebasename, filejoin = file.dirname, file.basename, file.join local ostype, osname, osuname, ossetenv, osgetenv = os.type, os.name, os.uname, os.setenv, os.getenv +local sortedpairs = table.sortedpairs local P, S, R, C, Cs, Cc, lpegmatch = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.match @@ -332,7 +333,7 @@ if ostype == "unix" then rawset(t,k,v) end local colon = P(":") - for k, v in table.sortedpairs(prefixes) do + for k, v in sortedpairs(prefixes) do if p then p = P(k) + p else diff --git a/tex/context/base/mkiv/data-lst.lua b/tex/context/base/mkiv/data-lst.lua index f061393e5..8e0061c4d 100644 --- a/tex/context/base/mkiv/data-lst.lua +++ b/tex/context/base/mkiv/data-lst.lua @@ -9,46 +9,44 @@ if not modules then modules = { } end modules ['data-lst'] = { -- used in mtxrun, can be loaded later .. todo local type = type -local concat, sortedhash = table.concat,table.sortedhash +local sortedhash = table.sortedhash +local isdir = lfs.isdir -local resolvers = resolvers -local listers = resolvers.listers or { } -resolvers.listers = listers +local resolvers = resolvers +local listers = resolvers.listers or { } +resolvers.listers = listers -local resolveprefix = resolvers.resolve +local resolveprefix = resolvers.resolve +local configurationfiles = resolvers.configurationfiles +local expandedpathfromlist = resolvers.expandedpathfromlist +local splitpath = resolvers.splitpath +local knownvariables = resolvers.knownvariables -local report_lists = logs.reporter("resolvers","lists") -local report_resolved = logs.reporter("system","resolved") - -local function tabstr(str) - if type(str) == 'table' then - return concat(str," | ") - else - return str - end -end +local report_lists = logs.reporter("resolvers","lists") +local report_resolved = logs.reporter("system","resolved") function listers.variables(pattern) local result = resolvers.knownvariables(pattern) + local unset = { "unset" } for key, value in sortedhash(result) do report_lists(key) - report_lists(" env: %s",tabstr(value.environment or "unset")) - report_lists(" var: %s",tabstr(value.variable or "unset")) - report_lists(" exp: %s",tabstr(value.expansion or "unset")) - report_lists(" res: %s",tabstr(value.resolved or "unset")) + report_lists(" env: % | t",value.environment or unset) + report_lists(" var: % | t",value.variable or unset) + report_lists(" exp: % | t",value.expansion or unset) + report_lists(" res: % | t",value.resolved or unset) end end function listers.configurations() - local configurations = resolvers.configurationfiles() + local configurations = configurationfiles() for i=1,#configurations do report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") - local list = resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + local list = expandedpathfromlist(splitpath(resolvers.luacnfspec)) for i=1,#list do local li = resolveprefix(list[i]) - if lfs.isdir(li) then + if isdir(li) then report_resolved("path - %s",li) else report_resolved("path + %s",li) diff --git a/tex/context/base/mkiv/data-lua.lua b/tex/context/base/mkiv/data-lua.lua index 3022ae550..03d899e84 100644 --- a/tex/context/base/mkiv/data-lua.lua +++ b/tex/context/base/mkiv/data-lua.lua @@ -10,8 +10,7 @@ if not modules then modules = { } end modules ['data-lua'] = { local package, lpeg = package, lpeg -local gsub = string.gsub -local concat = table.concat +local loadfile = loadfile local addsuffix = file.addsuffix local P, S, Cs, lpegmatch = lpeg.P, lpeg.S, lpeg.Cs, lpeg.match @@ -25,6 +24,8 @@ local methods = helpers.methods or { } local resolvers = resolvers local resolveprefix = resolvers.resolve +local expandedpaths = resolvers.expandedpathlistfromvariable +local findfile = resolvers.findfile helpers.report = logs.reporter("resolvers","libraries") @@ -50,9 +51,9 @@ function helpers.cleanpath(path) -- hm, don't we have a helper for this? return resolveprefix(lpegmatch(pattern,path)) end -local loadedaslib = helpers.loadedaslib -local registerpath = helpers.registerpath -local lualibfile = helpers.lualibfile +local loadedaslib = helpers.loadedaslib +local registerpath = helpers.registerpath +local lualibfile = helpers.lualibfile local luaformatpaths local libformatpaths @@ -61,7 +62,7 @@ local function getluaformatpaths() if not luaformatpaths then luaformatpaths = { } for i=1,#luaformats do - registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + registerpath("lua format","lua",luaformatpaths,expandedpaths(luaformats[i])) end end return luaformatpaths @@ -71,7 +72,7 @@ local function getlibformatpaths() if not libformatpaths then libformatpaths = { } for i=1,#libformats do - registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + registerpath("lib format","lib",libformatpaths,expandedpaths(libformats[i])) end end return libformatpaths @@ -81,8 +82,8 @@ local function loadedbyformat(name,rawname,suffixes,islib,what) local trace = helpers.trace local report = helpers.report for i=1,#suffixes do -- so we use findfile and not a lookup loop - local format = suffixes[i] - local resolved = resolvers.findfile(name,format) or "" + local format = suffixes[i] + local resolved = findfile(name,format) or "" if trace then report("%s format, identifying %a using format %a",what,name,format) end diff --git a/tex/context/base/mkiv/data-met.lua b/tex/context/base/mkiv/data-met.lua index bb8929577..8004ad2cd 100644 --- a/tex/context/base/mkiv/data-met.lua +++ b/tex/context/base/mkiv/data-met.lua @@ -6,9 +6,12 @@ if not modules then modules = { } end modules ['data-met'] = { license = "see context related readme files" } -local find, format = string.find, string.format -local sequenced = table.sequenced +local type = type +local find = string.find local addurlscheme, urlhashed = url.addscheme, url.hashed +local collapsepath, joinfile = file.collapsepath, file.join + +local report_methods = logs.reporter("resolvers","methods") local trace_locating = false local trace_methods = false @@ -16,31 +19,38 @@ local trace_methods = false trackers.register("resolvers.locating", function(v) trace_methods = v end) trackers.register("resolvers.methods", function(v) trace_methods = v end) ---~ trace_methods = true - -local report_methods = logs.reporter("resolvers","methods") - -local allocate = utilities.storage.allocate - -local resolvers = resolvers +local allocate = utilities.storage.allocate +local resolvers = resolvers local registered = { } local function splitmethod(filename) -- todo: filetype in specification if not filename then - return { scheme = "unknown", original = filename } + return { + scheme = "unknown", + original = filename, + } end if type(filename) == "table" then return filename -- already split end - filename = file.collapsepath(filename,".") -- hm, we should keep ./ in some cases - + filename = collapsepath(filename,".") -- hm, we should keep ./ in some cases if not find(filename,"://",1,true) then - return { scheme = "file", path = filename, original = filename, filename = filename } + return { + scheme = "file", + path = filename, + original = filename, + filename = filename, + } end - local specification = url.hashed(filename) + local specification = urlhashed(filename) if not specification.scheme or specification.scheme == "" then - return { scheme = "file", path = filename, original = filename, filename = filename } + return { + scheme = "file", + path = filename, + original = filename, + filename = filename, + } else return specification end @@ -53,7 +63,7 @@ end -- if type(filename) == "table" then -- return filename -- already split -- end --- return url.hashed(filename) +-- return urlhashed(filename) -- end resolvers.splitmethod = splitmethod -- bad name but ok @@ -64,11 +74,12 @@ resolvers.splitmethod = splitmethod -- bad name but ok local function methodhandler(what,first,...) -- filename can be nil or false local method = registered[what] if method then - local how, namespace = method.how, method.namespace + local how = method.how + local namespace = method.namespace if how == "uri" or how == "url" then local specification = splitmethod(first) - local scheme = specification.scheme - local resolver = namespace and namespace[scheme] + local scheme = specification.scheme + local resolver = namespace and namespace[scheme] if resolver then if trace_methods then report_methods("resolving, method %a, how %a, handler %a, argument %a",what,how,scheme,first) @@ -112,7 +123,10 @@ end resolvers.methodhandler = methodhandler function resolvers.registermethod(name,namespace,how) - registered[name] = { how = how or "tag", namespace = namespace } + registered[name] = { + how = how or "tag", + namespace = namespace + } namespace["byscheme"] = function(scheme,filename,...) if scheme == "file" then return methodhandler(name,filename,...) @@ -122,7 +136,7 @@ function resolvers.registermethod(name,namespace,how) end end -local concatinators = allocate { notfound = file.join } -- concatinate paths +local concatinators = allocate { notfound = joinfile } -- concatinate paths local locators = allocate { notfound = function() end } -- locate databases local hashers = allocate { notfound = function() end } -- load databases local generators = allocate { notfound = function() end } -- generate databases diff --git a/tex/context/base/mkiv/data-out.lua b/tex/context/base/mkiv/data-out.lua index 11304c2ce..6cd02a22e 100644 --- a/tex/context/base/mkiv/data-out.lua +++ b/tex/context/base/mkiv/data-out.lua @@ -6,13 +6,12 @@ if not modules then modules = { } end modules ['data-out'] = { license = "see context related readme files" } -local allocate = utilities.storage.allocate -local resolvers = resolvers +local allocate = utilities.storage.allocate +local resolvers = resolvers local registermethod = resolvers.registermethod -local savers = allocate { helpers = { } } - -resolvers.savers = savers +local savers = allocate { helpers = { } } +resolvers.savers = savers registermethod("savers", savers, "uri") diff --git a/tex/context/base/mkiv/data-pre.lua b/tex/context/base/mkiv/data-pre.lua index 5e3020b70..f7df8b918 100644 --- a/tex/context/base/mkiv/data-pre.lua +++ b/tex/context/base/mkiv/data-pre.lua @@ -25,6 +25,8 @@ if not modules then modules = { } end modules ['data-pre'] = { -- version : operating system version -- release : operating system release +local insert, remove = table.insert, table.remove + local resolvers = resolvers local prefixes = resolvers.prefixes @@ -121,8 +123,10 @@ prefixes.path = prefixes.pathname -- This one assumes that inputstack is set (used in the tex loader). It is a momentary resolve -- as the top of the input stack changes. +local inputstack = { } +local stackpath = resolvers.stackpath + local function toppath() - local inputstack = resolvers.inputstack -- dependency, actually the code should move but it's if not inputstack then -- more convenient to keep it here return "." end @@ -139,7 +143,7 @@ end -- stucture. local function jobpath() - local path = resolvers.stackpath() + local path = stackpath() if not path or path == "" then return "." else @@ -147,8 +151,18 @@ local function jobpath() end end -resolvers.toppath = toppath -resolvers.jobpath = jobpath +local function pushinputname(name) + insert(inputstack,name) +end + +local function popinputname(name) + return remove(inputstack) +end + +resolvers.toppath = toppath +resolvers.jobpath = jobpath +resolvers.pushinputname = pushinputname +resolvers.popinputname = popinputname -- This hook sit into the resolver: diff --git a/tex/context/base/mkiv/data-res.lua b/tex/context/base/mkiv/data-res.lua index 15d9f3db9..01b8f1aca 100644 --- a/tex/context/base/mkiv/data-res.lua +++ b/tex/context/base/mkiv/data-res.lua @@ -10,7 +10,9 @@ if not modules then modules = { } end modules ['data-res'] = { local gsub, find, lower, upper, match, gmatch = string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch local concat, insert, remove = table.concat, table.insert, table.remove -local next, type, rawget = next, type, rawget +local next, type, rawget, loadfile = next, type, rawget, loadfile +local mergedtable = table.merged + local os = os local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg @@ -30,6 +32,8 @@ local is_qualified_path = file.is_qualified_path local allocate = utilities.storage.allocate local settings_to_array = utilities.parsers.settings_to_array +local urlhasscheme = url.hasscheme + local getcurrentdir = lfs.currentdir local isfile = lfs.isfile local isdir = lfs.isdir @@ -63,11 +67,19 @@ local ostype, osname, osenv, ossetenv, osgetenv = os.type, os.name, os.env, os.s resolvers.cacheversion = "1.100" resolvers.configbanner = "" resolvers.homedir = environment.homedir -resolvers.criticalvars = allocate { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF", "TEXMF", "TEXOS" } resolvers.luacnfname = "texmfcnf.lua" resolvers.luacnffallback = "contextcnf.lua" resolvers.luacnfstate = "unknown" +local criticalvars = { + "SELFAUTOLOC", + "SELFAUTODIR", + "SELFAUTOPARENT", + "TEXMFCNF", + "TEXMF", + "TEXOS", +} + -- The web2c tex binaries as well as kpse have built in paths for the configuration -- files and there can be a depressing truckload of them. This is actually the weak -- spot of a distribution. So we don't want: @@ -145,14 +157,27 @@ local suffixmap = resolvers.suffixmap resolvers.defaultsuffixes = { "tex" } -- "mkiv", "cld" -- too tricky -local instance = nil -- the current one (fast access) +local instance = nil -- the current one (fast access) + +-- forward declarations + +local variable +local expansion +local setenv +local getenv + +-- done + +local formatofsuffix = resolvers.formatofsuffix +local splitpath = resolvers.splitpath +local splitmethod = resolvers.splitmethod -- An instance has an environment (coming from the outside, kept raw), variables -- (coming from the configuration file), and expansions (variables with nested -- variables replaced). One can push something into the outer environment and -- its internal copy, but only the later one will be the raw unprefixed variant. -function resolvers.setenv(key,value,raw) +setenv = function(key,value,raw) if instance then -- this one will be consulted first when we stay inside -- the current environment (prefixes are not resolved here) @@ -167,7 +192,7 @@ end -- Beware we don't want empty here as this one can be called early on -- and therefore we use rawget. -local function getenv(key) +getenv = function(key) local value = rawget(instance.environment,key) if value and value ~= "" then return value @@ -178,115 +203,121 @@ local function getenv(key) end resolvers.getenv = getenv -resolvers.env = getenv +resolvers.setenv = setenv -- We are going to use some metatable trickery where we backtrack from -- expansion to variable to environment. -local function resolvevariable(k) - return instance.expansions[k] -end - local dollarstripper = lpeg.stripper("$") local inhibitstripper = P("!")^0 * Cs(P(1)^0) -local somevariable = P("$") / "" -local somekey = C(R("az","AZ","09","__","--")^1) -local somethingelse = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "") - + P(";") * (P(";") / "") - + P(1) -local variableexpander = Cs( (somevariable * (somekey/resolvevariable) + somethingelse)^1 ) +local expandedvariable, resolvedvariable do -local cleaner = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";" -local variablecleaner = Cs((cleaner + P(1))^0) + local function resolveinstancevariable(k) + return instance.expansions[k] + end -local somevariable = R("az","AZ","09","__","--")^1 / resolvevariable -local variable = (P("$")/"") * (somevariable + (P("{")/"") * somevariable * (P("}")/"")) -local variableresolver = Cs((variable + P(1))^0) + local p_variable = P("$") / "" + local p_key = C(R("az","AZ","09","__","--")^1) + local p_whatever = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "") + + P(";") * (P(";") / "") + + P(1) + local variableexpander = Cs( (p_variable * (p_key/resolveinstancevariable) + p_whatever)^1 ) -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end + local p_cleaner = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";" + local variablecleaner = Cs((p_cleaner + P(1))^0) -function resolvers.reset() + local p_variable = R("az","AZ","09","__","--")^1 / resolveinstancevariable + local p_variable = (P("$")/"") * (p_variable + (P("{")/"") * p_variable * (P("}")/"")) + local variableresolver = Cs((p_variable + P(1))^0) - -- normally we only need one instance but for special cases we can (re)load one so - -- we stick to this model. + expandedvariable = function(var) + return lpegmatch(variableexpander,var) or var + end - if trace_locating then - report_resolving("creating instance") - end - - local environment = { } - local variables = { } - local expansions = { } - local order = { } - - instance = { - environment = environment, - variables = variables, - expansions = expansions, - order = order, - files = { }, - setups = { }, - found = { }, - foundintrees = { }, - hashes = { }, - hashed = { }, - pathlists = false,-- delayed - specification = { }, - lists = { }, - data = { }, -- only for loading - fakepaths = { }, - remember = true, - diskcache = true, - renewcache = false, - renewtree = false, - loaderror = false, - savelists = true, - pattern = nil, -- lists - force_suffixes = true, - pathstack = { }, - } - - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v = order[i][k] - if v ~= nil then - t[k] = v - return v + function resolvers.reset() + + -- normally we only need one instance but for special cases we can (re)load one so + -- we stick to this model. + + if trace_locating then + report_resolving("creating instance") + end + + local environment = { } + local variables = { } + local expansions = { } + local order = { } + + instance = { + environment = environment, + variables = variables, + expansions = expansions, + order = order, + files = { }, + setups = { }, + found = { }, + foundintrees = { }, + hashes = { }, + hashed = { }, + pathlists = false,-- delayed + specification = { }, + lists = { }, + data = { }, -- only for loading + fakepaths = { }, + remember = true, + diskcache = true, + renewcache = false, + renewtree = false, + loaderror = false, + savelists = true, + pattern = nil, -- lists + force_suffixes = true, + pathstack = { }, + } + + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v = order[i][k] + if v ~= nil then + t[k] = v + return v + end end - end - if v == nil then - v = "" - end - t[k] = v - return v - end) + if v == nil then + v = "" + end + t[k] = v + return v + end) - setmetatableindex(environment, function(t,k) - local v = osgetenv(k) - if v == nil then - v = variables[k] - end - if v ~= nil then - v = checkedvariable(v) or "" - end - v = resolvers.repath(v) -- for taco who has a : separated osfontdir - t[k] = v - return v - end) + local repath = resolvers.repath - setmetatableindex(expansions, function(t,k) - local v = environment[k] - if type(v) == "string" then - v = lpegmatch(variableresolver,v) - v = lpegmatch(variablecleaner,v) - end - t[k] = v - return v - end) + setmetatableindex(environment, function(t,k) + local v = osgetenv(k) + if v == nil then + v = variables[k] + end + if v ~= nil then + v = checkedvariable(v) or "" + end + v = repath(v) -- for taco who has a : separated osfontdir + t[k] = v + return v + end) + + setmetatableindex(expansions, function(t,k) + local v = environment[k] + if type(v) == "string" then + v = lpegmatch(variableresolver,v) + v = lpegmatch(variablecleaner,v) + end + t[k] = v + return v + end) + + end end @@ -305,40 +336,44 @@ local function reset_caches() instance.pathlists = false end -local slash = P("/") +local makepathexpression do -local pathexpressionpattern = Cs ( -- create lpeg instead (2013/2014) - Cc("^") * ( - Cc("%") * S(".-") - + slash^2 * P(-1) / "/.*" - -- + slash^2 / "/.-/" - -- + slash^2 / "/[^/]*/*" -- too general - + slash^2 / "/" - + (1-slash) * P(-1) * Cc("/") - + P(1) - )^1 * Cc("$") -- yes or no $ -) + local slash = P("/") -local cache = { } + local pathexpressionpattern = Cs ( -- create lpeg instead (2013/2014) + Cc("^") * ( + Cc("%") * S(".-") + + slash^2 * P(-1) / "/.*" + -- + slash^2 / "/.-/" + -- + slash^2 / "/[^/]*/*" -- too general + + slash^2 / "/" + + (1-slash) * P(-1) * Cc("/") + + P(1) + )^1 * Cc("$") -- yes or no $ + ) -local function makepathexpression(str) - if str == "." then - return "^%./$" - else - local c = cache[str] - if not c then - c = lpegmatch(pathexpressionpattern,str) - cache[str] = c + local cache = { } + + makepathexpression = function(str) + if str == "." then + return "^%./$" + else + local c = cache[str] + if not c then + c = lpegmatch(pathexpressionpattern,str) + cache[str] = c + end + return c end - return c end + end local function reportcriticalvariables(cnfspec) if trace_locating then - for i=1,#resolvers.criticalvars do - local k = resolvers.criticalvars[i] - local v = resolvers.getenv(k) or "unknown" -- this one will not resolve ! + for i=1,#criticalvars do + local k = criticalvars[i] + local v = getenv(k) or "unknown" -- this one will not resolve ! report_resolving("variable %a set to %a",k,v) end report_resolving() @@ -361,7 +396,7 @@ local function identify_configuration_files() resolvers.luacnfstate = "environment" end reportcriticalvariables(cnfspec) - local cnfpaths = expandedpathfromlist(resolvers.splitpath(cnfspec)) + local cnfpaths = expandedpathfromlist(splitpath(cnfspec)) local function locatecnf(luacnfname,kind) for i=1,#cnfpaths do @@ -398,6 +433,8 @@ end local function load_configuration_files() local specification = instance.specification + local setups = instance.setups + local order = instance.order if #specification > 0 then local luacnfname = resolvers.luacnfname for i=1,#specification do @@ -407,7 +444,6 @@ local function load_configuration_files() local realname = resolveprefix(filename) -- no shortcut local blob = loadfile(realname) if blob then - local setups = instance.setups local data = blob() local parent = data and data.parent if parent then @@ -418,7 +454,7 @@ local function load_configuration_files() local parentdata = blob() if parentdata then report_resolving("loading configuration file %a",filename) - data = table.merged(parentdata,data) + data = mergedtable(parentdata,data) end end end @@ -454,7 +490,7 @@ local function load_configuration_files() -- we push the value into the main environment (osenv) so -- that it takes precedence over the default one and therefore -- also over following definitions - resolvers.setenv("TEXMFCNF",cnfspec) -- resolves prefixes + setenv("TEXMFCNF",cnfspec) -- resolves prefixes -- we now identify and load the specified configuration files instance.specification = { } identify_configuration_files() @@ -476,7 +512,7 @@ local function load_configuration_files() elseif trace_locating then report_resolving("skipping configuration file %a (no valid format)",filename) end - instance.order[#instance.order+1] = instance.setups[pathname] + order[#order+1] = setups[pathname] if instance.loaderror then break end @@ -486,6 +522,13 @@ local function load_configuration_files() end end +-- forward declarations: + +local expandedpathlist +local unexpandedpathlist + +-- done + function resolvers.configurationfiles() return instance.specification or { } end @@ -507,7 +550,7 @@ end local function locate_file_databases() -- todo: cache:// and tree:// (runtime) - local texmfpaths = resolvers.expandedpathlist("TEXMF") + local texmfpaths = expandedpathlist("TEXMF") if #texmfpaths > 0 then for i=1,#texmfpaths do local path = collapsepath(texmfpaths[i]) @@ -516,7 +559,7 @@ local function locate_file_databases() if stripped ~= "" then local runtime = stripped == path path = cleanpath(path) - local spec = resolvers.splitmethod(stripped) + local spec = splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme == "file") then stripped = "tree:///" .. stripped elseif spec.scheme == "cache" or spec.scheme == "file" then @@ -552,11 +595,13 @@ local function generate_file_databases() end local function save_file_databases() -- will become cachers - for i=1,#instance.hashes do - local hash = instance.hashes[i] + local hashes = instance.hashes + local files = instance.files + for i=1,#hashes do + local hash = hashes[i] local cachename = hash.name if hash.cache then - local content = instance.files[cachename] + local content = files[cachename] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",cachename) @@ -569,8 +614,9 @@ local function save_file_databases() -- will become cachers end function resolvers.renew(hashname) + local files = instance.files if hashname and hashname ~= "" then - local expanded = resolvers.expansion(hashname) or "" + local expanded = expansion(hashname) or "" if expanded ~= "" then if trace_locating then report_resolving("identifying tree %a from %a",expanded,hashname) @@ -588,7 +634,7 @@ function resolvers.renew(hashname) end methodhandler('generators',hashname) -- could be shared - local content = instance.files[hashname] + local content = files[hashname] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",hashname) @@ -618,35 +664,43 @@ local function load_databases() end function resolvers.appendhash(type,name,cache) - -- safeguard ... tricky as it's actually a bug when seen twice - if not instance.hashed[name] then + local hashed = instance.hashed + local hashes = instance.hashes + if hashed[name] then + -- safeguard ... tricky as it's actually a bug when seen twice + else if trace_locating then report_resolving("hash %a appended",name) end - insert(instance.hashes, { type = type, name = name, cache = cache } ) - instance.hashed[name] = cache + insert(hashes, { type = type, name = name, cache = cache } ) + hashed[name] = cache end end function resolvers.prependhash(type,name,cache) - -- safeguard ... tricky as it's actually a bug when seen twice - if not instance.hashed[name] then + local hashed = instance.hashed + local hashes = instance.hashes + if hashed[name] then + -- safeguard ... tricky as it's actually a bug when seen twice + else if trace_locating then report_resolving("hash %a prepended",name) end - insert(instance.hashes, 1, { type = type, name = name, cache = cache } ) - instance.hashed[name] = cache + insert(hashes, 1, { type = type, name = name, cache = cache } ) + hashed[name] = cache end end function resolvers.extendtexmfvariable(specification) -- crap, we could better prepend the hash - local t = resolvers.splitpath(getenv("TEXMF")) -- okay? - insert(t,1,specification) - local newspec = concat(t,",") -- not ; - if instance.environment["TEXMF"] then - instance.environment["TEXMF"] = newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"] = newspec + local environment = instance.environment + local variables = instance.variables + local texmftrees = splitpath(getenv("TEXMF")) -- okay? + insert(texmftrees,1,specification) + texmftrees = concat(texmftrees,",") -- not ; + if environment["TEXMF"] then + environment["TEXMF"] = texmftrees + elseif variables["TEXMF"] then + variables["TEXMF"] = texmftrees else -- weird end @@ -654,8 +708,8 @@ function resolvers.extendtexmfvariable(specification) -- crap, we could better p end function resolvers.splitexpansions() - local ie = instance.expansions - for k,v in next, ie do + local expansions = instance.expansions + for k, v in next, expansions do local t, tn, h, p = { }, 0, { }, splitconfigurationpath(v) for kk=1,#p do local vv = p[kk] @@ -665,10 +719,10 @@ function resolvers.splitexpansions() h[vv] = true end end - if #t > 1 then - ie[k] = t + if tn > 1 then + expansions[k] = t else - ie[k] = t[1] + expansions[k] = t[1] end end end @@ -682,26 +736,31 @@ function resolvers.datastate() return caches.contentstate() end -function resolvers.variable(name) +variable = function(name) + local variables = instance.variables local name = name and lpegmatch(dollarstripper,name) - local result = name and instance.variables[name] + local result = name and variables[name] return result ~= nil and result or "" end -function resolvers.expansion(name) +expansion = function(name) + local expansions =instance.expansions local name = name and lpegmatch(dollarstripper,name) - local result = name and instance.expansions[name] + local result = name and expansions[name] return result ~= nil and result or "" end -function resolvers.unexpandedpathlist(str) - local pth = resolvers.variable(str) - local lst = resolvers.splitpath(pth) +resolvers.variable = variable +resolvers.expansion = expansion + +unexpandedpathlist = function(str) + local pth = variable(str) + local lst = splitpath(pth) return expandedpathfromlist(lst) end function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(unexpandedpathlist(str)) end function resolvers.pushpath(name) @@ -736,8 +795,8 @@ end local done = { } function resolvers.resetextrapaths() - local ep = instance.extra_paths - if not ep then + local extra_paths = instance.extra_paths + if not extra_paths then done = { } instance.extra_paths = { } elseif #ep > 0 then @@ -760,8 +819,8 @@ function resolvers.registerextrapath(paths,subpaths) end local paths = settings_to_array(paths) local subpaths = settings_to_array(subpaths) - local ep = instance.extra_paths or { } - local oldn = #ep + local extra_paths = instance.extra_paths or { } + local oldn = #extra_paths local newn = oldn local nofpaths = #paths local nofsubpaths = #subpaths @@ -774,7 +833,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps = p .. "/" .. s if not done[ps] then newn = newn + 1 - ep[newn] = cleanpath(ps) + extra_paths[newn] = cleanpath(ps) done[ps] = true end end @@ -784,7 +843,7 @@ function resolvers.registerextrapath(paths,subpaths) local p = paths[i] if not done[p] then newn = newn + 1 - ep[newn] = cleanpath(p) + extra_paths[newn] = cleanpath(p) done[p] = true end end @@ -793,17 +852,17 @@ function resolvers.registerextrapath(paths,subpaths) for i=1,oldn do for j=1,nofsubpaths do local s = subpaths[j] - local ps = ep[i] .. "/" .. s + local ps = extra_paths[i] .. "/" .. s if not done[ps] then newn = newn + 1 - ep[newn] = cleanpath(ps) + extra_paths[newn] = cleanpath(ps) done[ps] = true end end end end if newn > 0 then - instance.extra_paths = ep -- register paths + instance.extra_paths = extra_paths -- register paths end if newn ~= oldn then reset_caches() @@ -812,8 +871,9 @@ end function resolvers.pushextrapath(path) local paths = settings_to_array(path) - if instance.extra_stack then - insert(instance.extra_stack,1,paths) + local extra_stack = instance.extra_stack + if extra_stack then + insert(extra_stack,1,paths) else instance.extra_stack = { paths } end @@ -821,9 +881,10 @@ function resolvers.pushextrapath(path) end function resolvers.popextrapath() - if instance.extra_stack then + local extra_stack = instance.extra_stack + if extra_stack then reset_caches() - return remove(instance.extra_stack,1) + return remove(extra_stack,1) end end @@ -856,17 +917,17 @@ local function made_list(instance,list,extra_too) end end if extra_too then + local extra_stack = instance.extra_stack + local extra_paths = instance.extra_paths -- first the stacked paths - local es = instance.extra_stack - if es and #es > 0 then - for k=1,#es do - add(es[k]) + if extra_stack and #extra_stack > 0 then + for k=1,#extra_stack do + add(extra_stack[k]) end end -- then the extra paths - local ep = instance.extra_paths - if ep and #ep > 0 then - add(ep) + if extra_paths and #extra_paths > 0 then + add(extra_paths) end end -- last the formal paths @@ -874,21 +935,7 @@ local function made_list(instance,list,extra_too) return new end -function resolvers.cleanpathlist(str) - local t = resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i] = collapsepath(cleanpath(t[i])) - end - end - return t -end - -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) -end - -function resolvers.expandedpathlist(str,extra_too) +expandedpathlist = function(str,extra_too) if not str then return { } elseif instance.savelists then -- hm, what if two cases, with and without extra_too @@ -896,45 +943,65 @@ function resolvers.expandedpathlist(str,extra_too) local lists = instance.lists local lst = lists[str] if not lst then - local l = made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + local l = made_list(instance,splitpath(expansion(str)),extra_too) lst = expandedpathfromlist(l) lists[str] = lst end return lst else - local lst = resolvers.splitpath(resolvers.expansion(str)) + local lst = splitpath(expansion(str)) return made_list(instance,expandedpathfromlist(lst),extra_too) end end -function resolvers.expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ // +resolvers.expandedpathlist = expandedpathlist +resolvers.unexpandedpathlist = unexpandedpathlist + +function resolvers.cleanpathlist(str) + local t = expandedpathlist(str) + if t then + for i=1,#t do + t[i] = collapsepath(cleanpath(t[i])) + end + end + return t +end + +function resolvers.expandpath(str) + return joinpath(expandedpathlist(str)) +end + +local function expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ // str = lpegmatch(dollarstripper,str) local tmp = resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp ~= "" and tmp or str) + return expandedpathlist(tmp ~= "" and tmp or str) end function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(expandedpathlistfromvariable(str)) end +resolvers.expandedpathlistfromvariable = expandedpathlistfromvariable + function resolvers.cleanedpathlist(v) -- can be cached if needed - local t = resolvers.expandedpathlist(v) + local t = expandedpathlist(v) for i=1,#t do - t[i] = resolvers.resolve(resolvers.cleanpath(t[i])) + t[i] = resolveprefix(cleanpath(t[i])) end return t end function resolvers.expandbraces(str) -- output variable and brace expansion of STRING - local pth = expandedpathfromlist(resolvers.splitpath(str)) + local pth = expandedpathfromlist(splitpath(str)) return joinpath(pth) end function resolvers.registerfilehash(name,content,someerror) + local files = instance.files if content then - instance.files[name] = content + files[name] = content else - instance.files[name] = { } + files[name] = { } if somerror == true then -- can be unset instance.loaderror = someerror end @@ -998,10 +1065,11 @@ local function collect_files(names) -- potential files .. sort of too much when pathname = "/" .. pathname .. "$" end local hashes = instance.hashes + local files = instance.files for h=1,#hashes do local hash = hashes[h] local hashname = hash.name - local content = hashname and instance.files[hashname] + local content = hashname and files[hashname] if content then if trace_details then report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) @@ -1089,7 +1157,6 @@ local function find_analyze(filename,askedformat,allresults) if askedformat == "" then if filesuffix == "" or not suffixmap[filesuffix] then local defaultsuffixes = resolvers.defaultsuffixes - local formatofsuffix = resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname = filename .. '.' .. defaultsuffixes[i] wantedfiles[#wantedfiles+1] = forcedname @@ -1099,7 +1166,7 @@ local function find_analyze(filename,askedformat,allresults) end end else - filetype = resolvers.formatofsuffix(filename) + filetype = formatofsuffix(filename) if trace_locating then report_resolving("using suffix based filetype %a",filetype) end @@ -1163,11 +1230,11 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) -- t local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat] if format_suffixes then for i=1,#format_suffixes do - local s = format_suffixes[i] - forcedname = filename .. "." .. s + local suffix = format_suffixes[i] + forcedname = filename .. "." .. suffix if isreadable(forcedname) then if trace_locating then - report_resolving("no suffix, forcing format filetype %a", s) + report_resolving("no suffix, forcing format filetype %a", suffix) end return "qualified", { forcedname } end @@ -1182,7 +1249,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) -- t local savedformat = askedformat local format = savedformat or "" if format == "" then - askedformat = resolvers.formatofsuffix(suffix) + askedformat = formatofsuffix(suffix) end if not format then askedformat = "othertextfiles" -- kind of everything, maybe all @@ -1244,7 +1311,7 @@ end local function makepathlist(list,filetype) local typespec = resolvers.variableofformat(filetype) - local pathlist = resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) -- only extra path with user files + local pathlist = expandedpathlist(typespec,filetype and usertypes[filetype]) -- only extra path with user files local entry = { } if pathlist and #pathlist > 0 then for k=1,#pathlist do @@ -1255,7 +1322,7 @@ local function makepathlist(list,filetype) local expression = makepathexpression(pathname) local barename = gsub(pathname,"/+$","") barename = resolveprefix(barename) - local scheme = url.hasscheme(barename) + local scheme = urlhasscheme(barename) local schemename = gsub(barename,"%.%*$",'') -- after scheme -- local prescanned = path ~= pathname -- ^!! -- local resursive = find(pathname,'//$') @@ -1480,10 +1547,8 @@ collect_instance_files = function(filename,askedformat,allresults) -- uses neste return { } end askedformat = askedformat or "" - filename = collapsepath(filename,".") - - filename = gsub(filename,"^%./",getcurrentdir().."/") -- we will merge dir.expandname and collapse some day - + filename = collapsepath(filename,".") + filename = gsub(filename,"^%./",getcurrentdir().."/") -- we will merge dir.expandname and collapse some day if allresults then -- no need for caching, only used for tracing local filetype, wantedfiles = find_analyze(filename,askedformat) @@ -1495,7 +1560,9 @@ collect_instance_files = function(filename,askedformat,allresults) -- uses neste { find_onpath (filename,filetype,wantedfiles,true) }, { find_otherwise(filename,filetype,wantedfiles,true) }, } - local result, status, done = { }, { }, { } + local result = { } + local status = { } + local done = { } for k, r in next, results do local method, list = r[1], r[2] if method and list then @@ -1570,6 +1637,9 @@ local function findfiles(filename,filetype,allresults) if not filename or filename == "" then return { } end + if allresults == nil then + allresults = true + end local result, status = collect_instance_files(filename,filetype or "",allresults) if not result or #result == 0 then local lowered = lower(filename) @@ -1580,32 +1650,29 @@ local function findfiles(filename,filetype,allresults) return result or { }, status end -function resolvers.findfiles(filename,filetype) - if not filename or filename == "" then - -- weird ... why called then - return "" - else - return findfiles(filename,filetype,true) - end -end - -function resolvers.findfile(filename,filetype) +local function findfile(filename,filetype) if not filename or filename == "" then - -- weird ... why called then return "" else return findfiles(filename,filetype,false)[1] or "" end end +resolvers.findfiles = findfiles +resolvers.findfile = findfile + +resolvers.find_file = findfile -- obsolete +resolvers.find_files = findfiles -- obsolete + function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) + local hashes = instance.hashes + local files = instance.files local base = filebasename(filename) local result = { } - local hashes = instance.hashes -- local function okay(hash,path,name) local found = methodhandler('concatinators',hash.type,hash.name,path,name) @@ -1617,7 +1684,7 @@ local function findgivenfiles(filename,allresults) -- for k=1,#hashes do local hash = hashes[k] - local content = instance.files[hash.name] + local content = files[hash.name] if content then local path, name = lookup(content,base) if not path then @@ -1660,15 +1727,16 @@ end -- why bother local function findwildcardfiles(filename,allresults,result) + local files = instance.files + local hashes = instance.hashes + -- local result = result or { } local base = filebasename(filename) local dirn = filedirname(filename) local path = lower(lpegmatch(makewildcard,dirn) or dirn) local name = lower(lpegmatch(makewildcard,base) or base) - local files = instance.files -- if find(name,"*",1,true) then - local hashes = instance.hashes local function okay(found,path,base,hashname,hashtype) if find(found,path) then local full = methodhandler('concatinators',hashtype,hashname,found,base) @@ -1709,7 +1777,6 @@ local function findwildcardfiles(filename,allresults,result) end end -- - local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] local hashname = hash.name @@ -1745,18 +1812,30 @@ function resolvers.findwildcardfile(filename) return findwildcardfiles(filename,false)[1] or "" end --- main user functions +do -function resolvers.automount() - -- implemented later -end + local starttiming = statistics.starttiming + local stoptiming = statistics.stoptiming + local elapsedtime = statistics.elapsedtime + + function resolvers.starttiming() + starttiming(instance) + end + + function resolvers.stoptiming() + stoptiming(instance) + end + + function resolvers.loadtime() + return elapsedtime(instance) + end -function resolvers.starttiming() - statistics.starttiming(instance) end -function resolvers.stoptiming() - statistics.stoptiming(instance) +-- main user functions + +function resolvers.automount() + -- implemented later (maybe a one-time setter like textopener) end function resolvers.load(option) @@ -1772,10 +1851,6 @@ function resolvers.load(option) return files and next(files) and true end -function resolvers.loadtime() - return statistics.elapsedtime(instance) -end - local function report(str) if trace_locating then report_resolving(str) -- has already verbose @@ -1806,13 +1881,11 @@ function resolvers.dowithfilesandreport(command, files, ...) -- will move end end --- obsolete - -- resolvers.varvalue = resolvers.variable -- output the value of variable $STRING. --- resolvers.expandvar = resolvers.expansion -- output variable expansion of STRING. +-- resolvers.expandvar = expansion -- output variable expansion of STRING. function resolvers.showpath(str) -- output search path for file type NAME - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) + return joinpath(expandedpathlist(resolvers.formatofvariable(str))) end function resolvers.registerfile(files, name, path) @@ -1828,7 +1901,7 @@ function resolvers.registerfile(files, name, path) end function resolvers.dowithpath(name,func) - local pathlist = resolvers.expandedpathlist(name) + local pathlist = expandedpathlist(name) for i=1,#pathlist do func("^"..cleanpath(pathlist[i])) end @@ -1844,7 +1917,7 @@ function resolvers.locateformat(name) local fullname = addsuffix(barename,"fmt") local fmtname = caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname == "" then - fmtname = resolvers.findfile(fullname) + fmtname = findfile(fullname) fmtname = cleanpath(fmtname) end if fmtname ~= "" then @@ -1864,7 +1937,7 @@ function resolvers.locateformat(name) end function resolvers.booleanvariable(str,default) - local b = resolvers.expansion(str) + local b = expansion(str) if b == "" then return default else @@ -1875,6 +1948,7 @@ end function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead local hashes = instance.hashes + local files = instance.files for i=1,#hashes do local hash = hashes[i] local blobtype = hash.type @@ -1886,7 +1960,7 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, if before then before(blobtype,blobpath,pattern) end - for path, name in filtered(instance.files[blobpath],pattern) do + for path, name in filtered(files[blobpath],pattern) do if type(path) == "string" then checked = checked + 1 if handle(blobtype,blobpath,path,name) then @@ -1908,12 +1982,6 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, end end -local obsolete = resolvers.obsolete or { } -resolvers.obsolete = obsolete - -resolvers.find_file = resolvers.findfile obsolete.find_file = resolvers.findfile -resolvers.find_files = resolvers.findfiles obsolete.find_files = resolvers.findfiles - -- moved here function resolvers.knownvariables(pattern) diff --git a/tex/context/base/mkiv/data-sch.lua b/tex/context/base/mkiv/data-sch.lua index a4a0b43cc..d869520ba 100644 --- a/tex/context/base/mkiv/data-sch.lua +++ b/tex/context/base/mkiv/data-sch.lua @@ -6,9 +6,14 @@ if not modules then modules = { } end modules ['data-sch'] = { license = "see context related readme files" } +-- this one will be redone / localized + local load, tonumber = load, tonumber -local gsub, concat, format = string.gsub, table.concat, string.format +local gsub, format = string.gsub, string.format +local sortedhash, concat = table.sortedhash, table.concat local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders +local addsuffix, suffix, splitbase = file.addsuffix, file.suffix, file.splitbase +local md5hex = md5.hex local trace_schemes = false trackers.register("resolvers.schemes",function(v) trace_schemes = v end) local report_schemes = logs.reporter("resolvers","schemes") @@ -39,7 +44,7 @@ end -- end function cleaners.strip(specification) -- keep suffixes - local path, name = file.splitbase(specification.original) + local path, name = splitbase(specification.original) if path == "" then return (gsub(name,"[^%a%d%.]+","-")) else @@ -48,7 +53,7 @@ function cleaners.strip(specification) -- keep suffixes end function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return addsuffix(md5hex(specification.original),suffix(specification.path)) end local cleaner = cleaners.strip @@ -149,10 +154,10 @@ schemes.install = install local function http_handler(specification,cachename) local tempname = cachename .. ".tmp" - local f = io.open(tempname,"wb") + local handle = io.open(tempname,"wb") local status, message = http.request { url = specification.original, - sink = ltn12.sink.file(f) + sink = ltn12.sink.file(handle) } if not status then os.remove(tempname) @@ -169,13 +174,13 @@ install('ftp') statistics.register("scheme handling time", function() local l, r, nl, nr = { }, { }, 0, 0 - for k, v in table.sortedhash(loaded) do + for k, v in sortedhash(loaded) do if v > 0 then nl = nl + 1 l[nl] = k .. ":" .. v end end - for k, v in table.sortedhash(reused) do + for k, v in sortedhash(reused) do if v > 0 then nr = nr + 1 r[nr] = k .. ":" .. v @@ -183,10 +188,10 @@ statistics.register("scheme handling time", function() end local n = nl + nr if n > 0 then - l = nl > 0 and concat(l) or "none" - r = nr > 0 and concat(r) or "none" + if nl == 0 then l = { "none" } end + if nr == 0 then r = { "none" } end return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes), n, threshold, l, r) + statistics.elapsedtime(schemes), n, threshold, concat(l," "), concat(l," ")) else return nil end diff --git a/tex/context/base/mkiv/data-tex.lua b/tex/context/base/mkiv/data-tex.lua index 8f978a204..e7c4941ca 100644 --- a/tex/context/base/mkiv/data-tex.lua +++ b/tex/context/base/mkiv/data-tex.lua @@ -6,20 +6,25 @@ if not modules then modules = { } end modules ['data-tex'] = { license = "see context related readme files" } +local tostring, tonumber, type = tostring, tonumber, type local char, find = string.char, string.find -local insert, remove = table.insert, table.remove local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) local report_tex = logs.reporter("resolvers","tex") -local resolvers = resolvers local sequencers = utilities.sequencers -local methodhandler = resolvers.methodhandler -local splitlines = string.splitlines local utffiletype = utf.filetype local setmetatableindex = table.setmetatableindex +local loaddata = io.loaddata +----- readall = io.readall + +local resolvers = resolvers +local methodhandler = resolvers.methodhandler +local loadbinfile = resolvers.loadbinfile +local pushinputname = resolvers.pushinputname +local popinputname = resolvers.popinputname -- local fileprocessor = nil -- local lineprocessor = nil @@ -36,8 +41,9 @@ local textlineactions = sequencers.new { results = "str", } -local helpers = resolvers.openers.helpers -local appendgroup = sequencers.appendgroup +local helpers = resolvers.openers.helpers +local appendgroup = sequencers.appendgroup +local appendaction = sequencers.appendaction helpers.textfileactions = textfileactions helpers.textlineactions = textlineactions @@ -53,10 +59,6 @@ appendgroup(textlineactions,"after" ) -- user local ctrl_d = char( 4) -- unix local ctrl_z = char(26) -- windows -resolvers.inputstack = resolvers.inputstack or { } - -local inputstack = resolvers.inputstack - ---------------------------------------- local lpegmatch = lpeg.match @@ -84,18 +86,25 @@ end ----------------------------------------- -function helpers.textopener(tag,filename,filehandle,coding) +local wideutfcoding = { + ["utf-16-be"] = utf.utf16_to_utf8_be_t, + ["utf-16-le"] = utf.utf16_to_utf8_le_t, + ["utf-32-be"] = utf.utf32_to_utf8_be_t, + ["utf-32-le"] = utf.utf32_to_utf8_le_t, +} + +local function textopener(tag,filename,filehandle,coding) local lines local t_filehandle = type(filehandle) if not filehandle then - lines = io.loaddata(filename) + lines = loaddata(filename) elseif t_filehandle == "string" then lines = filehandle elseif t_filehandle == "table" then lines = filehandle else - lines = filehandle:read("*a") -- io.readall(filehandle) ... but never that large files anyway - -- lines = io.readall(filehandle) + lines = filehandle:read("*a") -- readall(filehandle) ... but never that large files anyway + -- lines = readall(filehandle) filehandle:close() end if type(lines) == "string" then @@ -103,14 +112,9 @@ function helpers.textopener(tag,filename,filehandle,coding) if trace_locating then report_tex("%a opener: %a opened using method %a",tag,filename,coding) end - if coding == "utf-16-be" then - lines = utf.utf16_to_utf8_be_t(lines) - elseif coding == "utf-16-le" then - lines = utf.utf16_to_utf8_le_t(lines) - elseif coding == "utf-32-be" then - lines = utf.utf32_to_utf8_be_t(lines) - elseif coding == "utf-32-le" then - lines = utf.utf32_to_utf8_le_t(lines) + local wideutf = wideutfcoding[coding] + if wideutf then + lines = wideutf(lines) else -- utf8 or unknown (could be a mkvi file) local runner = textfileactions.runner if runner then @@ -125,19 +129,17 @@ function helpers.textopener(tag,filename,filehandle,coding) if lines[noflines] == "" then -- maybe some special check is needed lines[noflines] = nil end - logs.show_open(filename) - insert(inputstack,filename) + pushinputname(filename) local currentline, noflines = 0, noflines local t = { filename = filename, noflines = noflines, -- currentline = 0, close = function() + local usedname = popinputname() -- should match filename if trace_locating then report_tex("%a closer: %a closed",tag,filename) end - logs.show_close(filename) - remove(inputstack) t = nil end, reader = function(self) @@ -176,6 +178,8 @@ function helpers.textopener(tag,filename,filehandle,coding) return t end +helpers.settextopener(textopener) -- can only be done once + function resolvers.findtexfile(filename,filetype) return methodhandler('finders',filename,filetype) end @@ -191,7 +195,7 @@ end function resolvers.loadtexfile(filename,filetype) -- todo: optionally apply filters - local ok, data, size = resolvers.loadbinfile(filename, filetype) + local ok, data, size = loadbinfile(filename, filetype) return data or "" end @@ -202,14 +206,14 @@ local function installhandler(namespace,what,where,func) where, func = "after", where end if where == "before" or where == "after" then - sequencers.appendaction(namespace,where,func) + appendaction(namespace,where,func) else report_tex("installing input %a handlers in %a is not possible",what,tostring(where)) end end -function resolvers.installinputlinehandler(...) installhandler(helpers.textlineactions,"line",...) end -function resolvers.installinputfilehandler(...) installhandler(helpers.textfileactions,"file",...) end +function resolvers.installinputlinehandler(...) installhandler(textlineactions,"line",...) end +function resolvers.installinputfilehandler(...) installhandler(textfileactions,"file",...) end -- local basename = file.basename -- resolvers.installinputlinehandler(function(str,filename,linenumber,noflines) diff --git a/tex/context/base/mkiv/data-tmp.lua b/tex/context/base/mkiv/data-tmp.lua index ae7f65149..bfced1e64 100644 --- a/tex/context/base/mkiv/data-tmp.lua +++ b/tex/context/base/mkiv/data-tmp.lua @@ -7,29 +7,32 @@ if not modules then modules = { } end modules ['data-tmp'] = { } --[[ldx-- -<p>This module deals with caching data. It sets up the paths and -implements loaders and savers for tables. Best is to set the -following variable. When not set, the usual paths will be -checked. Personally I prefer the (users) temporary path.</p> +<p>This module deals with caching data. It sets up the paths and implements +loaders and savers for tables. Best is to set the following variable. When not +set, the usual paths will be checked. Personally I prefer the (users) temporary +path.</p> </code> TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. </code> -<p>Currently we do no locking when we write files. This is no real -problem because most caching involves fonts and the chance of them -being written at the same time is small. We also need to extend -luatools with a recache feature.</p> +<p>Currently we do no locking when we write files. This is no real problem +because most caching involves fonts and the chance of them being written at the +same time is small. We also need to extend luatools with a recache feature.</p> --ldx]]-- -local format, lower, gsub, concat = string.format, string.lower, string.gsub, table.concat ------ serialize, serializetofile = table.serialize, table.tofile -- overloaded so no local -local concat = table.concat -local mkdirs, isdir, isfile = dir.mkdirs, lfs.isdir, lfs.isfile -local addsuffix, is_writable, is_readable = file.addsuffix, file.is_writable, file.is_readable -local formatters = string.formatters local next, type = next, type ------ pcall, loadfile = pcall, loadfile +local pcall, loadfile, collectgarbage = pcall, loadfile, collectgarbage +local format, lower, gsub = string.format, string.lower, string.gsub +local concat, serialize, fastserialize, serializetofile = table.concat, table.serialize, table.fastserialize, table.tofile +local mkdirs, expanddirname, isdir, isfile = dir.mkdirs, dir.expandname, lfs.isdir, lfs.isfile +local is_writable, is_readable = file.is_writable, file.is_readable +local collapsepath, joinfile, addsuffix, dirname = file.collapsepath, file.join, file.addsuffix, file.dirname +local savedata = file.savedata +local formatters = string.formatters +local osexit, osdate, osuuid = os.exit, os.date, os.uuid +local removefile = os.remove +local md5hex = md5.hex local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) @@ -37,45 +40,80 @@ local trace_cache = false trackers.register("resolvers.cache", function(v local report_caches = logs.reporter("resolvers","caches") local report_resolvers = logs.reporter("resolvers","caching") -local resolvers = resolvers -local cleanpath = resolvers.cleanpath +local resolvers = resolvers +local cleanpath = resolvers.cleanpath +local resolvepath = resolvers.resolve + +local luautilities = utilities.lua -- intermezzo -local directive_cleanup = false directives.register("system.compile.cleanup", function(v) directive_cleanup = v end) -local directive_strip = false directives.register("system.compile.strip", function(v) directive_strip = v end) +do + + local directive_cleanup = false directives.register("system.compile.cleanup", function(v) directive_cleanup = v end) + local directive_strip = false directives.register("system.compile.strip", function(v) directive_strip = v end) -local compile = utilities.lua.compile + local compilelua = luautilities.compile + + function luautilities.compile(luafile,lucfile,cleanup,strip) + if cleanup == nil then cleanup = directive_cleanup end + if strip == nil then strip = directive_strip end + return compilelua(luafile,lucfile,cleanup,strip) + end -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup == nil then cleanup = directive_cleanup end - if strip == nil then strip = directive_strip end - return compile(luafile,lucfile,cleanup,strip) end -- end of intermezzo -caches = caches or { } -local caches = caches +caches = caches or { } +local caches = caches +local writable = nil +local readables = { } +local usedreadables = { } -local luasuffixes = utilities.lua.suffixes +local compilelua = luautilities.compile +local luasuffixes = luautilities.suffixes -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.tree = false -caches.force = true -caches.ask = false -caches.relocate = false -caches.defaults = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } +caches.base = caches.base or "luatex-cache" -- can be local +caches.more = caches.more or "context" -- can be local +caches.defaults = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } -directives.register("system.caches.fast", function(v) caches.fast = true end) -directives.register("system.caches.direct",function(v) caches.direct = true end) +local direct_cache = false -- true is faster but may need huge amounts of memory +local fast_cache = false +local cache_tree = false -local writable, readables, usedreadables = nil, { }, { } +directives.register("system.caches.direct",function(v) direct_cache = true end) +directives.register("system.caches.fast", function(v) fast_cache = true end) -- we could use a metatable for writable and readable but not yet +local function configfiles() + return concat(resolvers.configurationfiles(),";") +end + +local function hashed(tree) + tree = gsub(tree,"[\\/]+$","") + tree = lower(tree) + local hash = md5hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end + +local function treehash() + local tree = configfiles() + if not tree or tree == "" then + return false + else + return hashed(tree) + end +end + +caches.hashed = hashed +caches.treehash = treehash +caches.configfiles = configfiles + local function identify() -- Combining the loops makes it messy. First we check the format cache path -- and when the last component is not present we try to create it. @@ -84,9 +122,9 @@ local function identify() for k=1,#texmfcaches do local cachepath = texmfcaches[k] if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) - cachepath = file.collapsepath(cachepath) + cachepath = resolvepath(cachepath) + cachepath = cleanpath(cachepath) + cachepath = collapsepath(cachepath) local valid = isdir(cachepath) if valid then if is_readable(cachepath) then @@ -95,16 +133,14 @@ local function identify() writable = cachepath end end - elseif not writable and caches.force then - local cacheparent = file.dirname(cachepath) - if is_writable(cacheparent) and true then -- we go on anyway (needed for mojca's kind of paths) - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("path %a created",cachepath) - writable = cachepath - readables[#readables+1] = cachepath - end + elseif not writable then + local cacheparent = dirname(cachepath) + if is_writable(cacheparent) then -- we go on anyway (needed for mojca's kind of paths) + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable = cachepath + readables[#readables+1] = cachepath end end end @@ -119,8 +155,8 @@ local function identify() local cachepath = texmfcaches[k] cachepath = resolvers.expansion(cachepath) -- was getenv if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) + cachepath = resolvepath(cachepath) + cachepath = cleanpath(cachepath) local valid = isdir(cachepath) if valid and is_readable(cachepath) then if not writable and is_writable(cachepath) then @@ -136,25 +172,27 @@ local function identify() -- quit. if not writable then report_caches("fatal error: there is no valid writable cache path defined") - os.exit() + osexit() elseif #readables == 0 then report_caches("fatal error: there is no valid readable cache path defined") - os.exit() + osexit() end -- why here - writable = dir.expandname(resolvers.cleanpath(writable)) -- just in case - -- moved here - local base, more, tree = caches.base, caches.more, caches.tree or caches.treehash() -- we have only one writable tree + writable = expanddirname(cleanpath(writable)) -- just in case + -- moved here ( we have only one writable tree) + local base = caches.base + local more = caches.more + local tree = cache_tree or treehash() -- we have only one writable tree if tree then - caches.tree = tree + cache_tree = tree writable = mkdirs(writable,base,more,tree) for i=1,#readables do - readables[i] = file.join(readables[i],base,more,tree) + readables[i] = joinfile(readables[i],base,more,tree) end else writable = mkdirs(writable,base,more) for i=1,#readables do - readables[i] = file.join(readables[i],base,more) + readables[i] = joinfile(readables[i],base,more) end end -- end @@ -194,30 +232,8 @@ function caches.usedpaths(separator) end end -function caches.configfiles() - return concat(resolvers.configurationfiles(),";") -end - -function caches.hashed(tree) - tree = gsub(tree,"[\\/]+$","") - tree = lower(tree) - local hash = md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %a, hash %a",tree,hash) - end - return hash -end - -function caches.treehash() - local tree = caches.configfiles() - if not tree or tree == "" then - return false - else - return caches.hashed(tree) - end -end - -local r_cache, w_cache = { }, { } -- normally w in in r but who cares +local r_cache = { } +local w_cache = { } local function getreadablepaths(...) local tags = { ... } @@ -228,7 +244,7 @@ local function getreadablepaths(...) if #tags > 0 then done = { } for i=1,#readables do - done[i] = file.join(readables[i],...) + done[i] = joinfile(readables[i],...) end else done = readables @@ -254,31 +270,21 @@ local function getwritablepath(...) return done end -caches.getreadablepaths = getreadablepaths -caches.getwritablepath = getwritablepath - --- this can be tricky as we can have a pre-generated format while at the same time --- use e.g. a home path where we have updated file databases and so maybe we need --- to check first if we do have a writable one - --- function caches.getfirstreadablefile(filename,...) --- local rd = getreadablepaths(...) --- for i=1,#rd do --- local path = rd[i] --- local fullname = file.join(path,filename) --- if is_readable(fullname) then --- usedreadables[i] = true --- return fullname, path --- end --- end --- return caches.setfirstwritablefile(filename,...) --- end +local function setfirstwritablefile(filename,...) + local wr = getwritablepath(...) + local fullname = joinfile(wr,filename) + return fullname, wr +end --- next time we have an issue, we can test this instead: +local function setluanames(path,name) + return + format("%s/%s.%s",path,name,luasuffixes.tma), + format("%s/%s.%s",path,name,luasuffixes.tmc) +end -function caches.getfirstreadablefile(filename,...) +local function getfirstreadablefile(filename,...) -- check if we have already written once - local fullname, path = caches.setfirstwritablefile(filename,...) + local fullname, path = setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname, path -- , true end @@ -286,7 +292,7 @@ function caches.getfirstreadablefile(filename,...) local rd = getreadablepaths(...) for i=1,#rd do local path = rd[i] - local fullname = file.join(path,filename) + local fullname = joinfile(path,filename) if is_readable(fullname) then usedreadables[i] = true return fullname, path -- , false @@ -296,21 +302,19 @@ function caches.getfirstreadablefile(filename,...) return fullname, path -- , true end -function caches.setfirstwritablefile(filename,...) - local wr = getwritablepath(...) - local fullname = file.join(wr,filename) - return fullname, wr -end - -function caches.define(category,subcategory) -- not used - return function() - return getwritablepath(category,subcategory) - end -end +caches.getreadablepaths = getreadablepaths +caches.getwritablepath = getwritablepath +caches.setfirstwritablefile = setfirstwritablefile +caches.getfirstreadablefile = getfirstreadablefile +caches.setluanames = setluanames -function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma), format("%s/%s.%s",path,name,luasuffixes.tmc) -end +-- -- not used: +-- +-- function caches.define(category,subcategory) +-- return function() +-- return getwritablepath(category,subcategory) +-- end +-- end -- This works best if the first writable is the first readable too. In practice -- we can have these situations for file databases: @@ -328,17 +332,17 @@ function caches.loaddata(readables,name,writable) local path = readables[i] local loader = false local state = false - local tmaname, tmcname = caches.setluanames(path,name) + local tmaname, tmcname = setluanames(path,name) if isfile(tmcname) then state, loader = pcall(loadfile,tmcname) end if not loader and isfile(tmaname) then -- can be different paths when we read a file database from disk - local tmacrap, tmcname = caches.setluanames(writable,name) + local tmacrap, tmcname = setluanames(writable,name) if isfile(tmcname) then state, loader = pcall(loadfile,tmcname) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) if isfile(tmcname) then state, loader = pcall(loadfile,tmcname) end @@ -356,23 +360,23 @@ function caches.loaddata(readables,name,writable) end function caches.is_writable(filepath,filename) - local tmaname, tmcname = caches.setluanames(filepath,filename) + local tmaname, tmcname = setluanames(filepath,filename) return is_writable(tmaname) end local saveoptions = { compact = true } -function caches.savedata(filepath,filename,data) - local tmaname, tmcname = caches.setluanames(filepath,filename) - data.cache_uuid = os.uuid() - if caches.fast then - file.savedata(tmaname,table.fastserialize(data,true)) - elseif caches.direct then - file.savedata(tmaname,table.serialize(data,true,saveoptions)) +function caches.savedata(filepath,filename,data,fast) + local tmaname, tmcname = setluanames(filepath,filename) + data.cache_uuid = osuuid() + if fast or fast_cache then + savedata(tmaname,fastserialize(data,true)) + elseif direct_cache then + savedata(tmaname,serialize(data,true,saveoptions)) else - table.tofile(tmaname,data,true,saveoptions) + serializetofile(tmaname,data,true,saveoptions) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) end -- moved from data-res: @@ -385,9 +389,9 @@ end function caches.loadcontent(cachename,dataname,filename) if not filename then - local name = caches.hashed(cachename) - local full, path = caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - filename = file.join(path,name) + local name = hashed(cachename) + local full, path = getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename = joinfile(path,name) end local state, blob = pcall(loadfile,addsuffix(filename,luasuffixes.luc)) if not blob then @@ -427,9 +431,9 @@ end function caches.savecontent(cachename,dataname,content,filename) if not filename then - local name = caches.hashed(cachename) - local full, path = caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - filename = file.join(path,name) -- is full + local name = hashed(cachename) + local full, path = setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename = joinfile(path,name) -- is full end local luaname = addsuffix(filename,luasuffixes.lua) local lucname = addsuffix(filename,luasuffixes.luc) @@ -440,17 +444,17 @@ function caches.savecontent(cachename,dataname,content,filename) type = dataname, root = cachename, version = resolvers.cacheversion, - date = os.date("%Y-%m-%d"), - time = os.date("%H:%M:%S"), + date = osdate("%Y-%m-%d"), + time = osdate("%H:%M:%S"), content = content, - uuid = os.uuid(), + uuid = osuuid(), } - local ok = io.savedata(luaname,table.serialize(data,true)) + local ok = savedata(luaname,serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - if utilities.lua.compile(luaname,lucname) then + if compilelua(luaname,lucname) then if trace_locating then report_resolvers("%a compiled to %a",dataname,lucname) end @@ -459,7 +463,7 @@ function caches.savecontent(cachename,dataname,content,filename) if trace_locating then report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) end - os.remove(lucname) + removefile(lucname) end elseif trace_locating then report_resolvers("unable to save %a in %a (access error)",dataname,luaname) diff --git a/tex/context/base/mkiv/data-tre.lua b/tex/context/base/mkiv/data-tre.lua index c4d43e3eb..da3d1f14f 100644 --- a/tex/context/base/mkiv/data-tre.lua +++ b/tex/context/base/mkiv/data-tre.lua @@ -6,6 +6,10 @@ if not modules then modules = { } end modules ['data-tre'] = { license = "see context related readme files" } +-- A tree search is rather dumb ... there is some basic caching of searched trees +-- but nothing is cached over runs ... it's also a wildcard one so we cannot use +-- the normal scanner. + -- tree://e:/temporary/mb-mp/**/drawing.jpg -- tree://e:/temporary/mb-mp/**/Drawing.jpg -- tree://t:./**/tufte.tex @@ -28,83 +32,119 @@ local trace_locating = false trackers.register("resolvers.locating", function(v local report_trees = logs.reporter("resolvers","trees") -local resolvers = resolvers -local resolveprefix = resolvers.resolve -local notfound = resolvers.finders.notfound -local lookup = resolvers.get_from_content +local resolvers = resolvers +local finders = resolvers.finders +local openers = resolvers.openers +local loaders = resolvers.loaders +local locators = resolvers.locators +local hashers = resolvers.hashers +local generators = resolvers.generators --- A tree search is rather dumb ... there is some basic caching of searched trees --- but nothing is cached over runs ... it's also a wildcard one so we cannot use --- the normal scanner. +do -local collectors = { } -local found = { } - -function resolvers.finders.tree(specification) -- to be adapted to new formats - local spec = specification.filename - local okay = found[spec] - if okay == nil then - if spec ~= "" then - local path = dirname(spec) - local name = basename(spec) - if path == "" then - path = "." - end - local names = collectors[path] - if not names then - local pattern = find(path,"/%*+$") and path or (path .. "/*") - names = globdir(pattern) - collectors[path] = names - end - local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$" - for i=1,#names do - local fullname = names[i] - if find(fullname,pattern) then - found[spec] = fullname - return fullname + local collectors = { } + local found = { } + local notfound = finders.notfound + + function finders.tree(specification) -- to be adapted to new formats + local spec = specification.filename + local okay = found[spec] + if okay == nil then + if spec ~= "" then + local path = dirname(spec) + local name = basename(spec) + if path == "" then + path = "." end - end - -- let's be nice: - local pattern = lower(pattern) - for i=1,#names do - local fullname = lower(names[i]) - if find(fullname,pattern) then - if isfile(fullname) then + local names = collectors[path] + if not names then + local pattern = find(path,"/%*+$") and path or (path .. "/*") + names = globdir(pattern) + collectors[path] = names + end + local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$" + for i=1,#names do + local fullname = names[i] + if find(fullname,pattern) then found[spec] = fullname return fullname - else - -- no os name mapping - break + end + end + -- let's be nice: + local pattern = lower(pattern) + for i=1,#names do + local fullname = lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec] = fullname + return fullname + else + -- no os name mapping + break + end end end end + okay = notfound() -- false + found[spec] = okay end - okay = notfound() -- false - found[spec] = okay + return okay end - return okay + end -function resolvers.locators.tree(specification) - local name = specification.filename - local realname = resolveprefix(name) -- no shortcut - if realname and realname ~= '' and isdir(realname) then - if trace_locating then - report_trees("locator %a found",realname) +do + + local resolveprefix = resolvers.resolve + local appendhash = resolvers.appendhash + + local function dolocate(specification) + local name = specification.filename + local realname = resolveprefix(name) -- no shortcut + if realname and realname ~= '' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + appendhash('tree',name,false) -- don't cache + elseif trace_locating then + report_trees("locator %a not found",name) end - resolvers.appendhash('tree',name,false) -- don't cache - elseif trace_locating then - report_trees("locator %a not found",name) end + + locators.tree = dolocate + locators.dirlist = dolocate + locators.dirfile = dolocate + end -function resolvers.hashers.tree(specification) - local name = specification.filename - if trace_locating then - report_trees("analyzing %a",name) + +do + + local filegenerator = generators.file + + generators.dirlist = filegenerator + generators.dirfile = filegenerator + +end + +do + + local filegenerator = generators.file + local methodhandler = resolvers.methodhandler + + local function dohash(specification) + local name = specification.filename + if trace_locating then + report_trees("analyzing %a",name) + end + methodhandler("hashers",name) + filegenerator(specification) end - resolvers.methodhandler("hashers",name) - resolvers.generators.file(specification) + + hashers.tree = dohash + hashers.dirlist = dohash + hashers.dirfile = dohash + end -- This is a variation on tree lookups but this time we do cache in the given @@ -112,120 +152,131 @@ end -- for instance trees with 50K xml files plus a similar amount of resources to -- deal and we don't want too much overhead. -local collectors = { } -local splitter = lpeg.splitat("/**/") -local stripper = lpeg.replacer { [P("/") * P("*")^1 * P(-1)] = "" } - -table.setmetatableindex(collectors, function(t,k) - local rootname = lpegmatch(stripper,k) - local dataname = joinname(rootname,"dirlist") - local content = caches.loadcontent(dataname,"files",dataname) - if not content then - -- path branch usecache onlyonce tolerant - content = resolvers.scanfiles(rootname,nil,nil,false,true) -- so we accept crap - caches.savecontent(dataname,"files",content,dataname) - end - t[k] = content - return content -end) +local resolve do + + local collectors = { } + local splitter = lpeg.splitat("/**/") + local stripper = lpeg.replacer { [P("/") * P("*")^1 * P(-1)] = "" } + + local loadcontent = caches.loadcontent + local savecontent = caches.savecontent + + local notfound = finders.notfound + local scanfiles = resolvers.scanfiles + local lookup = resolvers.get_from_content -local function checked(root,p,n) - if p then - if type(p) == "table" then - for i=1,#p do - local fullname = joinname(root,p[i],n) + table.setmetatableindex(collectors, function(t,k) + local rootname = lpegmatch(stripper,k) + local dataname = joinname(rootname,"dirlist") + local content = loadcontent(dataname,"files",dataname) + if not content then + -- path branch usecache onlyonce tolerant + content = scanfiles(rootname,nil,nil,false,true) -- so we accept crap + savecontent(dataname,"files",content,dataname) + end + t[k] = content + return content + end) + + local function checked(root,p,n) + if p then + if type(p) == "table" then + for i=1,#p do + local fullname = joinname(root,p[i],n) + if isfile(fullname) then -- safeguard + return fullname + end + end + else + local fullname = joinname(root,p,n) if isfile(fullname) then -- safeguard return fullname end end - else - local fullname = joinname(root,p,n) - if isfile(fullname) then -- safeguard - return fullname - end end + return notfound() end - return notfound() -end --- no funny characters in path but in filename permitted .. sigh - -local function resolve(specification) -- can be called directly too - local filename = specification.filename - -- inspect(specification) - if filename ~= "" then - local root, rest = lpegmatch(splitter,filename) - if root and rest then - local path, name = dirname(rest), basename(rest) - if name ~= rest then - local content = collectors[root] - local p, n = lookup(content,name) - if not p then - return notfound() - end - local pattern = ".*/" .. path .. "$" - local istable = type(p) == "table" - if istable then - for i=1,#p do - local pi = p[i] - if pi == path or find(pi,pattern) then - local fullname = joinname(root,pi,n) - if isfile(fullname) then -- safeguard - return fullname + -- no funny characters in path but in filename permitted .. sigh + + resolve = function(specification) -- can be called directly too + local filename = specification.filename + -- inspect(specification) + if filename ~= "" then + local root, rest = lpegmatch(splitter,filename) + if root and rest then + local path, name = dirname(rest), basename(rest) + if name ~= rest then + local content = collectors[root] + local p, n = lookup(content,name) + if not p then + return notfound() + end + local pattern = ".*/" .. path .. "$" + local istable = type(p) == "table" + if istable then + for i=1,#p do + local pi = p[i] + if pi == path or find(pi,pattern) then + local fullname = joinname(root,pi,n) + if isfile(fullname) then -- safeguard + return fullname + end end end + elseif p == path or find(p,pattern) then + local fullname = joinname(root,p,n) + if isfile(fullname) then -- safeguard + return fullname + end end - elseif p == path or find(p,pattern) then - local fullname = joinname(root,p,n) - if isfile(fullname) then -- safeguard - return fullname + local queries = specification.queries + if queries and queries.option == "fileonly" then + return checked(root,p,n) + else + return notfound() end end - local queries = specification.queries - if queries and queries.option == "fileonly" then - return checked(root,p,n) - else - return notfound() - end + end + local path = dirname(filename) + local name = basename(filename) + local root = lpegmatch(stripper,path) + local content = collectors[path] + local p, n = lookup(content,name) + if p then + return checked(root,p,n) end end - local path, name = dirname(filename), basename(filename) - local root = lpegmatch(stripper,path) - local content = collectors[path] - local p, n = lookup(content,name) - if p then - return checked(root,p,n) - end + return notfound() end - return notfound() -end -resolvers.finders .dirlist = resolve -resolvers.locators .dirlist = resolvers.locators .tree -resolvers.hashers .dirlist = resolvers.hashers .tree -resolvers.generators.dirlist = resolvers.generators.file -resolvers.openers .dirlist = resolvers.openers .file -resolvers.loaders .dirlist = resolvers.loaders .file - -function resolvers.finders.dirfile(specification) - local queries = specification.queries - if queries then - queries.option = "fileonly" - else - specification.queries = { option = "fileonly" } + finders.dirlist = resolve + + function finders.dirfile(specification) + local queries = specification.queries + if queries then + queries.option = "fileonly" + else + specification.queries = { option = "fileonly" } + end + return resolve(specification) end - return resolve(specification) + end -resolvers.locators .dirfile = resolvers.locators .dirlist -resolvers.hashers .dirfile = resolvers.hashers .dirlist -resolvers.generators.dirfile = resolvers.generators.dirlist -resolvers.openers .dirfile = resolvers.openers .dirlist -resolvers.loaders .dirfile = resolvers.loaders .dirlist +do --- local locate = collectors[ [[E:\temporary\mb-mp]] ] --- local locate = collectors( [[\\storage-2\resources\mb-mp]] ) + local fileopener = openers.file + local fileloader = loaders.file + + openers.dirlist = fileopener + loaders.dirlist = fileloader + + openers.dirfile = fileopener + loaders.dirfile = fileloader + +end -- print(resolvers.findtexfile("tree://e:/temporary/mb-mp/**/VB_wmf_03_vw_01d_ant.jpg")) -- print(resolvers.findtexfile("tree://t:/**/tufte.tex")) diff --git a/tex/context/base/mkiv/data-use.lua b/tex/context/base/mkiv/data-use.lua index ccd44007c..7df416b65 100644 --- a/tex/context/base/mkiv/data-use.lua +++ b/tex/context/base/mkiv/data-use.lua @@ -6,48 +6,60 @@ if not modules then modules = { } end modules ['data-use'] = { license = "see context related readme files" } -local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find +local format = string.format local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) local report_mounts = logs.reporter("resolvers","mounts") local resolvers = resolvers +local findfile = resolvers.findfile --- we will make a better format, maybe something xml or just text or lua - -resolvers.automounted = resolvers.automounted or { } - -function resolvers.automount(usecache) - local mountpaths = resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths == 0) and usecache then - mountpaths = caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths > 0 then - resolvers.starttiming() - for k=1,#mountpaths do - local root = mountpaths[k] - local f = io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then -- or %W - -- skip - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %a",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() - end - end - resolvers.stoptiming() - end -end +-- -- This should mount a zip file so that we can run from zip files but I never went +-- -- on with it. It's really old code, from right when we started with luatex and +-- -- mkiv. Nowadays I'd use a lua specification file instead of a line based url.tmi, +-- -- file so just to be modern I patched it but it's untested. This is a normally a +-- -- startup-only feature. +-- +-- do +-- +-- local mounted = { } +-- +-- function resolvers.automount(usecache) +-- local mountpaths = resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) +-- if (not mountpaths or #mountpaths == 0) and usecache then +-- mountpaths = caches.getreadablepaths("mount") +-- end +-- if mountpaths and #mountpaths > 0 then +-- resolvers.starttiming() +-- for k=1,#mountpaths do +-- local root = mountpaths[k] +-- local list = table.load("automount.lua") +-- if list then +-- local archives = list.archives +-- if archives then +-- for i=1,#archives do +-- local archive = archives[i] +-- local already = false +-- for i=1,#mounted do +-- if archive == mounted[i] then +-- already = true +-- break +-- end +-- end +-- if not already then +-- mounted[#mounted+1] = archive +-- resolvers.usezipfile(archive) +-- end +-- end +-- end +-- end +-- end +-- resolvers.stoptiming() +-- end +-- end +-- +-- end -- status info @@ -63,7 +75,7 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) - local luvdata = { enginebanner = enginebanner, formatbanner = formatbanner, - sourcehash = md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcehash = md5.hex(io.loaddata(findfile(sourcefile)) or "unknown"), sourcefile = sourcefile, luaversion = LUAVERSION, formatid = LUATEXFORMATID, @@ -92,7 +104,7 @@ function statistics.checkfmtstatus(texname) if lfs.isfile(luvname) then local luv = dofile(luvname) if luv and luv.sourcefile then - local sourcehash = md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local sourcehash = md5.hex(io.loaddata(findfile(luv.sourcefile)) or "unknown") local luvbanner = luv.enginebanner or "?" if luvbanner ~= enginebanner then return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) diff --git a/tex/context/base/mkiv/data-vir.lua b/tex/context/base/mkiv/data-vir.lua index 5a96a1d21..4b19ee537 100644 --- a/tex/context/base/mkiv/data-vir.lua +++ b/tex/context/base/mkiv/data-vir.lua @@ -7,7 +7,7 @@ if not modules then modules = { } end modules ['data-vir'] = { } local type = type -local formatters, validstrings = string.formatters, string.valid +local formatters = string.formatters local trace_virtual = false local report_virtual = logs.reporter("resolvers","virtual") @@ -16,8 +16,7 @@ trackers.register("resolvers.locating", function(v) trace_virtual = v end) trackers.register("resolvers.virtual", function(v) trace_virtual = v end) local resolvers = resolvers - -local finders, openers, loaders, savers = resolvers.finders, resolvers.openers, resolvers.loaders, resolvers.savers +local savers = resolvers.savers local data = { } local n = 0 -- hm, number can be query @@ -38,6 +37,9 @@ function savers.virtual(specification,content,suffix) return filename end +local finders = resolvers.finders +local notfound = finders.notfound + function finders.virtual(specification) local original = specification.original local d = data[original] @@ -50,10 +52,14 @@ function finders.virtual(specification) if trace_virtual then report_virtual("finder: unknown file %a",original) end - return finders.notfound() + return notfound() end end +local openers = resolvers.openers +local notfound = openers.notfound +local textopener = openers.helpers.textopener + function openers.virtual(specification) local original = specification.original local d = data[original] @@ -64,15 +70,18 @@ function openers.virtual(specification) data[original] = nil -- when we comment this we can have error messages -- With utf-8 we signal that no regime is to be applied! -- characters.showstring(d) - return openers.helpers.textopener("virtual",original,d,"utf-8") + return textopener("virtual",original,d,"utf-8") else if trace_virtual then report_virtual("opener: file %a not found",original) end - return openers.notfound() + return notfound() end end +local loaders = resolvers.loaders +local notfound = loaders.notfound + function loaders.virtual(specification) local original = specification.original local d = data[original] @@ -86,5 +95,5 @@ function loaders.virtual(specification) if trace_virtual then report_virtual("loader: file %a not loaded",original) end - return loaders.notfound() + return notfound() end diff --git a/tex/context/base/mkiv/data-zip.lua b/tex/context/base/mkiv/data-zip.lua index 388ae0c10..8d914b842 100644 --- a/tex/context/base/mkiv/data-zip.lua +++ b/tex/context/base/mkiv/data-zip.lua @@ -26,7 +26,16 @@ zip:///texmf-mine.zip?tree=/tex/texmf-projects </typing> --ldx]]-- -local resolvers = resolvers +local resolvers = resolvers +local findfile = resolvers.findfile +local registerfile = resolvers.registerfile +local splitmethod = resolvers.splitmethod +local prependhash = resolvers.prependhash +local starttiming = resolvers.starttiming +local extendtexmf = resolvers.extendtexmfvariable +local stoptiming = resolvers.stoptiming + +local urlquery = url.query zip = zip or { } local zip = zip @@ -50,8 +59,10 @@ if zipfiles then validfile = zipfiles.found wholefile = zipfiles.unzip + local listzip = zipfiles.list + traversezip = function(zfile) - return ipairs(zipfiles.list(zfile)) + return ipairs(listzip(zfile)) end local streams = utilities.streams @@ -120,13 +131,13 @@ local function validzip(str) -- todo: use url splitter end end -function zip.openarchive(name) +local function openarchive(name) if not name or name == "" then return nil else local arch = archives[name] if not arch then - local full = resolvers.findfile(name) or "" + local full = findfile(name) or "" arch = full ~= "" and openzip(full) or false archives[name] = arch end @@ -134,16 +145,19 @@ function zip.openarchive(name) end end -function zip.closearchive(name) +local function closearchive(name) if not name or (name == "" and archives[name]) then closezip(archives[name]) archives[name] = nil end end +zip.openarchive = openarchive +zip.closearchive = closearchive + function resolvers.locators.zip(specification) local archive = specification.filename - local zipfile = archive and archive ~= "" and zip.openarchive(archive) -- tricky, could be in to be initialized tree + local zipfile = archive and archive ~= "" and openarchive(archive) -- tricky, could be in to be initialized tree if trace_locating then if zipfile then report_zip("locator: archive %a found",archive) @@ -153,14 +167,6 @@ function resolvers.locators.zip(specification) end end -function resolvers.hashers.zip(specification) - local archive = specification.filename - if trace_locating then - report_zip("loading file %a",archive) - end - resolvers.usezipfile(specification.original) -end - function resolvers.concatinators.zip(zipfile,path,name) -- ok ? if not path or path == "" then return format('%s?name=%s',zipfile,name) @@ -169,14 +175,17 @@ function resolvers.concatinators.zip(zipfile,path,name) -- ok ? end end -function resolvers.finders.zip(specification) +local finders = resolvers.finders +local notfound = finders.notfound + +function finders.zip(specification) local original = specification.original - local archive = specification.filename + local archive = specification.filename if archive then - local query = url.query(specification.query) + local query = urlquery(specification.query) local queryname = query.name if queryname then - local zfile = zip.openarchive(archive) + local zfile = openarchive(archive) if zfile then if trace_locating then report_zip("finder: archive %a found",archive) @@ -197,17 +206,21 @@ function resolvers.finders.zip(specification) if trace_locating then report_zip("finder: %a not found",original) end - return resolvers.finders.notfound() + return notfound() end -function resolvers.openers.zip(specification) +local openers = resolvers.openers +local notfound = openers.notfound +local textopener = openers.helpers.textopener + +function openers.zip(specification) local original = specification.original - local archive = specification.filename + local archive = specification.filename if archive then - local query = url.query(specification.query) + local query = urlquery(specification.query) local queryname = query.name if queryname then - local zfile = zip.openarchive(archive) + local zfile = openarchive(archive) if zfile then if trace_locating then report_zip("opener; archive %a opened",archive) @@ -217,7 +230,7 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: file %a found",queryname) end - return resolvers.openers.helpers.textopener('zip',original,handle) + return textopener('zip',original,handle) elseif trace_locating then report_zip("opener: file %a not found",queryname) end @@ -229,24 +242,26 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -function resolvers.loaders.zip(specification) +local loaders = resolvers.loaders +local notfound = loaders.notfound + +function loaders.zip(specification) local original = specification.original - local archive = specification.filename + local archive = specification.filename if archive then - local query = url.query(specification.query) + local query = urlquery(specification.query) local queryname = query.name if queryname then - local zfile = zip.openarchive(archive) + local zfile = openarchive(archive) if zfile then if trace_locating then report_zip("loader: archive %a opened",archive) end local data = wholefile(zfile,queryname) if data then - logs.show_load(original) if trace_locating then report_zip("loader; file %a loaded",original) end @@ -262,52 +277,28 @@ function resolvers.loaders.zip(specification) if trace_locating then report_zip("loader: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -- zip:///somefile.zip -- zip:///somefile.zip?tree=texmf-local -> mount -function resolvers.usezipfile(archive) - local specification = resolvers.splitmethod(archive) -- to be sure - local archive = specification.filename - if archive and not registeredfiles[archive] then - local z = zip.openarchive(archive) - if z then - local tree = url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering: archive %a",archive) - end - resolvers.starttiming() - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) -- resets hashes too - registeredfiles[archive] = z - resolvers.registerfilehash(archive,resolvers.registerzipfile(z,tree)) - resolvers.stoptiming() - elseif trace_locating then - report_zip("registering: unknown archive %a",archive) - end - elseif trace_locating then - report_zip("registering: archive %a not found",archive) - end -end - -function resolvers.registerzipfile(z,tree) +local function registerzipfile(z,tree) local names = { } local files = { } -- somewhat overkill .. todo local remap = { } -- somewhat overkill .. todo local n = 0 local filter = tree == "" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) - local register = resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end + starttiming() for i in traversezip(z) do local filename = i.filename local path, name = match(filename,filter) if not path then n = n + 1 - register(names,filename,"") + registerfile(names,filename,"") local usedname = lower(filename) files[usedname] = "" if usedname ~= filename then @@ -325,6 +316,7 @@ function resolvers.registerzipfile(z,tree) -- directory end end + stoptiming() report_zip("registering: %s files registered",n) return { -- metadata = { }, @@ -332,3 +324,36 @@ function resolvers.registerzipfile(z,tree) remap = remap, } end + +local function usezipfile(archive) + local specification = splitmethod(archive) -- to be sure + local archive = specification.filename + if archive and not registeredfiles[archive] then + local z = openarchive(archive) + if z then + local tree = urlquery(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + prependhash('zip',archive) + extendtexmf(archive) -- resets hashes too + registeredfiles[archive] = z + registerfilehash(archive,registerzipfile(z,tree)) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end + +resolvers.usezipfile = usezipfile +resolvers.registerzipfile = registerzipfile + +function resolvers.hashers.zip(specification) + local archive = specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + usezipfile(specification.original) +end diff --git a/tex/context/base/mkiv/driv-shp.lua b/tex/context/base/mkiv/driv-shp.lua index 2d242b1a1..5e4d9a29a 100644 --- a/tex/context/base/mkiv/driv-shp.lua +++ b/tex/context/base/mkiv/driv-shp.lua @@ -54,8 +54,9 @@ local findtail = nuts.tail local nextdir = nuts.traversers.dir local nextnode = nuts.traversers.node -local rangedimensions = node.direct.rangedimensions +local rangedimensions = node.direct.rangedimensions -- nuts ? local effectiveglue = nuts.effective_glue +local start_of_par = nuts.start_of_par local texget = tex.get @@ -423,21 +424,23 @@ local function reset_state() -- page_v_origin = 0 -- trueinch end -local function dirstackentry(t,k) - local v = { - cur_h = 0, - cur_v = 0, - ref_h = 0, - ref_v = 0, - } - t[k] = v - return v -end - +-- local function dirstackentry(t,k) +-- local v = { +-- cur_h = 0, +-- cur_v = 0, +-- ref_h = 0, +-- ref_v = 0, +-- } +-- t[k] = v +-- return v +-- end + +----- dirstack = { } local dirstack = setmetatableindex(dirstackentry) local function reset_dir_stack() - dirstack = setmetatableindex(dirstackentry) + -- dirstack = setmetatableindex(dirstackentry) + dirstack = { } end local function flushlatelua(current,h,v) @@ -802,38 +805,69 @@ local hlist_out, vlist_out do local dir, cancel = getdirection(current) if cancel then local ds = dirstack[current] - ref_h = ds.ref_h - ref_v = ds.ref_v - cur_h = ds.cur_h - cur_v = ds.cur_v - pos_r = dir - else - local enddir, width = calculate_width_to_enddir(this_box,current) - local ds = dirstack[enddir] - ds.cur_h = cur_h + width - if dir ~= pos_r then + if ds then + ref_h = ds.ref_h + ref_v = ds.ref_v cur_h = ds.cur_h - end - if enddir ~= current then - local ds = dirstack[enddir] - ds.cur_v = cur_v - ds.ref_h = ref_h - ds.ref_v = ref_v - setdirection(enddir,pos_r) - end - if pos_r == righttoleft_code then - pos_h = ref_h - cur_h + cur_v = ds.cur_v else - pos_h = ref_h + cur_h + -- pardir end - pos_v = ref_v - cur_v - -- synced - ref_h = pos_h - ref_v = pos_v - cur_h = 0 - cur_v = 0 pos_r = dir - end + else +-- local enddir, width = calculate_width_to_enddir(this_box,current) +-- local ds = dirstack[enddir] +-- ds.cur_h = cur_h + width +-- if dir ~= pos_r then +-- cur_h = ds.cur_h +-- end +-- if enddir ~= current then +-- -- local ds = dirstack[enddir] +-- ds.cur_v = cur_v +-- ds.ref_h = ref_h +-- ds.ref_v = ref_v +-- setdirection(enddir,pos_r) +-- end +-- if pos_r == righttoleft_code then +-- pos_h = ref_h - cur_h +-- else +-- pos_h = ref_h + cur_h +-- end +-- pos_v = ref_v - cur_v +-- -- synced +-- ref_h = pos_h +-- ref_v = pos_v +-- cur_h = 0 +-- cur_v = 0 +-- pos_r = dir +-- end +local enddir, width = calculate_width_to_enddir(this_box,current) +local new_h = cur_h + width +if dir ~= pos_r then + cur_h = new_h +end +if enddir ~= current then + dirstack[enddir] = { + cur_h = new_h, + cur_v = cur_v, + ref_h = ref_h, + ref_v = ref_v, + } + setdirection(enddir,pos_r) +end +if pos_r == righttoleft_code then + pos_h = ref_h - cur_h +else + pos_h = ref_h + cur_h +end +pos_v = ref_v - cur_v +-- synced +ref_h = pos_h +ref_v = pos_v +cur_h = 0 +cur_v = 0 +pos_r = dir +end elseif id == whatsit_code then if subtype == literalwhatsit_code then flushliteral(current,pos_h,pos_v) @@ -857,6 +891,11 @@ local hlist_out, vlist_out do end elseif id == marginkern_code then cur_h = cur_h + getkern(current) +-- elseif id == localpar_code and start_of_par(current) then +-- local pardir = getdirection(current) or lefttoright_code +-- if pardir == righttoleft_code then +-- end +-- end end -- There is no gain in skipping over this when we have zero progression -- and such. diff --git a/tex/context/base/mkiv/file-job.lua b/tex/context/base/mkiv/file-job.lua index fd9a5a120..2c854b504 100644 --- a/tex/context/base/mkiv/file-job.lua +++ b/tex/context/base/mkiv/file-job.lua @@ -46,7 +46,7 @@ local dirname = file.dirname local is_qualified_path = file.is_qualified_path local cleanpath = resolvers.cleanpath -local inputstack = resolvers.inputstack +local toppath = resolvers.toppath local resolveprefix = resolvers.resolve local hasscheme = url.hasscheme @@ -789,7 +789,7 @@ end local function autoname(name) if name == "*" then - name = nameonly(inputstack[#inputstack] or name) + name = nameonly(toppath() or name) end return name end diff --git a/tex/context/base/mkiv/font-nod.lua b/tex/context/base/mkiv/font-nod.lua index f359ef04f..8c0f6e441 100644 --- a/tex/context/base/mkiv/font-nod.lua +++ b/tex/context/base/mkiv/font-nod.lua @@ -77,6 +77,7 @@ local copy_node_list = nuts.copy_list local hpack_node_list = nuts.hpack local flush_node_list = nuts.flush_list local protect_glyphs = nuts.protect_glyphs +local start_of_par = nuts.start_of_par local nextnode = nuts.traversers.node local nextglyph = nuts.traversers.glyph @@ -395,7 +396,7 @@ function step_tracers.codes(i,command,space) local char, id = isglyph(c) if char then showchar(char,id) - elseif id == dir_code or (id == localpar_code and getsubtype(c) == 0) then + elseif id == dir_code or (id == localpar_code and start_of_par(c)) then context("[%s]",getdirection(c) or "?") elseif id == disc_code then local pre, post, replace = getdisc(c) diff --git a/tex/context/base/mkiv/font-ots.lua b/tex/context/base/mkiv/font-ots.lua index 5091fd765..c37008d0e 100644 --- a/tex/context/base/mkiv/font-ots.lua +++ b/tex/context/base/mkiv/font-ots.lua @@ -213,6 +213,7 @@ local find_node_tail = nuts.tail local flush_node_list = nuts.flush_list local flush_node = nuts.flush_node local end_of_math = nuts.end_of_math +local start_of_par = nuts.start_of_par local setmetatable = setmetatable local setmetatableindex = table.setmetatableindex @@ -3804,7 +3805,7 @@ do local initialrl = 0 - if getid(head) == localpar_code and getsubtype(head) == 0 then + if getid(head) == localpar_code and start_of_par(head) then initialrl = pardirstate(head) elseif direction == 1 or direction == "TRT" then initialrl = -1 @@ -3958,7 +3959,7 @@ do elseif id == dir_code then topstack, rlmode = txtdirstate(start,dirstack,topstack,rlparmode) start = getnext(start) - -- elseif id == localpar_code then + -- elseif id == localpar_code and start_of_par(start) then -- rlparmode, rlmode = pardirstate(start) -- start = getnext(start) else @@ -4042,7 +4043,7 @@ do elseif id == dir_code then topstack, rlmode = txtdirstate(start,dirstack,topstack,rlparmode) start = getnext(start) - -- elseif id == localpar_code then + -- elseif id == localpar_code and start_of_par(start) then -- rlparmode, rlmode = pardirstate(start) -- start = getnext(start) else @@ -4153,7 +4154,7 @@ do elseif id == dir_code then topstack, rlmode = txtdirstate(start,dirstack,topstack,rlparmode) start = getnext(start) - -- elseif id == localpar_code then + -- elseif id == localpar_code and start_of_par(start) then -- rlparmode, rlmode = pardirstate(start) -- start = getnext(start) else diff --git a/tex/context/base/mkiv/font-shp.lua b/tex/context/base/mkiv/font-shp.lua index fd59d689b..d4805dab2 100644 --- a/tex/context/base/mkiv/font-shp.lua +++ b/tex/context/base/mkiv/font-shp.lua @@ -214,12 +214,7 @@ local function loadoutlines(cache,filename,sub,instance) end local function cachethem(cache,hash,data) - local fast = caches.fast - if compact_streams then - caches.fast = true - end - containers.write(cache,hash,data) - caches.fast = fast + containers.write(cache,hash,data,compact_streams) -- arg 4 aka fast return containers.read(cache,hash) -- frees old mem end diff --git a/tex/context/base/mkiv/font-sol.lua b/tex/context/base/mkiv/font-sol.lua index 002f9df13..2017afcb2 100644 --- a/tex/context/base/mkiv/font-sol.lua +++ b/tex/context/base/mkiv/font-sol.lua @@ -82,6 +82,7 @@ local hpack_nodes = nuts.hpack local insert_node_before = nuts.insert_before local insert_node_after = nuts.insert_after local protect_glyphs = nuts.protect_glyphs +local start_of_par = nuts.start_of_par local nextnode = nuts.traversers.next local nexthlist = nuts.traversers.hlist @@ -434,7 +435,7 @@ function splitters.split(head) -- best also pass the direction end local direction, pop = getdirection(current) r2l = not pop and direction == righttoleft - elseif id == localpar_code and getsubtype(current) == 0 then + elseif id == localpar_code and start_of_par(current) then if start then flush() -- very unlikely as this starts a paragraph end diff --git a/tex/context/base/mkiv/font-tpk.lua b/tex/context/base/mkiv/font-tpk.lua index bdfb0c2ee..05ecdd45c 100644 --- a/tex/context/base/mkiv/font-tpk.lua +++ b/tex/context/base/mkiv/font-tpk.lua @@ -1296,9 +1296,9 @@ do end --- inspect(readers.loadtfmvf(resolvers.find_file("mi-iwonari.tfm"))) --- inspect(readers.loadtfm(resolvers.find_file("texnansi-palatinonova-regular.tfm"))) --- inspect(readers.loadtfm(resolvers.find_file("cmex10.tfm"))) --- inspect(readers.loadtfm(resolvers.find_file("cmr10.tfm"))) +-- inspect(readers.loadtfmvf(resolvers.findfile("mi-iwonari.tfm"))) +-- inspect(readers.loadtfm(resolvers.findfile("texnansi-palatinonova-regular.tfm"))) +-- inspect(readers.loadtfm(resolvers.findfile("cmex10.tfm"))) +-- inspect(readers.loadtfm(resolvers.findfile("cmr10.tfm"))) -- local t = readers.loadtfmvf("texnansi-lte50019.tfm") -- inspect(t) diff --git a/tex/context/base/mkiv/grph-chk.lua b/tex/context/base/mkiv/grph-chk.lua index f286dedd6..d16106234 100644 --- a/tex/context/base/mkiv/grph-chk.lua +++ b/tex/context/base/mkiv/grph-chk.lua @@ -15,6 +15,7 @@ local report_inclusion = logs.reporter("graphics","inclusion") local report_bitmap = logs.reporter("graphics","bitmap") local checkers = figures.checkers +local genericchecker = checkers.generic local placeholder = graphics.bitmaps.placeholder @@ -27,6 +28,7 @@ local placeholder = graphics.bitmaps.placeholder -- but opens up some possibilities (like merging fonts) that I will look into some -- day. + function checkers.pdf(data) local request = data.request local used = data.used @@ -96,7 +98,7 @@ function checkers.pdf(data) end end end - return checkers.generic(data) + return genericchecker(data) end local function wrappedidentify(identify,filename) @@ -163,7 +165,7 @@ function checkers.jpg(data) end end end - return checkers.generic(data) + return genericchecker(data) end function checkers.jp2(data) -- idem as jpg @@ -199,7 +201,7 @@ function checkers.jp2(data) -- idem as jpg end end end - return checkers.generic(data) + return genericchecker(data) end function checkers.png(data) -- same as jpg (for now) @@ -245,5 +247,5 @@ function checkers.png(data) -- same as jpg (for now) end end end - return checkers.generic(data) + return genericchecker(data) end diff --git a/tex/context/base/mkiv/l-macro.lua b/tex/context/base/mkiv/l-macro.lua index 24a3d07bc..98cf20d05 100644 --- a/tex/context/base/mkiv/l-macro.lua +++ b/tex/context/base/mkiv/l-macro.lua @@ -273,7 +273,7 @@ macros.loaded = loaded function required(name,trace) local filename = file.addsuffix(name,"lua") - local fullname = resolvers and resolvers.find_file(filename) or filename + local fullname = resolvers and resolvers.findfile(filename) or filename if not fullname or fullname == "" then return false end diff --git a/tex/context/base/mkiv/lpdf-emb.lua b/tex/context/base/mkiv/lpdf-emb.lua index 1cb858571..c24a9173c 100644 --- a/tex/context/base/mkiv/lpdf-emb.lua +++ b/tex/context/base/mkiv/lpdf-emb.lua @@ -343,7 +343,7 @@ local loadmapfile, loadmapline, getmapentry do if not encoding then return end - local pfbfile = resolvers.find_file(m[3],"pfb") + local pfbfile = resolvers.findfile(m[3],"pfb") if not pfbfile or pfbfile == "" then return end diff --git a/tex/context/base/mkiv/lpdf-lmt.lua b/tex/context/base/mkiv/lpdf-lmt.lua index e79a5a17f..b09f0a4f8 100644 --- a/tex/context/base/mkiv/lpdf-lmt.lua +++ b/tex/context/base/mkiv/lpdf-lmt.lua @@ -1363,6 +1363,21 @@ local wrapup, registerpage do end end + -- We can have this, but then via codeinjections etc. Later. + + -- function structures.pages.swapthem() + -- local n = lpdf.getnofpages() + -- local t = { } + -- for i=1,n do + -- t[i] = i + -- end + -- for i=2,math.odd(n) and n or (n-1),2 do + -- t[i] = i+1 + -- t[i+1] = i + -- end + -- lpdf.setpageorder(t) + -- end + wrapup = function(driver) -- hook (to reshuffle pages) diff --git a/tex/context/base/mkiv/lpdf-mis.lua b/tex/context/base/mkiv/lpdf-mis.lua index c5422b49e..7410ed9ab 100644 --- a/tex/context/base/mkiv/lpdf-mis.lua +++ b/tex/context/base/mkiv/lpdf-mis.lua @@ -332,7 +332,7 @@ local plusspecs = { direction = "L2R", }, [v_righttoleft] ={ - direction = "R2R", + direction = "R2L", }, [v_nomenubar] ={ nomenubar = true, diff --git a/tex/context/base/mkiv/luat-lib.mkiv b/tex/context/base/mkiv/luat-lib.mkiv index 5b14b1d11..8c80072e5 100644 --- a/tex/context/base/mkiv/luat-lib.mkiv +++ b/tex/context/base/mkiv/luat-lib.mkiv @@ -57,12 +57,12 @@ \registerctxluafile{data-res}{} \registerctxluafile{data-inp}{} \registerctxluafile{data-out}{} -\registerctxluafile{data-fil}{} +\registerctxluafile{data-fil}{} % opener gets overloaded in data-tex -\registerctxluafile{data-pre}{} -\registerctxluafile{data-tex}{} +\registerctxluafile{data-pre}{} % after data-res +\registerctxluafile{data-bin}{} % before data-tex +\registerctxluafile{data-tex}{} % after data-pre \registerctxluafile{data-vir}{} -\registerctxluafile{data-bin}{} \registerctxluafile{data-zip}{} %registerctxluafile{data-crl}{} \registerctxluafile{data-sch}{} diff --git a/tex/context/base/mkiv/luat-mac.lua b/tex/context/base/mkiv/luat-mac.lua index 91148bc32..e6789a233 100644 --- a/tex/context/base/mkiv/luat-mac.lua +++ b/tex/context/base/mkiv/luat-mac.lua @@ -22,6 +22,7 @@ local rep, sub = string.rep, string.sub local setmetatable = setmetatable local filesuffix = file.suffix local convertlmxstring = lmx and lmx.convertstring +local savedata = io.savedata local pushtarget, poptarget = logs.pushtarget, logs.poptarget @@ -195,35 +196,27 @@ local checker = P("%") * (1 - newline - P("macros"))^0 -- maybe namespace -local macros = { } resolvers.macros = macros +local resolvers = resolvers + +local macros = { } +resolvers.macros = macros + +local loadtexfile = resolvers.loadtexfile function macros.preprocessed(str,strip) return lpegmatch(parser,str,1,strip) end function macros.convertfile(oldname,newname) -- beware, no testing on oldname == newname - local data = resolvers.loadtexfile(oldname) + local data = loadtexfile(oldname) data = interfaces.preprocessed(data) or "" -- interfaces not yet defined - io.savedata(newname,data) + savedata(newname,data) end function macros.version(data) return lpegmatch(checker,data) end --- function macros.processmkvi(str,filename) --- if filename and filesuffix(filename) == "mkvi" or lpegmatch(checker,str) == "mkvi" then --- local oldsize = #str --- str = lpegmatch(parser,str,1,true) or str --- pushtarget("logfile") --- report_macros("processed mkvi file %a, delta %s",filename,oldsize-#str) --- poptarget() --- end --- return str --- end --- --- utilities.sequencers.appendaction(resolvers.openers.helpers.textfileactions,"system","resolvers.macros.processmkvi") - -- the document variables hack is temporary local processors = { } @@ -305,40 +298,33 @@ end macros.processmklx = macros.processmkvi -local sequencers = utilities.sequencers - -if sequencers then - - sequencers.appendaction(resolvers.openers.helpers.textfileactions,"system","resolvers.macros.processmk") - sequencers.appendaction(resolvers.openers.helpers.textfileactions,"system","resolvers.macros.processmkvi") - -end - -- bonus -if resolvers.schemes then +local schemes = resolvers.schemes + +if schemes then local function handler(protocol,name,cachename) local hashed = url.hashed(name) local path = hashed.path if path and path ~= "" then - local str = resolvers.loadtexfile(path) + local str = loadtexfile(path) if validvi(path,str) then -- already done automatically - io.savedata(cachename,str) + savedata(cachename,str) else local result = lpegmatch(parser,str,1,true) or str pushtarget("logfile") report_macros("processed scheme %a, delta %s",filename,#str-#result) poptarget() - io.savedata(cachename,result) + savedata(cachename,result) end end return cachename end - resolvers.schemes.install('mkvi',handler,1) - resolvers.schemes.install('mklx',handler,1) + schemes.install('mkvi',handler,1) + schemes.install('mklx',handler,1) end @@ -430,7 +416,7 @@ local encodepattern = Cs((encodecomment + 1)^0) local decodecomment = P(commentsignal) / "%%%%" -- why doubles here? local decodepattern = Cs((decodecomment + 1)^0) -function resolvers.macros.encodecomment(str) +function macros.encodecomment(str) if txtcatcodes and tex.catcodetable == txtcatcodes then return lpegmatch(encodepattern,str) or str else @@ -438,7 +424,7 @@ function resolvers.macros.encodecomment(str) end end -function resolvers.macros.decodecomment(str) -- normally not needed +function macros.decodecomment(str) -- normally not needed return txtcatcodes and lpegmatch(decodepattern,str) or str end @@ -446,9 +432,22 @@ end -- resolvers.macros.encodecommentpattern = encodepattern -- resolvers.macros.decodecommentpattern = decodepattern -function resolvers.macros.enablecomment(thecatcodes) - if not txtcatcodes then - txtcatcodes = thecatcodes or catcodes.numbers.txtcatcodes - utilities.sequencers.appendaction(resolvers.openers.helpers.textlineactions,"system","resolvers.macros.encodecomment") +local sequencers = utilities.sequencers +local appendaction = sequencers and sequencers.appendaction + +if appendaction then + + local textlineactions = resolvers.openers.helpers.textlineactions + local textfileactions = resolvers.openers.helpers.textfileactions + + appendaction(textfileactions,"system","resolvers.macros.processmk") + appendaction(textfileactions,"system","resolvers.macros.processmkvi") + + function macros.enablecomment(thecatcodes) + if not txtcatcodes then + txtcatcodes = thecatcodes or catcodes.numbers.txtcatcodes + appendaction(textlineactions,"system","resolvers.macros.encodecomment") + end end + end diff --git a/tex/context/base/mkiv/mlib-lua.lua b/tex/context/base/mkiv/mlib-lua.lua index 666025154..83ce152e2 100644 --- a/tex/context/base/mkiv/mlib-lua.lua +++ b/tex/context/base/mkiv/mlib-lua.lua @@ -1041,15 +1041,16 @@ end do - local mprint = mp.print - local fprint = mp.fprint - local qprint = mp.quoted - local getwhd = job.positions.whd - local getxy = job.positions.xy - local getposition = job.positions.position - local getpage = job.positions.page - local getregion = job.positions.region - local getmacro = tokens.getters.macro + local mprint = mp.print + local fprint = mp.fprint + local qprint = mp.quoted + local jobpositions = job.positions + local getwhd = jobpositions.whd + local getxy = jobpositions.xy + local getposition = jobpositions.position + local getpage = jobpositions.page + local getregion = jobpositions.region + local getmacro = tokens.getters.macro function mp.positionpath(name) local w, h, d = getwhd(name) diff --git a/tex/context/base/mkiv/mult-def.lua b/tex/context/base/mkiv/mult-def.lua index 68deb6084..1e22aaa86 100644 --- a/tex/context/base/mkiv/mult-def.lua +++ b/tex/context/base/mkiv/mult-def.lua @@ -9629,6 +9629,10 @@ return { ["pe"]="شماره‌خود", ["ro"]="numarpropriu", }, + ["packcriterium"]={ + ["en"]="packcriterium", + ["nl"]="opelkaarcriterium", + }, ["page"]={ ["cs"]="stranka", ["de"]="seite", diff --git a/tex/context/base/mkiv/node-aux.lua b/tex/context/base/mkiv/node-aux.lua index e116a2cdc..ef884987b 100644 --- a/tex/context/base/mkiv/node-aux.lua +++ b/tex/context/base/mkiv/node-aux.lua @@ -473,13 +473,34 @@ end -- end -- end -function nuts.setparproperty(action,...) - local tail = tonut(texnest[texnest.ptr].tail) - while tail do - if getid(tail) == localpar_code then - return action(tail,...) - else +do + + local localparcodes = nodes.localparcodes + local hmodepar_code = localparcodes.vmode_par + local vmodepar_code = localparcodes.hmode_par + + local getsubtype = nuts.getsubtype + + function nuts.setparproperty(action,...) + local tail = tonut(texnest[texnest.ptr].tail) + while tail do + if getid(tail) == localpar_code then + local s = getsubtype(tail) + if s == hmodepar_code or s == vmodepar_code then + return action(tail,...) + else + -- something is wrong here + end + end tail = getprev(tail) end end + + local getsubtype = nodes.getsubtype + + function nodes.start_of_par(n) + local s = getsubtype(n) + return s == hmodepar_code or s == vmodepar_code + end + end diff --git a/tex/context/base/mkiv/node-dir.lua b/tex/context/base/mkiv/node-dir.lua index 3f0cba67e..abfd4a8c7 100644 --- a/tex/context/base/mkiv/node-dir.lua +++ b/tex/context/base/mkiv/node-dir.lua @@ -31,13 +31,18 @@ local dirvalues = nodes.dirvalues local lefttoright = dirvalues.lefttoright local righttoleft = dirvalues.righttoleft -local localparnewgraf_code = 0 +local localparcodes = nodes.localparcodes +local hmodepar_code = localparcodes.vmode_par +local vmodepar_code = localparcodes.hmode_par -local function newstack(head,direction) +function nodes.dirstack(head,direction) local stack = { } local top = 0 - if head and getid(head) == localpar_code and getsubtype(head) == localparnewgraf_code then - direction = getdirection(head) + if head and getid(head) == localpar_code then + local s = getsubtype(head) + if s == hmodepar_code or s == vmodepar_code then + direction = getdirection(head) + end end if not direction then direction = lefttoright diff --git a/tex/context/base/mkiv/node-ini.lua b/tex/context/base/mkiv/node-ini.lua index ce0a473ba..b4346cfe0 100644 --- a/tex/context/base/mkiv/node-ini.lua +++ b/tex/context/base/mkiv/node-ini.lua @@ -7,15 +7,13 @@ if not modules then modules = { } end modules ['node-ini'] = { } --[[ldx-- -<p>Most of the code that had accumulated here is now separated in -modules.</p> +<p>Most of the code that had accumulated here is now separated in modules.</p> --ldx]]-- --- this module is being reconstructed --- --- todo: datatype table per node type - --- todo: query names with new node.subtypes +-- I need to clean up this module as it's a bit of a mess now. The latest luatex +-- has most tables but we have a few more in luametatex. Also, some are different +-- between these engines. We started out with hardcoded tables, that then ended +-- up as comments and are now gone (as they differ per engine anyway). local next, type, tostring = next, type, tostring local gsub = string.gsub @@ -23,36 +21,34 @@ local concat, remove = table.concat, table.remove local sortedhash, sortedkeys, swapped = table.sortedhash, table.sortedkeys, table.swapped --[[ldx-- -<p>Access to nodes is what gives <l n='luatex'/> its power. Here we -implement a few helper functions. These functions are rather optimized.</p> +<p>Access to nodes is what gives <l n='luatex'/> its power. Here we implement a +few helper functions. These functions are rather optimized.</p> --ldx]]-- --[[ldx-- -<p>When manipulating node lists in <l n='context'/>, we will remove -nodes and insert new ones. While node access was implemented, we did -quite some experiments in order to find out if manipulating nodes -in <l n='lua'/> was feasible from the perspective of performance.</p> - -<p>First of all, we noticed that the bottleneck is more with excessive -callbacks (some gets called very often) and the conversion from and to -<l n='tex'/>'s datastructures. However, at the <l n='lua'/> end, we -found that inserting and deleting nodes in a table could become a -bottleneck.</p> - -<p>This resulted in two special situations in passing nodes back to -<l n='tex'/>: a table entry with value <type>false</type> is ignored, -and when instead of a table <type>true</type> is returned, the -original table is used.</p> - -<p>Insertion is handled (at least in <l n='context'/> as follows. When -we need to insert a node at a certain position, we change the node at -that position by a dummy node, tagged <type>inline</type> which itself -has_attribute the original node and one or more new nodes. Before we pass -back the list we collapse the list. Of course collapsing could be built -into the <l n='tex'/> engine, but this is a not so natural extension.</p> - -<p>When we collapse (something that we only do when really needed), we -also ignore the empty nodes. [This is obsolete!]</p> +<p>When manipulating node lists in <l n='context'/>, we will remove nodes and +insert new ones. While node access was implemented, we did quite some experiments +in order to find out if manipulating nodes in <l n='lua'/> was feasible from the +perspective of performance.</p> + +<p>First of all, we noticed that the bottleneck is more with excessive callbacks +(some gets called very often) and the conversion from and to <l n='tex'/>'s +datastructures. However, at the <l n='lua'/> end, we found that inserting and +deleting nodes in a table could become a bottleneck.</p> + +<p>This resulted in two special situations in passing nodes back to <l n='tex'/>: +a table entry with value <type>false</type> is ignored, and when instead of a +table <type>true</type> is returned, the original table is used.</p> + +<p>Insertion is handled (at least in <l n='context'/> as follows. When we need to +insert a node at a certain position, we change the node at that position by a +dummy node, tagged <type>inline</type> which itself has_attribute the original +node and one or more new nodes. Before we pass back the list we collapse the +list. Of course collapsing could be built into the <l n='tex'/> engine, but this +is a not so natural extension.</p> + +<p>When we collapse (something that we only do when really needed), we also +ignore the empty nodes. [This is obsolete!]</p> --ldx]]-- -- local gf = node.direct.getfield @@ -76,206 +72,25 @@ tex.magicconstants = { -- we use tex.constants for something else trueinch = 4736286, } --- local listcodes = allocate { --- [0] = "unknown", --- [1] = "line", --- [2] = "box", --- [3] = "indent", --- [4] = "alignment", -- row or column --- [5] = "cell", --- [6] = "equation", --- [7] = "equationnumber", --- } - -local listcodes = mark(getsubtypes("list")) - --- local rulecodes = allocate { --- [0] = "normal", --- [1] = "box", --- [2] = "image", --- [3] = "empty", --- [4] = "user", --- } - -local rulecodes = mark(getsubtypes("rule")) - -if not rulecodes[5] then - rulecodes[5] = "over" - rulecodes[6] = "under" - rulecodes[7] = "fraction" - rulecodes[8] = "radical" -end - --- local dircodes = mark(getsubtypes("dir")) - -dircodes = allocate { - [0] = "normal", - [1] = "cancel", -} - --- local glyphcodes = allocate { --- [ 1] = "character", --- [ 2] = "ligature", --- [ 4] = "ghost", --- [ 8] = "left", --- [16] = "right", --- } - -local glyphcodes = mark(getsubtypes("glyph")) - --- local disccodes = allocate { --- [0] = "discretionary", -- \discretionary --- [1] = "explicit", -- \- --- [2] = "automatic", -- following a - --- [3] = "regular", -- by hyphenator: simple --- [4] = "first", -- by hyphenator: hard first item --- [5] = "second", -- by hyphenator: hard second item --- } - -local disccodes = mark(getsubtypes("disc")) - --- local gluecodes = allocate { --- [ 0] = "userskip", --- [ 1] = "lineskip", --- [ 2] = "baselineskip", --- [ 3] = "parskip", --- [ 4] = "abovedisplayskip", --- [ 5] = "belowdisplayskip", --- [ 6] = "abovedisplayshortskip", --- [ 7] = "belowdisplayshortskip", --- [ 8] = "leftskip", --- [ 9] = "rightskip", --- [ 10] = "topskip", --- [ 11] = "splittopskip", --- [ 12] = "tabskip", --- [ 13] = "spaceskip", --- [ 14] = "xspaceskip", --- [ 15] = "parfillskip", --- [ 16] = "mathskip", -- experiment --- [ 17] = "thinmuskip", --- [ 18] = "medmuskip", --- [ 19] = "thickmuskip", --- [ 98] = "conditionalmathglue", --- [ 99] = "muskip", --- [100] = "leaders", --- [101] = "cleaders", --- [102] = "xleaders", --- [103] = "gleaders", --- } - -local gluecodes = mark(getsubtypes("glue")) - --- local leadercodes = allocate { --- [100] = "leaders", --- [101] = "cleaders", --- [102] = "xleaders", --- [103] = "gleaders", --- } - -local leadercodes = mark(getsubtypes("leader")) - --- local fillcodes = allocate { --- [0] = "stretch", --- [1] = "fi", --- [2] = "fil", --- [3] = "fill", --- [4] = "filll", --- } - -local fillcodes = mark(getsubtypes("fill")) - --- for now: - -local boundarycodes = allocate { - [0] = "cancel", - [1] = "user", - [2] = "protrusion", - [3] = "word", -} - --- local boundarycodes = mark(getsubtypes("boundary")) - --- local penaltycodes = allocate { -- unfortunately not used (yet) --- [ 0] = "userpenalty", --- } - -local penaltycodes = mark(getsubtypes("penalty")) - -table.setmetatableindex(penaltycodes,function(t,k) return "userpenalty" end) -- not used anyway - --- local kerncodes = allocate { --- [0] = "fontkern", --- [1] = "userkern", --- [2] = "accentkern", --- [3] = "italiccorrection", --- } - -local kerncodes = mark(getsubtypes("kern")) - --- local margincodes = allocate { --- [0] = "left", --- [1] = "right", --- } - -local margincodes = mark(getsubtypes("marginkern")) - --- local mathcodes = allocate { --- [0] = "beginmath", --- [1] = "endmath", --- } - -local mathcodes = mark(getsubtypes("math")) - --- local noadcodes = allocate { -- simple nodes --- [ 0] = "ord", --- [ 1] = "opdisplaylimits", --- [ 2] = "oplimits", --- [ 3] = "opnolimits", --- [ 4] = "bin", --- [ 5] = "rel", --- [ 6] = "open", --- [ 7] = "close", --- [ 8] = "punct", --- [ 9] = "inner", --- [10] = "under", --- [11] = "over", --- [12] = "vcenter", --- } - -local noadcodes = mark(getsubtypes("noad")) - --- local radicalcodes = allocate { --- [0] = "radical", --- [1] = "uradical", --- [2] = "uroot", --- [3] = "uunderdelimiter", --- [4] = "uoverdelimiter", --- [5] = "udelimiterunder", --- [6] = "udelimiterover", --- } - -local radicalcodes = mark(getsubtypes("radical")) - --- local accentcodes = allocate { --- [0] = "bothflexible", --- [1] = "fixedtop", --- [2] = "fixedbottom", --- [3] = "fixedboth", --- } - -local accentcodes = mark(getsubtypes("accent")) - --- local fencecodes = allocate { --- [0] = "unset", --- [1] = "left", --- [2] = "middle", --- [3] = "right", --- [4] = "no", --- } - -local fencecodes = mark(getsubtypes("fence")) - --- maybe we also need fractioncodes +local listcodes = mark(getsubtypes("list")) +local rulecodes = mark(getsubtypes("rule")) +local dircodes = mark(getsubtypes("dir")) +local glyphcodes = mark(getsubtypes("glyph")) +local disccodes = mark(getsubtypes("disc")) +local gluecodes = mark(getsubtypes("glue")) +local leadercodes = mark(getsubtypes("leader")) +local fillcodes = mark(getsubtypes("fill")) +local boundarycodes = mark(getsubtypes("boundary")) +local penaltycodes = mark(getsubtypes("penalty")) +local kerncodes = mark(getsubtypes("kern")) +local margincodes = mark(getsubtypes("marginkern")) +local mathcodes = mark(getsubtypes("math")) +local noadcodes = mark(getsubtypes("noad")) +local radicalcodes = mark(getsubtypes("radical")) +local accentcodes = mark(getsubtypes("accent")) +local fencecodes = mark(getsubtypes("fence")) +----- fractioncodes = mark(getsubtypes("fraction")) +local localparcodes = allocate { [0] = "new_graf", "local_box", "hmode_par", "penalty", "math" } -- only in luametatex now local function simplified(t) local r = { } @@ -356,6 +171,7 @@ margincodes = allocate(swapped(margincodes,margincodes)) disccodes = allocate(swapped(disccodes,disccodes)) accentcodes = allocate(swapped(accentcodes,accentcodes)) fencecodes = allocate(swapped(fencecodes,fencecodes)) +localparcodes = allocate(swapped(localparcodes,localparcodes)) rulecodes = allocate(swapped(rulecodes,rulecodes)) leadercodes = allocate(swapped(leadercodes,leadercodes)) usercodes = allocate(swapped(usercodes,usercodes)) @@ -364,6 +180,12 @@ dirvalues = allocate(swapped(dirvalues,dirvalues)) gluevalues = allocate(swapped(gluevalues,gluevalues)) literalvalues = allocate(swapped(literalvalues,literalvalues)) +if not gluecodes.indentskip then + gluecodes.indentskip = gluecodes.userskip + gluecodes.lefthangskip = gluecodes.userskip + gluecodes.righthangskip = gluecodes.userskip +end + if CONTEXTLMTXMODE > 0 then whatcodes.literal = 0x1 whatcodes[0x1] = "literal" whatcodes.latelua = 0x2 whatcodes[0x2] = "latelua" @@ -399,6 +221,7 @@ nodes.disccodes = disccodes nodes.accentcodes = accentcodes nodes.radicalcodes = radicalcodes nodes.fencecodes = fencecodes +nodes.localparcodes = localparcodes nodes.rulecodes = rulecodes nodes.leadercodes = leadercodes nodes.usercodes = usercodes @@ -420,12 +243,14 @@ nodes.subtypes = allocate { [nodecodes.glyph] = glyphcodes, [nodecodes.hlist] = listcodes, [nodecodes.kern] = kerncodes, + [nodecodes.localpar] = localparcodes, [nodecodes.marginkern] = margincodes, [nodecodes.math] = mathcodes, [nodecodes.noad] = noadcodes, [nodecodes.penalty] = penaltycodes, [nodecodes.radical] = radicalcodes, [nodecodes.rule] = rulecodes, + -- [nodecodes.user] = usercodes, [nodecodes.vlist] = listcodes, [nodecodes.whatsit] = whatcodes, } @@ -447,6 +272,8 @@ nodes.literalvalues = literalvalues -- more friendly glyphcodes.glyph = glyphcodes.character +localparcodes.vmode_par = localparcodes.new_graf + listcodes.row = listcodes.alignment listcodes.column = listcodes.alignment @@ -533,4 +360,3 @@ if CONTEXTLMTXMODE > 0 then end end - diff --git a/tex/context/base/mkiv/node-ini.mkiv b/tex/context/base/mkiv/node-ini.mkiv index 35013c4b8..685614bdb 100644 --- a/tex/context/base/mkiv/node-ini.mkiv +++ b/tex/context/base/mkiv/node-ini.mkiv @@ -22,7 +22,7 @@ \registerctxluafile{node-nut}{} \registerctxluafile{node-res}{} %registerctxluafile{node-ppt}{} % experimental, not used so probably useless -\registerctxluafile{node-dir}{} +%registerctxluafile{node-dir}{} % experimental, not yet (and maybe never) used \registerctxluafile{node-aux}{} \registerctxluafile{node-tst}{} \registerctxluafile{node-tra}{} % we might split it off (module) diff --git a/tex/context/base/mkiv/node-ltp.lua b/tex/context/base/mkiv/node-ltp.lua index 0b2790cc7..3e4fea0b1 100644 --- a/tex/context/base/mkiv/node-ltp.lua +++ b/tex/context/base/mkiv/node-ltp.lua @@ -286,8 +286,6 @@ local protrusionboundary_code = boundarycodes.protrusion local leaders_code = leadercodes.leaders -local localpar_code = nodecodes.localpar - local userkern_code = kerncodes.userkern local italickern_code = kerncodes.italiccorrection local fontkern_code = kerncodes.fontkern @@ -2845,8 +2843,6 @@ do local adjust_tail = adjust_head and find_tail(adjust_head) local pre_adjust_tail = pre_adjust_head and find_tail(pre_adjust_head) - new_dir_stack(hpack_dir) - local checked_expansion = false if cal_expand_ratio then @@ -2983,7 +2979,7 @@ do end adjust_tail = find_tail(list) elseif id == dir_code then - hpack_dir = checked_line_dir(stack,current) or hpack_dir + -- no need to deal with directions here (as we only support two) elseif id == marginkern_code then local width = getwidth(current) if cal_expand_ratio then @@ -3018,9 +3014,7 @@ do local delta = width - natural if delta == 0 then - setfield(hlist,"glue_sign",0) - setfield(hlist,"glue_order",0) - setfield(hlist,"glue_set",0) + setglue(hlist,0,0,0) -- set order sign elseif delta > 0 then -- natural width smaller than requested width local order = (total_stretch[4] ~= 0 and 4 or total_stretch[3] ~= 0 and 3) or @@ -3057,13 +3051,9 @@ do end local tso = total_stretch[order] if tso ~= 0 then - setfield(hlist,"glue_sign",1) - setfield(hlist,"glue_order",order) - setfield(hlist,"glue_set",delta/tso) + setglue(hlist,delta/tso,order,1) -- set order sign else - setfield(hlist,"glue_sign",0) - setfield(hlist,"glue_order",order) - setfield(hlist,"glue_set",0) + setglue(hlist,0,order,0) -- set order sign end if font_expand_ratio ~= 0 then -- todo @@ -3113,13 +3103,9 @@ do end local tso = total_shrink[order] if tso ~= 0 then - setfield(hlist,"glue_sign",2) - setfield(hlist,"glue_order",order) - setfield(hlist,"glue_set",-delta/tso) + setglue(hlist,-delta/tso,order,2) -- set order sign else - setfield(hlist,"glue_sign",0) - setfield(hlist,"glue_order",order) - setfield(hlist,"glue_set",0) + setglue(hlist,0,order,0) -- set order sign end if font_expand_ratio ~= 0 then -- todo diff --git a/tex/context/base/mkiv/node-nut.lua b/tex/context/base/mkiv/node-nut.lua index 6c4b034c9..7174bd475 100644 --- a/tex/context/base/mkiv/node-nut.lua +++ b/tex/context/base/mkiv/node-nut.lua @@ -204,6 +204,7 @@ nuts.writable_spec = direct.writable_spec nuts.write = direct.write nuts.mlist_to_hlist = direct.mlist_to_hlist nuts.has_dimensions = direct.has_dimensions +nuts.start_of_par = direct.start_of_par if not nuts.mlist_to_hlist then @@ -403,6 +404,23 @@ end -- for now +if not nuts.start_of_par then + + local localparcodes = nodes.localparcodes + local hmodepar_code = localparcodes.vmode_par + local vmodepar_code = localparcodes.hmode_par + + local getsubtype = nuts.getsubtype + + function nuts.start_of_par(n) + local s = getsubtype(n) + return s == hmodepar_code or s == vmodepar_code + end + +end + +-- for now + if not nuts.getpre then local d_getdisc = direct.getdisc diff --git a/tex/context/base/mkiv/node-ref.lua b/tex/context/base/mkiv/node-ref.lua index 6828fab0d..ac43a84a3 100644 --- a/tex/context/base/mkiv/node-ref.lua +++ b/tex/context/base/mkiv/node-ref.lua @@ -77,6 +77,7 @@ local getdimensions = nuts.dimensions local getrangedimensions = nuts.rangedimensions local traverse = nuts.traverse local find_node_tail = nuts.tail +local start_of_par = nuts.start_of_par local nodecodes = nodes.nodecodes local gluecodes = nodes.gluecodes @@ -419,7 +420,7 @@ local function inject_areas(head,attribute,make,stack,done,skip,parent,pardir,tx local direction, pop = getdirection(current) txtdir = not pop and direction -- we might need a stack elseif id == localpar_code then - if getsubtype(current) == 0 then + if start_of_par(current) then pardir = getdirection(current) end else diff --git a/tex/context/base/mkiv/node-res.lua b/tex/context/base/mkiv/node-res.lua index 1a3605410..b0a1e1c65 100644 --- a/tex/context/base/mkiv/node-res.lua +++ b/tex/context/base/mkiv/node-res.lua @@ -338,7 +338,7 @@ end function nutpool.textdir(dir) -- obsolete ! local t = copy_nut(textdir) if dir then - setdir(t,dir) + setdirection(t,dir) end return t end diff --git a/tex/context/base/mkiv/node-tra.lua b/tex/context/base/mkiv/node-tra.lua index e2aaf8bbe..c7556aba6 100644 --- a/tex/context/base/mkiv/node-tra.lua +++ b/tex/context/base/mkiv/node-tra.lua @@ -83,6 +83,8 @@ local whatsit_code = nodecodes.whatsit local dimenfactors = number.dimenfactors local formatters = string.formatters +local start_of_par = nuts.start_of_par + -- this will be reorganized: function nodes.showlist(head, message) @@ -165,7 +167,7 @@ local function tosequence(start,stop,compact) elseif id == dir_code then local d, p = getdirection(start) n = n + 1 ; t[n] = "[<" .. (p and "-" or "+") .. d .. ">]" -- todo l2r etc - elseif id == localpar_code and getsubtype(start) == 0 then + elseif id == localpar_code and start_of_par(current) then n = n + 1 ; t[n] = "[<" .. getdirection(start) .. ">]" -- todo l2r etc elseif compact then n = n + 1 ; t[n] = "[]" diff --git a/tex/context/base/mkiv/page-lay.mkiv b/tex/context/base/mkiv/page-lay.mkiv index 2b801b957..d115b588d 100644 --- a/tex/context/base/mkiv/page-lay.mkiv +++ b/tex/context/base/mkiv/page-lay.mkiv @@ -166,6 +166,8 @@ \def\layoutdistance#1#2{\ifdim\zeropoint<#1#2\else\zeropoint\fi} +% \newconditional\reverselayout + \def\page_layouts_set_dimensions {\global\naturalmarginwidth \layoutparameter\c!margin \global\naturaledgewidth \layoutparameter\c!edge @@ -194,8 +196,7 @@ \global\headerdistance \layoutdistance\headerheight \naturalheaderdistance \global\textdistance \naturaltextdistance \global\footerdistance \layoutdistance\footerheight \naturalfooterdistance - \global\bottomdistance \layoutdistance\bottomheight \naturalbottomdistance - } + \global\bottomdistance \layoutdistance\bottomheight \naturalbottomdistance} \def\page_layouts_set_distances % local in \setreducedvsize {\headerdistance\layoutdistance\headerheight\naturalheaderdistance @@ -729,6 +730,7 @@ +\rightedgetotal \relax % \page_layouts_check_next % here ? + \page_layouts_check_direction \page_layouts_calculate_extras \page_target_check_centering \calculatehsizes @@ -736,6 +738,21 @@ \page_layouts_check_pseudo_columns \page_backgrounds_recalculate} +\def\page_layouts_check_direction + {\edef\p_direction{\layoutparameter\c!direction}% + \ifx\p_direction\v!reverse + \globalswapdimens\naturalleftedgedistance \naturalrightedgedistance + \globalswapdimens\naturalleftmargindistance\naturalrightmargindistance + \globalswapdimens\leftedgedistance \rightedgedistance + \globalswapdimens\leftmargindistance \rightmargindistance + \globalswapdimens\leftmarginwidth \rightmarginwidth + \globalswapdimens\leftedgewidth \rightedgewidth + \globalswapdimens\backspace \cutspace + \expandafter\setsystemmode + \else + \expandafter\resetsystemmode + \fi{reverselayout}} + \def\page_layouts_check_pseudo_columns {\global\layoutcolumns\layoutparameter\c!columns \global\layoutcolumndistance\layoutparameter\c!columndistance @@ -1408,6 +1425,7 @@ \c!dx=\zeropoint, \c!dy=\zeropoint, \c!grid=\v!no, + \c!direction=\v!normal, \c!preset=, \c!setups=\systemsetupsprefix\s!default, \c!clipoffset=\zeropoint, diff --git a/tex/context/base/mkiv/spac-ver.lua b/tex/context/base/mkiv/spac-ver.lua index 1410e3278..33b81e5c7 100644 --- a/tex/context/base/mkiv/spac-ver.lua +++ b/tex/context/base/mkiv/spac-ver.lua @@ -153,6 +153,7 @@ local remove_node = nuts.remove local count_nodes = nuts.countall local hpack_node = nuts.hpack local vpack_node = nuts.vpack +local start_of_par = nuts.start_of_par local nextnode = nuts.traversers.node local nexthlist = nuts.traversers.hlist @@ -291,7 +292,7 @@ end local function validvbox(parentid,list) if parentid == hlist_code then local id = getid(list) - if id == localpar_code and getsubtype(list) == 0 then + if id == localpar_code and start_of_par(list) then list = getnext(list) if not next then return nil @@ -327,7 +328,7 @@ local function already_done(parentid,list,a_snapmethod) -- todo: done when only -- problem: any snapped vbox ends up in a line if list and parentid == hlist_code then local id = getid(list) - if id == localpar_code and getsubtype(list) == 0 then + if id == localpar_code and start_of_par(list) then list = getnext(list) if not list then return false @@ -2063,6 +2064,7 @@ end do local outer = texnest[0] + local enabled = true local trace = false local report = logs.reporter("vspacing") diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf Binary files differindex ac45cbe4e..42fe96dea 100644 --- a/tex/context/base/mkiv/status-files.pdf +++ b/tex/context/base/mkiv/status-files.pdf diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf Binary files differindex 726425def..40037aac1 100644 --- a/tex/context/base/mkiv/status-lua.pdf +++ b/tex/context/base/mkiv/status-lua.pdf diff --git a/tex/context/base/mkiv/strc-itm.mklx b/tex/context/base/mkiv/strc-itm.mklx index c456bf470..60fe3ee74 100644 --- a/tex/context/base/mkiv/strc-itm.mklx +++ b/tex/context/base/mkiv/strc-itm.mklx @@ -467,26 +467,17 @@ \setvalue{\??itemgroupkeyword\v!seven }{\letitemgroupparameter\c!n\plusseven} \setvalue{\??itemgroupkeyword\v!eight }{\letitemgroupparameter\c!n\pluseight} \setvalue{\??itemgroupkeyword\v!nine }{\letitemgroupparameter\c!n\plusnine} -%setvalue{\??itemgroupkeyword\v!standard }{\setupcurrentitemgroup -% [\c!width =1.5\emwidth,% -% \c!distance =.5\emwidth,% -% \c!factor =0,% -% \c!inner =,% -% \c!beforehead=,% -% \c!afterhead =\blank,% -% \c!before =\blank,% -% \c!inbetween =\blank,% -% \c!after =\blank]} -\setvalue{\??itemgroupkeyword\v!standard }{\setitemgroupparameter\c!width {1.5\emwidth}% - \setitemgroupparameter\c!distance {.5\emwidth}% - \letitemgroupparameter\c!factor \!!zerocount - \letitemgroupparameter\c!inner \empty - \letitemgroupparameter\c!beforehead\empty - \letitemgroupparameter\c!afterhead \blank - \letitemgroupparameter\c!before \blank - \letitemgroupparameter\c!inbetween \blank - \letitemgroupparameter\c!after \blank} - +\setvalue{\??itemgroupkeyword\v!standard }{\setitemgroupparameter\c!width {1.5\emwidth}% + \setitemgroupparameter\c!distance {.5\emwidth}% + %\letitemgroupparameter\c!factor \!!zerocount + \letitemgroupparameter\c!factor {\zerocount}% + \setitemgroupparameter\c!packcriterium{\zerocount}% + \letitemgroupparameter\c!inner \empty + \letitemgroupparameter\c!beforehead \empty + \letitemgroupparameter\c!afterhead \blank + \letitemgroupparameter\c!before \blank + \letitemgroupparameter\c!inbetween \blank + \letitemgroupparameter\c!after \blank} \def\strc_itemgroups_initialize_local {\setfalse\c_strc_itemgroups_inline @@ -882,6 +873,12 @@ \fi \fi % + \ifconditional\c_strc_itemgroups_pack + \ifnum\currentnofitems>\itemgroupparameter\c!packcriterium\relax\else + \setfalse\c_strc_itemgroups_pack + \fi + \fi + % \ifnum\c_strc_itemgroups_nesting=\plusone % NIEUW \doadaptleftskip {\itemgroupparameter\c!margin}% \doadaptleftskip {\itemgroupparameter\c!leftmargin}% @@ -1836,6 +1833,7 @@ \c!lefttext=(, \c!righttext=), \c!start=1, + \c!packcriterium=\zerocount, \c!criterium=\v!all, % permits 0 and negative numbers %\c!option=, \c!textdistance=\v!space, % none big medium small <dimension> diff --git a/tex/context/base/mkiv/strc-itm.mkvi b/tex/context/base/mkiv/strc-itm.mkvi index f835aa352..cbbefd5db 100644 --- a/tex/context/base/mkiv/strc-itm.mkvi +++ b/tex/context/base/mkiv/strc-itm.mkvi @@ -467,26 +467,17 @@ \setvalue{\??itemgroupkeyword\v!seven }{\letitemgroupparameter\c!n\plusseven} \setvalue{\??itemgroupkeyword\v!eight }{\letitemgroupparameter\c!n\pluseight} \setvalue{\??itemgroupkeyword\v!nine }{\letitemgroupparameter\c!n\plusnine} -%setvalue{\??itemgroupkeyword\v!standard }{\setupcurrentitemgroup -% [\c!width =1.5\emwidth,% -% \c!distance =.5\emwidth,% -% \c!factor =0,% -% \c!inner =,% -% \c!beforehead=,% -% \c!afterhead =\blank,% -% \c!before =\blank,% -% \c!inbetween =\blank,% -% \c!after =\blank]} -\setvalue{\??itemgroupkeyword\v!standard }{\setitemgroupparameter\c!width {1.5\emwidth}% - \setitemgroupparameter\c!distance {.5\emwidth}% - \letitemgroupparameter\c!factor \!!zerocount - \letitemgroupparameter\c!inner \empty - \letitemgroupparameter\c!beforehead\empty - \letitemgroupparameter\c!afterhead \blank - \letitemgroupparameter\c!before \blank - \letitemgroupparameter\c!inbetween \blank - \letitemgroupparameter\c!after \blank} - +\setvalue{\??itemgroupkeyword\v!standard }{\setitemgroupparameter\c!width {1.5\emwidth}% + \setitemgroupparameter\c!distance {.5\emwidth}% + %\letitemgroupparameter\c!factor \!!zerocount + \letitemgroupparameter\c!factor {\zerocount}% + \setitemgroupparameter\c!packcriterium{\zerocount}% + \letitemgroupparameter\c!inner \empty + \letitemgroupparameter\c!beforehead \empty + \letitemgroupparameter\c!afterhead \blank + \letitemgroupparameter\c!before \blank + \letitemgroupparameter\c!inbetween \blank + \letitemgroupparameter\c!after \blank} \def\strc_itemgroups_initialize_local {\setfalse\c_strc_itemgroups_inline @@ -882,6 +873,12 @@ \fi \fi % + \ifconditional\c_strc_itemgroups_pack + \ifnum\currentnofitems>\itemgroupparameter\c!packcriterium\relax\else + \setfalse\c_strc_itemgroups_pack + \fi + \fi + % \ifnum\c_strc_itemgroups_nesting=\plusone % NIEUW \doadaptleftskip {\itemgroupparameter\c!margin}% \doadaptleftskip {\itemgroupparameter\c!leftmargin}% @@ -1835,6 +1832,7 @@ \c!lefttext=(, \c!righttext=), \c!start=1, + \c!packcriterium=\zerocount, \c!criterium=\v!all, % permits 0 and negative numbers %\c!option=, \c!textdistance=\v!space, % none big medium small <dimension> diff --git a/tex/context/base/mkiv/strc-ref.lua b/tex/context/base/mkiv/strc-ref.lua index da2d50120..c85b3338a 100644 --- a/tex/context/base/mkiv/strc-ref.lua +++ b/tex/context/base/mkiv/strc-ref.lua @@ -427,7 +427,7 @@ end local function synchronizepage(reference) -- non public helper reference.realpage = texgetcount("realpageno") - if jobpositions.used then + if jobpositions.used() then reference.x, reference.y = getpos() end end diff --git a/tex/context/base/mkiv/strc-ref.mkvi b/tex/context/base/mkiv/strc-ref.mkvi index 9fa04c9a6..e774117fa 100644 --- a/tex/context/base/mkiv/strc-ref.mkvi +++ b/tex/context/base/mkiv/strc-ref.mkvi @@ -1065,6 +1065,21 @@ \def\referencecolumnnumber {\clf_referencecolumn} +% So we need : instead of \ but that's only lmtx: +% +% \def\referencecolumnnumber +% {\numexpr\dimexpr\clf_referenceposx-\cutspace\relax:\dimexpr\makeupwidth/\nofcolumns\relax+\plusone\relax} +% +% Tacos patch of the older one, kept here as illustration +% +% \def\referencecolumnnumber +% {\numexpr +% \dimexpr\clf_referenceposx-\cutspace-\makeupwidth/(2*\nofcolumns)\relax +% /\dimexpr \makeupwidth/ \nofcolumns \relax +% +\plusone +% \relax} + + \unexpanded\def\tracedpagestate {{\blue\tttf(\ifcase\referencepagedetail unknown\or same\or previous\or next\or above\or below\else unknown\fi)}} diff --git a/tex/context/base/mkiv/trac-deb.lua b/tex/context/base/mkiv/trac-deb.lua index 72294e587..a6174034c 100644 --- a/tex/context/base/mkiv/trac-deb.lua +++ b/tex/context/base/mkiv/trac-deb.lua @@ -6,104 +6,27 @@ if not modules then modules = { } end modules ['trac-deb'] = { license = "see context related readme files" } -local lpeg, status = lpeg, status +-- This is an old mechanism, a result of some experiments in the early days of +-- luatex and mkiv, but still nice anyway. -local lpegmatch = lpeg.match +local status = status + +local tonumber, tostring, type = tonumber, tostring, type local format, concat, match, find, gsub = string.format, table.concat, string.match, string.find, string.gsub -local tonumber, tostring = tonumber, tostring -- maybe tracers -> tracers.tex (and tracers.lua for current debugger) ------ report_tex = logs.reporter("tex error") ------ report_lua = logs.reporter("lua error") -local report_nl = logs.newline -local report_str = logs.writer - -tracers = tracers or { } -local tracers = tracers - -tracers.lists = { } -local lists = tracers.lists - -tracers.strings = { } -local strings = tracers.strings - -local texgetdimen = tex.getdimen -local texgettoks = tex.gettoks -local texgetcount = tex.getcount -local texgethelp = tex.gethelptext or function() end -local fatalerror = tex.fatalerror - -local implement = interfaces.implement - -strings.undefined = "undefined" - -lists.scratch = { - 0, 2, 4, 6, 8 -} - -lists.internals = { - 'p:hsize', 'p:parindent', 'p:leftskip','p:rightskip', - 'p:vsize', 'p:parskip', 'p:baselineskip', 'p:lineskip', 'p:topskip' -} - -lists.context = { - 'd:lineheight', - 'c:realpageno', 'c:userpageno', 'c:pageno', 'c:subpageno' -} - -local types = { - ['d'] = tracers.dimen, - ['c'] = tracers.count, - ['t'] = tracers.toks, - ['p'] = tracers.primitive -} - -local splitboth = lpeg.splitat(":") -local splittype = lpeg.firstofsplit(":") -local splitname = lpeg.secondofsplit(":") - -function tracers.type(csname) - return lpegmatch(splittype,csname) -end - -function tracers.name(csname) - return lpegmatch(splitname,csname) or csname -end - -function tracers.cs(csname) - local tag, name = lpegmatch(splitboth,csname) - if name and types[tag] then - return types[tag](name) - else - return tracers.primitive(csname) - end -end - -function tracers.dimen(name) - local d = texgetdimen(name) - return d and number.topoints(d) or strings.undefined -end - -function tracers.count(name) - return texgetcount(name) or strings.undefined -end - -function tracers.toks(name,limit) - local t = texgettoks(name) - return t and string.limit(t,tonumber(limit) or 40) or strings.undefined -end - -function tracers.primitive(name) - return tex[name] or strings.undefined -end +----- report_tex = logs.reporter("tex error") +----- report_lua = logs.reporter("lua error") +local report_nl = logs.newline +local report_str = logs.writer -function tracers.knownlist(name) - local l = lists[name] - return l and #l > 0 -end +tracers = tracers or { } +local tracers = tracers local savedluaerror = nil +local usescitelexer = nil +local quitonerror = true local function errorreporter(luaerror) local category = luaerror and "lua error" or "tex error" @@ -123,41 +46,46 @@ function tracers.showlines(filename,linenumber,offset,luaerrorline) end end end - local lines = data and string.splitlines(data) - if lines and #lines > 0 then - if luaerrorline and luaerrorline > 0 then - -- lua error: linenumber points to last line - local start = "\\startluacode" - local stop = "\\stopluacode" - local n = linenumber - for i=n,1,-1 do - local line = lines[i] - if not line then - break - elseif find(line,start) then - n = i + luaerrorline - 1 - if n <= linenumber then - linenumber = n + local scite = usescitelexer and require("util-sci") + if scite then + return utilities.scite.tohtml(data,"tex",linenumber or true,false) + else + local lines = data and string.splitlines(data) + if lines and #lines > 0 then + if luaerrorline and luaerrorline > 0 then + -- lua error: linenumber points to last line + local start = "\\startluacode" + local stop = "\\stopluacode" + local n = linenumber + for i=n,1,-1 do + local line = lines[i] + if not line then + break + elseif find(line,start) then + n = i + luaerrorline - 1 + if n <= linenumber then + linenumber = n + end + break end - break end end - end - offset = tonumber(offset) or 10 - linenumber = tonumber(linenumber) or 10 - local start = math.max(linenumber - offset,1) - local stop = math.min(linenumber + offset,#lines) - if stop > #lines then - return "<linenumber past end of file>" - else - local result, fmt = { }, "%" .. #tostring(stop) .. "d %s %s" - for n=start,stop do - result[#result+1] = format(fmt,n,n == linenumber and ">>" or " ",lines[n]) + offset = tonumber(offset) or 10 + linenumber = tonumber(linenumber) or 10 + local start = math.max(linenumber - offset,1) + local stop = math.min(linenumber + offset,#lines) + if stop > #lines then + return "<linenumber past end of file>" + else + local result, fmt = { }, "%" .. #tostring(stop) .. "d %s %s" + for n=start,stop do + result[#result+1] = format(fmt,n,n == linenumber and ">>" or " ",lines[n]) + end + return concat(result,"\n") end - return concat(result,"\n") + else + return "<empty file>" end - else - return "<empty file>" end end @@ -180,8 +108,6 @@ local function processerror(offset) -- print("[[ last location : " .. tostring(status.lastwarninglocation or "<unset>") .. " ]]") -- print("[[ last context : " .. tostring(status.lasterrorcontext or "<unset>") .. " ]]") - local inputstack = resolvers.inputstack --- local filename = inputstack[#inputstack-1] or status.filename -- weird, why -1, something is wrong local filename = status.filename local linenumber = tonumber(status.linenumber) or 0 local lastcontext = status.lasterrorcontext @@ -200,13 +126,12 @@ local function processerror(offset) lastluaerror = lastluaerror, -- can be the same as lasttexerror luaerrorline = luaerrorline, lastcontext = lastcontext, - lasttexhelp = texgethelp(), + lasttexhelp = tex.gethelptext and tex.gethelptext() or nil, } end -- so one can overload the printer if (really) needed - if fatalerror then callback.register("terminal_input",function(what) if what == "*" then @@ -219,13 +144,15 @@ else -- tex.print("\\nonstopmode") end -local quitonerror = true - directives.register("system.quitonerror",function(v) quitonerror = toboolean(v) -- tex.print("\\errorstopmode") end) +directives.register("system.usescitelexer",function(v) + usescitelexer = toboolean(v) +end) + local busy = false function tracers.printerror(specification) @@ -294,16 +221,10 @@ end luatex.wrapup(function() os.remove(file.addsuffix(tex.jobname .. "-error","log")) end) local function processwarning(offset) - -- local inputstack = resolvers.inputstack - -- local filename = inputstack[#inputstack] or status.filename - -- local linenumber = tonumber(status.linenumber) or 0 local lastwarning = status.lastwarningstring or "?" local lastlocation = status.lastwarningtag or "?" resetmessages() tracers.printwarning { - -- filename = filename, - -- linenumber = linenumber, - -- offset = tonumber(offset) or 10, lastwarning = lastwarning , lastlocation = lastlocation, } @@ -335,20 +256,27 @@ lmx = lmx or { } lmx.htmfile = function(name) return environment.jobname .. "-status.html" end lmx.lmxfile = function(name) return resolvers.findfile(name,'tex') end +local function reportback(lmxname,default,variables) + if lmxname == false then + return variables + else + local name = lmx.show(type(lmxname) == "string" and lmxname or default,variables) + if name then + logs.report("context report","file: %s",name) + end + end +end + function lmx.showdebuginfo(lmxname) local variables = { ['title'] = 'ConTeXt Debug Information', ['color-background-one'] = lmx.get('color-background-green'), ['color-background-two'] = lmx.get('color-background-blue'), } - if lmxname == false then - return variables - else - lmx.show(lmxname or 'context-debug.lmx',variables) - end + reportback(lmxname,"context-debug.lmx",variables) end -function lmx.showerror(lmxname) +local function showerror(lmxname) local filename, linenumber, errorcontext = status.filename, tonumber(status.linenumber) or 0, "" if not filename then filename, errorcontext = 'unknown', 'error in filename' @@ -366,17 +294,18 @@ function lmx.showerror(lmxname) ['filename'] = filename, ['errorcontext'] = errorcontext, } - if lmxname == false then - return variables - else - lmx.show(lmxname or 'context-error.lmx',variables) - end + reportback(lmxname,"context-error.lmx",variables) luatex.abort() end -function lmx.overloaderror() - callback.register('show_error_hook', function() lmx.showerror() end) -- prevents arguments being passed - callback.register('show_lua_error_hook', function() lmx.showerror() end) -- prevents arguments being passed +lmx.showerror = showerror + +function lmx.overloaderror(v) + if v == "scite" then + usescitelexer = true + end + callback.register('show_error_hook', function() showerror() end) -- prevents arguments being passed + callback.register('show_lua_error_hook', function() showerror() end) -- prevents arguments being passed end directives.register("system.showerror", lmx.overloaderror) @@ -416,6 +345,8 @@ directives.register("system.showerror", lmx.overloaderror) -- end -- end) +local implement = interfaces.implement + implement { name = "showtrackers", actions = trackers.show } implement { name = "enabletrackers", actions = trackers.enable, arguments = "string" } implement { name = "disabletrackers", actions = trackers.disable, arguments = "string" } diff --git a/tex/context/base/mkiv/trac-log.lua b/tex/context/base/mkiv/trac-log.lua index be6fff488..1469441b3 100644 --- a/tex/context/base/mkiv/trac-log.lua +++ b/tex/context/base/mkiv/trac-log.lua @@ -845,38 +845,6 @@ local nesting = 0 local verbose = false local hasscheme = url.hasscheme -function logs.show_open(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- nesting = nesting + 1 - -- report_files("level %s, opening %s",nesting,name) - -- else - -- write(formatters["(%s"](name)) -- tex adds a space - -- end - -- end -end - -function logs.show_close(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, closing %s",nesting,name) - -- nesting = nesting - 1 - -- else - -- write(")") -- tex adds a space - -- end - -- end -end - -function logs.show_load(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, loading %s",nesting+1,name) - -- else - -- write(formatters["(%s)"](name)) - -- end - -- end -end - -- there may be scripts out there using this: local simple = logs.reporter("comment") diff --git a/tex/context/base/mkiv/trac-set.lua b/tex/context/base/mkiv/trac-set.lua index d43fd6d0d..960c4ef84 100644 --- a/tex/context/base/mkiv/trac-set.lua +++ b/tex/context/base/mkiv/trac-set.lua @@ -9,8 +9,9 @@ if not modules then modules = { } end modules ['trac-set'] = { -- might become u -- maybe this should be util-set.lua local type, next, tostring, tonumber = type, next, tostring, tonumber +local print = print local concat, sortedhash = table.concat, table.sortedhash -local format, find, lower, gsub, topattern = string.format, string.find, string.lower, string.gsub, string.topattern +local formatters, find, lower, gsub, topattern = string.formatters, string.find, string.lower, string.gsub, string.topattern local is_boolean = string.is_boolean local settings_to_hash = utilities.parsers.settings_to_hash local allocate = utilities.storage.allocate @@ -254,8 +255,8 @@ end local enable, disable, register, list, show = setters.enable, setters.disable, setters.register, setters.list, setters.show -function setters.report(setter,...) - print(format("%-15s : %s\n",setter.name,format(...))) +function setters.report(setter,fmt,...) + print(formatters["%-15s : %s\n"](setter.name,formatters[fmt](...))) end local function default(setter,name) diff --git a/tex/context/base/mkiv/trac-vis.lua b/tex/context/base/mkiv/trac-vis.lua index bbc18df7f..48fff2e1a 100644 --- a/tex/context/base/mkiv/trac-vis.lua +++ b/tex/context/base/mkiv/trac-vis.lua @@ -889,6 +889,9 @@ local ruledglue do [gluecodes.spaceskip] = "SP", [gluecodes.xspaceskip] = "XS", [gluecodes.parfillskip] = "PF", + [gluecodes.indentskip] = "IN", + [gluecodes.lefthangskip] = "LH", + [gluecodes.righthangskip] = "RH", [gluecodes.thinmuskip] = "MS", [gluecodes.medmuskip] = "MM", [gluecodes.thickmuskip] = "ML", diff --git a/tex/context/base/mkiv/typo-chr.lua b/tex/context/base/mkiv/typo-chr.lua index dd3071e4f..cabfc4ab1 100644 --- a/tex/context/base/mkiv/typo-chr.lua +++ b/tex/context/base/mkiv/typo-chr.lua @@ -99,8 +99,9 @@ local wordboundary_code = boundarycodes.word local texgetnest = tex.getnest -- to be used local texsetcount = tex.setcount -local flush_node = node.flush_node -local flush_list = node.flush_list +local flush_node = nodes.flush_node +local flush_list = nodes.flush_list +----- start_of_par = nodes.start_of_par local settexattribute = tex.setattribute local punctuation = characters.is_punctuation @@ -175,7 +176,7 @@ local function pickup(head,tail,str) while true do local prev = first.prev if prev and prev[a_marked] == attr then - if prev.id == localpar_code then -- and prev.subtype == 0 + if prev.id == localpar_code then -- and start_of_par(prev) break else first = prev diff --git a/tex/context/base/mkiv/typo-dha.lua b/tex/context/base/mkiv/typo-dha.lua index c0bbdc396..9a6e39f42 100644 --- a/tex/context/base/mkiv/typo-dha.lua +++ b/tex/context/base/mkiv/typo-dha.lua @@ -65,6 +65,7 @@ local insert_node_before = nuts.insert_before local insert_node_after = nuts.insert_after local remove_node = nuts.remove local end_of_math = nuts.end_of_math +local start_of_par = nuts.start_of_par local nodepool = nuts.pool @@ -331,7 +332,7 @@ local function process(start) end textdir = autodir setprop(current,"direction",true) - elseif id == localpar_code and getsubtype(current) == 0 then + elseif id == localpar_code and start_of_par(current) then local direction = getdirection(current) if direction == righttoleft_code then autodir = -1 diff --git a/tex/context/base/mkiv/typo-drp.lua b/tex/context/base/mkiv/typo-drp.lua index 2f51e5dc0..50eb765f2 100644 --- a/tex/context/base/mkiv/typo-drp.lua +++ b/tex/context/base/mkiv/typo-drp.lua @@ -62,6 +62,7 @@ local new_kern = nodepool.kern local insert_before = nuts.insert_before local insert_after = nuts.insert_after local remove_node = nuts.remove +local start_of_par = nuts.start_of_par local nextnode = nuts.traversers.node local nextglyph = nuts.traversers.glyph @@ -333,7 +334,7 @@ end -- we can count ... when all done, we can disable ... function initials.handler(head) - if getid(head) == localpar_code then + if getid(head) == localpar_code and start_of_par(head) then local settings = getprop(head,a_initial) if settings then disableaction("processors","typesetters.initials.handler") diff --git a/tex/context/base/mkiv/typo-dua.lua b/tex/context/base/mkiv/typo-dua.lua index 3db5f510a..0cec0e08e 100644 --- a/tex/context/base/mkiv/typo-dua.lua +++ b/tex/context/base/mkiv/typo-dua.lua @@ -85,6 +85,7 @@ local setdirection = nuts.setdirection local remove_node = nuts.remove local insert_node_after = nuts.insert_after local insert_node_before = nuts.insert_before +local start_of_par = nuts.start_of_par local nodepool = nuts.pool local new_direction = nodepool.direction @@ -336,7 +337,7 @@ local function get_baselevel(head,list,size,direction) -- This is an adapted version: if direction == lefttoright_code or direction == righttoleft_code then return direction, true - elseif getid(head) == localpar_code and getsubtype(head) == 0 then + elseif getid(head) == localpar_code and start_of_par(head) then direction = getdirection(head) if direction == lefttoright_code or direction == righttoleft_code then return direction, true @@ -774,7 +775,7 @@ local function apply_to_list(list,size,head,pardir) enddir = false end elseif begindir then - if id == localpar_code and getsubtype(current) == 0 then + if id == localpar_code and start_of_par(current) then -- localpar should always be the 1st node local d = new_direction(begindir) -- setprop(d,"directions",true) @@ -816,7 +817,7 @@ local function apply_to_list(list,size,head,pardir) end local function process(head,direction,only_one,where) - -- This is an adapted version: + -- This is an adapted version: local list, size = build_list(head) local baselevel, dirfound = get_baselevel(head,list,size,direction) if not dirfound and trace_details then diff --git a/tex/context/base/mkiv/typo-dub.lua b/tex/context/base/mkiv/typo-dub.lua index f27e40476..f1bd38fcc 100644 --- a/tex/context/base/mkiv/typo-dub.lua +++ b/tex/context/base/mkiv/typo-dub.lua @@ -73,6 +73,7 @@ local setattrlist = nuts.setattrlist local remove_node = nuts.remove local insert_node_after = nuts.insert_after local insert_node_before = nuts.insert_before +local start_of_par = nuts.start_of_par local nodepool = nuts.pool local new_direction = nodepool.direction @@ -397,7 +398,7 @@ end local function get_baselevel(head,list,size,direction) if direction == lefttoright_code or direction == righttoleft_code then return direction, true - elseif getid(head) == localpar_code and getsubtype(head) == 0 then + elseif getid(head) == localpar_code and start_of_par(head) then direction = getdirection(head) if direction == lefttoright_code or direction == righttoleft_code then return direction, true @@ -907,7 +908,7 @@ local function apply_to_list(list,size,head,pardir) enddir = false end elseif begindir then - if id == localpar_code and getsubtype(current) == 0 then + if id == localpar_code and start_of_par(current) then -- localpar should always be the 1st node local d = new_direction(begindir) -- setprop(d,"directions",true) diff --git a/tex/context/base/mkiv/typo-duc.lua b/tex/context/base/mkiv/typo-duc.lua index f91fc4d64..d4c56e300 100644 --- a/tex/context/base/mkiv/typo-duc.lua +++ b/tex/context/base/mkiv/typo-duc.lua @@ -77,6 +77,7 @@ local properties = nodes.properties.data local remove_node = nuts.remove local insert_node_after = nuts.insert_after local insert_node_before = nuts.insert_before +local start_of_par = nuts.start_of_par local nodepool = nuts.pool local new_direction = nodepool.direction @@ -412,7 +413,7 @@ end local function get_baselevel(head,list,size,direction) if direction == lefttoright_code or direction == righttoleft_code then return direction, true - elseif getid(head) == localpar_code and getsubtype(head) == 0 then + elseif getid(head) == localpar_code and start_of_par(head) then direction = getdirection(head) if direction == lefttoright_code or direction == righttoleft_code then return direction, true @@ -954,7 +955,7 @@ local function apply_to_list(list,size,head,pardir) enddir = false end elseif begindir then - if id == localpar_code and getsubtype(current) == 0 then + if id == localpar_code and start_of_par(current) then -- localpar should always be the 1st node head, current = insert_node_after(head,current,new_direction(begindir)) begindir = nil diff --git a/tex/context/base/mkiv/typo-fln.lua b/tex/context/base/mkiv/typo-fln.lua index 3124e80c7..6cca92480 100644 --- a/tex/context/base/mkiv/typo-fln.lua +++ b/tex/context/base/mkiv/typo-fln.lua @@ -71,6 +71,7 @@ local insert_node_after = nuts.insert_after local remove_node = nuts.remove local getdimensions = nuts.dimensions local hpack_node_list = nuts.hpack +local start_of_par = nuts.start_of_par local nodepool = nuts.pool local newpenalty = nodepool.penalty @@ -369,7 +370,7 @@ end actions[v_default] = actions[v_line] function firstlines.handler(head) - if getid(head) == localpar_code then + if getid(head) == localpar_code and start_of_par(head) then local settings = getprop(head,a_firstline) if settings then disableaction("processors","typesetters.firstlines.handler") diff --git a/tex/context/base/mkiv/typo-lin.lua b/tex/context/base/mkiv/typo-lin.lua index 758d0d502..15e543c52 100644 --- a/tex/context/base/mkiv/typo-lin.lua +++ b/tex/context/base/mkiv/typo-lin.lua @@ -129,6 +129,8 @@ local noflines = 0 -- This is the third version, a mix between immediate (prestice lines) and delayed -- as we don't want anchors that are not used. +-- I will make a better variant once lmtx is stable i.e. less clutter. + local function finalize(prop,key) -- delayed calculations local line = prop.line local hsize = prop.hsize diff --git a/tex/context/base/mkiv/util-sci.lua b/tex/context/base/mkiv/util-sci.lua index a3fcbc928..778991ea6 100644 --- a/tex/context/base/mkiv/util-sci.lua +++ b/tex/context/base/mkiv/util-sci.lua @@ -18,10 +18,10 @@ utilities.scite = scite local report = logs.reporter("scite") do - local lexerroot = "c:/data/system/scite/wscite/context/lexers" - if not lexerroot then - lexerroot = file.dirname(resolvers.find_file("scite-context-lexer.lua")) - end +-- local lexerroot = "c:/data/system/scite/wscite/context/lexers" +-- if not lexerroot then + lexerroot = file.dirname(resolvers.findfile("scite-context-lexer.lua")) +-- end if lfs.isdir(lexerroot) then package.extraluapath(lexerroot) package.extraluapath(lexerroot.."/themes") @@ -84,13 +84,15 @@ scite.loadedlexers = loadedlexers scite.knownlexers = knownlexers scite.loadscitelexer = loadscitelexer -local f_fore_bold = formatters['.%s { display: inline ; font-weight: bold ; color: #%02X%02X%02X ; }'] -local f_fore_none = formatters['.%s { display: inline ; font-weight: normal ; color: #%02X%02X%02X ; }'] -local f_none_bold = formatters['.%s { display: inline ; font-weight: bold ; }'] -local f_none_none = formatters['.%s { display: inline ; font-weight: normal ; }'] +local f_fore_bold = formatters['.%s { display:inline; font-weight:bold; color:#%02X%02X%02X; }'] +local f_fore_none = formatters['.%s { display:inline; font-weight:normal; color:#%02X%02X%02X; }'] +local f_none_bold = formatters['.%s { display:inline; font-weight:bold; }'] +local f_none_none = formatters['.%s { display:inline; font-weight:normal; }'] local f_div_class = formatters['<div class="%s">%s</div>'] local f_linenumber = formatters['<div class="linenumber">%s</div>\n'] -local f_div_number = formatters['.linenumber { display: inline-block ; font-weight: normal ; width: %sem ; margin-right: 2em ; padding-right: .25em ; text-align: right ; background-color: #C7C7C7 ; }'] +local f_lineerror = formatters['<div class="linenumber lineerror">%s</div>\n'] +local f_div_number = formatters['.linenumber { display:inline-block; font-weight:normal; width:%sem; margin-right:2em; padding-right:.25em; text-align:right; color:#000000; }'] -- background-color:#C7C7C7; +local s_div_error = '.lineerror { font-weight:bold; color:#FFFFFF; background-color:#000000; }' local replacer_regular = lpeg.replacer { ["<"] = "<", @@ -99,6 +101,7 @@ local replacer_regular = lpeg.replacer { } local linenumber = 0 +local lineerror = 0 local linenumbers = { } local replacer_numbered = lpeg.replacer { @@ -107,7 +110,7 @@ local replacer_numbered = lpeg.replacer { ["&"] = "&", [lpeg.patterns.newline] = function() linenumber = linenumber + 1 - linenumbers[linenumber] = f_linenumber(linenumber) + linenumbers[linenumber] = (linenumber == lineerror and f_lineerror or f_linenumber)(linenumber) return "\n" end, } @@ -153,6 +156,7 @@ local function exportstyled(lexer,text,numbered) local b = 0 linenumber = 0 linenumbers = { } + lineerror = tonumber(numbered) or 0 local replacer = numbered and replacer_numbered or replacer_regular local n = #result for i=1,n,2 do @@ -169,12 +173,11 @@ local function exportstyled(lexer,text,numbered) end start = position end - buffer = concat(buffer) - return buffer, concat(linenumbers) + return concat(buffer), concat(linenumbers) end local function exportcsslinenumber() - return f_div_number(#tostring(linenumber)/2+1) + return f_div_number(#tostring(linenumber)/2+1) .. "\n" .. s_div_error end local htmlfile = utilities.templates.replacer([[ @@ -188,7 +191,7 @@ local htmlfile = utilities.templates.replacer([[ %numberstyles% --></style> <body> - <table style="padding:0; margin:0;"> + <table style="padding:0; margin:0; background-color:#FFFFFF;"> <tr> <td><pre>%linenumbers%</pre></td> <td><pre>%lexedcontent%</pre></td> @@ -198,15 +201,30 @@ local htmlfile = utilities.templates.replacer([[ </html> ]]) +local htmlblob = utilities.templates.replacer([[ +<style type="text/css"><!-- +%lexingstyles% +%numberstyles% +--></style> +<table style="padding:0; margin:0; background-color:#FFFFFF;"> + <tr> + <td><pre>%linenumbers%</pre></td> + <td><pre>%lexedcontent%</pre></td> + </tr> +</table> +]]) + function scite.tohtml(data,lexname,numbered,title) local source, lines = exportstyled(loadedlexers[lexname],data or "",numbered) - return htmlfile { - lexedcontent = source, -- before numberstyles - lexingstyles = exportcsslexing(), - numberstyles = exportcsslinenumber(), - title = title or "context source file", - linenumbers = lines, - } + if source then + return (title == false and htmlblob or htmlfile) { + lexedcontent = source, -- before numberstyles + lexingstyles = exportcsslexing(), + numberstyles = exportcsslinenumber(), + title = title or "context source file", + linenumbers = lines, + } + end end local function maketargetname(name) diff --git a/tex/context/fonts/mkiv/type-imp-opendyslexic.mkiv b/tex/context/fonts/mkiv/type-imp-opendyslexic.mkiv index 1f366b8eb..bdae0e261 100644 --- a/tex/context/fonts/mkiv/type-imp-opendyslexic.mkiv +++ b/tex/context/fonts/mkiv/type-imp-opendyslexic.mkiv @@ -35,6 +35,8 @@ % \input tufte % \stoptext +% \setupinterlinespace[line=1.535\bodyfontsize] % older fonts have a weird exheight + \starttypescriptcollection[opendyslexic] \definefontfeature diff --git a/tex/context/interface/mkii/keys-cs.xml b/tex/context/interface/mkii/keys-cs.xml index 5f8f2ff5e..3014b72d1 100644 --- a/tex/context/interface/mkii/keys-cs.xml +++ b/tex/context/interface/mkii/keys-cs.xml @@ -235,6 +235,7 @@ <cd:variable name='handwritten' value='rukopisne'/> <cd:variable name='hang' value='zaveseni'/> <cd:variable name='hanging' value='visici'/> + <cd:variable name='hangingboth' value='hangingboth'/> <cd:variable name='head' value='hlavicka'/> <cd:variable name='header' value='zahlavi'/> <cd:variable name='headintext' value='headintext'/> @@ -1050,6 +1051,7 @@ <cd:constant name='overprint' value='overprint'/> <cd:constant name='ownerpassword' value='ownerpassword'/> <cd:constant name='ownnumber' value='vlastnicislo'/> + <cd:constant name='packcriterium' value='packcriterium'/> <cd:constant name='page' value='stranka'/> <cd:constant name='pageboundaries' value='hranicestranky'/> <cd:constant name='pagecolor' value='barvastranky'/> @@ -1059,6 +1061,7 @@ <cd:constant name='pageconversion' value='pageconversion'/> <cd:constant name='pageconversionset' value='pageconversionset'/> <cd:constant name='pageleft' value='pageleft'/> + <cd:constant name='pagemethod' value='pagemethod'/> <cd:constant name='pagenumber' value='cislostranky'/> <cd:constant name='pageprefix' value='pageprefix'/> <cd:constant name='pageprefixconnector' value='pageprefixconnector'/> diff --git a/tex/context/interface/mkii/keys-de.xml b/tex/context/interface/mkii/keys-de.xml index e8e797c4e..42d642544 100644 --- a/tex/context/interface/mkii/keys-de.xml +++ b/tex/context/interface/mkii/keys-de.xml @@ -235,6 +235,7 @@ <cd:variable name='handwritten' value='handschrift'/> <cd:variable name='hang' value='haengend'/> <cd:variable name='hanging' value='haengend'/> + <cd:variable name='hangingboth' value='hangingboth'/> <cd:variable name='head' value='kopf'/> <cd:variable name='header' value='kopfzeile'/> <cd:variable name='headintext' value='headintext'/> @@ -1050,6 +1051,7 @@ <cd:constant name='overprint' value='overprint'/> <cd:constant name='ownerpassword' value='ownerpassword'/> <cd:constant name='ownnumber' value='eigenenummer'/> + <cd:constant name='packcriterium' value='packcriterium'/> <cd:constant name='page' value='seite'/> <cd:constant name='pageboundaries' value='seitenbegrenzung'/> <cd:constant name='pagecolor' value='seitenfarbe'/> diff --git a/tex/context/interface/mkii/keys-nl.xml b/tex/context/interface/mkii/keys-nl.xml index cb6ebc68d..344a874f0 100644 --- a/tex/context/interface/mkii/keys-nl.xml +++ b/tex/context/interface/mkii/keys-nl.xml @@ -235,6 +235,7 @@ <cd:variable name='handwritten' value='handschrift'/> <cd:variable name='hang' value='hang'/> <cd:variable name='hanging' value='hangend'/> + <cd:variable name='hangingboth' value='beidehangend'/> <cd:variable name='head' value='kop'/> <cd:variable name='header' value='hoofd'/> <cd:variable name='headintext' value='kopintekst'/> @@ -1050,6 +1051,7 @@ <cd:constant name='overprint' value='overprint'/> <cd:constant name='ownerpassword' value='ownerpassword'/> <cd:constant name='ownnumber' value='eigennummer'/> + <cd:constant name='packcriterium' value='opelkaarcriterium'/> <cd:constant name='page' value='pagina'/> <cd:constant name='pageboundaries' value='paginaovergangen'/> <cd:constant name='pagecolor' value='paginakleur'/> @@ -1059,6 +1061,7 @@ <cd:constant name='pageconversion' value='pageconversion'/> <cd:constant name='pageconversionset' value='pageconversionset'/> <cd:constant name='pageleft' value='pageleft'/> + <cd:constant name='pagemethod' value='paginamethode'/> <cd:constant name='pagenumber' value='paginanummer'/> <cd:constant name='pageprefix' value='pageprefix'/> <cd:constant name='pageprefixconnector' value='pageprefixconnector'/> diff --git a/tex/context/interface/mkii/keys-ro.xml b/tex/context/interface/mkii/keys-ro.xml index ca739d2b2..0bde14e86 100644 --- a/tex/context/interface/mkii/keys-ro.xml +++ b/tex/context/interface/mkii/keys-ro.xml @@ -235,6 +235,7 @@ <cd:variable name='handwritten' value='scrismanual'/> <cd:variable name='hang' value='suspenda'/> <cd:variable name='hanging' value='suspendat'/> + <cd:variable name='hangingboth' value='hangingboth'/> <cd:variable name='head' value='antet'/> <cd:variable name='header' value='antet'/> <cd:variable name='headintext' value='headintext'/> @@ -1050,6 +1051,7 @@ <cd:constant name='overprint' value='overprint'/> <cd:constant name='ownerpassword' value='ownerpassword'/> <cd:constant name='ownnumber' value='numarpropriu'/> + <cd:constant name='packcriterium' value='packcriterium'/> <cd:constant name='page' value='pagina'/> <cd:constant name='pageboundaries' value='marginipagina'/> <cd:constant name='pagecolor' value='culoarepagina'/> @@ -1059,6 +1061,7 @@ <cd:constant name='pageconversion' value='pageconversion'/> <cd:constant name='pageconversionset' value='pageconversionset'/> <cd:constant name='pageleft' value='pageleft'/> + <cd:constant name='pagemethod' value='pagemethod'/> <cd:constant name='pagenumber' value='numarpagina'/> <cd:constant name='pageprefix' value='pageprefix'/> <cd:constant name='pageprefixconnector' value='pageprefixconnector'/> diff --git a/tex/context/interface/mkiv/context-en.xml b/tex/context/interface/mkiv/context-en.xml index 7666fcf11..a652126ab 100644 --- a/tex/context/interface/mkiv/context-en.xml +++ b/tex/context/interface/mkiv/context-en.xml @@ -17847,6 +17847,9 @@ <cd:parameter name="grid"> <cd:inherit name="definegridsnapping"/> </cd:parameter> + <cd:parameter name="packcriterium"> + <cd:constant type="cd:number"/> + </cd:parameter> <cd:inherit name="setupcounter"/> </cd:assignments> </cd:arguments> @@ -18507,6 +18510,9 @@ <cd:parameter name="grid"> <cd:inherit name="definegridsnapping"/> </cd:parameter> + <cd:parameter name="packcriterium"> + <cd:constant type="cd:number"/> + </cd:parameter> <cd:inherit name="setupcounter"/> </cd:assignments> </cd:arguments> @@ -20136,6 +20142,10 @@ <cd:parameter name="artoffset"> <cd:constant type="cd:dimension"/> </cd:parameter> + <cd:parameter name="direction"> + <cd:constant default="yes" type="normal"/> + <cd:constant type="reverse"/> + </cd:parameter> </cd:assignments> </cd:arguments> </cd:command> diff --git a/tex/context/interface/mkiv/i-context.pdf b/tex/context/interface/mkiv/i-context.pdf Binary files differindex 94d34d7b3..d57e3f0e4 100644 --- a/tex/context/interface/mkiv/i-context.pdf +++ b/tex/context/interface/mkiv/i-context.pdf diff --git a/tex/context/interface/mkiv/i-itemgroup.xml b/tex/context/interface/mkiv/i-itemgroup.xml index 810024351..a3271d12f 100644 --- a/tex/context/interface/mkiv/i-itemgroup.xml +++ b/tex/context/interface/mkiv/i-itemgroup.xml @@ -191,6 +191,9 @@ <cd:parameter name="grid"> <cd:inherit name="definegridsnapping"/> </cd:parameter> + <cd:parameter name="packcriterium"> + <cd:constant type="cd:number"/> + </cd:parameter> <cd:inherit name="setupcounter"/> </cd:assignments> </cd:define> diff --git a/tex/context/interface/mkiv/i-layout.xml b/tex/context/interface/mkiv/i-layout.xml index 4370cc4f5..0d256e967 100644 --- a/tex/context/interface/mkiv/i-layout.xml +++ b/tex/context/interface/mkiv/i-layout.xml @@ -208,6 +208,10 @@ <cd:parameter name="artoffset"> <cd:constant type="cd:dimension"/> </cd:parameter> + <cd:parameter name="direction"> + <cd:constant type="normal" default="yes"/> + <cd:constant type="reverse"/> + </cd:parameter> </cd:assignments> </cd:arguments> </cd:command> @@ -382,4 +386,4 @@ </cd:arguments> </cd:command> -</cd:interface>
\ No newline at end of file +</cd:interface> diff --git a/tex/context/interface/mkiv/i-readme.pdf b/tex/context/interface/mkiv/i-readme.pdf Binary files differindex 9d9ba6c34..ed1635f00 100644 --- a/tex/context/interface/mkiv/i-readme.pdf +++ b/tex/context/interface/mkiv/i-readme.pdf diff --git a/tex/generic/context/luatex/luatex-basics-nod.lua b/tex/generic/context/luatex/luatex-basics-nod.lua index 172fcc7f7..be82277bd 100644 --- a/tex/generic/context/luatex/luatex-basics-nod.lua +++ b/tex/generic/context/luatex/luatex-basics-nod.lua @@ -279,3 +279,14 @@ if not nuts.setreplace then end end + +do + + local getsubtype = nuts.getsubtype + + function nuts.start_of_par(n) + local s = getsubtype(n) + return s == 0 or s == 2 -- sorry, hardcoded, won't change anyway + end + +end diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index c409933f5..1a11fc13b 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua -- parent file : c:/data/develop/context/sources/luatex-fonts.lua --- merge date : 11/14/19 17:07:21 +-- merge date : 11/25/19 17:28:15 do -- begin closure to overcome local limits and interference @@ -4702,6 +4702,7 @@ if not modules then modules={} end modules ['data-con']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local setmetatable=setmetatable local format,lower,gsub=string.format,string.lower,string.gsub local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) @@ -4709,16 +4710,21 @@ local trace_storage=false trackers.register("resolvers.storage",function(v) tra containers=containers or {} local containers=containers containers.usecache=true +local getwritablepath=caches.getwritablepath +local getreadablepaths=caches.getreadablepaths +local cacheiswritable=caches.is_writable +local loaddatafromcache=caches.loaddata +local savedataincache=caches.savedata local report_containers=logs.reporter("resolvers","containers") local allocated={} local mt={ __index=function(t,k) if k=="writable" then - local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable=getwritablepath(t.category,t.subcategory) or { "." } t.writable=writable return writable elseif k=="readables" then - local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables=getreadablepaths(t.category,t.subcategory) or { "." } t.readables=readables return readables end @@ -4749,7 +4755,7 @@ function containers.define(category,subcategory,version,enabled) end end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable,name) + return container.enabled and caches and cacheiswritable(container.writable,name) end function containers.is_valid(container,name) if name and name~="" then @@ -4763,7 +4769,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name,container.writable) + stored=loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -4779,17 +4785,20 @@ function containers.read(container,name) end return stored end -function containers.write(container,name,data) +function containers.write(container,name,data,fast) if data then data.cache_version=container.version if container.enabled and caches then - local unique,shared=data.unique,data.shared - data.unique,data.shared=nil,nil - caches.savedata(container.writable,name,data) + local unique=data.unique + local shared=data.shared + data.unique=nil + data.shared=nil + savedataincache(container.writable,name,data,fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique,data.shared=unique,shared + data.unique=unique + data.shared=shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) @@ -5024,6 +5033,13 @@ if not nuts.setreplace then setfield(n,"replace",h) end end +do + local getsubtype=nuts.getsubtype + function nuts.start_of_par(n) + local s=getsubtype(n) + return s==0 or s==2 + end +end end -- closure @@ -26281,6 +26297,7 @@ local find_node_tail=nuts.tail local flush_node_list=nuts.flush_list local flush_node=nuts.flush_node local end_of_math=nuts.end_of_math +local start_of_par=nuts.start_of_par local setmetatable=setmetatable local setmetatableindex=table.setmetatableindex local nextnode=nuts.traversers.node @@ -29251,7 +29268,7 @@ do checkstep(head) end local initialrl=0 - if getid(head)==localpar_code and getsubtype(head)==0 then + if getid(head)==localpar_code and start_of_par(head) then initialrl=pardirstate(head) elseif direction==1 or direction=="TRT" then initialrl=-1 |