%D \module
%D   [       filefile=core-uti,
%D        version=1997.03.31,
%D          title=\CONTEXT\ Core Macros,
%D       subtitle=Utility File Handling,
%D         author=Hans Hagen,
%D           date=\currentdate,
%D      copyright={PRAGMA / Hans Hagen \& Ton Otten}]
%C
%C This module is part of the \CONTEXT\ macro||package and is
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
%C details.

\writestatus{loading}{Context Core Macros / Utility File Handling}

\unprotect

% todo : safe lan etc too
% todo : load all commands at once (tok)
% todo : merge status info patch into tui file (language, encoding, etc),

% Utility-file
%
% De onderstaande macro's ondersteunen het gebruik van de
% zogeheten utility-file. Alle extern onder te brengen
% informatie wordt opgeslagen in de file \jobname.tui, tenzij
% er selectief pagina's worden gezet. In dat geval wordt de
% file \jobname.tmp gebruikt. Informatie wordt ingelezen uit
% de file \jobname.tuo, welke door TeXUtil wordt aangemaakt.

\edef\utilityversion{1998.07.07} % was: 1996.03.15  % status variables
\edef\utilityversion{1998.12.20} % was: 1998.07.07  % index attributes
\edef\utilityversion{2003.07.19} % was: 1998.12.20  % object pages

% Bepaalde commando's worden als string weggeschreven. Deze
% zijn aan het eind van deze file gedefinieerd.

% Om een opbouw van spaties te voorkomen (???) moet ^^M een
% andere betekenis krijgen:
%
% \catcode`\^^M=14 (comment)
%
% read file
%
% \catcode`\^^M=5  (end of line)

\newwrite\uti
\newif\ifutilitydone

\def\@@utilityerrormessage
  {\showmessage\m!systems8\empty
   \globallet\@@utilityerrormessage\relax}

\def\thisisutilityversion#1%
  {\doifelse\utilityversion{#1}%
     {\checksectionseparator}
     {\@@utilityerrormessage\resetutilities\endinput}}

\def\checksectionseparator               % catches backward compatibility conflict
  {\doifnot\sectionseparator:\endinput}  % this dependency may go in a few years

\def\dosplitofffoliopart[#1::#2::#3]{#3}

\def\thisissectionseparator#1%
  {\bgroup
   \globallet\checksectionseparator\relax
   \convertcommand \sectionseparator\to\asciiA
   \convertargument               #1\to\asciiB
   \expanded{\gdef\noexpand\dosplitofffoliopart[####1\sectionseparator
     \sectionseparator####2\sectionseparator\sectionseparator####3]{####3}}%
   \ifx\asciiA\asciiB
     \egroup
   \else
     \egroup
     % todo \@@utilityerrormessage
     \resetutilities
     \endinput
   \fi}

\def\writeutility         {\write\uti}
\def\writeutilitycommand#1{\write\uti{c \string#1}}

% less tokens
%
% \def\immediatewriteutility       {\immediate\writeutility}
% \def\immediatewriteutilitycommand{\immediate\writeutilitycommand}
%
% more flexible (for overloading)

\def\immediatewriteutility         {\immediate\write\uti}
\def\immediatewriteutilitycommand#1{\immediate\write\uti{c \string#1}}

% as in:

\def\cwriteutility#1%
  {\write\uti{\noexpand\checkedutility{\number\nofshipouts}{#1}}}

\def\cwriteutilitycommand#1%
  {\write\uti{\noexpand\checkedutility{\number\nofshipouts}{c \string#1}}}

% \def\cimmediatewriteutility
%   {\immediate\write\uti}
%
% \def\cimmediatewriteutilitycommand#1%
%   {\immediate\write\uti{c \string#1}}

\let\checkedutility\secondoftwoarguments

\def\docheckedutility#1#2{\ifnum#1=\nofshipouts#2\else\letterpercent\fi}

\prependtoks
  \let\checkedutility\docheckedutility
\to \everybeforeshipout

\edef\testbytesequence{\rawcharacter{7}\rawcharacter{27}\rawcharacter{227}}

\def\thisisbytesequence#1%
  {\ifx\testbytesequence\empty\else
     \convertcommand\testbytesequence\to\testbytesequence
     \convertargument#1\to\ascii
     \ifx\testbytesequence\ascii \else
       \writestatus\m!systems{possible problem with 8 bit output}%
     \fi
   \fi
   \global\let\thisisbytesequence\gobbleoneargument}

% Better use marks.

\def\checkutilities
  {\let\writeutility                \cwriteutility
   \let\writeutilitycommand         \cwriteutilitycommand
  %\let\immediatewriteutility       \cimmediatewriteutility
  %\let\immediatewriteutilitycommand\cimmediatewriteutilitycommand
   \let\checkutilities              \relax}

% this trickery is used in s-pre-50

\def\openutilities
  {\immediate\openout\uti\jobname.\f!inputextension
   \immediatewriteutilitycommand{\thisissectionseparator{\sectionseparator}}% for the moment
   \immediatewriteutilitycommand{\thisisutilityversion  {\utilityversion  }}% in this order
   \immediatewriteutilitycommand{\thisisbytesequence    {\testbytesequence}}}

\def\closeutilities
  {\immediate\closeout\uti % niet echt nodig
   \reportutilityproblems
   % should be a message :
   \let\writeutilitycommand         \gobbleoneargument
   \let\writeutility                \gobbleoneargument
   \let\immediatewriteutilitycommand\gobbleoneargument
   \let\immediatewriteutility       \gobbleoneargument}

\def\reopenutilities
  {\immediate\closeout\uti
   \openutilities}

\def\abortutilitygeneration
  {\immediatewriteutilitycommand\utilitygenerationaborted
   \immediatewriteutility{q {quit}}}

\def\utilitygenerationaborted
  {\showmessage\m!systems{21}\empty
   \globallet\utilitygenerationaborted\endinput
   \gdef\reportutilityproblems{\showmessage\m!systems{22}\empty}%
   \endinput}

\def\savecurrentvalue#1#2%
  {\immediatewriteutilitycommand{\initializevariable\string#1{#2}}}

\let\initializevariable\gdef

\def\disableinitializevariables
  {\globallet\initializevariable\gobbletwoarguments}

\let\reportutilityproblems=\relax

\let\utilityresetlist=\empty

% the original
%
% \def\addutilityreset#1%
%   {\addtocommalist{\s!reset#1}\utilityresetlist}
%
% \def\resetutilities%
%   {\processcommacommand[\utilityresetlist]\getvalue}
%
% the more efficient
%
% \def\addutilityreset#1%
%   {\addtocommalist{#1}\utilityresetlist}
%
% \def\doresetutility#1%
%   {\getvalue{\s!reset#1}}
%
% \def\resetutilities%
%   {\processcommacommand[\utilityresetlist]\doresetutility}
%
% the fastest, about two times, but who cares, since this
% can be neglected

\newtoks\utilityresetlist

\def\addutilityreset#1%
  {\@EA\appendtoks\csname\s!reset#1\endcsname\to\utilityresetlist}

\def\resetutilities
  {\the\utilityresetlist}

% #1=type
% #2=file
% #3=melding

% #4=voor
% #5=na

% Er wordt gegroepeerd. Als binnen een lijst (bijvoorbeeld) de
% \leftskip is aangepast, maar nog geen \par is gegeven, dan
% geldt buiten de groep de oude \leftskip. Aan #5 kan dan
% ook \par worden meegegeven om de paragraaf af te sluiten.

\newif\ifdoinpututilities
\newif\ifunprotectutilities   % voor't geval er \v!xxxxxx's zijn

% no longer needed, since texutil is now multi platform
%
% \def\utilitycheckmessage%
%   {\showmessage{\m!systems}{12}{}%
%    \globallet\utilitycheckmessage=\relax}
%
% \def\saveutilityline#1 #2\txen% tricky maar ok, want achter \command
%   {\if     #1c% commands      % in \ascii staat een spatie; #1 kan
%      \write\scratchwrite{#2}% % \par in stringvorm zijn (eof)!
%    \else\if#1s% synoniems
%      \utilitycheckmessage
%    \else\if#1r% registers
%      \utilitycheckmessage
%    \fi\fi\fi}
%
% \def\checkutilityfile%
%   {\doiflocfileelse{\jobname.\f!outputextension}
%      {}
%      {\doiflocfileelse{\jobname.\f!inputextension}
%         {\bgroup
%          \showmessage{\m!systems}{11}{}%
%          \openout\scratchwrite=\jobname.\f!outputextension
%          \openlocin\scratchread{\jobname.\f!inputextension}%
%          \def\doprocessline%
%            {\ifeof\scratchread
%               \def\doprocessline{\closein\scratchread}%
%             \else
%               \read\scratchread to \ascii
%               \convertcommand\ascii\to\ascii
%               \expandafter\saveutilityline\ascii\txen
%             \fi
%             \doprocessline}%
%          \doprocessline
%          \closeout\scratchwrite
%          \egroup}
%         {}}}

\def\currentutilityfilename{\jobname}

\long\def\doutilities#1#2#3#4#5% % introduceren in utility file
  {\restorecatcodes
   \resetutilities
   % more than one utility thing can be handled in one pass,
   % for instance lists, so we process ##1 as list
   \def\douticommand##1{\csname\s!set##1\endcsname}%
   \processcommacommand[#1]\douticommand
   \begingroup
   \def\currentutilityfilename{#2}%
   \notesenabledfalse
   \doinpututilitiestrue
   \global\utilitydonefalse
   \catcode`\\=\@@escape
   \catcode`\{=\@@begingroup
   \catcode`\}=\@@endgroup
   \catcode`\%=\@@comment\relax
   \pushendofline % geeft problemen zodra andere file wordt ingelezen
   \ifunprotectutilities % nog nodig ?
     \unprotect
   \fi
   \ifnum\catcode`\@=\@@active \else
     \catcode`\@=\@@letter % permits expanded commands with \@'s
   \fi
   \ifnum\catcode`\!=\@@active \else
     \catcode`\!=\@@letter % permits multilingual constants
   \fi
   #4%
   \the\everybeforeutilityread
   \readjobfile{#2.\f!outputextension}\donothing\donothing
   \the\everyafterutilityread
   #5%
   \relax
   \ifunprotectutilities
     \protect
   \fi
   \popendofline
   \ifutilitydone\else
     \doifsomething{#3}
       {\showmessage\m!systems9{{#3}}%
        \ifvoorlopig
          \blank
          \type{[\currentmessagetext]}%
          \blank
        \fi}%
   \fi
   \disableinitializevariables
   \endgroup}

% Saving the sort vector:

\def\savesortkeys
  {\startnointerference
   \def\flushsortkey##1##2##3##4%
     {\convertargument{##1}{##2}{##3}{##4}\to\ascii
      \immediatewriteutility{k {\currentlanguage}{\currentencoding}\ascii}}%
   \let\definesortkey\flushsortkey
   \flushsortkeys
   \globallet\savesortkeys\relax
   \stopnointerference}

\prependtoks \savesortkeys \to \everystarttext

% Commando's ten behoeve van two-pass lists. In principe
% kan alles in een keer worden ingelezen. Omdat de macro's
% groeien is de kans groot dat het (main) geheugen door
% (de)allocatie volloopt. Vandaar dat we het toch maar niet
% doen.
%
% \definetwopasslist{\s!xxx}
%
% \gettwopassdata{\s!xxx}
% \getfrompassdata{\s!xxx}{n}       n=index (getal)
% \findtwopassdata{\s!xxx}{tag}     bijvoorbeeld {label:}
% \iftwopassdatafound
% \twopassdata
%
% \twopassentry{\s!xxx}{nr}{data}   nr alleen voor testdoeleinden
%
% also:
%
% \definerawpasslist{\s!xxx}
% \moverawpasslist\s!xxx\to\somemacro

\let\alltwopasslists\empty % with    0,0 -> stepwise commalist
\let\allrawpasslists\empty % without 0,0 -> raw commalist

\newif\iftwopassdatafound

\def\settwopassentries
  {\def\twopassentry##1{\executeifdefined{@@##1\s!pass}\gobbletwoarguments}}

\def\resettwopassentries
  {\let\twopassentry\gobblethreearguments}

\addutilityreset{twopassentries}

\resettwopassentries

% \def\twopassentry#1%
%   {\executeifdefined{@@#1\s!pass}\gobbletwoarguments}

%\def\appendtwopasselement#1#2#3%
%  {%\debuggerinfo{\m!systems}{twopass data #1 - #2 = #3}%
%   \@EA\ifx\csname#1:\s!list\endcsname\empty
%     \setxvalue{#1:\s!list}{#3}%
%   \else
%     \setxvalue{#1:\s!list}{\getvalue{#1:\s!list},#3}%
%   \fi}

\def\appendtwopasselement#1#2#3% can sometimes become a large list
  {%\debuggerinfo{\m!systems}{twopass data #1 - #2 = #3}%
   \expandafter\xdef\csname#1:\s!list\endcsname
     {\@EA\ifx\csname#1:\s!list\endcsname\empty \else
        \csname#1:\s!list\endcsname,\fi#3}}

\def\dodefinetwopasslist#1%
  {\doifundefined{#1:\s!list}
     {%\debuggerinfo\m!systems{defining twopass class #1}%
      \doglobal\addutilityreset{#1\s!pass}%
      \setgvalue{\s!set  #1\s!pass}{\dosettwopasslist  {#1}}%
      \setgvalue{\s!reset#1\s!pass}{\doresettwopasslist{#1}}%
      \getvalue {\s!reset#1\s!pass}}}

\def\dosettwopasslist#1%
  {\letgvalue{\s!set#1\s!pass}\gobbletwoarguments
   \setgvalue{@@#1\s!pass}{\appendtwopasselement{#1}}}

\def\doresettwopasslist#1%
  {\letgvalue{@@#1\s!pass}\gobbletwoarguments}

\def\definetwopasslist#1%
  {\expanded{\dodefinetwopasslist{#1}}%
   \doglobal\addtocommalist{#1}\alltwopasslists}

\def\definerawpasslist#1%
  {\expanded{\dodefinetwopasslist{#1}}%
   \doglobal\addtocommalist{#1}\allrawpasslists}

\def\doloadtwopassdata#1%
  {\doifundefined{#1:\s!list}
     {\startnointerference
      \letgvalueempty{#1:\s!list}%
      \protectlabels
      \doutilities{twopassentries,#1\s!pass}\jobname\empty\relax\relax
      \ifx\twopassdata\empty\else
        \appendtwopasselement{#1}{0}\twopassdata
      \fi
      \stopnointerference}}

\def\loadtwopassdata
  {\ifx\alltwopasslists\empty\else
     \def\twopassdata{0,0}% end condition
     \processcommacommand[\alltwopasslists]\doloadtwopassdata
     \globallet\alltwopassdata\empty
   \fi
   \ifx\allrawpasslists\empty\else
     \let\twopassdata\empty
     \processcommacommand[\allrawpasslists]\doloadtwopassdata
     \globallet\allrawpassdata\empty
   \fi}

\def\moverawpasslist#1#2% erases the old one, like the others do
  {\loadtwopassdata
   \@EA\let\@EA#2\csname#1:\s!list\endcsname
   \@EA\let\csname#1:\s!list\endcsname\empty}

\let\twopassdata=\empty

%\def\dogettwopassdata[#1,#2]#3%
%  {\doifelse{#1}{0} % \ifcase truukje gaat fout
%     {\twopassdatafoundfalse
%      \let\twopassdata\empty}
%     {\twopassdatafoundtrue
%      \setxvalue{#3:\s!list}{#2}%
%      \edef\twopassdata{#1}}}

\def\notwopassdata{0}

% \def\dogettwopassdata[#1,#2]#3%
%   {\edef\twopassdata{#1}%
%    \ifx\twopassdata\notwopassdata
%      \twopassdatafoundfalse
%      \let\twopassdata\empty
%    \else
%      \twopassdatafoundtrue
%      \setxvalue{#3:\s!list}{#2}%
%    \fi}
%
% \def\gettwopassdata#1%
%   {\loadtwopassdata
%   %\edef\!!stringa{\csname#1:\s!list\endcsname}%
%   %\debuggerinfo{\m!systems}{twopass get #1 - \!!stringa}%
%   %\expandafter\dogettwopassdata\expandafter[\!!stringa]{#1}}
%    \expanded{\dogettwopassdata[\csname#1:\s!list\endcsname]}{#1}}
%
% \def\findtwopassdata#1#2%
%   {\loadtwopassdata
%    \expanded{\dofindtwopassdata{#1}{#2}}}
%
% \def\dofindtwopassdata#1#2%
%   {\edef\!!stringa{,\csname#1:\s!list\endcsname}%
%   %\debuggerinfo{\m!systems}{twopass find #2 - \!!stringa}%
%    \def\dodofindtwopassdata[##1,##2#2##3,##4]%
%      {\edef\twopassdata{##3}%
%       \ifx\twopassdata\empty
%         \twopassdatafoundfalse
%       \else
%         \twopassdatafoundtrue
%       \fi}%
%    \@EA\dodofindtwopassdata\@EA[\!!stringa,#2,#2,]}
%
% \def\getfirsttwopassdata#1%
%   {\loadtwopassdata
%    \edef\!!stringa{\getvalue{#1:\s!list}}%
%    \expandafter\dogetfirsttwopassdata\expandafter[\!!stringa]{#1}}
%
% \def\dogetfirsttwopassdata[#1,#2]#3%
%   {\doifelse{#1}{0}
%      {\twopassdatafoundfalse
%       \let\twopassdata\empty}
%      {\twopassdatafoundtrue
%       \edef\twopassdata{#1}}}
%
% \def\getlasttwopassdata#1%
%   {\loadtwopassdata
%    \edef\twopassdata{0}\twopassdatafoundfalse
%    \newcounter\noftwopassitems
%    \def\docommando##1%
%      {\doifnot{##1}{0}
%         {\increment\noftwopassitems
%          \edef\twopassdata{##1}\twopassdatafoundtrue}}%
%    \processcommacommand[\getvalue{#1:\s!list}]\docommando}
%
% \def\getfromtwopassdata#1#2%
%   {\loadtwopassdata
%    \getfromcommacommand[\csname#1:\s!list\endcsname][#2]%
%    \doifelsenothing{\commalistelement}
%      {\twopassdatafoundfalse
%       \let\twopassdata\empty}
%      {\twopassdatafoundtrue
%       \let\twopassdata\commalistelement}}

% todo: store each entry in hash, an load all uti commands at once

\def\dogettwopassdata[#1,#2]#3%
  {\edef\twopassdata{#1}%
   \ifx\twopassdata\notwopassdata
     \twopassdatafoundfalse
     \let\twopassdata\empty
   \else
     \twopassdatafoundtrue
     \setxvalue{#3:\s!list}{#2}%
   \fi}

\def\gettwopassdata#1%
  {\loadtwopassdata
   \@EAEAEA\dogettwopassdata\@EA\@EA\@EA[\csname#1:\s!list\endcsname]{#1}}

\def\findtwopassdata#1#2%
  {\loadtwopassdata
   \expanded{\dofindtwopassdata{#1}{#2}}}

\def\dofindtwopassdata#1#2%
  {\def\dodofindtwopassdata[##1,##2#2##3,##4]{\edef\twopassdata{##3}}%
   \@EAEAEA\dodofindtwopassdata\@EA\@EA\@EA[\@EA\@EA\@EA,\csname#1:\s!list\endcsname,#2,#2,]%
   \ifx\twopassdata\empty
     \twopassdatafoundfalse
   \else
     \twopassdatafoundtrue
   \fi}

\def\getfirsttwopassdata#1%
  {\loadtwopassdata
   \@EAEAEA\dogetfirsttwopassdata\@EA\@EA\@EA[\csname#1:\s!list\endcsname]{#1}}

\def\dogetfirsttwopassdata[#1,#2]#3%
  {\edef\twopassdata{#1}%
   \ifx\twopassdata\notwopassdata
     \twopassdatafoundfalse
     \let\twopassdata\empty
   \else
     \twopassdatafoundtrue
   \fi}

\def\dogetlasttwopassdata#1%
  {\edef\nexttwopassdata{#1}%
   \ifx\nexttwopassdata\notwopassdata \else
     \let\twopassdata\nexttwopassdata
     \advance\scratchcounter \plusone
     \twopassdatafoundtrue
   \fi}

\def\getlasttwopassdata#1%
  {\loadtwopassdata
   \scratchcounter\zerocount
   \@EAEAEA\rawprocesscommalist\@EA\@EA\@EA[\csname#1:\s!list\endcsname]\dogetlasttwopassdata
   \edef\noftwopassitems{\the\scratchcounter}%
   \iftwopassdatafound\else
     \let\twopassdata\empty
   \fi}

\def\getfromtwopassdata#1#2%
  {\loadtwopassdata
   \@EAEAEA\getfromcommalist\@EA\@EA\@EA[\csname#1:\s!list\endcsname][#2]%
   \ifx\commalistelement\empty
     \twopassdatafoundfalse
     \let\twopassdata\empty
   \else
     \twopassdatafoundtrue
     \let\twopassdata\commalistelement
   \fi}

% Default-instellingen (verborgen)

\prependtoks \resetutilities \to \everyjob

% left over

\def\plaatsvolledig#1#2#3#4%   kop, ref, tit, do
  {#1[#2]{#3}#4\page[\v!yes]}

% Experiment
%
%\installprogram{Hello World}

\def\installprogram#1%
  {\immediatewriteutility{e p {#1}}}

% \writeplugindata{texutil}{{alpha}}
% \writeplugindata{texutil}{{beta}}
% \writeplugindata{texutil}{{gamma}}
% \writeplugindata{texutil}{{delta}}
%
% \loadplugindata {plugintest}

\def\immediatewriteplugindata#1#2%
  {\immediatewriteutility{p u {#1} #2}}

\def\writeplugindata#1#2%
  {\writeutility{p u {#1} #2}}

\def\loadplugindata#1%
  {\doutilities{#1}\jobname\empty\relax\relax}

% \plugincommand{\command{}{}{}}
%
% this way we can catch undefined commands

\long\def\plugincommand#1%
  {\doplugincommand#1\relax}

\long\def\doplugincommand#1%
  {\ifx#1\undefined
     \expandafter\noplugincommand
   \else
     \expandafter#1%
   \fi}

% shorter:
%
% \long\def\doplugincommand#1%
%   {\ifx#1\undefined\expandafter\noplugincommand\fi#1}

\long\def\noplugincommand#1\relax
  {}

% \addutilityreset{plugintest}
%
% \def\resetplugintest{\let\plugintest\gobbletwoarguments}
% \def\setplugintest  {\let\plugintest\writestatus}
%
% \installplugin
%   {plugintest}
%   {\let\plugintest\gobbletwoarguments}
%   {\let\plugintest\writestatus}

\long\def\installplugin#1#2#3%
  {\addutilityreset          {#1}%
   \long\setvalue{\s!reset#1}{#2}%
   \long\setvalue{\s!set  #1}{#3}}

\protect \endinput