%D \module
%D   [       file=catc-ini,
%D        version=2006.09.18,
%D          title=\CONTEXT\ System Macros,
%D       subtitle=Catcode Handling,
%D         author=Hans Hagen,
%D           date=\currentdate,
%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
%C
%C This module is part of the \CONTEXT\ macro||package and is
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
%C details.

\registerctxluafile{catc-ini}{autosuffix}

\unprotect

%D A long standing wish has been the availability of catcode arrays. Because
%D traditional \TEX\ does not provide this we implement a fake method in the \MKII\
%D file. There is some overlap in code with \MKII\ but we take that for granted.
%D Also, in \MKIV\ less active characters are used.

% \begingroup
%     \letcharcode\newlineasciicode\relax \xdef\outputnewlinechar{\Uchar\newlineasciicode}
% \endgroup

% \endlinechar = \endoflineasciicode % appended to input lines
% \newlinechar = \newlineasciicode   % can be used in write

% rather special and used in writing to file: \let\par\outputnewlinechar

% \permanent\protected\def\initializenewlinechar % operating system dependent
%   {\begingroup
%    \enforced\letcharcode\newlineasciicode\relax
%    \newlinechar\newlineasciicode
%    \xdef\outputnewlinechar{\Uchar\newlineasciicode}%
%    \endgroup}

%D We predefine some prefixes ahead of syst-aux and mult-sys. We reserve 8 slots for
%D catcodes. (This active mess probably needs an update some day.)

% \installsystemnamespace{catcodelet} % let : \let
% \installsystemnamespace{catcodedef} % def : \def
% \installsystemnamespace{catcodeued} % ued : \protected\def
% \installsystemnamespace{catcodeget} %       \meaning

\installsystemnamespace{catcodetablet}
\installsystemnamespace{catcodetablen}

\newcount\c_syst_catcodes_n \c_syst_catcodes_n\zerocount % 0 = signal, so advance before allocate

% \newcount\c_syst_catcodes_a
% \newcount\c_syst_catcodes_b
% \newcount\c_syst_catcodes_c

\permanent\protected\def\newcatcodetable#1% we could move the cctdefcounter to lua
  {\global\advance\c_syst_catcodes_n\plusone
   \gdefcsname\??catcodetablen\number\c_syst_catcodes_n\endcsname{\string#1}% logging
   \immutable\integerdef#1\c_syst_catcodes_n
   \ctxlua{catcodes.register("\expandafter\gobbleoneargument\string#1",\number#1)}}

\newtoks\everysetdefaultcatcodes

\everysetdefaultcatcodes % this might get dropped
  {\catcode\backslashasciicode\othercatcode
   \catcode\endoflineasciicode\othercatcode
   \catcode\spaceasciicode    \othercatcode
   \catcode\commentasciicode  \othercatcode
   \catcode\delasciicode      \othercatcode}

\permanent\protected\def\startcatcodetable#1#2\stopcatcodetable
  {\begingroup
   \catcodetable\inicatcodes
   \the\everysetdefaultcatcodes
   #2%
   \savecatcodetable#1\relax
   \endgroup}

\permanent\let\stopcatcodetable\relax

\permanent\protected\def\startextendcatcodetable#1#2\stopextendcatcodetable
  {\begingroup
   \catcodetable#1\relax
   \globaldefs\plusone
   #2%
   \globaldefs\zerocount
   \endgroup}

\permanent\let\stopextendcatcodetable\relax

\permanent\protected\def\permitcircumflexescape % to be used grouped
  {\catcode\circumflexasciicode\superscriptcatcode}

\aliased\let\permitcaretescape\permitcircumflexescape

\newconstant\defaultcatcodetable

% ==
%
% \protected\def\startextendcatcodetable#1#2\stopextendcatcodetable
%   {\bgroup
%    \scratchcounter\the\catcodetable
%    \catcodetable #1 #2
%    \catcodetable\scratchcounter
%    \egroup}

%D The next command can be defined in a cleaner way in the MkIV way but we want
%D to have a fast one with a minimal chance for interference. Do we still need
%D this complex mechanism? Probably not. Future versions of \MKIV\ might only use
%D active characters for very special cases. Older files demonstrate this old
%D hackery tilde abuse.

% %D Once a catcode is assigned, the next assignments will happen faster. However,
% %D redefinitions probably happen seldom so it's sort of overkill. We also need to
% %D take care of the initial (shared between catcode regfimes) binding.
%
% \permanent\protected\def\letcatcodecommand{\afterassignment\syst_catcodes_let_a\c_syst_catcodes_a}
% \permanent\protected\def\defcatcodecommand{\afterassignment\syst_catcodes_def_a\c_syst_catcodes_a} % obsolete
% \permanent\protected\def\uedcatcodecommand{\afterassignment\syst_catcodes_ued_a\c_syst_catcodes_a} % obsolete
%
% \def\syst_catcodes_let_a{\afterassignment\syst_catcodes_let_b\c_syst_catcodes_b}
% \def\syst_catcodes_def_a{\afterassignment\syst_catcodes_def_b\c_syst_catcodes_b}
% \def\syst_catcodes_ued_a{\afterassignment\syst_catcodes_ued_b\c_syst_catcodes_b}
%
% % The two step definition is used because we need to fetch the third argument.
%
% \def\syst_catcodes_let_b % each time
%   {\ifcsname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname
%      \expandafter\lastnamedcs
%    \else
%      \expandafter\syst_catcodes_let_c
%    \fi}
%
% \def\syst_catcodes_def_b % each time
%   {\ifcsname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname
%      \expandafter\lastnamedcs
%    \else
%      \expandafter\syst_catcodes_def_c
%    \fi}
%
% \def\syst_catcodes_ued_b % each time
%   {\ifcsname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname
%      \expandafter\lastnamedcs
%    \else
%      \expandafter\syst_catcodes_ued_c
%    \fi}
%
% \def\syst_catcodes_let_c % only first time
%   {\frozen\enforced\gdefcsname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname\expandafter
%      {\expandafter\enforced\letcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}%
%    \syst_catcodes_reinstate_unexpanded
%    \csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}
%
% \def\syst_catcodes_def_c % only first time (we could use \normalexpanded here)
%   {\frozen\enforced\gdefcsname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname
%      \expandafter##\expandafter1\expandafter
%        {\expandafter\frozen\expandafter\enforced\defcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}%
%    \syst_catcodes_reinstate_normal
%    \csname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}
%
% \def\syst_catcodes_ued_c % only first time
%   {\frozen\enforced\gdefcsname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname
%      \expandafter##\expandafter1\expandafter
%        {\expandafter\frozen\expandafter\enforced\expandafter\protected\defcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}%
%    \syst_catcodes_reinstate_unexpanded
%    \csname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}
%
% %D We can simplify this a bit (not that critical):
%
% \def\syst_catcodes_let_b
%   {\afterassignment\syst_catcodes_let_c\let\m_syst_catcodes_temp}
%
% \def\syst_catcodes_let_c
%   {\enforced\letcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname\m_syst_catcodes_temp
%    \protected\edef\m_syst_catcodes_temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}%
%    \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp}
%
% \def\syst_catcodes_let_c
%   {\letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp}
%
% % not that much gain:
%
% \def\syst_catcodes_let_c % only first time
%   {\expandafter\integerdef\csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname\c_syst_catcodes_b
%    \letcsname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname\m_syst_catcodes_temp
%    \protected\edef\m_syst_catcodes_temp{\noexpand\catcodecommand\csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}%
%    \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp}

%D This can be used when a direct definition has been done and the selector has been
%D lost. I really need to get rid of this ...

% \permanent\def\reinstatecatcodecommand{\afterassignment\syst_catcodes_reinstate_normal\c_syst_catcodes_b}
%
% \let\m_syst_catcodes_temp\relax
%
% \def\syst_catcodes_reinstate_normal
%   {\edef\m_syst_catcodes_temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}%
%    \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp}
%
% \def\syst_catcodes_reinstate_unexpanded
%   {\protected\edef\m_syst_catcodes_temp{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}%
%    \letcharcode\c_syst_catcodes_b\m_syst_catcodes_temp}
%
% \permanent\def\catcodecommand#1%
%   {\csname\??catcodeget\number
%      \ifcsname\??catcodeget\number\currentcatcodetable:\number#1\endcsname
%        \currentcatcodetable \else \defaultcatcodetable
%      \fi
%    :\number#1\endcsname}

%D For now, will become just letcharcode:

\permanent\protected\def\letcatcodecommand{\afterassignment\letcharcode\scratchcounter}

%D \startbuffer
%D \def\foo{foo}
%D \start
%D   \pushactivechar    |   \letcharcode124 \foo test||test\par
%D   \popactivechar     |                        test||test\par
%D \stop
%D \start
%D   \pushactivecharcode124 \letcharcode124 \foo test||test\par
%D   \popactivecharcode 124                      test||test\par
%D \stop
%D \stopbuffer
%D
%D \typebuffer \getbuffer

\permanent\protected\def\pushactivechar#1%
  {\expandafter\let\expandafter\m_active\csname\csactive#1\endcsname
  %\expandafter\let\expandafter\m_active\csname\Uchar"FFFF\Uchar\expandafter`\string#1\endcsname
   \pushmacro\m_active}

\permanent\protected\def\popactivechar#1%
  {\popmacro\m_active
  %\letcsname\Uchar"FFFF\Uchar\expandafter`\string#1\endcsname\m_active
   \letcsname\csactive#1\endcsname\m_active}

\permanent\protected\def\pushactivecharcode{\afterassignment\syst_active_push\integerdef\c_active_char_code}
\permanent\protected\def\popactivecharcode {\afterassignment\syst_active_pop\integerdef \c_active_char_code}

\permanent\protected\def\syst_active_push
  {\expandafter\let\expandafter\m_active\csname\csactive\Uchar\c_active_char_code\endcsname
  %\expandafter\let\expandafter\m_active\csname\Uchar"FFFF\Uchar\c_active_char_code\endcsname
   \pushmacro\m_active}

\permanent\protected\def\syst_active_pop
  {\popmacro\m_active
  %\letcsname\Uchar"FFFF\Uchar\c_active_char_code\endcsname\m_active
   \letcsname\csactive\Uchar\c_active_char_code\endcsname\m_active}

%D \macros
%D   {restorecatcodes,pushcatcodetable,popcatcodetable}
%D
%D We're not finished dealing \CATCODES\ yet. In \CONTEXT\ we use only one auxiliary
%D file, which deals with tables of contents, registers, two pass tracking,
%D references etc. This file, as well as files concerning graphics, is processed
%D when needed, which can be in the mid of typesetting verbatim. However, when
%D reading in data in verbatim mode, we should temporary restore the normal
%D \CATCODES, and that's exactly what the next macros do. Saving the catcodes can be
%D disabled by saying \type {\localcatcodestrue}. In \MKIV\ instead we can push and
%D pop catcode tables and as we keep track of used tables users seldom need to deal
%D with this themselves.

\newcount\c_syst_catcodes_level

\permanent\protected\def\pushcatcodetable
  {\advance\c_syst_catcodes_level\plusone
   \syst_catcodes_trace_push
   \expandafter\integerdef\csname\??catcodetablet\number\c_syst_catcodes_level\endcsname\currentcatcodetable}

\permanent\protected\def\popcatcodetable
  {\ifcase\c_syst_catcodes_level
     \syst_catcodes_trace_nesting_error
   \else
     \catcodetable\csname\??catcodetablet\number\c_syst_catcodes_level\endcsname
     \syst_catcodes_trace_pop
     \advance\c_syst_catcodes_level\minusone
   \fi}

\protected\def\syst_catcodes_trace_nesting_error
  {\immediate\write\statuswrite{}%
   \immediate\write\statuswrite{Fatal error: catcode push/pop mismatch. Fix this! (restore level: \number\c_syst_catcodes_level)}\wait\end
   \immediate\write\statuswrite{}}

\permanent\protected\def\restorecatcodes % takes previous level
  {\ifnum\c_syst_catcodes_level>\plusone
     \catcodetable\csname\??catcodetablet\number\numexpr\c_syst_catcodes_level-1\relax\endcsname
   \fi}

\permanent\protected\def\setcatcodetable#1%
  {\catcodetable#1%
   \syst_catcodes_trace_set}

%D Handy for debugging (not that we ever use(d) it):
%D
%D \starttyping
%D \tracecatcodetables
%D \stoptyping

\permanent\protected\def\tracecatcodetables
  {\def\syst_catcodes_trace_set {\syst_catcodes_trace{set  \catcodetablename\space                                at \number\c_syst_catcodes_level}}%
   \def\syst_catcodes_trace_push{\syst_catcodes_trace{push \catcodetablename\space from \syst_catcodes_prev\space at \number\c_syst_catcodes_level}}%
   \def\syst_catcodes_trace_pop {\syst_catcodes_trace{pop  \catcodetablename\space to   \syst_catcodes_prev\space at \number\c_syst_catcodes_level}}}

\def\syst_catcodes_trace#1{\immediate\write\statuswrite{[#1]}}

\def\syst_catcodes_prev
  {\ifnum\numexpr\c_syst_catcodes_level-1\relax>\zerocount
     \csname\??catcodetablen\number\csname\??catcodetablet\number\numexpr\c_syst_catcodes_level-1\relax\endcsname\endcsname
   \else
     -%
   \fi}

\permanent\def\catcodetablename
  {\ifnum\currentcatcodetable>\zerocount
     \csname\??catcodetablen\number\currentcatcodetable\endcsname
   \else
     -%
   \fi}

\let\syst_catcodes_trace_set \empty
\let\syst_catcodes_trace_push\empty
\let\syst_catcodes_trace_pop \empty

% \tracecatcodetables

\protect

%D We still have to define these so let's do that now:

\newcatcodetable \inicatcodes
\initcatcodetable\inicatcodes

\mutable\let\currentcatcodetable\catcodetable

\endinput