%D \module
%D   [       file=catc-ini,
%D        version=2006.09.18,
%D          title=\CONTEXT\ System Macros,
%D       subtitle=Catcode Handling,
%D         author=Hans Hagen,
%D           date=\currentdate,
%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
%C
%C This module is part of the \CONTEXT\ macro||package and is
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
%C details.

%D We've split the functionality of syst-cat.* over more files
%D now so that we can load more selectively.

\registerctxluafile{catc-ini}{1.001}

\unprotect

%D A long standing wish has been the availability of catcode
%D arrays. Because traditional \TEX\ does not provide this we
%D implement a fake method in the \MKII\ file. There is some
%D overlap in code with \MKII\ but we take that for granted.

\setnewconstant\escapecatcode             0
\setnewconstant\begingroupcatcode         1
\setnewconstant\endgroupcatcode           2
\setnewconstant\mathshiftcatcode          3
\setnewconstant\alignmentcatcode          4
\setnewconstant\endoflinecatcode          5
\setnewconstant\parametercatcode          6
\setnewconstant\superscriptcatcode        7
\setnewconstant\subscriptcatcode          8
\setnewconstant\ignorecatcode             9
\setnewconstant\spacecatcode             10
\setnewconstant\lettercatcode            11
\setnewconstant\othercatcode             12  % finally obsolete: \let\other \othercatcode
\setnewconstant\activecatcode            13  % finally obsolete: \let\active\activecatcode
\setnewconstant\commentcatcode           14
\setnewconstant\invalidcatcode           15

\setnewconstant\tabasciicode              9
\setnewconstant\newlineasciicode         10  % don't confuse this one with \endoflineasciicode
\setnewconstant\formfeedasciicode        12
\setnewconstant\endoflineasciicode       13  % somewhat messy but this can be the active \par
\setnewconstant\endoffileasciicode       26
\setnewconstant\spaceasciicode           32
\setnewconstant\exclamationmarkasciicode 33 % ! used in namespace protection
\setnewconstant\doublequoteasciicode     34 % "
\setnewconstant\hashasciicode            35
\setnewconstant\dollarasciicode          36
\setnewconstant\commentasciicode         37
\setnewconstant\ampersandasciicode       38
\setnewconstant\singlequoteasciicode     39 % '
\setnewconstant\primeasciicode           39 % '
\setnewconstant\forwardslashasciicode    47 % /
\setnewconstant\colonasciicode           58
\setnewconstant\lessthanasciicode        60 % < used as alternative verbatim {
\setnewconstant\morethanasciicode        62 % > used as alternative verbatim }
\setnewconstant\questionmarkasciicode    63 % ? used in namespace protection
\setnewconstant\atsignasciicode          64 % @ used in namespace protection
\setnewconstant\backslashasciicode       92 % `\\
\setnewconstant\circumflexasciicode      94
\setnewconstant\underscoreasciicode      95
\setnewconstant\leftbraceasciicode      123 % `\{
\setnewconstant\barasciicode            124 % `\|
\setnewconstant\rightbraceasciicode     125 % `\}
\setnewconstant\tildeasciicode          126 % `\~
\setnewconstant\delasciicode            127

\begingroup
    \catcode \tabasciicode       \activecatcode  \gdef\activetabtoken      {^^I}
                                                 \gdef\outputnewlinechar   {^^J}
    \catcode \formfeedasciicode  \activecatcode  \gdef\activeformfeedtoken {^^L}
    \catcode \endoflineasciicode \activecatcode  \gdef\activeendoflinetoken{^^M}
\endgroup

% \endlinechar = \endoflineasciicode % appended to input lines
% \newlinechar = \newlineasciicode   % can be used in write

% rather special and used in writing to file: \let\par\outputnewlinechar

\def\initializenewlinechar % operating system dependent
  {\begingroup
   \newlinechar\newlineasciicode
   \xdef\outputnewlinechar{^^J}%
   \endgroup}

%D We predefine some prefixes ahead of syst-aux and mult-sys. We reserve 8 slots
%D for catcodes.

\def\??catcodelet   {1>>} % let : \let
\def\??catcodedef   {2>>} % def : \def
\def\??catcodeued   {3>>} % ued : \unexpanded\def
\def\??catcodeget   {4>>} %       \meaning

\def\??catcodetablet{5>>}
\def\??catcodetablen{6>>}

\newcount\c_syst_catcodes_n \c_syst_catcodes_n\zerocount % 0 = signal, so advance before allocate
\newcount\c_syst_catcodes_a
\newcount\c_syst_catcodes_b
\newcount\c_syst_catcodes_c

\normalprotected\def\newcatcodetable#1% we could move the cctdefcounter to lua
  {\global\advance\c_syst_catcodes_n\plusone
   \expandafter\xdef\csname\??catcodetablen\number\c_syst_catcodes_n\endcsname{\string#1}% logging
   \newconstant#1%
   #1\c_syst_catcodes_n
   \ctxlua{catcodes.register("\expandafter\gobbleoneargument\string#1",\number#1)}}

\newtoks \everysetdefaultcatcodes

\everysetdefaultcatcodes % this might get dropped
  {\catcode\backslashasciicode\othercatcode
   \catcode\endoflineasciicode\othercatcode
   \catcode\spaceasciicode    \othercatcode
   \catcode\commentasciicode  \othercatcode
   \catcode\delasciicode      \othercatcode}

\long\normalprotected\def\startcatcodetable#1#2\stopcatcodetable
  {\begingroup
   \catcodetable\inicatcodes
   \the\everysetdefaultcatcodes
   #2%
   \savecatcodetable#1\relax
   \endgroup}

\let\stopcatcodetable\relax

\long\normalprotected\def\startextendcatcodetable#1#2\stopextendcatcodetable
  {\begingroup
   \catcodetable#1\relax
   \globaldefs\plusone
   #2%
   \globaldefs\zerocount
   \endgroup}

\let\stopextendcatcodetable\relax

\normalprotected\def\permitcircumflexescape % to be used grouped
  {\catcode\circumflexasciicode\superscriptcatcode}

\let\permitcaretescape\permitcircumflexescape

% ==
%
% \long\normalprotected\def\startextendcatcodetable#1#2\stopextendcatcodetable
%   {\bgroup
%    \scratchcounter\the\catcodetable
%    \catcodetable #1 #2
%    \catcodetable\scratchcounter
%    \egroup}

%D The next command can be defined in a cleaner way in the
%D Mk IV file but we want to have a fast one with a minimal
%D chance for interference. Do we still need this complex
%D mechanism? Future versions of \MKIV\ might only use
%D active characters for very special cases.

\setnewconstant\c_syst_catcodes_hack\tildeasciicode

%D Once a catcode is assigned, the next assignments will happen
%D faster.

\def\letcatcodecommand{\afterassignment\syst_catcodes_let_a\c_syst_catcodes_a}
\def\defcatcodecommand{\afterassignment\syst_catcodes_def_a\c_syst_catcodes_a}
\def\uedcatcodecommand{\afterassignment\syst_catcodes_ued_a\c_syst_catcodes_a}

\def\syst_catcodes_let_a{\afterassignment\syst_catcodes_let_b\c_syst_catcodes_b}
\def\syst_catcodes_def_a{\afterassignment\syst_catcodes_def_b\c_syst_catcodes_b}
\def\syst_catcodes_ued_a{\afterassignment\syst_catcodes_ued_b\c_syst_catcodes_b}

\def\syst_catcodes_let_b % each time
  {\ifcsname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname
     \csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname
   \else
     \expandafter\syst_catcodes_let_c
   \fi}

\def\syst_catcodes_def_b % each time
  {\ifcsname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname
     \csname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname
   \else
     \expandafter\syst_catcodes_def_c
   \fi}

\def\syst_catcodes_ued_b % each time
  {\ifcsname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname
     \csname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname
   \else
     \expandafter\syst_catcodes_ued_c
   \fi}

\def\syst_catcodes_let_c % only first time
  {\expandafter\gdef\csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname\expandafter
     {\expandafter\let\csname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}%
   \syst_catcodes_reinstate_unexpanded
   \csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}


\def\syst_catcodes_def_c % only first time (we could use \normalexpanded here)
  {\expandafter\gdef\csname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname
     \expandafter##\expandafter1\expandafter
       {\expandafter\def\csname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}%
   \syst_catcodes_reinstate_normal
   \csname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}

\def\syst_catcodes_ued_c % only first time
  {\expandafter\gdef\csname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname
     \expandafter##\expandafter1\expandafter
       {\expandafter\normalprotected\expandafter\def\csname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}%
   \syst_catcodes_reinstate_unexpanded
   \csname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}

\def\reinstatecatcodecommand{\afterassignment\syst_catcodes_reinstate_normal\c_syst_catcodes_b}

\def\syst_catcodes_reinstate_normal % can be used when a direct definition has been done
  {\begingroup                        % and the selector has been lost
   \uccode\c_syst_catcodes_hack\c_syst_catcodes_b
   \catcode\uccode\c_syst_catcodes_hack\activecatcode
   \uppercase{\xdef~{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}}%
   \endgroup}

\def\syst_catcodes_reinstate_unexpanded % can be used when a direct definition has been done
  {\begingroup                            % and the selector has been lost
   \uccode\c_syst_catcodes_hack\c_syst_catcodes_b
   \catcode\uccode\c_syst_catcodes_hack\activecatcode
   \uppercase{\normalprotected\xdef~{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}}%
   \endgroup}

\newconstant\defaultcatcodetable

\def\catcodecommand#1%
  {\csname\??catcodeget\number
     \ifcsname\??catcodeget\number\currentcatcodetable:\number#1\endcsname
       \currentcatcodetable \else \defaultcatcodetable
     \fi
   :\number#1\endcsname}

%D \macros
%D   {restorecatcodes,pushcatcodetable,popcatcodetable}
%D
%D We're not finished dealing \CATCODES\ yet. In \CONTEXT\ we
%D use only one auxiliary file, which deals with tables of
%D contents, registers, two pass tracking, references etc. This
%D file, as well as files concerning graphics, is processed when
%D needed, which can be in the mid of typesetting verbatim.
%D However, when reading in data in verbatim mode, we should
%D temporary restore the normal \CATCODES, and that's exactly
%D what the next macros do. Saving the catcodes can be
%D disabled by saying \type{\localcatcodestrue}.

\newcount\c_syst_catcodes_level

\normalprotected\def\pushcatcodetable
  {\advance\c_syst_catcodes_level\plusone
   \syst_catcodes_trace_push
   \expandafter\chardef\csname\??catcodetablet\number\c_syst_catcodes_level\endcsname\currentcatcodetable}

\normalprotected\def\popcatcodetable
  {\ifcase\c_syst_catcodes_level
     \syst_catcodes_trace_nesting_error
   \else
     \expandafter\catcodetable\csname\??catcodetablet\number\c_syst_catcodes_level\endcsname
     \syst_catcodes_trace_pop
     \advance\c_syst_catcodes_level\minusone
   \fi}

\normalprotected\def\syst_catcodes_trace_nesting_error
  {\immediate\write16{}%
   \immediate\write16{Fatal error: catcode push/pop mismatch. Fix this! (restore level: \number\c_syst_catcodes_level)}\wait\end
   \immediate\write16{}}

\normalprotected\def\restorecatcodes % takes previous level
  {\ifnum\c_syst_catcodes_level>\plusone
     \expandafter\catcodetable\csname\??catcodetablet\number\numexpr\c_syst_catcodes_level-1\relax\endcsname
   \fi}

% \newtoks\everycatcodetable

\normalprotected\def\setcatcodetable#1%
  {\catcodetable#1%
%  \the\everycatcodetable
   \syst_catcodes_trace_set}

%D Handy for debugging:
%D
%D \starttyping
%D \tracecatcodetables
%D \stoptyping

\normalprotected\def\tracecatcodetables
  {\def\syst_catcodes_trace_set {\syst_catcodes_trace{set  \catcodetablename\space                                  at \number\c_syst_catcodes_level}}%
   \def\syst_catcodes_trace_push{\syst_catcodes_trace{push \catcodetablename\space from \syst_catcodes_prev\space at \number\c_syst_catcodes_level}}%
   \def\syst_catcodes_trace_pop {\syst_catcodes_trace{pop  \catcodetablename\space to   \syst_catcodes_prev\space at \number\c_syst_catcodes_level}}}

\def\syst_catcodes_trace#1{\immediate\write16{[#1]}}

\def\syst_catcodes_prev
  {\ifnum\numexpr\c_syst_catcodes_level-1\relax>\zerocount
     \csname\??catcodetablen\number\csname\??catcodetablet\number\numexpr\c_syst_catcodes_level-1\relax\endcsname\endcsname
   \else
     -%
   \fi}

\def\catcodetablename
  {\ifnum\currentcatcodetable>\zerocount
     \csname\??catcodetablen\number\currentcatcodetable\endcsname
   \else
     -%
   \fi}

\let\syst_catcodes_trace_set \empty
\let\syst_catcodes_trace_push\empty
\let\syst_catcodes_trace_pop \empty

% \tracecatcodetables

\protect

%D We still have to define these so let's do that now:

\newcatcodetable \inicatcodes
\initcatcodetable\inicatcodes

\let\currentcatcodetable\catcodetable

\endinput