Merge branch 'master' into unstable

1 files changed, 289 insertions, 183 deletions

diff --git a/luatexbase-cctb.dtx b/luatexbase-cctb.dtx
index abcd6da..5eeefc9 100644
--- a/luatexbase-cctb.dtx
+++ b/luatexbase-cctb.dtx
@@ -1,8 +1,5 @@
 % \iffalse meta-comment
 %
-% Template : look for cctb DATE DESC VERSION !!!
-% and 'derived files' !!!
-%
 % Written in 2009, 2010 by Manuel Pégourié-Gonnard and Élie Roux.
 %     <mpg@elzevir.fr>
 %     <elie.roux@telecom-bretagne.eu>
@@ -11,18 +8,14 @@
 %
 % This work consists of the main source file luatexbase-cctb.dtx
 % and the derived files
-%    luatexbase-cctb.sty cctb.lua ...
+%    luatexbase-cctb.sty luatexbase.cctb.lua
+%    test-cctb-plain.tex test-cctb-latex.tex
 %
 % Unpacking:
 %    tex luatexbase-cctb.dtx
 % Documentation:
 %    pdflatex luatexbase-cctb.dtx
 %
-%    The class ltxdoc loads the configuration file ltxdoc.cfg
-%    if available. Here you can specify further options, e.g.
-%    use A4 as paper format:
-%       \PassOptionsToClass{a4paper}{article}
-%
 %<*ignore>
 \begingroup
   \def\x{LaTeX2e}%
@@ -51,7 +44,8 @@ This work is under the CC0 license.
 
 This work consists of the main source file luatexbase-cctb.dtx
 and the derived files
-   luatexbase-cctb.sty cctb.lua ...
+   luatexbase-cctb.sty luatexbase.cctb.lua
+   test-cctb-plain.tex test-cctb-latex.tex
 
 \endpreamble
 
@@ -62,6 +56,12 @@ and the derived files
   \file{luatexbase-cctb.sty}{\from{luatexbase-cctb.dtx}{texpackage}}%
 }
 
+\generate{%
+  \usedir{doc/luatex/luatexbase}%
+  \file{test-cctb-plain.tex}{\from{luatexbase-cctb.dtx}{testplain}}%
+  \file{test-cctb-latex.tex}{\from{luatexbase-cctb.dtx}{testlatex}}%
+}
+
 \def\MetaPrefix{-- }
 
 \def\luapostamble{%
@@ -73,7 +73,7 @@ and the derived files
 
 \generate{%
   \usedir{tex/luatex/luatexbase}%
-  \file{cctb.lua}{\from{luatexbase-cctb.dtx}{luamodule}}%
+  \file{luatexbase.cctb.lua}{\from{luatexbase-cctb.dtx}{luamodule}}%
 }
 
 \obeyspaces
@@ -94,12 +94,8 @@ and the derived files
 \fi
 %</ignore>
 %<*driver>
-\NeedsTeXFormat{LaTeX2e}
-\ProvidesFile{luatexbase-cctb.drv}
-  [DATE DESC]
 \documentclass{ltxdoc}
-\EnableCrossrefs
-\CodelineIndex
+\input{lltxb-dtxstyle}
 \begin{document}
   \DocInput{luatexbase-cctb.dtx}%
 \end{document}
@@ -124,189 +120,278 @@ and the derived files
 %   Grave accent  \`     Left brace    \{     Vertical bar  \|
 %   Right brace   \}     Tilde         \~}
 %
-% \GetFileInfo{luatexbase-cctb.drv}
-%
-% \title{The \textsf{luatexbase-cctb} package}
-% \date{DATE}
+% \title{The \pk{luatexbase-cctb} package}
+% \date{v0.1 2010-03-26}
 % \author{%
-%  Manuel P\'egouri\'e-Gonnard \\ \texttt{mpg@elzevir.fr} \and
+%   Manuel P\'egouri\'e-Gonnard \\ \texttt{mpg@elzevir.fr} \and
 %   \'Elie Roux \\ \texttt{elie.roux@telecom-bretagne.eu}}
 %
 % \maketitle
 %
 % \begin{abstract}
+% In addition to the registers existing in \tex and \etex, \luatex introduces
+% a new concept: catcode tables. This package takes care of catcode table
+% allocation just like Plain TeX and LaTeX do for other registers.
 % \end{abstract}
 %
 % \section{Documentation}
 %
+% The main macro defined here is |\newluatexcatcodetable|. It behaves the same
+% as |\newbox|. Additonally, the newly allocated catcode table is initialised
+% to the catcodes of Ini\tex.  In order to help you define the catcode tables
+% (once they are allocated), two helper macros are available.
+%
+% \begin{quote}
+% \cs{luatexsetcatcoderange}\marg{from}\marg{to}\marg{value}
+% \end{quote}
+% Set all characters code in the range \meta{from}--\meta{to} to the given
+% catcode \meta{value}.
+%
+% \begin{quote}
+% \cs{setluatexcatcodetable}\marg{table}\marg{catcode statements}
+% \end{quote}
+% Set a previously allocated \meta{table} to the catcodes given by executing
+% \meta{catcode statements}. Note that \meta{table} must not be the current
+% active catcode table. You may, however, load another catcode table in your
+% \meta{catcode statements}.
+%
+% For your convenience, a few catcode tables are prefined:
+% \begin{itemize}
+% \item |\CatcodeTableIniTeX|: Ini\tex catcodes.
+% \item |\CatcodeTableString|: the catcode regime used by |\string| and
+% |\meaning|: everything has catcode 12, except space (U+0020) that has
+% catcode 10.
+% \item |\CatcodeTableOther|: everything (included space) has catcode 12.
+% \item |\CatcodeTableLaTeX|: basic \LaTeXe\ catcodes.
+% \item |\CatcodeTableLaTeXAtLetter|: same as above, but |@| is a letter.
+% \item |\CatcodeTableExpl|: catcodes used by \latex{}3 with |\ExplSyntaxOn|.
+% Be aware that this does not provide the exact same environment as
+% |\ExplSyntaxOn|: most noticibly, some booleans are not set, and
+% |\endlinechar| is not adjusted (it should be 32).
+% \end{itemize}
+%
+% Various Lua function accept a catcode table number as argument. In order to
+% use them, the package writer needs to know the number of an allocated
+% catcode table. Since |\chardef| is used for the definition of the control
+% sequence, this is rather easy to do. However, for extra ease of use, the
+% numbers are also directly accessible from Lua as the value of the table
+% |luatextra.catcodetables|, whose keys is the name of the control sequence
+% (without any leading backslash). Moreover, nickames are available for the
+% predefined catcode tables:
+% \begin{itemize}
+% \item |CatcodeTableIniTeX| = |ini|,
+% \item |CatcodeTableString| = |string|,
+% \item |CatcodeTableOther| = |other|,
+% \item |CatcodeTableLaTeX| = |latex|,
+% \item |CatcodeTableLaTeXAtLetter| = |latex-atletter| = |latex-package|,
+% \item |CatcodeTableExpl| = |expl| = |expl3|,
+% \end{itemize}
+%
 %    \section{Implementation}
 %
-%    \subsection{\TeX\ package}
+%    \subsection{\tex package}
 %
 %    \begin{macrocode}
 %<*texpackage>
 %    \end{macrocode}
 %
-%    Here we allocate catcodetables the same way we handle attributes.
+%    \subsubsection{Preliminaries}
+%
+%    Reload protection, especially for \plaintex.
 %
 %    \begin{macrocode}
-
-\newcount\luatexcatcodetabledefcounter
-
-\luatexcatcodetabledefcounter = 1
-
+                \csname lltxb@cctb@loaded\endcsname
+\expandafter\let\csname lltxb@cctb@loaded\endcsname\endinput
+%    \end{macrocode}
+%
+%    Package declaration.
+%
+%    \begin{macrocode}
+\begingroup
+  \expandafter\ifx\csname ProvidesPackage\endcsname\relax
+    \def\x#1[#2]{\immediate\write16{Package: #1 #2}}
+  \else
+    \let\x\ProvidesPackage
+  \fi
+\expandafter\endgroup
+\x{luatexbase-cctb}[2010/03/26 v0.1  Catcodetable allocation for LuaTeX  (mpg)]
+%    \end{macrocode}
+%
+%    Make sure \luatex is used.
+%
+%    \begin{macrocode}
+\begingroup\expandafter\expandafter\expandafter\endgroup
+\expandafter\ifx\csname RequirePackage\endcsname\relax
+  \input ifluatex.sty
+\else
+  \RequirePackage{ifluatex}
+\fi
+\ifluatex\else
+  \begingroup
+    \expandafter\ifx\csname PackageWarningNoLine\endcsname\relax
+      \def\x#1#2{\begingroup\newlinechar10
+        \immediate\write16{Package #1 warning: #2}\endgroup}
+    \else
+      \let\x\PackageWarningNoLine
+    \fi
+  \expandafter\endgroup
+  \x{luatexbase-cctb}{LuaTeX is required for this package.^^J
+    Aborting package loading.}
+  \expandafter\endinput
+\fi
+%    \end{macrocode}
+%
+%    Make sure the catcode of @ is correct, especially for \plaintex.
+%
+%    \begin{macrocode}
+\expandafter\edef\csname lltxb@cctb@AtEnd\endcsname{%
+  \catcode64 \the\catcode64\relax}
+\catcode64 11
+%    \end{macrocode}
+%
+%    \subsubsection{Main content}
+%
+%    Load the supporting Lua module.
+%
+%    \begin{macrocode}
+\directlua{dofile(kpse.find_file('luatexbase.cctb.lua', 'lua'))}
+%    \end{macrocode}
+%
+%    The allocation macro. Allocate tables starting with 1, since table 0 is
+%    reserved for IniTeX catcodes by LuaTeX.
+%
+%    \begin{macrocode}
+\newcount\lltxb@catcodetable@alloc
+\lltxb@catcodetable@alloc\z@
 \def\newluatexcatcodetable#1{%
-  \ifnum\luatexcatcodetabledefcounter<1114110\relax % 0x10FFFF is maximal \chardef
-    \global\advance\luatexcatcodetabledefcounter by 1\relax %
-    \allocationnumber=\luatexcatcodetabledefcounter %
-    \global\chardef#1=\allocationnumber %
-    \luadirect{%
-      luatextra.catcodetabledef_from_tex([[\noexpand#1]], '\number\allocationnumber')}%
-    \wlog{\string#1=\string\catcodetable\the\allocationnumber}%
-  \else %
-    \errmessage{No room for a new \string\catcodetable}%
-  \fi %
-}
-
+  \ifnum\lltxb@catcodetable@alloc<65535\relax
+    \global\advance\lltxb@catcodetable@alloc\@ne
+    \allocationnumber\lltxb@catcodetable@alloc
+    \global\chardef#1\allocationnumber
+    \luatexinitcatcodetable\allocationnumber
+    \begingroup\escapechar\m@ne \expandafter\endgroup
+    \directlua{luatextra.catcodetabledef_from_tex(
+      '\luatexluaescapestring{\string#1}', '\number\allocationnumber')}%
+    \wlog{\string#1=\string\luatexcatcodetable\the\allocationnumber}%
+  \else
+    \errmessage{No room for a new \string\luatexcatcodetable}%
+  \fi}
 %    \end{macrocode}
 %
-%    A small patch to manage the catcode of \@ in Plain, and to get two new
-%    counters in Plain too.
+%    A small patch to get two new counters in Plain too.
 %
 %    \begin{macrocode}
-
-\expandafter\edef\csname ltxtra@AtEnd\endcsname{%
-  \catcode64 \the\catcode64\relax
-}
-
-\catcode 64=11\relax
-
 \expandafter\ifx\csname @tempcnta\endcsname\relax
   \csname newcount\endcsname\@tempcnta
 \fi
 \expandafter\ifx\csname @tempcntb\endcsname\relax
   \csname newcount\endcsname\@tempcntb
 \fi
-
 %    \end{macrocode}
 %
-%    A macro that sets the catcode of a range of characters. The first
-%    parameter is the character number of the first character of the range,
-%    the second parameter is one for the last character, and the third
-%    parameter is the catcode we want them to have.
+%    Set the catcodes for a range of characters.
 %
 %    \begin{macrocode}
-
 \def\luatexsetcatcoderange#1#2#3{%
   \edef\luaSCR@temp{%
     \noexpand\@tempcnta=\the\@tempcnta
     \noexpand\@tempcntb=\the\@tempcntb
     \noexpand\count@=\the\count@
-    \relax
-  }%
+    \relax}%
   \@tempcnta=#1\relax
   \@tempcntb=#2\relax
   \count@=#3\relax
   \loop\unless\ifnum\@tempcnta>\@tempcntb
     \catcode\@tempcnta=\count@
-    \advance\@tempcnta by 1\relax
+    \advance\@tempcnta\@ne
   \repeat
-  \luaSCR@temp
-}
-
+  \luaSCR@temp}
 %    \end{macrocode}
 %
-%    Finally we create several catcodetables that may be used by the user.
-%    These are:
-%
-%    \begin{itemize}
-%    \item \texttt{\string\CatcodeTableIniTeX}: the base \TeX\ catcodes
-%    \item \texttt{\string\CatcodeTableString}: almost all characters have
-%    catcode 12
-%    \item \texttt{\string\CatcodeTableOther}: all characters have catcode 12
-%    (even space)
-%    \item \texttt{\string\CatcodeTableLaTeX}: the \LaTeX\ classical catcodes
-%    \item \texttt{\string\CatcodeTableLaTeXAtLetter}: the \LaTeX\ classical
-%    catcodes and |@| letter
-%    \item \texttt{\string\CatcodeTableExpl}: the expl3 catcodes
-%    \end{itemize}
+%    Set a catcode table.
+%
+%    \begin{macrocode}
+\def\setluatexcatcodetable#1#2{%
+  \begingroup
+  #2%
+  \luatexsavecatcodetable#1%
+  \endgroup}
+%    \end{macrocode}
+%
+%    The |IniTeX| catcode table needs no extra initialisation.
 %
 %    \begin{macrocode}
-
 \newluatexcatcodetable\CatcodeTableIniTeX
+%    \end{macrocode}
+%
+%    The |String| and |Other| catcode tables.
+%
+%    \begin{macrocode}
 \newluatexcatcodetable\CatcodeTableString
+\setluatexcatcodetable\CatcodeTableString{%
+  \luatexcatcodetable\CatcodeTableIniTeX
+  \catcode0   12 % nul
+  \catcode13  12 % carriage return
+  \catcode37  12 % percent
+  \luatexsetcatcoderange{65}{90}{12}% A-Z
+  \luatexsetcatcoderange{97}{122}{12}% a-z
+  \catcode92  12 % backslash
+  \catcode127 12 }
+%    \end{macrocode}
+%
+%    \begin{macrocode}
 \newluatexcatcodetable\CatcodeTableOther
+\setluatexcatcodetable\CatcodeTableOther{%
+  \luatexcatcodetable\CatcodeTableString
+  \catcode32 12 }
+%    \end{macrocode}
+%
+%    Tables for \LaTeXe.
+%
+%    \begin{macrocode}
 \newluatexcatcodetable\CatcodeTableLaTeX
+\setluatexcatcodetable\CatcodeTableLaTeX{%
+  \luatexcatcodetable\CatcodeTableIniTeX
+  \luatexsetcatcoderange{0}{31}{15}%
+  \catcode9   10 % tab
+  \catcode12  13 % form feed
+  \catcode13  5  % carriage return
+  \catcode35  6  % hash
+  \catcode36  3  % dollar
+  \catcode38  4  % ampersand
+  \catcode94  7  % circumflex
+  \catcode95  8  % underscore
+  \catcode123 1  % brace left
+  \catcode125 2  % brace right
+  \catcode126 13 % tilde
+  \catcode127 15 }
+%    \end{macrocode}
+%
+%    \begin{macrocode}
 \newluatexcatcodetable\CatcodeTableLaTeXAtLetter
+\setluatexcatcodetable\CatcodeTableLaTeXAtLetter{%
+  \luatexcatcodetable\CatcodeTableLaTeX
+  \catcode64=11 }
+%    \end{macrocode}
+%
+%    A table for \pk{expl3}.
+%
+%    \begin{macrocode}
 \newluatexcatcodetable\CatcodeTableExpl
-\initluatexcatcodetable\CatcodeTableIniTeX
-
-\expandafter\ifx\csname @firstofone\endcsname\relax
-  \long\def\@firstofone#1{#1}%
-\fi
-
-\begingroup
-  \def\@makeother#1{\catcode#1=12\relax}%
-  \@firstofone{%
-    \luatexcatcodetable\CatcodeTableIniTeX
-    \begingroup
-      \luatexsetcatcoderange{0}{8}{15}%
-      \catcode9=10  % tab
-      \catcode11=15 %
-      \catcode12=13 % form feed
-      \luatexsetcatcoderange{14}{31}{15}%
-      \catcode35=6 % hash
-      \catcode36=3 % dollar
-      \catcode38=4 % ampersand
-      \catcode94=7 % circumflex
-      \catcode95=8 % underscore
-      \catcode123=1 % brace left
-      \catcode125=2 % brace right
-      \catcode126=13 % tilde
-      \catcode127=15 %
-      \saveluatexcatcodetable\CatcodeTableLaTeX
-      \catcode64=11 %
-      \saveluatexcatcodetable\CatcodeTableLaTeXAtLetter
-    \endgroup
-    \begingroup
-      \luatexsetcatcoderange{0}{8}{15}%
-      \catcode9=9 % tab ignored
-      \catcode11=15 %
-      \catcode12=13 % form feed
-      \luatexsetcatcoderange{14}{31}{15}%
-      \catcode32=9 % space is ignored
-      \catcode35=6 % hash mark is macro parameter character
-      \catcode36=3 % dollar (not so sure about the catcode...)
-      \catcode38=4 % ampersand
-      \catcode58=11 % colon letter
-      \catcode94=7 % circumflex is superscript character
-      \catcode95=11 % underscore letter
-      \catcode123=1 % left brace is begin-group character
-      \catcode125=2 % right brace is end-group character
-      \catcode126=10 % tilde is a space char.
-      \catcode127=15 %
-      \saveluatexcatcodetable\CatcodeTableExpl
-    \endgroup
-    \@makeother{0}% nul
-    \@makeother{13}% carriage return
-    \@makeother{37}% percent
-    \@makeother{92}% backslash
-    \@makeother{127}%
-    \luatexsetcatcoderange{65}{90}{12}% A-Z
-    \luatexsetcatcoderange{97}{122}{12}% a-z
-    \saveluatexcatcodetable\CatcodeTableString
-    \@makeother{32}% space
-    \saveluatexcatcodetable\CatcodeTableOther
-  \endgroup
-}
-
-\ltxtra@AtEnd
-
-\luadirect{luatextra.catcodetable_do_shortcuts()}
-
+\setluatexcatcodetable\CatcodeTableExpl{%
+  \luatexcatcodetable\CatcodeTableLaTeX
+  \catcode126 10 % tilde is a space char
+  \catcode32  9  % space is ignored
+  \catcode9   9  % tab also ignored
+  \catcode95  11 % underscore letter
+  \catcode58  11 % colon letter
+  }
 %    \end{macrocode}
 %
+%    Finally do the shortcuts.
+%
 %    \begin{macrocode}
+\directlua{luatextra.catcodetable_do_shortcuts()}
 %</texpackage>
 %    \end{macrocode}
 %
@@ -314,6 +399,7 @@ and the derived files
 %
 %    \begin{macrocode}
 %<*luamodule>
+module('luatextra', package.seeall)
 %    \end{macrocode}
 %
 %    In the same way, the table \texttt{tex.catcodetablenumber} contains the
@@ -321,60 +407,80 @@ and the derived files
 %    \texttt{\string\newluacatcodetable}.
 %
 %    \begin{macrocode}
-
-luatextra.catcodetables = {}
-
-tex.catcodetablenumber = luatextra.catcodetables
-
-function luatextra.catcodetabledef_from_tex(name, number)
-    truename = name:gsub('[\\ ]', '')
-    luatextra.catcodetables[truename] = tonumber(number)
+catcodetables = {}
+tex.catcodetablenumber = catcodetables
+function catcodetabledef_from_tex(name, number)
+    catcodetables[name] = tonumber(number)
 end
-
 %    \end{macrocode}
 %
-%    With this function we create some shortcuts for a
-%    better readability in lua code. This makes
-%    |tex.catcodetablenumber.latex| equivalent to 
+%    With this function we create some shortcuts for a better readability in
+%    lua code. This makes |tex.catcodetablenumber.latex| equivalent to
 %    |tex.catcodetablenumber['CatcodeTableLaTeX']|.
 %
 %    \begin{macrocode}
-
-function luatextra.catcodetable_do_shortcuts()
-    local cat = tex.catcodetablenumber
-    local val = cat['CatcodeTableLaTeX']
-    if val then
-      cat['latex'] = val
-    end
-    val = cat['CatcodeTableLaTeXAtLetter']
-    if val then
-      cat['latex-package'] = val
-      cat['latex-atletter'] = val
-    end
-    val = cat['CatcodeTableIniTeX']
-    if val then
-      cat['ini'] = val
-    end
-    val = cat['CatcodeTableExpl']
-    if val then
-      cat['expl3'] = val
-      cat['expl'] = val
-    end
-    val = cat['CatcodeTableString']
-    if val then
-      cat['string'] = val
-    end
-    val = cat['CatcodeTableOther']
-    if val then
-      cat['other'] = val
-    end
+function catcodetable_do_shortcuts()
+    local cat = luatextra.catcodetables
+    cat['latex']                = cat.CatcodeTableLaTeX
+    cat['latex-package']        = cat.CatcodeTableLaTeXAtLetter
+    cat['latex-atletter']       = cat.CatcodeTableLaTeXAtLetter
+    cat['ini']                  = cat.CatcodeTableIniTeX
+    cat['expl3']                = cat.CatcodeTableExpl
+    cat['expl']                 = cat.CatcodeTableExpl
+    cat['string']               = cat.CatcodeTableString
+    cat['other']                = cat.CatcodeTableOther
 end
-
 %    \end{macrocode}
 %
 %    \begin{macrocode}
 %</luamodule>
 %    \end{macrocode}
 %
+%    \section{Test files}
+%
+%    The tests done are very basic: we just make sure that the package loads
+%    correctly and the macros don't generate any error, under both LaTeX en
+%    Plain TeX.
+%
+%    \begin{macrocode}
+%<testplain>\input luatexbase-cctb.sty
+%<testlatex>\RequirePackage{luatexbase-cctb}
+%<*testplain,testlatex>
+\newluatexcatcodetable\testcctb
+\directlua{assert(luatextra.catcodetables.testcctb)}
+%    \end{macrocode}
+%
+%    Also check that the catcodetable's number is remembered well,
+%    independently of the current value of |\escapechar|.
+
+%    \begin{macrocode}
+\begingroup
+\escapechar64
+\newluatexcatcodetable\anothercctb
+\endgroup
+\directlua{assert(luatextra.catcodetables.anothercctb)}
+%    \end{macrocode}
+%
+%    Now, play a little bit with predefined tables.
+%
+%    \begin{macrocode}
+\luatexcatcodetable\CatcodeTableLaTeXAtLetter
+\ifnum\catcode64=11 \else \ERROR \fi
+\luatexcatcodetable\CatcodeTableLaTeX
+\ifnum\catcode64=12 \else \ERROR \fi
+%<testlatex>\documentclass{minimal}
+\directlua{%
+  tex.sprint('\string\\setbox0=\string\\hbox{')
+  tex.sprint(luatextra.catcodetables.string, "\string\\undef # _^&")
+  tex.sprint('}')
+  }
+%    \end{macrocode}
+%
+%    \begin{macrocode}
+%</testplain,testlatex>
+%<testplain>\bye
+%<testlatex>\stop
+%    \end{macrocode}
+%
 % \Finale
 \endinput