%D \module
%D   [       file=m-database,
%D        version=2006.04.23,
%D          title=\CONTEXT\ Modules,
%D       subtitle=Database Thingies,
%D         author=Hans Hagen\& Taco Hoekwater,
%D           date=\currentdate,
%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]

%C This module is part of the \CONTEXT\ macro||package and is
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
%C details.

\unprotect

% % % to be added to mult-* files
%
% % % todo: \dontcollectseparatedlist via k/v

\definesystemvariable{ls}

\def\c!first     {first}
\def\c!last      {last}
\def\c!quotechar {quotechar}

\let\@NX\noexpand

%D   {processquotedlist}
%D
%D An even more general list processing macro is the
%D following one:
%D
%D \starttyping
%D \processquotedlist{beginsym}{endsym}{separator}\docommand list
%D \stoptyping
%D
%D This one supports arbitrary open and close symbols as well
%D as user defined separators.
%D
%D \starttyping
%D \processquotedlist(){,}{"}\docommand(a=>b=>c=>d)
%D \stoptyping

\def\processquotedlist#1#2#3#4#5%
  {\def\csvquotechar{#4}%
   \edef\doconvertcsvquoteditem#4##1#4##2%
     {\@NX\ifx##2#3%
       \let\@NX\next\@NX\doconvertcsvlist
       \def\@NX\arg{#3}%
       \edef\@NX\temp{##1}%
       \@NX\expanded{\@NX#5{\@NX\temp}}%
     \@NX\else
       \let\@NX\next\@NX\redoconvertcsvquoteditem
       \def\@NX\arg{##1\@NX\csvquotechar}%
     \@NX\fi
     \@NX\expandafter\@NX\next\@NX\arg}%
   \edef\redoconvertcsvquoteditem##1#4##2%
     {\@NX\ifx##2#3%
       \let\@NX\next\@NX\doconvertcsvlist
       \def\@NX\arg{#3}%
       \edef\@NX\temp{##1}%
       \@NX\expanded{\@NX#5{\@NX\temp}}%
     \@NX\else
       \let\@NX\next\@NX\redoconvertcsvquoteditem
       \def\@NX\arg{##1\@NX\csvquotechar}%
     \@NX\fi
     \@NX\expandafter\@NX\next\@NX\arg}%
   \edef\doconvertcsvitem##1#3%
      {\edef\@NX\temp{##1}%
       \@NX\expanded{\@NX#5{\@NX\temp}}%
       \@NX\doconvertcsvlist#3}%
   \edef\doconvertcsvlist#3##1%
     {\@NX\ifx##1\@NX#2%
        \let\@NX\next \@NX\gobbleoneargument
      \@NX\else\@NX\ifx##1#4%
         \let\@NX\next \@NX\doconvertcsvquoteditem
      \@NX\else
         \let\@NX\next \@NX\doconvertcsvitem
      \@NX\fi\@NX\fi \@NX\next##1}%
   \doconvertcsvitem}

\gdef\doprocessseparatedquoteditem#1%
  {\appendseparatedlistparameter\c!left%
   \appendseparatedlistcontent{#1}%
   \appendseparatedlistparameter\c!right}

% a version more robust with regard to {a a} b c d situations:

\edef\detokenizedrelax{\detokenize{\relax}}

\edef\processseplistseparator{\detokenize{,}}%

\def\dodefineprocessseplist#1#2% separator \docommand
  {\edef\processseplistseparator{\detokenize{#1}}%
   \def\dodoprocessseplist##1##2#1%
     {\edef\!!stringa{\detokenize{##1}}%
      \ifx\detokenizedrelax\!!stringa
        \expandafter\nodoprocessseplist
    % \else\ifx\!!stringa#1%
      \else\ifx\!!stringa\processseplistseparator
        #2{}%
        #2{##2}%
        \expandafter\expandafter\expandafter\dodoprocessseplist
      \else
        #2{##1##2}%
        \expandafter\expandafter\expandafter\dodoprocessseplist
      \fi\fi}%
   \def\doprocessseplist##1\relax
     {\dodoprocessseplist##1#1\relax#1\relax\relax\end}}

\def\nodoprocessseplist#1\end
  {}

\long\def\processseplist#1#2#3\relax raw version
  {\dodefineprocessseplist{#1}{#2}%
   \dodoprocessseplist#3#1\relax#1\relax\relax\end}

% \dodefineprocessseplist{,}\test
% \dodoprocessseplist{,}a,b,c\relax,\relax\relax\end
% \doprocessseplista,b,c\relax

% \def\test#1{[#1]}
% \startlines
% \processseplist{,}\test ,2,,\relax
% \processseplist{,}\test ,,,44\relax
% \processseplist{,}\test ,,33,44\relax
% \processseplist{,}\test 11,,33,44\relax
% \processseplist{,}\test 1,2,3,4\relax
% \stoplines

\newtoks\separatedlistdata

\def\appendseparatedlistparameter#1%
  {\@EAEAEA\appendtoks\csname\??ls\currentseparatedlist#1\endcsname\to\separatedlistdata}

\def\appendseparatedlistcontent#1%
  {\appendtoks#1\to\separatedlistdata}

\def\flushseparatedlistdata
  {\the\separatedlistdata
   \separatedlistdata\emptytoks}

\def\initializeseparatedlistdata
  {\separatedlistdata{\egroup}}

\def\dontcollectseparatedlist
  {\def\dodoprocessseparatedfileline
     {\the\separatedlistdata
      \separatedlistdata\emptytoks
      \doprocessseparatedfileline}%
   \def\dodoprocessseparatedline
     {\the\separatedlistdata
      \separatedlistdata\emptytoks
      \doprocessseparatedline}%
   \let\flushseparatedlistdata\egroup
   \let\initializeseparatedlistdata\donothing}

\chardef\separatedlistmode\zerocount

\def\setcurrentlistseparator
  {\edef\currentlistseparator
     {\executeifdefined
        {\??ls::\csname\??ls\currentseparatedlist\c!separator\endcsname}%
        {\csname\??ls\currentseparatedlist\c!separator\endcsname}}%
   \doifvalue{\??ls\currentseparatedlist\c!separator}{tab}
     {\catcode`\^^I=12\relax}%
   \ifx\currentlistseparator\empty\def\currentlistseparator{,}\fi}

\bgroup \catcode`\^^I=12
    \setgvalue{\??ls::tab}{	}
    \setgvalue{\??ls::space}{ }
    \setgvalue{\??ls::comma}{,}
\egroup

\def\doprocessseparatedfileline
  {\ifeof\scratchread
     \ifcase\separatedlistmode\appendseparatedlistparameter\c!after\fi
     \immediate\closein\scratchread
     \expandafter\flushseparatedlistdata
   \else\ifx\line\empty
     % skip, can be comment
     \read\scratchread to\line
     \@EA\dodoprocessseparatedfileline
   \else
     \appendseparatedlistparameter{\ifcase\separatedlistmode\c!first\else\c!command\fi}%
     \ifx\currentlistquotechar\empty%
       \expandafter\doprocessseplist\line\relax
     \else
       \expanded{\processquotedlist{}{\noexpand\end}%
          {\currentlistseparator}{\currentlistquotechar}%
            \noexpand\doprocessseparatedquoteditem \line\currentlistseparator\noexpand\end}%
     \fi
     \ifcase\separatedlistmode\appendseparatedlistparameter\c!last\fi
     \read\scratchread to\line
     \@EAEAEA\dodoprocessseparatedfileline
   \fi\fi}

\def\dodoprocessseparatedfileline
  {\doprocessseparatedfileline}

\def\doprocessseparatedfile[#1][#2]%
  {\bgroup
   \edef\currentseparatedlist{#1}%
   \doifdefined{\??ls\currentseparatedlist\c!command}{\chardef\separatedlistmode\plusone}%
   \setcurrentlistseparator
   \edef\currentlistquotechar{\csname\??ls\currentseparatedlist\c!quotechar\endcsname}%
   \expandafter\dodefineprocessseplist\expandafter{\currentlistseparator}\doprocessseparateditem
   \initializeseparatedlistdata
   \directsetup{\currentseparatedlist:\executeifdefined{\??ls\currentseparatedlist\c!setups}\s!default}%
   \ifcase\separatedlistmode\appendseparatedlistparameter\c!before\fi
   \endlinechar\minusone
   \ignorelines
   \catcode`\#\@@comment
   \immediate\openin\scratchread=#2\relax % todo: \doopenin
   \read\scratchread to\line
   \doprocessseparatedfileline}

\def\dostartseparatedlist#1[#2]%
  {\bgroup
   \edef\currentseparatedlist{#2}%
   \doifdefined{\??ls\currentseparatedlist\c!command}{\chardef\separatedlistmode\plusone}%
   \obeylines
   \let#1\relax
   \def\separateslistend{#1}%
   \setcurrentlistseparator
   \edef\currentlistquotechar{\csname\??ls\currentseparatedlist\c!quotechar\endcsname}%
   \expandafter\dodefineprocessseplist\expandafter{\currentlistseparator}\doprocessseparateditem
   \directsetup{\currentseparatedlist:\executeifdefined{\??ls\currentseparatedlist\c!setups}\s!default}%
   \initializeseparatedlistdata
   \ifcase\separatedlistmode\appendseparatedlistparameter\c!before\fi
   \dodostartseparatedlist}

\def\redoprocessseparatedline#1%
  {\def\!!stringa{#1}%
   \ifx\!!stringa\separateslistend
     \ifcase\separatedlistmode\appendseparatedlistparameter\c!after\fi
     \expandafter\flushseparatedlistdata
   \else%
     \appendseparatedlistparameter{\ifcase\separatedlistmode\c!first\else\c!command\fi}%
     \ifx\currentlistquotechar\empty%
       \doprocessseplist#1\relax
     \else%
       \defconvertedargument\csvdata{#1}%
       \expanded{\processquotedlist{}{\noexpand\end}%
           {\currentlistseparator}{\currentlistquotechar}%
             \noexpand\doprocessseparatedquoteditem \csvdata\currentlistseparator\noexpand\end}%
     \fi
     \ifcase\separatedlistmode\appendseparatedlistparameter\c!last\fi
     \expandafter\dodoprocessseparatedline
   \fi}

\def\doprocessseparatedline
  {\doifnextbgroupelse\xdoprocessseparatedline\ydoprocessseparatedline}

\def\dodoprocessseparatedline
  {\doprocessseparatedline}

\def\doprocessseparateditem#1%
  {\ifcase\separatedlistmode
     \appendseparatedlistparameter\c!left
     \appendseparatedlistcontent{#1}%
     \appendseparatedlistparameter\c!right
   \else
     \appendseparatedlistcontent{{#1}}%
   \fi}


\bgroup  \obeylines

  \gdef\dodostartseparatedlist#1
    {\doprocessseparatedline}

  \gdef\xdoprocessseparatedline#1#2
    {\redoprocessseparatedline{{{#1}}#2}}

  \gdef\ydoprocessseparatedline#1
    {\redoprocessseparatedline{#1}}

\egroup

\startsetups CSV:unix
  \catcode`\#=\@@comment
\stopsetups

\def\defineseparatedlist
  {\dodoubleempty\dodefineseparatedlist}

\def\dodefineseparatedlist[#1][#2]%
  {\setvalue{\e!start#1}{\expandafter\dostartseparatedlist\csname\e!stop#1\endcsname[#1]}%
   \getparameters
     [\??ls#1]
     [\c!separator=,
      \c!quotechar=,
      \c!first=,
      \c!left=,
      \c!before=,
      \c!right=,
      \c!last=,
      \c!after=,
%       \c!command=,
      #2]}

\def\setupseparatedlist
  {\dodoubleempty\dosetupseparatedlist}

\def\dosetupseparatedlist[#1]% [#2]
  {\getparameter[\??ls#1]} % [#2]

\def\startseparatedlist[#1]%
  {\dostartseparatedlist\stopseparatedlist[#1]}

\def\processseparatedfile
  {\dodoubleargument\doprocessseparatedfile}

\protect \doifnotmode{demo}{\endinput}

\defineseparatedlist
  [CSV]
  [separator={,},
   first=\bTR,last=\eTR,
   left=\bTD,right=\eTD,
   before=\bTABLE,after=\eTABLE]

\startseparatedlist[CSV]
a,b,c
d,e,f
\stopseparatedlist

\startCSV
a,b,c
d,e,f
\stopCSV

\defineseparatedlist
  [CSV]
  [separator={,},
   quotechar={"},
   first=\NC,last=\NR,
   left=,right=\NC,
   before={\starttabulate[|l|l|l|]},after=\stoptabulate]

\startCSV
a,b,"c,d"
d,"""",f
\stopCSV

\defineseparatedlist
  [CSV]
  [separator={ },
   first=\NC,last=\NR,
   left=,right=\NC,
   before={\starttabulate[|l|l|l|]},after=\stoptabulate]

\startCSV
a b c
d e f
\stopCSV

\defineseparatedlist
  [CSV]
  [setups=unix,
   first=\NC,last=\NR,
   left=,right=\NC,
   before={\starttabulate[|l|l|l|]},after=\stoptabulate]

% \startsetups CSV:unix
%     \catcode`\#=\@@comment
% \stopsetups

%  %1,2,3
%  1,2,3
%  # 4,5,6
%  4,5,6

\processseparatedfile[CSV][test.dat]

\defineseparatedlist
  [CSVX]
  [command=\Whatever,
   separator={,},
   first=\bTR,last=\eTR,
   left=\bTD,right=\eTD,
   before=\bTABLE,after=\eTABLE]

\def\Whatever#1#2#3{[#1][#2][#3]\endgraf}

\startseparatedlist[CSVX]
a,b,c
d,e,f
\stopseparatedlist

\defineseparatedlist[CSV]
  [separator=comma,
   before=\bTABLE, after=\eTABLE,
   first=\bTR, last=\eTR,
   left=\bTD, right=\eTD]

\startCSV
a,b,c,č
d,e,f,š
\stopCSV

\enableregime[utf]

\defineseparatedlist[X][separator=X,left=(,right=),first=\endgraf,last=\endgraf]
\defineseparatedlist[Y][separator=Y,left=(,right=),first=\endgraf,last=\endgraf]

\startX
aXb
Xc
čXš
\stopX

\startY
aYb
Yc
čYš
\stopY

\stoptext