%D \module
%D   [       file=supp-lan,
%D        version=1997.03.20,
%D          title=\CONTEXT\ Support Macros,
%D       subtitle=Language Options,
%D         author=Hans Hagen,
%D           date=\currentdate,
%D      copyright={PRAGMA / Hans Hagen \& Ton Otten}]
%C
%C This module is part of the \CONTEXT\ macro||package and is
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
%C details.

%D This module needs a drastic update: can be made simplier
%D and faster; code can move to core module.

%D Also, commenting the previous versions code will clear up
%D some memory.

%D \gdef\starttest
%D   {\blank
%D    \noindent
%D    \halign\bgroup\tt##\hskip2em&##\hskip2em&##\cr}
%D
%D \gdef\stoptest
%D   {\egroup
%D    \blank}
%D
%D \gdef\test#1%
%D   {\convertargument#1\to\ascii\ascii&\hyphenatedword{#1}&#1\cr}

%D One of \TEX's strong points in building paragraphs is the way
%D hyphenations are handled. Although for real good hyphenation
%D of non||english languages some extensions to the program are
%D needed, fairly good results can be reached with the standard
%D mechanisms and an additional macro, at least in Dutch.

\unprotect

\writestatus{loading}{Context Support Macros / Language Options}

%D \CONTEXT\ originates in the wish to typeset educational
%D materials, especially in a technical environment. In
%D production oriented environments, a lot of compound words
%D are used. Because the Dutch language poses no limits on
%D combining words, we often favor putting dashes between those
%D words, because it facilitates reading, at least for those
%D who are not that accustomed to it.
%D
%D In \TEX\ compound words, separated by a hyphen, are not
%D hyphenated at all. In spite of the multiple pass paragraph
%D typesetting this can lead to parts of words sticking into
%D the margin. The solution lays in saying \type
%D {spoelwater||terugwinunit} instead of \type
%D {spoelwater-terugwinunit}. By using a one character command
%D like \type {|}, delimited by the same character \type {|},
%D we get ourselves both a decent visualization (in \TEXEDIT\
%D and colored verbatim we color these commands yellow) and an
%D efficient way of combining words.
%D
%D The sequence \type{||} simply leads to two words connected by
%D a hyphen. Because we want to distinguish such a hyphen from
%D the one inserted when \TEX\ hyphenates a word, we use a bit
%D longer one.
%D
%D \hyphenation {spoel-wa-ter te-rug-win-unit}
%D
%D \starttest
%D \test {spoelwater||terugwinunit}
%D \stoptest
%D
%D As we already said, the \type{|} is a command. This commands
%D accepts an optional argument before it's delimiter, which is
%D also a \type{|}.
%D
%D \hyphenation {po-ly-meer che-mie}
%D
%D \starttest
%D \test {polymeer|*|chemie}
%D \stoptest
%D
%D Arguments like \type{*} are not interpreted and inserted
%D directly, in contrary to arguments like:
%D
%D \starttest
%D \test {polymeer|~|chemie}
%D \test {|(|polymeer|)|chemie}
%D \test {polymeer|(|chemie|)| }
%D \stoptest
%D
%D Although such situations seldom occur |<|we typeset thousands
%D of pages before we encountered one that forced us to enhance
%D this mechanism|>| we also have to take care of comma's.
%D
%D \hyphenation {uit-stel-len}
%D
%D \starttest
%D \test {op||, in|| en uitstellen}
%D \stoptest
%D
%D The next special case (concerning quotes) was brought to my
%D attention by Piet Tutelaers, one of the driving forces
%D behind rebuilding hyphenation patterns for the dutch
%D language.\footnote{In 1996 the spelling of the dutch
%D language has been slightly reformed which made this topic
%D actual again.} We'll also take care of this case.
%D
%D \starttest
%D \test {AOW|'|er}
%D \test {cd|'|tje}
%D \test {ex|-|PTT|'|er}
%D \test {rock|-|'n|-|roller}
%D \stoptest
%D
%D Tobias Burnus pointed out that I should also support
%D something like
%D
%D \starttest
%D \test {well|_|known}
%D \stoptest
%D
%D to stress the compoundness of hyphenated words.
%D
%D Of course we also have to take care of the special case:
%D
%D \starttest
%D \test {text||color and ||font}
%D \stoptest

%D \macros
%D   {installdiscretionaries}
%D
%D The mechanism described here is one of the older inner parts
%D of \CONTEXT. The most recent extensions concerns some
%D special cases as well as the possibility to install other
%D characters as delimiters. The prefered way of specifying
%D compound words is using \type{||}, which is installed by:
%D
%D \starttyping
%D \installdiscretionaries || -
%D \stoptyping
%D
%D Some alternative definitions are:
%D
%D \startbuffer
%D \installdiscretionaries ** -
%D \installdiscretionaries ++ -
%D \installdiscretionaries // -
%D \installdiscretionaries ~~ -
%D \stopbuffer
%D
%D \typebuffer
%D
%D after which we can say:
%D
%D \bgroup
%D \getbuffer
%D \starttest
%D \test {test**test**test}
%D \test {test++test++test}
%D \test {test//test//test}
%D \test {test~~test~~test}
%D \stoptest
%D \egroup

%D \macros
%D   {compoundhyphen,
%D    beginofsubsentence,endofsubsentence}
%D
%D Now let's go to the macros. First we define some variables.
%D In the main \CONTEXT\ modules these can be tuned by a setup
%D command. Watch the (maybe) better looking compound hyphen.

% I've added \hbox's so that in mathmode we get proper chars

\def\compoundhyphen     {\hbox{-\kern-.25ex-}}
\def\beginofsubsentence {\hbox{---}}
\def\endofsubsentence   {\hbox{---}}

%D The last two variables are needed for subsentences
%D |<|like this one|>| which we did not yet mention.
%D
%D We want to enable breaking but at the same time don't want
%D compound characters like |-| or || to be separated from the
%D words. \TEX\ hackers will recognise the next two macro's:

\def\prewordbreak  {\penalty\plustenthousand\hskip\zeropoint\relax}
\def\postwordbreak {\penalty\zerocount\prewordbreak}

%D We first show the original implementation, which only
%D supports \type{|} as command and delimiter. Before
%D activating \type{|} we save it's value:
%D
%D \starttyping
%D \edef\domathmodediscretionary{\string|}
%D \stoptyping
%D
%D after which we're ready to define it's meaning to:
%D
%D \starttyping
%D \catcode`\|=\@@active
%D
%D \unexpanded\def|%
%D   {\ifmmode
%D      \expandafter\domathmodediscretionary
%D    \else
%D      \expandafter\dotextmodediscretionary
%D    \fi}
%D \stoptyping
%D
%D We need a two stage \type{\futurelet} because we want to
%D look ahead for both the compound character definition and
%D the (optional) comma that follows it, and because we want to
%D prevent that \TEX\ puts this comma on the next line. We use
%D \type{\next} for easy and fast checking of the argument, we
%D save this argument (which can consist of more tokens) and
%D also save the character following the \type{|#1|} in
%D \type{\nextnext}.
%D
%D \starttyping
%D \def\dotextmodediscretionary%
%D   {\bgroup
%D    \futurelet\next\dodotextmodediscretionary}
%D
%D \def\dodotextmodediscretionary#1|%
%D   {\def\betweendiscretionaries{#1}%
%D    \futurelet\nextnext\dododotextmodediscretionary}
%D \stoptyping
%D
%D The main macro consists of quite some \type{\ifx} tests
%D while \type{\checkafterdiscretionary} handles the commas.
%D We show the simplified version here:
%D
%D \starttyping
%D \def\dododotextmodediscretionary%
%D   {\let\nextnextnext=\egroup
%D    \ifx     |\next
%D      \checkafterdiscretionary
%D      \prewordbreak\hbox{\compoundhyphen\nextnext}\allowbreak\postwordbreak
%D    \else\ifx=\next
%D      \prewordbreak\compoundhyphen
%D    \else\ifx~\next
%D      \discretionary{-}{}{\thinspace}\postwordbreak
%D    \else\ifx(\next
%D      \prewordbreak\discretionary{}{(-}{(}\prewordbreak
%D    \else\ifx)\next
%D      \prewordbreak\discretionary{-)}{}{)}\prewordbreak
%D    \else\ifx'\next
%D      \prewordbreak\discretionary{-}{}{'}\postwordbreak
%D    \else
%D      \checkafterdiscretionary
%D      \prewordbreak\hbox{\betweendiscretionaries\nextnext}\allowbreak\postwordbreak
%D    \fi\fi\fi\fi\fi\fi
%D    \nextnextnext}
%D
%D \def\checkafterdiscretionary%
%D   {\ifx,\nextnext
%D      \def\nextnextnext{\afterassignment\egroup\let\next=}%
%D    \else
%D      \let\nextnext=\relax
%D    \fi}
%D \stoptyping
%D
%D Handling \type{(} and \type{)} is a a bit special, because
%D \TEX\ sees them as decent hyphenation points, according to
%D their \type{\lccode} being non||zero. For the same reason,
%D later on in this module we cannot manipulate the
%D \type{\lccode} but take the \type{\uccode}.

%D The most recent implementation is more advanced. As
%D demonstrated we can install delimiters, like:
%D
%D \starttyping
%D \installdiscretionaries || \compoundhyphen
%D \stoptyping
%D
%D This time we have to use a bit more clever way of saving the
%D math mode specification of the character we're going to
%D make active. We also save the user supplied compound hyphen.
%D We show the a bit more traditional implementation first.
%D
%D \starttyping
%D \def\installdiscretionaries#1%
%D   {\catcode`#1\@@other
%D    \expandafter\doinstalldiscretionaries\string#1}
%D
%D \def\doinstalldiscretionaries#1%
%D   {\setvalue{mathmodediscretionary#1}{#1}%
%D    \catcode`#1\@@active
%D    \dodoinstalldiscretionaries}
%D
%D \def\dodoinstalldiscretionaries#1#2%
%D   {\setvalue{textmodediscretionary\string#1}{#2}%
%D    \unexpanded\def#1{\discretionarycommand#1}}
%D \stoptyping
%D
%D A bit more \CATCODE\ and character trickery enables us to
%D discard the two intermediate steps. This trick originates
%D on page~394 of the \TEX book, in the appendix full of
%D dirty tricks. The second argument has now become redundant,
%D but I decided to reserve it for future use. At least it
%D remembers us of the symmetry.

\beginTEX

\def\installdiscretionaries#1#2#3%
  {\convertargument#1\to\ascii
   \setevalue{\strippedcsname\mathmodediscretionary\string#1}{\ascii}%
   \setvalue {\strippedcsname\textmodediscretionary\string#1}{#3}%
   \catcode`#1=\@@active
   \scratchcounter=\the\uccode`~
   \uccode`~=`#1
   \uppercase{\unexpanded\def~{\discretionarycommand~}}%
   \uccode`~=\scratchcounter}

\endTEX

\beginETEX \detokenize

\def\installdiscretionaries#1#2#3%
  {\setevalue{\strippedcsname\mathmodediscretionary\string#1}{\detokenize{#1}}%
   \setvalue {\strippedcsname\textmodediscretionary\string#1}{#3}%
   \catcode`#1=\@@active
   \scratchcounter=\the\uccode`~
   \uccode`~=`#1
   \uppercase{\unexpanded\def~{\discretionarycommand~}}%
   \uccode`~=\scratchcounter}

\endETEX

\def\dohandlemathmodebar#1%
  {\getvalue{\strippedcsname\mathmodediscretionary\string#1}}

\def\discretionarycommand
  {\relax\ifmmode
     \expandafter\dohandlemathmodebar
   \else
     \expandafter\dotextmodediscretionary
   \fi}

%D The next piece of code is a torture test for this previous
%D macro. The \type {\relax} before the \type {\ifmmode} is
%D needed because of the alignment scanner (in \ETEX\ this
%D problem is not present because there a protected macro is
%D not expanded. Thanks to Tobias Burnus for providing this
%D example.
%D
%D \startformula
%D   \left|f(x_n)-{1\over2}\right| =
%D      {\cases{|{1\over2}-x_n| &for $0\le x_n < {1\over2}$\cr
%D              |x_n-{1\over2}| &for ${1\over2}<x_n\le1$   \cr}}
%D \stopformula

%D \macros
%D   {defineactivecharacter}
%D
%D In \type {lang-ger.tex} we define some language specific
%D commands for german input, like:
%D
%D \starttyping
%D \startlanguagespecifics[de]
%D   \installcompoundcharacter "a {{\moveaccent{-.1ex}\"a\midworddiscretionary}}
%D \stoplanguagespecifics
%D \stoptyping
%D
%D Occasionally we have to redefine such an active character,
%D like in:
%D
%D \starttyping
%D \startencoding[pdfdoc]
%D   \startlanguagespecifics[de]
%D     \defineactivecharacter " {\"}
%D   \stoplanguagespecifics
%D \stopencoding
%D \stoptyping
%D
%D Here is the command we used:

% \def\next{#2}% \unexpanded\def\next is wrong in non||etex
% \uppercase{\let~\next}%

\def\defineactivecharacter #1 #2%
  {\catcode`#1=\@@active
   \scratchcounter=\the\uccode`~
   \uccode`~=`#1\relax
   \uppercase{\def\next{~}}%
  %\expandafter\unexpanded\expandafter\def\next{#2}%
   \expandafter\def\next{#2}% unexpanded goes wrong in pdfdoc
   \uccode`~=\scratchcounter}

%D The next alternative also supports character numbers:

\def\defineactivecharacter #1 #2%
  {\scratchcounter\the\uccode`~
   \expandafter\doifnumberelse\expandafter{\string#1}
     {\catcode #1=\@@active \uccode`~= #1\relax}
     {\catcode`#1=\@@active \uccode`~=`#1\relax}%
   \uppercase{\def\next{~}}%
  %\expandafter\unexpanded\expandafter\def\next{#2}%
   \expandafter\def\next{#2}% unexpanded goes wrong in pdfdoc
   \uccode`~\scratchcounter}

%D One reason for abusing the \type{~} is that it is already
%D an active character, so it is unlikely to appear as argument.

%D The next implementation is about 20\% faster; 1M
%D definitions on a 1Gig machine takes about 19 seconds
%D (against 24 for the previous alternative).

\chardef\activehackcode=`~

% \def\defineactivecharacter #1 %
%   {\scratchcounter\the\uccode\activehackcode
%    \uccode\activehackcode\expandafter\doifnumberelse
%      \expandafter{\string#1}\empty`#1%
%    \catcode\uccode\activehackcode\@@active
%    \uppercase{\def\next{~}}%
%    \uccode\activehackcode\scratchcounter
%    \expandafter\def\next}% unexpanded goes wrong in pdfdoc

%D But ...

\def\defineactivecharacter #1 #2%
  {\scratchcounter\uccode\activehackcode
   \uccode\activehackcode\expandafter\doifnumberelse
     \expandafter{\string#1}\empty`#1%
   \catcode\uccode\activehackcode\@@active
   \uppercase{\def\next{~}}%
   \uccode\activehackcode\scratchcounter
   % unexpanded goes wrong in pdfdoc
   \@EA\@EA\@EA\def\@EA\next\@EA{\@EA\dohandleactivecharacter\next{#2}}}

\let\dohandleactivecharacter\secondoftwoarguments

\def\donthandleactivecharacter#1#2{\noexpand#1}

%D A goody:

\def\makecharacteractive #1 %
  {\catcode`#1\active}

%D Although adapting character codes and making characters
%D active can interfere with other features of macropackages,
%D normally there should be no problems with things like:
%D
%D \starttyping
%D \installdiscretionary || +
%D \installdiscretionary ++ =
%D \stoptyping
%D
%D The real work is done by the next set of macros. We have
%D to use a double \type{\futurelet} because we have to take
%D following characters into account.

\def\dotextmodediscretionary#1%
  {\bgroup
   \def\dodotextmodediscretionary##1#1%
     {\def\betweendiscretionary{##1}%
      \futurelet\nextnext\dododotextmodediscretionary}%
   \let\discretionarycommand=#1%
   \def\textmodediscretionary%
     {\getvalue{\strippedcsname\textmodediscretionary\string#1}}%
   \futurelet\next\dodotextmodediscretionary}

\def\dododotextmodediscretionary
  {\let\nextnextnext\egroup
   \ifx\discretionarycommand\next
     \checkafterdiscretionary
     \bgroup
       \checkbeforediscretionary
       \prewordbreak\hbox{\textmodediscretionary\nextnext}\allowbreak\postwordbreak
     \egroup
   \else\ifx=\next
     \prewordbreak\textmodediscretionary
   \else\ifx~\next
     \prewordbreak\discretionary{-}{}{\thinspace}\postwordbreak
   \else\ifx_\next
     \prewordbreak\discretionary{\textmodediscretionary}
       {\textmodediscretionary}{\textmodediscretionary}\prewordbreak
   \else\ifx(\next
     \ifdim\lastskip>\zeropoint\relax
       (\prewordbreak
     \else
       \prewordbreak\discretionary{}{(-}{(}\prewordbreak
     \fi
   \else\ifx)\next
     \ifx\nextnext\blankspace
       \prewordbreak)\relax
     \else
       \prewordbreak\discretionary{-)}{}{)}\prewordbreak
     \fi
   \else\ifx'\next
     \prewordbreak\discretionary{-}{}{'}\postwordbreak
   \else\ifx<\next
     \beginofsubsentence\prewordbreak\beginofsubsentencespacing
   \else\ifnum\uccode`>=\nextuccode
     \endofsubsentencespacing\prewordbreak\endofsubsentence
   \else
     \checkafterdiscretionary
     \bgroup
       \checkbeforediscretionary
       \prewordbreak
       \discretionary{\hbox{\betweendiscretionary}}{}{\hbox{\betweendiscretionary}}%
       \allowbreak\postwordbreak
     \egroup
   \fi\fi\fi\fi\fi\fi\fi\fi\fi
   \nextnextnext}

\def\checkbeforediscretionary
  {\ifvmode\dontleavehmode\fi
   \ifhmode
     \begingroup
     \setbox\scratchbox\lastbox
     \ifdim\wd\scratchbox=\zeropoint
       \let\postwordbreak\prewordbreak
     \fi
     \box\scratchbox\relax
     \endgroup
  \fi}

\def\checkafterdiscretionary
  {\ifx,\nextnext
     \def\nextnextnext{\afterassignment\egroup\let\next=}%
   \else
     \let\nextnext\relax
   \fi}

%D The macro \type{\checkbeforediscretionary} takes care of
%D loners like \type{||word}, while it counterpart
%D \type{\checkafterdiscretionary} is responsible for handling
%D the comma.

%D \macros
%D   {beginofsubsentencespacing,endofsubsentencespacing}
%D
%D In the previous macros we provided two hooks which can be
%D used to support nested sub||sentences. In \CONTEXT\ these
%D hooks are used to insert a small space when needed.

\let\beginofsubsentencespacing=\relax
\let\endofsubsentencespacing  =\relax

%D Before we show some more tricky alternative, we first install
%D the mechanism:

\installdiscretionaries || \compoundhyphen

%D \macros
%D   {fakecompoundhyphen}
%D
%D In headers and footers as well as in active pieces of text
%D we need a dirty hack. Try to imagine what is needed to
%D savely break the next text across a line and at the same
%D time make the words interactive.
%D
%D \starttyping
%D \goto{Some||Long||Word}
%D \stoptyping

\def\currentspaceskip
  {\fontdimen2\font\!!plus\fontdimen3\font\!!minus\fontdimen4\font\relax}

% \def\fakecompoundhyphen% wrong
%   {\def|##1|{\compoundhyphen\nobreak\hskip-\currentspaceskip\allowbreak}}

\ifx\newsignal\undefined

   \let\fakecompoundhyphen\relax

\else

  \newsignal\compoundbreakpoint

  % \def\fakecompoundhyphen%
  %   {\def|##1|%
  %      {\doifelsenothing{##1}{\compoundhyphen}{##1}%
  %       \kern\compoundbreakpoint\allowbreak}}

  \def\fakecompoundhyphen
    {\def\|{\mathortext\vert\dofakecompoundhyphen}}

  \def\dofakecompoundhyphen
    {\def##1|%
       {\doifelsenothing{##1}\compoundhyphen{##1}%
        \kern\compoundbreakpoint\allowbreak}}

\fi

%D One of the drawbacks of this mechanism is that characters can
%D be made active afterwards. The next alternative can be used
%D in such situations.  This time we don't compare the arguments
%D directly but use the \type{\uccode}'s instead. \TEX\
%D initializes these codes of the alphabetics glyphs to their
%D uppercase counterparts. Normally the other characters remain
%D zero. If so, we can use the \type{\uccode} as a signal.

%D \macros
%D   {enableactivediscretionaries}
%D
%D The more advanced mechanism is activated by calling:
%D
%D \starttyping
%D \enableactivediscretionaries
%D \stoptyping
%D
%D which is defined as:

\def\enableactivediscretionaries
  {\uccode`'=`'\relax \uccode`~=`~\relax \uccode`_=`_\relax
   \uccode`(=`(\relax \uccode`)=`)\relax \uccode`==`=\relax
   \uccode`<=`<\relax \uccode`>=`>\relax
   \let\dotextmodediscretionary    \activedotextmodediscretionary
   \let\dododotextmodediscretionary\activedododotextmodediscretionary}

%D We only have to redefine two macros. While saving the
%D \type{\uccode} in a macro we have to take care of empty
%D arguments, like in \type{||}.

\def\activedotextmodediscretionary#1%
  {\bgroup
   \def\dodotextmodediscretionary##1#1%
     {\def\betweendiscretionary{##1}%
      \def\nextuccode####1####2\relax%
        {\ifcat\noexpand####1\noexpand\relax
           \edef\nextuccode{0}%
         \else
           \edef\nextuccode{\the\uccode`####1}%
         \fi}%
      \nextuccode##1@\relax
      \futurelet\nextnext\dododotextmodediscretionary}%
   \let\discretionarycommand=#1%
   \def\textmodediscretionary%
     {\getvalue{\strippedcsname\textmodediscretionary\string#1}}%
   \futurelet\next\dodotextmodediscretionary}

%D This time we use \type{\ifnum}:

\def\activedododotextmodediscretionary
  {\let\nextnextnext\egroup
   \ifx\discretionarycommand\next
     \checkafterdiscretionary
     \bgroup
       \checkbeforediscretionary
       \prewordbreak\hbox{\textmodediscretionary\nextnext}\allowbreak\postwordbreak
     \egroup
   \else\ifnum\uccode`==\nextuccode
     \prewordbreak\textmodediscretionary
   \else\ifnum\uccode`~=\nextuccode
     \prewordbreak\discretionary{-}{}{\thinspace}\postwordbreak
   \else\ifnum\uccode`_=\nextuccode
     \prewordbreak\discretionary{\textmodediscretionary}
       {\textmodediscretionary}{\textmodediscretionary}\prewordbreak
   \else\ifnum\uccode`(=\nextuccode
     \ifdim\lastskip>\zeropoint\relax
       (\prewordbreak
     \else
       \prewordbreak\discretionary{}{(-}{(}\prewordbreak
     \fi
   \else\ifnum\uccode`)=\nextuccode
     \ifx\nextnext\blankspace
       \prewordbreak)\relax
     \else
       \prewordbreak\discretionary{-)}{}{)}\prewordbreak
     \fi
   \else\ifnum\uccode`'=\nextuccode
     \prewordbreak\discretionary{-}{}{'}\postwordbreak
   \else\ifnum\uccode`<=\nextuccode
     \beginofsubsentence\prewordbreak\beginofsubsentencespacing
   \else\ifnum\uccode`>=\nextuccode
     \endofsubsentencespacing\prewordbreak\endofsubsentence
   \else
     \checkafterdiscretionary
     \bgroup
       \checkbeforediscretionary
      %\prewordbreak\hbox{\betweendiscretionary\nextnext}\allowbreak
       \prewordbreak
       \discretionary{\hbox{\betweendiscretionary}}{}{\hbox{\betweendiscretionary}}%
       \allowbreak\postwordbreak
     \egroup
   \fi\fi\fi\fi\fi\fi\fi\fi\fi
   \nextnextnext} % no lookahead in commands

%D Now we can safely do things like: \enableactivediscretionaries
%D
%D \starttyping
%D \catcode`<=\@@active  \def<{hello there}
%D \catcode`>=\@@active  \def>{hello there}
%D \catcode`(=\@@active  \def({hello there}
%D \catcode`)=\@@active  \def){hello there}
%D \stoptyping
%D
%D In normal day||to||day production of texts this kind of
%D activation is seldom used.\footnote{In the \CONTEXT\ manual
%D the \type{<} and \type{>} are made active and used for some
%D cross||reference trickery.} If so, we have to take care of
%D the math mode explicitly, just like we did when making
%D \type{|} active. It can be confusing too, especially when we
%D load macropackages afterwards that make use of \type{<} in
%D \type{\ifnum} or \type{\ifdim} statements.

%D And then came the Polish users, and who can deny them? Like
%D the German \TEX\ users demand an active \type {"}, some
%D Polish users like using the \type {/}. The next alternative
%D is a bit slower but far the most robust alternative. As a
%D bonus it also offers \type {|^|} and even \type {|||} which
%D both result in a breakable|^|bar.
%D
%D The trick we use here is to convert the argument to a
%D string and compare this string to string'd tokens.

\def\activedotextmodediscretionary#1%
  {\bgroup
   \let\nextnextnext\egroup
   \def\handlecompoundcharacter##1% new, needed for polish
     {\getvalue{\@nc@\string##1}}%  where / is active
   \def\next##1#1%
     {\def\next{\activedododotextmodediscretionary#1{##1}}%
      \futurelet\nextnext\next}%
   \next}

\def\activedododotextmodediscretionary#1#2%
  {\convertargument#2\to\discretionarytoken
   \def\textmodediscretionary
     {\getvalue{\strippedcsname\textmodediscretionary\string#1}}%
   \ifx#1\nextnext % takes care of ||| and +++ and ......
     \prewordbreak\discretionary{\hbox{$#1$}}{}{\hbox{$#1$}}%
       \allowbreak\postwordbreak
     \def\nextnextnext{\afterassignment\egroup\let\next=}%
   \else\ifx\discretionarytoken\empty
     \checkafterdiscretionary
     \bgroup
       \checkbeforediscretionary
       % this was:
       % \prewordbreak\hbox{\textmodediscretionary\nextnext}\allowbreak\postwordbreak
       % but an hbox blocks a possible \discretionary
       \prewordbreak\textmodediscretionary\nextnext\allowbreak\postwordbreak
     \egroup
   \else\convertargument=\to\next\ifx\next\discretionarytoken
     \prewordbreak\textmodediscretionary
   \else\convertargument~\to\next\ifx\next\discretionarytoken
     \prewordbreak\discretionary{-}{}{\thinspace}\postwordbreak
   \else\convertargument_\to\next\ifx\next\discretionarytoken
     \prewordbreak\discretionary{\textmodediscretionary}
       {\textmodediscretionary}{\textmodediscretionary}\prewordbreak
   \else\convertargument(\to\next\ifx\next\discretionarytoken
     \ifdim\lastskip>\zeropoint\relax
       (\prewordbreak
     \else
       \prewordbreak\discretionary{}{(-}{(}\prewordbreak
     \fi
   \else\convertargument)\to\next\ifx\next\discretionarytoken
     \ifx\nextnext\blankspace
       \prewordbreak)\relax
     \else\ifx\nextnext\space
       \prewordbreak)\relax
     \else
       \prewordbreak\discretionary{-)}{}{)}\prewordbreak
     \fi\fi
   \else\convertargument'\to\next\ifx\next\discretionarytoken
     \prewordbreak\discretionary{-}{}{'}\postwordbreak
   \else\convertargument<\to\next\ifx\next\discretionarytoken
     \beginofsubsentence\prewordbreak\beginofsubsentencespacing
   \else\convertargument>\to\next\ifx\next\discretionarytoken
     \endofsubsentencespacing\prewordbreak\endofsubsentence
   \else\convertargument^\to\next\ifx\next\discretionarytoken
     \prewordbreak\discretionary{\hbox{$|$}}{}{\hbox{$|$}}%
       \allowbreak\postwordbreak
   \else
     \checkafterdiscretionary
     \bgroup
       \checkbeforediscretionary
       \prewordbreak
       \discretionary{\hbox{#2}}{}{\hbox{#2}}%
       \allowbreak\postwordbreak
     \egroup
   \fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi
   \nextnextnext} % no lookahead in commands

%D Since most things in \CONTEXT\ are configurable, we
%D slightly change the previous definition so that we can
%D install new functionality outside this module. We also
%D support lookahead (over the egroup).

\def\@tmd@{@@tmd@@}

\def\activedododotextmodediscretionary#1#2%
  {\convertargument#2\to\discretionarytoken
   \def\textmodediscretionary%
     {\getvalue{\strippedcsname\textmodediscretionary\string#1}}%
   \ifx\discretionarytoken\empty
     \ifx#1\nextnext % takes care of ||| and +++ and ......
       \prewordbreak\discretionary{\hbox{$#1$}}{}{\hbox{$#1$}}%
       \allowbreak\postwordbreak
       \def\nextnextnext{\afterassignment\egroup\let\next=}%
     \else
       \checkafterdiscretionary
       \bgroup
         \checkbeforediscretionary
         % the next line has been changed (20050203)
         % \prewordbreak\hbox{\textmodediscretionary\nextnext}\allowbreak\postwordbreak
         % but an hbox blocks a possible \discretionary
         \prewordbreak\textmodediscretionary\nextnext\allowbreak\postwordbreak
       \egroup
     \fi
   \else\expandafter\ifx\csname\@tmd@\discretionarytoken\endcsname\relax
     \checkafterdiscretionary
     \bgroup
       \checkbeforediscretionary
       \prewordbreak
       \discretionary{\hbox{#2}}{}{\hbox{#2}}%
       \allowbreak\postwordbreak
     \egroup
   \else\ifx\nextnextnext\egroup % so we can properly do things afterward
     \@EA\egroup
     \@EA\let\@EA\nextnextnext\@EA\empty
     \csname\@tmd@\discretionarytoken\endcsname
   \else
     \csname\@tmd@\discretionarytoken\endcsname
   \fi\fi\fi
   \nextnextnext} % lookahead in commands

\def\definetextmodediscretionary #1
  {\convertargument#1\to\ascii
   \setvalue{\@tmd@\ascii}}

\definetextmodediscretionary {} % empty case, also handled in parser
  {\prewordbreak\compoundhyphen\allowbreak\postwordbreak}

\definetextmodediscretionary ~
  {\prewordbreak\discretionary{-}{}{\thinspace}\postwordbreak}

\definetextmodediscretionary _
  {\prewordbreak
   \discretionary{\compoundhyphen}{\compoundhyphen}{\compoundhyphen}%
   \prewordbreak}

\definetextmodediscretionary (
  {\ifdim\lastskip>\zeropoint
     (\prewordbreak
   \else
     \prewordbreak\discretionary{}{(-}{(}\prewordbreak
   \fi}

\definetextmodediscretionary )
  {\ifx\nextnext\blankspace
     \prewordbreak)\relax
   \else\ifx\nextnext\space
     \prewordbreak)\relax
   \else
     \prewordbreak\discretionary{-)}{}{)}\prewordbreak
   \fi\fi}

\definetextmodediscretionary '
  {\prewordbreak\discretionary{-}{}{'}\postwordbreak}

\definetextmodediscretionary <
  {\beginofsubsentence\prewordbreak\beginofsubsentencespacing}

\definetextmodediscretionary >
  {\endofsubsentencespacing\prewordbreak\endofsubsentence}

% \definetextmodediscretionary . % not yet definitive
%   {\prewordbreak\midsentence\prewordbreak}

\definetextmodediscretionary =
  {\prewordbreak\midsentence\prewordbreak} % {\prewordbreak\compoundhyphen}

\definetextmodediscretionary ^
  {\prewordbreak\discretionary{\hbox{$|$}}{}{\hbox{$|$}}%
   \allowbreak\postwordbreak}

% french

\definetextmodediscretionary :
  {\removeunwantedspaces\prewordbreak\kern\hspaceamount\empty{:}:}

\definetextmodediscretionary ;
  {\removeunwantedspaces\prewordbreak\kern\hspaceamount\empty{;};}

\definetextmodediscretionary ?
  {\removeunwantedspaces\prewordbreak\kern\hspaceamount\empty{?}?}

\definetextmodediscretionary !
  {\removeunwantedspaces\prewordbreak\kern\hspaceamount\empty{!}!}

\definetextmodediscretionary *
  {\prewordbreak\discretionary{-}{}{\kern.05em}\prewordbreak}

% spanish

\definetextmodediscretionary ??
  {\prewordbreak\questiondown}

\definetextmodediscretionary !!
  {\prewordbreak\exclamdown}

\ifx\hspaceamount\undefined

  \def\hspaceamount#1#2{\kern.16667em}

\fi

%D Since we don't have to bother about active characters any
%D longer, we end up with a pretty simple activating macro:

\def\enableactivediscretionaries
  {\let\dotextmodediscretionary=\activedotextmodediscretionary}

%D Done somewhere else: \type {\enableactivediscretionaries}.

%D \macros
%D   {directdiscretionary}
%D
%D In those situations where the nature of characters is
%D less predictable, we can use the more direct approach:

%D Beware: an \type {\ignorespaces} in a definition works
%D okay here, but not in the main mechanism because there
%D we have \type {\nextnextnext}.

\unexpanded\def\directdiscretionary#1%
  {\convertargument#1\to\discretionarytoken
   \let\textmodediscretionary\compoundhyphen
   \expandafter\ifx\csname\@tmd@\string#1\endcsname\relax
     \prewordbreak
     \discretionary{\hbox{#1}}{}{\hbox{#1}}%
     \allowbreak\postwordbreak
   \else
     \csname\@tmd@\string#1\endcsname
   \fi}

%D \macros
%D   {installcompoundcharacter}
%D
%D When Tobias Burnus started translating the dutch manual of
%D \PPCHTEX\ into german, he suggested to let \CONTEXT\ support
%D the \type{german.sty} method of handling compound
%D characters, especially the umlaut. This package is meant for
%D use with \PLAIN\ \TEX\ as well as \LATEX.
%D
%D I decided to implement compound character support as
%D versatile as possible. As a result one can define his own
%D compound character support, like:
%D
%D \starttyping
%D \installcompoundcharacter "a {\"a}
%D \installcompoundcharacter "e {\"e}
%D \installcompoundcharacter "i {\"i}
%D \installcompoundcharacter "u {\"u}
%D \installcompoundcharacter "o {\"o}
%D \installcompoundcharacter "s {\SS}
%D \stoptyping
%D
%D or even
%D
%D \starttyping
%D \installcompoundcharacter "ck {\discretionary {k-}{k}{ck}}
%D \installcompoundcharacter "ff {\discretionary{ff-}{f}{ff}}
%D \stoptyping
%D
%D The support is not limited to alphabetic characters, so the
%D next definition is also valid.
%D
%D \starttyping
%D \installcompoundcharacter ". {.\doifnextcharelse{\spacetoken}{}{\kern.125em}}
%D \stoptyping
%D
%D The implementation looks familiar and uses the same tricks as
%D mentioned earlier in this module. We take care of two
%D arguments, which complicates things a bit.

\def\@nc@{@nc@} % normal character
\def\@nn@{@nn@} % normal catcode
\def\@cc@{@cc@} % compound character
\def\@cs@{@cs@} % compound characters

% \def\installcompoundcharacter #1#2#3 #4% {#4} no grouping
%   {\setvalue{\@nc@\string#1}{\char`#1}%
%    \ifnum\catcode`#1=\@@active \else
%      \setevalue{\@nn@\string#1}{\number\catcode`#1}% new
%    \fi
%    \def\!!stringa{#3}%
%    \ifx\!!stringa\empty
%      \setvalue{\@cc@\string#1\string#2}{#4}%
%    \else
%      \setvalue{\@cs@\string#1\string#2\string#3}{#4}%
%    \fi
%    \catcode`#1=\@@active
%    \scratchcounter=\the\uccode`~
%    \uccode`~=`#1
%    \uppercase{\unexpanded\def~{\handlecompoundcharacter~}}%
%    \uccode`~=\scratchcounter}

\def\installcompoundcharacter #1#2#3 #4% {#4} no grouping
  {\chardef\thecompoundcharacter`#1%
  %\@EA\def\csname\@nc@\string#1\endcsname{\char`#1}%
   \@EA\chardef\csname\@nc@\string#1\endcsname\thecompoundcharacter
   \ifnum\catcode\thecompoundcharacter=\@@active \else
     \@EA\edef\csname\@nn@\string#1\endcsname
       {\number\catcode\thecompoundcharacter}%
   \fi
   \def\!!stringa{#3}%
   \@EA\def\csname\ifx\!!stringa\empty
     \@cc@\string#1\string#2%
   \else
     \@cs@\string#1\string#2\string#3%
   \fi\endcsname{#4}%
   \catcode\thecompoundcharacter\@@active
   \scratchcounter\uccode\activehackcode
   \uccode\activehackcode\thecompoundcharacter
   \uppercase{\unexpanded\def~{\handlecompoundcharacter~}}%
   \uccode\activehackcode\scratchcounter}

%D A compound character can be reset with the following
%D command.
%D
%D \starttyping
%D \restorecompoundcharacter /
%D \stoptyping

\def\restorecompoundcharacter#1% new
  {\catcode`#1=\csname\@nn@\string#1\endcsname\relax}

%D We can also ignore definitions (needed in for instance \XML). Beware,
%D this macro is supposed to be used grouped!

\def\ignorecompoundcharacter
  {\def\installcompoundcharacter##1 ##2{}}

%D In handling the compound characters we have to take care of
%D \type{\bgroup} and \type{\egroup} tokens, so we end up with
%D a multi||step interpretation macro. We look ahead for a
%D \type{\bgroup}, \type{\egroup} or \type{\blankspace}. Being
%D no user of this mechanism, the credits for testing them goes
%D to Tobias Burnus, the first german user of \CONTEXT.
%D
%D We define these macros as \type{\long} because we can
%D expect \type{\par} tokens. We need to look into the future
%D with \type{\futurelet} to prevent spaces from
%D disappearing.

% \def\handlecompoundcharacter#1%
%   {\def\dohandlecompoundcharacter%
%      {\ifx\next\bgroup
%        %\def\next{\dodohandlecompoundcharacter#1}%    % handle "{ee} -> \"ee
%        %\let\next\relax                               % forget "{ee} -> ee
%         \def\next{\handlecompoundcharacterone#1}%     % ignore "{ee} -> "ee
%       \else\ifx\next\egroup
%         \def\next{\getvalue{\@nc@\string#1}}%
%       \else\ifx\next\blankspace
%         \def\next{\getvalue{\@nc@\string#1}}%
%       \else
%         \def\next{\dodohandlecompoundcharacter#1}%
%       \fi\fi\fi
%       \next}%
%    \futurelet\next\dohandlecompoundcharacter}
%
% \def\dodohandlecompoundcharacter#1#2%
%   {\def\dododohandlecompoundcharacter% Keep it here and
%      {\ifx\next\bgroup               % preserve spaces!
%         \def\next{\handlecompoundcharacterone#1#2}%
%       \else\ifx\next\egroup
%         \def\next{\handlecompoundcharacterone#1#2}%
%       \else\ifx\next\blankspace
%         \def\next{\handlecompoundcharacterone#1#2}%
%       \else
%         \def\next{\handlecompoundcharactertwo#1#2}%
%       \fi\fi\fi
%       \next}%
%    \futurelet\next\dododohandlecompoundcharacter}

\def\handlecompoundcharacter#1%
  {\def\xhandlecompoundcharacter{\dohandlecompoundcharacter{#1}}%
   \futurelet\next\xhandlecompoundcharacter}

\def\dohandlecompoundcharacter
  {\ifx\next\bgroup
    %\@EA\dodohandlecompoundcharacter % handle "{ee} -> \"ee
    %\@EA\gobbleoneargument           % forget "{ee} -> ee
     \@EA\handlecompoundcharacterone  % ignore "{ee} -> "ee
   \else\ifx\next\egroup
     \@EAEAEA\donohandlecompoundcharacter
   \else\ifx\next\blankspace
     \@EA\@EAEAEA\@EA\donohandlecompoundcharacter
   \else
     \@EA\@EAEAEA\@EA\dodohandlecompoundcharacter
   \fi\fi\fi}

\def\donohandlecompoundcharacter#1{\csname\@nc@\string#1\endcsname}

\def\dododohandlecompoundcharacter
  {\ifx\next\bgroup
     \@EA\handlecompoundcharacterone
   \else\ifx\next\egroup
     \@EAEAEA\handlecompoundcharacterone
   \else\ifx\next\blankspace
     \@EA\@EAEAEA\@EA\handlecompoundcharacterone
   \else
     \@EA\@EAEAEA\@EA\handlecompoundcharactertwo
   \fi\fi\fi}

\def\dodohandlecompoundcharacter#1#2% preserve space
  {\def\xdodohandlecompoundcharacter{\dododohandlecompoundcharacter#1#2}%
   \futurelet\next\xdodohandlecompoundcharacter}

%D Besides taken care of the grouping and space tokens, we have
%D to deal with three situations. First we look if the next
%D character equals the first one, if so, then we just insert
%D the original. Next we look if indeed a compound character is
%D defined. We either execute the compound character or just
%D insert the first. So we have
%D
%D \starttyping
%D <key><key>  <key><known>  <key><unknown>
%D \stoptyping
%D
%D In later modules we will see how these commands are used.

\beginTEX

\long\def\handlecompoundcharacterone#1#2%
  {\if\string#1\string#2% was: \ifx#1#2%
    %\def\next{\getvalue{\@nc@\string#1}\getvalue{\@nc@\string#2}}%
     \def\next{\getvalue{\@nc@\string#1}}%
   \else\expandafter\ifx\csname\@cc@\string#1\string#2\endcsname\relax
     \def\next{\getvalue{\@nc@\string#1}#2}%
   \else
     \def\next{\getvalue{\@cc@\string#1\string#2}}%
   \fi\fi
   \next}

\long\def\handlecompoundcharactertwo#1#2#3%
  {\if\string#1\string#2% was: \ifx#1#2%
    %\def\next{\getvalue{\@nc@\string#1}\getvalue{\@nc@\string#2}#3}%
     \def\next{\getvalue{\@nc@\string#1}#3}%
   \else\@EA\ifx\csname\@cs@\string#1\string#2\string#3\endcsname\relax
     \expandafter\ifx\csname\@cc@\string#1\string#2\endcsname\relax
       \def\next{\getvalue{\@nc@\string#1}#2#3}%
     \else
       \def\next{\getvalue{\@cc@\string#1\string#2}#3}%
     \fi
   \else
     \def\next{\getvalue{\@cs@\string#1\string#2\string#3}}%
   \fi\fi
   \next}

\endTEX

\beginETEX \ifcsname

\long\def\handlecompoundcharacterone#1#2%
  {\if\string#1\string#2% was: \ifx#1#2%
    %\def\next{\getvalue{\@nc@\string#1}\getvalue{\@nc@\string#2}}%
     \def\next{\getvalue{\@nc@\string#1}}%
   \else\ifcsname\@cc@\string#1\string#2\endcsname
     \def\next{\getvalue{\@cc@\string#1\string#2}}%
   \else
     \def\next{\getvalue{\@nc@\string#1}#2}%
   \fi\fi
   \next}

\long\def\handlecompoundcharactertwo#1#2#3%
  {\if\string#1\string#2% was: \ifx#1#2%
    %\def\next{\getvalue{\@nc@\string#1}\getvalue{\@nc@\string#2}#3}%
     \def\next{\getvalue{\@nc@\string#1}#3}%
   \else\ifcsname\@cs@\string#1\string#2\string#3\endcsname
     \def\next{\getvalue{\@cs@\string#1\string#2\string#3}}%
   \else\ifcsname\@cc@\string#1\string#2\endcsname
     \def\next{\getvalue{\@cc@\string#1\string#2}#3}%
   \else
     \def\next{\getvalue{\@nc@\string#1}#2#3}%
   \fi\fi\fi
   \next}

\endETEX

%D For very obscure applications (see for an application \type
%D {lang-sla.tex}) we provide:

\def\dosimplifiedcompoundcharacter#1%
  {#1}

\beginTEX

\def\simplifiedcompoundcharacter#1#2%
  {\@EA\ifx\csname\@cc@\string#1\string#2\endcsname\relax
     #2%
   \else
     \@EA\@EA\@EA\dosimplifiedcompoundcharacter\csname\@cc@\string#1\string#2\endcsname
   \fi}

\endTEX

\beginETEX \ifcsname

\def\simplifiedcompoundcharacter#1#2%
  {\ifcsname\@cc@\string#1\string#2\endcsname
     \@EA\@EA\@EA\dosimplifiedcompoundcharacter\csname\@cc@\string#1\string#2\endcsname
   \else
     #2%
   \fi}

\endETEX

%D \macros
%D   {nonbreakablespace}
%D
%D The following macro is taken from plain \TEX.

\def\nonbreakablespace{\penalty\plustenthousand\ }

\let~\nonbreakablespace

% under testing:
%
% \unexpanded\def~{\nonbreakablespace}

%D \macros
%D   {midworddiscretionary}
%D
%D If needed, one can add a discretionary hyphen using \type
%D {\midworddiscretionary}. This macro does the same as
%D \PLAIN\ \TEX's \type {\-}, but, like the ones implemented
%D earlier, this one also looks ahead for spaces and grouping
%D tokens.

\def\domidworddiscretionary
  {\ifx\next\blankspace\else
   \ifx\next\bgroup    \else
   \ifx\next\egroup    \else
     \discretionary{-}{}{}%
   \fi\fi\fi}

\def\midworddiscretionary%
  {\futurelet\next\domidworddiscretionary}

%D \macros
%D   {hyphenatedurl}
%D
%D For those who want to put full \URL's in a text, we offer
%D
%D \startbuffer
%D \hyphenatedurl{http://optimist.optimist/optimist/optimist.optimist#optimist}
%D \stopbuffer
%D
%D \typebuffer
%D
%D which breaks at the appropriate places. Watch the \type{#}
%D hack.
%D
%D When passed as argument, like in \type {\goto}, one needs
%D to substitute a \type {\\} for each \type{#}.
%D
%D \startbuffer
%D \hyphenatedurl{http://this.is.a.rather/strange/reference#indeed}
%D \stopbuffer
%D
%D \typebuffer

\ifx\\\undefined \let\\\crlf \fi

\chardef\urlsplitmode=1

% 0 => don't split
% 1 => . : na, rest voor
% 2 => alles na
% 3 => alles voor

% \bgroup \catcode`\~=\active \catcode`\/=\active

% Why not convert to ascii first? I will redo this one!

% \unexpanded\gdef\hyphenatedurl#1% {}{} handles accents
%   {\bgroup
%    \obeyhyphens
%    \def\splitbefore##1%
%      {\setbox\scratchbox=\hbox{##1{}{}}%
%       \ifcase\urlsplitmode
%         \box\scratchbox
%       \or
%         \postwordbreak\box\scratchbox\prewordbreak
%       \or
%         \prewordbreak\discretionary{\box\scratchbox}{}{\box\scratchbox}\prewordbreak
%       \else
%         \postwordbreak\box\scratchbox\prewordbreak
%       \fi}%
%    \def\splitafter##1%
%      {\ifcase\urlsplitmode
%         ##1{}{}%
%       \or
%         \prewordbreak\discretionary{##1{}{}}{}{##1{}{}}\prewordbreak
%       \or
%         \prewordbreak\discretionary{##1{}{}}{}{##1{}{}}\prewordbreak
%       \else
%         \prewordbreak\discretionary{}{##1{}{}}{##1{}{}}\prewordbreak
%       \fi}%
%    \def\flushurl%
%      {\savedurl\let\savedurl\empty}%
%    \def\\%
%      {\spliturl\#}%
%    \let\~=\lettertilde \let~=\~%
%    \let\/=\letterslash \let/=\/%
%    \let\savedurl\empty
%    \handletokens#1\with\scanurl
%    \egroup}
%
% Better (a mere copy with \dohyphens):

% \bgroup \catcode`\~=\active \catcode`\/=\active
%
% \unexpanded\gdef\hyphenatedurl#1% {}{} handles accents
%   {\bgroup
%    \ifnum\hyphenpenalty<10000 \else
%      \def\discretionary##1##2##3{##1\allowbreak##2}%
%    \fi
%    \obeyhyphens
%    \def\splitbefore##1%
%      {\setbox\scratchbox=\hbox{##1{}{}}%
%       \ifcase\urlsplitmode
%         \box\scratchbox
%       \or
%         \postwordbreak\box\scratchbox\prewordbreak
%       \or
%         \prewordbreak\discretionary{\box\scratchbox}{}{\box\scratchbox}\prewordbreak
%       \else
%         \postwordbreak\box\scratchbox\prewordbreak
%       \fi}%
%    \def\splitafter##1%
%      {\ifcase\urlsplitmode
%         ##1{}{}%
%       \or
%         \prewordbreak\discretionary{##1{}{}}{}{##1{}{}}\prewordbreak
%       \or
%         \prewordbreak\discretionary{##1{}{}}{}{##1{}{}}\prewordbreak
%       \else
%         \prewordbreak\discretionary{}{##1{}{}}{##1{}{}}\prewordbreak
%       \fi}%
%    \def\flushurl%
%      {\savedurl\let\savedurl\empty}%
%    \def\\%
%      {\spliturl\#}%
%    \let\~=\lettertilde\let~=\~%
%    \let\/=\letterslash\let/=\/%
%    \let\savedurl\empty
%    \handletokens#1\with\scanurl\savedurl
%    \egroup}
%
% \egroup

\bgroup \catcode`\~=\active \catcode`\/=\active

\unexpanded\gdef\hyphenatedurl#1% {}{} handles accents
  {\bgroup
   \ifnum\hyphenpenalty<10000 \else
     \def\discretionary##1##2##3{##1\allowbreak##2}%
   \fi
   \obeyhyphens
   \def\splitbefore##1%
     {\setbox\scratchbox=\hbox{##1{}{}}%
      \ifcase\urlsplitmode
        \box\scratchbox
      \or
        \postwordbreak\box\scratchbox\prewordbreak
      \or
        \prewordbreak\discretionary{\box\scratchbox}{}{\box\scratchbox}\prewordbreak
      \else
        \postwordbreak\box\scratchbox\prewordbreak
      \fi}%
   \def\splitafter##1%
     {\ifcase\urlsplitmode
        ##1{}{}%
      \or
        \prewordbreak\discretionary{##1{}{}}{}{##1{}{}}\prewordbreak
      \or
        \prewordbreak\discretionary{##1{}{}}{}{##1{}{}}\prewordbreak
      \else
        \prewordbreak\discretionary{}{##1{}{}}{##1{}{}}\prewordbreak
      \fi}%
   \def\splitanyway##1%
     {\prewordbreak##1\prewordbreak}%
   \def\flushurl%
     {\savedurl\let\savedurl\empty}%
   \def\\%
     {\spliturl\#}%
   \let\~=\lettertilde\let~=\~%
   \let\/=\letterslash\let/=\/%
   \let\savedurl\empty
   \scratchcounter\zerocount % used for hyphenmethod
   \handletokens#1\with\scanurl\savedurl
   \egroup}

\egroup

%D This would be better, but it spoils \type {\~} and so:
%D
%D \starttyping
%D \convertargument#1\to\ascii
%D \expandafter\handletokens\ascii\with\scanurl
%D \stoptyping

% \def\scanurl#1%
%   {\ifx#1\~%
%      \flushurl\splitbefore\~%
%    \else\ifx#1\#%
%      \flushurl\splitbefore\#%
%    \else\ifx#1\&%
%      \flushurl\splitbefore\&%
%    \else\ifx#1\%%
%      \flushurl\splitbefore\%%
%    \else\ifx#1\_%
%      \flushurl\splitbefore\_%
%    \else\if\noexpand#1\relax
%      #1%
%    \else\ifnum\catcode`#1=8
%      \flushurl\splitbefore\_%
%    \else\ifnum\catcode`#1=6
%      \flushurl\splitbefore\#%
%    \else\ifnum\catcode`#1=4
%      \flushurl\splitbefore\&%
%    \else\if#1\lettertilde
%      \flushurl\splitbefore\~%
%    \else\if#1\letterpercent
%      \flushurl\splitbefore\%%
%    \else\if#1\letterunderscore
%      \flushurl\splitbefore\_%
%    \else\if#1\letterquestionmark
%      \flushurl\splitafter\letterquestionmark
%    \else\if#1\letterat
%      \flushurl\splitafter\letterat
%    \else\if#1\letterslash
%      \edef\savedurl{\savedurl\letterslash}%
%    \else\if#1+%
%      \flushurl\splitafter+%
%    \else\if#1:%
%      \flushurl\splitafter:%
%    \else\if#1.%
%      \flushurl\splitafter.%
%    \else\if#1(%
%      \flushurl\splitbefore(%
%    \else\if#1)%
%      \flushurl\splitafter)%
%    \else
%      \ifx\savedurl\empty\else
%        \splitbefore\savedurl
%        \let\savedurl\empty
%      \fi
%      #1%
%    \fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi}

\chardef\urlhyphenmethod=0

\def\scanurl#1%
  {\advance\scratchcounter\plusone
   \ifx#1\blankspace
     \flushurl\splitanyway\normalspace
   \else\ifx#1\ %
     \flushurl\splitanyway\normalspace
   \else\ifx#1\space
     \flushurl\splitanyway\normalspace
   \else\ifx#1\~%
     \flushurl\splitbefore\~%
   \else\ifx#1\#%
     \flushurl\splitbefore\#%
   \else\ifx#1\&%
     \flushurl\splitbefore\&%
   \else\ifx#1\%%
     \flushurl\splitbefore\%%
   \else\ifx#1\_%
     \flushurl\splitbefore\_%
   \else\if\noexpand#1\relax
     #1%
   \else\ifnum\catcode`#1=8
     \flushurl\splitbefore\_%
   \else\ifnum\catcode`#1=6
     \flushurl\splitbefore\#%
   \else\ifnum\catcode`#1=4
     \flushurl\splitbefore\&%
   \else\if#1\lettertilde
     \flushurl\splitbefore\~%
   \else\if#1\letterpercent
     \flushurl\splitbefore\%%
   \else\if#1\letterunderscore
     \flushurl\splitbefore\_%
   \else\if#1\letterquestionmark
     \flushurl\splitafter\letterquestionmark
   \else\if#1\letterat
     \flushurl\splitafter\letterat
   \else\if#1\letterslash
     \edef\savedurl{\savedurl\letterslash}%
   \else\if#1+%
     \flushurl\splitafter+%
   \else\if#1:%
     \flushurl\splitafter:%
   \else\if#1.%
     \flushurl\splitafter.%
   \else\if#1(%
     \flushurl\splitbefore(%
   \else\if#1)%
     \flushurl\splitafter)%
   \else
     \ifx\savedurl\empty\else
       \splitbefore\savedurl
       \let\savedurl\empty
     \fi
     \ifcase\urlhyphenmethod
       #1%
     \else
       \ifnum\scratchcounter>\plusthree % so, \http: will not break
         \edef\savedurl{#1}%
       \else
         #1%
       \fi
     \fi
   \fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi\fi}

% \setupinteraction[state=start]
% \def\gotoURL#1{\useURL[foo][#1]\goto{\url[foo]}[url(foo)]}
% \starttext
% \endgraf \chardef\urlhyphenmethod=0
% \hsize1pt\gotoURL{http://www.physik.fu-berlin.de/SomeVeryVeryVeryLongDirectory/And/AQuiteLongFileName.html}
% \endgraf \chardef\urlhyphenmethod=1
% \hsize1pt\gotoURL{http://www.physik.fu-berlin.de/SomeVeryVeryVeryLongDirectory/And/AQuiteLongFileName.html}
% \stoptext

%D When Joop Susan asked (on the \CONTEXT\ mailing list) how
%D to handle url's passed as argument, the following solutions
%D came to my mind:
%D
%D \starttyping
%D \def\whateverurl#1%
%D   {{\def~{\string~}\useURL[dummy][#1]\goto{\url[dummy]}[URL(dummy)]}}
%D
%D \def\whateverurl#1%
%D   {{\let~\lettertilde\useURL[dummy][#1]\goto{\url[dummy]}[URL(dummy)]}}
%D
%D \def\whateverurl#1%
%D   {\convertargument#1\to\ascii
%D    \expanded{\useURL[dummy][\ascii]}\goto{\url[dummy]}[URL(dummy)]}
%D \stoptyping

%D \macros
%D   {hyphenatedfile}
%D
%D For the moment we treat filenames in a similar way,
%D
%D \starttyping
%D \hyphenatedfile{here/there/filename.suffix}
%D \stoptyping

\let\hyphenatedfile\hyphenatedurl

% to be finished
%
% \def\hyphenatedstring#1%
%   {\bgroup
%    \nohyphens
%    \def\next##1{##1\doif{##1}{-}{\allowbreak}}%
%    \handletokens#1\with\next
%    \egroup}
%
% {\hsize1cm\hyphenatedstring{ABXXXXXXXXXXC-12345-12345}}

%D \macros
%D   {disablediscretionaries,disablecompoundcharacter}
%D
%D Occasionally we need to disable this mechanism. For the
%D moment we assume that \type {|} is used.

\def\disablediscretionaries
  {\def|##1|{\string##1}%
   \def\directdiscretionary##1{\string##1}}

\def\disablecompoundcharacters
  {\let\handlecompoundcharacter\string}

\protect \endinput