% \iffalse
%% File: l3toks.dtx Copyright (C) 1990-1998 LaTeX3 project
%
%<*dtx>
          \ProvidesFile{l3toks.dtx}
%</dtx>
%<package>\NeedsTeXFormat{LaTeX2e}
%<package>\ProvidesPackage{l3toks}
%<driver> \ProvidesFile{l3toks.drv}
% \fi
%         \ProvidesFile{l3toks.dtx}
          [1998/05/14 v1.0g L3 Experimental Token Registers]
%
% \iffalse
%<*driver>
\documentclass{l3doc}

\begin{document}
\DocInput{l3toks.dtx}
\end{document}
%</driver>
% \fi
%
%
% \GetFileInfo{l3toks.dtx}
% \title{The \textsf{l3toks} package\thanks{This file
%         has version number \fileversion, last
%         revised \filedate.}\\
% Token Registers}
% \author{\Team}
% \date{\filedate}
% \maketitle
%
% \section {Token Registers}
% 
% There is a second form beside token list pointers in which \LaTeX3
% stores token lists, namely the internal \TeX{} token registers.
% Functions dealing with these registers got the prefix "\toks_". Unlike
% token list pointers we have an accessing function as one can see
% below.
% 
% The main difference between <toks> (token registers) and <tlp> (token
% list pointers) is their behavior regarding expansion. While <tlp>'s
% expand fully (i.e., until only unexpandable tokens are left) inside
% an argument that is subject to expansion (i.e., denote by |x|)
% <toks>'s expand always only up to one level, i.e., passing their
% contents without further expansion.
% 
% \subsection{Functions}
% 
% \begin{function}{\toks_new:N}
% \begin{syntax}
%    "\toks_new:N" <toks>
% \end{syntax}
% Defines <toks> to be a new token list register.
% token list pointer.
% \begin{texnote}
% This is the \LaTeX3 allocation for what was called \tn{newtoks} in
% plain \TeX.
% \end{texnote}
% \end{function}
% 
% \begin{function}{%
%                  \toks_set:Nn |
%                  \toks_set:No |
%                  \toks_set:Nx |
%                  \toks_gset:Nn |
%                  \toks_gset:No |
%                  \toks_gset:Nx}
% \begin{syntax}
%   "\toks_set:Nn" <toks> "{"<token list>"}"
% \end{syntax}
% Defines <toks> to hold the token list <token list>. Global
% variants of this command assign the value globally the other variants
% expand the <token list> up to a certain level before the assignment
% or interpret the <token list> as a character list and form a control
% sequence out of it.
% \begin{texnote}
% "\toks_set:Nn" could have been specified in plain \TeX{} by <toks> "="
% "{" <token list> "}" but all other functions have no counterpart in
% plain \TeX. Additionally the functions above will check for correct
% local and global assignments, something that isn't available in plain
% \TeX.
% \end{texnote}
% \end{function}
% 
% \begin{function}{%
%                  \toks_gset_eq:NN |
% }
% \begin{syntax}
%   "\toks_gset_eq:NN" <toks1> <toks2>
% \end{syntax}
% The <toks1>  globally set to the value of <toks2>. Don't try to use
% "\toks_gset:Nn" for this purpose if the second argument is also a
% token register.
% \end{function}
% 
% \begin{function}{%
%                  \toks_clear:N |
%                  \toks_gclear:N |
% }
% \begin{syntax}
%   "\toks_clear:N" <toks>
% \end{syntax}
% The <toks> is locally or globally cleared.
% \end{function}
% 
% \begin{function}{%
%                  \toks_put_left:Nn |
%                  \toks_gput_left:Nn |
%                  \toks_put_right:Nn |
%                  \toks_gput_right:Nn |
%                  \toks_gput_right:Nx |
% }
% \begin{syntax}
%   "\toks_put_left:Nn" <toks> "{"<token list>"}"
% \end{syntax}
% These functions will append <token list> to the left or right of
% <toks>. Assignment is done either locally or globally. If possible
% append to the right since this operation is faster.
% \end{function}
% 
% \begin{function}{\toks_use:N}
% \begin{syntax}
%   "\toks_use:N" <toks>
% \end{syntax}
% Accesses the contents of <toks>. Contrary to token list pointers
% <toks> can't be access simply by calling them directly.
% \begin{texnote}
% Something like \tn{the} <toks>.
% \end{texnote}
% \end{function}
% 
% \begin{function}{\toks_use_clear:N |
%                  \toks_use_gclear:N}
% \begin{syntax}
%   "\toks_use_clear:N" <toks>
% \end{syntax}
% Accesses the contents of <toks> and clears (locally or globally) it
% afterwards. Actually the clearing operation is done in a way that does
% not prohibit the access of the following tokens in the input stream
% with functions stored in the token register. In other words this
% function is not exactly the same as calling "\toks_use:N" <toks>
% "\toks_clear:N" <toks> in sequence.
% \end{function}
% 
% \subsection{Predicates}
% 
% \begin{function}{\toks_empty:NTF}
% \begin{syntax}
%   "\toks_empty:NTF" <toks> "{"<true code>"}{"<false code>"}"
% \end{syntax}
% Tests if <toks> is empty.
% \end{function}
% 
% \subsection{Variable and constants}
% 
% \begin{variable}{\c_empty_toks}
% Constant that is always empty.
% \end{variable}
% 
% \begin{variable}{%
%                  \l_tmpa_toks |
%                  \l_tmpb_toks |
%                  \g_tmpa_toks |
%                  \g_tmpb_toks
% }
% Scratch register for immediate use. They are not used by conditionals
% or predicate functions.
% \end{variable}
% 
% \subsubsection{Internal functions}
% 
% \begin{function}{\toks_put_left_aux:w}
% Used by "\toks_put_left:Nn" and its variants.
% \end{function}
% 
% \begin{function}{\def_toks:D}
% Primitive function for defining a <cs> to correspond to a token
% register should not be used by a programmer.
% \begin{texnote}
% This function was named \tn{toksdef}.
% \end{texnote}
% \end{function}
% 
% \StopEventually{}
%
%    \section{The code}
%
%    We start by ensuring that the required packages are loaded.
%    We check for \texttt{l3expan} since this a basic package that is
%    essential for use of any higher-level package.
%    \begin{macrocode}
%<package&check>\RequirePackage{l3chk}\par
%<package>\RequirePackage{l3expan}\par
%<*package>
%    \end{macrocode}
%
% \begin{macro}{\toks_new:N}
%    Allocates a new token register. This function is already defined above.
%    \begin{macrocode}
\def_new:Npn \toks_new:N {}
\let:NN \toks_new:N \newtoks   % nick from LaTeX for the moment
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\toks_clear:N}
% \begin{macro}{\toks_gclear:N}
%    These functions clear a token register, either locally or globally.
%    \begin{macrocode}
\def_new:Npn \toks_clear:N #1{#1\c_empty_toks
%<*check>
   \chk_local_or_pref_global:N #1
%</check>
}
\def_new:Npn \toks_gclear:N {
%<*check>
   \pref_global_chk:
%</check>
%<-check> \pref_global:D
   \toks_clear:N}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\toks_use:N}
%    This function just returns the contents of a token register.
%    \begin{macrocode}
\let_new:NN \toks_use:N \the_internal:D
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\toks_use_clear:N}
% \begin{macro}{\toks_use_gclear:N}
%    These functions clear a token register (locally or globally) after
%    returning the contents.
%
%    They make sure that clearing the register does not
%    interfere with following tokens. In other words, the contents of
%    the register might operate on what follows in the input stream.
%    A direct implementation will save one |\exp_after:NN| but for the
%    sake of checking we do it this way now.
%    \begin{macrocode}
\def_new:Npn \toks_use_clear:N#1{
   \exp_after:NN
     \toks_clear:N
   \exp_after:NN
        #1
     \toks_use:N#1}
\def_new:Npn \toks_use_gclear:N{
%<*check>
   \pref_global_chk:
%</check>
%<-check> \pref_global:D
   \toks_use_clear:N}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\toks_put_left:Nn}
% \begin{macro}{\toks_gput_left:Nn}
%    |\toks_put_left:Nn |\zz{toks}\zv{stuff\/} adds the tokens of {\sl
%    stuff\/} on the `left-side' of the token register \m{toks}.
%    \begin{macrocode}
\def_new:Npn \toks_put_left:Nn #1{\exp_after:NN\toks_put_left_aux:w
                        \toks_use:N #1\q_stop #1}
\def_new:Npn \toks_gput_left:Nn {
%<*check>
   \pref_global_chk:
%</check>
%<-check> \pref_global:D
   \toks_put_left:Nn}
%    \end{macrocode}
%    A helper function for |\toks_put_left:Nn|. Its arguments are 
%    subsequently the tokens of \zv{stuff\/}, the token register
%    \zz{toks} and the current contents of \zz{toks}.
%    \begin{macrocode}
\def_long_new:Npn \toks_put_left_aux:w #1\q_stop #2#3{#2{#3#1}
%<*check>
     \chk_local_or_pref_global:N #2
%</check>
}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\toks_put_right:Nn}
% \begin{macro}{\toks_gput_right:Nn}
% \begin{macro}{\toks_gput_right:Nx}
%    These macros add a list of tokens to the right of a token register.
%    \begin{macrocode}
\def_long_new:Npn \toks_put_right:Nn #1#2{#1\exp_after:NN{\toks_use:N #1#2}
%<*check>
        \chk_local_or_pref_global:N #1
%</check>
}
\def_new:Npn \toks_gput_right:Nn {
%<*check>
   \pref_global_chk:
%</check>
%<-check> \pref_global:D
   \toks_put_right:Nn}
%    \end{macrocode}
%    |\toks_gput_right:Nx| expands its (second) argument.
%    \begin{macrocode}
\def_new:Npn \toks_gput_right:Nx {\exp_args:Nnx\toks_gput_right:Nn}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\toks_set:Nn}
% \begin{macro}{\toks_set:No}
% \begin{macro}{\toks_set:Nx}
%    |\toks_set:Nn|\m{toks}\m{stuff} stores \m{stuff} without expansion
%    in \m{toks}. |\toks_set:No| and |\toks_set:Nx| expand \m{stuff} once
%    and fully.
%    \begin{macrocode}
%<*check>
\def_new:Npn \toks_set:Nn #1{\chk_local:N #1#1}
%</check>
%    \end{macrocode}
%    If we don't check if \m{toks} is a local register then the 
%    |\toks_set:Nn| function has nothing to do.
%    \begin{macrocode}
%<-check> \let_new:NN \toks_set:Nn\use_noop:
\def_new:Npn \toks_set:No {\exp_args:NNo \toks_set:Nn}
\def_new:Npn \toks_set:Nx {\exp_args:NNx \toks_set:Nn}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\toks_gset:Nn}
% \begin{macro}{\toks_gset:No}
% \begin{macro}{\toks_gset:Nx}
%    These functions are the global variants of the above.
%    \begin{macrocode}
%<*check>
\def_new:Npn \toks_gset:Nn #1{\chk_global:N #1\pref_global:D#1}
%</check>
%<-check> \let_new:NN \toks_gset:Nn\pref_global:D
\def_new:Npn \toks_gset:No {\exp_args:NNo \toks_gset:Nn}
\def_new:Npn \toks_gset:Nx {\exp_args:NNx \toks_gset:Nn}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
%  \begin{macro}{\toks_set_eq:NN}
%  \begin{macro}{\toks_gset_eq:NN}
%    |\toks_set_eq:NN|\m{toks1}\m{toks2} copies the contents of \m{toks2}
%    in \m{toks1}.
%    \begin{macrocode}
%<*check>
\def_new:Npn\toks_set_eq:NN#1#2{
   \chk_local:N#1
   \chk_var_or_const:N#2
   #1#2}
\def_new:Npn\toks_gset_eq:NN#1#2{
   \chk_global:N#1
   \chk_var_or_const:N#2
   \pref_global:D#1#2}
%</check>
%<-check> \let_new:NN \toks_set_eq:NN  \use_noop:
%<-check> \let_new:NN \toks_gset_eq:NN \pref_global:D
%    \end{macrocode}
%  \end{macro}
%  \end{macro}
%
% \begin{macro}{\toks_empty:NTF}
%    |\toks_empty:NTF|\m{toks}\m{true code}\m{false code} tests if 
%    a token register is empty and executes either \m{true code} or 
%    \m{false code}.
%    \begin{macrocode}
\def_new:Npn\toks_empty:NTF#1{
   \tlp_set:No\l_testa_tlp{\toks_use:N#1}
   \tlp_empty:NTF\l_testa_tlp}
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\l_tmpa_toks}
% \begin{macro}{\l_tmpb_toks}
% \begin{macro}{\l_tmpc_toks}
% \begin{macro}{\g_tmpa_toks}
% \begin{macro}{\g_tmpb_toks}
% \begin{macro}{\g_tmpc_toks}
%    Some scratch register \ldots
%    \begin{macrocode}
\tex_toksdef:D \l_tmpa_toks = 255
\toks_new:N \l_tmpb_toks
\toks_new:N \l_tmpc_toks
\toks_new:N \g_tmpa_toks
\toks_new:N \g_tmpb_toks
\toks_new:N \g_tmpc_toks
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\c_empty_toks}
%    And here is a constant, which is a (permanently) empty token register.
%    \begin{macrocode}
\toks_new:N \c_empty_toks
%    \end{macrocode}
% \end{macro}
%
%    Show token usage:
%    \begin{macrocode}
%</package>
%<*showmemory>
\showMemUsage
%</showmemory>
%    \end{macrocode}