summaryrefslogtreecommitdiff
path: root/macros/latex/contrib/tokenizer/tokenizer.tex
blob: c8f628776080b4050848f9463b82a4eaf097e0aa (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
% Copyright 2003 Sascha Herpers
%
% This file can be redistributed and/or modified under the terms
% of the LaTeX Project Public License Distributed from CTAN
% archives in directory macros/latex/base/lppl.txt; either
% version 1 of the License, or any later version.
%
%
% 2003-05-22
% Sascha Herpers
% herpers@wiso.uni-koeln.de
\documentclass[draft]{article}
\usepackage{tokenizer}
\usepackage{verbatim}
\usepackage{tabularx}

\setlength{\parindent}{0pt}

\begin{document}
\title{\texttt{tokenizer.sty}}
\author{Sascha Herpers\thanks{Herpers@wiso.uni-koeln.de}}
\date{\today} \maketitle
\begin{abstract}
\noindent This package provides a command \verb+\GetTokens+, which
splits a comma separated list of strings into
tokens.\footnote{Thanks to Harald Harders for his support.}\\

\noindent Version: 1.1.0 (May 26, 2003)
\end{abstract}

\section{Usage}
This package defines the following command:
\begin{center}
\verb+\GetTokens{<name1>}{<name2>}{<source>}+.
\end{center}

\verb+\GetTokens+ can be used to split a comma separated list of
strings passed as \verb+<source>+ into two tokens at the first
encounter of a comma. While doing so, two commands with the name
\verb+<name1>+ and \verb+<name2>+ are defined. The string up to
the first comma of \verb+<source>+ is assigned as value for
\verb+\<name1>+ and the rest is assigned to \verb+\<name2>+.\\

By default the tokens are left as is, i.\,e. leading and trailing
spaces are not removed. However, if this behavior is not
desirable, you can use the package option \textbf{trim}.
Specifying this option causes each token to be stripped off
leading and trailing spaces.\footnote{Omitting the trim options
has the same result as using the \textbf{notrim} option.}\\

As a byproduct to the trim option the tokenizer package defines
the following command:
\begin{center}
\verb+\TrimSpaces{<source>}+
\end{center}
\verb+\TrimSpaces+ can be used remove spaces from the text passed
as first parameters.

\section{Example}
The example shown in listing 1 illustrates the use of
\verb+\GetTokens+. Firstly, a source string \verb+\Source+ is
created, which contains the strings to be separated. The following
while statement loops until there are no more tokens to process.
\verb+\GetTokens+ is called and the separated tokens are stored in
two commands \verb+\TokenOne+ and \verb+\TokenTwo+, which are
created by \verb+\GetTokens+. Lastly, \verb+\Source+ is replaced
by the remainder string contained in \verb+\TokenTwo+.\\


\hrule
\begin{verbatim}
 \def\Source{ this , is , a , short , test }
 The string \emph{\Source} contains the following tokens:\\
 \whiledo{\not\equal{\Source}{}}
 {
     \GetTokens{TokenOne}{TokenTwo}{\Source}
     \hspace*{.3cm}$\bullet$ [\TokenOne]\\
     \let\Source\TokenTwo
 }
\end{verbatim}
\hrule
\begin{center}
Listing 1: example usage of \verb+\GetTokens+
\end{center}

This is the output produced by the above example:\\
\begin{center}
    \begin{minipage}{9cm}
        \def\Source{ this , is , a , short , test }
        The string \emph{\Source} contains the following tokens:\\
        \whiledo{\not\equal{\Source}{}}
        {
            \GetTokens{TokenOne}{TokenTwo}{\Source}
            \hspace*{.3cm}$\bullet$ [\TokenOne]\\
            \let\Source\TokenTwo
        }
    \end{minipage}
\end{center}

\newpage
\section{History}
\begin{tabularx}{\textwidth}{lX}
 \hline
 date & change\\
 \hline
 05/26/03 & added packages options \textbf{trim} and \textbf{notrim}\\
\end{tabularx}

\end{document}