summaryrefslogtreecommitdiff
path: root/macros/latex/contrib/tokenizer
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
committerNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
commite0c6872cf40896c7be36b11dcc744620f10adf1d (patch)
tree60335e10d2f4354b0674ec22d7b53f0f8abee672 /macros/latex/contrib/tokenizer
Initial commit
Diffstat (limited to 'macros/latex/contrib/tokenizer')
-rw-r--r--macros/latex/contrib/tokenizer/tokenizer.pdfbin0 -> 85487 bytes
-rw-r--r--macros/latex/contrib/tokenizer/tokenizer.sty127
-rw-r--r--macros/latex/contrib/tokenizer/tokenizer.tex109
3 files changed, 236 insertions, 0 deletions
diff --git a/macros/latex/contrib/tokenizer/tokenizer.pdf b/macros/latex/contrib/tokenizer/tokenizer.pdf
new file mode 100644
index 0000000000..6fe0c2a2c2
--- /dev/null
+++ b/macros/latex/contrib/tokenizer/tokenizer.pdf
Binary files differ
diff --git a/macros/latex/contrib/tokenizer/tokenizer.sty b/macros/latex/contrib/tokenizer/tokenizer.sty
new file mode 100644
index 0000000000..b4fcb6e39e
--- /dev/null
+++ b/macros/latex/contrib/tokenizer/tokenizer.sty
@@ -0,0 +1,127 @@
+% Copyright 2003 Sascha Herpers
+%
+% This file can be redistributed and/or modified under the terms
+% of the LaTeX Project Public License Distributed from CTAN
+% archives in directory macros/latex/base/lppl.txt; either
+% version 1 of the License, or any later version.
+%
+%
+% 2003-05-22
+% Sascha Herpers
+% herpers@wiso.uni-koeln.de
+%
+%------------------------------------------------------------------------------
+% History
+% (mm/dd/yy) change
+% -----------------------------------------------------------------------------
+% 05/26/03 + added new options trim and notrim
+% If 'trim' is set, each first token is stripped from leading and
+% trailing spaces. 'notrim' is the default and leaves the token as
+% is.
+% + added new public command \TrimSpaces{<Token>}
+%------------------------------------------------------------------------------
+ \ProvidesPackage{tokenizer}[2003/05/26]
+ \RequirePackage{ifthen}
+
+%
+% local variables
+%------------------------------------------------------------------------------
+\def\SH@true{true} % used with trim/notrim
+\def\SH@false{false} % used with trim/notrim
+
+%
+% Options
+%------------------------------------------------------------------------------
+\DeclareOption{trim}{\edef\SH@trimtokens{\SH@true}} % trim tokens
+\DeclareOption{notrim}{\edef\SH@trimtokens{\SH@false}} % do not trim tokens
+\ProcessOptions
+
+%
+% \SH@GetTokens<Token1>,<Token2>
+% Saves <Token1> and <Token2> in two commands. The names of the commands
+% are derived from the value of \SH@FirstArgName and \SH@SecondArgName.
+%------------------------------------------------------------------------------
+\def\SH@GetTokens#1,#2\@empty{%
+ \def\SH@token{#1}
+ \ifx\SH@trimtokens\SH@true % strip spaces if requested
+ \TrimSpaces\SH@token
+ \fi
+ \SH@DefineCommand{\SH@FirstArgName}{\SH@token}
+ \SH@DefineCommand{\SH@SecondArgName}{#2}
+}
+
+%
+% \SH@CheckTokenSep<Token1>,<Token2>
+% Checks the arguments for the existence of a token separator.
+% This Command must be called with a trailing comma!
+% If a comma was found \SH@TokenValid is set to true, false otherwise.
+% Example 1: <t1>,<t2>,
+% This is split into <t1> and <t2>, and because the second token is
+% not empty \SH@TokenValid is set to true.
+% Example 2: <t1>,
+% This is split into <t1> and an empty second token and thus evaluates
+% to false.
+% Remark: If you omit the trailing comma when calling \SH@CheckTokenSep
+% example 2 will fail to compile.
+%------------------------------------------------------------------------------
+\def\SH@CheckTokenSep#1,#2\@empty{%
+ \def\SH@CTSArgTwo{#2}%
+ \ifx\SH@CTSArgTwo\@empty
+ \edef\SH@TokenValid{\SH@false}
+ \else
+ \edef\SH@TokenValid{\SH@true}
+ \fi
+}
+
+%
+% \GetTokens{<Target1>}{<Target2>}{<Source>}
+% Splits <Source> into two tokens at the first encounter of a comma.
+% The first token is saved in a newly created command with the name passed
+% as <Target1> and the second token likewise.
+%------------------------------------------------------------------------------
+\newcommand\GetTokens[3]%
+ {%
+ % save the names of the commands to receive the separated tokens
+ \def\SH@FirstArgName{#1}%
+ \def\SH@SecondArgName{#2}%
+ % check for the token separator
+ \expandafter\SH@CheckTokenSep#3,\@empty%
+ \ifx\SH@TokenValid\SH@true
+ % separator found, split the source string
+ \expandafter\SH@GetTokens#3\@empty%
+ \else
+ % there was no separator, return the complete source string as
+ % first token
+ \def\SH@token{#3}
+ \ifx\SH@trimtokens\SH@true % strip spaces if requested
+ \expandafter\TrimSpaces\SH@token
+ \fi
+ \SH@DefineCommand{\SH@FirstArgName}{\SH@token}
+ \SH@DefineCommand{\SH@SecondArgName}{}
+ \fi
+ }
+
+%
+% \DefineCommand{<Name>}{<Value>}
+% Creates a new command \<Name> with the value passed as second parameter.
+%------------------------------------------------------------------------------
+\def\SH@DefineCommand#1#2{%
+ \expandafter\def\csname #1\endcsname{#2}%
+}
+
+%
+% \TrimSpaces{<Token>}
+% This command removes leading and trailing spaces from <Token>. I found
+% the original code on the web. The original author was Michael Downes, who
+% provided the code as an answer to 'around the bend' question #15.
+%------------------------------------------------------------------------------
+\catcode`\Q=3
+\def\TrimSpaces#1{%
+ \begingroup
+ \aftergroup\toks\aftergroup0\aftergroup{%
+ \expandafter\trimb\expandafter\noexpand#1Q Q}%
+ \global\edef#1{\the\toks0}%
+}
+\def\trimb#1 Q{\trimc#1Q}
+\def\trimc#1Q#2{\afterassignment\endgroup \vfuzz\the\vfuzz#1}
+\catcode`\Q=11
diff --git a/macros/latex/contrib/tokenizer/tokenizer.tex b/macros/latex/contrib/tokenizer/tokenizer.tex
new file mode 100644
index 0000000000..c8f6287760
--- /dev/null
+++ b/macros/latex/contrib/tokenizer/tokenizer.tex
@@ -0,0 +1,109 @@
+% Copyright 2003 Sascha Herpers
+%
+% This file can be redistributed and/or modified under the terms
+% of the LaTeX Project Public License Distributed from CTAN
+% archives in directory macros/latex/base/lppl.txt; either
+% version 1 of the License, or any later version.
+%
+%
+% 2003-05-22
+% Sascha Herpers
+% herpers@wiso.uni-koeln.de
+\documentclass[draft]{article}
+\usepackage{tokenizer}
+\usepackage{verbatim}
+\usepackage{tabularx}
+
+\setlength{\parindent}{0pt}
+
+\begin{document}
+\title{\texttt{tokenizer.sty}}
+\author{Sascha Herpers\thanks{Herpers@wiso.uni-koeln.de}}
+\date{\today} \maketitle
+\begin{abstract}
+\noindent This package provides a command \verb+\GetTokens+, which
+splits a comma separated list of strings into
+tokens.\footnote{Thanks to Harald Harders for his support.}\\
+
+\noindent Version: 1.1.0 (May 26, 2003)
+\end{abstract}
+
+\section{Usage}
+This package defines the following command:
+\begin{center}
+\verb+\GetTokens{<name1>}{<name2>}{<source>}+.
+\end{center}
+
+\verb+\GetTokens+ can be used to split a comma separated list of
+strings passed as \verb+<source>+ into two tokens at the first
+encounter of a comma. While doing so, two commands with the name
+\verb+<name1>+ and \verb+<name2>+ are defined. The string up to
+the first comma of \verb+<source>+ is assigned as value for
+\verb+\<name1>+ and the rest is assigned to \verb+\<name2>+.\\
+
+By default the tokens are left as is, i.\,e. leading and trailing
+spaces are not removed. However, if this behavior is not
+desirable, you can use the package option \textbf{trim}.
+Specifying this option causes each token to be stripped off
+leading and trailing spaces.\footnote{Omitting the trim options
+has the same result as using the \textbf{notrim} option.}\\
+
+As a byproduct to the trim option the tokenizer package defines
+the following command:
+\begin{center}
+\verb+\TrimSpaces{<source>}+
+\end{center}
+\verb+\TrimSpaces+ can be used remove spaces from the text passed
+as first parameters.
+
+\section{Example}
+The example shown in listing 1 illustrates the use of
+\verb+\GetTokens+. Firstly, a source string \verb+\Source+ is
+created, which contains the strings to be separated. The following
+while statement loops until there are no more tokens to process.
+\verb+\GetTokens+ is called and the separated tokens are stored in
+two commands \verb+\TokenOne+ and \verb+\TokenTwo+, which are
+created by \verb+\GetTokens+. Lastly, \verb+\Source+ is replaced
+by the remainder string contained in \verb+\TokenTwo+.\\
+
+
+\hrule
+\begin{verbatim}
+ \def\Source{ this , is , a , short , test }
+ The string \emph{\Source} contains the following tokens:\\
+ \whiledo{\not\equal{\Source}{}}
+ {
+ \GetTokens{TokenOne}{TokenTwo}{\Source}
+ \hspace*{.3cm}$\bullet$ [\TokenOne]\\
+ \let\Source\TokenTwo
+ }
+\end{verbatim}
+\hrule
+\begin{center}
+Listing 1: example usage of \verb+\GetTokens+
+\end{center}
+
+This is the output produced by the above example:\\
+\begin{center}
+ \begin{minipage}{9cm}
+ \def\Source{ this , is , a , short , test }
+ The string \emph{\Source} contains the following tokens:\\
+ \whiledo{\not\equal{\Source}{}}
+ {
+ \GetTokens{TokenOne}{TokenTwo}{\Source}
+ \hspace*{.3cm}$\bullet$ [\TokenOne]\\
+ \let\Source\TokenTwo
+ }
+ \end{minipage}
+\end{center}
+
+\newpage
+\section{History}
+\begin{tabularx}{\textwidth}{lX}
+ \hline
+ date & change\\
+ \hline
+ 05/26/03 & added packages options \textbf{trim} and \textbf{notrim}\\
+\end{tabularx}
+
+\end{document}