tokenizer

git-svn-id: svn://tug.org/texlive/trunk@1400 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 127 insertions, 0 deletions

diff --git a/Master/texmf-dist/tex/latex/tokenizer/tokenizer.sty b/Master/texmf-dist/tex/latex/tokenizer/tokenizer.sty
new file mode 100644
index 00000000000..3336d0037ed
--- /dev/null
+++ b/Master/texmf-dist/tex/latex/tokenizer/tokenizer.sty
@@ -0,0 +1,127 @@
+% Copyright 2003 Sascha Herpers
+%
+% This file can be redistributed and/or modified under the terms
+% of the LaTeX Project Public License Distributed from CTAN
+% archives in directory macros/latex/base/lppl.txt; either
+% version 1 of the License, or any later version.
+%
+%
+% 2003-05-22
+% Sascha Herpers
+% herpers@wiso.uni-koeln.de
+%
+%------------------------------------------------------------------------------
+% History
+% (mm/dd/yy) change
+% -----------------------------------------------------------------------------
+%  05/26/03  + added new options trim and notrim
+%              If 'trim' is set, each first token is stripped from leading and
+%              trailing spaces. 'notrim' is the default and leaves the token as
+%              is.
+%            + added new public command \TrimSpaces{<Token>}
+%------------------------------------------------------------------------------
+ \ProvidesPackage{tokenizer}[2003/05/26]
+ \RequirePackage{ifthen}
+
+%
+% local variables
+%------------------------------------------------------------------------------
+\def\SH@true{true}                       % used with trim/notrim
+\def\SH@false{false}                     % used with trim/notrim
+
+%
+% Options
+%------------------------------------------------------------------------------
+\DeclareOption{trim}{\edef\SH@trimtokens{\SH@true}}     % trim tokens
+\DeclareOption{notrim}{\edef\SH@trimtokens{\SH@false}}  % do not trim tokens
+\ProcessOptions
+
+%
+% \SH@GetTokens<Token1>,<Token2>
+%     Saves <Token1> and <Token2> in two commands. The names of the commands
+%     are derived from the value of \SH@FirstArgName and \SH@SecondArgName.
+%------------------------------------------------------------------------------
+\def\SH@GetTokens#1,#2\@empty{%
+       \def\SH@token{#1}
+       \ifx\SH@trimtokens\SH@true        % strip spaces if requested
+           \TrimSpaces\SH@token
+       \fi
+       \SH@DefineCommand{\SH@FirstArgName}{\SH@token}
+       \SH@DefineCommand{\SH@SecondArgName}{#2}
+}
+
+%
+% \SH@CheckTokenSep<Token1>,<Token2>
+%     Checks the arguments for the existence of a token separator.
+%     This Command must be called with a trailing comma!
+%     If a comma was found \SH@TokenValid is set to true, false otherwise.
+%     Example 1: <t1>,<t2>,
+%        This is split into <t1> and <t2>, and because the second token is
+%        not empty \SH@TokenValid is set to true.
+%     Example 2: <t1>,
+%        This is split into <t1> and an empty second token and thus evaluates
+%        to false.
+%     Remark: If you omit the trailing comma when calling \SH@CheckTokenSep
+%             example 2 will fail to compile.
+%------------------------------------------------------------------------------
+\def\SH@CheckTokenSep#1,#2\@empty{%
+    \def\SH@CTSArgTwo{#2}%
+    \ifx\SH@CTSArgTwo\@empty
+        \edef\SH@TokenValid{\SH@false}
+    \else
+        \edef\SH@TokenValid{\SH@true}
+    \fi
+}
+
+%
+% \GetTokens{<Target1>}{<Target2>}{<Source>}
+%     Splits <Source> into two tokens at the first encounter of a comma.
+%     The first token is saved in a newly created command with the name passed
+%     as <Target1> and the second token likewise.
+%------------------------------------------------------------------------------
+\newcommand\GetTokens[3]%
+    {%
+        % save the names of the commands to receive the separated tokens
+        \def\SH@FirstArgName{#1}%
+        \def\SH@SecondArgName{#2}%
+        % check for the token separator
+        \expandafter\SH@CheckTokenSep#3,\@empty%
+        \ifx\SH@TokenValid\SH@true
+            % separator found, split the source string
+            \expandafter\SH@GetTokens#3\@empty%
+        \else
+            % there was no separator, return the complete source string as
+            % first token
+            \def\SH@token{#3}
+            \ifx\SH@trimtokens\SH@true   % strip spaces if requested
+                    \expandafter\TrimSpaces\SH@token
+            \fi
+            \SH@DefineCommand{\SH@FirstArgName}{\SH@token}
+            \SH@DefineCommand{\SH@SecondArgName}{}
+        \fi
+    }
+
+%
+% \DefineCommand{<Name>}{<Value>}
+%     Creates a new command \<Name> with the value passed as second parameter.
+%------------------------------------------------------------------------------
+\def\SH@DefineCommand#1#2{%
+  \expandafter\def\csname #1\endcsname{#2}%
+}
+
+%
+% \TrimSpaces{<Token>}
+%     This command removes leading and trailing spaces from <Token>. I found
+%     the original code on the web. The original author was Michael Downes, who
+%     provided the code as an answer to 'around the bend' question #15.
+%------------------------------------------------------------------------------
+\catcode`\Q=3
+\def\TrimSpaces#1{%
+  \begingroup
+  \aftergroup\toks\aftergroup0\aftergroup{%
+  \expandafter\trimb\expandafter\noexpand#1Q Q}%
+  \global\edef#1{\the\toks0}%
+}
+\def\trimb#1 Q{\trimc#1Q}
+\def\trimc#1Q#2{\afterassignment\endgroup \vfuzz\the\vfuzz#1}
+\catcode`\Q=11