summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2011-08-01 00:47:44 +0000
committerKarl Berry <karl@freefriends.org>2011-08-01 00:47:44 +0000
commit07db9a5835ec1aaa508040c50590c2d0f8c70d84 (patch)
tree4ba49a28061e02b74dbff30725ff21abfaaf996f
parentaec2bbc3994991ebc6c5f4dbc90ad15c55f1a1fb (diff)
texcount 2.3 (31jul11)
git-svn-id: svn://tug.org/texlive/trunk@23293 c570f23f-e606-0410-a88d-b1316a301751
-rwxr-xr-xBuild/source/texk/texlive/linked_scripts/texcount/texcount.pl3001
-rwxr-xr-xBuild/source/texk/texlive/linked_scripts/texlive/tlmgr.pl13
-rw-r--r--Master/texmf-dist/doc/support/texcount/QuickReference.pdfbin130756 -> 138221 bytes
-rw-r--r--Master/texmf-dist/doc/support/texcount/QuickReference.tex101
-rw-r--r--Master/texmf-dist/doc/support/texcount/README2
-rw-r--r--Master/texmf-dist/doc/support/texcount/TeXcount.pdfbin148874 -> 224304 bytes
-rw-r--r--Master/texmf-dist/doc/support/texcount/TeXcount.tex468
-rw-r--r--Master/texmf-dist/doc/support/texcount/macros.tex9
-rw-r--r--Master/texmf-dist/doc/support/texcount/sub_addrules.tex25
-rw-r--r--Master/texmf-dist/doc/support/texcount/sub_options.tex114
-rw-r--r--Master/texmf-dist/doc/support/texcount/sub_tc_other.tex14
-rwxr-xr-xMaster/texmf-dist/scripts/texcount/texcount.pl3001
12 files changed, 4990 insertions, 1758 deletions
diff --git a/Build/source/texk/texlive/linked_scripts/texcount/texcount.pl b/Build/source/texk/texlive/linked_scripts/texcount/texcount.pl
index 96fac5cd220..69fb4b5002f 100755
--- a/Build/source/texk/texlive/linked_scripts/texcount/texcount.pl
+++ b/Build/source/texk/texlive/linked_scripts/texcount/texcount.pl
@@ -2,38 +2,370 @@
use strict;
use warnings;
use Term::ANSIColor;
+use Text::Wrap;
use Encode;
-use POSIX qw(locale_h);
-use locale;
-setlocale(LC_CTYPE,"no_NO");
+use utf8; # Because the script itself is UTF-8 encoded
-my $versionnumber="2.2";
-my $versiondate="2009 Apr 30";
+##### Version information
+
+my $versionnumber="2.3";
+my $versiondate="2011 Jul 30";
+
+###### Set global settings and variables
+
+### Global data about TeXcount
+my %GLOBALDATA=
+ ('versionnumber' => $versionnumber
+ ,'versiondate' => $versiondate
+ ,'maintainer' => "Einar Andreas Rodland"
+ ,'copyrightyears' => "2008-2011"
+ ,'website' => "http://app.uio.no/ifi/texcount/"
+ );
+
+### Options and states
+
+# Outer object (for error reports not added by a TeX object)
+my $Main=getMain();
+
+# Global options and settings
+my $htmlstyle=0; # Flag to print HTML
+my $encoding=undef; # Selected input encoding (default will be guess)
+my @encodingGuessOrder=qw/ascii utf8 latin1/; # Encoding guessing order
+my $outputEncoding; # Encoding used for output
+my @AlphabetScripts=qw/Digit Is_alphabetic/; # Letters minus logograms: defined later
+my @LogogramScripts=qw/Ideographic Katakana Hiragana Thai Lao Hangul/; # Scripts counted as whole words
+my $WordPattern; # Regex matching a word (defined in apply_language_options())
+
+# Parsing rules options
+my $includeTeX=0; # Flag to parse included files
+my $includeBibliography=0; # Flag to include bibliography
+my %substitutions; # Substitutions to make globally
+
+# Counting options
+my @sumweights; # Set count weights for computing sum
+my $optionWordFreq=0; # Count words of this frequency, or don't count if 0
+my $optionWordClassFreq=0; # Count words per word class (language) if set
+
+# Parsing details options
+my $strictness=0; # Flag to check for undefined groups
+my $verbose=0; # Level of verbosity
+my $showcodes=1; # Flag to show overview of colour codes (2=force)
+my $showstates=0; # Flag to show internal state in verbose log
+my $showsubcounts=0; # Write subcounts if #>this, or not (if 0)
+my $separatorstyleregex='^word'; # Styles (regex) after which separator should be added
+my $separator=''; # Separator to add after words/tokens
+
+# Final summary output options
+my $showVersion=0; # Indicator that version info be included (1) or not (-1)
+my $totalflag=0; # Flag to write only total summary
+my $briefsum=0; # Flag to set brief summary
+my $outputtemplate; # Output template
+my $finalLineBreak=1; # Add line break at end
+
+# Global settings
+my $optionFast=1; # Flag inticating fast method
+
+# Global variables and internal states (for internal use only)
+my $blankline=0; # Number of blank lines printed
+my $errorcount=0; # Number of errors in parsing
+my %warnings=(); # Warnings
+my %WordFreq; # Hash for counting words
+
+# String data storage
+my $STRINGDATA;
###### Set CMD specific settings and variables
-# Options and states
-my $verbose=0;
-my $showcodes=1;
-my $showstates=0;
-my $showsubcounts=0;
-my $htmlstyle=0;
-my $includeTeX=0;
-my $briefsum=0;
-my @sumweights;
-my $utf8flag=0;
-
-# Global variables
-my $blankline=0;
-my $errorcount=0;
+## Preset command line options
+# List of options (stings) separated by comma,
+# e.g. ("-inc","-v") to parse included files and
+# give verbose output by default.
+my @StartupOptions=();
# CMD specific global variables
-my $totalflag=0;
-my @filelist;
-my $workdir;
-my $globalworkdir="";
+my @filelist; # List of files to parse
+my $workdir; # Root directory (taken from filename)
+my $globalworkdir=""; # Overrules workdir (default=present root)
+my $fileFromSTDIN=0; # Flag to set input from STDIN
+my $_STDIN_="<STDIN>"; # File name to represent STDIN
+
+# CMD specific settings
+$Text::Wrap::columns=76; # Page width for wrapped output
+
+###### Set status identifiers and methods
+
+
+### Counter indices from 0 to $SIZE_CNT-1
+# 0: Number of files
+# 1: Text words
+# 2: Header words
+# 3: Caption words
+# 4: Number of headers
+# 5: Number of floating environments
+# 6: Number of inlined math
+# 7: Number of displayed math
+my $SIZE_CNT=8;
+my $CNT_FILE=0;
+my $CNT_WORDS_TEXT=1;
+my $CNT_WORDS_HEADER=2;
+my $CNT_WORDS_CAPTION=3;
+my $CNT_COUNT_HEADER=4;
+my $CNT_COUNT_FLOAT=5;
+my $CNT_COUNT_INLINEMATH=6;
+my $CNT_COUNT_DISPLAYMATH=7;
-###### Set global settings and variables
+# Labels used to describe the counts
+my @countlabel=('Files','Words in text','Words in headers',
+ 'Words in float captions','Number of headers','Number of floats',
+ 'Number of math inlines','Number of math displayed');
+
+### Parsing statuses
+## Note that status numbers are used in rule definitions and should
+## not be changed.
+#
+## Status for regions that should not be counted
+# 0 = exclude from count
+## Statuses for regions in which words should not be counted
+# -1 = float (exclude, but include captions)
+# -2 = strong exclude, ignore begin-end groups
+# -3 = stronger exclude, do not parse macro parameters
+# -4 = ignore everything except end marker: even {
+# -9 = preamble (between \documentclass and \begin{document})
+## Statuses for regions in which words should be counted
+# 1 = text
+# 2 = header text
+# 3 = float text
+## Status change: not used in parsing, but to switch status then ignore contents
+# 6 = switch to inlined math
+# 7 = switch to displayed math
+## Note that positive statuses must correspond to CNT codes!
+#
+my $STATUS_IGNORE=0;
+my $STATUS_FLOAT=-1;
+my $STATUS_EXCLUDE_STRONG=-2;
+my $STATUS_EXCLUDE_STRONGER=-3;
+my $STATUS_EXCLUDE_ALL=-4;
+my $STATUS_PREAMBLE=-9;
+my $STATUS_TEXT=1;
+my $STATUS_TEXT_HEADER=2;
+my $STATUS_TEXT_FLOAT=3;
+my $STATUS_TO_INLINEMATH=6;
+my $STATUS_TO_DISPLAYMATH=7;
+
+# When combining two statuses, use the first one; list must be complete!
+my @STATUS_PRIORITY_LIST=(-4,-3,-2,-1,0,-9,3,2,1);
+
+# Status: is a text status..."include status" is more correct
+sub status_is_text {
+ my $st=shift @_;
+ return ($st>0);
+}
+
+# Status: get CNT corresponding to text status (or undef)
+sub status_text_cnt {
+ my $st=shift @_;
+ if ($st==$STATUS_TEXT) {return $CNT_WORDS_TEXT;}
+ if ($st==$STATUS_TEXT_HEADER) {return $CNT_WORDS_HEADER;}
+ if ($st==$STATUS_TEXT_FLOAT) {return $CNT_WORDS_CAPTION;}
+ return undef;
+}
+
+# Status: is an exclude status
+sub status_is_exclude {
+ my $st=shift @_;
+ return ($st<0);
+}
+
+# Status: \begin and \end should be processed
+sub status_inc_envir {
+ my $st=shift @_;
+ return ($st>-2);
+}
+
+# Map status number to CNT value
+sub status_to_cnt {
+ my $st=shift @_;
+ if ($st==$STATUS_TO_INLINEMATH) {return $CNT_COUNT_INLINEMATH;}
+ if ($st==$STATUS_TO_DISPLAYMATH) {return $CNT_COUNT_DISPLAYMATH;}
+ return undef;
+}
+
+# Status as text
+sub status_to_text {
+ my $st=shift @_;
+ return $st;
+}
+
+# Status as text
+sub status_to_style {
+ return 'word'.status_to_text(@_);
+}
+
+### Token types
+# -1: space
+# 0: comment
+# 1: word (or other forms of text or text components)
+# 2: symbol (not word, e.g. punctuation)
+# 3: macro
+# 4: curly braces {}
+# 5: brackets []
+# 6: maths
+# 9: line break in file
+# 999: end of line or blank line
+# 666: TeXcount instruction (%TC:instruction)
+my $TOKEN_SPACE=-1;
+my $TOKEN_COMMENT=0;
+my $TOKEN_WORD=1;
+my $TOKEN_SYMBOL=2;
+my $TOKEN_MACRO=3;
+my $TOKEN_BRACE=4;
+my $TOKEN_BRACKET=5;
+my $TOKEN_MATH=6;
+my $TOKEN_LINEBREAK=9;
+my $TOKEN_TC=666;
+my $TOKEN_END=999;
+
+###### Set global definitions
+
+
+### Break points
+# Definition of macros that define break points that start a new subcount.
+# The values given are used as labels.
+my %BreakPointsOptions;
+$BreakPointsOptions{'none'}={};
+$BreakPointsOptions{'part'}={%{$BreakPointsOptions{'none'}},'\part'=>'Part'};
+$BreakPointsOptions{'chapter'}={%{$BreakPointsOptions{'part'}},'\chapter'=>'Chapter'};
+$BreakPointsOptions{'section'}={%{$BreakPointsOptions{'chapter'}},'\section'=>'Section'};
+$BreakPointsOptions{'subsection'}={%{$BreakPointsOptions{'section'}},'\subsection'=>'Subsection'};
+$BreakPointsOptions{'default'}=$BreakPointsOptions{'subsection'};
+my %BreakPoints=%{$BreakPointsOptions{'none'}};
+
+### Print styles
+# Definition of different print styles: maps of class labels
+# to ANSI codes. Class labels are as used by HTML styles.
+my @STYLES=();
+my %STYLE;
+$STYLES[0]={'error'=>'bold red'};
+$STYLES[1]={%{$STYLES[0]},''=>'normal',' '=>'normal',
+ 'word1'=>'blue','word2'=>'bold blue','word3'=>'blue',
+ 'grouping'=>'red','document'=>'red','mathgroup'=>'magenta',
+ 'state'=>'cyan underline','cumsum'=>'yellow'};
+$STYLES[2]={%{$STYLES[1]},
+ 'command'=>'green','ignore'=>'cyan',
+ 'exclcommand'=>'yellow','exclgroup'=>'yellow','exclmath'=>'yellow'};
+$STYLES[3]={%{$STYLES[2]},
+ 'tc'=>'bold yellow','comment'=>'yellow','option'=>'yellow',
+ 'fileinclude'=>'bold green'};
+$STYLES[4]={%{$STYLES[3]}};
+
+###### Define what a word is and language options
+
+
+# Patters matching a letter. Should be a single character or
+# ()-enclosed regex for substitution into word pattern regex.
+my @LetterMacros=qw/ae AE o O aa AA oe OE ss
+ alpha beta gamma delta epsilon zeta eta theta iota kappa lamda
+ mu nu xi pi rho sigma tau upsilon phi chi psi omega
+ Gamma Delta Theta Lambda Xi Pi Sigma Upsilon Phi Psi Omega
+ /;
+my $specialchars='\\\\('.join('|',@LetterMacros).')(\{\}|\s*|\b)';
+my $modifiedchars='\\\\[\'\"\`\~\^\=](@|\{@\})';
+my %NamedLetterPattern;
+$NamedLetterPattern{'restricted'}='@';
+$NamedLetterPattern{'default'}='('.join('|','@',$modifiedchars,$specialchars).')';
+$NamedLetterPattern{'relaxed'}=$NamedLetterPattern{'default'};
+my $LetterPattern=$NamedLetterPattern{'default'};
+
+# List of regexp patterns that should be analysed as words.
+# Use @ to represent a letter, will be substituted with $LetterPattern.
+# Named patterns may replace or be appended to the original patterns.
+my %NamedWordPattern;
+$NamedWordPattern{'letters'}='@';
+$NamedWordPattern{'words'}='(@+|\{@+\})([\-\'\.]?(@+|\{@+\}))*';
+my @WordPatterns=($NamedWordPattern{'words'});
+
+### Macro option regexp list
+# List of regexp patterns to be gobbled as macro option in and after
+# a macro.
+my %NamedMacroOptionPattern;
+$NamedMacroOptionPattern{'default'}='\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]';
+$NamedMacroOptionPattern{'relaxed'}='\[[^\[\]\n]*\]';
+$NamedMacroOptionPattern{'restricted'}=$NamedMacroOptionPattern{'default'};
+my $MacroOptionPattern=$NamedMacroOptionPattern{'default'};
+
+### Alternative language encodings
+my %NamedEncodingGuessOrder;
+$NamedEncodingGuessOrder{'chinese'}=[qw/utf8 gb2312 big5/];
+$NamedEncodingGuessOrder{'japanese'}=[qw/utf8 euc-jp iso-2022-jp jis shiftjis/];
+$NamedEncodingGuessOrder{'korean'}=[qw/utf8 euc-kr iso-2022-kr/];
+
+
+###### Define character classes (alphabets)
+
+
+### Character classes to use as Unicode properties
+
+# Character group representing digits 0-9 (more restrictive than Digits)
+sub Is_digit { return <<END;
+0030\t0039
+END
+}
+
+# Character group representing letters (excluding logograms)
+sub Is_alphabetic { return <<END;
++utf8::Alphabetic
+-utf8::Ideographic
+-utf8::Katakana
+-utf8::Hiragana
+-utf8::Thai
+-utf8::Lao
+-utf8::Hangul
+END
+}
+
+# Character group representing letters (excluding logograms)
+sub Is_alphanumeric { return <<END;
++utf8::Alphabetic
++utf8::Digit
+-utf8::Ideographic
+-utf8::Katakana
+-utf8::Hiragana
+-utf8::Thai
+-utf8::Lao
+-utf8::Hangul
+END
+}
+
+# Character class for punctuation excluding special characters
+sub Is_punctuation { return <<END;
++utf8::Punctuation
+-0024\t0025
+-005c
+-007b\007e
+END
+}
+
+# Character group representing CJK characters
+sub Is_cjk { return <<END;
++utf8::Han
++utf8::Katakana
++utf8::Hiragana
++utf8::Hangul
+END
+}
+
+# Character group for CJK punctuation characters
+sub Is_cjkpunctuation { return <<END;
+3000\t303f
+2018\t201f
+ff01\tff0f
+ff1a\tff1f
+ff3b\tff3f
+ff5b\tff65
+END
+}
+
+###### Define core rules
### Macros for headers
# Macros that identify headers: i.e. following token or
@@ -43,8 +375,26 @@ my $globalworkdir="";
# only role of defining it here is that the counter for the number
# of headers is incremented by one.
my %TeXheader=('\title'=>[2],'\part'=>[2],'\chapter'=>[2],
- '\section'=>[2],'\subsection'=>[2],'\subsubsection'=>[2],
- '\paragraph'=>[2],'\subparagraph'=>[2]);
+ '\section'=>[2],'\subsection'=>[2],'\subsubsection'=>[2],
+ '\paragraph'=>[2],'\subparagraph'=>[2]);
+
+### Macros indicating package inclusion
+# Will always be assumed to take one parameter (plus options).
+my %TeXpackageinc=('\usepackage'=>1);
+
+### Macros that are counted within the preamble
+# The preamble is the text between \documentclass and \begin{document}.
+# Text and macros in the preamble is ignored unless specified here. The
+# value is the status (1=text, 2=header, etc.) they should be interpreted as.
+# Note that only the first unit (token or {...} block) is counted.
+my %TeXpreamble=(
+ '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3],
+ '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3],
+ '\title'=>[2]);
+
+### In floats: include only specific macros
+# Macros used to identify caption text within floats.
+my %TeXfloatinc=('\caption'=>[3]);
### How many tokens to gobble after macro
# Each macro is assumed to gobble up a given number of
@@ -61,11 +411,11 @@ my %TeXheader=('\title'=>[2],'\part'=>[2],'\chapter'=>[2],
# For macros not specified here, the default value is 0: i.e.
# no tokens are excluded, but [...] options are. Header macros
# specified in %TeXheader are automatically included here.
-my %TeXmacro=(%TeXheader,
+my %TeXmacro=(%TeXheader,%TeXpreamble,%TeXfloatinc,
'\documentclass'=>1,'\documentstyle'=>1,'\usepackage'=>1, '\hyphenation'=>1,
'\pagestyle'=>1,'\thispagestyle'=>1, '\pagenumbering'=>1,'\markboth'=>1, '\markright'=>1,
'\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3],
- '\newenvironment'=>[-3,-3,-3], 'renewenvironment'=>[-3,-3,-3],
+ '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3],
'\newfont'=>2,'\newtheorem'=>2,'\bibliographystyle'=>1, '\bibliography'=>1,
'\parbox'=>1, '\marginpar'=>[3],'\makebox'=>0, '\raisebox'=>1, '\framebox'=>0,
'\newsavebox'=>1, '\sbox'=>1, '\savebox'=>2, '\usebox'=>1,'\rule'=>2,
@@ -80,29 +430,21 @@ my %TeXmacro=(%TeXheader,
'\cline'=>1, '\multicolumn'=>3,'\typeout'=>1, '\typein'=>1,
'beginlist'=>2, 'beginminipage'=>1, 'begintabular'=>1,
'beginthebibliography'=>1,'beginlrbox'=>1,
- '\begin'=>1,'\end'=>1,'\title'=>[2]);
+ '\begin'=>1,'\end'=>1,'\title'=>[2],
+ '\addtocontents'=>2,'\addcontentsline'=>3,
+ '\uppercase'=>0,'\lowercase'=>0);
### Macros that should be counted as one or more words
# Macros that represent text may be declared here. The value gives
# the number of words the macro represents.
my %TeXmacroword=('\LaTeX'=>1,'\TeX'=>1);
-### Macros that are counted within the preamble
-# The preamble is the text between \documentclass and \begin{document}.
-# Text and macros in the preamble is ignored unless specified here. The
-# value is the status (1=text, 2=header, etc.) they should be interpreted as.
-# Note that only the first unit (token or {...} block) is counted.
-my %TeXpreamble=('\title'=>[2],
- '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3],
- '\newenvironment'=>[-3,-3,-3], 'renewenvironment'=>[-3,-3,-3],
- );
-
### Begin-End groups
# Identified as begin-end groups, and define =>state. The
# states used corresponds to the elements of the count array, and
# are:
# 0: Not included
-# 1: Text, word included in text count
+# 1: Text, words included in text count
# 2: Header, words included in header count
# 3: Float caption, words included in float caption count
# 6: Inline mathematics, words not counted
@@ -117,79 +459,41 @@ my %TeXpreamble=('\title'=>[2],
# Note that some environments may only exist within math-mode, and
# therefore need not be defined here: in fact, they should not as it
# is not clear if they will be in inlined or displayed math.
-#
my %TeXgroup=('document'=>1,'letter'=>1,'titlepage'=>0,
'center'=>1,'flushleft'=>1,'flushright'=>1,
- 'abstract'=>1,'quote'=>1,'quotation'=>1,'verse'=>1,'minipage'=>1,'verbatim'=>1,
+ 'abstract'=>1,'quote'=>1,'quotation'=>1,'verse'=>1,'minipage'=>1,
+ 'verbatim'=>-4,'tikzpicture'=>-4,
'description'=>1,'enumerate'=>1,'itemize'=>1,'list'=>1,
'theorem'=>1,'lemma'=>1,'definition'=>1,'corollary'=>1,'example'=>1,
'math'=>6,'displaymath'=>7,'equation'=>7,'eqnarray'=>7,'align'=>7,
+ 'equation*'=>7,'eqnarray*'=>7,'align*'=>7,
'figure'=>-1,'float'=>-1,'picture'=>-1,'table'=>-1,
'tabbing'=>0,'tabular'=>0,'thebibliography'=>0,'lrbox'=>0);
-### In floats: include only specific macros
-# Macros used to identify caption text within floats.
-my %TeXfloatinc=('\caption'=>[3]);
-
### Macros for including tex files
# Allows \macro{file} or \macro file. If the value is 0, the filename will
# be used as is; if it is 1, the filetype .tex will be added if the
# filename is without filetype; if it is 2, the filetype .tex will be added.
my %TeXfileinclude=('\input'=>1,'\include'=>2);
-### Count labels
-# Labels used to describe the counts
-my @countlabel=('Files','Words in text','Words in headers',
- 'Words in float captions','Number of headers','Number of floats',
- 'Number of math inlines','Number of math displayed');
+###### Define package specific rules
-### Break points
-# Definition of macros that define break points that start a new subcount.
-# The values given are used as labels.
-my %BreakPointsOptions;
-$BreakPointsOptions{'none'}={};
-$BreakPointsOptions{'part'}={%{$BreakPointsOptions{'none'}},'\part'=>'Part'};
-$BreakPointsOptions{'chapter'}={%{$BreakPointsOptions{'part'}},'\chapter'=>'Chapter'};
-$BreakPointsOptions{'section'}={%{$BreakPointsOptions{'chapter'}},'\section'=>'Section'};
-$BreakPointsOptions{'subsection'}={%{$BreakPointsOptions{'section'}},'\subsection'=>'Subsection'};
-$BreakPointsOptions{'default'}=$BreakPointsOptions{'subsection'};
-my %BreakPoints=%{$BreakPointsOptions{'none'}};
+### Package rule definitions
-### Print styles
-# Definition of different print styles: maps of class labels
-# to ANSI codes. Class labels are as used by HTML styles.
-my @STYLES=();
-my %STYLE;
-$STYLES[0]={'error'=>'bold red'};
-$STYLES[1]={%{$STYLES[0]},
- 'word1'=>'blue','word2'=>'bold blue','word3'=>'blue',
- 'grouping'=>'red','document'=>'red','mathgroup'=>'magenta',
- 'state'=>'cyan underline','sumcount'=>'yellow'};
-$STYLES[2]={%{$STYLES[1]},
- 'command'=>'green','exclcommand'=>'yellow','exclgroup'=>'yellow','exclmath'=>'yellow',
- 'ignore'=>'cyan'};
-$STYLES[3]={%{$STYLES[2]},
- 'tc'=>'bold yellow','comment'=>'yellow','option'=>'yellow',
- 'fileinclude'=>'bold green'};
-$STYLES[4]={%{$STYLES[3]}};
+my %PackageTeXmacro=(); # TeXmacro definitions per package
+my %PackageTeXmacroword=(); # TeXmacroword definitions per package
+my %PackageTeXheader=(); # TeXheader definitions per package
+my %PackageTeXgroup=(); # TeXgroup definitions per package
+my %PackageTeXfileinclude=(); # TeXgroup definitions per package
-### Word regexp pattern list
-# List of regexp patterns that should be analysed as words.
-# Use @ to represent a letter, will be substituted with $LetterPattern.
-my @WordPatterns=('(@+\.)+@+\.?','@+([\-\']@+)*');
-my $specialchars='\\\\(ae|AE|o|O|aa|AA)';
-my $modifiedchars='\\\\[\'\"\`\~\^\=](\w|\{\w\})';
-my $LetterPattern='\w';
-my $LetterPatternRelaxed='([\w\-\']|'.$modifiedchars.'|'.$specialchars.'(\{\})?|\{'.$specialchars.'\}|\{\w\})';
-my %NamedWordPattern;
-$NamedWordPattern{'chinese'}='\p{script=Han}';
-$NamedWordPattern{'japanese'}='(\p{script=Han}|\p{script=Hiragana}|\p{script=Katakana})';
+# Rules for package psfig
+$PackageTeXmacro{'psfig'}={('\psfig'=>1)};
-### Macro option regexp list
-# List of regexp patterns to be gobbled as macro option in and after
-# a macro.
-my @MacroOptionPatterns=('\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]');
-my @MacroOptionPatternsRelaxed=('\[[^\[\]\n]*\]');
+# Rules for bibliography inclusion
+$PackageTeXmacroword{'%incbib'}={'thebibliography'=>1};
+$PackageTeXmacro{'%incbib'}={};
+$PackageTeXgroup{'%incbib'}={'thebibliography'=>1};
+$PackageTeXfileinclude{'%incbib'}={'\bibliography'=>'bbl'};
###### Main script
@@ -197,6 +501,7 @@ my @MacroOptionPatternsRelaxed=('\[[^\[\]\n]*\]');
###################################################
MAIN(@ARGV);
+exit; # Just to make sure it ends here...
###################################################
@@ -207,22 +512,40 @@ MAIN(@ARGV);
# MAIN ROUTINE: Handle arguments, then parse files
sub MAIN {
- my @args=@_;
+ my @args;
+ push @args,@StartupOptions;
+ push @args,@_;
+ Initialise();
+ Check_Arguments(@args);
my @toplevelfiles=Parse_Arguments(@args);
Apply_Options();
- if (scalar(@toplevelfiles)==0) {
- conditional_print_help_style()
- || print_error("No files specified.","p","error");
- } else {
+ if (scalar @toplevelfiles>0 || $fileFromSTDIN) {
+ if ($showVersion && !$htmlstyle && !($briefsum && $totalflag)) {
+ print "\n=== LaTeX word count (TeXcount version $versionnumber) ===\n\n";
+ }
conditional_print_help_style();
- my $totalcount=parse_file_list(@toplevelfiles);
+ my $totalcount=Parse_file_list(@toplevelfiles);
conditional_print_total($totalcount);
+ Report_Errors();
+ if ($optionWordFreq || $optionWordClassFreq) {print_word_freq();}
+ } elsif ($showcodes>1) {
+ conditional_print_help_style();
+ } else {
+ error($Main,'No files specified.');
}
- Report_ErrorCount();
Close_Output();
}
-# Checks arguments, exits on exit condition
+# Initialise, overrule initial settings, etc.
+sub Initialise {
+ _option_subcount();
+ # Windows settings
+ if ($^O=~/^MSWin/) {
+ option_ansi_colours(0);
+ }
+}
+
+# Check arguments, exit on exit condition
sub Check_Arguments {
my @args=@_;
if (!@args) {
@@ -230,26 +553,28 @@ sub Check_Arguments {
print_syntax();
print_reference();
exit;
- } elsif ($args[0]=~/^(\-?\-(h|\?|help)|\/(\?|h))$/) {
+ } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))$/) {
print_help();
exit;
- } elsif ($args[0]=~/^\-?\-(ver|version)$/) {
+ } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))=/) {
+ print_help_on("$'");
+ exit;
+ } elsif ($args[0]=~/^--?(ver|version)$/) {
print_version();
exit;
- } elsif ($args[0]=~/^\-?\-(lic|license)$/) {
+ } elsif ($args[0]=~/^--?(lic|license|licence)$/) {
print_license();
exit;
}
return 1;
}
-# Parses arguments, sets options (global) and returns file list
+# Parse arguments, set options (global) and return file list
sub Parse_Arguments {
my @args=@_;
- Check_Arguments(@args);
my @files;
- foreach my $arg (@ARGV) {
- if (Parse_Option($arg)) {next;}
+ foreach my $arg (@args) {
+ if (parse_option($arg)) {next;}
if ($arg=~/^\-/) {
print 'Invalid opton '.$arg."\n";
print_syntax();
@@ -262,49 +587,88 @@ sub Parse_Arguments {
}
# Parse individual option parameters
-sub Parse_Option {
+sub parse_option {
my $arg=shift @_;
- return parse_options_parsing($arg)
- || parse_options_sums($arg)
+ return parse_options_preset($arg)
+ || parse_options_parsing($arg)
+ || parse_options_counts($arg)
|| parse_options_output($arg)
|| parse_options_format($arg)
;
}
+# Parse presetting options
+sub parse_options_preset {
+ my $arg=shift @_;
+ if ($arg=~/^-(opt|option|options|optionfile)=/) {
+ _parse_optionfile($');
+ }
+ else {return 0;}
+ return 1;
+}
+
+# Parse parsing options
sub parse_options_parsing {
my $arg=shift @_;
- if ($arg eq '-inc') {$includeTeX=1;}
+ if ($arg eq '-') {$fileFromSTDIN=1;}
+ elsif ($arg eq '-merge') {$includeTeX=2;}
+ elsif ($arg eq '-inc') {$includeTeX=1;}
elsif ($arg eq '-noinc') {$includeTeX=0;}
+ elsif ($arg =~/^-(includepackage|incpackage|package|pack)=(.*)$/) {include_package($2);}
+ elsif ($arg eq '-incbib') {$includeBibliography=1;}
+ elsif ($arg eq '-nobib') {$includeBibliography=0;}
elsif ($arg eq '-dir') {$globalworkdir=undef;}
- elsif ($arg=~/^-dir=(.*)$/) {$globalworkdir=$1;}
- elsif ($arg=~/^-(utf8|unicode)$/) {$utf8flag=1;}
- elsif ($arg=~/^-(ch|chinese|zhongwen)$/) {
- $utf8flag=1;
- @WordPatterns=($NamedWordPattern{'chinese'},@WordPatterns);
- }
- elsif ($arg=~/^-(jp|japanese)$/) {
- $utf8flag=1;
- @WordPatterns=($NamedWordPattern{'japanese'},@WordPatterns);
+ elsif ($arg=~/^-dir=(.*)$/) {
+ $globalworkdir=$1;
+ $globalworkdir=~s:([^\/\\])$:$1\/:;
}
+ elsif ($arg =~/^-(enc|encode|encoding)=(.+)$/) {$encoding=$2;}
+ elsif ($arg =~/^-(utf8|unicode)$/) {$encoding='utf8';}
+ elsif ($arg =~/^-(alpha(bets?)?)=(.*)$/) {set_script_options(\@AlphabetScripts,$3);}
+ elsif ($arg =~/^-(logo(grams?)?)=(.*)$/) {set_script_options(\@LogogramScripts,$3);}
+ elsif ($arg =~/^-([-a-z]+)$/ && set_language_option($1)) {}
elsif ($arg eq '-relaxed') {
- @MacroOptionPatterns=@MacroOptionPatternsRelaxed;
- $LetterPattern=$LetterPatternRelaxed;
+ $MacroOptionPattern=$NamedMacroOptionPattern{'relaxed'};
+ $LetterPattern=$NamedLetterPattern{'relaxed'};
+ }
+ elsif ($arg eq '-restricted') {
+ $MacroOptionPattern=$NamedMacroOptionPattern{'restricted'};
+ $LetterPattern=$NamedLetterPattern{'restricted'};
}
else {return 0;}
return 1;
}
-sub parse_options_sums {
+# Parse count and summation options
+sub parse_options_counts {
my $arg=shift @_;
- if ($arg=~/^-sum(=(.+))?$/) {option_sum($2);}
- elsif ($arg=~/^-(sub|subcounts?)(=(.+))?$/) {option_subcount($3);}
+ if ($arg =~/^-sum(=(.+))?$/) {_option_sum($2);}
+ elsif ($arg =~/^-nosum/) {@sumweights=();}
+ elsif ($arg =~/^-(sub|subcounts?)(=(.+))?$/) {_option_subcount($3);}
+ elsif ($arg =~/^-(nosub|nosubcounts?)/) {$showsubcounts=0;}
+ elsif ($arg eq '-freq') {$optionWordFreq=1;}
+ elsif ($arg =~/^-freq=(\d+)$/) {$optionWordFreq=$1;}
+ elsif ($arg eq '-stat') {$optionWordClassFreq=1;}
else {return 0;}
return 1;
}
-sub option_subcount {
+# Apply sum option
+sub _option_sum {
my $arg=shift @_;
- $showsubcounts=1;
+ if (!defined $arg) {
+ @sumweights=(1,1,1,0,0,1,1);
+ } elsif ($arg=~/^(\d+(\.\d*)?(,\d+(\.\d*)?){0,6})$/) {
+ @sumweights=split(',',$1);
+ } else {
+ print STDERR "Warning: Option value ".$arg." not valid, ignoring option.\n";
+ }
+}
+
+# Apply subcount options
+sub _option_subcount {
+ my $arg=shift @_;
+ $showsubcounts=2;
if (!defined $arg) {
%BreakPoints=%{$BreakPointsOptions{'default'}};
} elsif (my $option=$BreakPointsOptions{$arg}) {
@@ -315,55 +679,106 @@ sub option_subcount {
}
}
-sub option_sum {
+# Parse output and verbosity options
+sub parse_options_output {
my $arg=shift @_;
- if (!defined $arg) {
+ if ($arg eq '-strict') {$strictness=1;}
+ elsif ($arg eq '-v0') {$verbose=0;}
+ elsif ($arg eq '-v1') {$verbose=1;}
+ elsif ($arg eq '-v2') {$verbose=2;}
+ elsif ($arg eq '-v3' || $arg eq '-v') {$verbose=3;}
+ elsif ($arg eq '-v4') {$verbose=3; $showstates=1;}
+ elsif ($arg =~/^-showstates?$/ ) {$showstates=1;}
+ elsif ($arg =~/^-(q|quiet)$/ ) {$verbose=-1;}
+ elsif ($arg =~/^-(template)=(.*)$/ ) {_set_output_template($2);}
+ elsif ($arg eq '-split') {$optionFast=1;}
+ elsif ($arg eq '-nosplit') {$optionFast=0;}
+ elsif ($arg eq '-showver') {$showVersion=1;}
+ elsif ($arg eq '-nover') {$showVersion=-1;}
+ elsif ($arg =~/^-nosep(s|arators?)?$/ ) {$separator='';}
+ elsif ($arg =~/^-sep(arators?)?=(.*)$/ ) {$separator=$2;}
+ else {return 0;}
+ return 1;
+}
+
+# Set output template
+sub _set_output_template {
+ my $template=shift @_;
+ $outputtemplate=$template;
+ if ($template=~/\{(S|SUM)[\?\}]/i && !@sumweights) {
@sumweights=(1,1,1,0,0,1,1);
- } elsif ($arg=~/^(\d+(,\d+){0,6})$/) {
- @sumweights=split(',',$1);
- } else {
- print STDERR "Warning: Option value ".$arg." not valid, ignoring option.\n";
+ }
+ if ($template=~/\{SUB\?/i && !$showsubcounts) {
+ _option_subcount();
}
}
+# Parse output formating options
sub parse_options_format {
my $arg=shift @_;
- if ($arg eq '-brief') {$briefsum=1;}
+ if ($arg eq '-brief') {$briefsum=1;}
elsif ($arg eq '-total') {$totalflag=1;}
+ elsif ($arg eq '-0') {$briefsum=1;$totalflag=1;$verbose=-1;$finalLineBreak=0;}
elsif ($arg eq '-1') {$briefsum=1;$totalflag=1;$verbose=-1;}
- elsif ($arg eq "-html" ) {option_no_colours();$htmlstyle = 2;}
- elsif ($arg eq "-htmlcore" ) {option_no_colours();$htmlstyle = 1;}
- elsif ($arg=~/^\-(nocol|nc$)/) {option_no_colours();}
- elsif ($arg eq '-codes') {
- $showcodes=2;
- if ($verbose==0) {$verbose=3;}
- }
+ elsif ($arg eq '-html' ) {option_ansi_colours(0);$htmlstyle = 2;}
+ elsif ($arg eq '-htmlcore' ) {option_ansi_colours(0);$htmlstyle = 1;}
+ elsif ($arg =~/^\-(nocol|nc$)/) {option_ansi_colours(0);}
+ elsif ($arg =~/^\-(col)$/) {option_ansi_colours(1);}
+ elsif ($arg eq '-codes') {$showcodes=2;}
elsif ($arg eq '-nocodes') {$showcodes=0;}
else {return 0;}
return 1;
}
-sub parse_options_output {
+# Include options from option file
+sub _parse_optionfile {
+ my $filename=shift @_;
+ open(FH,"<",$filename)
+ || die "Option file not found: ".$filename."\n";
+ my @options=<FH>;
+ close(FH);
+ s/^\s*(#.*|)//s for @options;
+ my $text=join('',@options);
+ $text=~s/(\n|\r|\r\n)\s*\\//g;
+ @options=split("\n",$text);
+ foreach my $arg (@options) {
+ __optionfile_tc($arg)
+ || parse_option($arg)
+ || die "Invalid option ".$arg." in ".$filename."\n";
+ }
+}
+
+# Parse option file TC options
+sub __optionfile_tc {
my $arg=shift @_;
- if ($arg eq "-v0") {$verbose=0;}
- elsif ($arg eq "-v1") {$verbose=1;}
- elsif ($arg eq '-vv' || $arg eq '-v2') {$verbose=2;}
- elsif ($arg eq '-vvv' || $arg eq '-v3' || $arg eq '-v') {$verbose=3;}
- elsif ($arg eq '-vvvv' || $arg eq '-v4') {$verbose=3; $showstates=1;}
- elsif ($arg =~ /^\-showstates?$/ ) {$showstates=1;}
- elsif ($arg =~ /^-(q|-?quiet)$/ ) {$verbose=-1;}
- else {return 0;}
+ $arg=~s/^\%\s*// || return 0;
+ if ($arg=~/^subst\s+(\\\w+)\s+/i) {
+ $substitutions{$1}=$';
+ } elsif ($arg=~/^(\w+)\s+([\\]*\w+)\s+([^\s\n]+)(\s+([0-9]+))?/i) {
+ tc_macro_param_option($1,$2,$3,$5) || die "Invalid TC option: ".$arg."\n";
+ } else {
+ print "Invalid TC option format: ".$arg."\n";
+ return 0;
+ }
return 1;
}
# Parse file list and return total count
-sub parse_file_list {
- my @filelist=@_;
- my $listtotalcount=new_count("TOTAL COUNT");
- for my $file (<@filelist>) {
- my $filetotalcount=parse_file($file);
- add_count($listtotalcount,$filetotalcount);
+sub Parse_file_list {
+ my @files=@_;
+ my $listtotalcount=new_count("Total");
+ foreach (@files) {s/\\/\//g; s/ /\\ /g;}
+ if (@files) {
+ @files=<@files>; # For the sake of Windows: expand wildcards!
+ for my $file (@files) {
+ my $filetotalcount=parse_file($file);
+ add_to_total($listtotalcount,$filetotalcount);
+ }
}
+ if ($fileFromSTDIN) {
+ my $filetotalcount=parse_file($_STDIN_);
+ add_to_total($listtotalcount,$filetotalcount);
+ }
return $listtotalcount;
}
@@ -375,30 +790,36 @@ sub parse_file {
$workdir=$file;
$workdir =~ s/^((.*[\\\/])?)[^\\\/]+$/$1/;
}
- @filelist=($file);
- if ($htmlstyle) {print "\n<div class='filegroup'>\n";}
- my $filetotalcount=new_count("SUM COUNT FOR ".$file);
+ if ($htmlstyle && ($verbose || !$totalflag)) {print "\n<div class='filegroup'>\n";}
+ my $filetotalcount=new_count("File(s) total: ".$file);
+ @filelist=();
+ _add_file($filetotalcount,$file);
foreach my $f (@filelist) {
- my $tex=TeXfile($f);
- my $fpath=$f;
- $fpath=~s/^((.*[\\\/])?)[^\\\/]+$/$1/;
- if (!defined $tex) {
- print STDERR "File not found or not readable: ".$f."\n";
- formatprint("File not found or not readable: ".$f."\n","p","error");
- } else {
- parse($tex);
- my $filecount=add_subcount($tex);
- if (!$totalflag) {
- print_count($filecount);
- print "\n";
- }
- add_count($filetotalcount,$filecount);
- }
+ _add_file($filetotalcount,$f,"Included file: ".$f);
}
- if ($htmlstyle) {print "</div>\n\n";}
+ if (!$totalflag && get_count($filetotalcount,$CNT_FILE)>1) {
+ if ($htmlstyle) {formatprint("Sum of files: ".$file."\n",'h2');}
+ print_count($filetotalcount,'sumcount');
+ }
+ if ($htmlstyle && ($verbose || !$totalflag)) {print "</div>\n\n";}
return $filetotalcount;
}
+# Parse single file, included files will be appended to @filelist.
+sub _add_file {
+ my ($filetotalcount,$f,$title)=@_;
+ my $tex=TeXfile($f,$title);
+ my $fpath=$f;
+ $fpath=~s/^((.*[\\\/])?)[^\\\/]+$/$1/;
+ if (!defined $tex) {
+ error($Main,'File not found or not readable: '.$f);
+ } else {
+ parse($tex);
+ my $filecount=next_subcount($tex);
+ if (!$totalflag) {print_count($filecount);}
+ add_to_total($filetotalcount,$filecount);
+ }
+}
######
###### Subroutines
@@ -407,102 +828,272 @@ sub parse_file {
###### CMD specific implementations
-sub add_file_to_list {
- my $fname=shift @_;
- push @filelist,$workdir.$fname;
-}
-
-sub print_with_style {
- my ($text,$style,$colour)=@_;
- #if ($utf8flag || $htmlstyle) {utf8::encode($text);}
- if ($htmlstyle) {
- print "<span class='".$style."'>".$text."</span>";
+# Add file to list of files scheduled for parsing
+sub include_file {
+ my ($tex,$fname)=@_;
+ my $fpath=$workdir.$fname;
+ if ($includeTeX==2) {
+ my $bincode=read_binary($fpath) || BLOCK {
+ error($tex,"File $fpath not found.");
+ return;
+ };
+ flush_next($tex);
+ line_return(0,$tex);
+ prepend_code($tex,$bincode,$fname);
} else {
- print Term::ANSIColor::colored($text,$colour);
+ push @filelist,$fpath;
}
}
-sub option_no_colours {
- $ENV{'ANSI_COLORS_DISABLED'} = 1;
-}
-
# Print count (total) if conditions are met
sub conditional_print_total {
my $sumcount=shift @_;
- if ($totalflag || get_count($sumcount,0)>1) {
+ if ($totalflag || number_of_subcounts($sumcount)>1) {
if ($totalflag && $briefsum && @sumweights) {
- print total_count($sumcount);
+ print get_sum_count($sumcount),"\n";
} else {
- if ($htmlstyle) {
- formatprint("Total word count",'h2');
- }
- print_count($sumcount);
+ if ($htmlstyle) {formatprint("Total word count",'h2');}
+ print_count($sumcount,'sumcount');
}
}
}
+# Set or unset use of ANSI colours
+sub option_ansi_colours {
+ my $flag=shift @_;
+ $ENV{'ANSI_COLORS_DISABLED'} = $flag?undef:1;
+}
+
+# Print text using ANSI colours
+sub ansiprint {
+ my ($text,$colour)=@_;
+ print Term::ANSIColor::colored($text,$colour);
+}
+
###### Option handling
# Apply options to set values
sub Apply_Options {
+ if ($showcodes>1 && $verbose<1) {$verbose=3;}
%STYLE=%{$STYLES[$verbose]};
- if ($utf8flag) {binmode STDOUT,':utf8';}
+ if (defined $encoding && $encoding eq 'guess') {$encoding=undef;}
+ if (!defined $encoding) {
+ } elsif (ref(find_encoding($encoding))) {
+ if (!$htmlstyle) {$outputEncoding=$encoding;}
+ } else {
+ error($Main,"Unknown encoding $encoding ignored.");
+ error_details($Main,'Valid encodings are: '.wrap('','',join(', ',Encode->encodings(':all'))));
+ $encoding=undef;
+ }
+ if (!defined $outputEncoding) {$outputEncoding='utf8';}
+ binmode STDIN;
+ binmode STDOUT,':encoding('.$outputEncoding.')';
if ($htmlstyle>1) {html_head();}
- foreach (@WordPatterns) {
- s/\@/$LetterPattern/g;
+ flush_errorbuffer($Main);
+ apply_language_options();
+ if ($includeBibliography) {apply_include_bibliography();}
+}
+
+# Set or add scripts to array of scripts
+sub set_script_options {
+ my ($scriptset,$str)=@_;
+ if ($str=~s/^\+//) {} else {splice(@$scriptset,0,scalar $scriptset);}
+ my @scripts=split(/[+,]/,$str);
+ foreach my $scr (@scripts) {
+ $scr=~tr/_/ /;
+ if ($scr eq 'Alphabetic') {
+ warning($Main,'Using alphabetic instead of Unicode class Alphabetic');
+ $scr='alphabetic';
+ }
+ if ($scr=~/^[a-z]/) {$scr='Is_'.$scr;}
+ if (is_property_valid($scr)) {push @$scriptset,$scr;}
+ else {error($Main,"Unknown script $scr ignored.");}
}
}
+sub is_property_valid {
+ my $script=shift @_;
+ eval {' '=~/\p{$script}/};
+ if ($@) {return 0;} else {return 1;}
+}
+
+# Set language option, return language if applied, undef if not
+sub set_language_option {
+ my $language=shift @_;
+ if ($language=~/^(count\-?)all$/) {
+ @AlphabetScripts=qw/Digit Is_alphabetic/;
+ @LogogramScripts=qw/Ideographic Katakana Hiragana Thai Lao Hangul/;
+ } elsif ($language=~/^words(-?only)?$/) {
+ @LogogramScripts=();
+ } elsif ($language=~/^(ch|chinese|zhongwen)(-?only)?$/) {
+ @LogogramScripts=qw/Han/;
+ if (defined $2) {$LetterPattern=undef;}
+ @encodingGuessOrder=@{$NamedEncodingGuessOrder{'chinese'}};
+ return 'chinese';
+ } elsif ($language=~/^(jp|japanese)(-?only)?$/) {
+ @LogogramScripts=qw/Han Hiragana Katakana/;
+ if (defined $2) {$LetterPattern=undef;}
+ @encodingGuessOrder=@{$NamedEncodingGuessOrder{'japanese'}};
+ return 'japanese';
+ } elsif ($language=~/^(kr|korean)(-?only)?$/) {
+ @LogogramScripts=qw/Han Hangul/;
+ if (defined $2) {$LetterPattern=undef;}
+ @encodingGuessOrder=@{$NamedEncodingGuessOrder{'korean'}};
+ return 'korean';
+ } elsif ($language=~/^(kr|korean)-?words?(-?only)?$/) {
+ if (defined $2) {
+ @AlphabetScripts=qw/Hangul/;
+ @LogogramScripts=qw/Han/;
+ } else {
+ @AlphabetScripts=qw/Digit Is_alphabetic Hangul/;
+ @LogogramScripts=qw/Han Katakana Hiragana Thai Lao/;
+ }
+ @encodingGuessOrder=@{$NamedEncodingGuessOrder{'korean'}};
+ return 'korean-words';
+ } elsif ($language=~/^(char|character|letter)s?(-?only)?$/) {
+ @WordPatterns=($NamedWordPattern{'letters'});
+ if (defined $2) {@LogogramScripts=();}
+ $countlabel[1]='Letters in text';
+ $countlabel[2]='Letters in headers';
+ $countlabel[3]='Letters in captions';
+ return 'letter';
+ } else {
+ return undef;
+ }
+}
-###### TeX code handle
+# Apply language options
+sub apply_language_options {
+ my @tmp;
+ if (defined $LetterPattern && @AlphabetScripts && scalar @AlphabetScripts>0) {
+ @tmp=@AlphabetScripts;
+ foreach (@tmp) {$_='\\p{'.$_.'}';}
+ my $letterchars='['.join('',@tmp).']';
+ my $letter=$LetterPattern;
+ $letter=~s/@/$letterchars/g;
+ @WordPatterns=map { s/\@/$letter/g ; qr/$_/ } @WordPatterns;
+ } else {
+ @WordPatterns=();
+ }
+ if (@LogogramScripts && scalar @LogogramScripts>0) {
+ @tmp=@LogogramScripts;
+ foreach (@tmp) {$_='\\p{'.$_.'}';}
+ push @WordPatterns,'['.join('',@tmp).']';
+ }
+ if (scalar @WordPatterns==0) {
+ error($Main,'No script (alphabets or logograms) defined. Using fallback mode.');
+ push @WordPatterns,'\\w+';
+ }
+ $WordPattern=join '|',@WordPatterns;
+}
+# Apply incbib rule
+sub apply_include_bibliography {
+ include_package('%incbib');
+}
-sub TeXfile {
- my $filename=shift @_;
- my $file=read_file($filename) || return undef;
- return TeXcode($file,$filename);
+# Process package inclusion
+sub include_package {
+ my $incpackage=shift @_;
+ my $sub;
+ _add_to_hash_if_exists(\%TeXmacro,\%PackageTeXmacro,$incpackage);
+ _add_to_hash_if_exists(\%TeXmacroword,\%PackageTeXmacroword,$incpackage);
+ _add_to_hash_if_exists(\%TeXheader,\%PackageTeXheader,$incpackage);
+ _add_to_hash_if_exists(\%TeXgroup,\%PackageTeXgroup,$incpackage);
+ _add_to_hash_if_exists(\%TeXfileinclude,\%PackageTeXfileinclude,$incpackage);
}
-sub read_file {
- my $filename=shift @_;
- if ($utf8flag) {
- open(FH,"<:utf8",$filename) || return undef;
- } else {
- open(FH,"<".$filename) || return undef;
- }
- if ($verbose>0) {
- formatprint("File: ".$filename."\n",'h2');
- $blankline=0;
+# Add package rules if defined
+sub _add_to_hash_if_exists {
+ my ($target,$source,$name)=@_;
+ my $sub;
+ if ($sub=$source->{$name}) {
+ while (my ($key,$val)=each(%$sub)) {
+ $target->{$key}=$val;
+ }
}
- my @text=<FH>;
- close(FH);
- my $latexcode=join('',@text);
- if ($utf8flag) {
- $latexcode =~ s/^\x{feff}//;
+}
+
+# Process TC instruction
+sub tc_macro_param_option {
+ my ($instr,$macro,$param,$option)=@_;
+ if ($param=~/^\[([0-9,]+)\]$/) {$param=[split(',',$1)];}
+ if (!defined $option) {$option=1;}
+ if (($instr eq 'macro') || ($instr eq 'exclude')) {$TeXmacro{$macro}=$param;}
+ elsif ($instr eq 'header') {$TeXheader{$macro}=$param;$TeXmacro{$macro}=$param;}
+ elsif ($instr eq 'macroword') {$TeXmacroword{$macro}=$param;}
+ elsif ($instr eq 'preambleinclude') {$TeXpreamble{$macro}=$param;}
+ elsif ($instr eq 'group') {
+ $TeXmacro{'begin'.$macro}=$param;
+ $TeXgroup{$macro}=$option;
}
- return $latexcode;
+ elsif ($instr eq 'floatinclude') {$TeXfloatinc{$macro}=$param;}
+ elsif ($instr eq 'fileinclude') {$TeXfileinclude{$macro}=$param;}
+ elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;}
+ else {return 0;}
+ return 1;
}
-###### Parsing routines
+
+###### TeX code handle
+
-# Make TeXcode handle
+# Return object capable of capturing errors for use when
+# no TeXcode object is available.
+sub getMain {
+ my %main=();
+ $main{'errorcount'}=0;
+ $main{'errorbuffer'}=[];
+ $main{'warnings'}={};
+ return \%main;
+}
+
+
+## Make TeX handle for LaTeX code: the main TeXcount object
+# The "TeX object" is a data containser: a hash containing
+# - filename: name of LaTeX file being parsed
+# - filepath: path to LaTeX file
+# - countsum: count object with total count (incl. subcounts)
+# - subcount: count object for subcount (to be added to countsum)
+# - subcounts: list of subcounts
+# plus following elements used for the processing the LaTeX code
+# - line: the LaTeX paragraph being processed
+# - texcode: what remains of LaTeX code to process (after line)
+# - texlength: length of LaTeX code
+# - next: the next token, i.e. the one being processed
+# - type: the type of the next token
+# - style: the present output style (for verbose output)
+# - printstate: present parsing state (for verbose output only)
+# - eof: set once the end of the input is reached
+# which are passed to methods by passing the TeX object. It is used when parsing
+# the LaTeX code, and discarded once the parsing is done. During parsing, counts
+# are added to the subcount element; whenever a break point is encountered, the
+# subcount is added to the countsum (next_subcount) and a new subcount object
+# prepared. Note that this requires that the last subcount object be added to
+# the countsum once the end of the document is reached.
sub TeXcode {
- my ($texcode,$filename,$title)=@_;
- my %TeX=();
- $TeX{'filename'}=$filename;
- if (!defined $filename) {
- $TeX{'filepath'}='';
- } elsif ($filename=~/^(.*[\\\/])[^\\\/]+$/) {
- $TeX{'filepath'}=$1;
- } else {
- $TeX{'filepath'}='';
- }
+ my ($bincode,$filename,$title)=@_;
+ my $tex=_TeXcode_blank($filename,$title);
+ _TeXcode_setcode($tex,$bincode);
+ more_texcode($tex);
+ return $tex;
+}
+
+# Return a blank TeXcode object
+sub _TeXcode_blank {
+ my ($filename,$title)=@_;
if (defined $title) {}
- elsif (defined $filename) {$title="FILE: ".$filename;}
+ elsif (defined $filename) {$title="File: ".$filename;}
else {$title="Word count";}
- $TeX{'line'}=$texcode;
+ my %TeX=();
+ $TeX{'errorcount'}=0;
+ $TeX{'filename'}=$filename;
+ if (!defined $filename) {$TeX{'filepath'}='';}
+ elsif ($filename=~/^(.*[\\\/])[^\\\/]+$/) {$TeX{'filepath'}=$1;}
+ else {$TeX{'filepath'}='';}
+ $TeX{'line'}='';
$TeX{'next'}=undef;
$TeX{'type'}=undef;
$TeX{'style'}=undef;
@@ -510,27 +1101,216 @@ sub TeXcode {
$TeX{'eof'}=0;
my $countsum=new_count($title);
$TeX{'countsum'}=$countsum;
+ $countsum->{'TeXcode'}=\%TeX;
my $count=new_count("_top_");
- $TeX{'count'}=$count;
- inc_count(\%TeX,0);
+ $TeX{'subcount'}=$count;
+ inc_count(\%TeX,$CNT_FILE);
my @countlist=();
- $TeX{'countlist'}=\@countlist;
$countsum->{'subcounts'}=\@countlist;
return \%TeX;
}
+# Set the texcode element of the TeXcount object
+sub _TeXcode_setcode {
+ my ($tex,$bincode)=@_;
+ $tex->{'texcode'}=_prepare_texcode($tex,$bincode);
+ $tex->{'texlength'}=length($tex->{'texcode'});
+ # TODO: Comment out this test
+ if ($tex->{'texcode'} =~ /([[:^ascii:]])/) {warning($tex,"Code contains non-ASCII characters.");}
+}
+
+# Decode and return TeX/LaTeX code
+sub _prepare_texcode {
+ my ($tex,$texcode)=@_;
+ $texcode=_decode_texcode($tex,$texcode);
+ foreach my $key (keys %substitutions) {
+ my $value=$substitutions{$key};
+ $texcode=~s/(\w)\Q$key\E/$1 $value/g;
+ $texcode=~s/\Q$key\E/$value/g;
+ }
+ return $texcode;
+}
+
+# Return text decoder
+sub _decode_texcode {
+ my ($tex,$texcode)=@_;
+ my $decoder;
+ if (defined $encoding) {
+ $decoder=find_encoding($encoding);
+ eval {
+ $texcode=$decoder->decode($texcode);
+ };
+ if ($@) {
+ error($tex,'Decoding file/text using the '.$decoder->name.' encoding failed.');
+ }
+ } else {
+ ($texcode,$decoder)=_guess_encoding($texcode);
+ if (!ref($decoder)) {
+ error($tex,'Failed to identify encoding or incorrect encoding specified.');
+ $tex->{'encoding'}='[FAILED]';
+ return $texcode;
+ }
+ }
+ __set_encoding_name($tex,$decoder->name);
+ $texcode =~ s/^\x{feff}//; # Remove BOM (relevant for UTF only)
+ if ($texcode =~/\x{fffd}/ ) {
+ error($tex,'File/text was not valid '.$decoder->name.' encoded.');
+ }
+ return $texcode;
+}
+
+# Guess the right encoding to use
+sub _guess_encoding {
+ my ($texcode)=@_;
+ foreach my $enc (@encodingGuessOrder) {
+ my $dec=find_encoding($enc);
+ if (ref($dec)) {
+ eval {
+ $texcode=$dec->decode($texcode,Encode::FB_CROAK)
+ };
+ if (!$@) {return $texcode,$dec;}
+ }
+ }
+ return $texcode,undef;
+}
+
+# Set name of current encoding
+sub __set_encoding_name {
+ my ($tex,$enc)=@_;
+ my $cur=$tex->{'encoding'};
+ if (!defined $enc) {$enc='[FAILED]';} # Shouldn't happen here though...
+ if (!defined $cur) {}
+ elsif ($enc eq 'ascii') {$enc=$cur;}
+ elsif ($cur eq 'ascii') {}
+ elsif ($cur ne $enc) {
+ error($tex,"Mismatching encodings: $cur versus $enc.");
+ }
+ $tex->{'encoding'}=$enc;
+}
+
+# Apply substitution rule
+sub apply_substitution_rule {
+ my ($tex,$from,$to)=@_;
+ $tex->{'line'}=~s/(\w)\Q$from\E\b\s*/$1 $to/g;
+ $tex->{'line'}=~s/\Q$from\E\b\s*/$to/g;
+ $tex->{'texcode'}=~s/(\w)\Q$from\E\b\s*/$1 $to/g;
+ $tex->{'texcode'}=~s/\Q$from\E\b\s*/$to/g;
+}
+
+## Get more TeX code from texcode buffer if possible, return 1 if done
+sub more_texcode {
+ my ($tex)=@_;
+ if (!defined $tex->{'texcode'}) {return 0;}
+ if ( $optionFast && $tex->{'texcode'} =~ s/^.*?(\r{2,}|\n{2,}|(\r\n){2,})//s ) {
+ $tex->{'line'}.=$&;
+ return 1;
+ }
+ $tex->{'line'}.=$tex->{'texcode'};
+ $tex->{'texcode'}=undef;
+ return 1;
+}
+
+## Prepend LaTeX code to TeXcode object
+sub prepend_code {
+ my ($tex,$code,$fname)=@_;
+ my $prefix="\n% --- Start of included file ".$fname."\n";
+ my $suffix="\n% --- End of included file ".$fname."\n";
+ $code=_decode_texcode($tex,$code);
+ $tex->{'length'}+=length($code);
+ $tex->{'texcode'}=$prefix.$code.$suffix.$tex->{'line'}.$tex->{'texcode'};
+ $tex->{'line'}='';
+ more_texcode();
+}
+
+## Returns size of TeX code in bytes
+sub get_texsize {
+ my $tex=shift @_;
+ return $tex->{'texlength'}
+}
+
+
+###### TeX file reader
+
+
+# Read LaTeX file into TeX object
+sub TeXfile {
+ my ($filename,$title)=@_;
+ if ($filename eq $_STDIN_) {
+ if ($verbose>0) {
+ formatprint("File from STDIN\n",'h2');
+ $blankline=0;
+ }
+ return TeXcode(_read_stdin(),'STDIN',$title);
+ } else {
+ my $bincode=read_binary($filename) || return undef;
+ if ($verbose>0) {
+ formatprint("File: ".$filename."\n",'h2');
+ $blankline=0;
+ }
+ return TeXcode($bincode,$filename,$title);
+ }
+}
+
+# Read file to string without regard for encoding
+sub read_binary {
+ my $filename=shift @_;
+ open(FH,$filename) || return undef;
+ binmode(FH);
+ my $bincode;
+ read(FH,$bincode,-s FH);
+ close(FH);
+ return $bincode;
+}
+
+# Read file from STDIN
+sub _read_stdin {
+ my @text=<STDIN>;
+ my $latexcode=join('',@text);
+ return $latexcode;
+}
+
+###### Error handling
+
+
+# Add warning to list of registered warnings (optionally to be reported at the end)
+sub warning {
+ my ($tex,$text)=@_;
+ $tex->{'warnings'}->{$text}++;
+ $warnings{$text}++;
+}
+
+# Register error and print error message
+sub error {
+ my ($tex,$text,$type)=@_;
+ if (defined $type) {$text=$type.': '.$text;}
+ $errorcount++;
+ $tex->{'errorcount'}++;
+ #print STDERR $text,"\n";
+ if (my $err=$tex->{'errorbuffer'}) {push @$err,$text;}
+ if ($verbose>=0) {_print_error($text);}
+}
+
+# Print error details
+sub error_details {
+ my ($tex,$text)=@_;
+ print STDERR $text,"\n";
+}
+
+###### Parsing routines
+
+
# Parse LaTeX document
sub parse {
my ($tex)=@_;
- if ($htmlstyle && $verbose) {print "<p class=parse>\n";}
+ if ($htmlstyle && $verbose) {print "<div class='parse'><p>\n";}
while (!($tex->{'eof'})) {
- parse_unit($tex,1);
+ _parse_unit($tex,$STATUS_TEXT);
}
- if ($htmlstyle && $verbose) {print "</p>\n";}
+ if ($htmlstyle && $verbose) {print "</p></div>\n";}
}
# Parse one block or unit
-sub parse_unit {
+sub _parse_unit {
# Status:
# 0 = exclude from count
# 1 = text
@@ -539,256 +1319,388 @@ sub parse_unit {
# -1 = float (exclude)
# -2 = strong exclude, ignore begin-end groups
# -3 = stronger exclude, do not parse macro parameters
+ # -4 = ignore everything except end marker: even {
# -9 = preamble (between \documentclass and \begin{document})
my ($tex,$status,$end)=@_;
if (!defined $status) {
- print_error("CRITICAL ERROR: Undefined parser status!");
+ error($tex,'Undefined parser status!','CRITICAL ERROR');
exit;
} elsif (ref($status) eq 'ARRAY') {
- print_error("CRITICAL ERROR: Invalid parser status!");
+ error($tex,'Invalid parser status!','CRITICAL ERROR');
exit;
}
- my $substat;
if ($showstates) {
if (defined $end) {
- $tex->{'printstate'}=':'.$status.':'.$end.':';
+ $tex->{'printstate'}=':'.status_to_text($status).':'.$end.':';
} else {
- $tex->{'printstate'}=':'.$status.':';
+ $tex->{'printstate'}=':'.status_to_text($status).':';
}
flush_next($tex);
}
- while (defined (my $next=next_token($tex))) {
- # parse next token; or tokens until match with $end
+ while (defined (my $next=_next_token($tex))) {
+ # Parse next token until token matches $end
set_style($tex,"ignore");
- if ((defined $end) && ($end eq $next)) {
- # end of unit
- return;
- } elsif (!defined $next) {
- print_error("ERROR: End of file while waiting for ".$end);
- return;
- }
- if ($status==-9 && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) {
+ if ((defined $end) && ($end eq $next)) {return;}
+ # Determine how token should be interpreted
+ if ($status==$STATUS_PREAMBLE && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) {
# \begin{document}
- $status=1;
+ $status=$STATUS_TEXT;
}
- if ($next eq '\documentclass') {
+ if ($status==$STATUS_EXCLUDE_ALL) {
+ # Ignore everything
+ } elsif ($tex->{'type'}==$TOKEN_SPACE) {
+ # space or other code that should be passed through without styling
+ flush_next($tex,' ');
+ } elsif ($next eq '\documentclass') {
# starts preamble
set_style($tex,'document');
- gobble_option($tex);
- gobble_macro_parms($tex,1);
+ __gobble_option($tex);
+ __gobble_macro_parms($tex,1);
while (!($tex->{'eof'})) {
- parse_unit($tex,-9);
+ _parse_unit($tex,$STATUS_PREAMBLE);
}
- } elsif ($tex->{'type'}==666) {
+ } elsif ($tex->{'type'}==$TOKEN_TC) {
# parse TC instructions
- parse_tc($tex);
- } elsif ($tex->{'type'}==1) {
+ _parse_tc($tex,$next);
+ } elsif ($tex->{'type'}==$TOKEN_WORD) {
# word
- if ($status>0) {
- inc_count($tex,$status);
- set_style($tex,'word'.$status);
+ if (my $cnt=status_text_cnt($status)) {
+ _process_word($tex,$next,$status);
+ inc_count($tex,$cnt);
+ set_style($tex,status_to_style($status));
}
} elsif ($next eq '{') {
# {...}
- parse_unit($tex,$status,'}');
- } elsif ($tex->{'type'}==3 && $status==-3) {
+ _parse_unit($tex,$status,'}');
+ } elsif ($next eq '}') {
+ error($tex,'Encountered } without corresponding {.');
+ } elsif ($tex->{'type'}==$TOKEN_MACRO && $status==$STATUS_EXCLUDE_STRONGER) {
+ # ignore macro call
set_style($tex,'ignore');
- } elsif ($tex->{'type'}==3) {
+ } elsif ($tex->{'type'}==$TOKEN_MACRO) {
# macro call
- parse_macro($tex,$next,$status,$substat);
+ _parse_macro($tex,$next,$status);
} elsif ($next eq '$') {
# math inline
- parse_math($tex,$status,6,'$');
+ _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'$');
} elsif ($next eq '$$') {
# math display (unless already in inlined math)
if (!(defined $end && $end eq '$')) {
- parse_math($tex,$status,7,'$$');
+ _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'$$');
}
}
if (!defined $end) {return;}
}
+ if (defined $end) {
+ error($tex,'Reached end of file while waiting for '.$end.'.');
+ }
+}
+
+# Process word with a given status (>0, i.e. counted)
+sub _process_word {
+ my ($tex,$word,$status)=@_;
+ $WordFreq{$word}++;
}
-sub parse_macro {
- my ($tex,$next,$status,$substat)=@_;
+# Parse unit when next token is a macro
+sub _parse_macro {
+ my ($tex,$next,$status)=@_;
+ # substat = parameter status settings
+ my $substat;
if (my $label=$BreakPoints{$next}) {
- if ($tex->{'line'}=~ /^[*]?(\s*\[.*?\])*\s*\{(.+?)\}/ ) {
+ if ($tex->{'line'}=~ /^[*]?(\s*\[.*?\])*\s*\{((.|\{.*\})*)\}/ ) {
$label=$label.': '.$2;
}
- add_subcount($tex,$label);
+ next_subcount($tex,$label);
}
- set_style($tex,$status>0?'command':'exclcommand');
- if ($next eq '\begin' && $status!=-2) {
- parse_begin_end($tex,$status);
- } elsif (($status==-1) && ($substat=$TeXfloatinc{$next})) {
+ set_style($tex,status_is_text($status)?'command':'exclcommand');
+ if ($next eq '\begin' && status_inc_envir($status)) {
+ _parse_begin_end($tex,$status);
+ } elsif ($next eq '\end' && status_inc_envir($status)) {
+ error($tex,'Encountered \end without corresponding \begin');
+ } elsif ($next eq '\verb') {
+ _parse_verb_region($tex,$status);
+ } elsif ( (status_is_text($status) || $status==$STATUS_PREAMBLE)
+ && defined ($substat=$TeXpackageinc{$next}) ) {
+ _parse_include_package($tex,$substat);
+ } elsif (($status==$STATUS_FLOAT) && ($substat=$TeXfloatinc{$next})) {
# text included from float
set_style($tex,'command');
- gobble_macro_parms($tex,$substat);
- } elsif ($status==-9 && defined ($substat=$TeXpreamble{$next})) {
- # parse preamble include macros
+ __gobble_macro_parms($tex,$substat);
+ } elsif ($status==$STATUS_PREAMBLE && defined ($substat=$TeXpreamble{$next})) {
+ # parse preamble include macros
set_style($tex,'command');
- if (defined $TeXheader{$next}) {inc_count($tex,4);}
- gobble_macro_parms($tex,$substat,1);
- } elsif ($status<0) {
- # ignore
- gobble_option($tex);
+ if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);}
+ __gobble_macro_parms($tex,$substat,1);
+ } elsif (status_is_exclude($status)) {
+ # ignore
+ __gobble_option($tex);
} elsif ($next eq '\(') {
# math inline
- parse_math($tex,$status,6,'\)');
+ _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'\)');
} elsif ($next eq '\[') {
# math display
- parse_math($tex,$status,7,'\]');
+ _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'\]');
} elsif ($next eq '\def') {
# ignore \def...
$tex->{'line'} =~ s/^([^\{]*)\{/\{/;
flush_next($tex);
- print_style($1.' ','ignore');
- parse_unit($tex,-2);
+ print_style($1,'ignore');
+ _parse_unit($tex,$STATUS_EXCLUDE_STRONG);
} elsif (defined (my $addsuffix=$TeXfileinclude{$next})) {
- # include file: queue up for parsing
- parse_include_file($tex,$status,$addsuffix);
+ # include file: queue up for parsing
+ _parse_include_file($tex,$status,$addsuffix);
} elsif (defined ($substat=$TeXmacro{$next})) {
# macro: exclude options
- if (defined $TeXheader{$next}) {inc_count($tex,4);}
- gobble_macro_parms($tex,$substat,$status);
- } elsif (defined ($substat=$TeXmacroword{$next})) {
- # count macro as word (or a given number of words)
- inc_count($tex,$status,$substat);
+ if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);}
+ __count_macroword($tex,$next,$status);
+ __gobble_macro_parms($tex,$substat,$status);
+ } elsif (defined __count_macroword($tex,$next,$status)) {
+ # count macro as word (or a given number of words)
set_style($tex,'word'.$status);
} elsif ($next =~ /^\\[^\w\_]/) {
} else {
- gobble_option($tex);
+ __gobble_option($tex);
}
}
-sub parse_tc {
- my ($tex)=@_;
- my $next=$tex->{'next'};
+# Parse TC instruction
+sub _parse_tc {
+ my ($tex,$next)=@_;
set_style($tex,'tc');
flush_next($tex);
- if (!($next=~s/^\%+TC:\s*(\w+)\s*// )) {
- print_error('Warning: TC command should have format %TC:instruction [macro] [parameters]');
+ $next=~s/^\%+TC:\s*(\w+)\s*//i || BLOCK {
+ error($tex,'TC command should have format %TC:instruction [macro] [parameters]');
return;
};
my $instr=$1;
+ $instr=~tr/[A-Z]/[a-z]/;
if ($instr=~/^(break)$/) {
- if ($instr eq 'break') {add_subcount($tex,$next);}
- } elsif ($next=~/^([\\]*\w+)\s+([^\s\n]+)(\s+([0-9]+))?/) {
+ if ($instr eq 'break') {next_subcount($tex,$next);}
+ } elsif ($instr=~/^(incbib|includebibliography)$/) {
+ $includeBibliography=1;
+ apply_include_bibliography();
+ } elsif ($instr eq 'ignore') {
+ __gobble_tc_ignore($tex);
+ } elsif ($instr eq 'endignore') {
+ error($tex,'TC:endignore without corresponding TC:ignore.');
+ } elsif ($instr eq 'newtemplate') {$outputtemplate='';
+ } elsif ($instr eq 'template') {$outputtemplate.=$next;
+ } elsif ($instr eq 'subst') {
+ if ($next=~/^(\\\S+)\s+/) {
+ my $from=$1;
+ my $to=$';
+ $substitutions{$from}=$to;
+ apply_substitution_rule($tex,$from,$to);
+ } else {
+ error($tex,'Invalid TC:subst format.');
+ }
+ } elsif ($next=~/^([\\]*\S+)\s+([^\s\n]+)(\s+(-?[0-9]+))?/) {
# Format = TC:word macro
my $macro=$1;
my $param=$2;
my $option=$4;
- if ($param=~/^\[([0-9,]+)\]$/) {$param=[split(',',$1)];}
- if (($instr eq 'macro') || ($instr eq 'exclude')) {$TeXmacro{$macro}=$param;}
- elsif ($instr eq 'header') {$TeXheader{$macro}=$param;$TeXmacro{$macro}=$param;}
- elsif ($instr eq 'macroword') {$TeXmacroword{$macro}=$param;}
- elsif ($instr eq 'preambleinclude') {$TeXpreamble{$macro}=$param;}
- elsif ($instr eq 'group') {
- $TeXmacro{'begin'.$macro}=$param;
- $TeXgroup{$macro}=$option;
- }
- elsif ($instr eq 'floatinclude') {$TeXfloatinc{$macro}=$param;}
- elsif ($instr eq 'fileinclude') {$TeXfileinclude{$macro}=$param;}
- elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;}
- else {print_error("Warning: Unknown TC command: ".$instr);}
- } elsif ($instr eq 'ignore') {
- tc_ignore_input($tex);
+ if (tc_macro_param_option($instr,$macro,$param,$option)) {}
+ else {error($tex,'Unknown TC command: '.$instr);}
} else {
- print_error("Warning: Invalid TC command format: ".$instr);
+ error($tex,'Invalid TC command format: '.$instr);
}
}
-sub tc_ignore_input {
+# Parse through ignored LaTeX code
+sub __gobble_tc_ignore {
my ($tex)=@_;
set_style($tex,'ignore');
- parse_unit($tex,-3,"%TC:endignore");
+ _parse_unit($tex,$STATUS_EXCLUDE_ALL,'%TC:endignore');
set_style($tex,'tc');
flush_next($tex);
}
-sub parse_math {
- my ($tex,$status,$substat,$end)=@_;
- my $localstyle=$status>0 ? 'mathgroup' : 'exclmath';
- if ($status>0) {inc_count($tex,$substat);}
+# Parse math formulae
+sub _parse_math {
+ my ($tex,$status,$cnt,$end)=@_;
+ my $localstyle;
+ if (status_is_text($status)) {
+ $localstyle='mathgroup';
+ inc_count($tex,$cnt);
+ } else {
+ $localstyle='exclmath';
+ }
set_style($tex,$localstyle);
- parse_unit($tex,0,$end);
+ _parse_unit($tex,$STATUS_IGNORE,$end);
set_style($tex,$localstyle);
}
-sub parse_begin_end {
+# Parse \verb region
+sub _parse_verb_region {
+ my ($tex,$status)=@_;
+ flush_next($tex);
+ set_style($tex,'ignore');
+ if (!($tex->{'line'} =~ s/^[^\s]// )) {
+ error($tex,'Invalid \verb: delimiter required.');
+ }
+ my $dlm=$&;
+ print_style($dlm,'command');
+ if (!($tex->{'line'} =~ s/^([^$dlm]*)($dlm)// )) {
+ error($tex,'Invalid \verb: could not find ending delimiter ('.$dlm.').');
+ }
+ print_style($1,'ignore');
+ print_style($2,'command');
+}
+
+# Parse begin-end group
+sub _parse_begin_end {
my ($tex,$status)=@_;
- my $localstyle=$status>0 ? 'grouping' : 'exclgroup';
- flush_style($tex,$localstyle);
- gobble_option($tex);
- my $groupname;
- if ($tex->{'line'} =~ s/^\{\s*([^\{\}]+)\s*\*?\}[ \t\r\f]*//) {
+ my $localstyle=status_is_text($status) ? 'grouping' : 'exclgroup';
+ flush_next_gobble_space($tex,$localstyle,$status);
+ #__gobble_option($tex); # no option before group name
+ my ($groupname,$next);
+ if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) {
# gobble group type
$groupname=$1;
print_style('{'.$1.'}',$localstyle);
- my $next='begin'.$1;
+ $next='begin'.$groupname;
if (defined (my $substat=$TeXmacro{$next})) {
- gobble_macro_parms($tex,$substat);
+ __gobble_macro_parms($tex,$substat);
}
} else {
- print_error("Warning: BEGIN group without type.");
+ $groupname='???'; $next='???';
+ error($tex,'Encountered \begin without environment name provided.');
}
# find group status (or leave unchanged)
- my $substat;
- defined ($substat=$TeXgroup{$1}) || ($substat=$status);
- if ($status<=0 && $status<$substat) {$substat=$status;}
- if (($status>0) && ($substat==-1)) {
+ my $substat=$TeXgroup{$1};
+ if (!defined $substat) {
+ $substat=$status;
+ if ($strictness>=1) {
+ warning($tex,"Using default rule for group ".$groupname);
+ }
+ } elsif (!status_is_text($status)) {
+ # Do not raise status
+ $substat=__new_status($substat,$status);
+ } elsif ($substat==$STATUS_FLOAT) {
# Count float
- inc_count($tex,5);
- }
- if ($status>0 and $substat>3) {
- # count item, exclude contents
- inc_count($tex,$substat);
+ inc_count($tex,$CNT_COUNT_FLOAT);
+ } elsif (my $cnt=status_to_cnt($substat)) {
+ # Count as given type (was: if $substat>3)
+ inc_count($tex,$cnt);
$substat=0;
+ } else {
+ # Use $substat value as given
}
- parse_unit($tex,$substat,'\end');
- if ($tex->{'line'} =~ s/^\{\s*([^\{\}]+)\s*\}[ \t\r\f]*//) {
- # gobble group type
- flush_style($tex,$localstyle);
- print_style('{'.$1.'}',$localstyle);
+ if ($includeBibliography && $groupname eq 'thebibliography' && $substat>0) {
+ # Add bibliography header
+ inc_count($tex,$CNT_COUNT_HEADER);
+ __count_macroword($tex,$groupname,$STATUS_TEXT_HEADER);
+ }
+ if (!status_inc_envir($substat)) {
+ # Keep parsing until appropriate end group arrives
+ while (!$tex->{'eof'}) {
+ _parse_unit($tex,$substat,'\end');
+ if ($tex->{'line'} =~ s/^\s*\{$groupname\}[ \t\r\f]*//) {
+ # gobble end group parameter
+ flush_next($tex,$localstyle,$status);
+ print_style($&,$localstyle);
+ return;
+ }
+ }
} else {
- print_error("Warning: END group without type while waiting to end ".$groupname.".");
+ # Parse until \end arrives, and check that it matches
+ _parse_unit($tex,$substat,'\end');
+ flush_next_gobble_space($tex,$localstyle,$status);
+ if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) {
+ # gobble group type
+ print_style('{'.$1.'}',$localstyle);
+ if ($groupname ne $1) {
+ error($tex,'Group \begin{'.$groupname.'} ended with end{'.$1.'}.');
+ }
+ } else {
+ error($tex,'Group ended while waiting for \end{'.$groupname.'}.');
+ }
}
}
-sub parse_include_file {
+# Parse and process file inclusion
+sub _parse_include_file {
my ($tex,$status,$addsuffix)=@_;
- $tex->{'line'} =~ s/^\{([^\{\}\s]+)\}// ||
- $tex->{'line'} =~ s/^\s*([^\{\}\%\\\s]+)// ||
+ if ($addsuffix eq 'bbl') {
+ _parse_include_bbl($tex,$status);
return;
+ }
flush_next($tex);
- if ($status>0) {
+ $tex->{'line'} =~ s/^\s*\{([^\{\}\s]+)\}//
+ || $tex->{'line'} =~ s/^\s*([^\{\}\%\\\s]+)//
+ || $tex->{'line'} =~ s/^\s*\{(.+?)\}//
+ || BLOCK {
+ error($tex,'Failed to read or interpret file name for inclusion.');
+ return;
+ };
+ if (status_is_text($status)) {
print_style($&,'fileinclude');
my $fname=$1;
if ($addsuffix==2) {$fname.='.tex';}
elsif ($addsuffix==1 && ($fname=~/^[^\.]+$/)) {$fname.='.tex';}
- if ($includeTeX) {add_file_to_list($fname);}
+ if ($includeTeX) {include_file($tex,$fname);}
} else {
print_style($&,'ignored');
}
}
-sub gobble_option {
- my $tex=shift @_;
- flush_next($tex);
- foreach my $pattern (@MacroOptionPatterns) {
- if ($tex->{'line'}=~s/^($pattern)//) {
- print_style($1,'option');
- return $1;
+# Parse and process bibliography file
+sub _parse_include_bbl {
+ my ($tex,$status)=@_;
+ __gobble_macro_parms($tex,1,$status);
+ if (status_is_text($status) && $includeBibliography) {
+ my $fname=$tex->{'filename'};
+ $fname=~s/\.\w+$/\.bbl/;
+ include_file($tex,$fname);
+ }
+}
+
+# Parse and process package inclusion
+sub _parse_include_package {
+ my ($tex)=@_;
+ set_style($tex,'document');
+ __gobble_option($tex);
+ if ( $tex->{'line'}=~s/^\{((\w+)(\s*,\s*\w+)*)\}// ) {
+ print_style('{'.$1.'}','document');
+ foreach (split(/\s*,\s*/,$1)) {
+ include_package($_);
}
+ } else {
+ _parse_unit($tex,$STATUS_IGNORE);
+ error($tex,"Could not recognise package list, ignoring it instead.");
+ }
+ __gobble_options($tex);
+}
+
+# Count macroword using given status
+sub __count_macroword {
+ my ($tex,$next,$status)=@_;
+ my $n=$TeXmacroword{$next};
+ if (defined $n && (my $cnt=status_text_cnt($status))) {
+ inc_count($tex,$cnt,$n);
+ }
+ return $n;
+}
+
+# Gobble next option, return option or undef if none
+sub __gobble_option {
+ my $tex=shift @_;
+ flush_next_gobble_space($tex);
+ if ($tex->{'line'}=~s/^($MacroOptionPattern)//) {
+ print_style($1,'option');
+ return $1;
}
return undef;
}
-sub gobble_options {
- while (gobble_option(@_)) {}
+# Gobble all options
+sub __gobble_options {
+ while (__gobble_option(@_)) {}
}
-sub gobble_macro_modifier {
+# Gobble macro modifyer (e.g. following *)
+sub __gobble_macro_modifier {
my $tex=shift @_;
flush_next($tex);
if ($tex->{'line'} =~ s/^\*//) {
@@ -798,45 +1710,48 @@ sub gobble_macro_modifier {
return undef;
}
-sub gobble_macro_parms {
+# Gobble macro parameters as specified in parm plus options
+sub __gobble_macro_parms {
my ($tex,$parm,$oldstat)=@_;
- my $i;
+ my $n;
if (ref($parm) eq 'ARRAY') {
- $i=scalar @{$parm};
+ $n=scalar @{$parm};
} else {
- $i=$parm;
- $parm=[0,0,0,0,0,0,0,0,0];
+ $n=$parm;
+ $parm=[($STATUS_IGNORE)x$n];
}
- if ($i>0) {gobble_macro_modifier($tex);}
- gobble_options($tex);
- for (my $j=0;$j<$i;$j++) {
- parse_unit($tex,new_status($parm->[$j],$oldstat));
- gobble_options($tex);
+ if ($n>0) {__gobble_macro_modifier($tex);}
+ __gobble_options($tex);
+ for (my $j=0;$j<$n;$j++) {
+ _parse_unit($tex,__new_status($parm->[$j],$oldstat));
+ __gobble_options($tex);
}
}
-sub new_status {
- my ($substat,$old)=@_;
- if (!defined $old) {return $substat;}
- if ($old==-3 || $substat==-3) {return -3;}
- if ($old==-2 || $substat==-2) {return -2;}
- if ($old==0 || $substat==0) {return 0;}
- if ($old==-9 || $substat==-9) {return -9;}
- if ($old>$substat) {return $old;}
- return $substat;
+# Return new parsing status given old and substatus
+sub __new_status {
+ my ($substat,$oldstat)=@_;
+ if (!defined $oldstat) {return $substat;}
+ foreach my $st (@STATUS_PRIORITY_LIST) {
+ if ($oldstat==$st || $substat==$st) {return $st;}
+ }
+ error($Main,'Could not determine new status in __new_status!','BUG');
+ return $oldstat;
}
-sub next_token {
+# Get next token skipping comments and flushing output buffer
+sub _next_token {
my $tex=shift @_;
my ($next,$type);
- if (defined $tex->{'next'}) {print_style($tex->{'next'}.' ',$tex->{'style'});}
+ my $style=$tex->{'style'};
+ if (defined $tex->{'next'}) {print_style($tex->{'next'},$tex->{'style'});}
$tex->{'style'}=undef;
- while (defined ($next=get_next_token($tex))) {
+ while (defined ($next=__get_next_token($tex))) {
$type=$tex->{'type'};
- if ($type==0) {
+ if ($type==$TOKEN_COMMENT) {
print_style($next,'comment');
- } elsif ($type==9) {
- if ($verbose>0) {line_return(1,$tex);}
+ } elsif ($type==$TOKEN_LINEBREAK) {
+ if ($verbose>0) {line_return(-1,$tex);}
} else {
return $next;
}
@@ -844,9 +1759,10 @@ sub next_token {
return $next;
}
-
-sub get_next_token {
+# Read, interpret and return next token
+sub __get_next_token {
# Token (or token group) category:
+ # -1: space
# 0: comment
# 1: word (or other forms of text or text components)
# 2: symbol (not word, e.g. punctuation)
@@ -859,64 +1775,112 @@ sub get_next_token {
# 666: TeXcount instruction (%TC:instruction)
my $tex=shift @_;
my $next;
- (defined ($next=get_token($tex,'\%+TC:\s*endignore\b[^\r\n]*',666))) && return "%TC:endignore";
- (defined ($next=get_token($tex,'\%+TC:[^\r\n]*',666))) && return $next;
- (defined ($next=get_token($tex,'\%[^\r\n]*',0))) && return $next;
- (defined ($next=get_token($tex,'(\r|\n|\r\n)',9))) && return $next;
- (defined ($next=get_token($tex,'\\\\[\{\}]',2))) && return $next;
- foreach my $pattern (@WordPatterns) {
- (defined ($next=get_token($tex,$pattern,1))) && return $next;
- }
- (defined ($next=get_token($tex,'[\"\'\`:\.,\(\)\[\]!\+\-\*=/\^\_\@\<\>\~\#\&]',2))) && return $next;
- (defined ($next=get_token($tex,'\\\\([a-zA-Z_]+|[^a-zA-Z_])',3))) && return $next;
- (defined ($next=get_token($tex,'[\{\}]',4))) && return $next;
- (defined ($next=get_token($tex,'[\[\]]',5))) && return $next;
- (defined ($next=get_token($tex,'\$\$',6))) && return $next;
- (defined ($next=get_token($tex,'\$',6))) && return $next;
- (defined ($next=get_token($tex,'.',999))) && return $next;
- (defined ($next=get_token($tex,'[^\s]+',999))) && return $next;
- $tex->{'eof'}=1;
+ my $ch;
+ while (!$tex->{'eof'}) {
+ $ch=substr($tex->{'line'},0,1);
+ if ($ch eq '') {
+ if (!more_texcode($tex)) {$tex->{'eof'}=1;}
+ next;
+ } elsif ($ch=~/^[ \t\f]/) {
+ $tex->{'line'}=~s/^([ \t\f]+)//;
+ return __set_token($tex,$1,$TOKEN_SPACE);
+ } elsif ($ch eq "\n" || $ch eq "\r") {
+ $tex->{'line'}=~s/^(\r\n?|\n)//;
+ return __set_token($tex,$1,$TOKEN_LINEBREAK);
+ } elsif ($tex->{'line'}=~s/^($WordPattern)//) {
+ return __set_token($tex,$1,$TOKEN_WORD);
+ } elsif ($ch eq '$') {
+ $tex->{'line'}=~s/^(\$\$?)//;
+ return __set_token($tex,$1,$TOKEN_MATH);
+ } elsif ($ch eq '{' || $ch eq '}') {
+ return __get_chtoken($tex,$ch,$TOKEN_BRACE);
+ } elsif ($ch eq '[' || $ch eq ']') {
+ return __get_chtoken($tex,$ch,$TOKEN_BRACKET);
+ } elsif ($ch=~/^['"`:.,()[]!+-*=\/^_@<>~#&]$/) {
+ return __get_chtoken($tex,$ch,$TOKEN_SYMBOL);
+ } elsif ($ch eq '%') {
+ if ($tex->{'line'}=~s/^(\%+TC:\s*endignore\b[^\r\n]*)//i) {
+ __set_token($tex,$1,$TOKEN_TC);
+ return "%TC:endignore";
+ }
+ if ($tex->{'line'}=~s/^(\%+[tT][cC]:[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_TC);}
+ if ($tex->{'line'}=~s/^(\%+[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_COMMENT);}
+ return __get_chtoken($tex,$ch,$TOKEN_COMMENT);
+ } elsif ($ch eq '\\') {
+ if ($tex->{'line'}=~s/^(\\[{}%])//) {return __set_token($tex,$1,$TOKEN_SYMBOL);}
+ if ($tex->{'line'}=~s/^(\\([a-zA-Z_]+|[^a-zA-Z_]))//) {return __set_token($tex,$1,$TOKEN_MACRO);}
+ return __get_chtoken($tex,$ch,$TOKEN_END);
+ } else {
+ return __get_chtoken($tex,$ch,$TOKEN_END);
+ }
+ }
return undef;
}
-sub get_token {
- my ($tex,$regexp,$type)=@_;
- if (!defined $regexp) {print_error("ERROR in get_token: undefined regex.");}
- if (!defined $tex->{'line'}) {print_error("ERROR in get_token: undefined tex-line. ".$tex->{'next'});}
- if ( $tex->{'line'} =~ s/^($regexp)[ \t\r\f]*// ) {
- $tex->{'next'}=$1;
- $tex->{'type'}=$type;
- return $1;
- }
- return undef;
+# Set next token and its type
+sub __set_token {
+ my ($tex,$next,$type)=@_;
+ $tex->{'next'}=$next;
+ $tex->{'type'}=$type;
+ return $next;
+}
+
+# Set character token and remove from line
+sub __get_chtoken {
+ my ($tex,$ch,$type)=@_;
+ $tex->{'line'}=substr($tex->{'line'},1);
+ $tex->{'next'}=$ch;
+ $tex->{'type'}=$type;
+ return $ch;
}
+
###### Count handling routines
+
+## Make new count object
+# The "count object" is a hash containing
+# - title: the title of the count (name of file, section, ...)
+# - counts: a list of numbers (the counts: files, text words, ...)
+# upon creation, but where the element
+# - subcounts: list of count objects (added by the TeX object)
+# may exist if the count contains subcounts. The elements of the
+# count are (by their index):
+# 0 = #files: counts the number of files
+# 1 = text words: counts the number of words in the text
+# 2 = header words: number of words in headers
+# 3 = caption words: number of words in float captions
+# 4 = #headers: number of headers
+# 5 = #floats: number of tables, figures, floats, etc.
+# 6 = #inline formulae: number of math elements in text ($...$)
+# 7 = #displayed formulae: number of displayed equations
sub new_count {
my ($title)=@_;
- my @cnt=(0,0,0,0,0,0,0,0);
- my %count=('count'=>\@cnt,'title'=>$title);
+ my @cnt=(0) x $SIZE_CNT;
+ my %count=('counts'=>\@cnt,'title'=>$title);
# files, text words, header words, float words,
# headers, floats, math-inline, math-display;
return \%count;
}
+# Increment TeX count for a given count type
sub inc_count {
- my ($tex,$type,$value)=@_;
- my $count=$tex->{'count'};
+ my ($tex,$cnt,$value)=@_;
+ my $count=$tex->{'subcount'};
if (!defined $value) {$value=1;}
- ${$count->{'count'}}[$type]+=$value;
+ ${$count->{'counts'}}[$cnt]+=$value;
}
+# Get count value for a given count type
sub get_count {
- my ($count,$type)=@_;
- return ${$count->{'count'}}[$type];
+ my ($count,$cnt)=@_;
+ return ${$count->{'counts'}}[$cnt];
}
-sub total_count {
- my ($count)=@_;
+# Compute sum count for a count object
+sub get_sum_count {
+ my $count=shift @_;
my $sum=0;
for (my $i=scalar(@sumweights);$i-->0;) {
$sum+=get_count($count,$i+1)*$sumweights[$i];
@@ -924,223 +1888,509 @@ sub total_count {
return $sum;
}
-sub print_count {
- my ($count,$header)=@_;
- if ($briefsum && @sumweights) {
- print_count_total($count,$header);
- } elsif ($briefsum) {
- if ($htmlstyle) {print "<p class='briefcount'>";}
- print_count_brief($count,$header);
- if ($htmlstyle) {print "</p>\n";}
+# Returns the number of subcounts
+sub number_of_subcounts {
+ my $count=shift @_;
+ if (my $subcounts=$count->{'subcounts'}) {
+ return scalar(@{$subcounts});
} else {
- print_count_details($count,$header);
+ return 0;
}
}
-sub print_count_with_header {
- my ($count,$header)=@_;
- if (!defined $header) {$header=$count->{'title'};}
- if (!defined $header) {$header="";}
- return $count,$header;
-}
+# Is a null count? (counts 1-7 zero, title starts with _)
+sub _count_is_null {
+ my $count=shift @_;
+ if (!$count->{'title'}=~/^_/) {return 0;}
+ for (my $i=1;$i<$SIZE_CNT;$i++) {
+ if (get_count($count,$i)>0) {return 0;}
+ }
+ return 1;
+}
-sub print_count_total {
- my ($count,$header)=print_count_with_header(@_);
- if ($htmlstyle) {print "<p class='count'>".$header;}
- print total_count($count);
- if ($htmlstyle) {print "</p>\n";}
- else {print ": ".$header;}
+# Add one count to another
+sub _add_to_count {
+ my ($a,$b)=@_;
+ for (my $i=0;$i<$SIZE_CNT;$i++) {
+ ${$a->{'counts'}}[$i]+=${$b->{'counts'}}[$i];
+ }
}
-sub print_count_brief {
- my ($count,$header)=print_count_with_header(@_);
- my $cnt=$count->{'count'};
- print ${$cnt}[1]."+".${$cnt}[2]."+".${$cnt}[3].
- " (".${$cnt}[4]."/".${$cnt}[5]."/".${$cnt}[6]."/".${$cnt}[7].") ".
- $header;
+# Add subcount to sum count and prepare new subcount
+sub next_subcount {
+ my ($tex,$title)=@_;
+ add_to_total($tex->{'countsum'},$tex->{'subcount'});
+ $tex->{'subcount'}=new_count($title);
+ return $tex->{'countsum'};
}
-sub print_count_details {
- my ($count,$header)=print_count_with_header(@_);
- if ($htmlstyle) {print "<dl class='count'>\n";}
- if (defined $header) {
- formatprint($header."\n",'dt','header');
+# Add count to total as subcount
+sub add_to_total {
+ my ($total,$count)=@_;
+ _add_to_count($total,$count);
+ if (!_count_is_null($count)) {
+ push @{$total->{'subcounts'}},$count;
+ $count->{'parentcount'}=$total;
}
- if (get_count($count,0)>1) {
- formatprint($countlabel[0].': ','dt');
- formatprint(get_count($count,0)."\n",'dd');
+}
+
+
+###### Result output routines
+
+
+# Close the output, e.g. adding HTML tail
+sub Close_Output {
+ if ($htmlstyle>1) {
+ html_tail();
}
- if (@sumweights) {
- formatprint('Sum count: ','dt');
- formatprint(total_count($count)."\n",'dd');
+}
+
+# Report if there were any errors occurring during parsing
+sub Report_Errors {
+ if (defined $outputtemplate) {return;}
+ if ( !$briefsum && !$totalflag && $verbose>=0 ) {
+ foreach (keys %warnings) {formatprint($_,"p","nb");print "\n";}
}
- for (my $i=1;$i<8;$i++) {
- formatprint($countlabel[$i].': ','dt');
- formatprint(get_count($count,$i)."\n",'dd');
+ if ($errorcount==0) {return;}
+ if ($briefsum && $totalflag) {print " ";}
+ if ($htmlstyle) {
+ print "<div class='error'><p>\n";
+ print "There were ".$errorcount." error(s) reported!\n";
+ print "</p></div>\n";
+ } elsif ($briefsum && $totalflag) {
+ print "(errors:".$errorcount.")";
+ } else {
+ print "(errors:".$errorcount.")\n";
}
- my $subcounts=$count->{'subcounts'};
- if ($showsubcounts && defined $subcounts && scalar(@{$subcounts})>1) {
- formatprint("Subcounts: text+headers+captions (#headers/#floats/#inlines/#displayed)\n",'dt');
- foreach my $subcount (@{$subcounts}) {
- if ($htmlstyle) {print "<dd class='briefcount'>";}
- print_count_brief($subcount);
- if ($htmlstyle) {print "</dd>";}
- print "\n";
+}
+
+# Print word frequencies (as text only)
+sub print_word_freq {
+ my ($word,$wd,$freq,%Freq,%Word,%Class);
+ my %regs;
+ foreach my $cg (@AlphabetScripts,@LogogramScripts) {
+ $regs{$cg}=qr/\p{$cg}/;
+ }
+ my $sum=0;
+ for $word (keys %WordFreq) {
+ $wd=lc $word;
+ $Freq{$wd}+=$WordFreq{$word};
+ $sum+=$WordFreq{$word};
+ $Word{$wd}=__lc_merge($word,$Word{$wd});
+ }
+ if ($htmlstyle) {print "<table class='stat'>\n<thead>\n";}
+ __print_word_freq("Word","Freq","th");
+ if ($htmlstyle) {print "</thead>\n";}
+ if ($optionWordClassFreq>0) {
+ for $word (keys %Freq) {$Class{__word_class($word,\%regs)}+=$Freq{$word};}
+ __print_sorted_freqs('langstat',\%Class);
+ }
+ if ($htmlstyle) {print "<tbody class='sumstat'>\n";}
+ __print_word_freq("All words",$sum,,'td','sum');
+ if ($htmlstyle) {print "</tbody>\n";}
+ if ($optionWordFreq>0) {__print_sorted_freqs('wordstat',\%Freq,\%Word,$optionWordFreq);}
+ if ($htmlstyle) {print "</table>\n";}
+}
+
+# Merge to words which have the same lower case
+sub __lc_merge {
+ my ($word,$w)=@_;
+ if (defined $w) {
+ for (my $i=length($word);$i-->0;) {
+ if (substr($word,$i,1) ne substr($w,$i,1)) {
+ substr($word,$i,1)=lc substr($word,$i,1);
+ }
}
}
- if ($htmlstyle) {print "</dl>\n";}
+ return $word;
}
-sub add_count {
- my ($a,$b)=@_;
- for (my $i=0;$i<8;$i++) {
- ${$a->{'count'}}[$i]+=${$b->{'count'}}[$i];
+# Return the word class based on script groups it contains
+sub __word_class {
+ my ($wd,$regs)=@_;
+ my @classes;
+ $wd=~s/\\\w+({})?/\\{}/g;
+ foreach my $name (keys %{$regs}) {
+ if ($wd=~$regs->{$name}) {push @classes,$name;}
}
+ my $cl=join('+',@classes);
+ if ($cl) {}
+ elsif ($wd=~/\\/) {$cl='(macro)';}
+ else {$cl='(unidentified)';}
+ return $cl;
}
-sub add_subcount {
- my ($tex,$title)=@_;
- add_count($tex->{'countsum'},$tex->{'count'});
- push @{$tex->{'countlist'}},$tex->{'count'};
- $tex->{'count'}=new_count($title);
- return $tex->{'countsum'};
+# Print table body containing word frequencies
+sub __print_sorted_freqs {
+ my ($class,$Freq,$Word,$min)=@_;
+ my $sum=0;
+ my ($word,$wd,$freq);
+ if (!defined $min) {$min=0;}
+ if ($htmlstyle) {print "<tbody class='",$class,"'>\n";}
+ else {print "---\n";}
+ for $wd (sort {$Freq->{$b} <=> $Freq->{$a}} keys %{$Freq}) {
+ if (defined $Word) {$word=$Word->{$wd};} else {$word=$wd;}
+ $freq=$Freq->{$wd};
+ if ($freq>=$min) {
+ $sum+=$freq;
+ __print_word_freq($word,$freq);
+ }
+ }
+ if ($min>0) {__print_word_freq("Sum of subset",$sum,'td','sum');}
+ if ($htmlstyle) {print "</tbody>\n";}
+}
+
+# Print one word freq line
+sub __print_word_freq {
+ my ($word,$freq,$tag,$class)=@_;
+ if (!defined $tag) {$tag='td';}
+ if (defined $class) {$class=' class=\''.$class.'\'';} else {$class='';}
+ if ($htmlstyle) {
+ print "<tr",$class,"><",$tag,">",$word,"</",$tag,"><",$tag,">",$freq,"</",$tag,"></tr>\n";
+ } else {
+ print $word,": ",$freq,"\n";
+ }
}
###### Printing routines
-sub set_style {
- my ($tex,$style)=@_;
- if (!(($tex->{'style'}) && ($tex->{'style'} eq '-'))) {$tex->{'style'}=$style;}
+# Print text using given style/colour
+sub print_with_style {
+ my ($text,$style,$colour)=@_;
+ if ($style eq ' ') {
+ print text_to_print($text);
+ } elsif ($style eq '') {
+ print text_to_print($text);
+ error($Main,'Empty style should not occur in print_with_style!','BUG');
+ } elsif ($htmlstyle) {
+ print "<span class='$style'>".text_to_print($text).'</span>';
+ } else {
+ ansiprint(text_to_print($text),$colour);
+ if ($style=~/$separatorstyleregex/) {print $separator;}
+ }
}
-sub flush_style {
- my ($tex,$style)=@_;
- set_style($tex,$style);
- flush_next($tex);
+# Prepare text string for print: convert special characters
+sub text_to_print {
+ my $text=join('',@_);
+ if ($htmlstyle) {
+ $text=~s/&/&amp;/g;
+ $text=~s/</&lt;/g;
+ $text=~s/>/&gt;/g;
+ $text=~s/[ \t]{2}/\&nbsp; /g;
+ }
+ return $text;
}
-sub line_return {
- my ($blank,$tex)=@_;
- if ($blank>$blankline) {
- if ((defined $tex) && @sumweights) {
- my $num=total_count($tex->{'count'});
- print_style(" [".$num."]","sumcount");
- }
- linebreak();
- $blankline++;
+# Print text, using appropriate tags for HTML
+sub formatprint {
+ my ($text,$tag,$class)=@_;
+ my $break=($text=~s/(\r\n?|\n)$//);
+ if ($htmlstyle && defined $tag) {
+ print '<'.$tag;
+ if ($class) {print " class='".$class."'";}
+ print '>'.text_to_print($text).'</'.$tag.'>';
+ } else {
+ print text_to_print($text);
}
+ if ($break) {print "\n";}
}
+# Add a line break to output
sub linebreak {
if ($htmlstyle) {print "<br>\n";} else {print "\n";}
}
-sub print_style {
- my ($text,$style,$state)=@_;
- (($verbose>=0) && (defined $text) && (defined $style)) || return 0;
- my $colour;
- ($colour=$STYLE{$style}) || return;
- if (($colour) && !($colour eq '-')) {
- print_with_style($text,$style,$colour);
- if ($state) {
- print_style($state,'state');
- }
- $blankline=-1;
- return 1;
+###### Routines for printing count summary
+
+
+# Print count summary for a count object
+sub print_count {
+ my ($count,$class)=@_;
+ if ($htmlstyle) {print "<div class='".($class||'count')."'>\n";}
+ if ($outputtemplate) {
+ _print_count_template($count,$outputtemplate);
+ } elsif ($briefsum && @sumweights) {
+ _print_sum_count($count);
+ } elsif ($briefsum) {
+ if ($htmlstyle) {print "<p class='count'>";}
+ _print_count_brief($count);
+ if ($htmlstyle) {print "</p>\n";}
} else {
- return 0;
+ _print_count_details($count);
}
+ if ($htmlstyle) {print "</div>\n";}
}
-sub print_error {
- my $text=shift @_;
- $errorcount++;
- if ($verbose>=0) {
- line_return(1);
- print_style("!!! ".$text." !!!",'error');
- line_return(1);
+# Return count,header,... list filling in header if missing
+sub __count_and_header {
+ my $count=shift @_;
+ my $header=__count_header($count);
+ return $count,$header,@_;
+}
+
+# Return count title or "" if missing
+sub __count_header {
+ my $count=shift @_;
+ return $count->{'title'}||'';
+}
+
+# Print total count (sum) for a given count object
+sub _print_sum_count {
+ my ($count,$header)=__count_and_header(@_);
+ if ($htmlstyle) {print "<p class='count'>".text_to_print($header).": ";}
+ print get_sum_count($count);
+ if ($htmlstyle) {print "</p>\n";}
+ else {print ": ".text_to_print($header);}
+ print "\n";
+}
+
+# Print brief summary of count object
+sub _print_count_brief {
+ my ($count,$header,$tag1,$tag2)=__count_and_header(@_);
+ my @cnt=@{$count->{'counts'}};
+ if ($htmlstyle && $tag1) {print "<".$tag1.">";}
+ print $cnt[$CNT_WORDS_TEXT]."+".$cnt[$CNT_WORDS_HEADER]."+".$cnt[$CNT_WORDS_CAPTION].
+ " (".$cnt[$CNT_COUNT_HEADER]."/".$cnt[$CNT_COUNT_FLOAT].
+ "/".$cnt[$CNT_COUNT_INLINEMATH]."/".$cnt[$CNT_COUNT_DISPLAYMATH].")";
+ if ($htmlstyle && $tag2) {
+ print "</".$tag1."><".$tag2.">";
+ $tag1=$tag2;
+ } else {print " ";}
+ print text_to_print($header);
+ if ($htmlstyle && $tag1) {print "</".$tag1.">";}
+ if ($finalLineBreak) {print "\n";}
+}
+
+# Print detailed summary of count object
+sub _print_count_details {
+ my ($count,$header)=__count_and_header(@_);
+ if ($htmlstyle) {print "<ul class='count'>\n";}
+ if ($header) {formatprint($header."\n",'li','header');}
+ if (my $tex=$count->{'TeXcode'}) {
+ if (!defined $encoding) {formatprint('Encoding: '.$tex->{'encoding'}."\n",'li');}
+ }
+ if (@sumweights) {formatprint('Sum count: '.get_sum_count($count)."\n",'li');}
+ for (my $i=1;$i<$SIZE_CNT;$i++) {
+ formatprint($countlabel[$i].': '.get_count($count,$i)."\n",'li');
}
+ if (get_count($count,$CNT_FILE)>1) {
+ formatprint($countlabel[$CNT_FILE].': '.get_count($count,$CNT_FILE)."\n",'li');
+ }
+ my $subcounts=$count->{'subcounts'};
+ if ($showsubcounts && defined $subcounts && scalar(@{$subcounts})>=$showsubcounts) {
+ formatprint("Subcounts:\n",'li');
+ if ($htmlstyle) {print "<span class='subcount'>\n";}
+ formatprint(" text+headers+captions (#headers/#floats/#inlines/#displayed)\n",'li','fielddesc');
+ foreach my $subcount (@{$subcounts}) {
+ print ' ';
+ _print_count_brief($subcount,'li');
+ }
+ if ($htmlstyle) {print "</span>\n";}
+ }
+ if ($htmlstyle) {print "</ul>\n";} else {print "\n";}
+}
+
+# Print summary of count object using template
+sub _print_count_template {
+ my ($count,$header,$template)=__count_and_header(@_);
+ $template=~s/\\n/\n/g;
+ if ($htmlstyle) {$template=~s/\n/<br>/g;}
+ my ($subtemplate,$posttemplate);
+ while ($template=~/\{SUB\?((.*?)\|)?(.*?)(\|(.*?))?\?SUB\}/is) {
+ __print_count_using_template($count,$`);
+ if (number_of_subcounts($count)>1) {
+ if (defined $2) {print $2;}
+ __print_subcounts_using_template($count,$3);
+ if (defined $5) {print $5;}
+ }
+ $template=$';
+ }
+ __print_count_using_template($count,$template);
}
-sub formatprint {
- my ($text,$tag,$class)=@_;
- my $break=($text=~s/\n$//);
- if ($htmlstyle && defined $tag) {
- print '<'.$tag;
- if ($class) {print " class='".$class."'";}
- print '>'.$text.'</'.$tag.'>';
+# Print counts using template
+sub __print_count_using_template {
+ my ($count,$template)=@_;
+ for (my $i=0;$i<$SIZE_CNT;$i++) {
+ $template=__process_template($template,$i,get_count($count,$i));
+ }
+ $template=~s/\{VER\}/$versionnumber/gi;
+ # TODO: Should base warnings and errors on TeXcode or Main object
+ $template=__process_template($template,"W|WARN|WARNING|WARNINGS",length(%warnings));
+ $template=__process_template($template,"E|ERR|ERROR|ERRORS",$errorcount);
+ $template=__process_template($template,"S|SUM",get_sum_count($count));
+ $template=__process_template($template,"T|TITLE",$count->{'title'}||"");
+ $template=__process_template($template,"SUB",number_of_subcounts($count));
+ print $template;
+}
+
+# Print subcounts using template
+sub __print_subcounts_using_template {
+ my ($count,$template)=@_;
+ my $subcounts=$count->{'subcounts'};
+ if ($template && defined $subcounts && scalar(@{$subcounts})>=$showsubcounts) {
+ foreach my $subcount (@{$subcounts}) {
+ __print_count_using_template($subcount,$template);
+ }
+ }
+}
+
+# Process template for specific label
+sub __process_template {
+ my ($template,$label,$value)=@_;
+ if ($value) {
+ $template=~s/\{($label)\?(.*?)(\|(.*?))?\?(\1)\}/$2/gis;
} else {
- print $text;
+ $template=~s/\{($label)\?(.*?)\|(.*?)\?(\1)\}/$3/gis;
+ $template=~s/\{($label)\?(.*?)\?(\1)\}//gis;
}
- if ($break) {print "\n";}
+ if (!defined $value) {$value="";}
+ $template=~s/\{($label)\}/$value/gis;
+ return $template;
}
+
+###### Routines for printing parsing details
+
+
+# Print next token
sub flush_next {
- my $tex=shift @_;
+ my ($tex,$style)=@_;
+ my $ret=undef;
+ if (defined $style && $style ne '') {
+ set_style($tex,$style);
+ }
if (defined $tex->{'next'}) {
- print_style($tex->{'next'}.' ',$tex->{'style'},$tex->{'printstate'});
+ $ret=print_style($tex->{'next'},$tex->{'style'},$tex->{'printstate'});
}
$tex->{'printstate'}=undef;
$tex->{'style'}='-';
+ return $ret;
+}
+
+# Print next token and gobble following spaces
+sub flush_next_gobble_space {
+ my ($tex,$style,$status)=@_;
+ my $ret=flush_next($tex,$style);
+ if (!defined $ret) {$ret=0;}
+ if (!defined $status) {$status=$STATUS_IGNORE;}
+ my $prt=($verbose>0);
+ if ($tex->{'line'}=~s/^([ \t\f]*)(\r\n?|\n)([ \t\f]*)//) {
+ if (!$prt) {
+ } elsif ($verbose>2 || $ret) {
+ print $1;
+ line_return(-1,$tex);
+ my $space=$3;
+ if ($htmlstyle) {$space=~s/ /\&nbsp;/g;}
+ print $space;
+ } else {
+ line_return(0,$tex);
+ }
+ } elsif ($tex->{'line'}=~s/^([ \t\f]*)//) {
+ if ($prt) {print $1;}
+ }
}
+# Set presentation style
+sub set_style {
+ my ($tex,$style)=@_;
+ if (!(($tex->{'style'}) && ($tex->{'style'} eq '-')) && $style ne '') {$tex->{'style'}=$style;}
+}
-# Close the output, e.g. adding HTML tail
-sub Close_Output {
- if ($htmlstyle>1) {
- html_tail();
+# Print text using the given style, and log state if given
+sub print_style {
+ my ($text,$style,$state)=@_;
+ (($verbose>=0) && (defined $text) && (defined $style)) || return 0;
+ my $colour;
+ ($colour=$STYLE{$style}) || return;
+ if (($colour) && !($colour eq '-')) {
+ print_with_style($text,$style,$colour);
+ if ($state) {print_style($state,'state');}
+ if ($style ne "cumsum") {$blankline=-1;}
+ return 1;
+ } else {
+ return 0;
}
}
-
-# Report if there were any errors occurring during parsing
-sub Report_ErrorCount {
- if ($errorcount==0) {return;}
- if ($briefsum && $totalflag) {print " ";}
- if ($htmlstyle) {
- print_error("Errors:".$errorcount,"p","error");
- } else {
- print "(errors:".$errorcount.")";
+# Conditional line return
+sub line_return {
+ my ($blank,$tex)=@_;
+ if ($blank<0 && $verbose<3) {$blank=1;}
+ if ($blank<0 || $blank>$blankline) {
+ if ((defined $tex) && @sumweights) {
+ my $num=get_sum_count($tex->{'subcount'});
+ print_style(" [".$num."]","cumsum");
+ }
+ linebreak();
+ $blankline++;
}
}
+# Print error message
+sub _print_error {
+ my $text=shift @_;
+ line_return(1);
+ print_style("!!! ".$text." !!!",'error');
+ line_return(1);
+}
+
+# Print errors in buffer and delete errorbuffer
+sub flush_errorbuffer {
+ my $source=shift @_;
+ my $err=$source->{'errorbuffer'} || return;
+ foreach (@$err) {_print_error($_);}
+ $source->{'errorbuffer'}=undef;
+}
+
+###### Print help on style/colour codes
-sub print_help_style {
+
+# Print output style codes if conditions are met
+sub conditional_print_help_style {
+ if ($showcodes) {_print_help_style();}
+ return $showcodes;
+}
+
+# Print help on output styles
+sub _print_help_style {
if ($verbose<=0) {return;}
+ if ($htmlstyle) {print "<div class='stylehelp'><p>";}
formatprint("Format/colour codes of verbose output:","h2");
print "\n\n";
- if ($htmlstyle) {print "<p class='stylehelp'>";}
- help_style_line('Text which is counted',"word1","counted as text words");
- help_style_line('Header and title text',"word2","counted as header words");
- help_style_line('Caption text and footnotes',"word3","counted as caption words");
- help_style_line("Ignored text or code","ignore","excluded or ignored");
- help_style_line('\documentclass',"document","document start, beginning of preamble");
- help_style_line('\macro',"command","macro not counted, but parameters may be");
- help_style_line('\macro',"exclcommand","macro in excluded region");
- help_style_line("[Macro options]","option","not counted");
- help_style_line('\begin{group} \end{group}',"grouping","begin/end group");
- help_style_line('\begin{group} \end{group}',"exclgroup","begin/end group in excluded region");
- help_style_line('$ $',"mathgroup","counted as one equation");
- help_style_line('$ $',"exclmath","equation in excluded region");
- help_style_line('% Comments',"comment","not counted");
- help_style_line('%TC:TeXcount instructions',"tc","not counted");
- help_style_line("File to include","fileinclude","not counted but file may be counted later");
+ _help_style_line('Text which is counted',"word1","counted as text words");
+ _help_style_line('Header and title text',"word2","counted as header words");
+ _help_style_line('Caption text and footnotes',"word3","counted as caption words");
+ _help_style_line("Ignored text or code","ignore","excluded or ignored");
+ _help_style_line('\documentclass',"document","document start, beginning of preamble");
+ _help_style_line('\macro',"command","macro not counted, but parameters may be");
+ _help_style_line('\macro',"exclcommand","macro in excluded region");
+ _help_style_line("[Macro options]","option","not counted");
+ _help_style_line('\begin{group} \end{group}',"grouping","begin/end group");
+ _help_style_line('\begin{group} \end{group}',"exclgroup","begin/end group in excluded region");
+ _help_style_line('$ $',"mathgroup","counted as one equation");
+ _help_style_line('$ $',"exclmath","equation in excluded region");
+ _help_style_line('% Comments',"comment","not counted");
+ _help_style_line('%TC:TeXcount instructions',"tc","not counted");
+ _help_style_line("File to include","fileinclude","not counted but file may be counted later");
if ($showstates) {
- help_style_line('[state]',"state","internal TeXcount state");
+ _help_style_line('[state]',"state","internal TeXcount state");
}
if (@sumweights) {
- help_style_line('[sumcount]',"sumcount","cumulative sum count");
+ _help_style_line('[cumsum]',"cumsum","cumulative sum count");
}
- help_style_line("ERROR","error","TeXcount error message");
- if ($htmlstyle) {print "</p>";}
+ _help_style_line("ERROR","error","TeXcount error message");
+ if ($htmlstyle) {print "</p></div>";}
print "\n\n";
}
-sub help_style_line {
+# Print one line of help
+sub _help_style_line {
my ($text,$style,$comment)=@_;
if ($htmlstyle) {
- $comment="&nbsp;&nbsp;....&nbsp;&nbsp;".$comment;
+ $comment="&nbsp;&nbsp;....&nbsp;&nbsp;".text_to_print($comment);
} else {
$comment=" .... ".$comment;
}
@@ -1150,27 +2400,175 @@ sub help_style_line {
}
}
-# Print output style codes if conditions are met
-sub conditional_print_help_style {
- if ($showcodes) {print_help_style();}
- return $showcodes;
+
+###### Help routines
+
+
+# Print TeXcount version
+sub print_version {
+ wprintstringdata('Version');
}
-###### HTML routines
+# Print TeXcount reference text
+sub print_reference {
+ wprintstringdata('Reference');
+}
+# Print TeXcount licence text
+sub print_license {
+ wprintstringdata('License');
+}
+# Print TeXcount parameter list
+sub print_syntax {
+ wprintstringdata('OptionsHead');
+ wprintstringdata('Options','@ - :');
+}
+# Print complete TeXcount help
+sub print_help {
+ print_help_title();
+ print_syntax();
+ print_help_text();
+ print_reference();
+}
+
+# Print help title
+sub print_help_title {
+ wprintstringdata('HelpTitle');
+}
+
+# Print help text
+sub print_help_text {
+ wprintstringdata('HelpText');
+ wprintstringdata('TCinstructions');
+}
+
+# Print help on specific macro or group
+sub print_help_on {
+ my $arg=shift @_;
+ my $def;
+ my %rules=(
+ '\documentclass' => 'Initiates LaTeX document preamble.',
+ '\begin' => 'Treatmend depends on group handling rules.',
+ '\def' => 'Excluded from count.',
+ '\verb' => 'Strong exclude for enclosed region.',
+ '$' => 'Opens or closes inlined equation',
+ '$$' => 'Opens or closes displayed equation.',
+ '\(' => 'Opens inlined equation.',
+ '\)' => 'Closes inlined equation initiated by \(.',
+ '\[' => 'Opens displayed equation.',
+ '\]' => 'Closes displayed equation initiated by \[.');
+ if (!defined $arg || $arg=~/^\s*$/) {
+ print "Specify macro or group name after the -h= option.\n";
+ return;
+ }
+ if ($def=$rules{$arg}) {
+ print "Special rule (hard coded) for $arg\n";
+ print $def."\n";
+ } elsif ($arg=~/^\\/) {
+ if ($def=$TeXfileinclude{$arg}) {
+ print "Rule for macro $arg\n";
+ print "Takes file name as parameter which is included in document.\n";
+ } elsif ($def=$TeXmacro{$arg}) {
+ print "Rule for macro $arg\n";
+ _print_rule_macro($arg,$def);
+ if ($def=$TeXheader{$arg}) {
+ print "This macro is counted as a header\n";
+ }
+ if ($def=$TeXfloatinc{$arg}) {
+ print "This macro is also counted inside floats as captions.\n";
+ }
+ } elsif ($def=$TeXmacroword{$arg}) {
+ print "Rule for macro $arg\n";
+ print "Count as ".$def." word(s).\n";
+ } else {
+ print "No macro rule defined for $arg.\nParameters treated as surrounding text.\n";
+ }
+ } else {
+ if ($def=$TeXgroup{$arg}) {
+ print "Rule for group $arg\n";
+ _print_rule_group($arg,$def);
+ } else {
+ print "No default group rule defined for $arg.\nContent handled as surrounding text.\n";
+ }
+ }
+}
+
+# Print macro handling rule
+sub _print_rule_macro {
+ my ($arg,$def)=@_;
+ my %rules=(
+ 0 => 'Exclude from count',
+ 1 => 'Count as text',
+ 2 => 'Count as header',
+ 3 => 'Count as caption',
+ -1 => 'Exclude as float (i.e. include captions)',
+ -2 => 'Strong exclude (ignore begin-end groups)',
+ -3 => 'Stronger exclude (ignore all macros)',
+ -9 => 'Exclude as preamble');
+ if (ref($def) eq 'ARRAY') {
+ print "Takes ".scalar(@{$def})." parameter(s):\n";
+ foreach my $i (@{$def}) {
+ print " - ".$rules{$i}."\n";
+ }
+ } else {
+ print "Takes ".$def." parameter(s), not included in counts.\n";
+ }
+}
+
+# Print group handling rule
+sub _print_rule_group {
+ my ($arg,$def)=@_;
+ my %rules=(
+ 0 => 'Not included',
+ 1 => 'Text, words included in text count',
+ 2 => 'Header, words included in header count',
+ 3 => 'Float caption, words included in float caption count',
+ 6 => 'Inline mathematics, words not counted',
+ 7 => 'Displayed mathematics, words not counted',
+ -1 => 'Float, not included, but looks for captions');
+ print "Rule used: ".$rules{$def}."\n";
+ if ($def=$TeXmacro{'begin'.$arg}) {
+ _print_rule_macro($def);
+ }
+}
+
+
+###### HTML routines
+
+
+# Print HTML header
sub html_head {
print "<html>\n<head>";
- if ($utf8flag) {
- print "\n<meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\">";
- }
- print '
+ print "\n<meta http-equiv='content-type' content='text/html; charset=$outputEncoding'>\n";
+ _print_html_style();
+ print "</head>\n\n<body>\n";
+ print "\n<h1>LaTeX word count";
+ if ($showVersion>0) {print " (version ",_html_version(),")"}
+ print "</h1>\n";
+}
+
+# Print HTML tail
+sub html_tail {
+ print "</body>\n\n</html>\n";
+}
+
+# Return version number using HTML
+sub _html_version {
+ my $htmlver=$versionnumber;
+ $htmlver=~s/\b(alpha|beta)\b/&$1;/g;
+ return $htmlver;
+}
+
+# Print HTML STYLE element
+sub _print_html_style {
+print <<END
<style>
<!--
body {width:auto;padding:5;margin:5;}
.error {font-weight:bold;color:#f00;font-style:italic;}
-.word1,.word2,.word3 {color: #009;}
+.word1,.word2,.word3 {color: #009; border-left: 1px solid #CDF; border-bottom: 1px solid #CDF;}
.word2 {font-weight: 700;}
.word3 {font-style: italic;}
.command {color: #c00;}
@@ -1184,208 +2582,289 @@ body {width:auto;padding:5;margin:5;}
.tc {color: #999; font-weight:bold;}
.comment {color: #999; font-style: italic;}
.state {color: #990; font-size: 70%;}
-.sumcount {color: #999; font-size: 80%;}
+.cumsum {color: #999; font-size: 80%;}
.fileinclude {color: #696; font-weight:bold;}
-dl.count {background: #cfc; color: 009;}
-dl.count dt.header {font-weight: bold; font-style: italic; float: none;}
-dl.count dt {clear: both; float: left; margin-right: .5em;}
-dl.count dd {font-weight: bold;}
-dl.count dd.briefcount {font-weight: 700; clear: both; font-size:80%; font-weight:normal; margin-left:8pt;}
.warning {color: #c00; font-weight: 700;}
-.parse, .count, .stylehelp, .filegroup {border: solid 1px #999; margin: 0pt; padding: 4pt;}
-.parse {font-size: 80%; background: #eef;}
-.parse {border-bottom:none;}
-.stylehelp {font-size: 80%; background: #ffc; margin-bottom: 8pt;}
-.filegroup {background: #efe; margin-bottom: 8pt;}
+
+div.filegroup, div.parse, div.stylehelp, div.count, div.sumcount, div.error {
+ border: solid 1px #999; margin: 4pt 0pt; padding: 4pt;
+}
+div.stylehelp {font-size: 80%; background: #fffff0; margin-bottom: 16pt;}
+div.filegroup {background: #dfd; margin-bottom: 16pt;}
+div.count {background: #ffe;}
+div.sumcount {background: #cec;}
+div.error {background: #fcc;}
+.parse {font-size: 80%; background: #f8fff8; border-bottom:none;}
+
+ul.count {list-style-type: none; margin: 4pt 0pt; padding: 0pt;}
+.count li.header {font-weight: bold; font-style: italic;}
+.subcount li.header {font-weight: normal; font-style: italic;}
+.subcount li {margin-left: 16pt; font-size: 80%;}
+.fielddesc {font-style: italic;}
+.nb {color: #900;}
+
+table.stat {border:2px solid #339; background:#666; border-collapse:collapse;}
+table.stat tr {border:1px solid #CCC;}
+table.stat th, table.stat td {padding:1pt 4pt;}
+table.stat col {padding:4pt;}
+table.stat thead {background: #CCF;}
+table.stat tbody {border:1px solid #333;}
+table.stat tbody.sumstat {background:#FFC;}
+table.stat tbody.langstat {background:#FEE;}
+table.stat tbody.wordstat {background:#EEF;}
+table.stat .sum {font-weight:bold; font-style:italic;}
-->
</style>
-</head>
-<body>
-<h1>LaTeX word count</h1>
-';
+END
}
-sub html_tail {
- print '</body></html>';
-}
+###### Read text data
-###### Help routines
+# Return the STRINGDATA hash (lazy instantiation)
+sub STRINGDATA {
+ if (!defined $STRINGDATA) {
+ my @DATA=<DATA>;
+ foreach (@DATA) {
+ $_=~s/\$\{(\w+)\}/__apply_globaldata($1)/ge;
+ }
+ $STRINGDATA=splitlines(':{3,}\s*(\w+)?',\@DATA);
+ }
+ return $STRINGDATA;
+}
-sub print_version {
- print "TeXcount version ".$versionnumber.", ".$versiondate.'.';
+# Return value from STRINGDATA hash
+sub StringData {
+ my $name=shift @_;
+ return STRINGDATA()->{$name};
}
-sub print_syntax {
- print '
-Syntax: TeXcount.pl [options] files
+# Insert value from GLOBALDATA
+sub __apply_globaldata {
+ my $name=shift @_;
+ if (my $value=$GLOBALDATA{$name}) {
+ return $value;
+ }
+ return '[['.$name.']]';
+}
-Options:
- -relaxed Uses relaxed rules for word and option handling:
- i.e. allows more general cases to be counted as
- either words or macros.
- -v Verbose (same as -v3)
- -v0 Do not present parsing details
- -v1 Verbose: print parsed words, mark formulae
- -v2 More verbose: also print ignored text
- -v3 Even more verbose: include comments and options
- -v4 Same as -v3 -showstate
- -showstate Show internal states (with verbose)
- -brief Only prints a brief, one line summary of counts
- -q, -quiet Quiet mode, no error messages (use is discouraged!)
- -sum, -sum= Make sum of all word and equation counts. May also
- use -sum=#[,#] with up to 7 numbers to indicate how
- each of the counts (text words, header words, caption
- words, #headers, #floats, #inlined formulae,
- #displayed formulae) are summed. The default sum (if
- only -sum is used) is the same as -sum=1,1,1,0,0,1,1.
- -sub, -sub= Generate subcounts. Option values are none, part,
- chapter, section or subsection. Default (-sub) is set
- to subsection, whereas unset is none. (Alternative
- option name is -subcount.)
- -nc, -nocol No colours (colours require ANSI)
- -html Output in HTML format
- -htmlcore Only HTML body contents
- -inc Include tex files included in the document
- -noinc Do not include included tex files (default)
- -total Do not give sums per file, only total sum.
- -1 Same as -brief and -total. Ensures there is only one
- line of output. If used in conjunction with -sum, the
- output will only be the total number. (NB: Character
- is the number one, not the letter L.)
- -dir, -dir= Specify the working directory using -dir=path.
- Remember that the path must end with \ or /. If only
- -dir is used, the directory of the parent file is used.
- -utf8, -unicode Turns on Unicode (UTF-8) for input and output. This
- is automatic with -chinese, and is required to handle
- e.g. Korean text. Note that the TeX file must be save
- in UTF-8 format (not e.g. GB2312 or Big5), or the
- result will be unpredictable.
- -ch, -chinese, -zhongwen Turns on support for Chinese characters.
- TeXcount will then count each Chinese character as a
- word. Automatically turns on -utf8.
- -jp, -japanese Turns on support for Japanese characters. TeXcount
- will count each Japanese character (kanji, hiragana,
- and katakana) as one word, i.e. not do any form of
- word segmentation. Automatically turns on -utf8.
- -codes Display output style code overview and explanation.
- This is on by default.
- -nocodes Do not display output style code overview.
- -h, -?, --help, /? Help
- --version Print version number
- --license License information
-';
+# Print value from STRINGDATA using wprintlines
+sub wprintstringdata {
+ my $name=shift @_;
+ my $data=StringData($name);
+ if (!defined $data) {
+ error($Main,"No StringData $name.",'BUG');
+ }
+ wprintlines(@_,@$data);
+}
+
+# Divide array of lines by identifying headers
+sub splitlines {
+ my ($pattern,$lines)=@_;
+ if (!defined $lines) {return;}
+ my $id=undef;
+ my %hash;
+ foreach my $line (@$lines) {
+ if ($line=~/^$pattern$/) {
+ $id=$1;
+ if (defined $id) {
+ $hash{$id}=[];
+ if (defined $2) {push @{$hash{$id}},$2;}
+ }
+ } elsif (defined $id) {
+ chomp $line;
+ push @{$hash{$id}},$line;
+ }
+ }
+ return \%hash;
}
-sub print_help {
- print '
+# Print string with word wrapping and indentation using
+# wprintlines.
+sub wprint {
+ my $text=shift @_;
+ my @lines=split(/\n/,$text);
+ wprintlines(@lines);
+}
+
+# Print with word wrapping and indentation. A line with
+# @ - :
+# sets two column tabs. A tab or multiple spaces is taken
+# to indicate indentation. If the first column value is
+# only a single '|', this is just an indication to skip
+# one tab.
+sub wprintlines {
+ my @lines=@_;
+ my $ind1=2;
+ my $ind2=6;
+ my $i;
+ foreach my $line (@lines) {
+ if ($line=~s/^@//) {
+ $ind2=1+index($line,':');
+ $ind1=1+index($line,'-');
+ if ($ind1<1) {$ind1=$ind2;}
+ next;
+ }
+ my $firstindent=0;
+ if ($line=~s/^(\t|\s{2,})(\S)/$2/) {$firstindent=$ind1;}
+ my $indent=$firstindent;
+ if ($line=~/^(.*\S)(\t|\s{2,})/) {
+ $indent=$ind2;
+ if ($1 eq '|') {$line=' ';}
+ else {$line=$1." ";}
+ $i=$indent-$firstindent-length($line);
+ if ($i>0) {$line.=' ' x $i;}
+ $line.=$';
+ }
+ print wrap(' ' x $firstindent,' ' x $indent,$line)."\n";
+ }
+}
+
+
+########################################
+##### TEXT DATA
+
+__DATA__
+
+::::::::::::::::::::::::::::::::::::::::
+:::::::::: Version
+TeXcount version ${versionnumber}, ${versiondate}.
+
+:::::::::: Reference
+The TeXcount script is copyright of ${maintainer} (${copyrightyears}) and published under the LaTeX Project Public Licence.
+
+Go to the TeXcount web page
+ ${website}
+for more information about the script, e.g. news, updates, help, usage tips, known issues and short-comings, or to access the script as a web application. Feedback such as problems or errors can be reported to einarro@ifi.uio.no.
+
+:::::::::: License
+TeXcount version ${versionnumber}
+
+Copyright ${copyrightyears} ${maintainer}
+
+The TeXcount script is published under the LaTeX Project Public License (LPPL)
+ http://www.latex-project.org/lppl.txt
+which grants you, the user, the right to use, modify and distribute the script. However, if the script is modified, you must change its name or use other technical means to avoid confusion.
+
+The script has LPPL status "maintained" with ${maintainer} being the current maintainer.
+
+::::::::::::::::::::::::::::::::::::::::
+:::::::::: HelpTitle
***************************************************************
-* TeXcount.pl '.$versionnumber.', '.$versiondate.'
+* TeXcount version ${versionnumber}, ${versiondate}
*
-Count words in TeX and LaTeX files, ignoring macros, tables,
-formulae, etc.
-';
- print_syntax();
- print '
-The script counts words as either words in the text, words in
-headers/titles or words in floats (figure/table captions).
-Macro options (i.e. \marco[...]) are ignored; macro parameters
-(i.e. \macro{...}) are counted or ignored depending on the
-macro, but by default counted. Begin-end groups are by default
-ignored and treated as \'floats\', though some (e.g. center) are
-counted.
-
-Unless -nocol (or -nc) has been specified, the output will be
-colour coded. Counted text is coloured blue with headers are in
-bold and in HTML output caption text is italicised.
-
-Mathematical formulae are not counted as words, but are instead
-counted separately with separate counts for inlined formulae
-and displayed formulae. Similarly, the number of headers and
-the number of \'floats\' are counted. Note that \'float\' is used
-here to describe anything defined in a begin-end group unless
-explicitly recognized as text or mathematics.
-
-The verbose options (-v1, -v2, -v3, showstate) produces output
-indicating how the text has been interpreted. Check this to
-ensure that words in the text has been interpreted as such,
-whereas mathematical formulae and text/non-text in begin-end
-groups have been correctly interpreted.
-
-Parsing instructions may be passed to TeXcount using comments
-in the LaTeX files on the format
+Count words in TeX and LaTeX files, ignoring macros, tables, formulae, etc.
+
+::::::::::::::::::::::::::::::::::::::::
+:::::::::: OptionsHead
+
+Syntax: TeXcount.pl [options] files
+
+Options:
+
+:::::::::: OptionsPrefix
+@ - :
+:::::::::: Options
+ -relaxed Uses relaxed rules for word and option handling: i.e. allows more general cases to be counted as either words or macros.
+ -restricted Restricts the rules for word and option handling.
+ -v Verbose (same as -v3).
+ -v0 Do not present parsing details.
+ -v1 Verbose: print parsed words, mark formulae.
+ -v2 More verbose: also print ignored text.
+ -v3 Even more verbose: include comments and options.
+ -v4 Same as -v3 -showstate.
+ -showstate Show internal states (with verbose).
+ -brief Only prints a brief, one line summary of counts.
+ -q, -quiet Quiet mode, no error messages. Use is discouraged!
+ -strict Strict mode, warns against begin-end groups for which rule are not defined.
+ -sum, -sum= Make sum of all word and equation counts. May also use -sum=#[,#] with up to 7 numbers to indicate how each of the counts (text words, header words, caption words, #headers, #floats, #inlined formulae, #displayed formulae) are summed. The default sum (if only -sum is used) is the same as -sum=1,1,1,0,0,1,1.
+ -nosum Do not compute sum.
+ -sub, -sub= Generate subcounts. Option values are none, part, chapter, section or subsection. Default (-sub) is set to subsection, whereas unset is none. (Alternative option name is -subcount.)
+ -nosub Do not generate subcounts.
+ -col Use ANSI colours in text output.
+ -nc, -nocol No colours (colours require ANSI).
+ -nosep, -noseparator No separating character/string added after each word (default).
+ -sep=, -separator= Separating character or string to be added after each word.
+ -html Output in HTML format.
+ -htmlcore Only HTML body contents.
+ -opt, -optionfile Read options/parameters from file.
+ - Read LaTeX code from STDIN.
+ -inc Parse included TeX files (as separate file).
+ -merge Merge included TeX files into code (in place).
+ -noinc Do not include included tex files (default).
+ -incbib Include bibliography in count, include bbl file if needed.
+ -nobib Do not include bibliography in count (default).
+ -incpackage= Include rules for the given package.
+ -total Do not give sums per file, only total sum.
+ -1 Same as -brief and -total. Ensures there is only one line of output. If used in conjunction with -sum, the output will only be the total number. (NB: Character is the number one, not the letter L.)
+ -template= Speficy an output template. Use {1},...,{7}, {SUM} and {TITLE} to include values, {1?...?1} etc. to conditionally include sections, {1?....|...?1} etc. to specify an alternative text if zero. To include subcounts, use {SUB?...?SUB} where ... is replaced with the template to use per subcount. Line shift may be specified using \\n.
+ -dir, -dir= Specify the working directory using -dir=path. Remember that the path must end with \\ or /. If only -dir is used, the directory of the parent file is used.
+ -enc=, -encoding= Specify encoding (default is to guess the encoding).
+ -utf8, -unicode Selects Unicode (UTF-8) for input and output. This is automatic with -chinese, and is required to handle e.g. Korean text. Note that the TeX file must be save in UTF-8 format (not e.g. GB2312 or Big5), or the result will be unpredictable.
+ -alpha=, -alphabets= List of Unicode character groups (or digit, alphabetic) permitted as letters. Names are separated by ',' or '+'. If list starts with '+', the alphabets will be added to those already included. The default is Digit+alphabetic.
+ -logo=, -logograms= List of Unicode character groups interpreted as whole word characters, e.g. Han for Chinese characters. Names are separated by ',' or '+'. If list starts with '+', the alphabets will be added to those already included. By default, this is set to include Ideographic, Katakana, Hiragana, Thai and Lao.
+ -ch, -chinese, -zhongwen Turns on support for Chinese characters. TeXcount will then count each Chinese character as a word.
+ -jp, -japanese Turns on support for Japanese characters. TeXcount will count each Japanese character (kanji, hiragana, and katakana) as one word, i.e. not do any form of word segmentation.
+ -kr, -korean Turns on support for Korean. This will count hangul and han characters, i.e. with no word separation. NB: Experimental!
+ -kr-words, -korean-words Turns on support for Korean words, i.e. hangul words separated by characters. Han characters are still counted as characters. NB: Experimental!
+ -ch-only, ..., -korean-words-only As options -chinese, ..., -korean-words, but also excludes letter-based words or trims down the character set to the minimum.
+ -char, -character, -letters Counts letters/characters instead of words. Note that spaces and punctuation is not counted.
+ -char-only, ..., -letters-only Like -letters, but counts alphabetic letters only.
+ -countall, -count-all The default setting in which all characters are included as either alphabets og logograms.
+ -freq Produce individual word frequency table.
+ -stat Produce statistics on language/script usage.
+ -codes Display output style code overview and explanation. This is on by default.
+ -nocodes Do not display output style code overview.
+ -h, -?, -help, /? Help text.
+ -h=, -?=, -help=, /?= Takes a macro or group name as option and returns a description of the rules for handling this if any are defined. If handling rule is package specific, use -incpackage=package name: -incpackage must come before -h= on the command line to take effect.
+ -ver, -version Print version number.
+ -lic, -license, -licence Licence information.
+
+::::::::::::::::::::::::::::::::::::::::
+:::::::::: HelpText
+The script counts words as either words in the text, words in headers/titles or words in floats (figure/table captions). Macro options (i.e. \\macro[...]) are ignored; macro parameters (i.e. \\macro{...}) are counted or ignored depending on the macro, but by default counted. Begin-end groups are by default ignored and treated as 'floats', though some (e.g. center) are counted.
+
+Mathematical formulae are not counted as words, but are instead counted separately with separate counts for inlined formulae and displayed formulae. Similarly, the number of headers and the number of 'floats' are counted. Note that 'float' is used here to describe anything defined in a begin-end group unless explicitly recognized as text or mathematics.
+
+The verbose options (-v1, -v2, -v3, showstate) produces output indicating how the text has been interpreted. Check this to ensure that words in the text has been interpreted as such, whereas mathematical formulae and text/non-text in begin-end groups have been correctly interpreted.
+
+Summary, as well as the verbose output, may be produced as text (default) or as HTML code using the -html option. The HTML may then be sent to file which may be viewed with you favourite browser.
+
+Under UNIX, unless -nocol (or -nc) has been specified, the output will be colour coded using ANSI colour codes. Counted text is coloured blue with headers are in bold and in HTML output caption text is italicised. Use 'less -r' instead of just 'less' to view output: the '-r' option makes less treat text formating codes properly. Windows does not support ANSI colour codes, and so this is turned off by default.
+
+:::::::::: TCinstructions
+Parsing instructions may be passed to TeXcount using comments in the LaTeX files on the format
+@ - :
%TC:instruction arguments
-where valid instructions for setting parsing rules, typically
-set at the start of the document (applies globally), are:
+and are used to control how TeXcount parses the document. The following instructions are used to set parsing rules which will apply to all subsequent parsing (including other files):
%TC:macro [macro] [param.states]
- macro handling rule, no. of and rules for parameters
+ | macro handling rule, no. of and rules for parameters
%TC:macroword [macro] [number]
- macro counted as a given number of words
+ | macro counted as a given number of words
%TC:header [macro] [param.states]
- header macro rule, as macro but counts as one header
+ | header macro rule, as macro but counts as one header
%TC:breakmacro [macro] [label]
- macro causing subcount break point
- %TC:group [name] [parsing-state]
- begin-end-group handling rule
+ | macro causing subcount break point
+ %TC:group [name] [param.states] [content-state]
+ | begin-end-group handling rule
%TC:floatinclude [macro] [param.states]
- as macro, but also counted inside floats
+ | as macro, but also counted inside floats
%TC:preambleinclude [macro] [param.states]
- as macro, but also counted inside the preamble
+ | as macro, but also counted inside the preamble
%TC:fileinclue [macro] [rule]
- file include, add .tex if rule=2, not if rule=0
-The [param.states] is used to indicate the number of parameters
-used by the macro and the rules of handling each of these: format
-is [#,#,...,#] with one number for each parameter, and main rules
-are 0 to ignore and 1 to count as text. Parsing instructions
-which may be used anywhere are:
+ | file include, add .tex if rule=2, not if rule=0, if missing when rule=1
+The [param.states] is used to indicate the number of parameters used by the macro and the rules of handling each of these: the format is [#,#,...,#] with # representing one number for each parameter giving the parsing state to use for that parameter, alternatively just a single number (#) indicating how many parameters to ignore (parsing state 0). The option [content-state] is used to give the parsing state to use for the contents of a begin-end group. The main parsing states are 0 to ignore and 1 to count as text.
+
+Parsing instructions which may be used anywhere are:
+@ - :
%TC:ignore start block to ignore
%TC:endignore end block to ignore
%TC:break [title] add subcount break point here
See the documentation for more details.
-Unix hint: Use \'less -r\' instead of just \'less\' to view output:
-the \'-r\' option makes less treat text formating codes properly.
+Command line options and most %TC commands (prefixed by % rather than %TC:) may be placed in an options file. This is particularly useful for defining your own output templates and macro handling rules.
-Windows hint: If your Windows interprets ANSI colour codes, lucky
-you! Otherwise, use the -nocol (or -nc) option with the verbose
-options or the output will be riddled with colour codes. Instead,
-you can use -html to produce HTML code, write this to file and
-view with your favourite browser.
-';
- print_reference();
-}
-
-sub print_reference {
- print '
-The TeXcount script is copyright of Einar Andreas Rødland (2008)
-and published under the LaTeX Project Public License.
-
-For more information about the script, e.g. news, updates, help,
-usage tips, known issues and short-comings, go to
- http://folk.uio.no/einarro/Comp/texwordcount.html
-or go to
- http://folk.uio.no/einarro/Services/texcount.html
-to access the script as a web service. Feedback such as problems
-or errors can be reported to einarro@ifi.uio.no.
-';
-}
-
-sub print_license {
- print 'TeXcount version '.$versionnumber.'
-
-Copyright 2008 Einar Andreas Rødland
-
-The TeXcount script is published under the LaTeX Project Public
-License (LPPL)
- http://www.latex-project.org/lppl.txt
-which grants you, the user, the right to use, modify and distribute
-the script. However, if the script is modified, you must change its
-name or use other technical means to avoid confusion.
-
-The script has LPPL status "maintained" with Einar Andreas
-Rødland being the current maintainer.
-';
-}
+::::::::::::::::::::::::::::::::::::::::
diff --git a/Build/source/texk/texlive/linked_scripts/texlive/tlmgr.pl b/Build/source/texk/texlive/linked_scripts/texlive/tlmgr.pl
index d9d8dedb8a6..411c774bb4a 100755
--- a/Build/source/texk/texlive/linked_scripts/texlive/tlmgr.pl
+++ b/Build/source/texk/texlive/linked_scripts/texlive/tlmgr.pl
@@ -1,12 +1,12 @@
#!/usr/bin/env perl
-# $Id: tlmgr.pl 23193 2011-07-21 06:05:07Z preining $
+# $Id: tlmgr.pl 23271 2011-07-29 23:20:41Z preining $
#
# Copyright 2008, 2009, 2010, 2011 Norbert Preining
# This file is licensed under the GNU General Public License version 2
# or any later version.
-my $svnrev = '$Revision: 23193 $';
-my $datrev = '$Date: 2011-07-21 08:05:07 +0200 (Thu, 21 Jul 2011) $';
+my $svnrev = '$Revision: 23271 $';
+my $datrev = '$Date: 2011-07-30 01:20:41 +0200 (Sat, 30 Jul 2011) $';
my $tlmgrrevision;
if ($svnrev =~ m/: ([0-9]+) /) {
$tlmgrrevision = $1;
@@ -6128,10 +6128,11 @@ other lines must look like
where the allowed keys are
C<gui-expertmode> (values 0 or 1),
-C<persistent-downloads> (values 0 or 1),
+C<persistent-downloads> (values 0 or 1), C<auto-remove> (values 0 or 1),
and C<gui-lang> (values like the command line arguments).
-C<persistent-downloads> and C<gui-lang> correspond to the command line
-options of the same name. C<gui-expertmode> switches between the full
+C<persistent-downloads>, C<gui-lang>, and C<auto-remove> correspond to the
+respective command line options of the same name.
+C<gui-expertmode> switches between the full
GUI and a simplified GUI with only the important and mostly used
settings.
diff --git a/Master/texmf-dist/doc/support/texcount/QuickReference.pdf b/Master/texmf-dist/doc/support/texcount/QuickReference.pdf
index 146b8595e07..e3a8d1f7e9b 100644
--- a/Master/texmf-dist/doc/support/texcount/QuickReference.pdf
+++ b/Master/texmf-dist/doc/support/texcount/QuickReference.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/support/texcount/QuickReference.tex b/Master/texmf-dist/doc/support/texcount/QuickReference.tex
index e6aeaf3eee8..16726e808d2 100644
--- a/Master/texmf-dist/doc/support/texcount/QuickReference.tex
+++ b/Master/texmf-dist/doc/support/texcount/QuickReference.tex
@@ -1,5 +1,4 @@
\documentclass{article}
-\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{a4wide}
\usepackage{times}
@@ -7,10 +6,9 @@
\include{macros}
-\title{\TeXcount{}\footnote{Copyright (2008-2009) of Einar Andreas Rødland, distributed
-under the \LaTeX{} Project Public Licence (LPPL).}~
-Quick Reference Guide\\
-Version 2.2
+\title{%
+\TeXcount{} Quick Reference Guide\\
+Version \version\copyrightfootnote
}
\begin{document}
@@ -22,71 +20,8 @@ Version 2.2
Syntax for running \TeXcount{}:
\codeline{texcount \textit{[options] [files]}}
where \code{texcount} refers to the TeXcount Perl-script, and the options may be amongst the following:
-\begin{description}
-\def\option[#1]{\item[\quad\code{#1}]}
-\def\alt#1{[#1]}
-\option[-v]Verbose (same as -v3).
-
-\option[-v0]No details (default).
-
-\option[-v1]Prints counted text, marks formulae.
-
-\option[-v2]Also prints ignored text.
-
-\option[-v3]Also includes comments and options.
-
-\option[-v4]Same as \code{-v3 -showstate}.
-
-\option[-showstate]Show internal states (with verbose).
-
-\option[-brief]Only prints a one line summary of the counts.
-
-\option[-q, -quiet]Quiet mode, does not print error messages. Use is
-discouraged, but it may be useful when piping the output into another
-application.
-
-\option[-total]Only give total sum, no per file sums.
-
-\option[-1]Same as specifying \code{-brief} and \code{-total}, and ensures there will only be one line of output. If used with \code{-sum}, the output will only be the total number.
-
-\option[-sub\alt{=\ldots}, -subcount\alt{=\ldots}]Generate subcounts. Valid option values are \code{none}, \code{part}, \code{chapter}, \code{section} and \code{subsection} (default), indicating at which level subcounts are generated.
-
-\option[-sum\alt{=n,n,\ldots}]Produces total sum, default being all words and formulae, but customizable to any weighted sum of the seven counts (list of weights for text words, header words, caption words, headers, floats, inlined formulae, displayed formulae).
-
-\option[-nc, -nocol]No colours (colours require ANSI).
-
-\option[-relaxed]Relaxes the rules for matching words and macro options.
-
-\option[-inc]Include tex files included in the document.
-
-\option[-noinc]Do not include included tex files (default).
-
-\option[-dir\alt{=\ldots}]Working directory, either taken from the first file given, or specified.
-
-\option[-utf8, -unicode]Turn on Unicode (UTF-8) support.
-
-\option[-ch, -chinese, -zhongwen]Turn on Chinese mode in which Chinese
-characters are counted. Switches on UTF-8 mode.
-
-\option[-jp, -japanese]Turn on Japanese mode in which Japanese
-characters (kanji and kana) are counted. Switches on UTF-8 mode.
-
-\option[-html]Output in HTML format.
-
-\option[-htmlcore]Only HTML body contents.
-
-\option[-codes]Display an overview of the colour codes. Can be used as a separate option to only display the colour codes, or together with files to parse.
-
-\option[-nocodes]Do not display overview of colour codes.
-
-\option[-h, -?, --help, /?]Help.
-
-\option[--version]Print version number.
-
-\option[--license]License information.
-
-\end{description}
+\input{sub_options}
\section{\TeXcount{} instructions embedded in \LaTeX{} documents}
@@ -96,13 +31,7 @@ Instructions to \TeXcount{} can be given from within the
\codeline{\%TC:\textit{instruction [name] parameters}}
where the name is use for instructions providing macro handling rules to give the name of the macro or group for which the rule applies.
-\begin{description}\def\option#1{\item[\bigcode{#1}]}
-
-\option{ignore}Indicates start of a region to be ignored. End region with the TC-instruction \code{endignore}.
-
-\option{break \textit{title}}Break point which initiates a new subcount. The title is used to identify the following region in the summary output.
-
-\end{description}
+\input{sub_tc_other}
\subsection{Adding macro handling rules}
@@ -110,24 +39,6 @@ The macro handling rules all take the format
\codeline{\%TC:\textit{instruction name parameters}}
where the name indicates the macro (with backslash) or group name for which the rule applies.
-\begin{description}\def\option[#1]{\item[\bigcode{#1}]}
-
-\option[macro]Define macro handling rule for specified macro. Takes one parameter which is either an integer or a \code{[]}-enclosed array of integers (e.g. \code{[0,1,0]}). An integer value $n$ indicates that the $n$ first parameters to the macro should be ignored. An array of length $n$ indicates that the first $n$ parameters should be handled, and the numbers of the array specifies the parsing status (see below) with which they should be parsed. Giving the number $n$ as parameter is equivalent to giving an array of $n$ zeroes (\code{[0,\ldots,0]}) as zero is the parsing status for ignoring text.
-
-\option[macroword]This defines the given macro to represent a certain number of words, where the number is given as the parameter.
-
-\option[header]Define macro to give a header. This is specified much as the macro rule. The added effect is that the header counter is increase by 1. Note, however, that you should specify a parameter array, otherwise none of the parameters will be parsed as header text. The parser status for header text is 2, so a standard header macro that uses the first parameter as header should be given the parameter \code{[2]}.
-
-\option[breakmacro]Specify that the given macro should cause a break point. Defining it as a header macro does not do this, nor is it required of a break point macro that it be a header (although I suppose in most cases of interest it will be).
-
-\option[group]This specifies a begin-end group with the given name (no backslash). It takes two further parameters. The first parameter speficies the macro rule following \code{\bs{begin}\{\textit{name}\}}. The second parameter specifies the parser status with which the contents should be parsed: e.g. $1$ for text (default rule), $0$ to ignore, $-1$ to specify a float (table, group, etc.) for which text should not be counted but captions should, $6$ and $7$ for inline or displated math.
-
-\option[floatinclude]This may be used to specify macros which should be counted when within float groups. The handling rules are spefified as for \code{macro}, but like with \code{header} an array parameter should be provided and parameters that should be counted as text in floats should be specified by parsing status 3. Thus, a macro that takes one parameter which should be counted as float/caption text should take the parameter \code{[3]}.
-
-\option[preambleinclude]The preamble, i.e. text between \code{\bs{documentclass}} and \code{\bs{begin}\{document\}}, if the document contains one, should generally not be included in the word count. However, there may be definitions, e.g. \code{\bs{title}\{title text\}}, that should still be counted. In order to be able to include these special cases, there is a preambleinclude rule in which one may speficy handling rules for macros within the preamble. Again, the rule is speficied like the \code{macro} rules, but since the default is to ignore text the only relevant rules to be specified require an array.
-
-\option[fileinclude]By default, \TeXcount{} does not automatically add files included in the document using \code{\bs{input}} or \code{\bs{include}}, but inclusion may be turned on by using the option \code{-inc}. If other macros are used to include files, these may be specifed by adding fileinclude rules for these macros. The specification takes one parameter: 0 if the file name should be used as provided, 1 if file type \code{.tex} should be added to files without a file type, and 2 if the file tyle \code{.tex} should always be added.
-
-\end{description}
+\input{sub_addrules}
\end{document}
diff --git a/Master/texmf-dist/doc/support/texcount/README b/Master/texmf-dist/doc/support/texcount/README
index 45512130d2f..2ca33b6e3b6 100644
--- a/Master/texmf-dist/doc/support/texcount/README
+++ b/Master/texmf-dist/doc/support/texcount/README
@@ -2,7 +2,7 @@ TeXcount is a Perl script that counts the number of words in the
text of LaTeX files. It has rules for handling most of the common
macros and provides colour coded output indicating which parts have
been counted. Go to
- http://folk.uio.no/einarro/Comp/texwordcount.html
+ http://app.uio.no/ifi/texcount/
for more information or to access the script online as a web service.
The package, i.e. the script and all accompanying files, is
diff --git a/Master/texmf-dist/doc/support/texcount/TeXcount.pdf b/Master/texmf-dist/doc/support/texcount/TeXcount.pdf
index d40319268d1..c1fbb094c99 100644
--- a/Master/texmf-dist/doc/support/texcount/TeXcount.pdf
+++ b/Master/texmf-dist/doc/support/texcount/TeXcount.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/support/texcount/TeXcount.tex b/Master/texmf-dist/doc/support/texcount/TeXcount.tex
index 78296cc2847..7d5e89110b1 100644
--- a/Master/texmf-dist/doc/support/texcount/TeXcount.tex
+++ b/Master/texmf-dist/doc/support/texcount/TeXcount.tex
@@ -1,34 +1,40 @@
\documentclass{article}
-\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{a4wide}
\usepackage{times}
+\usepackage{listings}
+\usepackage{url}
\include{macros}
%\parindent=0pt\parskip=8pt
+\lstset{basicstyle=\ttfamily\scriptsize,xleftmargin=2cm,xrightmargin=2cm}
+\lstset{basicstyle=\ttfamily\scriptsize,xleftmargin=2cm,xrightmargin=2cm}
-
-\title{\TeXcount\footnote{Copyright (2008-2009) of Einar Andreas Rødland, distributed
-under the \LaTeX{} Project Public Licence (LPPL).}\\
+\title{%
+\LARGE
+\TeXcount\\
\Large
Perl script for counting words in \LaTeX{} documents\\
-Version 2.2}
-\author{Einar Andreas Rødland}
+Version \version\copyrightfootnote
+}
+\author{Einar Andreas R{\o}dland}
\begin{document}
\maketitle
{\abstract%
-\TeXcount{} is a Perl script for counting words in \LaTeX{} documents. It recognizes most of the common macros, and has rules for which parameters to count and not to count; the main text is counted separately from the words in headers and in captions of figures and tables. Finally, it produces a colour coded version of the parsed document, either as a text document or as HTML to be viewed in a browser, indicating which parts of the document have been included in the count.
+\TeXcount{} is a Perl script for counting words in \LaTeX{} documents. It recognises most of the common macros, and has rules for which parameters to count and not to count; the main text is counted separately from the words in headers and in captions of figures and tables. Finally, it produces a colour coded version of the parsed document, either as a text document or as HTML to be viewed in a browser, indicating which parts of the document have been included in the count.
%
}
-\tableofcontents
+{\scriptsize\tableofcontents}
+\pagebreak
-\section{What does \TeXcount{} do?}
+
+\section{What \TeXcount{} does}
\TeXcount{} is a Perl script made for counting the words in a \LaTeX{} document. Since \LaTeX{} documents are formated using lots of macro instructions and often contain both mathematical formulae and floating tables and figures, this is no trivial task.
@@ -50,103 +56,122 @@ The primary focus of \TeXcount{} is to:
\end{itemize}
The last point on this list is one of the most important. Having an accurate word count is of little value unless you know that it is accurate; conversly, trusting an inaccurate word count can be potentially harmful, e.g. if you are submitting a paper or a report which has a strict word limit.
-\TeXcount{} handles complete \LaTeX{} documents, i.e. that start with \code{\bs{documentclass}} and has the text between \code{\bs{begin}\{document\}} and \code{\bs{end}\{document\}}, as well as partial documents made to be included in another \LaTeX{} document. Automatic inclusion of included documents is possible, but is by default turned off.
+\TeXcount{} handles complete \LaTeX{} documents, i.e. that start with \code{\bs{documentclass}} and has the text between \code{\bs{begin}\{document\}} and \code{\bs{end}\{document\}}, as well as partial documents made to be included in another \LaTeX{} document. However, in either case, it requires that all groups are closed: \code{\{\ldots\}} and \code{\bs{begin}\ldots\bs{end}}.
-Since \TeXcount{} relies on a relatively simple rules for how to handle the different macros and only performs limited checks on the validity of the \LaTeX{} document, it is your responsibility to make sure the document actually typesets in \LaTeX{} before running it through \TeXcount{}. Also, \TeXcount{} relies on handling macros taking parameters enclosed with \{ and \}, and on ignoring options enclosed by [ and ]: macros with significantly different syntax such as \code{\bs{vskip}} cannot be handled.
+Automatic parsing of included documents is possible, but is by default turned off. There are two options for turning this on: \code{-inc} and \code{-merge}. Turning it on using \code{-merge} will merge the included files into the main document. By using \code{-inc}, however, the included files are parsed separately rather than include the text into the appropriate location: this will perform a separate word count of the included document which is then later included in the total sum.
-\subsection{What \TeXcount{} does not do}
+Since \TeXcount{} relies on a relatively simple rules for handling the different macros and only performs limited checks on the validity of the \LaTeX{} document, it is your responsibility to make sure the document actually typesets in \LaTeX{} before running it through \TeXcount{}. Also, \TeXcount{} relies on handling macros taking parameters enclosed with \{ and \}, and on ignoring options enclosed by [ and ]: macros with significantly different syntax such as \code{\bs{vskip}} cannot be handled. There are also limitations on what may be contained in macro options enclosed in [], although this restriction may be relaxed by specifying the command line option \code{-relaxed}.
-While an ideal solution should be able to expand the macro instructions, thus being able to handle new macros, that would at it's worst require reimplementing much of \TeX{}, something that is clearly unrealistic. While some limited methods for handling new macros based on definitions in the file might be done, I have opted for a simpler solution: to define rules stating which paramters to count and which to ignore. Thus, \TeXcount{} cannot handle macros that may take a variable number of parameters. Nor can it handle macros that takes parameters on forms other than \code{\{parameter\}}.
+\subsection{What \TeXcount{} counts}
-In general, while \TeXcount{} does the parsing in some detail, it does not do it exacly as \TeX{} does it. In some respects there may therefore be critical difference: e.g. while \TeX{} reads one character at a time, \TeXcount{} reads one word at a time, so while \LaTeX{} would interpret \code{\bs{cite} me} as \code{\bs{}cite\{m\}e}, \TeXcount{} would interpret it like \code{\bs{cite}\{me\}}.
+Basically, \TeXcount{} has seven different counts plus an additional file count for use with total counts over a set of files. These and their indices (numbers used to identify them) are:
+\begin{description}
+\item[0. Number of files:] When multiple files are included, this is counted.
+\item[1. Text words:] Words that occur in the main text.
+\item[2. Header words:] Words that occur in headers, e.g. \code{\bs{title}} and \code{\bs{section}}.
+\item[3. Caption words:] Words that occur in figure and table captions.
+\item[4. Header count:] This counts the number of headers, i.e. each \code{\bs{section}} counts as 1.
+\item[5. Figure/float count:] This counts the number of floats and figures, e.g. \code{table} and \code{figure} groups.
+\item[6. Inline formulae:] This counts the number of inline formulae, i.e. \code{\$\ldots\$}.
+\item[7. Displayed formulae:] This counts the number of displayed formulae, e.g. \code{\bs{[}\ldots\bs{]}} or \code{equation} groups.
+\end{description}
+These are stored in an array and often referenced by their index: e.g. in the option \code{-sum=} which takes parameter values corresponding to counts 1 to 7, or \code{-tempate=} in which the counts are referred to by the indices 0 to 7.
-Another issue is that, since \TeXcount{} does not know how to expand macros, it cannot handle macros like \code{\bs{maketitle}}. Instead, it will count \code{\bs{title}\{title text\}} when it occurs.
+The primary role is to count the words. It is not entirely clear what should be considered words, so I have had to make some decisions. A sequence of letters is certainly a word. I also count acronyms like \textit{e.g.}, dashed words like \textit{over-all}, and \textit{it's} as one word. I have decided also to count numbers as words unless they are placed in a math environment. If \TeXcount{} breaks words that contain special characters, you may try the option \code{-relaxed} which extends the range of characters allowed as part of words.
-For users of languages containing letters other than A to Z, there may be an additional challenge. The script relies on Perl to recognize words as sequence of letters, and must therefore know which characters are considered to be letters. The Perl locale may be changed (as of now this is hard-coded!) to accommodate this, but this does not work for special letters encoded using \TeX{} or \LaTeX{} macros or codes: e.g. \code{\bs{aa}} and \code{\bs{"}a} will not be understood to be letters in the present implementation, whereas \code{\aa} and \code{\"a} will. I have added a relaxed mode where some special character and character modifying macros are allowed as parts of words, which may be activated with the option \code{-relaxed}.
+Alternatively, \TeXcount{} may be asked to count the number of letters/characters (not including spaces). It may also be set to count Chinese or Japanese characters.
+Mathematical formulae are not counted as words: it would be difficult to define a sensible rule for this. Instead, \TeXcount{} counts the number of inline formulae and displayed formulae separately. You may then decide on how to combine these counts with the word counts, e.g. using the \code{-sum} option.
-\section{Syntax and options}
+Text in headers (\code{\bs{title}}, \code{\bs{section}}, etc.) are counted separately: \TeXcount{} counts the number of headers as well as the number of words in headers. It may also provide subcounts for each of these by specifying the \code{-sub} option.
-\subsection{Running \TeXcount{}}
+Floating environments (or potentially floating environments) such as tables and figures are not counted as text, even if the cells of a table may containt text. However, if they have captions, these will be counted separately much like headers were. Footnotes are included in this count. By default, begin--end environments do not modify the parsing state: i.e. environments within the text are counted as text, etc. Rules for the most common environments, at least those that require non-default treatment, should be predefined, but you may have to add more rules if you use environments defined in packages or by yourself. If you wish to be warned against any groups names you use that lack a defined rule, set the option \code{-strict}.
-The command to run \TeXcount{} may vary slightly depending on the operating system and the local settings.
+Some macros are words by themselves: e.g. \code{\bs{LaTeX}}. These are counted as words provided the macro word rule has been defined for them, but you cannot expect \TeXcount{} to count something like \code{\bs{LaTeX}-word} or \code{\{\bs{TeX}\}count} as one word although the above explanation inicates that it should: \TeXcount{} will in both cases evaluate the macro and the following text separately and thus count them as separate entities. Since \TeXcount{} recognises \code{\bs{LaTeX}} and \code{\bs{TeX}} as single words, each of the two examples would end up being counted as two words.
-Under Linux/Unix, it should be sufficient to run \code{texcount.pl} provided it is in the PATH and has been made executable (\code{chmod u+x texcount.pl}). The first line of the file, the one starting with \code{\#!}, tells Linux/Unix which command to use to run the script. This may either direct to the \code{perl} command giving the full path to it (use \code{which perl} if you need to find out where \code{perl} is located) or, as is done in more recent versions of \TeXcount{}, contain the line \code{\#!/usr/bin/env perl} which should find the correct location for \code{perl} (provided the program \code{/usr/bin/env} is available).
+\subsection{What \TeXcount{} does not do}
-Under Windows, running \code{texcount} from the command line suffices if \code{texcount.pl} is in the path and pl-files are defined to run as Perl scripts.
+While an ideal solution should be able to expand the macro instructions, thus being able to handle new macros, that would at it's worst require reimplementing much of \TeX{}, something that is clearly unrealistic. Instead, I have opted for a simpler solution: to define rules stating which paramters to count and which to ignore and allowing for such rules to be added easily. Thus, \TeXcount{} cannot handle macros that may take a variable number of parameters. Nor can it handle macros that takes parameters on forms other than \code{\{parameter\}}. In particular, when interpreting macros, \TeXcount{} treats all \code{[\ldots]} blocks as macro options that should be excluded from the counts; \TeXcount{} has some restrictions on what it permits to go into an option in terms of size and characters.
-Alternatively, if the above methods do not work, you may have to execute it more exclicitly under Perl using \code{perl texcount.pl}. You then need to have the \code{perl} executable file in the path or give the explicit path.
+In general, while \TeXcount{} does the parsing in some detail, it does not do it exacly as \TeX{} does it. In some respects there may therefore be critical differences: e.g. while \TeX{} reads one character at a time, \TeXcount{} reads one word at a time, so while \LaTeX{} would interpret \code{\bs{cite} me} as \code{\bs{}cite\{m\}e}, \TeXcount{} would interpret it like \code{\bs{cite}\{me\}}.
-I will simply write \code{texcount} in this manual for the code to execute the script. Then, the syntax becomes
-\codeline{texcount \textit{[options] [files]}}
-where the options may be amongst the following:
-\begin{description}
-\def\option[#1]{\item[\quad\code{#1}]}
-\def\alt#1{[#1]}
+Another issue is that, since \TeXcount{} does not know how to expand macros, it cannot handle macros like \code{\bs{maketitle}} that add text to the document. With respect to \code{\bs{maketitle}}, I have instead set the rule for \code{\bs{title}\{title text\}} to count this as a header although it does not itself produce any text.
+
+\subsection{Problems to be aware of}
-\option[-v]Verbose (same as -v3).
+In most large documents, there will be cases where \TeXcount{} does not give an exact count. Reasons may be macros \TeXcount{} does not recognise, words that \TeXcount{} split in two (or more) because of special characters not recognised as letters, or options and parameters not counted which actually produce text. Some problems may also arise because it is not always clear what should be counted and \TeXcount{} implements one particular choice: counting numbers as letters/words, not counting formulae as words, not to count tables as text, etc. However, hopefully these should either consist of individual, infrequent errors which should have limited effect on the total count, or entire regions that are included or excluded for which the user may change the parsing rule to produce the desired count.
-\option[-v0]No details (default).
+There are, however, problems that may arise which are more fundamental and result in counts which are simply wrong rather than just inaccurate, or even make \TeXcount{} fail entirely.
-\option[-v1]Prints counted text, marks formulae.
+If \TeXcount{} fails to detect group endings properly, either closing \code{\{} or \code{\bs{end}}, it may end up ignoring major parts of the document. This should normally produce errors of some kind, although there may be cases when no errors are produced. However, by looking at the verbose output, it will be very clear that entire parts of the document has been excluded. Such problems may be cause by macros that allow unmatched group delimiters, and some effort has been made to minimise the risk of this at the cost of risking other but less critical errors: e.g. there are limits to what is permitted as macro options in order to ensure that a single unmatched \code{[} does not cause large parts of the document to be interpreted as a big option.
-\option[-v2]Also prints ignored text.
+For users of languages containing letters other than the Latin letters A to Z, there is a risk that \TeXcount{} may have difficulty identifying words correctly. The script relies on Perl to recognise words as sequence of letters, and must therefore know which characters are considered to be letters. Words containing letters not recognised by \TeXcount{} will tend to be split into two or more words, which can dramatically inflate the word count. The first step is to ensure that the file is read using the correct encoding: I generally suggest using the UTF-8 Unicode encoding, and from version 2.3. this is the default encoding used by \TeXcount{}, although other encodings may also be used. Unicode has good annotation of which characters are letters, and starting with version 2.3, \TeXcount{} uses Unicode internally to represent the text.
-\option[-v3]Also includes comments and options.
+While non-Latin letters like \code{\aa} and \code{\"a} should be recognised as letters, \TeX/\LaTeX codes using macros or special characters, such as \code{\bs{aa}} and \code{\bs{"}a}, are not immediately understood as letters. I have added patterns aimed at recognising these as well, but depending on the code you are writing, these patterns may either not be flexible enough to recognise all letter codes, or may be too flexible and recognise things it should not. I have added a relaxed mode (\code{-relaxed}) and a more restricted mode (\code{-restricted}) in which these patterns are more general or more constrained, but you should check how this performs on you actual texts by viewing the verbose output.
-\option[-v4]Same as \code{-v3 -showstate}.
-\option[-showstate]Show internal states (with verbose).
+\section{Syntax and options}
-\option[-brief]Only prints a one line summary of the counts.
+\subsection{Running \TeXcount{}}
-\option[-q, -quiet]Quiet mode, does not print error messages. Use is
-discouraged, but it may be useful when piping the output into another
-application.
+The command to run \TeXcount{} may vary slightly depending on the operating system and the local settings. You may also wish to rename it or define an alias.
-\option[-total]Only give total sum, no per file sums.
+Under Windows, running \code{texcount.pl} from the command line suffices if \code{texcount.pl} is in the path and pl-files are defined to run as Perl scripts.
-\option[-1]Same as specifying \code{-brief} and \code{-total}, and ensures there will only be one line of output. If used with \code{-sum}, the output will only be the total number.
+Under Linux/Unix, it should be sufficient to run \code{texcount.pl} provided it is in the PATH and has been made executable (\code{chmod u+x texcount.pl}). The first line of the file contains the line \code{\#!/usr/bin/env perl} which should find the correct location for \code{perl} (provided the program \code{/usr/bin/env} is available). If not, run \code{which perl} to locate Perl and replace the first line of the script with \code{\#!\textit{path}}.
-\option[-sub\alt{=\ldots}, -subcount\alt{=\ldots}]Generate subcounts. Valid option values are \code{none}, \code{part}, \code{chapter}, \code{section} and \code{subsection} (default), indicating at which level subcounts are generated.
+Alternatively, if the above methods do not work, you may have to run \TeXcount{} exclicitly in Perl by executing \code{perl texcount.pl}. You then need to have the \code{perl} executable in the path or give the explicit path.
-\option[-sum\alt{=n,n,\ldots}]Produces total sum, default being all words and formulae, but customizable to any weighted sum of the seven counts (list of weights for text words, header words, caption words, headers, floats, inlined formulae, displayed formulae).
+For simplicity, I will simply write \code{texcount.pl} in this manual for the code to execute the script. The syntax then becomes
+\codeline{texcount.pl \textit{[options] [files]}}
+where the options may be amongst the following:
-\option[-nc, -nocol]No colours (colours require ANSI).
+\input{sub_options}
-\option[-relaxed]Relaxes the rules for matching words and macro options.
+If more than one file is given, \TeXcount{} will perform the count on each of them printing the results for each, then print the total sum at the end. Note that files are parsed one by one in order of appearance and counts made per file; only afterwards are the totals computed.
-\option[-inc]Include tex files included in the document.
+\subsection{File encoding}
-\option[-noinc]Do not include included tex files (default).
+If your \TeX/\LaTeX{} document consists entirely of ASCII characters, there should be no problems with file encoding. However, if it contains non-ASCII characters, e.g. non-Latin letters such as \o{}, there are different ways in which these may be encoded in the files.
-\option[-dir\alt{=\ldots}]Working directory, either taken from the first file given, or specified.
+The main encoding supported by \TeXcount{} is UTF-8 (Unicode), and this is used to represent text internally in \TeXcount{}. In older versions of \TeXcount{}, Latin-1 (ISO-8859-1) was the default encoding, but this may cause problems when using non-Latin characters.%
+\footnote{%
+In Perl, which \TeXcount{} is written in, Latin-1 is the default. However, starting with version 2.3, \TeXcount{} has switched to using UTF-8 (Unicode) internally and will convert text to Unicode before processing: in older version, internal representation was UTF-8 or Latin-1 depending on the options used.}
+Both of these are compatible with ASCII: i.e. both are extensions of ASCII, so ASCII characters will be treated correctly by both encodings, but non-ASCII characters will be treated differently.
-\option[-utf8, -unicode]Turn on Unicode (UTF-8) support.
+From version 2.3 of \TeXcount{}, it is possible to specify other encodings using the \code{-encoding=} option. If no encoding is specified, \TeXcount{} will guess which encoding is used. By default, this guessing is limited to ASCII, UTF-8 and Latin-1. If other encodings are used, the automatic guessing is likely to pick Latin-1 since most files would result in valid Latin-1 code. If the \code{-chinese} or \code{-japanese} option is set, it will guess at other encodings, but still with UTF-8 as the first choice.
-\option[-ch, -chinese, -zhongwen]Turn on Chinese mode in which Chinese
-characters are counted. Switches on UTF-8 mode.
+I generally recommend using UTF-8 Unicode: this is increasingly being the new standard. Basically, Unicode contains the characters needed for all existing languages, enumerated from 0 and upwards (beyond 100000), which resolves to problem of requiring different character sets. Since there are more than 256 characters in Unicode, Unicode cannot be represented using one byte per character: UTF-8 is a way to encode the Unicode characters into a list of bytes so that ASCII characters (no. 0--127) are represented by one byte (same as in ASCII), while non-ASCII characters are represented using two or more bytes. Unicode may also be encoded using two bytes to represent each of Unicode characters 0--65535, which covers most of practical use, but this is less commonly used as a file format: it is, however, common for internal representation of strings in memory, as done by e.g. Java, so Perl is the odd one out in using UTF-8 for internal string representation.
-\option[-jp, -japanese]Turn on Japanese mode in which Japanese
-characters (kanji and kana) are counted. Switches on UTF-8 mode.
+If an encoding is specified using the \code{-encoding=} option, the input will be decoded from the specified encoding into UTF-8. If HTML output is specified, the output will be UTF-8. This ensures that all HTML produced is UTF-8, which is also the encoding specified in the HTML header. If text output is used, the specified encoding is used for the output. E.g. if you specify \code{-encoding=latin1}, \TeXcount{} will assume that all files are encoded in Latin-1, and will also produce the detailed output using Latin-1. For piping, i.e. option \code{-}, this is useful as it ensures the output has the same encoding as the input.
-\option[-html]Output in HTML format.
+For convenience, if no encoding is specified, \TeXcount{} will try to guess which encoding is the appropriate one. This is done simply by checking a specified list of encodings one by one until one is found that fits the text. The default is to check ASCII, then UTF-8, and finally Latin-1. If none fits, \TeXcount{} should try to decode the ASCII part of the text replacing non-ASCII characters with a wildcard character, although there may be cases when the decoding exits upon hitting an error. If Chinese or Japanese languages are specified, UTF-8 is tried first, then other encodings are checked depending on the language.
-\option[-htmlcore]Only HTML body contents.
+Note that if no encoding is specified and \TeXcount{} left to guess the appropriate encoding, all output will be UTF-8. Thus, letting \TeXcount{} guess the encoding may not be suitable when using \TeXcount{} in a pipe since the UTF-8 output may not be compatible with the encoding of the input. If multiple files are parsed, \TeXcount{} will guess the encoding separately for each file even if they are included (\code{-inc} or \code{-merge}) in a file with an identified encoding, and may thus end up selecting different encodings for different files.
-\option[-codes]Display an overview of the colour codes. Can be used as a separate option to only display the colour codes, or together with files to parse.
+\subsection{Language scripts, alphabets and character sets}
-\option[-nocodes]Do not display overview of colour codes.
+In additional to the traditional Latin letters, A-Z, a number of letters are recognised by Unicode as part of the extension of the Latin letters. Some languages, however, use entirely different character sets.
-\option[-h, -?, --help, /?]Help.
+By default, \TeXcount{} has been set up to recognise all alphabets. However, there is a distinction between alphabets like the Latin, Greek, Cyrillic, etc. in which words consists of multiple letters, and languages like Chinese in which each character should be counted as a word. For simplicity, we refer to these as \emph{alphabetic} characters and \emph{logograms}.\footnote{%
+Actually, these names are not completely accurate. A logogram is a script which represents a word or `meaningful unit', but e.g. the Japanese kana and Korean hangul are counted as words although they represent sound or syllables rather than meanings.}
+The options \code{-alphabets=} and \code{-logograms=} (or \code{-alpha=} and \code{-logo=} for short) allows you to specify which characters to use as either alphabetic letters or whole word characters. These take values that consist of Unicode properties separated by \code{,} or \code{+}. The default setting corresponds to
+\codeline{-alphabets=Digit,alphabetic}
+in which \code{alphabetic} is defined by \TeXcount{} as the Unicode \code{Alphabetic} class minus logographic script classes, and
+\codeline{-logograms=Ideographic,Hiragana,Katakana,Thai,Lao}
+which should cover Chinese characters (\code{Han}) as well as the Japanese characters (\code{Han} for the kanji, \code{Hiragana} and \code{Katakana} for the kana). Both options remove previous script settings, unless the list is prefixed by \code{+} in which case the scripts are added: e.g. \code{-logograms=+cjkpunctuation} will add the CJK punctuation characters (defined by \TeXcount) to the set of counted characters.
-\option[--version]Print version number.
+Applicable Unicode properties/scripts include \code{Digit}, \code{Latin}, \code{Greek}, \code{Cyrillic}, \code{Hebrew}, \code{Arabic}, \code{Han}, \code{Katakana}, \code{Hiragana}, and more.\footnote{A more complete overview is available at Wikipedia: \url{http://en.wikipedia.org/wiki/Script_(Unicode)}.}
-\option[--license]License information.
+In addition to the Unicode properties, \TeXcount{} has added a few additional character groups. The properties \code{alphabetic}, \code{digit} and \code{alphanumeric} are more restrictive than their Unicode name-sakes: \code{alphabetic} excludes the default logographic character sets, and \code{digit} consists only of 0--9 unlike Unicode \code{Digit} which includes numerals from other scripts. There is also \code{cjkpunctuation} which is intended to identify Chinese/Japanese/Korean punctuation.
-\end{description}
+Note that the Unicode properties are case sensitive. The native Unicode properties start with capital letters, whereas the properties defined by \TeXcount{} are all lower case. Invalid properties will be ignored.
-If more than one file is given, \TeXcount{} will perform the count on each of them printing the results for each, then print the total sum at the end.
+The options \code{-chinese} and \code{-japanese} still exist and simply restrict the logographic character sets. In addition, \code{-chinese-only} and \code{-japanese-only} will exclude alphabetic words from the counting, equivalent to \code{-alphabets=} with no script properties given. In addition, these options will change the lists of file encodings \TeXcount{} will try if no encoding is given.
+
+The option \code{-stat} has been added to produce overall word counts per script type. This uses the character classes specified in the \code{-alphabets=} and \code{-logograms=} options, so the default will be able to count which words are purely alphabetic and which contain numbers (or a combination of both), but will not distinguish between e.g. Latin and Greek. To do that, you would have to specify the script classes: e.g.
+\codeline{-alphabets=digit,Latin,Greek,Cyrillic}
+will count words containing the numbers 0--9, Latin letters (including the extended Latin character set), Greek letters and Cyrillic letters. Words may contain any combination of these: \TeXcount{} does not require that a word consist of only one type of script. Also, note that if \code{digit} had not been included, numbers would not be allowed to be part of or counted as words. The output statistics will then give the number of words containing each of these script classes (or combination).
\subsection{Parsing details}
@@ -154,15 +179,15 @@ By selecting one of the \code{-v} options, you can choose how much detail is pri
The output is colour coded with counted text coloured blue, other colours for other contexts. The colour coding is made using ANSI colour codes. These should work when printed directly to Linux xterm window, but need not work if piped through \code{more} or \code{less}: with \code{less} you need to use the option \code{-r} for the colours to be shown correcly.
-Under Windows or other operating systems, the ANSI colour codes cannot be expected to work, in which case the option \code{-nocol} (\code{-nc}) may be used to turn off colour coding; instead I recommend using HTML output which can be viewed in a browser.
+Under Windows or other operating systems, ANSI colour don't work and are turned off by default. Instead I recommend using HTML output which can be viewed in a browser.
-To print the details encoded as HTML code, use the option \code{-html}; alternatively, with \code{-htmlcore} only the HTML body is printed. On Windows, I suggest using the options \code{-html -v} to get full detail, save this to a HTML file, e.g. using
-\codeline{texcount -html -v texfile.tex > details.html}
-and then view the details.html file in a browser.
+To print the details encoded as HTML document, use the option \code{-html}. Alternatively, \code{-htmlcore} only outputs the HTML body. I suggest using the options \code{-html -v} to get full detail, save this to a HTML file, e.g. using
+\codeline{texcount.pl -html -v -sum \textit{files} > details.html}
+where \code{-sum} computes the total count of words and formulae (or \code{-sum=1,1,1} to only count words) and adds the cumulative count at the end of each line of the parsing details, and \code{-sub} is on by default which produces subcounts per section.
\subsection{Summary information}
-By default, \TeXcount{} outputs counts of text words, header words, caption words, number of headers, number of floats/figures, number of inlined formulae, and number of displayed formulae, and lists each of these counts. To shorten this to a one-line format, specify \code{-brief}.
+By default, \TeXcount{} outputs counts of text words, header words, caption words, number of headers, number of floats/figures, number of inlined formulae, and number of displayed formulae, and lists each of these counts. To shorten this to a one-line format per file, specify \code{-brief}.
To get \TeXcount{} to produce a total count, specify \code{-sum}: this will compute the sum of all words plus the number of formulae. A customized sum may be computed by speficying \code{-sum=n,n,\ldots} with up to seven numbers separated by commas giving the weight (0=don't count, 1=count once) of each of the seven counts: e.g. the default is equivalent to \code{-sum=1,1,1,0,0,1,1}. To count words only, use \code{-sum=1,1,1}. Higher weights may also be used, e.g. to count displayed formulae or floats/figures as a given number of words.
@@ -170,11 +195,15 @@ Specifying \code{-sum} has two main effects: the cumulative sum is added to the
For adding subcounts e.g. by sections, the option \code{-sub} (or \code{-subcount}) may be used. By default, this produces subcounts by part, chapter, section and subsection which are listed in a brief format. One may, however, specify \code{-sub=} followed by \code{part}, \code{chapter}, \code{section}, or \code{subsection} (default when given without value). Break points which initiate a new subcount may also be specified within the \LaTeX{} document using \code{\%TC:break name}.
-If included files are included in the count (\code{-inc}), counts per file will be produced followed by a total count. To suppress per file counts, the option \code{-total} may be used.
+If included files are included in the count (\code{-inc}), counts per file will be produced followed by a total count. Note that the counts for the included files are not included in the counts for the main document, and in particular is not included in the subcounts (e.g. per section). To suppress per file counts, the option \code{-total} may be used.
+
+By adding the option \code{-freq}, \TeXcount{} will output the word frequencies in order of descending frequency: this is only done for the total count, not per file. You may restrict the frequency table to words occurring at least $n$ times by specifying \code{-freq=\it{n}}. \TeXcount{} will count words irrespective of case, but the output will retain upper case where this is consistently used. Note that \TeXcount{} may not recognise that words are the same if they are written differently in the code, e.g. \code{{U}pper} and \code{Upper}.
+
+A frequency table for each script type (alphabetic, Han, etc. or script classes like Greek, Hebrew etc. if specified in \code{-alphabets=}) is produced by the option \code{-stat}.
\subsection{Parsing options}
-\TeXcount{} uses regular expressions to identify words and macro options. By default, these are moderately strict, but can be relaxed using the option \code{-relaxed}. This allows more general document elements to be identified as words or macro options, which may sometimes be desired, but may also have undesirable effects, so check the verbose output to verify that \TeXcount{} has counted the appropriate elements.
+\TeXcount{} uses regular expressions to identify words and macro options. By default, these have been set so as to fit most common usages. However, some users may find the default to be too strict, e.g. not recognise options that are long and contain less common symbols. More permissive patterns may be selected by using the option \code{-relaxed}. This allows more general document elements to be identified as words or macro options, which may sometimes be desired, but may also have undesirable effects, so check the verbose output to verify that \TeXcount{} has counted the appropriate elements. Conversely, if the default settings tends to combine words that should be counted as separate words, you may try the option \code{-restricted}.
Macro options, i.e. \code{[\ldots]} after macros and macro parameters are ignored. Since \TeXcount{} has no specific knowledge of which macros take options, this is a general rule. In order to avoid that uses of [\ldots] that are not macro options are mistaken as such, \TeXcount{} makes some restrictions on what may be contained in such an option. By default, this restriction is relatively strict under the assumption that it is better to count a few macro options as words than risk large fragments of text to be ignored. However, if your document contains macro options with more complicated values (e.g. certain special characters or macros), using \code{-relaxed} may help handle these correctly.
@@ -182,38 +211,31 @@ By default, \TeXcount{} does not allow special characters or macros to be part o
\subsection{File inclusion}
-If the option \code{-inc} is used, \TeXcount{} will automatically count documents that are included (using \code{\bs{input}} or \code{\bs{include}}. As when giving a list of files to count, it will print the sum per file and a total sum at the end. The default is \code{-noinc} indicating that included documents are not counted.
-
-By default, \TeXcount{} assumes paths are relative to the working directory. Alternatively, an explicit path may be given using \code{-dir=path}. Note that the path must end in the path delimiter symbol (depending on the operating system, \code{/} under Linux, \textbackslash under Windows). If only \code{-dir} is used, the path of the first file given will be used.
+By specifying \code{-inc} or \code{-merge}, \TeXcount{} will automatically count documents that are included using \code{\bs{input}} or \code{\bs{include}}. The difference between the two is that \code{-inc} analyses the included files separately, while \code{-merge} merges the included documents into the parent document. Thus, \code{-inc} will result in one count per file and a total sum at the end, while \code{-merge} will treat the merged document as if it was one file.
-Note that included documents are analysed as separate files: the text of included documents are thus not included where the \code{\bs{input}} or \code{\bs{include}} is located. This has two consequences. First, since word counts are produced per file, subcounts f.ex. by chapter will only include the text in the same file as the \code{\bs{chapter}} macro. Secondly, if TC-instructions to \TeXcount{} are embedded in the \LaTeX{} document, e.g. defining additional macro handling rules, these take effect in the order they are parsed by \TeXcount{}. Since included documents are parsed after the parent document, definitions in the parent document will be in effect for the included documents; definitions made in the included documents will only be in effect for subsequently included documents, not in the parent or previously included documents.
+The default option is \code{-noinc} indicating that included documents are not counted.
-\section{What \TeXcount{} counts}
+By default, \TeXcount{} assumes paths are relative to the present working directory. Alternatively, an explicit path may be given using \code{-dir=path}. Note that the path must end with the path delimiter symbol. If only \code{-dir} is used, the path of the main file (the one given to \TeXcount{} on the command line) will be used.
-The primary role is to count the words. It is not entirely clear what should be considered words, so I have had to make some decisions. A sequence of letters is certainly a word. I also count acronyms like `e.g.', dashed words like `over-all', and `it's' as one word. I have decided also to count numbers as words unless they are placed in a math environment.
+Note that when included documents are analysed as separate files, i.e. using \code{-inc}, the text of included documents is not included where the \code{\bs{input}} or \code{\bs{include}} is located. This has two consequences. First, since word counts are produced per file, subcounts, e.g. by chapter, will only include the text in the same file, not that of the included file. Secondly, if TC-instructions to \TeXcount{} are embedded in the \LaTeX{} document, e.g. defining additional macro handling rules, these take effect in the order they are parsed by \TeXcount{}. Since included documents are parsed after the parent document, definitions in the parent document will be in effect for the included documents; definitions made in the included documents will only be in effect for subsequently included documents, not in the parent or previously included documents.
-Mathematical formulae are not counted as words. Instead, \TeXcount{} counts the number of inline formulae (\code{\$maths\$}) and displayed formulae separately.
-Text in headers (\code{\bs{title}}, \code{\bs{section}}, etc.) are counted separately: it counts the number of headers as well as the number of words in headers.
-
-Floating environments (or potentially floating environments) such as tables and figures are not counted as text, even if the cells of a table may containt text. However, if they have captions, these will be counted separately much like headers were. By default, begin--end environments do not modify the parsing state: i.e. environments within the text are counted as text, etc. Rules for the most common environments, at least those that require non-default treatment, should be predefined, but you may have to add more rules if you use environments defined in packages or by yourself.
-
-Some macros are words by themselves: e.g. \code{\bs{LaTeX}}. These are counted as words, but you cannot expect \TeXcount{} to count something like \code{\bs{LaTeX}-word} or \code{\bs{\{}\bs{TeX}\bs{\}}count} as one word, although the above explanation inicates that it should: \TeXcount{} will in both cases evaluate the macro and the following text separately and thus count them as separate entities.
+\section{Macro handling rules}
+A few special macro handling rules are hard-coded into the \TeXcount{} script: i.e. the handling of those can only be changed by editing the script. However, \TeXcount{} primarily relies on a few general rules and macro and group handling rules that follow a specific pattern.
-\section{Macro handling rules}
+\subsection{General macro handling rules}
-A very few rules are hard-coded, e.g. that text between \code{\bs{documentclass}} and \code{\bs{begin}\{document\}} is the preamble of the \LaTeX{} document which should be default not be included in the count. However, most of the rules consists fall into a few general categories:
+The general macro handling rules fall into a few general categories:
\begin{description}
-\item[macro]In its simplest form, this type of rule just tells how many parameters to ignore following the macro. More generally, one may specify the number of parameters a macro takes and how each of these should be handled. Options enclosed in \code{[]} before, between and after parameters are also ignored; this also applies to macros not specified here, so for a macro with no rule, immediately following \code{[]}-options will be ignored. (This type of rule was called an exclude rule in older versions of
-\TeXcount{}, the reason being that the rule originally only gave the number of
+\item[macro]In its simplest form, this type of rule just tells how many parameters to ignore following the macro. More generally, one may specify the number of parameters a macro takes and how each of these should be handled. Options enclosed in \code{[]} before, between and after parameters are also ignored; this also applies to macros not specified here, so for a macro with no rule, immediately following \code{[]}-options will be ignored. (This type of rule was called an exclude rule in older versions of \TeXcount{}, the reason being that the rule originally only gave the number of
parameters to ignore following a given macro.)
\item[header]Some macros are specified to be counted as headers. This initially only indicates that the macro should cause the number of headers to be increased by one, but an additional rule is added to the macro-rule to count the following parameter as header text.
-\item[group]For groups enclosed by \code{\bs{begin}\{\textit{name}\}} and \code{\bs{end}\{\textit{name}\}}, there are rules specifying how the contents should be interpreted. A macro rule is added for \code{begin\textit{name}} (without the backslash!) which is \TeXcount{}'s internal representation of \code{\bs{begin}\{\textit{name}\}}. Some special characters will be ignored if following \code{\textit{name}}, e.g. \code{\bs{begin}\{equation*\}} will be treated as \code{\bs{begin}\{equation\}} since \code{*} is ignored.
+\item[group]For groups enclosed by \code{\bs{begin}\{\textit{name}\}} and \code{\bs{end}\{\textit{name}\}}, there are rules specifying how the contents should be interpreted. A macro rule is added for \code{begin\textit{name}} (without the backslash!) which is \TeXcount{}'s internal representation of \code{\bs{begin}\{\textit{name}\}}. Note that special characters like \code{*} may be part of the group name, e.g. as in \code{equation*} and rules for these need be specified\footnote{Previously, trailing \code{*} was supposed to be ignored so the same rule would apply to group \code{equation*} as to \code{equation}. However, due to a bug in a regular expression, this did not work as intended and I have decided not to follow that strategy and instead speficy these rules specifically.}.
\item[macroword]This type of rule indicates that the macro itself represents one or more words. Initially, \code{\bs{LaTeX}} and \code{\bs{TeX}} are defined with values 1 indicating that each represents one word.
@@ -221,39 +243,110 @@ parameters to ignore following a given macro.)
\item[float inclusion]Within floats (begin-end groups defined with parsing status $-1$) there may be texts that should still be counted: in particular captions. These are specified with the float inclusion rule.
-\item[file include]If \code{-inc} is specified, included files will also be parsed and the total presented at the end. Initially, \code{\bs{input}} and \code{\bs{include}} trigger file inclusion, but more file inclusion macros may be added.
-
\end{description}
A macro parameter is normally something on the form \code{\{something\}}; more generally it may be anything \TeXcount{} parses as a single unit (or token), e.g. a macro, but since \TeXcount{} parses word by word rather than character by character this may not always be correct if parameters are not \code{\{\}}-enclosed or macros.
+\subsection{Special macro handling rules}
+
+Some macros do not follow the pattern used by \TeXcount{} to represent macro handling rules. For some of these, special handling rules have been hard-coded into the \TeXcount{} script. For some, the macro syntax differs from the general rule, while in other cases the macros may trigger special processing.
+
+\begin{description}
+
+\item[file include]If \code{-inc} is specified, included files will also be parsed and the total presented at the end. Initially, \code{\bs{input}} and \code{\bs{include}} trigger file inclusion, but more file inclusion macros may be added to the \code{\%TeXfileinclude} hash. In addition to potentially triggering file inclusion, the syntax differs in that the file name need not be enclosed in \code{\{\ldots\}}.
+
+\item[package include]When packages are included using \code{\bs{usepackage}{name}}, \TeXcount{} will check for package specific macro handling rules to include. Initially, only \code{\bs{usepackage}} triggers package inclusion, but more macros may be added to the \code{\%TeXpackageinc} hash.
+
+\end{description}
+
+Complete \LaTeX{} documents should start with a \code{\bs{documentclass}} specification, then a preamble region which should not contain typeset text, before the main document starts with \code{\bs{begin}\{document\}}. However, \LaTeX{} files which are ment to be included into a document will not contain \code{\bs{documentclass}} and \code{\bs{begin}\{document\}}. A rule to recognise the preamble region is hard-coded into \TeXcount{}.
+
+Rules for identifying \code{\$\ldots\$}, \code{\$\$\ldots\$\$}, \code{\bs(\ldots\bs)}, and \code{\bs[\ldots\bs]} as formulae are hard-coded and basically parse until the closing token is encountered.
+
+The macros \code{\bs{def}} and \code{\bs{verb}} have hard-coded rules since these do not follow the pattern for macro handling rules, but may contain \LaTeX{} code which could seriously disrupt the parsing, e.g. by containing unclosed \code{\bs{begin}}. Macros like \code{\bs{newcommand}}, however, are handled by ordinary macro rules.
+
+The macro \code{\bs{biblography}} is handled to check if the bibliography file should be parsed. The \code{thebibliography} group is also handled differently, one difference being that a bibliography header is added to the count.
+
+\subsection{Package specific macro handling rules}
+
+Starting with version 2.3, \TeXcount{} can handle different sets of macro handling rules for different packages. When a package is included in the \LaTeX{} code or through the \code{-incpackage} option, rules defined for the given package is added.
+
+Note that \TeXcount{} is still doing the analyses sequentially. It is therefore critical that the package inclusion takes place before any use of the package which may make a difference if you are analysing several files. E.g. if the main file contains \code{\bs{input} setup}, any packages included in \code{setup.tex} will not apply to the main file since this is parsed before \TeXcount{} parses \code{setup.tex}.
+
+As of now, the package support is sparse since most macro handling rules have been included in the main set of rules.
+
+\subsection{Bibliography handling}
+
+By default, the bibliography is not included in the word count. If the \code{-incbib} option is specified, however, bibliography parsing is turned on. If the bibliography is included from the \code{bbl} file using the \code{\bs{bibliography}} macro, this will be parsed as if included with the \code{-inc} option. If \code{-merge} is specified together with \code{-incbib}, the bibliography will be merged into the document.
+
+Note that bibliography parsing may be non-trivial and depend on the bibliography style used, so the verbose output should be checked: some styles perform considerable formatting which may confuse \TeXcount{}. In addition, initials, page numbers, etc. will all be counted as words, which may result in a word count which is higher than intendet.
+
+\subsection{Adding or modifying macro handling rules}
+
+There are basically two different ways in which you can add additional macro handling rules, e.g. for your own macros, or modify existing rules: by modifying the \TeXcount{} script, or by adding the rules through \TeXcount{} instructions embedded in the \LaTeX{} code.
+
+The simplest method is to use \TeXcount{} instructions which are embedded in your \LaTeX{} document as \LaTeX{} comments on the format \code{\%TC:\textit{instruction}}. This approach is described in some detail in section \ref{sec:TC_addrule}.
+
+It is also possible to modify the \TeXcount{} code. The macro handling rules are mostly defined in the hash tables named \code{TeXmacro}, \code{TeXgroup}, etc., and editing these definitions is simple and does not required in-depth knowledge of Perl. A brief overview of the \TeXcount{} code is provided in section \ref{sec:code}.
+
\subsection{Cautions!}
Since the rules are of a relatively general nature, macros that have a great deal of flexibility are hard to deal with. In particular this applies to macros with a variable number of parameters or where the handling of the parameters are not constant.
Also, \code{[]}-options following macros and macro parameters are always ignored, and \TeXcount{} gives no flexibility in over-ruling that. Since options are, by definition of the term, meant to be optional, extending \TeXcount{} to handle them would require extensive reprogramming as well as require much more detailed macro definition rules than what is now possible.
-More critically, since \TeXcount{} does not really know which macros take options or not, just assumes that options should never be included, there is some risk of misinterpreting as an option something that is not: e.g. \code{\bs{bf}[text]}. This is not likely to be a frequent problem. However, if something like \code{\bs{bf}[a lot of text]} gets ignored because it is considered an option, it can influence the word count substantially. I have therefore been somewhat restrictive with what (and how much) can go into an option.
+More critically, since \TeXcount{} does not really know which macros take options or not, just assumes that options should never be included, there is some risk of misinterpreting as an option something that is not: e.g. \code{\bs{bf}[text]}. This is not likely to be a frequent problem. However, if something like \code{\bs{bf}[a lot of text]} gets ignored because it is considered an option, it can influence the word count substantially. I have therefore been somewhat restrictive with what (and how much) may go into an option. The default restriction on what may be allowed as an option may sometimes be too restrictive, causing \TeXcount{} to interpret options as text or macro parameters; you may use the command line option \code{-relaxed} to relax this restriction and allow more general options.
More advanced macros are not supported and can potentially confuse \TeXcount{}. In partcular, if you define macros that contain unbalanced begin--end groups, this will cause problems as \TeXcount{} needs to keep track of these to know where different groups start and end.
+\section{Output from \TeXcount{}}
+
+\TeXcount{} will by default provide a summary of the word and element counts. This may, however, be modified either by specifying \code{-brief} which reduces it to a one line summaryper file, \code{-total} to suppress per file summaries, or by providing an alternative template.
+
+If there are parsing errors, \TeXcount{} will print warnings about these. You may turn off this by specifying \code{-quiet} (\code{-q} for short), but there will still be an added comment about the number of errors in the final statistics to warn you of any errors.
+
+\subsection{Count statistics}
+
+The summary output will by default provide a summary of all counts: i.e. word counts for text, header and captions, and the number of floats/tables, headers, inlined and displayed formulae. You may combine these into a summary count by using the \code{-sum} option which by default gives the total number of words and formulae. You may choose briefer output formats by using the \code{-brief} option which produces a one-line summary of the counts. The option \code{-1} is the same as specifying \code{-brief -total} and will give only one line of output for the total only. Combining \code{-brief} with \code{-sum} will cause only the sum to be printed rather than the full set of counts.
+
+If multiple files are processed in one run, \TeXcount{} will by default provide summary statistics per file. If files are included (using the \code{-inc} option), summaries of all files are provided as well as the total. If there is more than one file, i.e. main \LaTeX{} documents provided in the command line, it will also write a total summary.
+
+In order to only write the total summary, use the option \code{-total}. If there is only one file processed, the result will be similar except that subcounts (counts per section etc.) are not provided with the total count.
+
+\subsection{Customising the output}
+
+You may specify an output template to use instead of the default output formats. This will replace the output per file or for the total with output produced using this template.
+
+The template is a string with codes for inserting the count values and titles. To specify it, use the option \code{-template="\textit{template}"}. The encapsulating \code{"\ldots"} are required if the template contains spaces. You may insert line shifts by using \code{\bs{n}}.
+
+The count numbers may be included by the codes \code{\{0\}} to \code{\{7\}} for the different counts: 0=number of files, 1=text words, etc. Codes \code{\{SUM\}}, \code{\{ERROR\}} and \code{\{TITLE\}} may be used to insert the count as specified by the \code{-sum} option, the number of parsing errors, the title (e.g. section name) and a header (same as title unless \TeXcount{} has replaced it). For the labels \code{SUM}, \code{\{ERROR\}} and \code{TITLE}, short forms \code{S}, \code{E} and \code{T} may be used.
+
+Conditional inclusion may be performed using the format \code{\{\textit{label}?\ldots?\textit{label}\}} where \code{\textit{label}} is one of \code{0} to \code{7}, \code{SUM}, \code{ERROR} or \code{TITLE} (or their alternative forms). The enclosed text will then be included only if the corresponding value exists and is non-zero. If you wish to include an alternative text when the value is non-existant or zero, use the format \code{\{\textit{label}?\textit{if non-zero}|\textit{if zero}?\textit{label}\}}.
+
+Subcounts, e.g. per section, may be included by using \code{\{SUB|\textit{template}|SUB\}} with a separate template text specified for the subcounts. This will only be included if there is more than one subcount, and in order to conditionally include prefix and suffix you may use \code{\{SUB?\textit{prefix}|\textit{template}|\textit{suffix}?SUB\}}.
+
+Note that you have to insert line shifts yourself. \TeXcount{} will only insert one line shift after each file count, and not after the total count: if you process only one file and want only to output the total sum without a line shift at the end, use \code{-sum -total -template="\{SUM\}"}, which should give the same output as \code{-1 -sum} when there are no parsing errors.
+
+
\section{\TeXcount{} instructions in the \LaTeX{} document}
It is possible to give some instructions to \TeXcount{} from within the
-\LaTeX{} document. The general format of these instructions are
-\codeline{\%TC:\textit{instruction [name] parameters}}
-where the name parameter is used with macro handling instructions to specify the macro or group name for which the rule applies.
+\LaTeX{} document. The general format of these instructions is
+\codeline{\%TC:\textit{instruction name parameters}}
+where \textit{name} is used with macro handling instructions to specify the macro or group name for which the rule applies and \textit{parameters} specify the details of how the macro and its parameters should be interpreted. In addition, there are some \TeXcount{} instructions that do not follow this syntax:
+%
+\input{sub_tc_other}
-Note that the comment should be on a separate line, and that instructions are case sensitive.
+The inital implementation of \TeXcount{} instructions required that the format was strictly adhered to: the comments should be a separate line and the instruction name was case sensitive. In particular the \code{ignore} instruction was problematic. However, in more recent version, at least from version 2.3, these \TeXcount{} instructions should are more robust.
\subsection{Ignoring segments of the file}
The TC-instruction \code{ignore}, later canceled by \code{endignore}, may be used to turn of all counting in a segment of the \LaTeX{} file. The ignored segment should thus be started by
\codeline{\%TC:ignore}
-end ended by
+and ended by
\codeline{\%TC:endignore}
-causing all text inbetween to be ignored. Note that \TeXcount{} still parses this text and may be affected by unbalanced brackets.
+causing all text inbetween to be ignored.\footnote{In older versions, \TeXcount{} would still parse this text and might thus be affected by unbalanced brackets. As of version 2.3, however, this should be fixed to make the ignore instruction more robust.}
\subsection{Adding subcount break points}
@@ -265,7 +358,7 @@ If you define new section macros or macros you wish to cause a break point, thes
\codeline{\%TC:breakmacro \textit{macro} \textit{label}}
This defines the given macro to cause a break point, and uses the given label to indicate the type of break (e.g. Section, Chapter, etc.).
-\subsection{Adding macro handling rules}
+\subsection{Adding macro handling rules}\label{sec:TC_addrule}
Adding your own macro handling rules is relatively simple. Although editing the script is possible, and not too difficult, this has the disadvantage that the modifications will be lost if updating to a new version of \TeXcount. A better and more flexible solution is to include instructions to \TeXcount{} in the \LaTeX{} documents, alternatively to make a definition file in which new macro handling rules are defined.
@@ -279,63 +372,170 @@ Note that macro handling rules are added successively throughout the session: i.
The instructions may be one of the following:
%
-\begin{description}\def\option[#1]{\item[\bigcode{#1}]}
+\input{sub_addrules}
+
+The parser status is used to dictate how each parameter should be parsed. E.g. if a macro has its parameter set defined by \code{[1,0,1]}, it means the first and third parameters are counted as text words (parser status $1$) whereas the second is ignored (parser status $0$). Another case is \code{\bs{renewcommand}} which is defined as \code{[-3,-2]}: the first parameter is to be read without interpreting the contents (which is going to be a macro name whose macro handling rules should not be applied here), and the second parameter should be ignored without requiring that begin--end groups be balanced. The different parsing states are:
+%
+\begin{description}
+\def\optionhead#1{\item[\textbf{#1}]}
+\def\option #1:{\item[{\makebox[7mm][r]{#1:}}]}
+\optionhead{States for ignoring text}
+\option 0: ignore text, i.e. do not count, but will still parse the code;
+\option -1: float, ignore text but look for \code{floatinclude} macros;
+\option -2: stronger ignore which ignore begin--end groups, e.g. to use in macro definitions where begin--end groups need not be balanced;
+\option -3: even stronger ignore, handles macros as isolated tokens without handling their parameters, to use with macro definitions like \code{\bs{newcommand}} and \code{\bs{def}};
+\option -4: ignore all, including unbalanced brackets (used by \code{\%TC:ignore} and the \code{verbatim} environment);
+\option -9: preamble, ignore text but look for \code{preambleinclude} macros.
+\optionhead{States for counting words}
+\option 1: count as text (i.e. count words);
+\option 2: count as header text;
+\option 3: count as float/caption text;
+\optionhead{Transitional states (internal/limited use)}
+\option 6: count as inlined math formulae;
+\option 7: count as displayed math formulae;
+\end{description}
+
+The two transitional states may be used to speficy rules for begin--end group contents, but not for macro parameters. They are transitional in the sense that the contents is parsed using the 0 state which ignores the contents.
+
+Here are some examples together with corresponding macro definitions:
+
+%TC:Ignore
+\begin{lstlisting}
+%TC:macroword \TeXcount 1
+\newcommand\TeXcount{{\TeX}count}
+
+%TC:macro \NB 1
+\newcommand\NB[1]{\marginpar{#1}}
+
+%TC:header \newsection [2,0]
+\newcommand\newsection[2]{\section{#1}\label{sec:#2}}
+
+%TC:group theorem 0 1
+\newtheorem{theorem}{Theorem}
+\end{lstlisting}
+%TC:EndIgnore
+
+The predefined rules can easily be read off the script file: they are hash maps defined at the beginning of the script with names \code{TeXmacro}, \code{TeXheader}, etc.
+
-\option[macro]Define macro handling rule for the given macro. Takes one parameter which is either an integer or a \code{[]}-enclosed array of integers (e.g. \code{[0,1,0]}). An integer value $n$ indicates that the $n$ first parameters to the macro should be ignored. An array of length $n$ indicates that the first $n$ parameters should be handled, and the numbers of the array specifies the parsing status (see below) with which they should be parsed. Giving the number $n$ as parameter is equivalent to giving an array of $n$ zeroes (\code{[0,\ldots,0]}) as zero is the parsing status for ignoring text.
+\section{Using an option file}
-\option[macroword]This defines the given macro to represent a certain number of words, where the number is given as the parameter.
+If you have a lot of settings, e.g. output template and TC commands for specifying parsing rules, you may place these into a file and include this using \code{-opt=\textit{file}}.
-\option[header]Define macro to give a header. This is specified much as the macro rule. The added effect is that the header counter is increase by 1. Note, however, that you should specify a parameter array, otherwise none of the parameters will be parsed as header text. The parser status for header text is 2, so a standard header macro that uses the first parameter as header should be given the parameter \code{[2]}.
+The format of this file is quite simple: each line is read as one option, so different options should not be placed on the same line. If some options are so long you need to break the line, e.g. for specifying an output template, you can do so by placing \code{\bs{}} at the start of lines that continue the previous line.
-\option[breakmacro]Specify that the given macro should cause a break point. Defining it as a header macro does not do this, nor is it required of a break point macro that it be a header (although I suppose in most cases of interest it will be).
+You may enter TC commands just as in the \LaTeX{} code by starting the line with \code{\%} instead of \code{TC:}. Using these, you may include specifications of parsing rules.
-\option[group]This specifies a begin-end group with the given name (no backslash). It takes two further parameters. The first parameter speficies the macro rule following \code{\bs{begin}\{\textit{name}\}}. The second parameter specifies the parser status with which the contents should be parsed: e.g. $1$ for text (default rule), $0$ to ignore, $-1$ to specify a float (table, group, etc.) for which text should not be counted but captions should, $6$ and $7$ for inline or displated math.
+Blank lines and lines starting with \code{\#} are ignored and may thus be used to add comments to the option file. So are leading spaces, which allows lines to be indented. Line breaks may be inserted by \code{\bs{n}}.
-\option[floatinclude]This may be used to specify macros which should be counted when within float groups. The handling rules are spefified as for \code{macro}, but like with \code{header} an array parameter should be provided and parameters that should be counted as text in floats should be specified by parsing status 3. Thus, a macro that takes one parameter which should be counted as float/caption text should take the parameter \code{[3]}.
+Here is an example which sets the total sum to be the number of words (not including formulae), subcounts by section, parses included files, and adds an output template.
-\option[preambleinclude]The preamble, i.e. text between \code{\bs{documentclass}} and \code{\bs{begin}\{document\}}, if the document contains one, should generally not be included in the word count. However, there may be definitions, e.g. \code{\bs{title}\{title text\}}, that should still be counted. In order to be able to include these special cases, there is a preambleinclude rule in which one may speficy handling rules for macros within the preamble. Again, the rule is speficied like the \code{macro} rules, but since the default is to ignore text the only relevant rules to be specified require an array.
+%TC:Ignore
+\begin{lstlisting}[frame=single]
+### Options to use with TeXcount
-\option[fileinclude]By default, \TeXcount{} does not automatically add files included in the document using \code{\bs{input}} or \code{\bs{include}}, but inclusion may be turned on by using the option \code{-inc}. If other macros are used to include files, these may be specifed by adding fileinclude rules for these macros. The specification takes one parameter: 0 if the file name should be used as provided, 1 if file type \code{.tex} should be added to files without a file type, and 2 if the file tyle \code{.tex} should always be added.
+# Counting options
+-sum=1,1,1
+-sub=section
+-inc
+
+# Macro rules
+%macro \url 1
+%group sourcecode 0 0
+%macroword \TeXcount 1
+
+# Path used in file inclusion (\chapterpath filename)
+%subst \chapterpath chap/
+
+# Output template
+-template=
+ \::: {title} :::\n
+ \Words: {sum}\n
+ \Formulae: {6} + {7}\n
+ \{5?Number of floats: {5}\n?5}
+ \{SUB? - {sum} words in {title}\n?SUB}
+\end{lstlisting}
+%TC:EndIgnore
+
+
+
+\section{Customising \TeXcount{}}
+
+\TeXcount{} is a self-contained Perl script: no external packages or resources required except that you need to have Perl installed to run it. Unfortunately, as with much of Perl code since Perl does not itself encourage structured programing, after expanding somewhat in size, it is not the most readable of codes. However, there may still be cases where you might yourself want to modify the code.
+
+There are some things that may be modified quite easily even without knowing Perl.
+
+\begin{description}
+
+\item[Preset startup options]On one of the first lines of the code, the list \code{@StartupOptions} is defined. A list is simply a sequence of values (an array) on the form \code{(\textit{value},\textit{value},\ldots)}. As it stands, this list is empty, but you may add startup options to be included prior to command line options when you run \TeXcount{}. E.g. if you change this to \code{("-inc")} it will automatically add the \code{-inc} option so you don't have to do that yourself every time you run \TeXcount{}.
+
+\item[Adding macro handling rules]While you may add macro handling rules using \code{\%TC:} commands either in the document or in a separate option file, this is inconvenient for large numbers of macros or if you want these rules always to be included. Also, you might want to add such rules for specific packages. In either case, it might be practical to add these directly to the \TeXcount{} code. \TeXcount{} stores the rules in hashes (maps from a key to a value) named \code{\%TeXmacro}, \code{\%TeXheader}, etc. There is more documentation on each of these in the code itself, and you may also inspect how rules have been defined for other macros and groups.
+
+\item[Output style]The ANSI colour codes for different levels of verbosity are encoded in the \code{\%STYLES} hashes and may be changed. The HTML style is encoded in the method \code{html\_head()} and is easily modified.
+
+\item[Character and word definitions]\TeXcount{} identifies words as those that match one of a given set of regular expressions (defined in \code{@WordPatterns}). Note that \code{@WordPatterns} is changed by options \code{-chinese}, \code{-japanese} and \code{-letters}. The pattern that is used within the word patterns to recognise letters is stored in \code{\$LetterPattern}. This is replaced if the \code{-relaxed} or \code{-restricted} option is set. Changing these definitions may be useful if you have special characters or wish to define words differently.
\end{description}
-The parser status is used to dictat how each parameter should be parsed. E.g. if a macro has its parameter set defined by \code{[1,0,1]}, it means the first and third parameters are counted as text words (parser status $1$) whereas the second is ignored (parser status $0$). Another case is \code{\bs{renewcommand}} which is defined as \code{[-3,-2]}: the first parameter is to be read without interpreting the contents (which is going to be a macro name whose macro handling rules should not be applied here), and the second parameter should be ignored without requiring that begin--end groups be balanced. The different parsing states are:
-%
-\begin{description}\def\option[#1]{\item[\quad\textbf{#1}]}
-\option[$0$:]ignore text, i.e. do not count, but will still parse the code;
-\option[$1$:]count as text (i.e. count words);
-\option[$2$:]count as header text;
-\option[$3$:]count as float/caption text;
-\option[$-1$:]float, ignore text but look for \code{floatinclude} macros;
-\option[$-2$:]stronger ignore which ignore begin--end groups, e.g. to use in macro definitions where begin--end groups need not be balanced;
-\option[$-3$:]even stronger ignore, handles macros as isolated tokens without handling their parameters, to use with macro definitions like \code{\bs{newcommand}} and \code{\bs{def}};
-\option[$-9$:]preamble, ignore text but look for \code{preambleinclude} macros.
+
+\section{Modifying the \TeXcount{} script}\label{sec:code}
+
+\TeXcount{} is written in Perl, and although hardly the best structured and documented code ever seen, I have tried to structure and document it somewhat. In particular, some parts of the code have been written with modifications in mind.
+
+Here's a quick walk-through of the code structure and comments on how easily the code may be modified. Some parts of the code are marked as \emph{CMD specific}. There are two version of the script: the CMD version intended for command line use, and the CGI version used with the web interface. The one you have is the CMD version.
+
+\begin{description}
+
+\item[\em INITIAL SETUP:] \textit{These set up global variables prior to execution.}
+
+\item[Settings and setup variables:] The start of the script sets of initial settings and variables. Many of these may be modified by command line options, but if you want to change the default behaviour these may be changed. However, note that there is a list \code{@StartupOptions} intended for this: initially, it is empty, but this is probably the simplest place the change startup options.
+
+\item[Internal states:] As of version 2.3, internal state identifiers (which are numerical codes) have been defined as \code{STATUS}, \code{TOKEN} and \code{CNT} variables, and these are also defined here. A few subroutines for interpreting these states have been included here, although most subroutines are defined after the main code, since they are intimately tied to the state's numerical values. None of these are intended to be modified.
+
+\item[Styles:] The style definitions basically define which elements to print for each of the verbosity levels. These map element names to ANSI colour codes. When used with HTML, the element names are used as tag classes. If you wish to change the ANSI colour scheme, or change which elements are written in each verbosity option, these may be changed.
+
+\item[Word pattern definitions:] This section contains regular expression patterns for identifying words and macro options. In addition, the additional character classes defined by \TeXcount{} are defined here. If you have special needs or wishes, modifying these definitions may be an option.
+
+\item[\TeXcount{} parsing rules:] This is the section in which the main rules for interpreting the \LaTeX{} code is specified: the exception is a few hard-coded rules that do not follow these general patterns. These are hashes that map the macro or group name to the macro handling rules. First, the default rules are defined, then packages specific rules are defined.
+
+\item[\em MAIN:] \textit{This is the top-level code which gets executed. All else is done through calls to subroutines.}
+
+\item[Main \TeXcount{} code:] This is the main code that is run. It is very simple: just a call to the method \code{MAIN} passing the command line options.
+
+\item[\em SUBROUTINES:] \textit{The subroutines are organised into blocks. Subroutines names use capital letters or initials if they are main routines (like public in other languages) to be used at the top-level, lower case if they may be used throughout but are considered to be lower-level subroutines, prefixed by one or two underscores (\_) if used only within the block.}
+
+\item[Main routines:] The \code{MAIN} routine gives the general processing flow. This in turn calls routines to parse to command line options, process/apply the options, parse the \TeX/\LaTeX{} files, and finally summarise the final results. The main routines are CMD specific.
+
+\item[CMD specific subroutines:] These are subroutine versions that are CMD specific, e.g. file inclusion and ANSI colours. Their location is somewhat illogical: logically, they might belong later together with related subroutines, but have been placed this early because they are specific to the CMD (or CGI) version.
+
+\item[Option handling:] After parsing the options, the option values are processed using these subroutines. Some of the option handling operations call on global variables, whereas some are more hard-coded. Like the global variables, if you have special wishes or needs, there may be parts here that can be modified quite easily to change default settings or effects of specific options.
+
+\item[\TeX{} object:] The main role of the \code{TeX} object (which is technically not an object in the ordinary sense but just a hash) is to be a container object which links to the \TeX/\LaTeX{} code, the word count object, etc. The \code{TeX} object pertaining to any parsed \TeX/\LaTeX{} file is passed along from subroutine to subroutine, usually called \code{\$tex}. The \code{Main} object produced by \code{getMain} is a simple substitute for the \code{TeX} object for use when none is available, e.g. to catch errors not specific to any particular \code{TeX} object.
+
+\item[File reading routines:] These are used to read files and STDIN.
+
+\item[Parsing routines:] These contain the main routines for parsing the \TeX/\LaTeX{} code. The main worker method is the \code{\_{}parse\_{}unit} which parses a block of code: the \emph{unit}. A unit of code may be a begin-end group (environment), a \{\} separated group, a macro option or parameter, etc. The parsing of one unit is determined by the parsing status, which is passed to the parsing method, and the end marker which indicates which token marks the end of the unit. Different subroutines are then used to process the different types of code: macros, begin-end groups, TC instructions, etc. Amongst these routines are also routines for converting the parsed code into tokens, which is done one token at the time which is then removed from the start of the code.
+
+\item[Count object and routines:] The count object contains the counters as an array, plus titles and labels; in addition it can contain a list of subcounts which are themselves count objects. The count object is used for each file, but also to summarise multiple files, and region counts within files (e.g. per section). The \code{TeX} object contains an active count object to which newly counted words, equations, etc. get added. However, each \code{TeX} object also has a summary count object which will contain the final sum.
+
+\item[Output routines:] First, there are some routines for general output, i.e. independent of specific \code{TeX} objects. There are then some routines for formatting output, e.g. for the verbose output. There are also routines for printing count summaries in various formats. A special set of routines exist for printing the verbose output itself, and some of these are also involved in the parsing.
+
+\item[Help functions:] These routines are used to print help.
+
+\item[HTML functions:] These are routines for producing HTML output. In particular, the HTML style is defined here and may be easily modified.
+
+\item[Text data:] Some texts are not hard-coded into the script, but added as text data at the end. There are some routines defined to handle the text data, and then the text data itself.
+
\end{description}
-Here are some examples together with corresponding macro definitions:
-%
-\codeline{%
-\%TC:macroword \bs{TeXcount} 1\\
-\bs{newcommand}\bs{TeXcount}\{\{\bs{TeX}\}count\}
-\\[8pt]
-\%TC:macro \bs{NB} 1\\
-\bs{newcommand}\bs{NB}[1]\{\bs{marginpar}\{\#1\}\}
-\\[8pt]
-\%TC:header \bs{newsection} [2,0]\\
-\bs{newcommand}\bs{newsection}[2]\{\bs{section}\{\#1\}\bs{label}\{sec:\#2\}\}
-\\[8pt]
-\%TC:group theorem 0 1\\
-\bs{newtheorem}\{theorem\}\{Theorem\}
-}
-%
-The predefined rules can easily be read off the script file: they are hash maps defined at the beginning of the script with names \code{TeXmacro}, \code{TeXheader}, etc.
+Perl will first process the setup section which defines global variables, arrays and hashes. It then executes the main section (consisting of the call to \code{MAIN}), whereafter it exits. The subroutines and text data follow after the \code{exit}.
\section{License}
The \TeXcount{} package---script and accompanying documents---is distributed
under the \LaTeX{} Project Public License (LPPL)
-\codeline{http://www.latex-project.org/lppl.txt}
+\codeline{\url{http://www.latex-project.org/lppl.txt}}
which grants you, the user, the right to use, modify and distribute
the script. However, if the script is modified, you must change its
name or use other technical means to avoid confusion with the original script.
diff --git a/Master/texmf-dist/doc/support/texcount/macros.tex b/Master/texmf-dist/doc/support/texcount/macros.tex
index 09e304c736f..b7f853f39ad 100644
--- a/Master/texmf-dist/doc/support/texcount/macros.tex
+++ b/Master/texmf-dist/doc/support/texcount/macros.tex
@@ -1,8 +1,17 @@
%% LaTeX macros
+% Set version number
+\newcommand\version{$2.3$}
+
+\newcommand\copyrightfootnote{
+\footnotetext{Copyright (2008-2011) of Einar Andreas R{\o}dland, distributed
+under the \LaTeX{} Project Public License (LPPL).}
+}
+
%TC:macroword \TeXcount 1
\newcommand\TeXcount{{\TeX}count}
+% Text formats
\newcommand\code[1]{\texttt{\small#1}}
\newcommand\bigcode[1]{\texttt{#1}}
\newcommand\codeline[1]{\begin{quote}\code{#1}\end{quote}}
diff --git a/Master/texmf-dist/doc/support/texcount/sub_addrules.tex b/Master/texmf-dist/doc/support/texcount/sub_addrules.tex
new file mode 100644
index 00000000000..6c702ea95c3
--- /dev/null
+++ b/Master/texmf-dist/doc/support/texcount/sub_addrules.tex
@@ -0,0 +1,25 @@
+% Subsection: TC-commands for adding macro rules
+
+\begin{description}
+\sloppy
+\def\option[#1]{\item[\bigcode{#1}]\hskip 0pt plus 10pt}
+
+\option[macro]Defines macro handling rule for the specified macro. It takes one parameter which is either an integer or a \code{[]}-enclosed array of integers (e.g. \code{[0,1,0]}). An integer value $n$ indicates that the $n$ first parameters to the macro should be ignored. An array of length $n$ indicates that the first $n$ parameters should be handled by the rule, and the numbers in the array specifies the parsing status (see below) with which they should be parsed. Giving the number $n$ as parameter is equivalent to giving an array of $n$ zeroes (\code{[0,\ldots,0]}) as zero is the parsing status for ignoring text. For all macros, also those for which no rules have been defined, options enclosed in \code{[]} between or after macros and their parameters are ignored.
+
+\option[macroword]This defines the given macro to be counted as a certain number of words, where the number is given as the parameter.
+
+\option[header]Define macro to be a header. This is specified as the macro rule, but has the added effect is that the header counter is increase by 1. Note, however, that you should specify a parameter array, otherwise none of the parameters will be parsed as header text. The parser status for header text is 2, so a standard header macro that uses the first parameter as header should be given the parameter \code{[2]}.
+
+\option[breakmacro]Specify that the given macro should cause a break point. Defining it as a header macro does not do this, nor is it required of a break point macro that it be a header (although I suppose in most cases of interest it will be).
+
+\option[group]This specifies a begin-end group with the given name (no backslash). It takes two further parameters. The first parameter speficies the macro rule following \code{\bs{begin}\{\textit{name}\}}. The second parameter specifies the parser status with which the contents should be parsed: e.g. $1$ for text (default rule), $0$ to ignore, $-1$ to specify a float (table, group, etc.) for which text should not be counted but captions should, $6$ and $7$ for inline or displated math.
+
+\option[floatinclude]This may be used to specify macros which should be counted when within float groups. The handling rules are spefified as for \code{macro}, but like with \code{header} an array parameter should be provided and parameters that should be counted as text in floats should be specified by parsing status 3. Thus, a macro that takes one parameter which should be counted as float/caption text should take the parameter \code{[3]}.
+
+\option[preambleinclude]The preamble, i.e. text between \code{\bs{documentclass}} and \code{\bs{begin}\{document\}}, if the document contains one, should generally not be included in the word count. However, there may be definitions, e.g. \code{\bs{title}\{title text\}}, that should still be counted. In order to be able to include these special cases, there is a preambleinclude rule in which one may speficy handling rules for macros within the preamble. Again, the rule is speficied like the \code{macro} rules, but since the default is to ignore text the only relevant rules to be specified require an array.
+
+\option[fileinclude]By default, \TeXcount{} does not automatically add files included in the document using \code{\bs{input}} or \code{\bs{include}}, but inclusion may be turned on by using the option \code{-inc}. If other macros are used to include files, these may be specifed by adding fileinclude rules for these macros. The specification takes one parameter: 0 if the file name should be used as provided, 1 if file type \code{.tex} should be added to files without a file type, and 2 if the file tyle \code{.tex} should always be added.
+
+\option[subst]This substitutes a macro (the first parameter) with any text (the remaining option). Substitution is performed only on the present file and on the text following the instruction. Note that substitution is performed directly on the \LaTeX{} code prior to parsing, and the verbose output will show the substituted text. E.g. \code{\%TC:subst \bs{test} TEST} will cause a following \code{\bs{newcommand}\bs{test}{TEST}} to be changed into \code{\bs{newcommand} TEST{TEST}}, which \TeXcount{} will interpret differently. Use with care!
+
+\end{description}
diff --git a/Master/texmf-dist/doc/support/texcount/sub_options.tex b/Master/texmf-dist/doc/support/texcount/sub_options.tex
new file mode 100644
index 00000000000..75fdd7509ad
--- /dev/null
+++ b/Master/texmf-dist/doc/support/texcount/sub_options.tex
@@ -0,0 +1,114 @@
+% Subsection: command line options
+
+\begin{description}
+\sloppy
+\def\option[#1]{\item[\quad\code{#1}]\hskip 0pt plus 10pt}
+\def\alt#1{[#1]}
+
+\option[-v]Verbose (same as \code{-v3}).
+
+\option[-v0]No details (default).
+
+\option[-v1]Prints counted text, marks formulae.
+
+\option[-v2]Also prints ignored text.
+
+\option[-v3]Also includes comments and options.
+
+\option[-v4]Same as \code{-v3 -showstate}.
+
+\option[-showstate]Show internal states (with verbose).
+
+\option[-brief]Only prints a one line summary of the counts for each file.
+
+\option[-q, -quiet]Quiet mode, does not print error messages. Use is discouraged, but it may be useful when piping the output into another application.
+
+\option[-strict]Prints a warning of begin-end groups for which no specific rule is defined.
+
+\option[-total]Only give total sum, no per file sums.
+
+\option[-1]Same as specifying \code{-brief} and \code{-total}, and ensures there will only be one line of output. If used with \code{-sum}, the output will only be the total number.
+
+\option[-0]Same as \code{-1}, i.e. \code{-brief} and \code{-total}, but does not put a line shift at the end. This may be useful when the one line output is to be used by another application, e.g. Emacs, for which the line shift would otherwise need to be stripped away.
+
+\option[-template="\ldots"]Speficy an output template which is used to generate the summary output for each file and for the total count. Codes \code{\{\textit{label}\}} is used to include values, where \code{\textit{label}} is one of \code{0} to \code{7} (for the counts), \code{SUM}, \code{ERROR} or \code{TITLE} (first character of label is sufficient). Conditional inclusion is done using \code{\{\textit{label}?\textit{text}?\textit{label}\}} or \code{\{\textit{label}?\textit{if non-zero}|\textit{if zero}?\textit{label}\}}. If the count contains at least two subcounts, use \code{\{SUB|\textit{template}|SUB\}} with a separate template for the subcounts, or \code{\{SUB?\textit{prefix}|\textit{template}|\textit{suffix}?SUB\}}.
+
+\option[-sub\alt{=\ldots}, -subcount\alt{=\ldots}]Generate subcounts. Valid option values are \code{none}, \code{part}, \code{chapter}, \code{section} and \code{subsection} (default), indicating at which level subcounts are generated. (On by default.)
+
+\option[-nosub]Do not generate subcounts.
+
+\option[-sum\alt{=n,n,\ldots}]Produces total sum, default being all words and formulae, but customizable to any weighted sum of the seven counts (list of weights for text words, header words, caption words, headers, floats, inlined formulae, displayed formulae).
+
+\option[-nosum]Do not generate total sum. (Default choice.)
+
+\option[-col]Use ANSI colour codes in verbose output. This requires ANSI colours which is used on Linux, but may not be available under Windows. On by default on non-Windows systems.
+
+\option[-nc, -nocol]No colours (colours require ANSI). Default under Windows.
+
+\option[-relaxed]Relaxes the rules for matching words and macro options.
+
+\option[-restricted]Restricts the rules for matching words and macro options.
+
+\option[-]Read \LaTeX{} code from STDIN.
+
+\option[-inc]Parse included files (as separate files).
+
+\option[-merge]Merge included files into document (in place).
+
+\option[-noinc]Do not parse or merge in included files (default).
+
+\option[-incbib]Include bibliography in count, include bbl file if needed.
+
+\option[-nobib]Do not include bibliography in count (default).
+
+\option[-incpackage=]Include rules for a given package.
+
+\option[-dir\alt{=\ldots}]Specify working directory which will serve as root for all include files. Default (\code{-dir=""}) is to use the present directory. Use \code{-dir} to use path of the main \LaTeX{} document.
+
+\option[-enc=, -encoding=]Specify encoding to use in input (and text output).
+
+\option[-utf8, -unicode]Use UTF-8 (Unicode) encoding. Same as \code{-encoding=utf8}.
+
+\option[-alpha=, -alphabets=]List of Unicode character groups (or digit, alphabetic) permitted as letters. Names are separated by \code{,} or \code{+}. If list starts with \code{+}, the alphabets will be added to those already included. The default is Digit+alphabetic.
+
+\option[-logo=, -logograms=]List of Unicode character groups interpreted as whole word characters, e.g. Han for Chinese characters. Names are separated by \code{,} or \code{+}. If list starts with \code{+}, the alphabets will be added to those already included. By default, this is set to include Ideographic, Katakana, Hiragana, Thai and Lao.
+
+\option[-ch, -chinese, -zhongwen]Turn on Chinese mode in which Chinese characters are counted. I recommend using UTF-8, although \TeXcount{} will also test other encodings (GB2312, Big5, Hz) if UTF-8 fails, and other encodings may be specified by \code{-encoding=}.
+
+\option[-jp, -japanese]Turn on Japanese mode in which Japanese characters (kanji and kana) are counted. I recommend using UTF-8, although \TeXcount{} will also test other encodings (e.g. EUC-JP) if UTF-8 fails, and other encodings may be specified by \code{-encoding=}.
+
+\option[-kr, -korean]Turn on Korean mode in which Korean characters (hangul and han) are counted. I recommend using UTF-8, although \TeXcount{} will also test other encodings (e.g. EUC-KR) if UTF-8 fails, and other encodings may be specified by \code{-encoding=}. \emph{NB: Support for Korean is experimental.}
+
+\option[-kr-words, -korean-words]Korean mode in which hangul words are counted (i.e. as words separated by spaces) rather than characters. Han characters are still counted as characters. See also \code{-korean}. \emph{NB: Support for Korean is experimental.}
+
+\option[-chinese-only, ..., -korean-words-only]As options \code{-chinese}, ..., \code{-korean-words}, but also excludes other alphabets (e.g. letter-based words) and logographic characters.
+
+\option[-char, -letter]Count letters instead of words. This count does not include spaces.
+
+\option[-html]Output in HTML format.
+
+\option[-htmlcore]Only HTML body contents.
+
+\option[-freq\alt{=\#}]Count individual word frequencies. Optionally, give minimal frequency required to be included in output.
+
+\option[-stat]Produce statistics on language usage, i.e. based on the alphabets and logograms included.
+
+\option[-codes]Display an overview of the colour codes. Can be used as a separate option to only display the colour codes, or together with files to parse.
+
+\option[-nocodes]Do not display overview of colour codes.
+
+\option[-opt=, -optionfile=]Reads options (command line parameters) from a specified text file. Should use one option per line. May also include TC options in the same format as specified in \LaTeX{} documents, but prefixed by \code{\%} rather than \code{\%TC:}. Blank lines and lines starting with \code{\#} are ignored; lines starting with \code{\bs{}} are considered to be continuations of the previous line.
+
+\option[-split, -nosplit]The \code{-split} option, which is on by default, speeds up handling of large files by splitting the file into paragraphs. To turn it off, use the \code{-nosplit} option.
+
+\option[-showver, -nover]Include version number in output with \code{-showver}; use \code{-nover} not to show it (default).
+
+\option[-h, -?, --help, /?]Help.
+
+\option[-h=, -?=, --help=, /?=]Help on particular macro or group name: gives the parsing rule for that macro or group if defined. You may have to use \code{-incpackage=}package if the rule is defined for a specific package, and this option must be placed \emph{before} the \code{-h=} option on the command line.
+
+\option[-ver, --version]Print version number.
+
+\option[-lic, --license]License information.
+
+\end{description}
diff --git a/Master/texmf-dist/doc/support/texcount/sub_tc_other.tex b/Master/texmf-dist/doc/support/texcount/sub_tc_other.tex
new file mode 100644
index 00000000000..4c949ebdead
--- /dev/null
+++ b/Master/texmf-dist/doc/support/texcount/sub_tc_other.tex
@@ -0,0 +1,14 @@
+%% Other TC instructions
+
+\begin{description}
+\def\option[#1]{\item[\bigcode{#1}]}
+
+\option[ignore]Indicates start of a region to be ignored. End region with the TC-instruction \code{endignore}.
+
+\option[break \textit{title}]Break point which initiates a new subcount. The title is used to identify the following region in the summary output.
+
+\option[incbib]Sets bibliography inclusion, same as running \TeXcount{} with the option \code{-incbib}.
+
+\end{description}
+
+
diff --git a/Master/texmf-dist/scripts/texcount/texcount.pl b/Master/texmf-dist/scripts/texcount/texcount.pl
index 96fac5cd220..69fb4b5002f 100755
--- a/Master/texmf-dist/scripts/texcount/texcount.pl
+++ b/Master/texmf-dist/scripts/texcount/texcount.pl
@@ -2,38 +2,370 @@
use strict;
use warnings;
use Term::ANSIColor;
+use Text::Wrap;
use Encode;
-use POSIX qw(locale_h);
-use locale;
-setlocale(LC_CTYPE,"no_NO");
+use utf8; # Because the script itself is UTF-8 encoded
-my $versionnumber="2.2";
-my $versiondate="2009 Apr 30";
+##### Version information
+
+my $versionnumber="2.3";
+my $versiondate="2011 Jul 30";
+
+###### Set global settings and variables
+
+### Global data about TeXcount
+my %GLOBALDATA=
+ ('versionnumber' => $versionnumber
+ ,'versiondate' => $versiondate
+ ,'maintainer' => "Einar Andreas Rodland"
+ ,'copyrightyears' => "2008-2011"
+ ,'website' => "http://app.uio.no/ifi/texcount/"
+ );
+
+### Options and states
+
+# Outer object (for error reports not added by a TeX object)
+my $Main=getMain();
+
+# Global options and settings
+my $htmlstyle=0; # Flag to print HTML
+my $encoding=undef; # Selected input encoding (default will be guess)
+my @encodingGuessOrder=qw/ascii utf8 latin1/; # Encoding guessing order
+my $outputEncoding; # Encoding used for output
+my @AlphabetScripts=qw/Digit Is_alphabetic/; # Letters minus logograms: defined later
+my @LogogramScripts=qw/Ideographic Katakana Hiragana Thai Lao Hangul/; # Scripts counted as whole words
+my $WordPattern; # Regex matching a word (defined in apply_language_options())
+
+# Parsing rules options
+my $includeTeX=0; # Flag to parse included files
+my $includeBibliography=0; # Flag to include bibliography
+my %substitutions; # Substitutions to make globally
+
+# Counting options
+my @sumweights; # Set count weights for computing sum
+my $optionWordFreq=0; # Count words of this frequency, or don't count if 0
+my $optionWordClassFreq=0; # Count words per word class (language) if set
+
+# Parsing details options
+my $strictness=0; # Flag to check for undefined groups
+my $verbose=0; # Level of verbosity
+my $showcodes=1; # Flag to show overview of colour codes (2=force)
+my $showstates=0; # Flag to show internal state in verbose log
+my $showsubcounts=0; # Write subcounts if #>this, or not (if 0)
+my $separatorstyleregex='^word'; # Styles (regex) after which separator should be added
+my $separator=''; # Separator to add after words/tokens
+
+# Final summary output options
+my $showVersion=0; # Indicator that version info be included (1) or not (-1)
+my $totalflag=0; # Flag to write only total summary
+my $briefsum=0; # Flag to set brief summary
+my $outputtemplate; # Output template
+my $finalLineBreak=1; # Add line break at end
+
+# Global settings
+my $optionFast=1; # Flag inticating fast method
+
+# Global variables and internal states (for internal use only)
+my $blankline=0; # Number of blank lines printed
+my $errorcount=0; # Number of errors in parsing
+my %warnings=(); # Warnings
+my %WordFreq; # Hash for counting words
+
+# String data storage
+my $STRINGDATA;
###### Set CMD specific settings and variables
-# Options and states
-my $verbose=0;
-my $showcodes=1;
-my $showstates=0;
-my $showsubcounts=0;
-my $htmlstyle=0;
-my $includeTeX=0;
-my $briefsum=0;
-my @sumweights;
-my $utf8flag=0;
-
-# Global variables
-my $blankline=0;
-my $errorcount=0;
+## Preset command line options
+# List of options (stings) separated by comma,
+# e.g. ("-inc","-v") to parse included files and
+# give verbose output by default.
+my @StartupOptions=();
# CMD specific global variables
-my $totalflag=0;
-my @filelist;
-my $workdir;
-my $globalworkdir="";
+my @filelist; # List of files to parse
+my $workdir; # Root directory (taken from filename)
+my $globalworkdir=""; # Overrules workdir (default=present root)
+my $fileFromSTDIN=0; # Flag to set input from STDIN
+my $_STDIN_="<STDIN>"; # File name to represent STDIN
+
+# CMD specific settings
+$Text::Wrap::columns=76; # Page width for wrapped output
+
+###### Set status identifiers and methods
+
+
+### Counter indices from 0 to $SIZE_CNT-1
+# 0: Number of files
+# 1: Text words
+# 2: Header words
+# 3: Caption words
+# 4: Number of headers
+# 5: Number of floating environments
+# 6: Number of inlined math
+# 7: Number of displayed math
+my $SIZE_CNT=8;
+my $CNT_FILE=0;
+my $CNT_WORDS_TEXT=1;
+my $CNT_WORDS_HEADER=2;
+my $CNT_WORDS_CAPTION=3;
+my $CNT_COUNT_HEADER=4;
+my $CNT_COUNT_FLOAT=5;
+my $CNT_COUNT_INLINEMATH=6;
+my $CNT_COUNT_DISPLAYMATH=7;
-###### Set global settings and variables
+# Labels used to describe the counts
+my @countlabel=('Files','Words in text','Words in headers',
+ 'Words in float captions','Number of headers','Number of floats',
+ 'Number of math inlines','Number of math displayed');
+
+### Parsing statuses
+## Note that status numbers are used in rule definitions and should
+## not be changed.
+#
+## Status for regions that should not be counted
+# 0 = exclude from count
+## Statuses for regions in which words should not be counted
+# -1 = float (exclude, but include captions)
+# -2 = strong exclude, ignore begin-end groups
+# -3 = stronger exclude, do not parse macro parameters
+# -4 = ignore everything except end marker: even {
+# -9 = preamble (between \documentclass and \begin{document})
+## Statuses for regions in which words should be counted
+# 1 = text
+# 2 = header text
+# 3 = float text
+## Status change: not used in parsing, but to switch status then ignore contents
+# 6 = switch to inlined math
+# 7 = switch to displayed math
+## Note that positive statuses must correspond to CNT codes!
+#
+my $STATUS_IGNORE=0;
+my $STATUS_FLOAT=-1;
+my $STATUS_EXCLUDE_STRONG=-2;
+my $STATUS_EXCLUDE_STRONGER=-3;
+my $STATUS_EXCLUDE_ALL=-4;
+my $STATUS_PREAMBLE=-9;
+my $STATUS_TEXT=1;
+my $STATUS_TEXT_HEADER=2;
+my $STATUS_TEXT_FLOAT=3;
+my $STATUS_TO_INLINEMATH=6;
+my $STATUS_TO_DISPLAYMATH=7;
+
+# When combining two statuses, use the first one; list must be complete!
+my @STATUS_PRIORITY_LIST=(-4,-3,-2,-1,0,-9,3,2,1);
+
+# Status: is a text status..."include status" is more correct
+sub status_is_text {
+ my $st=shift @_;
+ return ($st>0);
+}
+
+# Status: get CNT corresponding to text status (or undef)
+sub status_text_cnt {
+ my $st=shift @_;
+ if ($st==$STATUS_TEXT) {return $CNT_WORDS_TEXT;}
+ if ($st==$STATUS_TEXT_HEADER) {return $CNT_WORDS_HEADER;}
+ if ($st==$STATUS_TEXT_FLOAT) {return $CNT_WORDS_CAPTION;}
+ return undef;
+}
+
+# Status: is an exclude status
+sub status_is_exclude {
+ my $st=shift @_;
+ return ($st<0);
+}
+
+# Status: \begin and \end should be processed
+sub status_inc_envir {
+ my $st=shift @_;
+ return ($st>-2);
+}
+
+# Map status number to CNT value
+sub status_to_cnt {
+ my $st=shift @_;
+ if ($st==$STATUS_TO_INLINEMATH) {return $CNT_COUNT_INLINEMATH;}
+ if ($st==$STATUS_TO_DISPLAYMATH) {return $CNT_COUNT_DISPLAYMATH;}
+ return undef;
+}
+
+# Status as text
+sub status_to_text {
+ my $st=shift @_;
+ return $st;
+}
+
+# Status as text
+sub status_to_style {
+ return 'word'.status_to_text(@_);
+}
+
+### Token types
+# -1: space
+# 0: comment
+# 1: word (or other forms of text or text components)
+# 2: symbol (not word, e.g. punctuation)
+# 3: macro
+# 4: curly braces {}
+# 5: brackets []
+# 6: maths
+# 9: line break in file
+# 999: end of line or blank line
+# 666: TeXcount instruction (%TC:instruction)
+my $TOKEN_SPACE=-1;
+my $TOKEN_COMMENT=0;
+my $TOKEN_WORD=1;
+my $TOKEN_SYMBOL=2;
+my $TOKEN_MACRO=3;
+my $TOKEN_BRACE=4;
+my $TOKEN_BRACKET=5;
+my $TOKEN_MATH=6;
+my $TOKEN_LINEBREAK=9;
+my $TOKEN_TC=666;
+my $TOKEN_END=999;
+
+###### Set global definitions
+
+
+### Break points
+# Definition of macros that define break points that start a new subcount.
+# The values given are used as labels.
+my %BreakPointsOptions;
+$BreakPointsOptions{'none'}={};
+$BreakPointsOptions{'part'}={%{$BreakPointsOptions{'none'}},'\part'=>'Part'};
+$BreakPointsOptions{'chapter'}={%{$BreakPointsOptions{'part'}},'\chapter'=>'Chapter'};
+$BreakPointsOptions{'section'}={%{$BreakPointsOptions{'chapter'}},'\section'=>'Section'};
+$BreakPointsOptions{'subsection'}={%{$BreakPointsOptions{'section'}},'\subsection'=>'Subsection'};
+$BreakPointsOptions{'default'}=$BreakPointsOptions{'subsection'};
+my %BreakPoints=%{$BreakPointsOptions{'none'}};
+
+### Print styles
+# Definition of different print styles: maps of class labels
+# to ANSI codes. Class labels are as used by HTML styles.
+my @STYLES=();
+my %STYLE;
+$STYLES[0]={'error'=>'bold red'};
+$STYLES[1]={%{$STYLES[0]},''=>'normal',' '=>'normal',
+ 'word1'=>'blue','word2'=>'bold blue','word3'=>'blue',
+ 'grouping'=>'red','document'=>'red','mathgroup'=>'magenta',
+ 'state'=>'cyan underline','cumsum'=>'yellow'};
+$STYLES[2]={%{$STYLES[1]},
+ 'command'=>'green','ignore'=>'cyan',
+ 'exclcommand'=>'yellow','exclgroup'=>'yellow','exclmath'=>'yellow'};
+$STYLES[3]={%{$STYLES[2]},
+ 'tc'=>'bold yellow','comment'=>'yellow','option'=>'yellow',
+ 'fileinclude'=>'bold green'};
+$STYLES[4]={%{$STYLES[3]}};
+
+###### Define what a word is and language options
+
+
+# Patters matching a letter. Should be a single character or
+# ()-enclosed regex for substitution into word pattern regex.
+my @LetterMacros=qw/ae AE o O aa AA oe OE ss
+ alpha beta gamma delta epsilon zeta eta theta iota kappa lamda
+ mu nu xi pi rho sigma tau upsilon phi chi psi omega
+ Gamma Delta Theta Lambda Xi Pi Sigma Upsilon Phi Psi Omega
+ /;
+my $specialchars='\\\\('.join('|',@LetterMacros).')(\{\}|\s*|\b)';
+my $modifiedchars='\\\\[\'\"\`\~\^\=](@|\{@\})';
+my %NamedLetterPattern;
+$NamedLetterPattern{'restricted'}='@';
+$NamedLetterPattern{'default'}='('.join('|','@',$modifiedchars,$specialchars).')';
+$NamedLetterPattern{'relaxed'}=$NamedLetterPattern{'default'};
+my $LetterPattern=$NamedLetterPattern{'default'};
+
+# List of regexp patterns that should be analysed as words.
+# Use @ to represent a letter, will be substituted with $LetterPattern.
+# Named patterns may replace or be appended to the original patterns.
+my %NamedWordPattern;
+$NamedWordPattern{'letters'}='@';
+$NamedWordPattern{'words'}='(@+|\{@+\})([\-\'\.]?(@+|\{@+\}))*';
+my @WordPatterns=($NamedWordPattern{'words'});
+
+### Macro option regexp list
+# List of regexp patterns to be gobbled as macro option in and after
+# a macro.
+my %NamedMacroOptionPattern;
+$NamedMacroOptionPattern{'default'}='\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]';
+$NamedMacroOptionPattern{'relaxed'}='\[[^\[\]\n]*\]';
+$NamedMacroOptionPattern{'restricted'}=$NamedMacroOptionPattern{'default'};
+my $MacroOptionPattern=$NamedMacroOptionPattern{'default'};
+
+### Alternative language encodings
+my %NamedEncodingGuessOrder;
+$NamedEncodingGuessOrder{'chinese'}=[qw/utf8 gb2312 big5/];
+$NamedEncodingGuessOrder{'japanese'}=[qw/utf8 euc-jp iso-2022-jp jis shiftjis/];
+$NamedEncodingGuessOrder{'korean'}=[qw/utf8 euc-kr iso-2022-kr/];
+
+
+###### Define character classes (alphabets)
+
+
+### Character classes to use as Unicode properties
+
+# Character group representing digits 0-9 (more restrictive than Digits)
+sub Is_digit { return <<END;
+0030\t0039
+END
+}
+
+# Character group representing letters (excluding logograms)
+sub Is_alphabetic { return <<END;
++utf8::Alphabetic
+-utf8::Ideographic
+-utf8::Katakana
+-utf8::Hiragana
+-utf8::Thai
+-utf8::Lao
+-utf8::Hangul
+END
+}
+
+# Character group representing letters (excluding logograms)
+sub Is_alphanumeric { return <<END;
++utf8::Alphabetic
++utf8::Digit
+-utf8::Ideographic
+-utf8::Katakana
+-utf8::Hiragana
+-utf8::Thai
+-utf8::Lao
+-utf8::Hangul
+END
+}
+
+# Character class for punctuation excluding special characters
+sub Is_punctuation { return <<END;
++utf8::Punctuation
+-0024\t0025
+-005c
+-007b\007e
+END
+}
+
+# Character group representing CJK characters
+sub Is_cjk { return <<END;
++utf8::Han
++utf8::Katakana
++utf8::Hiragana
++utf8::Hangul
+END
+}
+
+# Character group for CJK punctuation characters
+sub Is_cjkpunctuation { return <<END;
+3000\t303f
+2018\t201f
+ff01\tff0f
+ff1a\tff1f
+ff3b\tff3f
+ff5b\tff65
+END
+}
+
+###### Define core rules
### Macros for headers
# Macros that identify headers: i.e. following token or
@@ -43,8 +375,26 @@ my $globalworkdir="";
# only role of defining it here is that the counter for the number
# of headers is incremented by one.
my %TeXheader=('\title'=>[2],'\part'=>[2],'\chapter'=>[2],
- '\section'=>[2],'\subsection'=>[2],'\subsubsection'=>[2],
- '\paragraph'=>[2],'\subparagraph'=>[2]);
+ '\section'=>[2],'\subsection'=>[2],'\subsubsection'=>[2],
+ '\paragraph'=>[2],'\subparagraph'=>[2]);
+
+### Macros indicating package inclusion
+# Will always be assumed to take one parameter (plus options).
+my %TeXpackageinc=('\usepackage'=>1);
+
+### Macros that are counted within the preamble
+# The preamble is the text between \documentclass and \begin{document}.
+# Text and macros in the preamble is ignored unless specified here. The
+# value is the status (1=text, 2=header, etc.) they should be interpreted as.
+# Note that only the first unit (token or {...} block) is counted.
+my %TeXpreamble=(
+ '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3],
+ '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3],
+ '\title'=>[2]);
+
+### In floats: include only specific macros
+# Macros used to identify caption text within floats.
+my %TeXfloatinc=('\caption'=>[3]);
### How many tokens to gobble after macro
# Each macro is assumed to gobble up a given number of
@@ -61,11 +411,11 @@ my %TeXheader=('\title'=>[2],'\part'=>[2],'\chapter'=>[2],
# For macros not specified here, the default value is 0: i.e.
# no tokens are excluded, but [...] options are. Header macros
# specified in %TeXheader are automatically included here.
-my %TeXmacro=(%TeXheader,
+my %TeXmacro=(%TeXheader,%TeXpreamble,%TeXfloatinc,
'\documentclass'=>1,'\documentstyle'=>1,'\usepackage'=>1, '\hyphenation'=>1,
'\pagestyle'=>1,'\thispagestyle'=>1, '\pagenumbering'=>1,'\markboth'=>1, '\markright'=>1,
'\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3],
- '\newenvironment'=>[-3,-3,-3], 'renewenvironment'=>[-3,-3,-3],
+ '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3],
'\newfont'=>2,'\newtheorem'=>2,'\bibliographystyle'=>1, '\bibliography'=>1,
'\parbox'=>1, '\marginpar'=>[3],'\makebox'=>0, '\raisebox'=>1, '\framebox'=>0,
'\newsavebox'=>1, '\sbox'=>1, '\savebox'=>2, '\usebox'=>1,'\rule'=>2,
@@ -80,29 +430,21 @@ my %TeXmacro=(%TeXheader,
'\cline'=>1, '\multicolumn'=>3,'\typeout'=>1, '\typein'=>1,
'beginlist'=>2, 'beginminipage'=>1, 'begintabular'=>1,
'beginthebibliography'=>1,'beginlrbox'=>1,
- '\begin'=>1,'\end'=>1,'\title'=>[2]);
+ '\begin'=>1,'\end'=>1,'\title'=>[2],
+ '\addtocontents'=>2,'\addcontentsline'=>3,
+ '\uppercase'=>0,'\lowercase'=>0);
### Macros that should be counted as one or more words
# Macros that represent text may be declared here. The value gives
# the number of words the macro represents.
my %TeXmacroword=('\LaTeX'=>1,'\TeX'=>1);
-### Macros that are counted within the preamble
-# The preamble is the text between \documentclass and \begin{document}.
-# Text and macros in the preamble is ignored unless specified here. The
-# value is the status (1=text, 2=header, etc.) they should be interpreted as.
-# Note that only the first unit (token or {...} block) is counted.
-my %TeXpreamble=('\title'=>[2],
- '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3],
- '\newenvironment'=>[-3,-3,-3], 'renewenvironment'=>[-3,-3,-3],
- );
-
### Begin-End groups
# Identified as begin-end groups, and define =>state. The
# states used corresponds to the elements of the count array, and
# are:
# 0: Not included
-# 1: Text, word included in text count
+# 1: Text, words included in text count
# 2: Header, words included in header count
# 3: Float caption, words included in float caption count
# 6: Inline mathematics, words not counted
@@ -117,79 +459,41 @@ my %TeXpreamble=('\title'=>[2],
# Note that some environments may only exist within math-mode, and
# therefore need not be defined here: in fact, they should not as it
# is not clear if they will be in inlined or displayed math.
-#
my %TeXgroup=('document'=>1,'letter'=>1,'titlepage'=>0,
'center'=>1,'flushleft'=>1,'flushright'=>1,
- 'abstract'=>1,'quote'=>1,'quotation'=>1,'verse'=>1,'minipage'=>1,'verbatim'=>1,
+ 'abstract'=>1,'quote'=>1,'quotation'=>1,'verse'=>1,'minipage'=>1,
+ 'verbatim'=>-4,'tikzpicture'=>-4,
'description'=>1,'enumerate'=>1,'itemize'=>1,'list'=>1,
'theorem'=>1,'lemma'=>1,'definition'=>1,'corollary'=>1,'example'=>1,
'math'=>6,'displaymath'=>7,'equation'=>7,'eqnarray'=>7,'align'=>7,
+ 'equation*'=>7,'eqnarray*'=>7,'align*'=>7,
'figure'=>-1,'float'=>-1,'picture'=>-1,'table'=>-1,
'tabbing'=>0,'tabular'=>0,'thebibliography'=>0,'lrbox'=>0);
-### In floats: include only specific macros
-# Macros used to identify caption text within floats.
-my %TeXfloatinc=('\caption'=>[3]);
-
### Macros for including tex files
# Allows \macro{file} or \macro file. If the value is 0, the filename will
# be used as is; if it is 1, the filetype .tex will be added if the
# filename is without filetype; if it is 2, the filetype .tex will be added.
my %TeXfileinclude=('\input'=>1,'\include'=>2);
-### Count labels
-# Labels used to describe the counts
-my @countlabel=('Files','Words in text','Words in headers',
- 'Words in float captions','Number of headers','Number of floats',
- 'Number of math inlines','Number of math displayed');
+###### Define package specific rules
-### Break points
-# Definition of macros that define break points that start a new subcount.
-# The values given are used as labels.
-my %BreakPointsOptions;
-$BreakPointsOptions{'none'}={};
-$BreakPointsOptions{'part'}={%{$BreakPointsOptions{'none'}},'\part'=>'Part'};
-$BreakPointsOptions{'chapter'}={%{$BreakPointsOptions{'part'}},'\chapter'=>'Chapter'};
-$BreakPointsOptions{'section'}={%{$BreakPointsOptions{'chapter'}},'\section'=>'Section'};
-$BreakPointsOptions{'subsection'}={%{$BreakPointsOptions{'section'}},'\subsection'=>'Subsection'};
-$BreakPointsOptions{'default'}=$BreakPointsOptions{'subsection'};
-my %BreakPoints=%{$BreakPointsOptions{'none'}};
+### Package rule definitions
-### Print styles
-# Definition of different print styles: maps of class labels
-# to ANSI codes. Class labels are as used by HTML styles.
-my @STYLES=();
-my %STYLE;
-$STYLES[0]={'error'=>'bold red'};
-$STYLES[1]={%{$STYLES[0]},
- 'word1'=>'blue','word2'=>'bold blue','word3'=>'blue',
- 'grouping'=>'red','document'=>'red','mathgroup'=>'magenta',
- 'state'=>'cyan underline','sumcount'=>'yellow'};
-$STYLES[2]={%{$STYLES[1]},
- 'command'=>'green','exclcommand'=>'yellow','exclgroup'=>'yellow','exclmath'=>'yellow',
- 'ignore'=>'cyan'};
-$STYLES[3]={%{$STYLES[2]},
- 'tc'=>'bold yellow','comment'=>'yellow','option'=>'yellow',
- 'fileinclude'=>'bold green'};
-$STYLES[4]={%{$STYLES[3]}};
+my %PackageTeXmacro=(); # TeXmacro definitions per package
+my %PackageTeXmacroword=(); # TeXmacroword definitions per package
+my %PackageTeXheader=(); # TeXheader definitions per package
+my %PackageTeXgroup=(); # TeXgroup definitions per package
+my %PackageTeXfileinclude=(); # TeXgroup definitions per package
-### Word regexp pattern list
-# List of regexp patterns that should be analysed as words.
-# Use @ to represent a letter, will be substituted with $LetterPattern.
-my @WordPatterns=('(@+\.)+@+\.?','@+([\-\']@+)*');
-my $specialchars='\\\\(ae|AE|o|O|aa|AA)';
-my $modifiedchars='\\\\[\'\"\`\~\^\=](\w|\{\w\})';
-my $LetterPattern='\w';
-my $LetterPatternRelaxed='([\w\-\']|'.$modifiedchars.'|'.$specialchars.'(\{\})?|\{'.$specialchars.'\}|\{\w\})';
-my %NamedWordPattern;
-$NamedWordPattern{'chinese'}='\p{script=Han}';
-$NamedWordPattern{'japanese'}='(\p{script=Han}|\p{script=Hiragana}|\p{script=Katakana})';
+# Rules for package psfig
+$PackageTeXmacro{'psfig'}={('\psfig'=>1)};
-### Macro option regexp list
-# List of regexp patterns to be gobbled as macro option in and after
-# a macro.
-my @MacroOptionPatterns=('\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]');
-my @MacroOptionPatternsRelaxed=('\[[^\[\]\n]*\]');
+# Rules for bibliography inclusion
+$PackageTeXmacroword{'%incbib'}={'thebibliography'=>1};
+$PackageTeXmacro{'%incbib'}={};
+$PackageTeXgroup{'%incbib'}={'thebibliography'=>1};
+$PackageTeXfileinclude{'%incbib'}={'\bibliography'=>'bbl'};
###### Main script
@@ -197,6 +501,7 @@ my @MacroOptionPatternsRelaxed=('\[[^\[\]\n]*\]');
###################################################
MAIN(@ARGV);
+exit; # Just to make sure it ends here...
###################################################
@@ -207,22 +512,40 @@ MAIN(@ARGV);
# MAIN ROUTINE: Handle arguments, then parse files
sub MAIN {
- my @args=@_;
+ my @args;
+ push @args,@StartupOptions;
+ push @args,@_;
+ Initialise();
+ Check_Arguments(@args);
my @toplevelfiles=Parse_Arguments(@args);
Apply_Options();
- if (scalar(@toplevelfiles)==0) {
- conditional_print_help_style()
- || print_error("No files specified.","p","error");
- } else {
+ if (scalar @toplevelfiles>0 || $fileFromSTDIN) {
+ if ($showVersion && !$htmlstyle && !($briefsum && $totalflag)) {
+ print "\n=== LaTeX word count (TeXcount version $versionnumber) ===\n\n";
+ }
conditional_print_help_style();
- my $totalcount=parse_file_list(@toplevelfiles);
+ my $totalcount=Parse_file_list(@toplevelfiles);
conditional_print_total($totalcount);
+ Report_Errors();
+ if ($optionWordFreq || $optionWordClassFreq) {print_word_freq();}
+ } elsif ($showcodes>1) {
+ conditional_print_help_style();
+ } else {
+ error($Main,'No files specified.');
}
- Report_ErrorCount();
Close_Output();
}
-# Checks arguments, exits on exit condition
+# Initialise, overrule initial settings, etc.
+sub Initialise {
+ _option_subcount();
+ # Windows settings
+ if ($^O=~/^MSWin/) {
+ option_ansi_colours(0);
+ }
+}
+
+# Check arguments, exit on exit condition
sub Check_Arguments {
my @args=@_;
if (!@args) {
@@ -230,26 +553,28 @@ sub Check_Arguments {
print_syntax();
print_reference();
exit;
- } elsif ($args[0]=~/^(\-?\-(h|\?|help)|\/(\?|h))$/) {
+ } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))$/) {
print_help();
exit;
- } elsif ($args[0]=~/^\-?\-(ver|version)$/) {
+ } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))=/) {
+ print_help_on("$'");
+ exit;
+ } elsif ($args[0]=~/^--?(ver|version)$/) {
print_version();
exit;
- } elsif ($args[0]=~/^\-?\-(lic|license)$/) {
+ } elsif ($args[0]=~/^--?(lic|license|licence)$/) {
print_license();
exit;
}
return 1;
}
-# Parses arguments, sets options (global) and returns file list
+# Parse arguments, set options (global) and return file list
sub Parse_Arguments {
my @args=@_;
- Check_Arguments(@args);
my @files;
- foreach my $arg (@ARGV) {
- if (Parse_Option($arg)) {next;}
+ foreach my $arg (@args) {
+ if (parse_option($arg)) {next;}
if ($arg=~/^\-/) {
print 'Invalid opton '.$arg."\n";
print_syntax();
@@ -262,49 +587,88 @@ sub Parse_Arguments {
}
# Parse individual option parameters
-sub Parse_Option {
+sub parse_option {
my $arg=shift @_;
- return parse_options_parsing($arg)
- || parse_options_sums($arg)
+ return parse_options_preset($arg)
+ || parse_options_parsing($arg)
+ || parse_options_counts($arg)
|| parse_options_output($arg)
|| parse_options_format($arg)
;
}
+# Parse presetting options
+sub parse_options_preset {
+ my $arg=shift @_;
+ if ($arg=~/^-(opt|option|options|optionfile)=/) {
+ _parse_optionfile($');
+ }
+ else {return 0;}
+ return 1;
+}
+
+# Parse parsing options
sub parse_options_parsing {
my $arg=shift @_;
- if ($arg eq '-inc') {$includeTeX=1;}
+ if ($arg eq '-') {$fileFromSTDIN=1;}
+ elsif ($arg eq '-merge') {$includeTeX=2;}
+ elsif ($arg eq '-inc') {$includeTeX=1;}
elsif ($arg eq '-noinc') {$includeTeX=0;}
+ elsif ($arg =~/^-(includepackage|incpackage|package|pack)=(.*)$/) {include_package($2);}
+ elsif ($arg eq '-incbib') {$includeBibliography=1;}
+ elsif ($arg eq '-nobib') {$includeBibliography=0;}
elsif ($arg eq '-dir') {$globalworkdir=undef;}
- elsif ($arg=~/^-dir=(.*)$/) {$globalworkdir=$1;}
- elsif ($arg=~/^-(utf8|unicode)$/) {$utf8flag=1;}
- elsif ($arg=~/^-(ch|chinese|zhongwen)$/) {
- $utf8flag=1;
- @WordPatterns=($NamedWordPattern{'chinese'},@WordPatterns);
- }
- elsif ($arg=~/^-(jp|japanese)$/) {
- $utf8flag=1;
- @WordPatterns=($NamedWordPattern{'japanese'},@WordPatterns);
+ elsif ($arg=~/^-dir=(.*)$/) {
+ $globalworkdir=$1;
+ $globalworkdir=~s:([^\/\\])$:$1\/:;
}
+ elsif ($arg =~/^-(enc|encode|encoding)=(.+)$/) {$encoding=$2;}
+ elsif ($arg =~/^-(utf8|unicode)$/) {$encoding='utf8';}
+ elsif ($arg =~/^-(alpha(bets?)?)=(.*)$/) {set_script_options(\@AlphabetScripts,$3);}
+ elsif ($arg =~/^-(logo(grams?)?)=(.*)$/) {set_script_options(\@LogogramScripts,$3);}
+ elsif ($arg =~/^-([-a-z]+)$/ && set_language_option($1)) {}
elsif ($arg eq '-relaxed') {
- @MacroOptionPatterns=@MacroOptionPatternsRelaxed;
- $LetterPattern=$LetterPatternRelaxed;
+ $MacroOptionPattern=$NamedMacroOptionPattern{'relaxed'};
+ $LetterPattern=$NamedLetterPattern{'relaxed'};
+ }
+ elsif ($arg eq '-restricted') {
+ $MacroOptionPattern=$NamedMacroOptionPattern{'restricted'};
+ $LetterPattern=$NamedLetterPattern{'restricted'};
}
else {return 0;}
return 1;
}
-sub parse_options_sums {
+# Parse count and summation options
+sub parse_options_counts {
my $arg=shift @_;
- if ($arg=~/^-sum(=(.+))?$/) {option_sum($2);}
- elsif ($arg=~/^-(sub|subcounts?)(=(.+))?$/) {option_subcount($3);}
+ if ($arg =~/^-sum(=(.+))?$/) {_option_sum($2);}
+ elsif ($arg =~/^-nosum/) {@sumweights=();}
+ elsif ($arg =~/^-(sub|subcounts?)(=(.+))?$/) {_option_subcount($3);}
+ elsif ($arg =~/^-(nosub|nosubcounts?)/) {$showsubcounts=0;}
+ elsif ($arg eq '-freq') {$optionWordFreq=1;}
+ elsif ($arg =~/^-freq=(\d+)$/) {$optionWordFreq=$1;}
+ elsif ($arg eq '-stat') {$optionWordClassFreq=1;}
else {return 0;}
return 1;
}
-sub option_subcount {
+# Apply sum option
+sub _option_sum {
my $arg=shift @_;
- $showsubcounts=1;
+ if (!defined $arg) {
+ @sumweights=(1,1,1,0,0,1,1);
+ } elsif ($arg=~/^(\d+(\.\d*)?(,\d+(\.\d*)?){0,6})$/) {
+ @sumweights=split(',',$1);
+ } else {
+ print STDERR "Warning: Option value ".$arg." not valid, ignoring option.\n";
+ }
+}
+
+# Apply subcount options
+sub _option_subcount {
+ my $arg=shift @_;
+ $showsubcounts=2;
if (!defined $arg) {
%BreakPoints=%{$BreakPointsOptions{'default'}};
} elsif (my $option=$BreakPointsOptions{$arg}) {
@@ -315,55 +679,106 @@ sub option_subcount {
}
}
-sub option_sum {
+# Parse output and verbosity options
+sub parse_options_output {
my $arg=shift @_;
- if (!defined $arg) {
+ if ($arg eq '-strict') {$strictness=1;}
+ elsif ($arg eq '-v0') {$verbose=0;}
+ elsif ($arg eq '-v1') {$verbose=1;}
+ elsif ($arg eq '-v2') {$verbose=2;}
+ elsif ($arg eq '-v3' || $arg eq '-v') {$verbose=3;}
+ elsif ($arg eq '-v4') {$verbose=3; $showstates=1;}
+ elsif ($arg =~/^-showstates?$/ ) {$showstates=1;}
+ elsif ($arg =~/^-(q|quiet)$/ ) {$verbose=-1;}
+ elsif ($arg =~/^-(template)=(.*)$/ ) {_set_output_template($2);}
+ elsif ($arg eq '-split') {$optionFast=1;}
+ elsif ($arg eq '-nosplit') {$optionFast=0;}
+ elsif ($arg eq '-showver') {$showVersion=1;}
+ elsif ($arg eq '-nover') {$showVersion=-1;}
+ elsif ($arg =~/^-nosep(s|arators?)?$/ ) {$separator='';}
+ elsif ($arg =~/^-sep(arators?)?=(.*)$/ ) {$separator=$2;}
+ else {return 0;}
+ return 1;
+}
+
+# Set output template
+sub _set_output_template {
+ my $template=shift @_;
+ $outputtemplate=$template;
+ if ($template=~/\{(S|SUM)[\?\}]/i && !@sumweights) {
@sumweights=(1,1,1,0,0,1,1);
- } elsif ($arg=~/^(\d+(,\d+){0,6})$/) {
- @sumweights=split(',',$1);
- } else {
- print STDERR "Warning: Option value ".$arg." not valid, ignoring option.\n";
+ }
+ if ($template=~/\{SUB\?/i && !$showsubcounts) {
+ _option_subcount();
}
}
+# Parse output formating options
sub parse_options_format {
my $arg=shift @_;
- if ($arg eq '-brief') {$briefsum=1;}
+ if ($arg eq '-brief') {$briefsum=1;}
elsif ($arg eq '-total') {$totalflag=1;}
+ elsif ($arg eq '-0') {$briefsum=1;$totalflag=1;$verbose=-1;$finalLineBreak=0;}
elsif ($arg eq '-1') {$briefsum=1;$totalflag=1;$verbose=-1;}
- elsif ($arg eq "-html" ) {option_no_colours();$htmlstyle = 2;}
- elsif ($arg eq "-htmlcore" ) {option_no_colours();$htmlstyle = 1;}
- elsif ($arg=~/^\-(nocol|nc$)/) {option_no_colours();}
- elsif ($arg eq '-codes') {
- $showcodes=2;
- if ($verbose==0) {$verbose=3;}
- }
+ elsif ($arg eq '-html' ) {option_ansi_colours(0);$htmlstyle = 2;}
+ elsif ($arg eq '-htmlcore' ) {option_ansi_colours(0);$htmlstyle = 1;}
+ elsif ($arg =~/^\-(nocol|nc$)/) {option_ansi_colours(0);}
+ elsif ($arg =~/^\-(col)$/) {option_ansi_colours(1);}
+ elsif ($arg eq '-codes') {$showcodes=2;}
elsif ($arg eq '-nocodes') {$showcodes=0;}
else {return 0;}
return 1;
}
-sub parse_options_output {
+# Include options from option file
+sub _parse_optionfile {
+ my $filename=shift @_;
+ open(FH,"<",$filename)
+ || die "Option file not found: ".$filename."\n";
+ my @options=<FH>;
+ close(FH);
+ s/^\s*(#.*|)//s for @options;
+ my $text=join('',@options);
+ $text=~s/(\n|\r|\r\n)\s*\\//g;
+ @options=split("\n",$text);
+ foreach my $arg (@options) {
+ __optionfile_tc($arg)
+ || parse_option($arg)
+ || die "Invalid option ".$arg." in ".$filename."\n";
+ }
+}
+
+# Parse option file TC options
+sub __optionfile_tc {
my $arg=shift @_;
- if ($arg eq "-v0") {$verbose=0;}
- elsif ($arg eq "-v1") {$verbose=1;}
- elsif ($arg eq '-vv' || $arg eq '-v2') {$verbose=2;}
- elsif ($arg eq '-vvv' || $arg eq '-v3' || $arg eq '-v') {$verbose=3;}
- elsif ($arg eq '-vvvv' || $arg eq '-v4') {$verbose=3; $showstates=1;}
- elsif ($arg =~ /^\-showstates?$/ ) {$showstates=1;}
- elsif ($arg =~ /^-(q|-?quiet)$/ ) {$verbose=-1;}
- else {return 0;}
+ $arg=~s/^\%\s*// || return 0;
+ if ($arg=~/^subst\s+(\\\w+)\s+/i) {
+ $substitutions{$1}=$';
+ } elsif ($arg=~/^(\w+)\s+([\\]*\w+)\s+([^\s\n]+)(\s+([0-9]+))?/i) {
+ tc_macro_param_option($1,$2,$3,$5) || die "Invalid TC option: ".$arg."\n";
+ } else {
+ print "Invalid TC option format: ".$arg."\n";
+ return 0;
+ }
return 1;
}
# Parse file list and return total count
-sub parse_file_list {
- my @filelist=@_;
- my $listtotalcount=new_count("TOTAL COUNT");
- for my $file (<@filelist>) {
- my $filetotalcount=parse_file($file);
- add_count($listtotalcount,$filetotalcount);
+sub Parse_file_list {
+ my @files=@_;
+ my $listtotalcount=new_count("Total");
+ foreach (@files) {s/\\/\//g; s/ /\\ /g;}
+ if (@files) {
+ @files=<@files>; # For the sake of Windows: expand wildcards!
+ for my $file (@files) {
+ my $filetotalcount=parse_file($file);
+ add_to_total($listtotalcount,$filetotalcount);
+ }
}
+ if ($fileFromSTDIN) {
+ my $filetotalcount=parse_file($_STDIN_);
+ add_to_total($listtotalcount,$filetotalcount);
+ }
return $listtotalcount;
}
@@ -375,30 +790,36 @@ sub parse_file {
$workdir=$file;
$workdir =~ s/^((.*[\\\/])?)[^\\\/]+$/$1/;
}
- @filelist=($file);
- if ($htmlstyle) {print "\n<div class='filegroup'>\n";}
- my $filetotalcount=new_count("SUM COUNT FOR ".$file);
+ if ($htmlstyle && ($verbose || !$totalflag)) {print "\n<div class='filegroup'>\n";}
+ my $filetotalcount=new_count("File(s) total: ".$file);
+ @filelist=();
+ _add_file($filetotalcount,$file);
foreach my $f (@filelist) {
- my $tex=TeXfile($f);
- my $fpath=$f;
- $fpath=~s/^((.*[\\\/])?)[^\\\/]+$/$1/;
- if (!defined $tex) {
- print STDERR "File not found or not readable: ".$f."\n";
- formatprint("File not found or not readable: ".$f."\n","p","error");
- } else {
- parse($tex);
- my $filecount=add_subcount($tex);
- if (!$totalflag) {
- print_count($filecount);
- print "\n";
- }
- add_count($filetotalcount,$filecount);
- }
+ _add_file($filetotalcount,$f,"Included file: ".$f);
}
- if ($htmlstyle) {print "</div>\n\n";}
+ if (!$totalflag && get_count($filetotalcount,$CNT_FILE)>1) {
+ if ($htmlstyle) {formatprint("Sum of files: ".$file."\n",'h2');}
+ print_count($filetotalcount,'sumcount');
+ }
+ if ($htmlstyle && ($verbose || !$totalflag)) {print "</div>\n\n";}
return $filetotalcount;
}
+# Parse single file, included files will be appended to @filelist.
+sub _add_file {
+ my ($filetotalcount,$f,$title)=@_;
+ my $tex=TeXfile($f,$title);
+ my $fpath=$f;
+ $fpath=~s/^((.*[\\\/])?)[^\\\/]+$/$1/;
+ if (!defined $tex) {
+ error($Main,'File not found or not readable: '.$f);
+ } else {
+ parse($tex);
+ my $filecount=next_subcount($tex);
+ if (!$totalflag) {print_count($filecount);}
+ add_to_total($filetotalcount,$filecount);
+ }
+}
######
###### Subroutines
@@ -407,102 +828,272 @@ sub parse_file {
###### CMD specific implementations
-sub add_file_to_list {
- my $fname=shift @_;
- push @filelist,$workdir.$fname;
-}
-
-sub print_with_style {
- my ($text,$style,$colour)=@_;
- #if ($utf8flag || $htmlstyle) {utf8::encode($text);}
- if ($htmlstyle) {
- print "<span class='".$style."'>".$text."</span>";
+# Add file to list of files scheduled for parsing
+sub include_file {
+ my ($tex,$fname)=@_;
+ my $fpath=$workdir.$fname;
+ if ($includeTeX==2) {
+ my $bincode=read_binary($fpath) || BLOCK {
+ error($tex,"File $fpath not found.");
+ return;
+ };
+ flush_next($tex);
+ line_return(0,$tex);
+ prepend_code($tex,$bincode,$fname);
} else {
- print Term::ANSIColor::colored($text,$colour);
+ push @filelist,$fpath;
}
}
-sub option_no_colours {
- $ENV{'ANSI_COLORS_DISABLED'} = 1;
-}
-
# Print count (total) if conditions are met
sub conditional_print_total {
my $sumcount=shift @_;
- if ($totalflag || get_count($sumcount,0)>1) {
+ if ($totalflag || number_of_subcounts($sumcount)>1) {
if ($totalflag && $briefsum && @sumweights) {
- print total_count($sumcount);
+ print get_sum_count($sumcount),"\n";
} else {
- if ($htmlstyle) {
- formatprint("Total word count",'h2');
- }
- print_count($sumcount);
+ if ($htmlstyle) {formatprint("Total word count",'h2');}
+ print_count($sumcount,'sumcount');
}
}
}
+# Set or unset use of ANSI colours
+sub option_ansi_colours {
+ my $flag=shift @_;
+ $ENV{'ANSI_COLORS_DISABLED'} = $flag?undef:1;
+}
+
+# Print text using ANSI colours
+sub ansiprint {
+ my ($text,$colour)=@_;
+ print Term::ANSIColor::colored($text,$colour);
+}
+
###### Option handling
# Apply options to set values
sub Apply_Options {
+ if ($showcodes>1 && $verbose<1) {$verbose=3;}
%STYLE=%{$STYLES[$verbose]};
- if ($utf8flag) {binmode STDOUT,':utf8';}
+ if (defined $encoding && $encoding eq 'guess') {$encoding=undef;}
+ if (!defined $encoding) {
+ } elsif (ref(find_encoding($encoding))) {
+ if (!$htmlstyle) {$outputEncoding=$encoding;}
+ } else {
+ error($Main,"Unknown encoding $encoding ignored.");
+ error_details($Main,'Valid encodings are: '.wrap('','',join(', ',Encode->encodings(':all'))));
+ $encoding=undef;
+ }
+ if (!defined $outputEncoding) {$outputEncoding='utf8';}
+ binmode STDIN;
+ binmode STDOUT,':encoding('.$outputEncoding.')';
if ($htmlstyle>1) {html_head();}
- foreach (@WordPatterns) {
- s/\@/$LetterPattern/g;
+ flush_errorbuffer($Main);
+ apply_language_options();
+ if ($includeBibliography) {apply_include_bibliography();}
+}
+
+# Set or add scripts to array of scripts
+sub set_script_options {
+ my ($scriptset,$str)=@_;
+ if ($str=~s/^\+//) {} else {splice(@$scriptset,0,scalar $scriptset);}
+ my @scripts=split(/[+,]/,$str);
+ foreach my $scr (@scripts) {
+ $scr=~tr/_/ /;
+ if ($scr eq 'Alphabetic') {
+ warning($Main,'Using alphabetic instead of Unicode class Alphabetic');
+ $scr='alphabetic';
+ }
+ if ($scr=~/^[a-z]/) {$scr='Is_'.$scr;}
+ if (is_property_valid($scr)) {push @$scriptset,$scr;}
+ else {error($Main,"Unknown script $scr ignored.");}
}
}
+sub is_property_valid {
+ my $script=shift @_;
+ eval {' '=~/\p{$script}/};
+ if ($@) {return 0;} else {return 1;}
+}
+
+# Set language option, return language if applied, undef if not
+sub set_language_option {
+ my $language=shift @_;
+ if ($language=~/^(count\-?)all$/) {
+ @AlphabetScripts=qw/Digit Is_alphabetic/;
+ @LogogramScripts=qw/Ideographic Katakana Hiragana Thai Lao Hangul/;
+ } elsif ($language=~/^words(-?only)?$/) {
+ @LogogramScripts=();
+ } elsif ($language=~/^(ch|chinese|zhongwen)(-?only)?$/) {
+ @LogogramScripts=qw/Han/;
+ if (defined $2) {$LetterPattern=undef;}
+ @encodingGuessOrder=@{$NamedEncodingGuessOrder{'chinese'}};
+ return 'chinese';
+ } elsif ($language=~/^(jp|japanese)(-?only)?$/) {
+ @LogogramScripts=qw/Han Hiragana Katakana/;
+ if (defined $2) {$LetterPattern=undef;}
+ @encodingGuessOrder=@{$NamedEncodingGuessOrder{'japanese'}};
+ return 'japanese';
+ } elsif ($language=~/^(kr|korean)(-?only)?$/) {
+ @LogogramScripts=qw/Han Hangul/;
+ if (defined $2) {$LetterPattern=undef;}
+ @encodingGuessOrder=@{$NamedEncodingGuessOrder{'korean'}};
+ return 'korean';
+ } elsif ($language=~/^(kr|korean)-?words?(-?only)?$/) {
+ if (defined $2) {
+ @AlphabetScripts=qw/Hangul/;
+ @LogogramScripts=qw/Han/;
+ } else {
+ @AlphabetScripts=qw/Digit Is_alphabetic Hangul/;
+ @LogogramScripts=qw/Han Katakana Hiragana Thai Lao/;
+ }
+ @encodingGuessOrder=@{$NamedEncodingGuessOrder{'korean'}};
+ return 'korean-words';
+ } elsif ($language=~/^(char|character|letter)s?(-?only)?$/) {
+ @WordPatterns=($NamedWordPattern{'letters'});
+ if (defined $2) {@LogogramScripts=();}
+ $countlabel[1]='Letters in text';
+ $countlabel[2]='Letters in headers';
+ $countlabel[3]='Letters in captions';
+ return 'letter';
+ } else {
+ return undef;
+ }
+}
-###### TeX code handle
+# Apply language options
+sub apply_language_options {
+ my @tmp;
+ if (defined $LetterPattern && @AlphabetScripts && scalar @AlphabetScripts>0) {
+ @tmp=@AlphabetScripts;
+ foreach (@tmp) {$_='\\p{'.$_.'}';}
+ my $letterchars='['.join('',@tmp).']';
+ my $letter=$LetterPattern;
+ $letter=~s/@/$letterchars/g;
+ @WordPatterns=map { s/\@/$letter/g ; qr/$_/ } @WordPatterns;
+ } else {
+ @WordPatterns=();
+ }
+ if (@LogogramScripts && scalar @LogogramScripts>0) {
+ @tmp=@LogogramScripts;
+ foreach (@tmp) {$_='\\p{'.$_.'}';}
+ push @WordPatterns,'['.join('',@tmp).']';
+ }
+ if (scalar @WordPatterns==0) {
+ error($Main,'No script (alphabets or logograms) defined. Using fallback mode.');
+ push @WordPatterns,'\\w+';
+ }
+ $WordPattern=join '|',@WordPatterns;
+}
+# Apply incbib rule
+sub apply_include_bibliography {
+ include_package('%incbib');
+}
-sub TeXfile {
- my $filename=shift @_;
- my $file=read_file($filename) || return undef;
- return TeXcode($file,$filename);
+# Process package inclusion
+sub include_package {
+ my $incpackage=shift @_;
+ my $sub;
+ _add_to_hash_if_exists(\%TeXmacro,\%PackageTeXmacro,$incpackage);
+ _add_to_hash_if_exists(\%TeXmacroword,\%PackageTeXmacroword,$incpackage);
+ _add_to_hash_if_exists(\%TeXheader,\%PackageTeXheader,$incpackage);
+ _add_to_hash_if_exists(\%TeXgroup,\%PackageTeXgroup,$incpackage);
+ _add_to_hash_if_exists(\%TeXfileinclude,\%PackageTeXfileinclude,$incpackage);
}
-sub read_file {
- my $filename=shift @_;
- if ($utf8flag) {
- open(FH,"<:utf8",$filename) || return undef;
- } else {
- open(FH,"<".$filename) || return undef;
- }
- if ($verbose>0) {
- formatprint("File: ".$filename."\n",'h2');
- $blankline=0;
+# Add package rules if defined
+sub _add_to_hash_if_exists {
+ my ($target,$source,$name)=@_;
+ my $sub;
+ if ($sub=$source->{$name}) {
+ while (my ($key,$val)=each(%$sub)) {
+ $target->{$key}=$val;
+ }
}
- my @text=<FH>;
- close(FH);
- my $latexcode=join('',@text);
- if ($utf8flag) {
- $latexcode =~ s/^\x{feff}//;
+}
+
+# Process TC instruction
+sub tc_macro_param_option {
+ my ($instr,$macro,$param,$option)=@_;
+ if ($param=~/^\[([0-9,]+)\]$/) {$param=[split(',',$1)];}
+ if (!defined $option) {$option=1;}
+ if (($instr eq 'macro') || ($instr eq 'exclude')) {$TeXmacro{$macro}=$param;}
+ elsif ($instr eq 'header') {$TeXheader{$macro}=$param;$TeXmacro{$macro}=$param;}
+ elsif ($instr eq 'macroword') {$TeXmacroword{$macro}=$param;}
+ elsif ($instr eq 'preambleinclude') {$TeXpreamble{$macro}=$param;}
+ elsif ($instr eq 'group') {
+ $TeXmacro{'begin'.$macro}=$param;
+ $TeXgroup{$macro}=$option;
}
- return $latexcode;
+ elsif ($instr eq 'floatinclude') {$TeXfloatinc{$macro}=$param;}
+ elsif ($instr eq 'fileinclude') {$TeXfileinclude{$macro}=$param;}
+ elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;}
+ else {return 0;}
+ return 1;
}
-###### Parsing routines
+
+###### TeX code handle
+
-# Make TeXcode handle
+# Return object capable of capturing errors for use when
+# no TeXcode object is available.
+sub getMain {
+ my %main=();
+ $main{'errorcount'}=0;
+ $main{'errorbuffer'}=[];
+ $main{'warnings'}={};
+ return \%main;
+}
+
+
+## Make TeX handle for LaTeX code: the main TeXcount object
+# The "TeX object" is a data containser: a hash containing
+# - filename: name of LaTeX file being parsed
+# - filepath: path to LaTeX file
+# - countsum: count object with total count (incl. subcounts)
+# - subcount: count object for subcount (to be added to countsum)
+# - subcounts: list of subcounts
+# plus following elements used for the processing the LaTeX code
+# - line: the LaTeX paragraph being processed
+# - texcode: what remains of LaTeX code to process (after line)
+# - texlength: length of LaTeX code
+# - next: the next token, i.e. the one being processed
+# - type: the type of the next token
+# - style: the present output style (for verbose output)
+# - printstate: present parsing state (for verbose output only)
+# - eof: set once the end of the input is reached
+# which are passed to methods by passing the TeX object. It is used when parsing
+# the LaTeX code, and discarded once the parsing is done. During parsing, counts
+# are added to the subcount element; whenever a break point is encountered, the
+# subcount is added to the countsum (next_subcount) and a new subcount object
+# prepared. Note that this requires that the last subcount object be added to
+# the countsum once the end of the document is reached.
sub TeXcode {
- my ($texcode,$filename,$title)=@_;
- my %TeX=();
- $TeX{'filename'}=$filename;
- if (!defined $filename) {
- $TeX{'filepath'}='';
- } elsif ($filename=~/^(.*[\\\/])[^\\\/]+$/) {
- $TeX{'filepath'}=$1;
- } else {
- $TeX{'filepath'}='';
- }
+ my ($bincode,$filename,$title)=@_;
+ my $tex=_TeXcode_blank($filename,$title);
+ _TeXcode_setcode($tex,$bincode);
+ more_texcode($tex);
+ return $tex;
+}
+
+# Return a blank TeXcode object
+sub _TeXcode_blank {
+ my ($filename,$title)=@_;
if (defined $title) {}
- elsif (defined $filename) {$title="FILE: ".$filename;}
+ elsif (defined $filename) {$title="File: ".$filename;}
else {$title="Word count";}
- $TeX{'line'}=$texcode;
+ my %TeX=();
+ $TeX{'errorcount'}=0;
+ $TeX{'filename'}=$filename;
+ if (!defined $filename) {$TeX{'filepath'}='';}
+ elsif ($filename=~/^(.*[\\\/])[^\\\/]+$/) {$TeX{'filepath'}=$1;}
+ else {$TeX{'filepath'}='';}
+ $TeX{'line'}='';
$TeX{'next'}=undef;
$TeX{'type'}=undef;
$TeX{'style'}=undef;
@@ -510,27 +1101,216 @@ sub TeXcode {
$TeX{'eof'}=0;
my $countsum=new_count($title);
$TeX{'countsum'}=$countsum;
+ $countsum->{'TeXcode'}=\%TeX;
my $count=new_count("_top_");
- $TeX{'count'}=$count;
- inc_count(\%TeX,0);
+ $TeX{'subcount'}=$count;
+ inc_count(\%TeX,$CNT_FILE);
my @countlist=();
- $TeX{'countlist'}=\@countlist;
$countsum->{'subcounts'}=\@countlist;
return \%TeX;
}
+# Set the texcode element of the TeXcount object
+sub _TeXcode_setcode {
+ my ($tex,$bincode)=@_;
+ $tex->{'texcode'}=_prepare_texcode($tex,$bincode);
+ $tex->{'texlength'}=length($tex->{'texcode'});
+ # TODO: Comment out this test
+ if ($tex->{'texcode'} =~ /([[:^ascii:]])/) {warning($tex,"Code contains non-ASCII characters.");}
+}
+
+# Decode and return TeX/LaTeX code
+sub _prepare_texcode {
+ my ($tex,$texcode)=@_;
+ $texcode=_decode_texcode($tex,$texcode);
+ foreach my $key (keys %substitutions) {
+ my $value=$substitutions{$key};
+ $texcode=~s/(\w)\Q$key\E/$1 $value/g;
+ $texcode=~s/\Q$key\E/$value/g;
+ }
+ return $texcode;
+}
+
+# Return text decoder
+sub _decode_texcode {
+ my ($tex,$texcode)=@_;
+ my $decoder;
+ if (defined $encoding) {
+ $decoder=find_encoding($encoding);
+ eval {
+ $texcode=$decoder->decode($texcode);
+ };
+ if ($@) {
+ error($tex,'Decoding file/text using the '.$decoder->name.' encoding failed.');
+ }
+ } else {
+ ($texcode,$decoder)=_guess_encoding($texcode);
+ if (!ref($decoder)) {
+ error($tex,'Failed to identify encoding or incorrect encoding specified.');
+ $tex->{'encoding'}='[FAILED]';
+ return $texcode;
+ }
+ }
+ __set_encoding_name($tex,$decoder->name);
+ $texcode =~ s/^\x{feff}//; # Remove BOM (relevant for UTF only)
+ if ($texcode =~/\x{fffd}/ ) {
+ error($tex,'File/text was not valid '.$decoder->name.' encoded.');
+ }
+ return $texcode;
+}
+
+# Guess the right encoding to use
+sub _guess_encoding {
+ my ($texcode)=@_;
+ foreach my $enc (@encodingGuessOrder) {
+ my $dec=find_encoding($enc);
+ if (ref($dec)) {
+ eval {
+ $texcode=$dec->decode($texcode,Encode::FB_CROAK)
+ };
+ if (!$@) {return $texcode,$dec;}
+ }
+ }
+ return $texcode,undef;
+}
+
+# Set name of current encoding
+sub __set_encoding_name {
+ my ($tex,$enc)=@_;
+ my $cur=$tex->{'encoding'};
+ if (!defined $enc) {$enc='[FAILED]';} # Shouldn't happen here though...
+ if (!defined $cur) {}
+ elsif ($enc eq 'ascii') {$enc=$cur;}
+ elsif ($cur eq 'ascii') {}
+ elsif ($cur ne $enc) {
+ error($tex,"Mismatching encodings: $cur versus $enc.");
+ }
+ $tex->{'encoding'}=$enc;
+}
+
+# Apply substitution rule
+sub apply_substitution_rule {
+ my ($tex,$from,$to)=@_;
+ $tex->{'line'}=~s/(\w)\Q$from\E\b\s*/$1 $to/g;
+ $tex->{'line'}=~s/\Q$from\E\b\s*/$to/g;
+ $tex->{'texcode'}=~s/(\w)\Q$from\E\b\s*/$1 $to/g;
+ $tex->{'texcode'}=~s/\Q$from\E\b\s*/$to/g;
+}
+
+## Get more TeX code from texcode buffer if possible, return 1 if done
+sub more_texcode {
+ my ($tex)=@_;
+ if (!defined $tex->{'texcode'}) {return 0;}
+ if ( $optionFast && $tex->{'texcode'} =~ s/^.*?(\r{2,}|\n{2,}|(\r\n){2,})//s ) {
+ $tex->{'line'}.=$&;
+ return 1;
+ }
+ $tex->{'line'}.=$tex->{'texcode'};
+ $tex->{'texcode'}=undef;
+ return 1;
+}
+
+## Prepend LaTeX code to TeXcode object
+sub prepend_code {
+ my ($tex,$code,$fname)=@_;
+ my $prefix="\n% --- Start of included file ".$fname."\n";
+ my $suffix="\n% --- End of included file ".$fname."\n";
+ $code=_decode_texcode($tex,$code);
+ $tex->{'length'}+=length($code);
+ $tex->{'texcode'}=$prefix.$code.$suffix.$tex->{'line'}.$tex->{'texcode'};
+ $tex->{'line'}='';
+ more_texcode();
+}
+
+## Returns size of TeX code in bytes
+sub get_texsize {
+ my $tex=shift @_;
+ return $tex->{'texlength'}
+}
+
+
+###### TeX file reader
+
+
+# Read LaTeX file into TeX object
+sub TeXfile {
+ my ($filename,$title)=@_;
+ if ($filename eq $_STDIN_) {
+ if ($verbose>0) {
+ formatprint("File from STDIN\n",'h2');
+ $blankline=0;
+ }
+ return TeXcode(_read_stdin(),'STDIN',$title);
+ } else {
+ my $bincode=read_binary($filename) || return undef;
+ if ($verbose>0) {
+ formatprint("File: ".$filename."\n",'h2');
+ $blankline=0;
+ }
+ return TeXcode($bincode,$filename,$title);
+ }
+}
+
+# Read file to string without regard for encoding
+sub read_binary {
+ my $filename=shift @_;
+ open(FH,$filename) || return undef;
+ binmode(FH);
+ my $bincode;
+ read(FH,$bincode,-s FH);
+ close(FH);
+ return $bincode;
+}
+
+# Read file from STDIN
+sub _read_stdin {
+ my @text=<STDIN>;
+ my $latexcode=join('',@text);
+ return $latexcode;
+}
+
+###### Error handling
+
+
+# Add warning to list of registered warnings (optionally to be reported at the end)
+sub warning {
+ my ($tex,$text)=@_;
+ $tex->{'warnings'}->{$text}++;
+ $warnings{$text}++;
+}
+
+# Register error and print error message
+sub error {
+ my ($tex,$text,$type)=@_;
+ if (defined $type) {$text=$type.': '.$text;}
+ $errorcount++;
+ $tex->{'errorcount'}++;
+ #print STDERR $text,"\n";
+ if (my $err=$tex->{'errorbuffer'}) {push @$err,$text;}
+ if ($verbose>=0) {_print_error($text);}
+}
+
+# Print error details
+sub error_details {
+ my ($tex,$text)=@_;
+ print STDERR $text,"\n";
+}
+
+###### Parsing routines
+
+
# Parse LaTeX document
sub parse {
my ($tex)=@_;
- if ($htmlstyle && $verbose) {print "<p class=parse>\n";}
+ if ($htmlstyle && $verbose) {print "<div class='parse'><p>\n";}
while (!($tex->{'eof'})) {
- parse_unit($tex,1);
+ _parse_unit($tex,$STATUS_TEXT);
}
- if ($htmlstyle && $verbose) {print "</p>\n";}
+ if ($htmlstyle && $verbose) {print "</p></div>\n";}
}
# Parse one block or unit
-sub parse_unit {
+sub _parse_unit {
# Status:
# 0 = exclude from count
# 1 = text
@@ -539,256 +1319,388 @@ sub parse_unit {
# -1 = float (exclude)
# -2 = strong exclude, ignore begin-end groups
# -3 = stronger exclude, do not parse macro parameters
+ # -4 = ignore everything except end marker: even {
# -9 = preamble (between \documentclass and \begin{document})
my ($tex,$status,$end)=@_;
if (!defined $status) {
- print_error("CRITICAL ERROR: Undefined parser status!");
+ error($tex,'Undefined parser status!','CRITICAL ERROR');
exit;
} elsif (ref($status) eq 'ARRAY') {
- print_error("CRITICAL ERROR: Invalid parser status!");
+ error($tex,'Invalid parser status!','CRITICAL ERROR');
exit;
}
- my $substat;
if ($showstates) {
if (defined $end) {
- $tex->{'printstate'}=':'.$status.':'.$end.':';
+ $tex->{'printstate'}=':'.status_to_text($status).':'.$end.':';
} else {
- $tex->{'printstate'}=':'.$status.':';
+ $tex->{'printstate'}=':'.status_to_text($status).':';
}
flush_next($tex);
}
- while (defined (my $next=next_token($tex))) {
- # parse next token; or tokens until match with $end
+ while (defined (my $next=_next_token($tex))) {
+ # Parse next token until token matches $end
set_style($tex,"ignore");
- if ((defined $end) && ($end eq $next)) {
- # end of unit
- return;
- } elsif (!defined $next) {
- print_error("ERROR: End of file while waiting for ".$end);
- return;
- }
- if ($status==-9 && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) {
+ if ((defined $end) && ($end eq $next)) {return;}
+ # Determine how token should be interpreted
+ if ($status==$STATUS_PREAMBLE && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) {
# \begin{document}
- $status=1;
+ $status=$STATUS_TEXT;
}
- if ($next eq '\documentclass') {
+ if ($status==$STATUS_EXCLUDE_ALL) {
+ # Ignore everything
+ } elsif ($tex->{'type'}==$TOKEN_SPACE) {
+ # space or other code that should be passed through without styling
+ flush_next($tex,' ');
+ } elsif ($next eq '\documentclass') {
# starts preamble
set_style($tex,'document');
- gobble_option($tex);
- gobble_macro_parms($tex,1);
+ __gobble_option($tex);
+ __gobble_macro_parms($tex,1);
while (!($tex->{'eof'})) {
- parse_unit($tex,-9);
+ _parse_unit($tex,$STATUS_PREAMBLE);
}
- } elsif ($tex->{'type'}==666) {
+ } elsif ($tex->{'type'}==$TOKEN_TC) {
# parse TC instructions
- parse_tc($tex);
- } elsif ($tex->{'type'}==1) {
+ _parse_tc($tex,$next);
+ } elsif ($tex->{'type'}==$TOKEN_WORD) {
# word
- if ($status>0) {
- inc_count($tex,$status);
- set_style($tex,'word'.$status);
+ if (my $cnt=status_text_cnt($status)) {
+ _process_word($tex,$next,$status);
+ inc_count($tex,$cnt);
+ set_style($tex,status_to_style($status));
}
} elsif ($next eq '{') {
# {...}
- parse_unit($tex,$status,'}');
- } elsif ($tex->{'type'}==3 && $status==-3) {
+ _parse_unit($tex,$status,'}');
+ } elsif ($next eq '}') {
+ error($tex,'Encountered } without corresponding {.');
+ } elsif ($tex->{'type'}==$TOKEN_MACRO && $status==$STATUS_EXCLUDE_STRONGER) {
+ # ignore macro call
set_style($tex,'ignore');
- } elsif ($tex->{'type'}==3) {
+ } elsif ($tex->{'type'}==$TOKEN_MACRO) {
# macro call
- parse_macro($tex,$next,$status,$substat);
+ _parse_macro($tex,$next,$status);
} elsif ($next eq '$') {
# math inline
- parse_math($tex,$status,6,'$');
+ _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'$');
} elsif ($next eq '$$') {
# math display (unless already in inlined math)
if (!(defined $end && $end eq '$')) {
- parse_math($tex,$status,7,'$$');
+ _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'$$');
}
}
if (!defined $end) {return;}
}
+ if (defined $end) {
+ error($tex,'Reached end of file while waiting for '.$end.'.');
+ }
+}
+
+# Process word with a given status (>0, i.e. counted)
+sub _process_word {
+ my ($tex,$word,$status)=@_;
+ $WordFreq{$word}++;
}
-sub parse_macro {
- my ($tex,$next,$status,$substat)=@_;
+# Parse unit when next token is a macro
+sub _parse_macro {
+ my ($tex,$next,$status)=@_;
+ # substat = parameter status settings
+ my $substat;
if (my $label=$BreakPoints{$next}) {
- if ($tex->{'line'}=~ /^[*]?(\s*\[.*?\])*\s*\{(.+?)\}/ ) {
+ if ($tex->{'line'}=~ /^[*]?(\s*\[.*?\])*\s*\{((.|\{.*\})*)\}/ ) {
$label=$label.': '.$2;
}
- add_subcount($tex,$label);
+ next_subcount($tex,$label);
}
- set_style($tex,$status>0?'command':'exclcommand');
- if ($next eq '\begin' && $status!=-2) {
- parse_begin_end($tex,$status);
- } elsif (($status==-1) && ($substat=$TeXfloatinc{$next})) {
+ set_style($tex,status_is_text($status)?'command':'exclcommand');
+ if ($next eq '\begin' && status_inc_envir($status)) {
+ _parse_begin_end($tex,$status);
+ } elsif ($next eq '\end' && status_inc_envir($status)) {
+ error($tex,'Encountered \end without corresponding \begin');
+ } elsif ($next eq '\verb') {
+ _parse_verb_region($tex,$status);
+ } elsif ( (status_is_text($status) || $status==$STATUS_PREAMBLE)
+ && defined ($substat=$TeXpackageinc{$next}) ) {
+ _parse_include_package($tex,$substat);
+ } elsif (($status==$STATUS_FLOAT) && ($substat=$TeXfloatinc{$next})) {
# text included from float
set_style($tex,'command');
- gobble_macro_parms($tex,$substat);
- } elsif ($status==-9 && defined ($substat=$TeXpreamble{$next})) {
- # parse preamble include macros
+ __gobble_macro_parms($tex,$substat);
+ } elsif ($status==$STATUS_PREAMBLE && defined ($substat=$TeXpreamble{$next})) {
+ # parse preamble include macros
set_style($tex,'command');
- if (defined $TeXheader{$next}) {inc_count($tex,4);}
- gobble_macro_parms($tex,$substat,1);
- } elsif ($status<0) {
- # ignore
- gobble_option($tex);
+ if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);}
+ __gobble_macro_parms($tex,$substat,1);
+ } elsif (status_is_exclude($status)) {
+ # ignore
+ __gobble_option($tex);
} elsif ($next eq '\(') {
# math inline
- parse_math($tex,$status,6,'\)');
+ _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'\)');
} elsif ($next eq '\[') {
# math display
- parse_math($tex,$status,7,'\]');
+ _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'\]');
} elsif ($next eq '\def') {
# ignore \def...
$tex->{'line'} =~ s/^([^\{]*)\{/\{/;
flush_next($tex);
- print_style($1.' ','ignore');
- parse_unit($tex,-2);
+ print_style($1,'ignore');
+ _parse_unit($tex,$STATUS_EXCLUDE_STRONG);
} elsif (defined (my $addsuffix=$TeXfileinclude{$next})) {
- # include file: queue up for parsing
- parse_include_file($tex,$status,$addsuffix);
+ # include file: queue up for parsing
+ _parse_include_file($tex,$status,$addsuffix);
} elsif (defined ($substat=$TeXmacro{$next})) {
# macro: exclude options
- if (defined $TeXheader{$next}) {inc_count($tex,4);}
- gobble_macro_parms($tex,$substat,$status);
- } elsif (defined ($substat=$TeXmacroword{$next})) {
- # count macro as word (or a given number of words)
- inc_count($tex,$status,$substat);
+ if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);}
+ __count_macroword($tex,$next,$status);
+ __gobble_macro_parms($tex,$substat,$status);
+ } elsif (defined __count_macroword($tex,$next,$status)) {
+ # count macro as word (or a given number of words)
set_style($tex,'word'.$status);
} elsif ($next =~ /^\\[^\w\_]/) {
} else {
- gobble_option($tex);
+ __gobble_option($tex);
}
}
-sub parse_tc {
- my ($tex)=@_;
- my $next=$tex->{'next'};
+# Parse TC instruction
+sub _parse_tc {
+ my ($tex,$next)=@_;
set_style($tex,'tc');
flush_next($tex);
- if (!($next=~s/^\%+TC:\s*(\w+)\s*// )) {
- print_error('Warning: TC command should have format %TC:instruction [macro] [parameters]');
+ $next=~s/^\%+TC:\s*(\w+)\s*//i || BLOCK {
+ error($tex,'TC command should have format %TC:instruction [macro] [parameters]');
return;
};
my $instr=$1;
+ $instr=~tr/[A-Z]/[a-z]/;
if ($instr=~/^(break)$/) {
- if ($instr eq 'break') {add_subcount($tex,$next);}
- } elsif ($next=~/^([\\]*\w+)\s+([^\s\n]+)(\s+([0-9]+))?/) {
+ if ($instr eq 'break') {next_subcount($tex,$next);}
+ } elsif ($instr=~/^(incbib|includebibliography)$/) {
+ $includeBibliography=1;
+ apply_include_bibliography();
+ } elsif ($instr eq 'ignore') {
+ __gobble_tc_ignore($tex);
+ } elsif ($instr eq 'endignore') {
+ error($tex,'TC:endignore without corresponding TC:ignore.');
+ } elsif ($instr eq 'newtemplate') {$outputtemplate='';
+ } elsif ($instr eq 'template') {$outputtemplate.=$next;
+ } elsif ($instr eq 'subst') {
+ if ($next=~/^(\\\S+)\s+/) {
+ my $from=$1;
+ my $to=$';
+ $substitutions{$from}=$to;
+ apply_substitution_rule($tex,$from,$to);
+ } else {
+ error($tex,'Invalid TC:subst format.');
+ }
+ } elsif ($next=~/^([\\]*\S+)\s+([^\s\n]+)(\s+(-?[0-9]+))?/) {
# Format = TC:word macro
my $macro=$1;
my $param=$2;
my $option=$4;
- if ($param=~/^\[([0-9,]+)\]$/) {$param=[split(',',$1)];}
- if (($instr eq 'macro') || ($instr eq 'exclude')) {$TeXmacro{$macro}=$param;}
- elsif ($instr eq 'header') {$TeXheader{$macro}=$param;$TeXmacro{$macro}=$param;}
- elsif ($instr eq 'macroword') {$TeXmacroword{$macro}=$param;}
- elsif ($instr eq 'preambleinclude') {$TeXpreamble{$macro}=$param;}
- elsif ($instr eq 'group') {
- $TeXmacro{'begin'.$macro}=$param;
- $TeXgroup{$macro}=$option;
- }
- elsif ($instr eq 'floatinclude') {$TeXfloatinc{$macro}=$param;}
- elsif ($instr eq 'fileinclude') {$TeXfileinclude{$macro}=$param;}
- elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;}
- else {print_error("Warning: Unknown TC command: ".$instr);}
- } elsif ($instr eq 'ignore') {
- tc_ignore_input($tex);
+ if (tc_macro_param_option($instr,$macro,$param,$option)) {}
+ else {error($tex,'Unknown TC command: '.$instr);}
} else {
- print_error("Warning: Invalid TC command format: ".$instr);
+ error($tex,'Invalid TC command format: '.$instr);
}
}
-sub tc_ignore_input {
+# Parse through ignored LaTeX code
+sub __gobble_tc_ignore {
my ($tex)=@_;
set_style($tex,'ignore');
- parse_unit($tex,-3,"%TC:endignore");
+ _parse_unit($tex,$STATUS_EXCLUDE_ALL,'%TC:endignore');
set_style($tex,'tc');
flush_next($tex);
}
-sub parse_math {
- my ($tex,$status,$substat,$end)=@_;
- my $localstyle=$status>0 ? 'mathgroup' : 'exclmath';
- if ($status>0) {inc_count($tex,$substat);}
+# Parse math formulae
+sub _parse_math {
+ my ($tex,$status,$cnt,$end)=@_;
+ my $localstyle;
+ if (status_is_text($status)) {
+ $localstyle='mathgroup';
+ inc_count($tex,$cnt);
+ } else {
+ $localstyle='exclmath';
+ }
set_style($tex,$localstyle);
- parse_unit($tex,0,$end);
+ _parse_unit($tex,$STATUS_IGNORE,$end);
set_style($tex,$localstyle);
}
-sub parse_begin_end {
+# Parse \verb region
+sub _parse_verb_region {
+ my ($tex,$status)=@_;
+ flush_next($tex);
+ set_style($tex,'ignore');
+ if (!($tex->{'line'} =~ s/^[^\s]// )) {
+ error($tex,'Invalid \verb: delimiter required.');
+ }
+ my $dlm=$&;
+ print_style($dlm,'command');
+ if (!($tex->{'line'} =~ s/^([^$dlm]*)($dlm)// )) {
+ error($tex,'Invalid \verb: could not find ending delimiter ('.$dlm.').');
+ }
+ print_style($1,'ignore');
+ print_style($2,'command');
+}
+
+# Parse begin-end group
+sub _parse_begin_end {
my ($tex,$status)=@_;
- my $localstyle=$status>0 ? 'grouping' : 'exclgroup';
- flush_style($tex,$localstyle);
- gobble_option($tex);
- my $groupname;
- if ($tex->{'line'} =~ s/^\{\s*([^\{\}]+)\s*\*?\}[ \t\r\f]*//) {
+ my $localstyle=status_is_text($status) ? 'grouping' : 'exclgroup';
+ flush_next_gobble_space($tex,$localstyle,$status);
+ #__gobble_option($tex); # no option before group name
+ my ($groupname,$next);
+ if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) {
# gobble group type
$groupname=$1;
print_style('{'.$1.'}',$localstyle);
- my $next='begin'.$1;
+ $next='begin'.$groupname;
if (defined (my $substat=$TeXmacro{$next})) {
- gobble_macro_parms($tex,$substat);
+ __gobble_macro_parms($tex,$substat);
}
} else {
- print_error("Warning: BEGIN group without type.");
+ $groupname='???'; $next='???';
+ error($tex,'Encountered \begin without environment name provided.');
}
# find group status (or leave unchanged)
- my $substat;
- defined ($substat=$TeXgroup{$1}) || ($substat=$status);
- if ($status<=0 && $status<$substat) {$substat=$status;}
- if (($status>0) && ($substat==-1)) {
+ my $substat=$TeXgroup{$1};
+ if (!defined $substat) {
+ $substat=$status;
+ if ($strictness>=1) {
+ warning($tex,"Using default rule for group ".$groupname);
+ }
+ } elsif (!status_is_text($status)) {
+ # Do not raise status
+ $substat=__new_status($substat,$status);
+ } elsif ($substat==$STATUS_FLOAT) {
# Count float
- inc_count($tex,5);
- }
- if ($status>0 and $substat>3) {
- # count item, exclude contents
- inc_count($tex,$substat);
+ inc_count($tex,$CNT_COUNT_FLOAT);
+ } elsif (my $cnt=status_to_cnt($substat)) {
+ # Count as given type (was: if $substat>3)
+ inc_count($tex,$cnt);
$substat=0;
+ } else {
+ # Use $substat value as given
}
- parse_unit($tex,$substat,'\end');
- if ($tex->{'line'} =~ s/^\{\s*([^\{\}]+)\s*\}[ \t\r\f]*//) {
- # gobble group type
- flush_style($tex,$localstyle);
- print_style('{'.$1.'}',$localstyle);
+ if ($includeBibliography && $groupname eq 'thebibliography' && $substat>0) {
+ # Add bibliography header
+ inc_count($tex,$CNT_COUNT_HEADER);
+ __count_macroword($tex,$groupname,$STATUS_TEXT_HEADER);
+ }
+ if (!status_inc_envir($substat)) {
+ # Keep parsing until appropriate end group arrives
+ while (!$tex->{'eof'}) {
+ _parse_unit($tex,$substat,'\end');
+ if ($tex->{'line'} =~ s/^\s*\{$groupname\}[ \t\r\f]*//) {
+ # gobble end group parameter
+ flush_next($tex,$localstyle,$status);
+ print_style($&,$localstyle);
+ return;
+ }
+ }
} else {
- print_error("Warning: END group without type while waiting to end ".$groupname.".");
+ # Parse until \end arrives, and check that it matches
+ _parse_unit($tex,$substat,'\end');
+ flush_next_gobble_space($tex,$localstyle,$status);
+ if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) {
+ # gobble group type
+ print_style('{'.$1.'}',$localstyle);
+ if ($groupname ne $1) {
+ error($tex,'Group \begin{'.$groupname.'} ended with end{'.$1.'}.');
+ }
+ } else {
+ error($tex,'Group ended while waiting for \end{'.$groupname.'}.');
+ }
}
}
-sub parse_include_file {
+# Parse and process file inclusion
+sub _parse_include_file {
my ($tex,$status,$addsuffix)=@_;
- $tex->{'line'} =~ s/^\{([^\{\}\s]+)\}// ||
- $tex->{'line'} =~ s/^\s*([^\{\}\%\\\s]+)// ||
+ if ($addsuffix eq 'bbl') {
+ _parse_include_bbl($tex,$status);
return;
+ }
flush_next($tex);
- if ($status>0) {
+ $tex->{'line'} =~ s/^\s*\{([^\{\}\s]+)\}//
+ || $tex->{'line'} =~ s/^\s*([^\{\}\%\\\s]+)//
+ || $tex->{'line'} =~ s/^\s*\{(.+?)\}//
+ || BLOCK {
+ error($tex,'Failed to read or interpret file name for inclusion.');
+ return;
+ };
+ if (status_is_text($status)) {
print_style($&,'fileinclude');
my $fname=$1;
if ($addsuffix==2) {$fname.='.tex';}
elsif ($addsuffix==1 && ($fname=~/^[^\.]+$/)) {$fname.='.tex';}
- if ($includeTeX) {add_file_to_list($fname);}
+ if ($includeTeX) {include_file($tex,$fname);}
} else {
print_style($&,'ignored');
}
}
-sub gobble_option {
- my $tex=shift @_;
- flush_next($tex);
- foreach my $pattern (@MacroOptionPatterns) {
- if ($tex->{'line'}=~s/^($pattern)//) {
- print_style($1,'option');
- return $1;
+# Parse and process bibliography file
+sub _parse_include_bbl {
+ my ($tex,$status)=@_;
+ __gobble_macro_parms($tex,1,$status);
+ if (status_is_text($status) && $includeBibliography) {
+ my $fname=$tex->{'filename'};
+ $fname=~s/\.\w+$/\.bbl/;
+ include_file($tex,$fname);
+ }
+}
+
+# Parse and process package inclusion
+sub _parse_include_package {
+ my ($tex)=@_;
+ set_style($tex,'document');
+ __gobble_option($tex);
+ if ( $tex->{'line'}=~s/^\{((\w+)(\s*,\s*\w+)*)\}// ) {
+ print_style('{'.$1.'}','document');
+ foreach (split(/\s*,\s*/,$1)) {
+ include_package($_);
}
+ } else {
+ _parse_unit($tex,$STATUS_IGNORE);
+ error($tex,"Could not recognise package list, ignoring it instead.");
+ }
+ __gobble_options($tex);
+}
+
+# Count macroword using given status
+sub __count_macroword {
+ my ($tex,$next,$status)=@_;
+ my $n=$TeXmacroword{$next};
+ if (defined $n && (my $cnt=status_text_cnt($status))) {
+ inc_count($tex,$cnt,$n);
+ }
+ return $n;
+}
+
+# Gobble next option, return option or undef if none
+sub __gobble_option {
+ my $tex=shift @_;
+ flush_next_gobble_space($tex);
+ if ($tex->{'line'}=~s/^($MacroOptionPattern)//) {
+ print_style($1,'option');
+ return $1;
}
return undef;
}
-sub gobble_options {
- while (gobble_option(@_)) {}
+# Gobble all options
+sub __gobble_options {
+ while (__gobble_option(@_)) {}
}
-sub gobble_macro_modifier {
+# Gobble macro modifyer (e.g. following *)
+sub __gobble_macro_modifier {
my $tex=shift @_;
flush_next($tex);
if ($tex->{'line'} =~ s/^\*//) {
@@ -798,45 +1710,48 @@ sub gobble_macro_modifier {
return undef;
}
-sub gobble_macro_parms {
+# Gobble macro parameters as specified in parm plus options
+sub __gobble_macro_parms {
my ($tex,$parm,$oldstat)=@_;
- my $i;
+ my $n;
if (ref($parm) eq 'ARRAY') {
- $i=scalar @{$parm};
+ $n=scalar @{$parm};
} else {
- $i=$parm;
- $parm=[0,0,0,0,0,0,0,0,0];
+ $n=$parm;
+ $parm=[($STATUS_IGNORE)x$n];
}
- if ($i>0) {gobble_macro_modifier($tex);}
- gobble_options($tex);
- for (my $j=0;$j<$i;$j++) {
- parse_unit($tex,new_status($parm->[$j],$oldstat));
- gobble_options($tex);
+ if ($n>0) {__gobble_macro_modifier($tex);}
+ __gobble_options($tex);
+ for (my $j=0;$j<$n;$j++) {
+ _parse_unit($tex,__new_status($parm->[$j],$oldstat));
+ __gobble_options($tex);
}
}
-sub new_status {
- my ($substat,$old)=@_;
- if (!defined $old) {return $substat;}
- if ($old==-3 || $substat==-3) {return -3;}
- if ($old==-2 || $substat==-2) {return -2;}
- if ($old==0 || $substat==0) {return 0;}
- if ($old==-9 || $substat==-9) {return -9;}
- if ($old>$substat) {return $old;}
- return $substat;
+# Return new parsing status given old and substatus
+sub __new_status {
+ my ($substat,$oldstat)=@_;
+ if (!defined $oldstat) {return $substat;}
+ foreach my $st (@STATUS_PRIORITY_LIST) {
+ if ($oldstat==$st || $substat==$st) {return $st;}
+ }
+ error($Main,'Could not determine new status in __new_status!','BUG');
+ return $oldstat;
}
-sub next_token {
+# Get next token skipping comments and flushing output buffer
+sub _next_token {
my $tex=shift @_;
my ($next,$type);
- if (defined $tex->{'next'}) {print_style($tex->{'next'}.' ',$tex->{'style'});}
+ my $style=$tex->{'style'};
+ if (defined $tex->{'next'}) {print_style($tex->{'next'},$tex->{'style'});}
$tex->{'style'}=undef;
- while (defined ($next=get_next_token($tex))) {
+ while (defined ($next=__get_next_token($tex))) {
$type=$tex->{'type'};
- if ($type==0) {
+ if ($type==$TOKEN_COMMENT) {
print_style($next,'comment');
- } elsif ($type==9) {
- if ($verbose>0) {line_return(1,$tex);}
+ } elsif ($type==$TOKEN_LINEBREAK) {
+ if ($verbose>0) {line_return(-1,$tex);}
} else {
return $next;
}
@@ -844,9 +1759,10 @@ sub next_token {
return $next;
}
-
-sub get_next_token {
+# Read, interpret and return next token
+sub __get_next_token {
# Token (or token group) category:
+ # -1: space
# 0: comment
# 1: word (or other forms of text or text components)
# 2: symbol (not word, e.g. punctuation)
@@ -859,64 +1775,112 @@ sub get_next_token {
# 666: TeXcount instruction (%TC:instruction)
my $tex=shift @_;
my $next;
- (defined ($next=get_token($tex,'\%+TC:\s*endignore\b[^\r\n]*',666))) && return "%TC:endignore";
- (defined ($next=get_token($tex,'\%+TC:[^\r\n]*',666))) && return $next;
- (defined ($next=get_token($tex,'\%[^\r\n]*',0))) && return $next;
- (defined ($next=get_token($tex,'(\r|\n|\r\n)',9))) && return $next;
- (defined ($next=get_token($tex,'\\\\[\{\}]',2))) && return $next;
- foreach my $pattern (@WordPatterns) {
- (defined ($next=get_token($tex,$pattern,1))) && return $next;
- }
- (defined ($next=get_token($tex,'[\"\'\`:\.,\(\)\[\]!\+\-\*=/\^\_\@\<\>\~\#\&]',2))) && return $next;
- (defined ($next=get_token($tex,'\\\\([a-zA-Z_]+|[^a-zA-Z_])',3))) && return $next;
- (defined ($next=get_token($tex,'[\{\}]',4))) && return $next;
- (defined ($next=get_token($tex,'[\[\]]',5))) && return $next;
- (defined ($next=get_token($tex,'\$\$',6))) && return $next;
- (defined ($next=get_token($tex,'\$',6))) && return $next;
- (defined ($next=get_token($tex,'.',999))) && return $next;
- (defined ($next=get_token($tex,'[^\s]+',999))) && return $next;
- $tex->{'eof'}=1;
+ my $ch;
+ while (!$tex->{'eof'}) {
+ $ch=substr($tex->{'line'},0,1);
+ if ($ch eq '') {
+ if (!more_texcode($tex)) {$tex->{'eof'}=1;}
+ next;
+ } elsif ($ch=~/^[ \t\f]/) {
+ $tex->{'line'}=~s/^([ \t\f]+)//;
+ return __set_token($tex,$1,$TOKEN_SPACE);
+ } elsif ($ch eq "\n" || $ch eq "\r") {
+ $tex->{'line'}=~s/^(\r\n?|\n)//;
+ return __set_token($tex,$1,$TOKEN_LINEBREAK);
+ } elsif ($tex->{'line'}=~s/^($WordPattern)//) {
+ return __set_token($tex,$1,$TOKEN_WORD);
+ } elsif ($ch eq '$') {
+ $tex->{'line'}=~s/^(\$\$?)//;
+ return __set_token($tex,$1,$TOKEN_MATH);
+ } elsif ($ch eq '{' || $ch eq '}') {
+ return __get_chtoken($tex,$ch,$TOKEN_BRACE);
+ } elsif ($ch eq '[' || $ch eq ']') {
+ return __get_chtoken($tex,$ch,$TOKEN_BRACKET);
+ } elsif ($ch=~/^['"`:.,()[]!+-*=\/^_@<>~#&]$/) {
+ return __get_chtoken($tex,$ch,$TOKEN_SYMBOL);
+ } elsif ($ch eq '%') {
+ if ($tex->{'line'}=~s/^(\%+TC:\s*endignore\b[^\r\n]*)//i) {
+ __set_token($tex,$1,$TOKEN_TC);
+ return "%TC:endignore";
+ }
+ if ($tex->{'line'}=~s/^(\%+[tT][cC]:[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_TC);}
+ if ($tex->{'line'}=~s/^(\%+[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_COMMENT);}
+ return __get_chtoken($tex,$ch,$TOKEN_COMMENT);
+ } elsif ($ch eq '\\') {
+ if ($tex->{'line'}=~s/^(\\[{}%])//) {return __set_token($tex,$1,$TOKEN_SYMBOL);}
+ if ($tex->{'line'}=~s/^(\\([a-zA-Z_]+|[^a-zA-Z_]))//) {return __set_token($tex,$1,$TOKEN_MACRO);}
+ return __get_chtoken($tex,$ch,$TOKEN_END);
+ } else {
+ return __get_chtoken($tex,$ch,$TOKEN_END);
+ }
+ }
return undef;
}
-sub get_token {
- my ($tex,$regexp,$type)=@_;
- if (!defined $regexp) {print_error("ERROR in get_token: undefined regex.");}
- if (!defined $tex->{'line'}) {print_error("ERROR in get_token: undefined tex-line. ".$tex->{'next'});}
- if ( $tex->{'line'} =~ s/^($regexp)[ \t\r\f]*// ) {
- $tex->{'next'}=$1;
- $tex->{'type'}=$type;
- return $1;
- }
- return undef;
+# Set next token and its type
+sub __set_token {
+ my ($tex,$next,$type)=@_;
+ $tex->{'next'}=$next;
+ $tex->{'type'}=$type;
+ return $next;
+}
+
+# Set character token and remove from line
+sub __get_chtoken {
+ my ($tex,$ch,$type)=@_;
+ $tex->{'line'}=substr($tex->{'line'},1);
+ $tex->{'next'}=$ch;
+ $tex->{'type'}=$type;
+ return $ch;
}
+
###### Count handling routines
+
+## Make new count object
+# The "count object" is a hash containing
+# - title: the title of the count (name of file, section, ...)
+# - counts: a list of numbers (the counts: files, text words, ...)
+# upon creation, but where the element
+# - subcounts: list of count objects (added by the TeX object)
+# may exist if the count contains subcounts. The elements of the
+# count are (by their index):
+# 0 = #files: counts the number of files
+# 1 = text words: counts the number of words in the text
+# 2 = header words: number of words in headers
+# 3 = caption words: number of words in float captions
+# 4 = #headers: number of headers
+# 5 = #floats: number of tables, figures, floats, etc.
+# 6 = #inline formulae: number of math elements in text ($...$)
+# 7 = #displayed formulae: number of displayed equations
sub new_count {
my ($title)=@_;
- my @cnt=(0,0,0,0,0,0,0,0);
- my %count=('count'=>\@cnt,'title'=>$title);
+ my @cnt=(0) x $SIZE_CNT;
+ my %count=('counts'=>\@cnt,'title'=>$title);
# files, text words, header words, float words,
# headers, floats, math-inline, math-display;
return \%count;
}
+# Increment TeX count for a given count type
sub inc_count {
- my ($tex,$type,$value)=@_;
- my $count=$tex->{'count'};
+ my ($tex,$cnt,$value)=@_;
+ my $count=$tex->{'subcount'};
if (!defined $value) {$value=1;}
- ${$count->{'count'}}[$type]+=$value;
+ ${$count->{'counts'}}[$cnt]+=$value;
}
+# Get count value for a given count type
sub get_count {
- my ($count,$type)=@_;
- return ${$count->{'count'}}[$type];
+ my ($count,$cnt)=@_;
+ return ${$count->{'counts'}}[$cnt];
}
-sub total_count {
- my ($count)=@_;
+# Compute sum count for a count object
+sub get_sum_count {
+ my $count=shift @_;
my $sum=0;
for (my $i=scalar(@sumweights);$i-->0;) {
$sum+=get_count($count,$i+1)*$sumweights[$i];
@@ -924,223 +1888,509 @@ sub total_count {
return $sum;
}
-sub print_count {
- my ($count,$header)=@_;
- if ($briefsum && @sumweights) {
- print_count_total($count,$header);
- } elsif ($briefsum) {
- if ($htmlstyle) {print "<p class='briefcount'>";}
- print_count_brief($count,$header);
- if ($htmlstyle) {print "</p>\n";}
+# Returns the number of subcounts
+sub number_of_subcounts {
+ my $count=shift @_;
+ if (my $subcounts=$count->{'subcounts'}) {
+ return scalar(@{$subcounts});
} else {
- print_count_details($count,$header);
+ return 0;
}
}
-sub print_count_with_header {
- my ($count,$header)=@_;
- if (!defined $header) {$header=$count->{'title'};}
- if (!defined $header) {$header="";}
- return $count,$header;
-}
+# Is a null count? (counts 1-7 zero, title starts with _)
+sub _count_is_null {
+ my $count=shift @_;
+ if (!$count->{'title'}=~/^_/) {return 0;}
+ for (my $i=1;$i<$SIZE_CNT;$i++) {
+ if (get_count($count,$i)>0) {return 0;}
+ }
+ return 1;
+}
-sub print_count_total {
- my ($count,$header)=print_count_with_header(@_);
- if ($htmlstyle) {print "<p class='count'>".$header;}
- print total_count($count);
- if ($htmlstyle) {print "</p>\n";}
- else {print ": ".$header;}
+# Add one count to another
+sub _add_to_count {
+ my ($a,$b)=@_;
+ for (my $i=0;$i<$SIZE_CNT;$i++) {
+ ${$a->{'counts'}}[$i]+=${$b->{'counts'}}[$i];
+ }
}
-sub print_count_brief {
- my ($count,$header)=print_count_with_header(@_);
- my $cnt=$count->{'count'};
- print ${$cnt}[1]."+".${$cnt}[2]."+".${$cnt}[3].
- " (".${$cnt}[4]."/".${$cnt}[5]."/".${$cnt}[6]."/".${$cnt}[7].") ".
- $header;
+# Add subcount to sum count and prepare new subcount
+sub next_subcount {
+ my ($tex,$title)=@_;
+ add_to_total($tex->{'countsum'},$tex->{'subcount'});
+ $tex->{'subcount'}=new_count($title);
+ return $tex->{'countsum'};
}
-sub print_count_details {
- my ($count,$header)=print_count_with_header(@_);
- if ($htmlstyle) {print "<dl class='count'>\n";}
- if (defined $header) {
- formatprint($header."\n",'dt','header');
+# Add count to total as subcount
+sub add_to_total {
+ my ($total,$count)=@_;
+ _add_to_count($total,$count);
+ if (!_count_is_null($count)) {
+ push @{$total->{'subcounts'}},$count;
+ $count->{'parentcount'}=$total;
}
- if (get_count($count,0)>1) {
- formatprint($countlabel[0].': ','dt');
- formatprint(get_count($count,0)."\n",'dd');
+}
+
+
+###### Result output routines
+
+
+# Close the output, e.g. adding HTML tail
+sub Close_Output {
+ if ($htmlstyle>1) {
+ html_tail();
}
- if (@sumweights) {
- formatprint('Sum count: ','dt');
- formatprint(total_count($count)."\n",'dd');
+}
+
+# Report if there were any errors occurring during parsing
+sub Report_Errors {
+ if (defined $outputtemplate) {return;}
+ if ( !$briefsum && !$totalflag && $verbose>=0 ) {
+ foreach (keys %warnings) {formatprint($_,"p","nb");print "\n";}
}
- for (my $i=1;$i<8;$i++) {
- formatprint($countlabel[$i].': ','dt');
- formatprint(get_count($count,$i)."\n",'dd');
+ if ($errorcount==0) {return;}
+ if ($briefsum && $totalflag) {print " ";}
+ if ($htmlstyle) {
+ print "<div class='error'><p>\n";
+ print "There were ".$errorcount." error(s) reported!\n";
+ print "</p></div>\n";
+ } elsif ($briefsum && $totalflag) {
+ print "(errors:".$errorcount.")";
+ } else {
+ print "(errors:".$errorcount.")\n";
}
- my $subcounts=$count->{'subcounts'};
- if ($showsubcounts && defined $subcounts && scalar(@{$subcounts})>1) {
- formatprint("Subcounts: text+headers+captions (#headers/#floats/#inlines/#displayed)\n",'dt');
- foreach my $subcount (@{$subcounts}) {
- if ($htmlstyle) {print "<dd class='briefcount'>";}
- print_count_brief($subcount);
- if ($htmlstyle) {print "</dd>";}
- print "\n";
+}
+
+# Print word frequencies (as text only)
+sub print_word_freq {
+ my ($word,$wd,$freq,%Freq,%Word,%Class);
+ my %regs;
+ foreach my $cg (@AlphabetScripts,@LogogramScripts) {
+ $regs{$cg}=qr/\p{$cg}/;
+ }
+ my $sum=0;
+ for $word (keys %WordFreq) {
+ $wd=lc $word;
+ $Freq{$wd}+=$WordFreq{$word};
+ $sum+=$WordFreq{$word};
+ $Word{$wd}=__lc_merge($word,$Word{$wd});
+ }
+ if ($htmlstyle) {print "<table class='stat'>\n<thead>\n";}
+ __print_word_freq("Word","Freq","th");
+ if ($htmlstyle) {print "</thead>\n";}
+ if ($optionWordClassFreq>0) {
+ for $word (keys %Freq) {$Class{__word_class($word,\%regs)}+=$Freq{$word};}
+ __print_sorted_freqs('langstat',\%Class);
+ }
+ if ($htmlstyle) {print "<tbody class='sumstat'>\n";}
+ __print_word_freq("All words",$sum,,'td','sum');
+ if ($htmlstyle) {print "</tbody>\n";}
+ if ($optionWordFreq>0) {__print_sorted_freqs('wordstat',\%Freq,\%Word,$optionWordFreq);}
+ if ($htmlstyle) {print "</table>\n";}
+}
+
+# Merge to words which have the same lower case
+sub __lc_merge {
+ my ($word,$w)=@_;
+ if (defined $w) {
+ for (my $i=length($word);$i-->0;) {
+ if (substr($word,$i,1) ne substr($w,$i,1)) {
+ substr($word,$i,1)=lc substr($word,$i,1);
+ }
}
}
- if ($htmlstyle) {print "</dl>\n";}
+ return $word;
}
-sub add_count {
- my ($a,$b)=@_;
- for (my $i=0;$i<8;$i++) {
- ${$a->{'count'}}[$i]+=${$b->{'count'}}[$i];
+# Return the word class based on script groups it contains
+sub __word_class {
+ my ($wd,$regs)=@_;
+ my @classes;
+ $wd=~s/\\\w+({})?/\\{}/g;
+ foreach my $name (keys %{$regs}) {
+ if ($wd=~$regs->{$name}) {push @classes,$name;}
}
+ my $cl=join('+',@classes);
+ if ($cl) {}
+ elsif ($wd=~/\\/) {$cl='(macro)';}
+ else {$cl='(unidentified)';}
+ return $cl;
}
-sub add_subcount {
- my ($tex,$title)=@_;
- add_count($tex->{'countsum'},$tex->{'count'});
- push @{$tex->{'countlist'}},$tex->{'count'};
- $tex->{'count'}=new_count($title);
- return $tex->{'countsum'};
+# Print table body containing word frequencies
+sub __print_sorted_freqs {
+ my ($class,$Freq,$Word,$min)=@_;
+ my $sum=0;
+ my ($word,$wd,$freq);
+ if (!defined $min) {$min=0;}
+ if ($htmlstyle) {print "<tbody class='",$class,"'>\n";}
+ else {print "---\n";}
+ for $wd (sort {$Freq->{$b} <=> $Freq->{$a}} keys %{$Freq}) {
+ if (defined $Word) {$word=$Word->{$wd};} else {$word=$wd;}
+ $freq=$Freq->{$wd};
+ if ($freq>=$min) {
+ $sum+=$freq;
+ __print_word_freq($word,$freq);
+ }
+ }
+ if ($min>0) {__print_word_freq("Sum of subset",$sum,'td','sum');}
+ if ($htmlstyle) {print "</tbody>\n";}
+}
+
+# Print one word freq line
+sub __print_word_freq {
+ my ($word,$freq,$tag,$class)=@_;
+ if (!defined $tag) {$tag='td';}
+ if (defined $class) {$class=' class=\''.$class.'\'';} else {$class='';}
+ if ($htmlstyle) {
+ print "<tr",$class,"><",$tag,">",$word,"</",$tag,"><",$tag,">",$freq,"</",$tag,"></tr>\n";
+ } else {
+ print $word,": ",$freq,"\n";
+ }
}
###### Printing routines
-sub set_style {
- my ($tex,$style)=@_;
- if (!(($tex->{'style'}) && ($tex->{'style'} eq '-'))) {$tex->{'style'}=$style;}
+# Print text using given style/colour
+sub print_with_style {
+ my ($text,$style,$colour)=@_;
+ if ($style eq ' ') {
+ print text_to_print($text);
+ } elsif ($style eq '') {
+ print text_to_print($text);
+ error($Main,'Empty style should not occur in print_with_style!','BUG');
+ } elsif ($htmlstyle) {
+ print "<span class='$style'>".text_to_print($text).'</span>';
+ } else {
+ ansiprint(text_to_print($text),$colour);
+ if ($style=~/$separatorstyleregex/) {print $separator;}
+ }
}
-sub flush_style {
- my ($tex,$style)=@_;
- set_style($tex,$style);
- flush_next($tex);
+# Prepare text string for print: convert special characters
+sub text_to_print {
+ my $text=join('',@_);
+ if ($htmlstyle) {
+ $text=~s/&/&amp;/g;
+ $text=~s/</&lt;/g;
+ $text=~s/>/&gt;/g;
+ $text=~s/[ \t]{2}/\&nbsp; /g;
+ }
+ return $text;
}
-sub line_return {
- my ($blank,$tex)=@_;
- if ($blank>$blankline) {
- if ((defined $tex) && @sumweights) {
- my $num=total_count($tex->{'count'});
- print_style(" [".$num."]","sumcount");
- }
- linebreak();
- $blankline++;
+# Print text, using appropriate tags for HTML
+sub formatprint {
+ my ($text,$tag,$class)=@_;
+ my $break=($text=~s/(\r\n?|\n)$//);
+ if ($htmlstyle && defined $tag) {
+ print '<'.$tag;
+ if ($class) {print " class='".$class."'";}
+ print '>'.text_to_print($text).'</'.$tag.'>';
+ } else {
+ print text_to_print($text);
}
+ if ($break) {print "\n";}
}
+# Add a line break to output
sub linebreak {
if ($htmlstyle) {print "<br>\n";} else {print "\n";}
}
-sub print_style {
- my ($text,$style,$state)=@_;
- (($verbose>=0) && (defined $text) && (defined $style)) || return 0;
- my $colour;
- ($colour=$STYLE{$style}) || return;
- if (($colour) && !($colour eq '-')) {
- print_with_style($text,$style,$colour);
- if ($state) {
- print_style($state,'state');
- }
- $blankline=-1;
- return 1;
+###### Routines for printing count summary
+
+
+# Print count summary for a count object
+sub print_count {
+ my ($count,$class)=@_;
+ if ($htmlstyle) {print "<div class='".($class||'count')."'>\n";}
+ if ($outputtemplate) {
+ _print_count_template($count,$outputtemplate);
+ } elsif ($briefsum && @sumweights) {
+ _print_sum_count($count);
+ } elsif ($briefsum) {
+ if ($htmlstyle) {print "<p class='count'>";}
+ _print_count_brief($count);
+ if ($htmlstyle) {print "</p>\n";}
} else {
- return 0;
+ _print_count_details($count);
}
+ if ($htmlstyle) {print "</div>\n";}
}
-sub print_error {
- my $text=shift @_;
- $errorcount++;
- if ($verbose>=0) {
- line_return(1);
- print_style("!!! ".$text." !!!",'error');
- line_return(1);
+# Return count,header,... list filling in header if missing
+sub __count_and_header {
+ my $count=shift @_;
+ my $header=__count_header($count);
+ return $count,$header,@_;
+}
+
+# Return count title or "" if missing
+sub __count_header {
+ my $count=shift @_;
+ return $count->{'title'}||'';
+}
+
+# Print total count (sum) for a given count object
+sub _print_sum_count {
+ my ($count,$header)=__count_and_header(@_);
+ if ($htmlstyle) {print "<p class='count'>".text_to_print($header).": ";}
+ print get_sum_count($count);
+ if ($htmlstyle) {print "</p>\n";}
+ else {print ": ".text_to_print($header);}
+ print "\n";
+}
+
+# Print brief summary of count object
+sub _print_count_brief {
+ my ($count,$header,$tag1,$tag2)=__count_and_header(@_);
+ my @cnt=@{$count->{'counts'}};
+ if ($htmlstyle && $tag1) {print "<".$tag1.">";}
+ print $cnt[$CNT_WORDS_TEXT]."+".$cnt[$CNT_WORDS_HEADER]."+".$cnt[$CNT_WORDS_CAPTION].
+ " (".$cnt[$CNT_COUNT_HEADER]."/".$cnt[$CNT_COUNT_FLOAT].
+ "/".$cnt[$CNT_COUNT_INLINEMATH]."/".$cnt[$CNT_COUNT_DISPLAYMATH].")";
+ if ($htmlstyle && $tag2) {
+ print "</".$tag1."><".$tag2.">";
+ $tag1=$tag2;
+ } else {print " ";}
+ print text_to_print($header);
+ if ($htmlstyle && $tag1) {print "</".$tag1.">";}
+ if ($finalLineBreak) {print "\n";}
+}
+
+# Print detailed summary of count object
+sub _print_count_details {
+ my ($count,$header)=__count_and_header(@_);
+ if ($htmlstyle) {print "<ul class='count'>\n";}
+ if ($header) {formatprint($header."\n",'li','header');}
+ if (my $tex=$count->{'TeXcode'}) {
+ if (!defined $encoding) {formatprint('Encoding: '.$tex->{'encoding'}."\n",'li');}
+ }
+ if (@sumweights) {formatprint('Sum count: '.get_sum_count($count)."\n",'li');}
+ for (my $i=1;$i<$SIZE_CNT;$i++) {
+ formatprint($countlabel[$i].': '.get_count($count,$i)."\n",'li');
}
+ if (get_count($count,$CNT_FILE)>1) {
+ formatprint($countlabel[$CNT_FILE].': '.get_count($count,$CNT_FILE)."\n",'li');
+ }
+ my $subcounts=$count->{'subcounts'};
+ if ($showsubcounts && defined $subcounts && scalar(@{$subcounts})>=$showsubcounts) {
+ formatprint("Subcounts:\n",'li');
+ if ($htmlstyle) {print "<span class='subcount'>\n";}
+ formatprint(" text+headers+captions (#headers/#floats/#inlines/#displayed)\n",'li','fielddesc');
+ foreach my $subcount (@{$subcounts}) {
+ print ' ';
+ _print_count_brief($subcount,'li');
+ }
+ if ($htmlstyle) {print "</span>\n";}
+ }
+ if ($htmlstyle) {print "</ul>\n";} else {print "\n";}
+}
+
+# Print summary of count object using template
+sub _print_count_template {
+ my ($count,$header,$template)=__count_and_header(@_);
+ $template=~s/\\n/\n/g;
+ if ($htmlstyle) {$template=~s/\n/<br>/g;}
+ my ($subtemplate,$posttemplate);
+ while ($template=~/\{SUB\?((.*?)\|)?(.*?)(\|(.*?))?\?SUB\}/is) {
+ __print_count_using_template($count,$`);
+ if (number_of_subcounts($count)>1) {
+ if (defined $2) {print $2;}
+ __print_subcounts_using_template($count,$3);
+ if (defined $5) {print $5;}
+ }
+ $template=$';
+ }
+ __print_count_using_template($count,$template);
}
-sub formatprint {
- my ($text,$tag,$class)=@_;
- my $break=($text=~s/\n$//);
- if ($htmlstyle && defined $tag) {
- print '<'.$tag;
- if ($class) {print " class='".$class."'";}
- print '>'.$text.'</'.$tag.'>';
+# Print counts using template
+sub __print_count_using_template {
+ my ($count,$template)=@_;
+ for (my $i=0;$i<$SIZE_CNT;$i++) {
+ $template=__process_template($template,$i,get_count($count,$i));
+ }
+ $template=~s/\{VER\}/$versionnumber/gi;
+ # TODO: Should base warnings and errors on TeXcode or Main object
+ $template=__process_template($template,"W|WARN|WARNING|WARNINGS",length(%warnings));
+ $template=__process_template($template,"E|ERR|ERROR|ERRORS",$errorcount);
+ $template=__process_template($template,"S|SUM",get_sum_count($count));
+ $template=__process_template($template,"T|TITLE",$count->{'title'}||"");
+ $template=__process_template($template,"SUB",number_of_subcounts($count));
+ print $template;
+}
+
+# Print subcounts using template
+sub __print_subcounts_using_template {
+ my ($count,$template)=@_;
+ my $subcounts=$count->{'subcounts'};
+ if ($template && defined $subcounts && scalar(@{$subcounts})>=$showsubcounts) {
+ foreach my $subcount (@{$subcounts}) {
+ __print_count_using_template($subcount,$template);
+ }
+ }
+}
+
+# Process template for specific label
+sub __process_template {
+ my ($template,$label,$value)=@_;
+ if ($value) {
+ $template=~s/\{($label)\?(.*?)(\|(.*?))?\?(\1)\}/$2/gis;
} else {
- print $text;
+ $template=~s/\{($label)\?(.*?)\|(.*?)\?(\1)\}/$3/gis;
+ $template=~s/\{($label)\?(.*?)\?(\1)\}//gis;
}
- if ($break) {print "\n";}
+ if (!defined $value) {$value="";}
+ $template=~s/\{($label)\}/$value/gis;
+ return $template;
}
+
+###### Routines for printing parsing details
+
+
+# Print next token
sub flush_next {
- my $tex=shift @_;
+ my ($tex,$style)=@_;
+ my $ret=undef;
+ if (defined $style && $style ne '') {
+ set_style($tex,$style);
+ }
if (defined $tex->{'next'}) {
- print_style($tex->{'next'}.' ',$tex->{'style'},$tex->{'printstate'});
+ $ret=print_style($tex->{'next'},$tex->{'style'},$tex->{'printstate'});
}
$tex->{'printstate'}=undef;
$tex->{'style'}='-';
+ return $ret;
+}
+
+# Print next token and gobble following spaces
+sub flush_next_gobble_space {
+ my ($tex,$style,$status)=@_;
+ my $ret=flush_next($tex,$style);
+ if (!defined $ret) {$ret=0;}
+ if (!defined $status) {$status=$STATUS_IGNORE;}
+ my $prt=($verbose>0);
+ if ($tex->{'line'}=~s/^([ \t\f]*)(\r\n?|\n)([ \t\f]*)//) {
+ if (!$prt) {
+ } elsif ($verbose>2 || $ret) {
+ print $1;
+ line_return(-1,$tex);
+ my $space=$3;
+ if ($htmlstyle) {$space=~s/ /\&nbsp;/g;}
+ print $space;
+ } else {
+ line_return(0,$tex);
+ }
+ } elsif ($tex->{'line'}=~s/^([ \t\f]*)//) {
+ if ($prt) {print $1;}
+ }
}
+# Set presentation style
+sub set_style {
+ my ($tex,$style)=@_;
+ if (!(($tex->{'style'}) && ($tex->{'style'} eq '-')) && $style ne '') {$tex->{'style'}=$style;}
+}
-# Close the output, e.g. adding HTML tail
-sub Close_Output {
- if ($htmlstyle>1) {
- html_tail();
+# Print text using the given style, and log state if given
+sub print_style {
+ my ($text,$style,$state)=@_;
+ (($verbose>=0) && (defined $text) && (defined $style)) || return 0;
+ my $colour;
+ ($colour=$STYLE{$style}) || return;
+ if (($colour) && !($colour eq '-')) {
+ print_with_style($text,$style,$colour);
+ if ($state) {print_style($state,'state');}
+ if ($style ne "cumsum") {$blankline=-1;}
+ return 1;
+ } else {
+ return 0;
}
}
-
-# Report if there were any errors occurring during parsing
-sub Report_ErrorCount {
- if ($errorcount==0) {return;}
- if ($briefsum && $totalflag) {print " ";}
- if ($htmlstyle) {
- print_error("Errors:".$errorcount,"p","error");
- } else {
- print "(errors:".$errorcount.")";
+# Conditional line return
+sub line_return {
+ my ($blank,$tex)=@_;
+ if ($blank<0 && $verbose<3) {$blank=1;}
+ if ($blank<0 || $blank>$blankline) {
+ if ((defined $tex) && @sumweights) {
+ my $num=get_sum_count($tex->{'subcount'});
+ print_style(" [".$num."]","cumsum");
+ }
+ linebreak();
+ $blankline++;
}
}
+# Print error message
+sub _print_error {
+ my $text=shift @_;
+ line_return(1);
+ print_style("!!! ".$text." !!!",'error');
+ line_return(1);
+}
+
+# Print errors in buffer and delete errorbuffer
+sub flush_errorbuffer {
+ my $source=shift @_;
+ my $err=$source->{'errorbuffer'} || return;
+ foreach (@$err) {_print_error($_);}
+ $source->{'errorbuffer'}=undef;
+}
+
+###### Print help on style/colour codes
-sub print_help_style {
+
+# Print output style codes if conditions are met
+sub conditional_print_help_style {
+ if ($showcodes) {_print_help_style();}
+ return $showcodes;
+}
+
+# Print help on output styles
+sub _print_help_style {
if ($verbose<=0) {return;}
+ if ($htmlstyle) {print "<div class='stylehelp'><p>";}
formatprint("Format/colour codes of verbose output:","h2");
print "\n\n";
- if ($htmlstyle) {print "<p class='stylehelp'>";}
- help_style_line('Text which is counted',"word1","counted as text words");
- help_style_line('Header and title text',"word2","counted as header words");
- help_style_line('Caption text and footnotes',"word3","counted as caption words");
- help_style_line("Ignored text or code","ignore","excluded or ignored");
- help_style_line('\documentclass',"document","document start, beginning of preamble");
- help_style_line('\macro',"command","macro not counted, but parameters may be");
- help_style_line('\macro',"exclcommand","macro in excluded region");
- help_style_line("[Macro options]","option","not counted");
- help_style_line('\begin{group} \end{group}',"grouping","begin/end group");
- help_style_line('\begin{group} \end{group}',"exclgroup","begin/end group in excluded region");
- help_style_line('$ $',"mathgroup","counted as one equation");
- help_style_line('$ $',"exclmath","equation in excluded region");
- help_style_line('% Comments',"comment","not counted");
- help_style_line('%TC:TeXcount instructions',"tc","not counted");
- help_style_line("File to include","fileinclude","not counted but file may be counted later");
+ _help_style_line('Text which is counted',"word1","counted as text words");
+ _help_style_line('Header and title text',"word2","counted as header words");
+ _help_style_line('Caption text and footnotes',"word3","counted as caption words");
+ _help_style_line("Ignored text or code","ignore","excluded or ignored");
+ _help_style_line('\documentclass',"document","document start, beginning of preamble");
+ _help_style_line('\macro',"command","macro not counted, but parameters may be");
+ _help_style_line('\macro',"exclcommand","macro in excluded region");
+ _help_style_line("[Macro options]","option","not counted");
+ _help_style_line('\begin{group} \end{group}',"grouping","begin/end group");
+ _help_style_line('\begin{group} \end{group}',"exclgroup","begin/end group in excluded region");
+ _help_style_line('$ $',"mathgroup","counted as one equation");
+ _help_style_line('$ $',"exclmath","equation in excluded region");
+ _help_style_line('% Comments',"comment","not counted");
+ _help_style_line('%TC:TeXcount instructions',"tc","not counted");
+ _help_style_line("File to include","fileinclude","not counted but file may be counted later");
if ($showstates) {
- help_style_line('[state]',"state","internal TeXcount state");
+ _help_style_line('[state]',"state","internal TeXcount state");
}
if (@sumweights) {
- help_style_line('[sumcount]',"sumcount","cumulative sum count");
+ _help_style_line('[cumsum]',"cumsum","cumulative sum count");
}
- help_style_line("ERROR","error","TeXcount error message");
- if ($htmlstyle) {print "</p>";}
+ _help_style_line("ERROR","error","TeXcount error message");
+ if ($htmlstyle) {print "</p></div>";}
print "\n\n";
}
-sub help_style_line {
+# Print one line of help
+sub _help_style_line {
my ($text,$style,$comment)=@_;
if ($htmlstyle) {
- $comment="&nbsp;&nbsp;....&nbsp;&nbsp;".$comment;
+ $comment="&nbsp;&nbsp;....&nbsp;&nbsp;".text_to_print($comment);
} else {
$comment=" .... ".$comment;
}
@@ -1150,27 +2400,175 @@ sub help_style_line {
}
}
-# Print output style codes if conditions are met
-sub conditional_print_help_style {
- if ($showcodes) {print_help_style();}
- return $showcodes;
+
+###### Help routines
+
+
+# Print TeXcount version
+sub print_version {
+ wprintstringdata('Version');
}
-###### HTML routines
+# Print TeXcount reference text
+sub print_reference {
+ wprintstringdata('Reference');
+}
+# Print TeXcount licence text
+sub print_license {
+ wprintstringdata('License');
+}
+# Print TeXcount parameter list
+sub print_syntax {
+ wprintstringdata('OptionsHead');
+ wprintstringdata('Options','@ - :');
+}
+# Print complete TeXcount help
+sub print_help {
+ print_help_title();
+ print_syntax();
+ print_help_text();
+ print_reference();
+}
+
+# Print help title
+sub print_help_title {
+ wprintstringdata('HelpTitle');
+}
+
+# Print help text
+sub print_help_text {
+ wprintstringdata('HelpText');
+ wprintstringdata('TCinstructions');
+}
+
+# Print help on specific macro or group
+sub print_help_on {
+ my $arg=shift @_;
+ my $def;
+ my %rules=(
+ '\documentclass' => 'Initiates LaTeX document preamble.',
+ '\begin' => 'Treatmend depends on group handling rules.',
+ '\def' => 'Excluded from count.',
+ '\verb' => 'Strong exclude for enclosed region.',
+ '$' => 'Opens or closes inlined equation',
+ '$$' => 'Opens or closes displayed equation.',
+ '\(' => 'Opens inlined equation.',
+ '\)' => 'Closes inlined equation initiated by \(.',
+ '\[' => 'Opens displayed equation.',
+ '\]' => 'Closes displayed equation initiated by \[.');
+ if (!defined $arg || $arg=~/^\s*$/) {
+ print "Specify macro or group name after the -h= option.\n";
+ return;
+ }
+ if ($def=$rules{$arg}) {
+ print "Special rule (hard coded) for $arg\n";
+ print $def."\n";
+ } elsif ($arg=~/^\\/) {
+ if ($def=$TeXfileinclude{$arg}) {
+ print "Rule for macro $arg\n";
+ print "Takes file name as parameter which is included in document.\n";
+ } elsif ($def=$TeXmacro{$arg}) {
+ print "Rule for macro $arg\n";
+ _print_rule_macro($arg,$def);
+ if ($def=$TeXheader{$arg}) {
+ print "This macro is counted as a header\n";
+ }
+ if ($def=$TeXfloatinc{$arg}) {
+ print "This macro is also counted inside floats as captions.\n";
+ }
+ } elsif ($def=$TeXmacroword{$arg}) {
+ print "Rule for macro $arg\n";
+ print "Count as ".$def." word(s).\n";
+ } else {
+ print "No macro rule defined for $arg.\nParameters treated as surrounding text.\n";
+ }
+ } else {
+ if ($def=$TeXgroup{$arg}) {
+ print "Rule for group $arg\n";
+ _print_rule_group($arg,$def);
+ } else {
+ print "No default group rule defined for $arg.\nContent handled as surrounding text.\n";
+ }
+ }
+}
+
+# Print macro handling rule
+sub _print_rule_macro {
+ my ($arg,$def)=@_;
+ my %rules=(
+ 0 => 'Exclude from count',
+ 1 => 'Count as text',
+ 2 => 'Count as header',
+ 3 => 'Count as caption',
+ -1 => 'Exclude as float (i.e. include captions)',
+ -2 => 'Strong exclude (ignore begin-end groups)',
+ -3 => 'Stronger exclude (ignore all macros)',
+ -9 => 'Exclude as preamble');
+ if (ref($def) eq 'ARRAY') {
+ print "Takes ".scalar(@{$def})." parameter(s):\n";
+ foreach my $i (@{$def}) {
+ print " - ".$rules{$i}."\n";
+ }
+ } else {
+ print "Takes ".$def." parameter(s), not included in counts.\n";
+ }
+}
+
+# Print group handling rule
+sub _print_rule_group {
+ my ($arg,$def)=@_;
+ my %rules=(
+ 0 => 'Not included',
+ 1 => 'Text, words included in text count',
+ 2 => 'Header, words included in header count',
+ 3 => 'Float caption, words included in float caption count',
+ 6 => 'Inline mathematics, words not counted',
+ 7 => 'Displayed mathematics, words not counted',
+ -1 => 'Float, not included, but looks for captions');
+ print "Rule used: ".$rules{$def}."\n";
+ if ($def=$TeXmacro{'begin'.$arg}) {
+ _print_rule_macro($def);
+ }
+}
+
+
+###### HTML routines
+
+
+# Print HTML header
sub html_head {
print "<html>\n<head>";
- if ($utf8flag) {
- print "\n<meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\">";
- }
- print '
+ print "\n<meta http-equiv='content-type' content='text/html; charset=$outputEncoding'>\n";
+ _print_html_style();
+ print "</head>\n\n<body>\n";
+ print "\n<h1>LaTeX word count";
+ if ($showVersion>0) {print " (version ",_html_version(),")"}
+ print "</h1>\n";
+}
+
+# Print HTML tail
+sub html_tail {
+ print "</body>\n\n</html>\n";
+}
+
+# Return version number using HTML
+sub _html_version {
+ my $htmlver=$versionnumber;
+ $htmlver=~s/\b(alpha|beta)\b/&$1;/g;
+ return $htmlver;
+}
+
+# Print HTML STYLE element
+sub _print_html_style {
+print <<END
<style>
<!--
body {width:auto;padding:5;margin:5;}
.error {font-weight:bold;color:#f00;font-style:italic;}
-.word1,.word2,.word3 {color: #009;}
+.word1,.word2,.word3 {color: #009; border-left: 1px solid #CDF; border-bottom: 1px solid #CDF;}
.word2 {font-weight: 700;}
.word3 {font-style: italic;}
.command {color: #c00;}
@@ -1184,208 +2582,289 @@ body {width:auto;padding:5;margin:5;}
.tc {color: #999; font-weight:bold;}
.comment {color: #999; font-style: italic;}
.state {color: #990; font-size: 70%;}
-.sumcount {color: #999; font-size: 80%;}
+.cumsum {color: #999; font-size: 80%;}
.fileinclude {color: #696; font-weight:bold;}
-dl.count {background: #cfc; color: 009;}
-dl.count dt.header {font-weight: bold; font-style: italic; float: none;}
-dl.count dt {clear: both; float: left; margin-right: .5em;}
-dl.count dd {font-weight: bold;}
-dl.count dd.briefcount {font-weight: 700; clear: both; font-size:80%; font-weight:normal; margin-left:8pt;}
.warning {color: #c00; font-weight: 700;}
-.parse, .count, .stylehelp, .filegroup {border: solid 1px #999; margin: 0pt; padding: 4pt;}
-.parse {font-size: 80%; background: #eef;}
-.parse {border-bottom:none;}
-.stylehelp {font-size: 80%; background: #ffc; margin-bottom: 8pt;}
-.filegroup {background: #efe; margin-bottom: 8pt;}
+
+div.filegroup, div.parse, div.stylehelp, div.count, div.sumcount, div.error {
+ border: solid 1px #999; margin: 4pt 0pt; padding: 4pt;
+}
+div.stylehelp {font-size: 80%; background: #fffff0; margin-bottom: 16pt;}
+div.filegroup {background: #dfd; margin-bottom: 16pt;}
+div.count {background: #ffe;}
+div.sumcount {background: #cec;}
+div.error {background: #fcc;}
+.parse {font-size: 80%; background: #f8fff8; border-bottom:none;}
+
+ul.count {list-style-type: none; margin: 4pt 0pt; padding: 0pt;}
+.count li.header {font-weight: bold; font-style: italic;}
+.subcount li.header {font-weight: normal; font-style: italic;}
+.subcount li {margin-left: 16pt; font-size: 80%;}
+.fielddesc {font-style: italic;}
+.nb {color: #900;}
+
+table.stat {border:2px solid #339; background:#666; border-collapse:collapse;}
+table.stat tr {border:1px solid #CCC;}
+table.stat th, table.stat td {padding:1pt 4pt;}
+table.stat col {padding:4pt;}
+table.stat thead {background: #CCF;}
+table.stat tbody {border:1px solid #333;}
+table.stat tbody.sumstat {background:#FFC;}
+table.stat tbody.langstat {background:#FEE;}
+table.stat tbody.wordstat {background:#EEF;}
+table.stat .sum {font-weight:bold; font-style:italic;}
-->
</style>
-</head>
-<body>
-<h1>LaTeX word count</h1>
-';
+END
}
-sub html_tail {
- print '</body></html>';
-}
+###### Read text data
-###### Help routines
+# Return the STRINGDATA hash (lazy instantiation)
+sub STRINGDATA {
+ if (!defined $STRINGDATA) {
+ my @DATA=<DATA>;
+ foreach (@DATA) {
+ $_=~s/\$\{(\w+)\}/__apply_globaldata($1)/ge;
+ }
+ $STRINGDATA=splitlines(':{3,}\s*(\w+)?',\@DATA);
+ }
+ return $STRINGDATA;
+}
-sub print_version {
- print "TeXcount version ".$versionnumber.", ".$versiondate.'.';
+# Return value from STRINGDATA hash
+sub StringData {
+ my $name=shift @_;
+ return STRINGDATA()->{$name};
}
-sub print_syntax {
- print '
-Syntax: TeXcount.pl [options] files
+# Insert value from GLOBALDATA
+sub __apply_globaldata {
+ my $name=shift @_;
+ if (my $value=$GLOBALDATA{$name}) {
+ return $value;
+ }
+ return '[['.$name.']]';
+}
-Options:
- -relaxed Uses relaxed rules for word and option handling:
- i.e. allows more general cases to be counted as
- either words or macros.
- -v Verbose (same as -v3)
- -v0 Do not present parsing details
- -v1 Verbose: print parsed words, mark formulae
- -v2 More verbose: also print ignored text
- -v3 Even more verbose: include comments and options
- -v4 Same as -v3 -showstate
- -showstate Show internal states (with verbose)
- -brief Only prints a brief, one line summary of counts
- -q, -quiet Quiet mode, no error messages (use is discouraged!)
- -sum, -sum= Make sum of all word and equation counts. May also
- use -sum=#[,#] with up to 7 numbers to indicate how
- each of the counts (text words, header words, caption
- words, #headers, #floats, #inlined formulae,
- #displayed formulae) are summed. The default sum (if
- only -sum is used) is the same as -sum=1,1,1,0,0,1,1.
- -sub, -sub= Generate subcounts. Option values are none, part,
- chapter, section or subsection. Default (-sub) is set
- to subsection, whereas unset is none. (Alternative
- option name is -subcount.)
- -nc, -nocol No colours (colours require ANSI)
- -html Output in HTML format
- -htmlcore Only HTML body contents
- -inc Include tex files included in the document
- -noinc Do not include included tex files (default)
- -total Do not give sums per file, only total sum.
- -1 Same as -brief and -total. Ensures there is only one
- line of output. If used in conjunction with -sum, the
- output will only be the total number. (NB: Character
- is the number one, not the letter L.)
- -dir, -dir= Specify the working directory using -dir=path.
- Remember that the path must end with \ or /. If only
- -dir is used, the directory of the parent file is used.
- -utf8, -unicode Turns on Unicode (UTF-8) for input and output. This
- is automatic with -chinese, and is required to handle
- e.g. Korean text. Note that the TeX file must be save
- in UTF-8 format (not e.g. GB2312 or Big5), or the
- result will be unpredictable.
- -ch, -chinese, -zhongwen Turns on support for Chinese characters.
- TeXcount will then count each Chinese character as a
- word. Automatically turns on -utf8.
- -jp, -japanese Turns on support for Japanese characters. TeXcount
- will count each Japanese character (kanji, hiragana,
- and katakana) as one word, i.e. not do any form of
- word segmentation. Automatically turns on -utf8.
- -codes Display output style code overview and explanation.
- This is on by default.
- -nocodes Do not display output style code overview.
- -h, -?, --help, /? Help
- --version Print version number
- --license License information
-';
+# Print value from STRINGDATA using wprintlines
+sub wprintstringdata {
+ my $name=shift @_;
+ my $data=StringData($name);
+ if (!defined $data) {
+ error($Main,"No StringData $name.",'BUG');
+ }
+ wprintlines(@_,@$data);
+}
+
+# Divide array of lines by identifying headers
+sub splitlines {
+ my ($pattern,$lines)=@_;
+ if (!defined $lines) {return;}
+ my $id=undef;
+ my %hash;
+ foreach my $line (@$lines) {
+ if ($line=~/^$pattern$/) {
+ $id=$1;
+ if (defined $id) {
+ $hash{$id}=[];
+ if (defined $2) {push @{$hash{$id}},$2;}
+ }
+ } elsif (defined $id) {
+ chomp $line;
+ push @{$hash{$id}},$line;
+ }
+ }
+ return \%hash;
}
-sub print_help {
- print '
+# Print string with word wrapping and indentation using
+# wprintlines.
+sub wprint {
+ my $text=shift @_;
+ my @lines=split(/\n/,$text);
+ wprintlines(@lines);
+}
+
+# Print with word wrapping and indentation. A line with
+# @ - :
+# sets two column tabs. A tab or multiple spaces is taken
+# to indicate indentation. If the first column value is
+# only a single '|', this is just an indication to skip
+# one tab.
+sub wprintlines {
+ my @lines=@_;
+ my $ind1=2;
+ my $ind2=6;
+ my $i;
+ foreach my $line (@lines) {
+ if ($line=~s/^@//) {
+ $ind2=1+index($line,':');
+ $ind1=1+index($line,'-');
+ if ($ind1<1) {$ind1=$ind2;}
+ next;
+ }
+ my $firstindent=0;
+ if ($line=~s/^(\t|\s{2,})(\S)/$2/) {$firstindent=$ind1;}
+ my $indent=$firstindent;
+ if ($line=~/^(.*\S)(\t|\s{2,})/) {
+ $indent=$ind2;
+ if ($1 eq '|') {$line=' ';}
+ else {$line=$1." ";}
+ $i=$indent-$firstindent-length($line);
+ if ($i>0) {$line.=' ' x $i;}
+ $line.=$';
+ }
+ print wrap(' ' x $firstindent,' ' x $indent,$line)."\n";
+ }
+}
+
+
+########################################
+##### TEXT DATA
+
+__DATA__
+
+::::::::::::::::::::::::::::::::::::::::
+:::::::::: Version
+TeXcount version ${versionnumber}, ${versiondate}.
+
+:::::::::: Reference
+The TeXcount script is copyright of ${maintainer} (${copyrightyears}) and published under the LaTeX Project Public Licence.
+
+Go to the TeXcount web page
+ ${website}
+for more information about the script, e.g. news, updates, help, usage tips, known issues and short-comings, or to access the script as a web application. Feedback such as problems or errors can be reported to einarro@ifi.uio.no.
+
+:::::::::: License
+TeXcount version ${versionnumber}
+
+Copyright ${copyrightyears} ${maintainer}
+
+The TeXcount script is published under the LaTeX Project Public License (LPPL)
+ http://www.latex-project.org/lppl.txt
+which grants you, the user, the right to use, modify and distribute the script. However, if the script is modified, you must change its name or use other technical means to avoid confusion.
+
+The script has LPPL status "maintained" with ${maintainer} being the current maintainer.
+
+::::::::::::::::::::::::::::::::::::::::
+:::::::::: HelpTitle
***************************************************************
-* TeXcount.pl '.$versionnumber.', '.$versiondate.'
+* TeXcount version ${versionnumber}, ${versiondate}
*
-Count words in TeX and LaTeX files, ignoring macros, tables,
-formulae, etc.
-';
- print_syntax();
- print '
-The script counts words as either words in the text, words in
-headers/titles or words in floats (figure/table captions).
-Macro options (i.e. \marco[...]) are ignored; macro parameters
-(i.e. \macro{...}) are counted or ignored depending on the
-macro, but by default counted. Begin-end groups are by default
-ignored and treated as \'floats\', though some (e.g. center) are
-counted.
-
-Unless -nocol (or -nc) has been specified, the output will be
-colour coded. Counted text is coloured blue with headers are in
-bold and in HTML output caption text is italicised.
-
-Mathematical formulae are not counted as words, but are instead
-counted separately with separate counts for inlined formulae
-and displayed formulae. Similarly, the number of headers and
-the number of \'floats\' are counted. Note that \'float\' is used
-here to describe anything defined in a begin-end group unless
-explicitly recognized as text or mathematics.
-
-The verbose options (-v1, -v2, -v3, showstate) produces output
-indicating how the text has been interpreted. Check this to
-ensure that words in the text has been interpreted as such,
-whereas mathematical formulae and text/non-text in begin-end
-groups have been correctly interpreted.
-
-Parsing instructions may be passed to TeXcount using comments
-in the LaTeX files on the format
+Count words in TeX and LaTeX files, ignoring macros, tables, formulae, etc.
+
+::::::::::::::::::::::::::::::::::::::::
+:::::::::: OptionsHead
+
+Syntax: TeXcount.pl [options] files
+
+Options:
+
+:::::::::: OptionsPrefix
+@ - :
+:::::::::: Options
+ -relaxed Uses relaxed rules for word and option handling: i.e. allows more general cases to be counted as either words or macros.
+ -restricted Restricts the rules for word and option handling.
+ -v Verbose (same as -v3).
+ -v0 Do not present parsing details.
+ -v1 Verbose: print parsed words, mark formulae.
+ -v2 More verbose: also print ignored text.
+ -v3 Even more verbose: include comments and options.
+ -v4 Same as -v3 -showstate.
+ -showstate Show internal states (with verbose).
+ -brief Only prints a brief, one line summary of counts.
+ -q, -quiet Quiet mode, no error messages. Use is discouraged!
+ -strict Strict mode, warns against begin-end groups for which rule are not defined.
+ -sum, -sum= Make sum of all word and equation counts. May also use -sum=#[,#] with up to 7 numbers to indicate how each of the counts (text words, header words, caption words, #headers, #floats, #inlined formulae, #displayed formulae) are summed. The default sum (if only -sum is used) is the same as -sum=1,1,1,0,0,1,1.
+ -nosum Do not compute sum.
+ -sub, -sub= Generate subcounts. Option values are none, part, chapter, section or subsection. Default (-sub) is set to subsection, whereas unset is none. (Alternative option name is -subcount.)
+ -nosub Do not generate subcounts.
+ -col Use ANSI colours in text output.
+ -nc, -nocol No colours (colours require ANSI).
+ -nosep, -noseparator No separating character/string added after each word (default).
+ -sep=, -separator= Separating character or string to be added after each word.
+ -html Output in HTML format.
+ -htmlcore Only HTML body contents.
+ -opt, -optionfile Read options/parameters from file.
+ - Read LaTeX code from STDIN.
+ -inc Parse included TeX files (as separate file).
+ -merge Merge included TeX files into code (in place).
+ -noinc Do not include included tex files (default).
+ -incbib Include bibliography in count, include bbl file if needed.
+ -nobib Do not include bibliography in count (default).
+ -incpackage= Include rules for the given package.
+ -total Do not give sums per file, only total sum.
+ -1 Same as -brief and -total. Ensures there is only one line of output. If used in conjunction with -sum, the output will only be the total number. (NB: Character is the number one, not the letter L.)
+ -template= Speficy an output template. Use {1},...,{7}, {SUM} and {TITLE} to include values, {1?...?1} etc. to conditionally include sections, {1?....|...?1} etc. to specify an alternative text if zero. To include subcounts, use {SUB?...?SUB} where ... is replaced with the template to use per subcount. Line shift may be specified using \\n.
+ -dir, -dir= Specify the working directory using -dir=path. Remember that the path must end with \\ or /. If only -dir is used, the directory of the parent file is used.
+ -enc=, -encoding= Specify encoding (default is to guess the encoding).
+ -utf8, -unicode Selects Unicode (UTF-8) for input and output. This is automatic with -chinese, and is required to handle e.g. Korean text. Note that the TeX file must be save in UTF-8 format (not e.g. GB2312 or Big5), or the result will be unpredictable.
+ -alpha=, -alphabets= List of Unicode character groups (or digit, alphabetic) permitted as letters. Names are separated by ',' or '+'. If list starts with '+', the alphabets will be added to those already included. The default is Digit+alphabetic.
+ -logo=, -logograms= List of Unicode character groups interpreted as whole word characters, e.g. Han for Chinese characters. Names are separated by ',' or '+'. If list starts with '+', the alphabets will be added to those already included. By default, this is set to include Ideographic, Katakana, Hiragana, Thai and Lao.
+ -ch, -chinese, -zhongwen Turns on support for Chinese characters. TeXcount will then count each Chinese character as a word.
+ -jp, -japanese Turns on support for Japanese characters. TeXcount will count each Japanese character (kanji, hiragana, and katakana) as one word, i.e. not do any form of word segmentation.
+ -kr, -korean Turns on support for Korean. This will count hangul and han characters, i.e. with no word separation. NB: Experimental!
+ -kr-words, -korean-words Turns on support for Korean words, i.e. hangul words separated by characters. Han characters are still counted as characters. NB: Experimental!
+ -ch-only, ..., -korean-words-only As options -chinese, ..., -korean-words, but also excludes letter-based words or trims down the character set to the minimum.
+ -char, -character, -letters Counts letters/characters instead of words. Note that spaces and punctuation is not counted.
+ -char-only, ..., -letters-only Like -letters, but counts alphabetic letters only.
+ -countall, -count-all The default setting in which all characters are included as either alphabets og logograms.
+ -freq Produce individual word frequency table.
+ -stat Produce statistics on language/script usage.
+ -codes Display output style code overview and explanation. This is on by default.
+ -nocodes Do not display output style code overview.
+ -h, -?, -help, /? Help text.
+ -h=, -?=, -help=, /?= Takes a macro or group name as option and returns a description of the rules for handling this if any are defined. If handling rule is package specific, use -incpackage=package name: -incpackage must come before -h= on the command line to take effect.
+ -ver, -version Print version number.
+ -lic, -license, -licence Licence information.
+
+::::::::::::::::::::::::::::::::::::::::
+:::::::::: HelpText
+The script counts words as either words in the text, words in headers/titles or words in floats (figure/table captions). Macro options (i.e. \\macro[...]) are ignored; macro parameters (i.e. \\macro{...}) are counted or ignored depending on the macro, but by default counted. Begin-end groups are by default ignored and treated as 'floats', though some (e.g. center) are counted.
+
+Mathematical formulae are not counted as words, but are instead counted separately with separate counts for inlined formulae and displayed formulae. Similarly, the number of headers and the number of 'floats' are counted. Note that 'float' is used here to describe anything defined in a begin-end group unless explicitly recognized as text or mathematics.
+
+The verbose options (-v1, -v2, -v3, showstate) produces output indicating how the text has been interpreted. Check this to ensure that words in the text has been interpreted as such, whereas mathematical formulae and text/non-text in begin-end groups have been correctly interpreted.
+
+Summary, as well as the verbose output, may be produced as text (default) or as HTML code using the -html option. The HTML may then be sent to file which may be viewed with you favourite browser.
+
+Under UNIX, unless -nocol (or -nc) has been specified, the output will be colour coded using ANSI colour codes. Counted text is coloured blue with headers are in bold and in HTML output caption text is italicised. Use 'less -r' instead of just 'less' to view output: the '-r' option makes less treat text formating codes properly. Windows does not support ANSI colour codes, and so this is turned off by default.
+
+:::::::::: TCinstructions
+Parsing instructions may be passed to TeXcount using comments in the LaTeX files on the format
+@ - :
%TC:instruction arguments
-where valid instructions for setting parsing rules, typically
-set at the start of the document (applies globally), are:
+and are used to control how TeXcount parses the document. The following instructions are used to set parsing rules which will apply to all subsequent parsing (including other files):
%TC:macro [macro] [param.states]
- macro handling rule, no. of and rules for parameters
+ | macro handling rule, no. of and rules for parameters
%TC:macroword [macro] [number]
- macro counted as a given number of words
+ | macro counted as a given number of words
%TC:header [macro] [param.states]
- header macro rule, as macro but counts as one header
+ | header macro rule, as macro but counts as one header
%TC:breakmacro [macro] [label]
- macro causing subcount break point
- %TC:group [name] [parsing-state]
- begin-end-group handling rule
+ | macro causing subcount break point
+ %TC:group [name] [param.states] [content-state]
+ | begin-end-group handling rule
%TC:floatinclude [macro] [param.states]
- as macro, but also counted inside floats
+ | as macro, but also counted inside floats
%TC:preambleinclude [macro] [param.states]
- as macro, but also counted inside the preamble
+ | as macro, but also counted inside the preamble
%TC:fileinclue [macro] [rule]
- file include, add .tex if rule=2, not if rule=0
-The [param.states] is used to indicate the number of parameters
-used by the macro and the rules of handling each of these: format
-is [#,#,...,#] with one number for each parameter, and main rules
-are 0 to ignore and 1 to count as text. Parsing instructions
-which may be used anywhere are:
+ | file include, add .tex if rule=2, not if rule=0, if missing when rule=1
+The [param.states] is used to indicate the number of parameters used by the macro and the rules of handling each of these: the format is [#,#,...,#] with # representing one number for each parameter giving the parsing state to use for that parameter, alternatively just a single number (#) indicating how many parameters to ignore (parsing state 0). The option [content-state] is used to give the parsing state to use for the contents of a begin-end group. The main parsing states are 0 to ignore and 1 to count as text.
+
+Parsing instructions which may be used anywhere are:
+@ - :
%TC:ignore start block to ignore
%TC:endignore end block to ignore
%TC:break [title] add subcount break point here
See the documentation for more details.
-Unix hint: Use \'less -r\' instead of just \'less\' to view output:
-the \'-r\' option makes less treat text formating codes properly.
+Command line options and most %TC commands (prefixed by % rather than %TC:) may be placed in an options file. This is particularly useful for defining your own output templates and macro handling rules.
-Windows hint: If your Windows interprets ANSI colour codes, lucky
-you! Otherwise, use the -nocol (or -nc) option with the verbose
-options or the output will be riddled with colour codes. Instead,
-you can use -html to produce HTML code, write this to file and
-view with your favourite browser.
-';
- print_reference();
-}
-
-sub print_reference {
- print '
-The TeXcount script is copyright of Einar Andreas Rødland (2008)
-and published under the LaTeX Project Public License.
-
-For more information about the script, e.g. news, updates, help,
-usage tips, known issues and short-comings, go to
- http://folk.uio.no/einarro/Comp/texwordcount.html
-or go to
- http://folk.uio.no/einarro/Services/texcount.html
-to access the script as a web service. Feedback such as problems
-or errors can be reported to einarro@ifi.uio.no.
-';
-}
-
-sub print_license {
- print 'TeXcount version '.$versionnumber.'
-
-Copyright 2008 Einar Andreas Rødland
-
-The TeXcount script is published under the LaTeX Project Public
-License (LPPL)
- http://www.latex-project.org/lppl.txt
-which grants you, the user, the right to use, modify and distribute
-the script. However, if the script is modified, you must change its
-name or use other technical means to avoid confusion.
-
-The script has LPPL status "maintained" with Einar Andreas
-Rødland being the current maintainer.
-';
-}
+::::::::::::::::::::::::::::::::::::::::