diff options
| -rwxr-xr-x | Build/source/texk/texlive/linked_scripts/texcount/texcount.pl | 2644 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/QuickReference.pdf | bin | 138221 -> 115139 bytes | |||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/QuickReference.tex | 17 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/TeXcount.pdf | bin | 224304 -> 247259 bytes | |||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/TeXcount.tex | 435 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/TechDoc.pdf | bin | 0 -> 235583 bytes | |||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/TechDoc.tex | 620 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/macros.tex | 56 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/sub_addrules.tex | 28 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/sub_options.tex | 44 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/support/texcount/sub_tc_other.tex | 22 | ||||
| -rwxr-xr-x | Master/texmf-dist/scripts/texcount/texcount.pl | 2644 |
12 files changed, 4629 insertions, 1881 deletions
diff --git a/Build/source/texk/texlive/linked_scripts/texcount/texcount.pl b/Build/source/texk/texlive/linked_scripts/texcount/texcount.pl index 69fb4b5002f..5a53b3bfb4d 100755 --- a/Build/source/texk/texlive/linked_scripts/texcount/texcount.pl +++ b/Build/source/texk/texlive/linked_scripts/texcount/texcount.pl @@ -1,15 +1,22 @@ #! /usr/bin/env perl use strict; use warnings; -use Term::ANSIColor; -use Text::Wrap; -use Encode; use utf8; # Because the script itself is UTF-8 encoded +use Encode; +use Text::Wrap; +use Term::ANSIColor; + +BEGIN { + if ($^O=~/^MSWin/) { + require Win32::Console::ANSI; + Win32::Console::ANSI::->import(); + } +} ##### Version information -my $versionnumber="2.3"; -my $versiondate="2011 Jul 30"; +my $versionnumber="3.0"; +my $versiondate="2013 Jul 29"; ###### Set global settings and variables @@ -17,9 +24,9 @@ my $versiondate="2011 Jul 30"; my %GLOBALDATA= ('versionnumber' => $versionnumber ,'versiondate' => $versiondate - ,'maintainer' => "Einar Andreas Rodland" - ,'copyrightyears' => "2008-2011" - ,'website' => "http://app.uio.no/ifi/texcount/" + ,'maintainer' => 'Einar Andreas Rodland' + ,'copyrightyears' => '2008-2013' + ,'website' => 'http://app.uio.no/ifi/texcount/' ); ### Options and states @@ -29,28 +36,32 @@ my $Main=getMain(); # Global options and settings my $htmlstyle=0; # Flag to print HTML +my $texcodeoutput=0; # Flag to convert output to valid TeX text my $encoding=undef; # Selected input encoding (default will be guess) my @encodingGuessOrder=qw/ascii utf8 latin1/; # Encoding guessing order my $outputEncoding; # Encoding used for output my @AlphabetScripts=qw/Digit Is_alphabetic/; # Letters minus logograms: defined later my @LogogramScripts=qw/Ideographic Katakana Hiragana Thai Lao Hangul/; # Scripts counted as whole words -my $WordPattern; # Regex matching a word (defined in apply_language_options()) # Parsing rules options my $includeTeX=0; # Flag to parse included files my $includeBibliography=0; # Flag to include bibliography my %substitutions; # Substitutions to make globally +my %IncludedPackages; # List of included packages # Counting options my @sumweights; # Set count weights for computing sum my $optionWordFreq=0; # Count words of this frequency, or don't count if 0 my $optionWordClassFreq=0; # Count words per word class (language) if set +my $optionMacroStat=0; # Count macro, environment and package usage # Parsing details options -my $strictness=0; # Flag to check for undefined groups -my $verbose=0; # Level of verbosity -my $showcodes=1; # Flag to show overview of colour codes (2=force) +my $strictness=0; # Flag to check for undefined environments +my $defaultVerbosity='0'; # Specification of default verbose output style +my $defaultprintlevel=0; # Flag indicating default level of verbose output +my $printlevel=undef; # Flag indicating level of verbose output my $showstates=0; # Flag to show internal state in verbose log +my $showcodes=1; # Flag to show overview of colour codes (2=force) my $showsubcounts=0; # Write subcounts if #>this, or not (if 0) my $separatorstyleregex='^word'; # Styles (regex) after which separator should be added my $separator=''; # Separator to add after words/tokens @@ -70,29 +81,47 @@ my $blankline=0; # Number of blank lines printed my $errorcount=0; # Number of errors in parsing my %warnings=(); # Warnings my %WordFreq; # Hash for counting words +my %MacroUsage; # Hash for counting macros, environments and packages + +# External sources +my $HTMLfile; # HTML file to use as HTML output template +my $CSSfile; # CSS file to use with HTML output +my $CSShref; # CSS reference to use with HTML output +my @htmlhead; # Lines to add to the HTML header +my $htmlopen; # text used to open the HTML file +my $htmlclose; # text used to close the HTML file # String data storage my $STRINGDATA; +# Other constants +my $_PARAM_='<param>'; # to identify parameter to _parse_unit +my $STRING_PARAMETER='{_}'; # used to log macro parameters +my $STRING_OPTIONAL_PARAM='[_]'; # used to log optional parameter +my $STRING_GOBBLED_OPTION='[]'; # used to log gobbled macro option +my $STRING_ERROR='<error>'; # used to log errors causing parsing to stop +my $REGEX_NUMBER=qr/^\d+$/; # regex to recognize a number + ###### Set CMD specific settings and variables ## Preset command line options # List of options (stings) separated by comma, -# e.g. ("-inc","-v") to parse included files and +# e.g. ('-inc','-v') to parse included files and # give verbose output by default. my @StartupOptions=(); # CMD specific global variables my @filelist; # List of files to parse -my $workdir; # Root directory (taken from filename) -my $globalworkdir=""; # Overrules workdir (default=present root) +my $globalworkdir='./'; # Overrules workdir (default=present root) +my $workdir; # Current directory (default=taken from filename) +my $auxdir; # Directory for auxilary files, e.g. bbl (default=$workdir) my $fileFromSTDIN=0; # Flag to set input from STDIN -my $_STDIN_="<STDIN>"; # File name to represent STDIN +my $_STDIN_='<STDIN>'; # File name to represent STDIN (must be '<...>'!) # CMD specific settings $Text::Wrap::columns=76; # Page width for wrapped output -###### Set status identifiers and methods +###### Set state identifiers and methods ### Counter indices from 0 to $SIZE_CNT-1 @@ -105,125 +134,227 @@ $Text::Wrap::columns=76; # Page width for wrapped output # 6: Number of inlined math # 7: Number of displayed math my $SIZE_CNT=8; +my $SIZE_CNT_DEFAULT=8; my $CNT_FILE=0; my $CNT_WORDS_TEXT=1; my $CNT_WORDS_HEADER=2; -my $CNT_WORDS_CAPTION=3; +my $CNT_WORDS_OTHER=3; my $CNT_COUNT_HEADER=4; my $CNT_COUNT_FLOAT=5; my $CNT_COUNT_INLINEMATH=6; my $CNT_COUNT_DISPLAYMATH=7; # Labels used to describe the counts -my @countlabel=('Files','Words in text','Words in headers', - 'Words in float captions','Number of headers','Number of floats', +my @countkey=('file','word','hword','oword','header','float','inmath','dsmath'); +my @countdesc=('Files','Words in text','Words in headers', + 'Words outside text (captions, etc.)','Number of headers','Number of floats/tables/figures', 'Number of math inlines','Number of math displayed'); -### Parsing statuses -## Note that status numbers are used in rule definitions and should -## not be changed. +# Map keywords to counters +my %key2cnt; +add_keys_to_hash(\%key2cnt,$CNT_FILE,0,'file'); +add_keys_to_hash(\%key2cnt,$CNT_WORDS_TEXT,1,'text','word','w','wd'); +add_keys_to_hash(\%key2cnt,$CNT_WORDS_HEADER,2,'headerword','hword','hw','hwd'); +add_keys_to_hash(\%key2cnt,$CNT_WORDS_OTHER,3,'otherword','other','oword','ow','owd'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_HEADER,4,'header','heading','head'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_FLOAT,5,'float','table','figure'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_INLINEMATH,6,'inline','inlinemath','imath','eq'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_DISPLAYMATH,7,'displaymath','dsmath','dmath','ds'); + + +### Token types +# Set in $tex->{'type'} by call to _next_token +my $TOKEN_SPACE=-1; +my $TOKEN_COMMENT=0; +my $TOKEN_WORD=1; # word (or other form of text or text component) +my $TOKEN_SYMBOL=2; # symbol (not word, e.g. punctuation) +my $TOKEN_MACRO=3; # macro (\name) +my $TOKEN_BRACE=4; # curly braces: { } +my $TOKEN_BRACKET=5; # brackets: [ ] +my $TOKEN_MATH=6; +my $TOKEN_LINEBREAK=9; # line break in file +my $TOKEN_TC=666; # TeXcount instructions (%TC:instr) +my $TOKEN_END=999; # end of line or blank line + + +### Parsing states # -## Status for regions that should not be counted -# 0 = exclude from count -## Statuses for regions in which words should not be counted -# -1 = float (exclude, but include captions) -# -2 = strong exclude, ignore begin-end groups -# -3 = stronger exclude, do not parse macro parameters -# -4 = ignore everything except end marker: even { -# -9 = preamble (between \documentclass and \begin{document}) -## Statuses for regions in which words should be counted -# 1 = text -# 2 = header text -# 3 = float text -## Status change: not used in parsing, but to switch status then ignore contents -# 6 = switch to inlined math -# 7 = switch to displayed math -## Note that positive statuses must correspond to CNT codes! +## States for regions that should not be counted +# IGNORE = exclude from count +# FLOAT = float (exclude, but include captions) +# EXCLUDE_STRONG = strong exclude, ignore environments +# EXCLUDE_STRONGER = stronger exclude, do not parse macro parameters +# EXCLUDE_ALL = ignore everything except end marker: even { +# PREAMBLE = preamble (between \documentclass and \begin{document}) +## States for regions in which words should be counted +# TEXT = text +# TEXT_HEADER = header text +# TEXT_FLOAT = float text +## State change: not used in parsing, but to switch state then ignore contents +# TO_INLINEMATH = switch to inlined math +# TO_DISPLAYMATH = switch to displayed math +## Other states +# _NULL = default state to use if none other is defined +# _OPTION = state used to indicate that the next parameter is an option +# _EXCLUDE_ = cutoff, state <= this represents excluded text +## NB: Presently, it is assumed that additional states is added as 8,9,..., +## e.g. that states added through TC:newcounter correspond to the added counters. # -my $STATUS_IGNORE=0; -my $STATUS_FLOAT=-1; -my $STATUS_EXCLUDE_STRONG=-2; -my $STATUS_EXCLUDE_STRONGER=-3; -my $STATUS_EXCLUDE_ALL=-4; -my $STATUS_PREAMBLE=-9; -my $STATUS_TEXT=1; -my $STATUS_TEXT_HEADER=2; -my $STATUS_TEXT_FLOAT=3; -my $STATUS_TO_INLINEMATH=6; -my $STATUS_TO_DISPLAYMATH=7; - -# When combining two statuses, use the first one; list must be complete! -my @STATUS_PRIORITY_LIST=(-4,-3,-2,-1,0,-9,3,2,1); - -# Status: is a text status..."include status" is more correct -sub status_is_text { +my $STATE_IGNORE=-1; +my $STATE_MATH=-2; +my $STATE_FLOAT=-10; +my $STATE_EXCLUDE_STRONG=-20; +my $STATE_EXCLUDE_STRONGER=-30; +my $STATE_EXCLUDE_ALL=-40; +my $STATE_PREAMBLE=-99; +my $STATE_TEXT=1; +my $STATE_TEXT_HEADER=2; +my $STATE_TEXT_FLOAT=3; +my $STATE_TO_HEADER=4; +my $STATE_TO_FLOAT=5; +my $STATE_TO_INLINEMATH=6; +my $STATE_TO_DISPLAYMATH=7; +my $__STATE_EXCLUDE_=-10; +my $__STATE_NULL=1; +my $_STATE_OPTION=-1000; +my $_STATE_NOOPTION=-1001; +my $_STATE_AUTOOPTION=-1002; + +# Counter key mapped to STATE +my $PREFIX_PARAM_OPTION=' '; # Prefix for parameter options/modifiers +my %key2state; +add_keys_to_hash(\%key2state,$STATE_TEXT,1,'text','word','w','wd'); +add_keys_to_hash(\%key2state,$STATE_TEXT_HEADER,2,'headertext','headerword','hword','hw','hwd'); +add_keys_to_hash(\%key2state,$STATE_TEXT_FLOAT,3,'otherword','other','oword','ow','owd'); +add_keys_to_hash(\%key2state,$STATE_TO_HEADER,4,'header','heading','head'); +add_keys_to_hash(\%key2state,$STATE_TO_FLOAT,5,'float','table','figure'); +add_keys_to_hash(\%key2state,$STATE_TO_INLINEMATH,6,'inline','inlinemath','imath','eq'); +add_keys_to_hash(\%key2state,$STATE_TO_DISPLAYMATH,7,'displaymath','dsmath','dmath','ds'); +add_keys_to_hash(\%key2state,$STATE_IGNORE,0,'ignore','x'); +add_keys_to_hash(\%key2state,$STATE_MATH,'ismath'); +add_keys_to_hash(\%key2state,$STATE_FLOAT,-1,'isfloat'); +add_keys_to_hash(\%key2state,$STATE_EXCLUDE_STRONG,-2,'xx'); +add_keys_to_hash(\%key2state,$STATE_EXCLUDE_STRONGER,-3,'xxx'); +add_keys_to_hash(\%key2state,$STATE_EXCLUDE_ALL,-4,'xall'); +add_keys_to_hash(\%key2state,$_STATE_OPTION,'[',' option',' opt',' optional'); +add_keys_to_hash(\%key2state,$_STATE_NOOPTION,'nooption','nooptions','noopt','noopts'); +add_keys_to_hash(\%key2state,$_STATE_AUTOOPTION,'autooption','autooptions','autoopt','autoopts'); + +# When combining two states, use the first one; list must be complete! +my @STATE_FIRST_PRIORITY=( + $STATE_EXCLUDE_ALL, + $STATE_EXCLUDE_STRONGER, + $STATE_EXCLUDE_STRONG, + $STATE_FLOAT, + $STATE_MATH, + $STATE_IGNORE, + $STATE_PREAMBLE, + $STATE_TO_FLOAT, + $STATE_TO_HEADER, + $STATE_TO_INLINEMATH, + $STATE_TO_DISPLAYMATH); +my @STATE_MID_PRIORITY=(); +my @STATE_LAST_PRIORITY=( + $STATE_TEXT_FLOAT, + $STATE_TEXT_HEADER, + $STATE_TEXT); + +# Map state to corresponding word counter +my %state2cnt=( + $STATE_TEXT => $CNT_WORDS_TEXT, + $STATE_TEXT_HEADER => $CNT_WORDS_HEADER, + $STATE_TEXT_FLOAT => $CNT_WORDS_OTHER); + +# Transition state mapped to content state and counter +my %transition2state=( + $STATE_TO_HEADER => [$STATE_TEXT_HEADER,$CNT_COUNT_HEADER], + $STATE_TO_INLINEMATH => [$STATE_MATH ,$CNT_COUNT_INLINEMATH], + $STATE_TO_DISPLAYMATH => [$STATE_MATH ,$CNT_COUNT_DISPLAYMATH], + $STATE_TO_FLOAT => [$STATE_FLOAT ,$CNT_COUNT_FLOAT]); + +# Parsing state descriptions (used for macro rule help) +my %state2desc=( + $STATE_IGNORE => 'ignore: do not count', + $STATE_MATH => 'math/equation contents', + $STATE_FLOAT => 'float (figure, etc.): ignore all but special macros', + $STATE_EXCLUDE_STRONG => 'strong exclude: ignore environments', + $STATE_EXCLUDE_STRONGER => 'stronger exclude: ignore environments and macro paramters', + $STATE_EXCLUDE_ALL => 'exlude all: even {, only scan for end marker', + $STATE_PREAMBLE => 'preamble: from \documentclass to \begin{document}', + $STATE_TEXT => 'text: count words', + $STATE_TEXT_HEADER => 'header text: count words as header words', + $STATE_TEXT_FLOAT => 'float text: count words as float words (e.g. captions)', + $STATE_TO_HEADER => 'header: count header, then count words as header words', + $STATE_TO_FLOAT => 'float: count float, then count words as float/other words', + $STATE_TO_INLINEMATH => 'inline math: count as inline math/equation', + $STATE_TO_DISPLAYMATH => 'displayed math: count as displayed math/equation'); + +# Parsing state presentation style +my %state2style=( + $STATE_TEXT => 'word', + $STATE_TEXT_HEADER => 'hword', + $STATE_TEXT_FLOAT => 'oword', + ); + +# State: is a text state..."include state" is more correct +sub state_is_text { my $st=shift @_; - return ($st>0); + return ($st>=$STATE_TEXT); } -# Status: get CNT corresponding to text status (or undef) -sub status_text_cnt { +# State: is a parsed/included region, text or preamble +sub state_is_parsed { my $st=shift @_; - if ($st==$STATUS_TEXT) {return $CNT_WORDS_TEXT;} - if ($st==$STATUS_TEXT_HEADER) {return $CNT_WORDS_HEADER;} - if ($st==$STATUS_TEXT_FLOAT) {return $CNT_WORDS_CAPTION;} - return undef; + return ($st>=$STATE_TEXT || $st==$STATE_PREAMBLE); } -# Status: is an exclude status -sub status_is_exclude { +# State: get CNT corresponding to text state (or undef) +sub state_text_cnt { my $st=shift @_; - return ($st<0); + return $state2cnt{$st}; } -# Status: \begin and \end should be processed -sub status_inc_envir { +# State: is an exclude state +sub state_is_exclude { my $st=shift @_; - return ($st>-2); + return ($st<=$__STATE_EXCLUDE_); } -# Map status number to CNT value -sub status_to_cnt { +# State: \begin and \end should be processed +sub state_inc_envir { my $st=shift @_; - if ($st==$STATUS_TO_INLINEMATH) {return $CNT_COUNT_INLINEMATH;} - if ($st==$STATUS_TO_DISPLAYMATH) {return $CNT_COUNT_DISPLAYMATH;} - return undef; + return ($st>$STATE_EXCLUDE_STRONG); } -# Status as text -sub status_to_text { +# State as text (used with printstate) +# TODO: Should do a conversion based on STATE values. +sub state_to_text { my $st=shift @_; return $st; } -# Status as text -sub status_to_style { - return 'word'.status_to_text(@_); +# Style to use with text state +sub state_to_style { + return $state2style{shift @_}; } -### Token types -# -1: space -# 0: comment -# 1: word (or other forms of text or text components) -# 2: symbol (not word, e.g. punctuation) -# 3: macro -# 4: curly braces {} -# 5: brackets [] -# 6: maths -# 9: line break in file -# 999: end of line or blank line -# 666: TeXcount instruction (%TC:instruction) -my $TOKEN_SPACE=-1; -my $TOKEN_COMMENT=0; -my $TOKEN_WORD=1; -my $TOKEN_SYMBOL=2; -my $TOKEN_MACRO=3; -my $TOKEN_BRACE=4; -my $TOKEN_BRACKET=5; -my $TOKEN_MATH=6; -my $TOKEN_LINEBREAK=9; -my $TOKEN_TC=666; -my $TOKEN_END=999; +# Add new counter with the given key and description +sub add_new_counter { + my ($key,$desc,$like)=@_; + my $state=$SIZE_CNT; + my $cnt=$SIZE_CNT; + $key=lc($key); + if (!defined $like){$like=$CNT_WORDS_OTHER;} + $key2cnt{$key}=$cnt; + push @countkey,$key; + push @countdesc,$desc; + if (defined $sumweights[$like]) {$sumweights[$cnt]=$sumweights[$like];} + $key2state{$key}=$state; + $state2cnt{$state}=$cnt; + $state2style{$state}='altwd'; + push @STATE_MID_PRIORITY,$state; + $SIZE_CNT++; +} ###### Set global definitions @@ -243,20 +374,55 @@ my %BreakPoints=%{$BreakPointsOptions{'none'}}; ### Print styles # Definition of different print styles: maps of class labels # to ANSI codes. Class labels are as used by HTML styles. -my @STYLES=(); -my %STYLE; -$STYLES[0]={'error'=>'bold red'}; -$STYLES[1]={%{$STYLES[0]},''=>'normal',' '=>'normal', - 'word1'=>'blue','word2'=>'bold blue','word3'=>'blue', - 'grouping'=>'red','document'=>'red','mathgroup'=>'magenta', - 'state'=>'cyan underline','cumsum'=>'yellow'}; -$STYLES[2]={%{$STYLES[1]}, - 'command'=>'green','ignore'=>'cyan', - 'exclcommand'=>'yellow','exclgroup'=>'yellow','exclmath'=>'yellow'}; -$STYLES[3]={%{$STYLES[2]}, - 'tc'=>'bold yellow','comment'=>'yellow','option'=>'yellow', - 'fileinclude'=>'bold green'}; -$STYLES[4]={%{$STYLES[3]}}; +my %STYLES; +my $STYLE_EMPTY=' '; +my $STYLE_BLOCK='-'; +my $NOSTYLE=' '; +$STYLES{'Errors'}={'error'=>'bold red'}; +$STYLES{'Words'}={'word'=>'blue','hword'=>'bold blue','oword'=>'blue','altwd'=>'blue'}; +$STYLES{'Macros'}={'cmd'=>'green','fileinc'=>'bold green'}; +$STYLES{'Options'}={'option'=>'yellow','optparm'=>'green'}; +$STYLES{'Ignored'}={'ignore'=>'cyan','math'=>'magenta'}; +$STYLES{'Excluded'}={'exclcmd'=>'yellow','exclenv'=>'yellow','exclmath'=>'yellow','mathcmd'=>'yellow'}; +$STYLES{'Groups'}={'document'=>'red','envir'=>'red','mathgroup'=>'magenta'}; +$STYLES{'Comments'}={'tc'=>'bold yellow','comment'=>'yellow'}; +$STYLES{'Sums'}={'cumsum'=>'yellow'}; +$STYLES{'States'}={'state'=>'cyan underline'}; +$STYLES{'<core>'}={%{$STYLES{'Errors'}},$STYLE_EMPTY=>$NOSTYLE,'<printlevel>'=>1}; +$STYLES{0}={%{$STYLES{'Errors'}},'<printlevel>'=>0}; +$STYLES{1}={%{$STYLES{'<core>'}},%{$STYLES{'Words'}},%{$STYLES{'Groups'}},%{$STYLES{'Sums'}}}; +$STYLES{2}={%{$STYLES{1}},%{$STYLES{'Macros'}},%{$STYLES{'Ignored'}},%{$STYLES{'Excluded'}}}; +$STYLES{3}={%{$STYLES{2}},%{$STYLES{'Options'}},%{$STYLES{'Comments'}},'<printlevel>'=>2}; +$STYLES{4}={%{$STYLES{3}},%{$STYLES{'States'}}}; +$STYLES{'All'}=$STYLES{4}; +my %STYLE=%{$STYLES{$defaultVerbosity}}; + +my @STYLE_LIST=('error','word','hword','oword','altwd', + 'ignore','document','cmd','exclcmd','option','optparm','envir','exclenv', + 'mathgroup','exclmath','math','mathcmd','comment','tc','fileinc','state','cumsum'); +my %STYLE_DESC=( + 'error' => 'ERROR: TeXcount error message', + 'word' => 'Text which is counted: counted as text words', + 'hword' => 'Header and title text: counted as header words', + 'oword' => 'Caption text and footnotes: counted as caption words', + 'altwd' => 'Words in user specified counters: counted in separate counters', + 'ignore' => 'Ignored text or code: excluded or ignored', + 'document' => '\documentclass: document start, beginning of preamble', + 'cmd' => '\macro: macro not counted, but parameters may be', + 'exclcmd' => '\macro: macro in excluded region', + 'option' => '[Macro options]: not counted', + 'optparm' => '[Optional parameter]: content parsed and styled as counted', + 'envir' => '\begin{name} \end{name}: environment', + 'exclenv' => '\begin{name} \end{name}: environment in excluded region', + 'mathgroup' => '$ $: counted as one equation', + 'exclmath' => '$ $: equation in excluded region', + 'math' => '2+2=4: maths (inside $...$ etc.)', + 'mathcmd' => '$\macro$: macros inside maths', + 'comment' => '% Comments: not counted', + 'tc' => '%TC:TeXcount instructions: not counted', + 'fileinc' => 'File to include: not counted but file may be counted later', + 'state' => '[state]: internal TeXcount state', + 'cumsum' => '[cumsum]: cumulative sum count'); ###### Define what a word is and language options @@ -279,18 +445,22 @@ my $LetterPattern=$NamedLetterPattern{'default'}; # List of regexp patterns that should be analysed as words. # Use @ to represent a letter, will be substituted with $LetterPattern. # Named patterns may replace or be appended to the original patterns. +# Apply_Options() results in a call to apply_language_options() which +# constructs $WordPattern based on $LetterPattern, @WordPatterns and +# alphabet/logogram settings. my %NamedWordPattern; $NamedWordPattern{'letters'}='@'; -$NamedWordPattern{'words'}='(@+|\{@+\})([\-\'\.]?(@+|\{@+\}))*'; +$NamedWordPattern{'words'}='(@+|@+\{@+\}|\{@+\}@+)([\-\'\.]?(@+|\{@+\}))*'; my @WordPatterns=($NamedWordPattern{'words'}); +my $WordPattern; # Regex matching a word (defined in apply_language_options()) ### Macro option regexp list # List of regexp patterns to be gobbled as macro option in and after # a macro. my %NamedMacroOptionPattern; -$NamedMacroOptionPattern{'default'}='\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]'; -$NamedMacroOptionPattern{'relaxed'}='\[[^\[\]\n]*\]'; -$NamedMacroOptionPattern{'restricted'}=$NamedMacroOptionPattern{'default'}; +$NamedMacroOptionPattern{'default'}='\[[^\[\]\n]*\]'; +$NamedMacroOptionPattern{'relaxed'}='\[[^\[\]\n]*(\n[^\[\]\n]+)\n?\]'; +$NamedMacroOptionPattern{'restricted'}='\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]'; my $MacroOptionPattern=$NamedMacroOptionPattern{'default'}; ### Alternative language encodings @@ -367,133 +537,287 @@ END ###### Define core rules -### Macros for headers -# Macros that identify headers: i.e. following token or -# {...} is counted as header. The =>[2] indicates transition to -# state 2 which is used within headers (although the value is -# actually never used). This is copied to %TeXmacro and the -# only role of defining it here is that the counter for the number -# of headers is incremented by one. -my %TeXheader=('\title'=>[2],'\part'=>[2],'\chapter'=>[2], - '\section'=>[2],'\subsection'=>[2],'\subsubsection'=>[2], - '\paragraph'=>[2],'\subparagraph'=>[2]); - ### Macros indicating package inclusion # Will always be assumed to take one parameter (plus options). -my %TeXpackageinc=('\usepackage'=>1); +# Gets added to TeXmacro. After that, values are not used, only membership. +my %TeXpackageinc=('\usepackage'=>1,'\RequirePackage'=>1); ### Macros that are counted within the preamble # The preamble is the text between \documentclass and \begin{document}. # Text and macros in the preamble is ignored unless specified here. The -# value is the status (1=text, 2=header, etc.) they should be interpreted as. +# value is the states (1=text, 2=header, etc.) they should be interpreted as. # Note that only the first unit (token or {...} block) is counted. -my %TeXpreamble=( - '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3], - '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3], - '\title'=>[2]); +# Gets added to TeXmacro. Is used within preambles only. +my %TeXpreamble; +add_keys_to_hash(\%TeXpreamble,['header'],'\title'); +add_keys_to_hash(\%TeXpreamble,['other'],'\thanks'); +add_keys_to_hash(\%TeXpreamble,['xxx','xxx'],'\newcommand','\renewcommand'); +add_keys_to_hash(\%TeXpreamble,['xxx','xxx','xxx'],'\newenvironment','\renewenvironment'); ### In floats: include only specific macros # Macros used to identify caption text within floats. -my %TeXfloatinc=('\caption'=>[3]); +# Gets added to TeXmacro. Is used within floats only. +my %TeXfloatinc=('\caption'=>['otherword']); ### How many tokens to gobble after macro # Each macro is assumed to gobble up a given number of # tokens (or {...} groups), as well as options [...] before, within # and after. The %TeXmacro hash gives a link from a macro -# (or beginNAME for begin-end groups without the backslash) +# (or beginNAME for environment with no the backslash) # to either an integer giving the number of tokens to ignore -# or to an array (specified as [num,num,...]) of length N where -# N is the number of tokens to be read with the macro and the -# array values tell how each is to be interpreted (see the status -# values: 0=ignore, 1=count, etc.). Thus specifying a number N is -# equivalent to specifying an array [0,...,0] of N zeros. +# or to an array (specified as [rule,rule,...]) of length N where +# N is the number of parameters to be read with the macro. The +# array values tell how each is to be interpreted (see the parser state +# keywords for valid values). Thus specifying a number N is +# equivalent to specifying an array of N 'ignore' rules. # # For macros not specified here, the default value is 0: i.e. -# no tokens are excluded, but [...] options are. Header macros -# specified in %TeXheader are automatically included here. -my %TeXmacro=(%TeXheader,%TeXpreamble,%TeXfloatinc, - '\documentclass'=>1,'\documentstyle'=>1,'\usepackage'=>1, '\hyphenation'=>1, - '\pagestyle'=>1,'\thispagestyle'=>1, '\pagenumbering'=>1,'\markboth'=>1, '\markright'=>1, - '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3], - '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3], - '\newfont'=>2,'\newtheorem'=>2,'\bibliographystyle'=>1, '\bibliography'=>1, - '\parbox'=>1, '\marginpar'=>[3],'\makebox'=>0, '\raisebox'=>1, '\framebox'=>0, - '\newsavebox'=>1, '\sbox'=>1, '\savebox'=>2, '\usebox'=>1,'\rule'=>2, - '\footnote'=>[3],'\label'=>1, '\ref'=>1, '\pageref'=>1, '\bibitem'=>1, - '\cite'=>1, '\citep'=>1, '\citet'=>1, '\citeauthor'=>1, '\citealt'=>1, '\nocite'=>1, - '\eqlabel'=>1, '\eqref'=>1,'\hspace'=>1, '\vspace'=>1, '\addvspace'=>1, - '\input'=>1, '\include'=>1, '\includeonly'=>1,'\includegraphics'=>1, - '\newlength'=>1, '\setlength'=>2, '\addtolength'=>2,'\settodepth'=>2, - '\settoheight'=>2, '\settowidth'=>2,'\newcounter'=>1, '\setcounter'=>2, - '\addtocounter'=>2,'\stepcounter'=>1, '\refstepcounter'=>1, '\usecounter'=>1, - '\alph'=>1, '\arabic'=>1, '\fnsymbol'=>1, '\roman'=>1, '\value'=>1, - '\cline'=>1, '\multicolumn'=>3,'\typeout'=>1, '\typein'=>1, - 'beginlist'=>2, 'beginminipage'=>1, 'begintabular'=>1, - 'beginthebibliography'=>1,'beginlrbox'=>1, - '\begin'=>1,'\end'=>1,'\title'=>[2], - '\addtocontents'=>2,'\addcontentsline'=>3, - '\uppercase'=>0,'\lowercase'=>0); - -### Macros that should be counted as one or more words -# Macros that represent text may be declared here. The value gives -# the number of words the macro represents. -my %TeXmacroword=('\LaTeX'=>1,'\TeX'=>1); - -### Begin-End groups -# Identified as begin-end groups, and define =>state. The -# states used corresponds to the elements of the count array, and -# are: -# 0: Not included -# 1: Text, words included in text count -# 2: Header, words included in header count -# 3: Float caption, words included in float caption count -# 6: Inline mathematics, words not counted -# 7: Displayed mathematics, words not counted -# -1: Float, not included, but looks for captions +# no tokens are excluded, but [...] options are. +my %TeXmacro=(%TeXpreamble,%TeXfloatinc,%TeXpackageinc); +add_keys_to_hash(\%TeXmacro,['text'], + '\textnormal','\textrm','\textit','\textbf','\textsf','\texttt','\textsc','\textsl','\textup','\textmd', + '\makebox','\mbox','\framebox','\fbox','\uppercase','\lowercase','\textsuperscript','\textsubscript', + '\citetext'); +add_keys_to_hash(\%TeXmacro,['[','text'], + '\item'); +add_keys_to_hash(\%TeXmacro,['[','ignore'], + '\linebreak','\nolinebreak','\pagebreak','\nopagebreak'); +add_keys_to_hash(\%TeXmacro,0, + '\maketitle','\indent','\noindent', + '\centering','\raggedright','\raggedleft','\clearpage','\cleardoublepage','\newline','\newpage', + '\smallskip','\medskip','\bigskip','\vfill','\hfill','\hrulefill','\dotfill', + '\normalsize','\small','\footnotesize','\scriptsize','\tiny','\large','\Large','\LARGE','\huge','\Huge', + '\normalfont','\em','\rm','\it','\bf','\sf','\tt','\sc','\sl', + '\rmfamily','\sffamily','\ttfamily','\upshape','\itshape','\slshape','\scshape','\mdseries','\bfseries', + '\selectfont', + '\tableofcontents','\listoftables','\listoffigures'); +add_keys_to_hash(\%TeXmacro,1, + '\begin','\end', + '\documentclass','\documentstyle','\hyphenation','\pagestyle','\thispagestyle', + '\author','\date', + '\bibliographystyle','\bibliography','\pagenumbering','\markright', + '\includeonly','\includegraphics','\special', + '\label','\ref','\pageref','\bibitem', + '\eqlabel','\eqref','\hspace','\vspace','\addvspace', + '\newsavebox','\usebox', + '\newlength','\newcounter','\stepcounter','\refstepcounter','\usecounter', + '\fontfamily','\fontseries', + '\alph','\arabic','\fnsymbol','\roman','\value', + '\typeout', '\typein','\cline'); +add_keys_to_hash(\%TeXmacro,2, + '\newfont','\newtheorem','\sbox','\savebox','\rule','\markboth', + '\setlength','\addtolength','\settodepth','\settoheight','\settowidth','\setcounter', + '\addtocontents','\addtocounter', + '\fontsize'); +add_keys_to_hash(\%TeXmacro,3,'\multicolumn','\addcontentsline'); +add_keys_to_hash(\%TeXmacro,6,'\DeclareFontShape'); +add_keys_to_hash(\%TeXmacro,['[','text','ignore'], + '\cite','\nocite','\citep','\citet','\citeauthor','\citeyear','\citeyearpar', + '\citealp','\citealt','\Citep','\Citet','\Citealp','\Citealt','\Citeauthor'); +add_keys_to_hash(\%TeXmacro,['ignore','text'],'\parbox','\raisebox'); +add_keys_to_hash(\%TeXmacro,['otherword'],'\marginpar','\footnote','\footnotetext'); +add_keys_to_hash(\%TeXmacro,['header'], + '\title','\part','\chapter','\section','\subsection','\subsubsection','\paragraph','\subparagraph'); +add_keys_to_hash(\%TeXmacro,['xxx','xxx','text'],'\multicolumn'); +add_keys_to_hash(\%TeXmacro,['xxx','xxx'],'\newcommand','\renewcommand'); +add_keys_to_hash(\%TeXmacro,['xxx','xxx','xxx'],'\newenvironment','\renewenvironment'); + +### Environments +# The %TeXenvir hash provides content parsing rules (parser states). +# Environments that are not defined will be counted as the surrounding text. # -# 4 and 5 are used to count number of headers and floats -# and are not used as states. -# -# Groups that are not defined will be counted as the surrounding text. +# Parameters taken by the \begin{environment} are defined in %TeXmacro. # # Note that some environments may only exist within math-mode, and # therefore need not be defined here: in fact, they should not as it # is not clear if they will be in inlined or displayed math. -my %TeXgroup=('document'=>1,'letter'=>1,'titlepage'=>0, - 'center'=>1,'flushleft'=>1,'flushright'=>1, - 'abstract'=>1,'quote'=>1,'quotation'=>1,'verse'=>1,'minipage'=>1, - 'verbatim'=>-4,'tikzpicture'=>-4, - 'description'=>1,'enumerate'=>1,'itemize'=>1,'list'=>1, - 'theorem'=>1,'lemma'=>1,'definition'=>1,'corollary'=>1,'example'=>1, - 'math'=>6,'displaymath'=>7,'equation'=>7,'eqnarray'=>7,'align'=>7, - 'equation*'=>7,'eqnarray*'=>7,'align*'=>7, - 'figure'=>-1,'float'=>-1,'picture'=>-1,'table'=>-1, - 'tabbing'=>0,'tabular'=>0,'thebibliography'=>0,'lrbox'=>0); +my %TeXenvir; +add_keys_to_hash(\%TeXenvir,'ignore', + 'titlepage','tabbing','tabular','tabular*','thebibliography','lrbox'); +add_keys_to_hash(\%TeXenvir,'text', + 'document','letter','center','flushleft','flushright', + 'abstract','quote','quotation','verse','minipage', + 'description','enumerate','itemize','list', + 'theorem','thm','lemma','definition','corollary','example','proof','pf'); +add_keys_to_hash(\%TeXenvir,'inlinemath', + 'math'); +add_keys_to_hash(\%TeXenvir,'displaymath', + 'displaymath','equation','equation*','eqnarray','eqnarray*','align','align*',); +add_keys_to_hash(\%TeXenvir,'float', + 'float','picture','figure','figure*','table','table*'); +add_keys_to_hash(\%TeXenvir,'xall', + 'verbatim','tikzpicture'); + +# Environment parameters +my $PREFIX_ENVIR='begin'; # Prefix used for environment names +add_keys_to_hash(\%TeXmacro,1, + 'beginthebibliography','beginlrbox','beginminipage'); +add_keys_to_hash(\%TeXmacro,2, + 'beginlist'); +add_keys_to_hash(\%TeXmacro,['ignore'], + 'beginletter'); +add_keys_to_hash(\%TeXmacro,['xxx'], + 'begintabular'); +add_keys_to_hash(\%TeXmacro,['ignore','xxx'], + 'begintabular*'); +add_keys_to_hash(\%TeXmacro,['[','text'], + 'begintheorem','beginthm','beginlemma','begindefinition','begincorollary','beginexample','beginproof','beginpf'); +add_keys_to_hash(\%TeXmacro,['nooptions'], + 'beginverbatim'); + +### Macros that should be counted as one or more words +# Macros that represent text may be declared here. The value gives +# the number of words the macro represents. +my %TeXmacrocount=('\LaTeX'=>1,'\TeX'=>1,'beginabstract'=>['header','headerword']); ### Macros for including tex files # Allows \macro{file} or \macro file. If the value is 0, the filename will # be used as is; if it is 1, the filetype .tex will be added if the # filename is without filetype; if it is 2, the filetype .tex will be added. -my %TeXfileinclude=('\input'=>1,'\include'=>2); +my %TeXfileinclude=('\input'=>'input','\include'=>'texfile'); + +### Convert state keys to codes +convert_hash(\%TeXpreamble,\&keyarray_to_state); +convert_hash(\%TeXfloatinc,\&keyarray_to_state); +convert_hash(\%TeXmacro,\&keyarray_to_state); +convert_hash(\%TeXmacrocount,\&keyarray_to_cnt); +convert_hash(\%TeXenvir,\&key_to_state); ###### Define package specific rules ### Package rule definitions +my %PackageTeXpreamble=(); # TeXpreamble definitions per package +my %PackageTeXpackageinc=(); # TeXpackageinc definitions per package +my %PackageTeXfloatinc=(); # TeXfloatinc definitions per package my %PackageTeXmacro=(); # TeXmacro definitions per package -my %PackageTeXmacroword=(); # TeXmacroword definitions per package -my %PackageTeXheader=(); # TeXheader definitions per package -my %PackageTeXgroup=(); # TeXgroup definitions per package -my %PackageTeXfileinclude=(); # TeXgroup definitions per package +my %PackageTeXmacrocount=(); # TeXmacrocount definitions per package +my %PackageTeXenvir=(); # TeXenvir definitions per package +my %PackageTeXfileinclude=(); # TeXfileinclude definitions per package +my %PackageSubpackage=(); # Subpackages to include (listed in array [...]) -# Rules for package psfig -$PackageTeXmacro{'psfig'}={('\psfig'=>1)}; # Rules for bibliography inclusion -$PackageTeXmacroword{'%incbib'}={'thebibliography'=>1}; -$PackageTeXmacro{'%incbib'}={}; -$PackageTeXgroup{'%incbib'}={'thebibliography'=>1}; -$PackageTeXfileinclude{'%incbib'}={'\bibliography'=>'bbl'}; +$PackageTeXmacrocount{'%incbib'}={'beginthebibliography'=>['header','hword']}; +$PackageTeXmacro{'%incbib'}={'\bibliography'=>1}; +$PackageTeXenvir{'%incbib'}={'thebibliography'=>'text'}; +$PackageTeXfileinclude{'%incbib'}={'\bibliography'=>'<bbl>'}; + +# Rules for package alltt +$PackageTeXenvir{'alltt'}={ + 'alltt'=>'xall'}; + +# Rules for package babel +# NB: Only core macros implemented, those expected found in regular documents +$PackageTeXenvir{'babel'}={ + 'otherlanguage'=>'text','otherlanguage*'=>'text'}; +$PackageTeXmacro{'babel'}={ + '\selectlanguage'=>1,'\foreignlanguage'=>['ignore','text'], + 'beginotherlanguage'=>1,'beginotherlanguage*'=>1}; + +# Rules for package comment +$PackageTeXenvir{'comment'}={ + 'comment'=>'xxx'}; + +# Rules for package color +$PackageTeXmacro{'color'}={ + '\textcolor'=>['ignore','text'],'\color'=>1,'\pagecolor'=>1,'\normalcolor'=>0, + '\colorbox'=>['ignore','text'],'\fcolorbox'=>['ignore','ignore','text'], + '\definecolor'=>3,\'DefineNamedColor'=>4}; + +# Rules for package endnotes +$PackageTeXmacro{'endnotes'}={'\endnote'=>['oword'],'\endnotetext'=>['oword'],'\addtoendnotetext'=>['oword']}; + +# Rules for package fancyhdr +$PackageTeXmacro{'fancyhdr'}={ + '\fancyhf'=>1,'\lhead'=>1,'\chead'=>1,'\rhead'=>1,'\lfoot'=>1,'\cfoot'=>1,'\rfoot'=>1}; + +# Rules for package geometry +$PackageTeXmacro{'geometry'}={ + '\geometry'=>1,'\newgeometry'=>1,'\restoregeometry'=>0,,'\savegeometry'=>1,'\loadgeometry'=>1}; + +# Rules for package graphicx +$PackageTeXmacro{'graphicx'}={ + '\DeclareGraphicsExtensions'=>1,'\graphicspath'=>1, + '\includegraphics'=>['[','ignore','ignore'], + '\includegraphics*'=>['[','ignore','[','ignore','[','ignore','ignore'], + '\rotatebox'=>1,'\scalebox'=>1,'\reflectbox'=>1,'\resizebox'=>1}; + +# Rules for package hyperref (urls themselves counted as one word) +# NB: \hyperref[label]{text} preferred over \hyperref{url}{category}{name}{text} +# NB: Macros for use in Form environments not implemented +$PackageTeXmacro{'hyperref'}={ + '\hyperref'=>['[','ignore','text'], + '\url'=>1,'\nolinkurl'=>1,'\href'=>['ignore','text'], + '\hyperlink'=>['ignore','text'],'\hypertarget'=>['ignore','text'], + '\hyperbaseurl'=>1,'\hyperimage'=>['ignore','text'],'\hyperdef'=>['ignore','ignore','text'], + '\phantomsection'=>0,'\autoref'=>1,'\autopageref'=>1, + '\hypersetup'=>1,'\urlstyle'=>1, + '\pdfbookmark'=>2,'\currentpdfbookmark'=>2,'\subpdfbookmark'=>2,'\belowpdfbookmark'=>2, + '\pdfstringref'=>2,'\texorpdfstring'=>['text','ignore'], + '\hypercalcbp'=>1,'\Acrobatmenu'=>2}; +$PackageTeXmacrocount{'hyperref'}={ + '\url'=>1,'\nolinkurl'=>1}; + +# Rules for package import +$PackageTeXfileinclude{'import'}={ + '\import'=>'dir file','\subimport'=>'subdir file', + '\inputfrom'=>'dir file','\subinputfrom'=>'subdir file', + '\includefrom'=>'dir file','\subincludefrom'=>'subdir file'}; + +# Rules for package inputenc +$PackageTeXmacro{'inputenc'}={ + '\inputencoding'=>1}; + +# Rules for package listings +$PackageTeXenvir{'listings'}={'lstlisting'=>'xall'}; +$PackageTeXmacro{'listings'}={'\lstset'=>['ignore'],'\lstinputlisting'=>['ignore']}; + +# Rules for package psfig +$PackageTeXmacro{'psfig'}={'\psfig'=>1}; + +# Rules for package sectsty +$PackageTeXmacro{'sectsty'}={ + '\allsectionsfont'=>1,'\partfont'=>1,'\chapterfont'=>1,'\sectionfont'=>1, + '\subsectionfont'=>1,'\subsubsectionfont'=>1,'\paragraphfont'=>1,'\subparagraphfont'=>1, + '\minisecfont'=>1,'\partnumberfont'=>1,'\parttitlefont'=>1,'\chapternumberfont'=>1, + '\chaptertitlefont'=>1,'\nohang'=>0}; + +# Rules for package setspace +$PackageTeXenvir{'setspace'}={ + 'singlespace'=>'text','singlespace*'=>'text','onehalfspace'=>'text','doublespace'=>'text', + 'spacing'=>'text'}; +$PackageTeXmacro{'setspace'}={ + 'beginspacing'=>1, + '\singlespacing'=>0,'\onehalfspacing'=>0,'\doublespacing'=>0, + '\setstretch'=>1,'\SetSinglespace'=>1}; + +# Rules for package subfiles +$PackageTeXfileinclude{'subfiles'}={ + '\subfile'=>'file'}; + +# Rules for package url +# NB: \url|...| variant not implemented, only \url{...} +# NB: \urldef{macro}{url} will not be counted +$PackageTeXmacro{'url'}={ + '\url'=>1,'\urldef'=>2,'\urlstyle'=>1,'\DeclareUrlCommand'=>['ignore','xxx']}; +$PackageTeXmacro{'setspace'}={ + '\url'=>1}; + +# Rules for package wrapfig +$PackageTeXenvir{'wrapfig'}={ + 'wrapfigure'=>'float','wraptable'=>'float'}; +$PackageTeXmacro{'wrapfig'}={ + 'beginwrapfigure'=>2,'beginwraptable'=>2}; + +# Rules for package xcolor (reimplements the color package) +# NB: only main macros (mostly from package color) included +$PackageTeXmacro{'xcolor'}={ + '\textcolor'=>['ignore','text'],'\color'=>1,'\pagecolor'=>1,'\normalcolor'=>0, + '\colorbox'=>['ignore','text'],'\fcolorbox'=>['ignore','ignore','text'], + '\definecolor'=>3,\'DefineNamedColor'=>4, + '\colorlet'=>2}; + ###### Main script @@ -523,13 +847,14 @@ sub MAIN { if ($showVersion && !$htmlstyle && !($briefsum && $totalflag)) { print "\n=== LaTeX word count (TeXcount version $versionnumber) ===\n\n"; } - conditional_print_help_style(); + conditional_print_style_list(); my $totalcount=Parse_file_list(@toplevelfiles); conditional_print_total($totalcount); Report_Errors(); if ($optionWordFreq || $optionWordClassFreq) {print_word_freq();} + if ($optionMacroStat) {print_macro_stat();} } elsif ($showcodes>1) { - conditional_print_help_style(); + conditional_print_style_list(); } else { error($Main,'No files specified.'); } @@ -540,7 +865,9 @@ sub MAIN { sub Initialise { _option_subcount(); # Windows settings - if ($^O=~/^MSWin/) { + if ($^O eq 'MSWin32') { + } elsif ($^O=~/^MSWin/) { + #DELETE: do not overrule colour setting option_ansi_colours(0); } } @@ -548,21 +875,33 @@ sub Initialise { # Check arguments, exit on exit condition sub Check_Arguments { my @args=@_; + my $arg=$args[0]; if (!@args) { print_version(); - print_syntax(); - print_reference(); + print_short_help(); exit; - } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))$/) { + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))$/) { print_help(); exit; - } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))=/) { - print_help_on("$'"); + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))=(.*)$/) { + print_help_on_rule($4); + exit; + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(opt|options?)$/) { + print_syntax(); + exit; + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(opt|options?)=(.*)$/) { + print_syntax_subset($5); exit; - } elsif ($args[0]=~/^--?(ver|version)$/) { + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(styles?)$/) { + print_help_on_styles(); + exit; + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(styles?)=(\w+)$/) { + print_help_on_styles($5); + exit; + } elsif ($arg=~/^--?(ver|version)$/) { print_version(); exit; - } elsif ($args[0]=~/^--?(lic|license|licence)$/) { + } elsif ($arg=~/^--?(lic|license|licence)$/) { print_license(); exit; } @@ -576,8 +915,8 @@ sub Parse_Arguments { foreach my $arg (@args) { if (parse_option($arg)) {next;} if ($arg=~/^\-/) { - print 'Invalid opton '.$arg."\n"; - print_syntax(); + print "Invalid opton $arg \n\n"; + print_short_help(); exit; } $arg=~s/\\/\//g; @@ -600,8 +939,8 @@ sub parse_option { # Parse presetting options sub parse_options_preset { my $arg=shift @_; - if ($arg=~/^-(opt|option|options|optionfile)=/) { - _parse_optionfile($'); + if ($arg=~/^-(opt|option|options|optionfile)=(.*)$/) { + _parse_optionfile($2); } else {return 0;} return 1; @@ -622,6 +961,11 @@ sub parse_options_parsing { $globalworkdir=$1; $globalworkdir=~s:([^\/\\])$:$1\/:; } + elsif ($arg eq '-auxdir') {$auxdir=undef;} + elsif ($arg=~/^-auxdir=(.*)$/) { + $auxdir=$1; + $auxdir=~s:([^\/\\])$:$1\/:; + } elsif ($arg =~/^-(enc|encode|encoding)=(.+)$/) {$encoding=$2;} elsif ($arg =~/^-(utf8|unicode)$/) {$encoding='utf8';} elsif ($arg =~/^-(alpha(bets?)?)=(.*)$/) {set_script_options(\@AlphabetScripts,$3);} @@ -649,6 +993,7 @@ sub parse_options_counts { elsif ($arg eq '-freq') {$optionWordFreq=1;} elsif ($arg =~/^-freq=(\d+)$/) {$optionWordFreq=$1;} elsif ($arg eq '-stat') {$optionWordClassFreq=1;} + elsif ($arg =~/^-macro-?(stat|freq)$/) {$optionMacroStat=1;} else {return 0;} return 1; } @@ -657,11 +1002,11 @@ sub parse_options_counts { sub _option_sum { my $arg=shift @_; if (!defined $arg) { - @sumweights=(1,1,1,0,0,1,1); + @sumweights=(0,1,1,1,0,0,1,1); } elsif ($arg=~/^(\d+(\.\d*)?(,\d+(\.\d*)?){0,6})$/) { - @sumweights=split(',',$1); + @sumweights=(0,split(',',$1)); } else { - print STDERR "Warning: Option value ".$arg." not valid, ignoring option.\n"; + print STDERR "Warning: Option value $arg not valid, ignoring option.\n"; } } @@ -674,7 +1019,7 @@ sub _option_subcount { } elsif (my $option=$BreakPointsOptions{$arg}) { %BreakPoints=%{$option}; } else { - print STDERR "Warning: Option value ".$arg." not valid, using default instead.\n"; + print STDERR "Warning: Option value $arg not valid, using default instead.\n"; %BreakPoints=%{$BreakPointsOptions{'default'}}; } } @@ -683,20 +1028,18 @@ sub _option_subcount { sub parse_options_output { my $arg=shift @_; if ($arg eq '-strict') {$strictness=1;} - elsif ($arg eq '-v0') {$verbose=0;} - elsif ($arg eq '-v1') {$verbose=1;} - elsif ($arg eq '-v2') {$verbose=2;} - elsif ($arg eq '-v3' || $arg eq '-v') {$verbose=3;} - elsif ($arg eq '-v4') {$verbose=3; $showstates=1;} + elsif ($arg eq '-v') {set_verbosity_options('3');} + elsif ($arg =~/^-v([0-4=+\-].*)/) {set_verbosity_options($1);} elsif ($arg =~/^-showstates?$/ ) {$showstates=1;} - elsif ($arg =~/^-(q|quiet)$/ ) {$verbose=-1;} + elsif ($arg =~/^-(q|quiet)$/ ) {$printlevel=-1;} elsif ($arg =~/^-(template)=(.*)$/ ) {_set_output_template($2);} elsif ($arg eq '-split') {$optionFast=1;} elsif ($arg eq '-nosplit') {$optionFast=0;} elsif ($arg eq '-showver') {$showVersion=1;} elsif ($arg eq '-nover') {$showVersion=-1;} - elsif ($arg =~/^-nosep(s|arators?)?$/ ) {$separator='';} - elsif ($arg =~/^-sep(arators?)?=(.*)$/ ) {$separator=$2;} + elsif ($arg =~/^-nosep(arator)?s?$/ ) {$separator='';} + elsif ($arg =~/^-sep(arator)?s?=(.*)$/ ) {$separator=$2;} + elsif ($arg =~/^-out=(.*)/ ) {close STDOUT; open STDOUT,'>',$1;} else {return 0;} return 1; } @@ -705,8 +1048,8 @@ sub parse_options_output { sub _set_output_template { my $template=shift @_; $outputtemplate=$template; - if ($template=~/\{(S|SUM)[\?\}]/i && !@sumweights) { - @sumweights=(1,1,1,0,0,1,1); + if ($template=~/\{(SUM)[\?\}]/i && !@sumweights) { + @sumweights=(0,1,1,1,0,0,1,1); } if ($template=~/\{SUB\?/i && !$showsubcounts) { _option_subcount(); @@ -718,14 +1061,19 @@ sub parse_options_format { my $arg=shift @_; if ($arg eq '-brief') {$briefsum=1;} elsif ($arg eq '-total') {$totalflag=1;} - elsif ($arg eq '-0') {$briefsum=1;$totalflag=1;$verbose=-1;$finalLineBreak=0;} - elsif ($arg eq '-1') {$briefsum=1;$totalflag=1;$verbose=-1;} - elsif ($arg eq '-html' ) {option_ansi_colours(0);$htmlstyle = 2;} - elsif ($arg eq '-htmlcore' ) {option_ansi_colours(0);$htmlstyle = 1;} + elsif ($arg eq '-0') {$briefsum=1;$totalflag=1;$printlevel=-1;$finalLineBreak=0;} + elsif ($arg eq '-1') {$briefsum=1;$totalflag=1;$printlevel=-1;} + elsif ($arg eq '-htmlcore' ) {option_ansi_colours(0);$htmlstyle=1;} + elsif ($arg eq '-html' ) {option_ansi_colours(0);$htmlstyle=2;} + elsif ($arg eq '-tex' ) {option_ansi_colours(0);$texcodeoutput=1;} elsif ($arg =~/^\-(nocol|nc$)/) {option_ansi_colours(0);} elsif ($arg =~/^\-(col)$/) {option_ansi_colours(1);} elsif ($arg eq '-codes') {$showcodes=2;} elsif ($arg eq '-nocodes') {$showcodes=0;} + elsif ($arg =~/^-htmlfile=(.+)$/) {$HTMLfile=$1;} + elsif ($arg =~/^-cssfile=(.+)$/) {$CSSfile=$1;} + elsif ($arg =~/^-css=file:(.+)$/) {$CSSfile=$1;} + elsif ($arg =~/^-css=(.+)$/) {$CSShref=$1;} else {return 0;} return 1; } @@ -733,8 +1081,8 @@ sub parse_options_format { # Include options from option file sub _parse_optionfile { my $filename=shift @_; - open(FH,"<",$filename) - || die "Option file not found: ".$filename."\n"; + open(FH,'<',$filename) + || die "Option file not found: $filename\n"; my @options=<FH>; close(FH); s/^\s*(#.*|)//s for @options; @@ -744,7 +1092,7 @@ sub _parse_optionfile { foreach my $arg (@options) { __optionfile_tc($arg) || parse_option($arg) - || die "Invalid option ".$arg." in ".$filename."\n"; + || die "Invalid option $arg in $filename.\n"; } } @@ -752,12 +1100,12 @@ sub _parse_optionfile { sub __optionfile_tc { my $arg=shift @_; $arg=~s/^\%\s*// || return 0; - if ($arg=~/^subst\s+(\\\w+)\s+/i) { - $substitutions{$1}=$'; + if ($arg=~/^subst\s+(\\\w+)\s+(.*)$/i) { + $substitutions{$1}=$2; } elsif ($arg=~/^(\w+)\s+([\\]*\w+)\s+([^\s\n]+)(\s+([0-9]+))?/i) { - tc_macro_param_option($1,$2,$3,$5) || die "Invalid TC option: ".$arg."\n"; + tc_macro_param_option($Main,$1,$2,$3,$5) || die "Invalid TC option: $arg\n"; } else { - print "Invalid TC option format: ".$arg."\n"; + print "Invalid TC option format: $arg\n"; return 0; } return 1; @@ -766,10 +1114,10 @@ sub __optionfile_tc { # Parse file list and return total count sub Parse_file_list { my @files=@_; - my $listtotalcount=new_count("Total"); + my $listtotalcount=new_count('Total'); foreach (@files) {s/\\/\//g; s/ /\\ /g;} - if (@files) { - @files=<@files>; # For the sake of Windows: expand wildcards! + if (@files) { + @files=<@files>; # For the sake of Windows: expand wildcards! for my $file (@files) { my $filetotalcount=parse_file($file); add_to_total($listtotalcount,$filetotalcount); @@ -778,47 +1126,49 @@ sub Parse_file_list { if ($fileFromSTDIN) { my $filetotalcount=parse_file($_STDIN_); add_to_total($listtotalcount,$filetotalcount); - } + } return $listtotalcount; } # Parse file and included files, and return total count sub parse_file { my $file=shift @_; - $workdir=$globalworkdir; - if (!defined $workdir) { + if (defined ($workdir=$globalworkdir)) {} + elsif ($file eq $_STDIN_) {$workdir='';} + else { $workdir=$file; $workdir =~ s/^((.*[\\\/])?)[^\\\/]+$/$1/; } - if ($htmlstyle && ($verbose || !$totalflag)) {print "\n<div class='filegroup'>\n";} - my $filetotalcount=new_count("File(s) total: ".$file); + if ($htmlstyle && ($printlevel>0 || !$totalflag)) {print "\n<div class='filegroup'>\n";} + my $filetotalcount=new_count('File(s) total: '.$file); @filelist=(); - _add_file($filetotalcount,$file); + _add_file($filetotalcount,[$file,$workdir],'File: '.$file); foreach my $f (@filelist) { - _add_file($filetotalcount,$f,"Included file: ".$f); + _add_file($filetotalcount,$f,'Included file: '.$f->[0]); } if (!$totalflag && get_count($filetotalcount,$CNT_FILE)>1) { - if ($htmlstyle) {formatprint("Sum of files: ".$file."\n",'h2');} + if ($htmlstyle) {formatprint("Sum of files: $file\n",'h2');} print_count($filetotalcount,'sumcount'); } - if ($htmlstyle && ($verbose || !$totalflag)) {print "</div>\n\n";} + if ($htmlstyle && ($printlevel>0 || !$totalflag)) {print "</div>\n\n";} return $filetotalcount; } # Parse single file, included files will be appended to @filelist. sub _add_file { - my ($filetotalcount,$f,$title)=@_; - my $tex=TeXfile($f,$title); - my $fpath=$f; - $fpath=~s/^((.*[\\\/])?)[^\\\/]+$/$1/; + my ($filetotalcount,$pathfile,$title)=@_; + my @paths=@$pathfile; + my $file=shift @paths; + my $tex=TeXfile($file,$title); if (!defined $tex) { - error($Main,'File not found or not readable: '.$f); - } else { - parse($tex); - my $filecount=next_subcount($tex); - if (!$totalflag) {print_count($filecount);} - add_to_total($filetotalcount,$filecount); + error($Main,'File not found or not readable: '.$file); + return; } + $tex->{'PATH'}=\@paths; + parse($tex); + my $filecount=next_subcount($tex); + if (!$totalflag) {print_count($filecount);} + add_to_total($filetotalcount,$filecount); } ###### @@ -830,21 +1180,59 @@ sub _add_file { # Add file to list of files scheduled for parsing sub include_file { - my ($tex,$fname)=@_; - my $fpath=$workdir.$fname; + my ($tex,$state,$file,$refparam)=@_; + my %params=%$refparam; + my $type=$params{'<type>'}; + my @paths=@{$tex->{'PATH'}}; + my $filepath; + if ($type eq '<bbl>' && defined $auxdir) { + $auxdir=~s/([\\\/])*$/\//; + if (defined $globalworkdir) {@paths=[$auxdir];} + elsif ($auxdir=~/^(\w:)?[\\\/]/) {@paths=[$auxdir];} + else {@paths=[$workdir.$auxdir]} + } + foreach my $key (split(/\s+/,$type)) { + my $value=$params{$key}; + if ($key eq 'dir') {unshift @paths,$workdir.$value;} + elsif ($key eq 'subdir') {unshift @paths,$paths[0].$value;} + } + if ($file=~/^(\w:)?[\\\/]/) {$filepath=$file;} + else { + $filepath=_find_file_in_path($tex,$file,@paths) || BLOCK { + error($tex,'File '.$file.' not found in path ['.join(';',@paths).'].'); + return; + }; + } if ($includeTeX==2) { - my $bincode=read_binary($fpath) || BLOCK { - error($tex,"File $fpath not found."); + my $bincode=read_binary($filepath) || BLOCK { + error($tex,"File $filepath not readable."); return; }; flush_next($tex); line_return(0,$tex); - prepend_code($tex,$bincode,$fname); + # DELETE: prepend_code($tex,$bincode,$file); + my $texstate=texcode_insert_text($tex,$bincode,$file,@paths); + parse_all($tex,$state); + texcode_restore_state($tex,$texstate); } else { - push @filelist,$fpath; + push @filelist,[$filepath,@paths]; } } +# Seach path list to find file +sub _find_file_in_path { + my $tex=shift @_; + my $file=shift @_; + foreach my $path (@_) { + if (!$path=~/[\\\/]$/) {$path.='/';} + my $filepath=$path.$file; + if (-e $filepath) {return $filepath;} + elsif ($filepath=~/\.tex$/i) {} + elsif (-e $filepath.'.tex') {return $filepath.'.tex';} + } + return undef; +} + # Print count (total) if conditions are met sub conditional_print_total { my $sumcount=shift @_; @@ -852,7 +1240,7 @@ sub conditional_print_total { if ($totalflag && $briefsum && @sumweights) { print get_sum_count($sumcount),"\n"; } else { - if ($htmlstyle) {formatprint("Total word count",'h2');} + if ($htmlstyle) {formatprint('Total word count','h2');} print_count($sumcount,'sumcount'); } } @@ -875,24 +1263,49 @@ sub ansiprint { # Apply options to set values sub Apply_Options { - if ($showcodes>1 && $verbose<1) {$verbose=3;} - %STYLE=%{$STYLES[$verbose]}; - if (defined $encoding && $encoding eq 'guess') {$encoding=undef;} - if (!defined $encoding) { - } elsif (ref(find_encoding($encoding))) { - if (!$htmlstyle) {$outputEncoding=$encoding;} - } else { - error($Main,"Unknown encoding $encoding ignored."); - error_details($Main,'Valid encodings are: '.wrap('','',join(', ',Encode->encodings(':all')))); - $encoding=undef; - } - if (!defined $outputEncoding) {$outputEncoding='utf8';} - binmode STDIN; - binmode STDOUT,':encoding('.$outputEncoding.')'; + apply_encoding_options(); if ($htmlstyle>1) {html_head();} flush_errorbuffer($Main); apply_language_options(); if ($includeBibliography) {apply_include_bibliography();} + if ($showcodes>1 && !($STYLE{'<printlevel>'})) {%STYLE=%{$STYLES{'All'}};} + if ($showstates) {set_verbosity_options('+States');} + if (!@sumweights) {set_verbosity_options('-Sums');} + (defined $printlevel) + || (defined ($printlevel=$STYLE{'<printlevel>'})) + || ($printlevel=$defaultprintlevel); +} + +# Set or add styles included in the verbose output +sub set_verbosity_options { + my $verb=shift @_; + my $st; + if ($defaultprintlevel<1) {$defaultprintlevel=1;} + if ( $verb=~s/^([0-4])// || $verb=~s/^=([0-4])?// ) { + my $key=$1; + if (!defined $key) {%STYLE=(%{$STYLES{'<core>'}});} + elsif ($st=$STYLES{$key}) {%STYLE=(%$st);} + else { + error($Main,"Unknown verbosity option '$key'."); + %STYLE=(%{$STYLES{'<core>'}}); + } + } + while ($verb=~s/^([\+\-]?)(\w+)//) { + my $add=!($1 eq '-'); + my $key=$2; + if ($st=$STYLES{$key}) { + if ($add) {@STYLE{keys %$st}=values %$st;} + else {delete @STYLE{keys %$st};} + } elsif ($st=$STYLES{'All'}->{$key}) { + if ($add) {$STYLE{$key}=$st;} + elsif ($STYLE{$key}) {delete $STYLE{$key};} + } else { + error($Main,"Unknown verbosity option '$key'."); + } + } + if ($verb ne '') { + error($Main,"Unknown verbosity option format: $verb"); + } } # Set or add scripts to array of scripts @@ -907,13 +1320,14 @@ sub set_script_options { $scr='alphabetic'; } if ($scr=~/^[a-z]/) {$scr='Is_'.$scr;} - if (is_property_valid($scr)) {push @$scriptset,$scr;} + if (_is_property_valid($scr)) {push @$scriptset,$scr;} else {error($Main,"Unknown script $scr ignored.");} } } -sub is_property_valid { +sub _is_property_valid { my $script=shift @_; + if (! $script=~/^\w+$/) {return 0;} eval {' '=~/\p{$script}/}; if ($@) {return 0;} else {return 1;} } @@ -954,15 +1368,31 @@ sub set_language_option { } elsif ($language=~/^(char|character|letter)s?(-?only)?$/) { @WordPatterns=($NamedWordPattern{'letters'}); if (defined $2) {@LogogramScripts=();} - $countlabel[1]='Letters in text'; - $countlabel[2]='Letters in headers'; - $countlabel[3]='Letters in captions'; + $countdesc[1]='Letters in text'; + $countdesc[2]='Letters in headers'; + $countdesc[3]='Letters in captions'; return 'letter'; } else { return undef; } } +# Set encoding based on encoding options, guess encoding if not set +sub apply_encoding_options { + if (defined $encoding && $encoding eq 'guess') {$encoding=undef;} + if (!defined $encoding) { + } elsif (ref(find_encoding($encoding))) { + if (!$htmlstyle) {$outputEncoding=$encoding;} + } else { + error($Main,"Unknown encoding $encoding ignored."); + error_details($Main,'Valid encodings are: '.wrap('','',join(', ',Encode->encodings(':all')))); + $encoding=undef; + } + if (!defined $outputEncoding) {$outputEncoding='utf8';} + binmode STDIN; + binmode STDOUT,':encoding('.$outputEncoding.')'; +} + # Apply language options sub apply_language_options { my @tmp; @@ -993,49 +1423,239 @@ sub apply_include_bibliography { include_package('%incbib'); } +# Process TC instruction, return 1 if applied, 0 if not (e.g. error) +sub tc_macro_param_option { + my ($tex,$instr,$macro,$param,$option)=@_; + if (!defined $param) { + error($tex,'%TC:'.$instr.' requires parameter rule specification.'); + return 0; + } + elsif ($instr eq 'macro') { + if (!($macro=~/^\\/)) { + warning($tex,'%TC:macro '.$macro.': should perhaps be \\'.$macro.'?'); + } + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$macro,$param); + } + elsif ($instr eq 'exclude') { + warning($tex,'%TC:exclude is deprecated. Use %TC:macro instead.'); + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$macro,$param); + } + elsif ($instr eq 'header') { + warning($tex,'%TC:header is deprecated. Instead use e.g. %TC:macro \\name [header].'); + $TeXmacrocount{$macro}=[$CNT_COUNT_HEADER]; + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$macro,$param); + } + elsif ($instr eq 'preambleinclude') { + return _tc_macro_set_param($tex,\%TeXpreamble,$instr,$macro,$param); + } + elsif ($instr eq 'floatinclude') { + return _tc_macro_set_param($tex,\%TeXfloatinc,$instr,$macro,$param); + } + elsif ($instr=~/^(group|envir)$/) { + if (!defined $option) { + error($tex,'TC:'.$instr.' requires contents rule specification.'); + return 0; + } + defined ($option=key_to_state($option,$tex)) || return 0; + $TeXenvir{$macro}=$option; + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$PREFIX_ENVIR.$macro,$param); + } + elsif ($instr=~/^(macrocount|macroword)$/) { + return _tc_macro_set_param($tex,\%TeXmacrocount,$instr,$macro,$param,\&key_to_cnt,$REGEX_NUMBER); + } + elsif ($instr eq 'fileinclude') { + # No parameter checking here, just pass on; errors reported when used instead. + $param=~s/^\[(.*)\]$/$1/; + $param=~s/,/ /g; + assert($param=~/^\w+(\s\w+)*$/,$tex,'Invalid %TC:fileinclude parameter: '.$param) + || return 0; + $TeXfileinclude{$macro}=$param; + } + elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;} + else { + error($tex,'Unknown TC command: '.$instr); + return 0; + } + return 1; +} + +# Convert TC command parameters and add to hash +sub _tc_macro_set_param { + my ($tex,$hash,$instr,$macro,$param,$keymap,$validregex)=@_; + if (!defined $keymap) {$keymap=\&key_to_state; $validregex=$REGEX_NUMBER;} + $param=__interpret_tc_parameter($tex,$param,$keymap,$validregex); + if (!defined $param) {return 0;} + $hash->{$macro}=$param; + return 1; +} + +# Convert [...] into array of states, optionally return value if a valid pattern +sub __interpret_tc_parameter { + my ($tex,$param,$keymap,$validregex)=@_; + if ((defined $validregex) && ($param=~$validregex)) { + return $param; + } elsif ($param=~/^\[([0-9a-zA-Z,:]*)\]$/) { + my @params; + my $value; + foreach my $key (split(',',$1)) { + if ($key=~/^(\w*):(\w+)$/) { + defined ($value=$keymap->($PREFIX_PARAM_OPTION.$1,$tex)) || return undef; + push @params,$value; + $key=$2; + } + defined ($value=$keymap->($key,$tex)) || return undef; + push @params,$value; + } + return [@params]; + } + error($tex,'Invalid TC command parameter: '.$param); + return undef; +} + + +###### Macro rules handling + + +# Takes hash, value, list of keys and adds mappings to hash +sub add_keys_to_hash { + my $hash=shift @_; + my $value=shift @_; + foreach (@_) { + $hash->{$_}=$value; + } +} + +# Add source hash to target hash, optionally converting values +sub __add_to_hash { + my $target=shift @_; + my $source=shift @_; + my $convert=shift @_; + if (!defined $convert) {$convert=&__dummy;} + while (my ($key,$val)=each(%$source)) { + $target->{$key}=$convert->($val,@_); + } +} + +# Convert hash by applying function to all values +sub convert_hash { + my $hash=shift @_; + my $convert=shift @_; + while (my ($key,$val)=each(%$hash)) { + $hash->{$key}=$convert->($val,@_); + } +} + +# Convert key to parser state constant +sub key_to_state { + my ($key,$tex)=@_; + return key_convert(lc($key),$tex,\%key2state,'Unknown state key {key} replaced by IGNORE.',$STATE_IGNORE); +} + +# Convert key array to parser state constants +sub keyarray_to_state { + my ($array,$tex)=@_; + return keyarray_convert($array,$tex,\&key_to_state); +} + +# Convert key to counter constant +sub key_to_cnt { + my ($key,$tex)=@_; + return key_convert(lc($key),$tex,\%key2cnt,'Unknown counter key {key} ignored.'); +} + +# Convert key array to counter constants +sub keyarray_to_cnt { + my ($array,$tex)=@_; + return keyarray_convert($array,$tex,\&key_to_cnt); +} + +# Convert key to value using map +sub key_convert { + my ($key,$tex,$map,$error,$default)=@_; + my $value=$map->{$key}; + if (!defined $value) { + if (!defined $tex) {$tex=$Main;} + $error=~s/\{key\}/$key/g; + error($tex,$error); + return $default; + } + return $value; +} + +# Convert state keys to state nos for arrays +sub keyarray_convert { + my ($array,$tex,$conv)=@_; + if (ref($array) eq 'ARRAY') { + my @ar; + foreach my $key (@$array) { + if (defined (my $cnt=$conv->($key,$tex))) {push @ar,$cnt;} + } + $array=\@ar; + } + return $array; +} + +# Convert transition state to parsing state, and increment counter +sub transition_to_content_state { + my ($tex,$state)=@_; + my $tr=$transition2state{$state}; + if (defined $tr) { + if (my $cnt=$tr->[1]) {inc_count($tex,$tr->[1]);} + return $tr->[0]; + } + return $state; +} + +# Remove all rules +sub remove_all_rules { + %TeXpackageinc=(); + %TeXpreamble=(); + %TeXfloatinc=(); + %TeXmacro=(); + %TeXmacrocount=(); + %TeXenvir=(); + %TeXfileinclude=(); + %IncludedPackages=(); +} + # Process package inclusion sub include_package { - my $incpackage=shift @_; - my $sub; - _add_to_hash_if_exists(\%TeXmacro,\%PackageTeXmacro,$incpackage); - _add_to_hash_if_exists(\%TeXmacroword,\%PackageTeXmacroword,$incpackage); - _add_to_hash_if_exists(\%TeXheader,\%PackageTeXheader,$incpackage); - _add_to_hash_if_exists(\%TeXgroup,\%PackageTeXgroup,$incpackage); - _add_to_hash_if_exists(\%TeXfileinclude,\%PackageTeXfileinclude,$incpackage); + my ($incpackage,$tex)=@_; + if (defined $IncludedPackages{$incpackage}) {return;} + $IncludedPackages{$incpackage}=1; + # Add rules for package, preamble and float inclusion, and add to macro rules + _add_package(\%TeXpackageinc,\%PackageTeXpackageinc,$incpackage,\&__dummy,$tex); + _add_package(\%TeXpreamble,\%PackageTeXpreamble,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXfloatinc,\%PackageTeXfloatinc,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacro,\%PackageTeXpackageinc,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacro,\%PackageTeXpreamble,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacro,\%PackageTeXfloatinc,$incpackage,\&keyarray_to_state,$tex); + # Add regular rules + _add_package(\%TeXmacro,\%PackageTeXmacro,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacrocount,\%PackageTeXmacrocount,$incpackage,\&keyarray_to_cnt,$tex); + _add_package(\%TeXenvir,\%PackageTeXenvir,$incpackage,\&key_to_state,$tex); + _add_package(\%TeXfileinclude,\%PackageTeXfileinclude,$incpackage,\&__dummy,$tex); + if (my $subpackages=$PackageSubpackage{$incpackage}) { + foreach my $sub (@{$subpackages}) { + include_package($sub,$tex); + } + } } # Add package rules if defined -sub _add_to_hash_if_exists { - my ($target,$source,$name)=@_; +sub _add_package { + my ($target,$source,$name,$convert,$tex)=@_; my $sub; if ($sub=$source->{$name}) { - while (my ($key,$val)=each(%$sub)) { - $target->{$key}=$val; - } + __add_to_hash($target,$sub,$convert); } } -# Process TC instruction -sub tc_macro_param_option { - my ($instr,$macro,$param,$option)=@_; - if ($param=~/^\[([0-9,]+)\]$/) {$param=[split(',',$1)];} - if (!defined $option) {$option=1;} - if (($instr eq 'macro') || ($instr eq 'exclude')) {$TeXmacro{$macro}=$param;} - elsif ($instr eq 'header') {$TeXheader{$macro}=$param;$TeXmacro{$macro}=$param;} - elsif ($instr eq 'macroword') {$TeXmacroword{$macro}=$param;} - elsif ($instr eq 'preambleinclude') {$TeXpreamble{$macro}=$param;} - elsif ($instr eq 'group') { - $TeXmacro{'begin'.$macro}=$param; - $TeXgroup{$macro}=$option; - } - elsif ($instr eq 'floatinclude') {$TeXfloatinc{$macro}=$param;} - elsif ($instr eq 'fileinclude') {$TeXfileinclude{$macro}=$param;} - elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;} - else {return 0;} - return 1; +# Dummy conversion function: returns arguments unchanged +sub __dummy { + return shift @_; } - ###### TeX code handle @@ -1053,11 +1673,14 @@ sub getMain { ## Make TeX handle for LaTeX code: the main TeXcount object # The "TeX object" is a data containser: a hash containing -# - filename: name of LaTeX file being parsed -# - filepath: path to LaTeX file +# - filepath: path of LaTeX file being parsed +# - filename: file name of LaTeX file +# - dirpath: path to directory containing LaTeX file +# - PATH: array of paths to search for included files # - countsum: count object with total count (incl. subcounts) # - subcount: count object for subcount (to be added to countsum) # - subcounts: list of subcounts +# - errorcount: the number of errors reported # plus following elements used for the processing the LaTeX code # - line: the LaTeX paragraph being processed # - texcode: what remains of LaTeX code to process (after line) @@ -1074,8 +1697,8 @@ sub getMain { # prepared. Note that this requires that the last subcount object be added to # the countsum once the end of the document is reached. sub TeXcode { - my ($bincode,$filename,$title)=@_; - my $tex=_TeXcode_blank($filename,$title); + my ($bincode,$file,$title)=@_; + my $tex=_TeXcode_blank($file,$title); _TeXcode_setcode($tex,$bincode); more_texcode($tex); return $tex; @@ -1083,16 +1706,21 @@ sub TeXcode { # Return a blank TeXcode object sub _TeXcode_blank { - my ($filename,$title)=@_; + my ($file,$title)=@_; if (defined $title) {} - elsif (defined $filename) {$title="File: ".$filename;} - else {$title="Word count";} + elsif (defined $file) {$title='File: '.$file;} + else {$title='Word count';} my %TeX=(); $TeX{'errorcount'}=0; - $TeX{'filename'}=$filename; - if (!defined $filename) {$TeX{'filepath'}='';} - elsif ($filename=~/^(.*[\\\/])[^\\\/]+$/) {$TeX{'filepath'}=$1;} - else {$TeX{'filepath'}='';} + $TeX{'warnings'}={}; + $TeX{'filepath'}=$file; + if (!defined $file) {} + elsif ($file=~/^<.+>$/) {$TeX{'filename'}=$file;} + elsif ($file=~/^(.*[\\\/])([^\\\/]+)$/) { + $TeX{'dirpath'}=$1; $TeX{'filename'}=$2; + } + else {$TeX{'dirpath'}=''; $TeX{'filename'}=$file;} + $TeX{'PATH'}=[]; $TeX{'line'}=''; $TeX{'next'}=undef; $TeX{'type'}=undef; @@ -1102,11 +1730,9 @@ sub _TeXcode_blank { my $countsum=new_count($title); $TeX{'countsum'}=$countsum; $countsum->{'TeXcode'}=\%TeX; - my $count=new_count("_top_"); + my $count=new_count('_top_'); $TeX{'subcount'}=$count; inc_count(\%TeX,$CNT_FILE); - my @countlist=(); - $countsum->{'subcounts'}=\@countlist; return \%TeX; } @@ -1115,8 +1741,6 @@ sub _TeXcode_setcode { my ($tex,$bincode)=@_; $tex->{'texcode'}=_prepare_texcode($tex,$bincode); $tex->{'texlength'}=length($tex->{'texcode'}); - # TODO: Comment out this test - if ($tex->{'texcode'} =~ /([[:^ascii:]])/) {warning($tex,"Code contains non-ASCII characters.");} } # Decode and return TeX/LaTeX code @@ -1137,9 +1761,7 @@ sub _decode_texcode { my $decoder; if (defined $encoding) { $decoder=find_encoding($encoding); - eval { - $texcode=$decoder->decode($texcode); - }; + eval {$texcode=$decoder->decode($texcode);}; if ($@) { error($tex,'Decoding file/text using the '.$decoder->name.' encoding failed.'); } @@ -1200,26 +1822,51 @@ sub apply_substitution_rule { ## Get more TeX code from texcode buffer if possible, return 1 if done sub more_texcode { my ($tex)=@_; - if (!defined $tex->{'texcode'}) {return 0;} - if ( $optionFast && $tex->{'texcode'} =~ s/^.*?(\r{2,}|\n{2,}|(\r\n){2,})//s ) { - $tex->{'line'}.=$&; + if ($tex->{'texcode'} eq '') {return 0;} + if ( $optionFast && $tex->{'texcode'} =~ s/^(.*?(\r{2,}|\n{2,}|(\r\n){2,}))//s ) { + $tex->{'line'}.=$1; # $1 ~ ${^MATCH} return 1; } $tex->{'line'}.=$tex->{'texcode'}; - $tex->{'texcode'}=undef; + $tex->{'texcode'}=''; return 1; } ## Prepend LaTeX code to TeXcode object sub prepend_code { - my ($tex,$code,$fname)=@_; - my $prefix="\n% --- Start of included file ".$fname."\n"; - my $suffix="\n% --- End of included file ".$fname."\n"; + my ($tex,$code,$filename)=@_; + my $prefix="\n% --- Start of included file $filename\n"; + my $suffix="\n% --- End of included file $filename\n"; $code=_decode_texcode($tex,$code); $tex->{'length'}+=length($code); $tex->{'texcode'}=$prefix.$code.$suffix.$tex->{'line'}.$tex->{'texcode'}; $tex->{'line'}=''; - more_texcode(); + more_texcode($tex); +} + +# Save and return TeXcode state, and replace with new text +sub texcode_insert_text { + my ($tex,$code,$filename,@path)=@_; + my %texstate=( + 'texcode'=>$tex->{'line'}.$tex->{'texcode'}, + 'eof'=>$tex->{'eof'}, + 'PATH'=>$tex->{'PATH'}); + my $prefix="\n% --- Start of included file $filename\n"; + my $suffix="\n% --- End of included file $filename\n"; + $code=_decode_texcode($tex,$code); + $tex->{'length'}+=length($code); + $tex->{'texcode'}=$prefix.$code.$suffix; + $tex->{'line'}=''; + more_texcode($tex); + return \%texstate; +} + +# Restore TeXcode state +sub texcode_restore_state { + my ($tex,$texstate)=@_; + while (my ($key,$value)=each %$texstate) { + $tex->{$key}=$value; + } } ## Returns size of TeX code in bytes @@ -1235,20 +1882,20 @@ sub get_texsize { # Read LaTeX file into TeX object sub TeXfile { my ($filename,$title)=@_; + my ($header,$bincode); if ($filename eq $_STDIN_) { - if ($verbose>0) { - formatprint("File from STDIN\n",'h2'); - $blankline=0; - } - return TeXcode(_read_stdin(),'STDIN',$title); - } else { - my $bincode=read_binary($filename) || return undef; - if ($verbose>0) { - formatprint("File: ".$filename."\n",'h2'); - $blankline=0; - } - return TeXcode($bincode,$filename,$title); + $header='File from STDIN'; + $bincode=_read_stdin(); + } else { + defined ($bincode=read_binary($filename)) || return undef; + $header='File: '.$filename; + } + if ($printlevel>0) { + formatprint($header."\n",'h2'); + $blankline=0; } + my $tex=TeXcode($bincode,$filename,$title); + return $tex; } # Read file to string without regard for encoding @@ -1275,8 +1922,10 @@ sub _read_stdin { # Add warning to list of registered warnings (optionally to be reported at the end) sub warning { my ($tex,$text)=@_; - $tex->{'warnings'}->{$text}++; $warnings{$text}++; + # TODO: should only add warnings to subcount, not to TeXcode object + $tex->{'warnings'}->{$text}++; + if (my $count=$tex->{'subcount'}) {$count->{'warnings'}->{$text}++}; } # Register error and print error message @@ -1285,9 +1934,9 @@ sub error { if (defined $type) {$text=$type.': '.$text;} $errorcount++; $tex->{'errorcount'}++; - #print STDERR $text,"\n"; + if (my $count=$tex->{'subcount'}) {$count->{'errorcount'}++}; if (my $err=$tex->{'errorbuffer'}) {push @$err,$text;} - if ($verbose>=0) {_print_error($text);} + else {print_error($text);} } # Print error details @@ -1296,172 +1945,221 @@ sub error_details { print STDERR $text,"\n"; } +# Make assertion: i.e. report error if not true and return truthfullness +sub assert { + my $assertion=shift @_; + if ($assertion) {return 1;} + error(@_); + return 0; +} + +# Print error message +sub print_error { + my $text=shift @_; + line_return(1); + if ($printlevel<0) { + print STDERR 'ERROR: ',$text,"\n"; + } elsif ($htmlstyle) { + print STDERR 'ERROR: ',$text,"\n"; + print_style($text,'error'); + } else { + print_style("!!! $text !!!",'error'); + } + line_return(1); +} + +# Print errors in buffer and delete errorbuffer +sub flush_errorbuffer { + my $source=shift @_; + my $err=$source->{'errorbuffer'} || return; + foreach (@$err) {print_error($_);} + $source->{'errorbuffer'}=undef; +} + ###### Parsing routines # Parse LaTeX document sub parse { my ($tex)=@_; - if ($htmlstyle && $verbose) {print "<div class='parse'><p>\n";} + if ($htmlstyle && $printlevel>0) {print "<div class='parse'><p>\n";} + parse_all($tex); + if ($htmlstyle && $printlevel>0) {print "</p></div>\n";} +} + +# Parse until end +sub parse_all { + my ($tex,$state)=@_; + if (!defined $state) {$state=$STATE_TEXT;} while (!($tex->{'eof'})) { - _parse_unit($tex,$STATUS_TEXT); + _parse_unit($tex,$state); } - if ($htmlstyle && $verbose) {print "</p></div>\n";} } -# Parse one block or unit +# Parse one block/unit with given state until end token +# If end token is set to $_PARAM_, unit is assumed to be a parameter +# and words parsed as individual letters (simple_token is set) sub _parse_unit { - # Status: - # 0 = exclude from count - # 1 = text - # 2 = header text - # 3 = float text - # -1 = float (exclude) - # -2 = strong exclude, ignore begin-end groups - # -3 = stronger exclude, do not parse macro parameters - # -4 = ignore everything except end marker: even { - # -9 = preamble (between \documentclass and \begin{document}) - my ($tex,$status,$end)=@_; - if (!defined $status) { - error($tex,'Undefined parser status!','CRITICAL ERROR'); - exit; - } elsif (ref($status) eq 'ARRAY') { - error($tex,'Invalid parser status!','CRITICAL ERROR'); - exit; + my ($tex,$state,$end)=@_; + my $simple_token=0; + ( assert(defined $state,$tex,'Undefined parser state!','BUG') + && assert(!ref($state),$tex,'Invalid parser state of type '.ref($state).'!','BUG') + ) || exit; + $state=transition_to_content_state($tex,$state); + _set_printstate($tex,$state,$end); + if (defined $end && $end eq $_PARAM_) { + $end=undef; + $simple_token=1; } - if ($showstates) { - if (defined $end) { - $tex->{'printstate'}=':'.status_to_text($status).':'.$end.':'; - } else { - $tex->{'printstate'}=':'.status_to_text($status).':'; - } - flush_next($tex); - } - while (defined (my $next=_next_token($tex))) { + my $next; + while (defined ($next=next_token($tex,$simple_token))) { # Parse next token until token matches $end - set_style($tex,"ignore"); + set_style($tex,'ignore'); + if ($state==$STATE_MATH) {set_style($tex,'math');} if ((defined $end) && ($end eq $next)) {return;} # Determine how token should be interpreted - if ($status==$STATUS_PREAMBLE && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) { + if ($state==$STATE_PREAMBLE && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) { # \begin{document} - $status=$STATUS_TEXT; + $state=$STATE_TEXT; } - if ($status==$STATUS_EXCLUDE_ALL) { - # Ignore everything + # Pick the first matching interpretation: + if ($state==$STATE_EXCLUDE_ALL) { + # ignore everything } elsif ($tex->{'type'}==$TOKEN_SPACE) { # space or other code that should be passed through without styling flush_next($tex,' '); - } elsif ($next eq '\documentclass') { - # starts preamble - set_style($tex,'document'); - __gobble_option($tex); - __gobble_macro_parms($tex,1); - while (!($tex->{'eof'})) { - _parse_unit($tex,$STATUS_PREAMBLE); - } } elsif ($tex->{'type'}==$TOKEN_TC) { # parse TC instructions _parse_tc($tex,$next); } elsif ($tex->{'type'}==$TOKEN_WORD) { # word - if (my $cnt=status_text_cnt($status)) { - _process_word($tex,$next,$status); + if (my $cnt=state_text_cnt($state)) { + _process_word($tex,$next,$state); inc_count($tex,$cnt); - set_style($tex,status_to_style($status)); + set_style($tex,state_to_style($state)); } } elsif ($next eq '{') { - # {...} - _parse_unit($tex,$status,'}'); + # {...} group + set_style($tex,'ignore'); + _parse_unit($tex,$state,'}'); + set_style($tex,'ignore'); } elsif ($next eq '}') { error($tex,'Encountered } without corresponding {.'); - } elsif ($tex->{'type'}==$TOKEN_MACRO && $status==$STATUS_EXCLUDE_STRONGER) { - # ignore macro call + } elsif ($state==$STATE_EXCLUDE_STRONGER) { + # ignore remaining tokens set_style($tex,'ignore'); + } elsif ($next eq '\documentclass') { + # defines document class and starts preamble + set_style($tex,'document'); + _parse_documentclass_params($tex); + while (!($tex->{'eof'})) { + _parse_unit($tex,$STATE_PREAMBLE); + } } elsif ($tex->{'type'}==$TOKEN_MACRO) { # macro call - _parse_macro($tex,$next,$status); + _parse_macro($tex,$next,$state); } elsif ($next eq '$') { # math inline - _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'$'); + _parse_math($tex,$state,$CNT_COUNT_INLINEMATH,'$'); } elsif ($next eq '$$') { # math display (unless already in inlined math) if (!(defined $end && $end eq '$')) { - _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'$$'); + _parse_math($tex,$state,$CNT_COUNT_DISPLAYMATH,'$$'); } + } elsif ($simple_token) { + #DELETE: simple tokens should be handled properly + #print STDERR '<',$next,'>',"\n"; + # handle as parameter that should not be counted + set_style($tex,'ignore'); } if (!defined $end) {return;} } - if (defined $end) { - error($tex,'Reached end of file while waiting for '.$end.'.'); - } + defined $end && error($tex,'Reached end of file while waiting for '.$end.'.'); } -# Process word with a given status (>0, i.e. counted) +# Print state +sub _set_printstate { + my ($tex,$state,$end)=@_; + $tex->{'printstate'}=':'.state_to_text($state).':'.(defined $end?$end.':':''); + flush_next($tex); +} + +# Process word with a given state (>0, i.e. counted) sub _process_word { - my ($tex,$word,$status)=@_; + my ($tex,$word,$state)=@_; $WordFreq{$word}++; } # Parse unit when next token is a macro sub _parse_macro { - my ($tex,$next,$status)=@_; - # substat = parameter status settings + my ($tex,$next,$state)=@_; my $substat; + my @macro=($next); if (my $label=$BreakPoints{$next}) { if ($tex->{'line'}=~ /^[*]?(\s*\[.*?\])*\s*\{((.|\{.*\})*)\}/ ) { $label=$label.': '.$2; } next_subcount($tex,$label); } - set_style($tex,status_is_text($status)?'command':'exclcommand'); - if ($next eq '\begin' && status_inc_envir($status)) { - _parse_begin_end($tex,$status); - } elsif ($next eq '\end' && status_inc_envir($status)) { + if ($state==$STATE_MATH) {set_style($tex,'mathcmd');} + else {set_style($tex,state_is_text($state)?'cmd':'exclcmd');} + if ($next eq '\begin' && state_inc_envir($state)) { + _parse_envir($tex,$state); + push @macro,'<envir>'; + } elsif ($next eq '\end' && state_inc_envir($state)) { error($tex,'Encountered \end without corresponding \begin'); + push @macro,$STRING_ERROR; } elsif ($next eq '\verb') { - _parse_verb_region($tex,$status); - } elsif ( (status_is_text($status) || $status==$STATUS_PREAMBLE) - && defined ($substat=$TeXpackageinc{$next}) ) { - _parse_include_package($tex,$substat); - } elsif (($status==$STATUS_FLOAT) && ($substat=$TeXfloatinc{$next})) { + _parse_verb_region($tex,$state); + } elsif (state_is_parsed($state) && defined $TeXpackageinc{$next} ) { + _parse_include_package($tex); + push @macro,'<package>'; + } elsif (state_is_parsed($state) && defined (my $def=$TeXfileinclude{$next})) { + # include file (merge in or queue up for parsing) + set_style($tex,'cmd'); + if (defined ($substat=$TeXmacro{'@pre'.$next})) {__gobble_macro_parms($tex,$substat,$state);} + __count_macrocount($tex,$next,$state); + _parse_include_file($tex,$state,$def); + push @macro,'<filespec>'; + if (defined ($substat=$TeXmacro{'@post'.$next})) {__gobble_macro_parms($tex,$substat,$state);} + } elsif (($state==$STATE_FLOAT) && ($substat=$TeXfloatinc{$next})) { # text included from float - set_style($tex,'command'); - __gobble_macro_parms($tex,$substat); - } elsif ($status==$STATUS_PREAMBLE && defined ($substat=$TeXpreamble{$next})) { - # parse preamble include macros - set_style($tex,'command'); - if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);} - __gobble_macro_parms($tex,$substat,1); - } elsif (status_is_exclude($status)) { + set_style($tex,'cmd'); + push @macro,__gobble_macro_parms($tex,$substat,$__STATE_NULL); + } elsif ($state==$STATE_PREAMBLE && defined ($substat=$TeXpreamble{$next})) { + # parse preamble include macros + set_style($tex,'cmd'); + __count_macrocount($tex,$next,$STATE_TEXT); + push @macro,__gobble_macro_parms($tex,$substat,$__STATE_NULL); + } elsif (state_is_exclude($state)) { # ignore - __gobble_option($tex); + push @macro,__gobble_options($tex),'/*ignored*/'; } elsif ($next eq '\(') { # math inline - _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'\)'); + _parse_math($tex,$state,$CNT_COUNT_INLINEMATH,'\)'); } elsif ($next eq '\[') { # math display - _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'\]'); - } elsif ($next eq '\def') { + _parse_math($tex,$state,$CNT_COUNT_DISPLAYMATH,'\]'); + } elsif ($next=~/^\\(def|edef|gdef|xdef)$/) { # ignore \def... $tex->{'line'} =~ s/^([^\{]*)\{/\{/; flush_next($tex); print_style($1,'ignore'); - _parse_unit($tex,$STATUS_EXCLUDE_STRONG); - } elsif (defined (my $addsuffix=$TeXfileinclude{$next})) { - # include file: queue up for parsing - _parse_include_file($tex,$status,$addsuffix); + _parse_unit($tex,$STATE_EXCLUDE_STRONG); + push @macro,'<macro>',$STRING_PARAMETER; } elsif (defined ($substat=$TeXmacro{$next})) { # macro: exclude options - if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);} - __count_macroword($tex,$next,$status); - __gobble_macro_parms($tex,$substat,$status); - } elsif (defined __count_macroword($tex,$next,$status)) { + __count_macrocount($tex,$next,$state); + push @macro,__gobble_macro_parms($tex,$substat,$state); + } elsif (defined __count_macrocount($tex,$next,$state)) { # count macro as word (or a given number of words) - set_style($tex,'word'.$status); - } elsif ($next =~ /^\\[^\w\_]/) { + } elsif ($next =~ /^\\[^\w\_]$/) { + # handle \<symbol> as single symbol macro + push @macro,__gobble_options($tex),'/*symbol*/'; } else { - __gobble_option($tex); + push @macro,__gobble_options($tex),'/*defaultrule*/'; } + $MacroUsage{join('',@macro)}++; } # Parse TC instruction @@ -1469,190 +2167,196 @@ sub _parse_tc { my ($tex,$next)=@_; set_style($tex,'tc'); flush_next($tex); - $next=~s/^\%+TC:\s*(\w+)\s*//i || BLOCK { - error($tex,'TC command should have format %TC:instruction [macro] [parameters]'); - return; - }; + assert($next=~s/^\%+TC:\s*(\w+)\s*//i + ,$tex,'TC command should have format %TC:instruction [[parameters]]') + || return; my $instr=$1; $instr=~tr/[A-Z]/[a-z]/; - if ($instr=~/^(break)$/) { - if ($instr eq 'break') {next_subcount($tex,$next);} + if ($instr eq 'break') {next_subcount($tex,$next); } elsif ($instr=~/^(incbib|includebibliography)$/) { $includeBibliography=1; apply_include_bibliography(); - } elsif ($instr eq 'ignore') { - __gobble_tc_ignore($tex); - } elsif ($instr eq 'endignore') { - error($tex,'TC:endignore without corresponding TC:ignore.'); + } elsif ($instr eq 'ignore') {__gobble_tc_ignore($tex); + } elsif ($instr eq 'endignore') {error($tex,'%TC:endignore without corresponding %TC:ignore.'); } elsif ($instr eq 'newtemplate') {$outputtemplate=''; } elsif ($instr eq 'template') {$outputtemplate.=$next; + } elsif ($instr eq 'insert') { + $tex->{'line'}="\n".$next.$tex->{'line'}; } elsif ($instr eq 'subst') { - if ($next=~/^(\\\S+)\s+/) { + if ($next=~/^(\\\S+)\s+(.*)$/) { my $from=$1; - my $to=$'; + my $to=$2; $substitutions{$from}=$to; apply_substitution_rule($tex,$from,$to); } else { - error($tex,'Invalid TC:subst format.'); + error($tex,'Invalid %TC:subst format.'); } - } elsif ($next=~/^([\\]*\S+)\s+([^\s\n]+)(\s+(-?[0-9]+))?/) { - # Format = TC:word macro + } elsif ($instr eq 'newcounter') { + assert($next=~s/^(\w+)(=(\w+))?\s*//,$tex,'Should have format %TC:newcounter {key}[={like-key}] {description}') + || return; + my $key=$1; + my $like=$3; + if ($next eq '') {$next=$key;} + add_new_counter($key,$next,$like); + } elsif ($next=~/^([\\]*\S+)\s+([^\s]+)(\s+(-?\w+))?/) { + # %TC:instr macro param option my $macro=$1; my $param=$2; my $option=$4; - if (tc_macro_param_option($instr,$macro,$param,$option)) {} - else {error($tex,'Unknown TC command: '.$instr);} + tc_macro_param_option($tex,$instr,$macro,$param,$option); } else { - error($tex,'Invalid TC command format: '.$instr); + error($tex,'Invalid TC command: '.$instr); } } -# Parse through ignored LaTeX code -sub __gobble_tc_ignore { - my ($tex)=@_; - set_style($tex,'ignore'); - _parse_unit($tex,$STATUS_EXCLUDE_ALL,'%TC:endignore'); - set_style($tex,'tc'); - flush_next($tex); -} - # Parse math formulae sub _parse_math { - my ($tex,$status,$cnt,$end)=@_; + my ($tex,$state,$cnt,$end)=@_; my $localstyle; - if (status_is_text($status)) { + if (state_is_text($state)) { $localstyle='mathgroup'; inc_count($tex,$cnt); } else { $localstyle='exclmath'; } set_style($tex,$localstyle); - _parse_unit($tex,$STATUS_IGNORE,$end); + _parse_unit($tex,$STATE_MATH,$end); set_style($tex,$localstyle); } # Parse \verb region sub _parse_verb_region { - my ($tex,$status)=@_; - flush_next($tex); + my ($tex,$state)=@_; + #flush_next($tex); + __gobble_macro_modifier($tex); set_style($tex,'ignore'); - if (!($tex->{'line'} =~ s/^[^\s]// )) { - error($tex,'Invalid \verb: delimiter required.'); - } - my $dlm=$&; - print_style($dlm,'command'); - if (!($tex->{'line'} =~ s/^([^$dlm]*)($dlm)// )) { - error($tex,'Invalid \verb: could not find ending delimiter ('.$dlm.').'); - } + assert($tex->{'line'} =~ s/^([^\s])//,$tex,'Invalid \verb: delimiter required.') + || return; + my $dlm=$1; + print_style($dlm,'cmd'); + assert($tex->{'line'}=~s/^([^\Q$dlm\E]*)(\Q$dlm\E)// + ,$tex,'Invalid \verb: could not find ending delimiter ('.$dlm.').') + || return; print_style($1,'ignore'); - print_style($2,'command'); + print_style($2,'cmd'); } -# Parse begin-end group -sub _parse_begin_end { - my ($tex,$status)=@_; - my $localstyle=status_is_text($status) ? 'grouping' : 'exclgroup'; - flush_next_gobble_space($tex,$localstyle,$status); - #__gobble_option($tex); # no option before group name - my ($groupname,$next); +# Parse environment +sub _parse_envir { + my ($tex,$state)=@_; + my $localstyle=state_is_text($state) ? 'envir' : 'exclenv'; + flush_next_gobble_space($tex,$localstyle,$state); + my ($envirname,$next); if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) { - # gobble group type - $groupname=$1; + # gobble environment name + $envirname=$1; + my @macro=('<envir:'.$envirname.'>'); print_style('{'.$1.'}',$localstyle); - $next='begin'.$groupname; - if (defined (my $substat=$TeXmacro{$next})) { - __gobble_macro_parms($tex,$substat); + $next=$PREFIX_ENVIR.$envirname; + __count_macrocount($tex,$next,$state); + if (defined (my $def=$TeXmacro{$next})) { + push @macro,__gobble_macro_parms($tex,$def,$__STATE_NULL); + } else { + push @macro,__gobble_options($tex); } + $MacroUsage{join('',@macro)}++; } else { - $groupname='???'; $next='???'; + $envirname='???'; $next='???'; error($tex,'Encountered \begin without environment name provided.'); } - # find group status (or leave unchanged) - my $substat=$TeXgroup{$1}; + # find new parsing state (or leave unchanged) + my $substat=$TeXenvir{$1}; if (!defined $substat) { - $substat=$status; + $substat=$state; if ($strictness>=1) { - warning($tex,"Using default rule for group ".$groupname); + warning($tex,'Using default rule for environment '.$envirname); } - } elsif (!status_is_text($status)) { - # Do not raise status - $substat=__new_status($substat,$status); - } elsif ($substat==$STATUS_FLOAT) { - # Count float - inc_count($tex,$CNT_COUNT_FLOAT); - } elsif (my $cnt=status_to_cnt($substat)) { - # Count as given type (was: if $substat>3) - inc_count($tex,$cnt); - $substat=0; } else { - # Use $substat value as given - } - if ($includeBibliography && $groupname eq 'thebibliography' && $substat>0) { - # Add bibliography header - inc_count($tex,$CNT_COUNT_HEADER); - __count_macroword($tex,$groupname,$STATUS_TEXT_HEADER); + $substat=__new_state($substat,$state); } - if (!status_inc_envir($substat)) { - # Keep parsing until appropriate end group arrives - while (!$tex->{'eof'}) { - _parse_unit($tex,$substat,'\end'); - if ($tex->{'line'} =~ s/^\s*\{$groupname\}[ \t\r\f]*//) { - # gobble end group parameter - flush_next($tex,$localstyle,$status); - print_style($&,$localstyle); - return; - } - } - } else { + if (state_inc_envir($substat)) { # Parse until \end arrives, and check that it matches _parse_unit($tex,$substat,'\end'); - flush_next_gobble_space($tex,$localstyle,$status); + flush_next_gobble_space($tex,$localstyle,$state); if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) { - # gobble group type + # gobble environment name print_style('{'.$1.'}',$localstyle); - if ($groupname ne $1) { - error($tex,'Group \begin{'.$groupname.'} ended with end{'.$1.'}.'); - } + assert($envirname eq $1,$tex,"Environment \\begin{$envirname} ended with \\end{$1}."); } else { - error($tex,'Group ended while waiting for \end{'.$groupname.'}.'); + error($tex,"Environment ended while waiting for \end{$envirname}."); + } + } else { + # Keep parsing until appropriate \end arrives ignoring all else + while (!$tex->{'eof'}) { + _parse_unit($tex,$substat,'\end'); + if ($tex->{'line'} =~ s/^(\s*)(\{\Q$envirname\E\})[ \t\r\f]*//) { + # gobble \end parameter and end environment + flush_next($tex,$localstyle,$state); + print_style($1,$localstyle); + print_style($2,$localstyle); + return; + } } } } # Parse and process file inclusion sub _parse_include_file { - my ($tex,$status,$addsuffix)=@_; - if ($addsuffix eq 'bbl') { - _parse_include_bbl($tex,$status); - return; - } + my ($tex,$state,$includetype)=@_; + my $include=state_is_parsed($state); + my $style=$include?'fileinc':'ignore'; + my $file; + my %params; flush_next($tex); - $tex->{'line'} =~ s/^\s*\{([^\{\}\s]+)\}// - || $tex->{'line'} =~ s/^\s*([^\{\}\%\\\s]+)// - || $tex->{'line'} =~ s/^\s*\{(.+?)\}// - || BLOCK { - error($tex,'Failed to read or interpret file name for inclusion.'); + $params{'<type>'}=$includetype; + if ($includetype eq '<bbl>') { + _parse_include_bbl($tex,$state,\%params); return; - }; - if (status_is_text($status)) { - print_style($&,'fileinclude'); - my $fname=$1; - if ($addsuffix==2) {$fname.='.tex';} - elsif ($addsuffix==1 && ($fname=~/^[^\.]+$/)) {$fname.='.tex';} - if ($includeTeX) {include_file($tex,$fname);} - } else { - print_style($&,'ignored'); + } + elsif ($includetype eq 'input') { + $tex->{'line'} =~ s/^(\s*\{([^\{\}\s]+)\})// + || $tex->{'line'} =~ s/^(\s*([^\{\}\%\\\s]+))// + || $tex->{'line'} =~ s/^(\s*\{(.+?)\})// + || BLOCK { + error($tex,'Failed to identify file name.'); + return; + }; + print_style($1,$style); + $file=$2; + } + else { + foreach my $param (split(/\s+|,/,$includetype)) { + if ($param=~/^<.*>$/) {next;} + $tex->{'line'} =~ s/(\s*\{)([^\{\}\%]*)(\})//|| BLOCK { + error($tex,"Failed to identify file parameter {$param}."); + return; + }; + print_style($1,'ignore'); + print_style($2,$style); + print_style($3,'ignore'); + if ($param eq 'file') {$file=$2;} + elsif ($param eq 'texfile') { + $file=$2; + if (!$file=~/\.tex$/i) {$file.='.tex';} + } + else {$params{$param}=$2;} + } + } + if (!defined $file) { + error($tex,'Undefined file name.'); + } + elsif ($includeTeX && $include) { + include_file($tex,$state,$file,\%params); } } # Parse and process bibliography file sub _parse_include_bbl { - my ($tex,$status)=@_; - __gobble_macro_parms($tex,1,$status); - if (status_is_text($status) && $includeBibliography) { - my $fname=$tex->{'filename'}; - $fname=~s/\.\w+$/\.bbl/; - include_file($tex,$fname); + my ($tex,$state,$params)=@_; + if ($includeBibliography && state_is_text($state)) { + my $filename=$tex->{'filename'}; + $filename=~s/(\.tex)?$/\.bbl/i; + include_file($tex,$state,$filename,$params); } } @@ -1661,26 +2365,53 @@ sub _parse_include_package { my ($tex)=@_; set_style($tex,'document'); __gobble_option($tex); - if ( $tex->{'line'}=~s/^\{((\w+)(\s*,\s*\w+)*)\}// ) { - print_style('{'.$1.'}','document'); + if ( $tex->{'line'}=~s/^\{(([\w\-]+)(\s*,\s*[\w\-]+)*)\}// ) { + print_style("{$1}",'document'); foreach (split(/\s*,\s*/,$1)) { - include_package($_); + $MacroUsage{"<package:$_>"}++; + include_package($_,$tex); } } else { - _parse_unit($tex,$STATUS_IGNORE); - error($tex,"Could not recognise package list, ignoring it instead."); + _parse_unit($tex,$STATE_IGNORE); + error($tex,'Could not recognise package list, ignoring it instead.'); } - __gobble_options($tex); } -# Count macroword using given status -sub __count_macroword { - my ($tex,$next,$status)=@_; - my $n=$TeXmacroword{$next}; - if (defined $n && (my $cnt=status_text_cnt($status))) { - inc_count($tex,$cnt,$n); +# Parse \documentclass parameters and include rules +sub _parse_documentclass_params { + my ($tex)=@_; + my $options=__gobble_option($tex); + if ( $tex->{'line'}=~s/^(\{\s*([^\{\}\s]+)\s*\})// ) { + print_style("$1",'document'); + $MacroUsage{"<documentclass:$2>"}++; + include_package("class%$2",$tex); + } else { + _parse_unit($tex,$STATE_IGNORE); + error($tex,'Could not identify document class.'); } - return $n; +} + +# Count macrocount using given state +sub __count_macrocount { + my ($tex,$next,$state)=@_; + my $def=$TeXmacrocount{$next}; + my $cnt; + if (!defined $def) {return undef;} + elsif (ref($def) eq 'ARRAY') { + # TODO: Is this an appropriate style to use? + set_style($tex,state_to_style($state)); + flush_next($tex); + foreach $cnt (@{$def}) { + if ($cnt==$CNT_WORDS_TEXT) {$cnt=state_text_cnt($state);} + inc_count($tex,$cnt); + } + } + elsif ($cnt=state_text_cnt($state)) { + set_style($tex,state_to_style($state)); + flush_next($tex); + inc_count($tex,$cnt,$def); + } + return $def; } # Gobble next option, return option or undef if none @@ -1694,64 +2425,104 @@ sub __gobble_option { return undef; } -# Gobble all options +# Gobble all options, return the number of gobble options sub __gobble_options { - while (__gobble_option(@_)) {} + my $n=0; + while (__gobble_option(@_)) {$n++} + return ($STRING_GOBBLED_OPTION)x$n; } # Gobble macro modifyer (e.g. following *) sub __gobble_macro_modifier { my $tex=shift @_; flush_next($tex); - if ($tex->{'line'} =~ s/^\*//) { + if ($tex->{'line'} =~ s/^(\*)//) { print_style($1,'option'); return $1; } - return undef; + return; } # Gobble macro parameters as specified in parm plus options sub __gobble_macro_parms { my ($tex,$parm,$oldstat)=@_; my $n; + my @ret; if (ref($parm) eq 'ARRAY') { $n=scalar @{$parm}; } else { $n=$parm; - $parm=[($STATUS_IGNORE)x$n]; + $parm=[($STATE_IGNORE)x$n]; } - if ($n>0) {__gobble_macro_modifier($tex);} - __gobble_options($tex); + # TODO: Optionally gobble macro modifier? + if ($n>0) {push @ret,__gobble_macro_modifier($tex);} + my $auto_gobble_options=1; for (my $j=0;$j<$n;$j++) { - _parse_unit($tex,__new_status($parm->[$j],$oldstat)); - __gobble_options($tex); + my $p=$parm->[$j]; + if ($p==$_STATE_OPTION) { + # Parse macro option + $p=$parm->[++$j]; + if ($tex->{'line'}=~s/^(\s*\[)//) { + flush_next_gobble_space($tex); + print_style($1,'optparm'); + push @ret,$STRING_OPTIONAL_PARAM; + _parse_unit($tex,__new_state($p,$oldstat),']'); + set_style($tex,'optparm'); + } + } elsif ($p==$_STATE_NOOPTION) { + $auto_gobble_options=0; + } elsif ($p==$_STATE_AUTOOPTION) { + $auto_gobble_options=1; + } else { + # Parse macro parameter + if ($auto_gobble_options) {push @ret,__gobble_options($tex);} + push @ret,$STRING_PARAMETER; + _parse_unit($tex,__new_state($p,$oldstat),$_PARAM_); + } } + #TODO: Drop default gobbling of option at end? + if ($auto_gobble_options) {push @ret,__gobble_options($tex);} + return @ret; } -# Return new parsing status given old and substatus -sub __new_status { +# Parse through ignored LaTeX code +sub __gobble_tc_ignore { + my ($tex)=@_; + set_style($tex,'ignore'); + _parse_unit($tex,$STATE_EXCLUDE_ALL,'%TC:endignore'); + set_style($tex,'tc'); + flush_next($tex); +} + +# Return new parsing state given old and substate +sub __new_state { my ($substat,$oldstat)=@_; if (!defined $oldstat) {return $substat;} - foreach my $st (@STATUS_PRIORITY_LIST) { + foreach my $st (@STATE_FIRST_PRIORITY,@STATE_MID_PRIORITY,@STATE_LAST_PRIORITY) { if ($oldstat==$st || $substat==$st) {return $st;} } - error($Main,'Could not determine new status in __new_status!','BUG'); + error($Main,'Could not determine new state in __new_state!','BUG'); return $oldstat; } + +###### Tokenisation routines + + # Get next token skipping comments and flushing output buffer -sub _next_token { - my $tex=shift @_; +sub next_token { + my ($tex,$simple_token)=@_; + if (!defined $simple_token) {$simple_token=0;} my ($next,$type); my $style=$tex->{'style'}; if (defined $tex->{'next'}) {print_style($tex->{'next'},$tex->{'style'});} $tex->{'style'}=undef; - while (defined ($next=__get_next_token($tex))) { + while (defined ($next=_get_next_token($tex,$simple_token))) { $type=$tex->{'type'}; if ($type==$TOKEN_COMMENT) { print_style($next,'comment'); } elsif ($type==$TOKEN_LINEBREAK) { - if ($verbose>0) {line_return(-1,$tex);} + if ($printlevel>0) {line_return(-1,$tex);} } else { return $next; } @@ -1760,20 +2531,9 @@ sub _next_token { } # Read, interpret and return next token -sub __get_next_token { - # Token (or token group) category: - # -1: space - # 0: comment - # 1: word (or other forms of text or text components) - # 2: symbol (not word, e.g. punctuation) - # 3: macro - # 4: curly braces {} - # 5: brackets [] - # 6: maths - # 9: line break in file - # 999: end of line or blank line - # 666: TeXcount instruction (%TC:instruction) - my $tex=shift @_; +# If simple_token is set, words cannot form tokens +sub _get_next_token { + my ($tex,$simple_token)=@_; my $next; my $ch; while (!$tex->{'eof'}) { @@ -1787,8 +2547,18 @@ sub __get_next_token { } elsif ($ch eq "\n" || $ch eq "\r") { $tex->{'line'}=~s/^(\r\n?|\n)//; return __set_token($tex,$1,$TOKEN_LINEBREAK); - } elsif ($tex->{'line'}=~s/^($WordPattern)//) { + } elsif (!$simple_token && $tex->{'line'}=~s/^($WordPattern)//o) { return __set_token($tex,$1,$TOKEN_WORD); + } elsif ($simple_token && $tex->{'line'}=~s/^(\w)//) { + # When parsing simple tokens (not words), only include single characters + # TODO: the handling of simple tokens should be improved + my $match=$1; + if ($match=~/^$WordPattern$/) {return __set_token($tex,$match,$TOKEN_WORD);} + else {return __set_token($tex,$match,$TOKEN_SYMBOL);} + } elsif ($ch eq '\\') { + if ($tex->{'line'}=~s/^(\\[{}%])//) {return __set_token($tex,$1,$TOKEN_SYMBOL);} + if ($tex->{'line'}=~s/^(\\([a-zA-Z@]+|[^a-zA-Z@[:^print:]]))//) {return __set_token($tex,$1,$TOKEN_MACRO);} + return __get_chtoken($tex,$ch,$TOKEN_END); } elsif ($ch eq '$') { $tex->{'line'}=~s/^(\$\$?)//; return __set_token($tex,$1,$TOKEN_MATH); @@ -1806,10 +2576,6 @@ sub __get_next_token { if ($tex->{'line'}=~s/^(\%+[tT][cC]:[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_TC);} if ($tex->{'line'}=~s/^(\%+[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_COMMENT);} return __get_chtoken($tex,$ch,$TOKEN_COMMENT); - } elsif ($ch eq '\\') { - if ($tex->{'line'}=~s/^(\\[{}%])//) {return __set_token($tex,$1,$TOKEN_SYMBOL);} - if ($tex->{'line'}=~s/^(\\([a-zA-Z_]+|[^a-zA-Z_]))//) {return __set_token($tex,$1,$TOKEN_MACRO);} - return __get_chtoken($tex,$ch,$TOKEN_END); } else { return __get_chtoken($tex,$ch,$TOKEN_END); } @@ -1840,30 +2606,26 @@ sub __get_chtoken { ## Make new count object -# The "count object" is a hash containing +# The "count object" is a hash containing: # - title: the title of the count (name of file, section, ...) # - counts: a list of numbers (the counts: files, text words, ...) -# upon creation, but where the element -# - subcounts: list of count objects (added by the TeX object) -# may exist if the count contains subcounts. The elements of the -# count are (by their index): -# 0 = #files: counts the number of files -# 1 = text words: counts the number of words in the text -# 2 = header words: number of words in headers -# 3 = caption words: number of words in float captions -# 4 = #headers: number of headers -# 5 = #floats: number of tables, figures, floats, etc. -# 6 = #inline formulae: number of math elements in text ($...$) -# 7 = #displayed formulae: number of displayed equations +# - subcounts: list of count objects (used by the TeX object) +# - warnings: list of warnings produced +# The elements are specified by the $CONT_* constants. sub new_count { my ($title)=@_; my @cnt=(0) x $SIZE_CNT; - my %count=('counts'=>\@cnt,'title'=>$title); - # files, text words, header words, float words, - # headers, floats, math-inline, math-display; + my %count=('counts'=>\@cnt,'title'=>$title,'subcounts'=>[],'errorcount'=>0,'warnings'=>{}); return \%count; } +# Ensure the count array is of the right size +sub update_count_size { + my $count=shift @_; + my $counts=$count->{'counts'}; + while (scalar @{$counts}<$SIZE_CNT) {push @$counts,0;} +} + # Increment TeX count for a given count type sub inc_count { my ($tex,$cnt,$value)=@_; @@ -1875,15 +2637,17 @@ sub inc_count { # Get count value for a given count type sub get_count { my ($count,$cnt)=@_; - return ${$count->{'counts'}}[$cnt]; + my $counts=$count->{'counts'}; + if ($cnt<scalar @{$counts}) {return ${$counts}[$cnt];} + else {return 0;} } # Compute sum count for a count object sub get_sum_count { my $count=shift @_; my $sum=0; - for (my $i=scalar(@sumweights);$i-->0;) { - $sum+=get_count($count,$i+1)*$sumweights[$i]; + for (my $i=scalar(@sumweights);$i-->1;) { + $sum+=get_count($count,$i)*$sumweights[$i]; } return $sum; } @@ -1898,6 +2662,20 @@ sub number_of_subcounts { } } +# Returns the number of warnings (not distinct) +sub number_of_warnings { + my $count=shift @_; + my $n=0; + foreach my $m (values %{$count->{'warnings'}}) {$n+=$m;} + return $n; +} + +# Returns the number of distinct warnings +sub number_of_distinct_warnings { + my $count=shift @_; + return scalar keys %{$count->{'warnings'}}; +} + # Is a null count? (counts 1-7 zero, title starts with _) sub _count_is_null { my $count=shift @_; @@ -1905,15 +2683,23 @@ sub _count_is_null { for (my $i=1;$i<$SIZE_CNT;$i++) { if (get_count($count,$i)>0) {return 0;} } + if (scalar keys %{$count->{'warnings'}}) {return 0;} + if ($count->{'errorcount'}) {return 0;} return 1; } # Add one count to another sub _add_to_count { my ($a,$b)=@_; + update_count_size($a); + update_count_size($b); for (my $i=0;$i<$SIZE_CNT;$i++) { ${$a->{'counts'}}[$i]+=${$b->{'counts'}}[$i]; } + while (my ($text,$n)=each %{$a->{'warnings'}}) { + $a->{'warnings'}->{$text}+=$n; + } + $a->{'errorcount'}+=$b->{'errorcount'}; } # Add subcount to sum count and prepare new subcount @@ -1928,39 +2714,38 @@ sub next_subcount { sub add_to_total { my ($total,$count)=@_; _add_to_count($total,$count); - if (!_count_is_null($count)) { - push @{$total->{'subcounts'}},$count; - $count->{'parentcount'}=$total; - } + if (_count_is_null($count)) {return;} + push @{$total->{'subcounts'}},$count; + $count->{'parentcount'}=$total; } - ###### Result output routines -# Close the output, e.g. adding HTML tail +# Close the output (STDOUT), e.g. adding HTML tail sub Close_Output { if ($htmlstyle>1) { html_tail(); } + close STDOUT; } # Report if there were any errors occurring during parsing sub Report_Errors { if (defined $outputtemplate) {return;} - if ( !$briefsum && !$totalflag && $verbose>=0 ) { - foreach (keys %warnings) {formatprint($_,"p","nb");print "\n";} + if ( !$briefsum && !$totalflag && $printlevel>=0 ) { + foreach (keys %warnings) {formatprint($_,'p','nb');print "\n";} } if ($errorcount==0) {return;} - if ($briefsum && $totalflag) {print " ";} + if ($briefsum && $totalflag) {print ' ';} if ($htmlstyle) { print "<div class='error'><p>\n"; - print "There were ".$errorcount." error(s) reported!\n"; + print "There were $errorcount error(s) reported!\n"; print "</p></div>\n"; } elsif ($briefsum && $totalflag) { - print "(errors:".$errorcount.")"; + print "(errors:$errorcount)"; } else { - print "(errors:".$errorcount.")\n"; + print "(errors:$errorcount)\n"; } } @@ -1979,14 +2764,14 @@ sub print_word_freq { $Word{$wd}=__lc_merge($word,$Word{$wd}); } if ($htmlstyle) {print "<table class='stat'>\n<thead>\n";} - __print_word_freq("Word","Freq","th"); + __print_word_freq('Word','Freq','th'); if ($htmlstyle) {print "</thead>\n";} if ($optionWordClassFreq>0) { for $word (keys %Freq) {$Class{__word_class($word,\%regs)}+=$Freq{$word};} __print_sorted_freqs('langstat',\%Class); } if ($htmlstyle) {print "<tbody class='sumstat'>\n";} - __print_word_freq("All words",$sum,,'td','sum'); + __print_word_freq('All words',$sum,'td','sum'); if ($htmlstyle) {print "</tbody>\n";} if ($optionWordFreq>0) {__print_sorted_freqs('wordstat',\%Freq,\%Word,$optionWordFreq);} if ($htmlstyle) {print "</table>\n";} @@ -2026,7 +2811,7 @@ sub __print_sorted_freqs { my $sum=0; my ($word,$wd,$freq); if (!defined $min) {$min=0;} - if ($htmlstyle) {print "<tbody class='",$class,"'>\n";} + if ($htmlstyle) {print "<tbody class='$class'>\n";} else {print "---\n";} for $wd (sort {$Freq->{$b} <=> $Freq->{$a}} keys %{$Freq}) { if (defined $Word) {$word=$Word->{$wd};} else {$word=$wd;} @@ -2036,7 +2821,7 @@ sub __print_sorted_freqs { __print_word_freq($word,$freq); } } - if ($min>0) {__print_word_freq("Sum of subset",$sum,'td','sum');} + if ($min>0) {__print_word_freq('Sum of subset',$sum,'td','sum');} if ($htmlstyle) {print "</tbody>\n";} } @@ -2044,27 +2829,45 @@ sub __print_sorted_freqs { sub __print_word_freq { my ($word,$freq,$tag,$class)=@_; if (!defined $tag) {$tag='td';} - if (defined $class) {$class=' class=\''.$class.'\'';} else {$class='';} + if (defined $class) {$class=" class='$class'";} else {$class='';} if ($htmlstyle) { - print "<tr",$class,"><",$tag,">",$word,"</",$tag,"><",$tag,">",$freq,"</",$tag,"></tr>\n"; + print "<tr$class><$tag>$word</$tag><$tag>$freq</$tag></tr>\n"; } else { - print $word,": ",$freq,"\n"; + print $word,': ',$freq,"\n"; + } +} + +# Print macro usage statistics +sub print_macro_stat { + if ($htmlstyle) {print "<table class='stat'>\n<thead>\n";} + __print_word_freq('Macro/envir','Freq','th'); + if ($htmlstyle) {print "</thead>\n";} + if ($htmlstyle) {print "<tbody class='macrostat'>\n";} + foreach my $name (sort keys %MacroUsage) { + my $freq=$MacroUsage{$name}; + $name=text_to_printable($name); + $name=text_to_print($name); + if ($htmlstyle) { + print "<tr><td>$name</td><td>$freq</td></tr>\n"; + } else { + print $name,': ',$freq,"\n"; + } } + if ($htmlstyle) {print "</tbody>\n";} + if ($htmlstyle) {print "</table>\n";} } + ###### Printing routines # Print text using given style/colour sub print_with_style { my ($text,$style,$colour)=@_; - if ($style eq ' ') { - print text_to_print($text); - } elsif ($style eq '') { + if ($style eq $NOSTYLE) { print text_to_print($text); - error($Main,'Empty style should not occur in print_with_style!','BUG'); } elsif ($htmlstyle) { - print "<span class='$style'>".text_to_print($text).'</span>'; + print "<span class='$style'>",text_to_print($text),'</span>'; } else { ansiprint(text_to_print($text),$colour); if ($style=~/$separatorstyleregex/) {print $separator;} @@ -2079,18 +2882,35 @@ sub text_to_print { $text=~s/</</g; $text=~s/>/>/g; $text=~s/[ \t]{2}/\ /g; + } elsif ($texcodeoutput) { + $text=~s/\\/\\textbackslash¤/g; + $text=~s/([%{}])/\\$1/g; + $text=~s/¤/{}/g; } return $text; } +# Return &#xxx; code for character ; +sub convert_to_charcode { + my $ch=shift @_; + return '&#'.ord($ch).';'; +} + +# Convert special characters to charcodes to make it printable ; +sub text_to_printable { + my $text=shift @_; + $text=~s/([^[:print:]])/convert_to_charcode($1)/gse; + return $text; +} + # Print text, using appropriate tags for HTML sub formatprint { my ($text,$tag,$class)=@_; my $break=($text=~s/(\r\n?|\n)$//); if ($htmlstyle && defined $tag) { print '<'.$tag; - if ($class) {print " class='".$class."'";} - print '>'.text_to_print($text).'</'.$tag.'>'; + if ($class) {print " class='$class'";} + print '>'.text_to_print($text)."</$tag>"; } else { print text_to_print($text); } @@ -2130,7 +2950,7 @@ sub __count_and_header { return $count,$header,@_; } -# Return count title or "" if missing +# Return count title or '' if missing sub __count_header { my $count=shift @_; return $count->{'title'}||''; @@ -2150,16 +2970,19 @@ sub _print_sum_count { sub _print_count_brief { my ($count,$header,$tag1,$tag2)=__count_and_header(@_); my @cnt=@{$count->{'counts'}}; - if ($htmlstyle && $tag1) {print "<".$tag1.">";} - print $cnt[$CNT_WORDS_TEXT]."+".$cnt[$CNT_WORDS_HEADER]."+".$cnt[$CNT_WORDS_CAPTION]. - " (".$cnt[$CNT_COUNT_HEADER]."/".$cnt[$CNT_COUNT_FLOAT]. - "/".$cnt[$CNT_COUNT_INLINEMATH]."/".$cnt[$CNT_COUNT_DISPLAYMATH].")"; + if ($htmlstyle && $tag1) {print '<',$tag1,'>';} + print $cnt[$CNT_WORDS_TEXT],'+',$cnt[$CNT_WORDS_HEADER],'+',$cnt[$CNT_WORDS_OTHER]; + for (my $i=$SIZE_CNT_DEFAULT;$i<$SIZE_CNT;$i++) { + if ($cnt[$i]) {print '+',$cnt[$i],$countkey[$i];} + } + print ' (',$cnt[$CNT_COUNT_HEADER],'/',$cnt[$CNT_COUNT_FLOAT], + '/',$cnt[$CNT_COUNT_INLINEMATH],'/',$cnt[$CNT_COUNT_DISPLAYMATH],')'; if ($htmlstyle && $tag2) { - print "</".$tag1."><".$tag2.">"; + print '</',$tag1,'><',$tag2,'>'; $tag1=$tag2; - } else {print " ";} + } else {print ' ';} print text_to_print($header); - if ($htmlstyle && $tag1) {print "</".$tag1.">";} + if ($htmlstyle && $tag1) {print '</',$tag1,'>';} if ($finalLineBreak) {print "\n";} } @@ -2173,10 +2996,10 @@ sub _print_count_details { } if (@sumweights) {formatprint('Sum count: '.get_sum_count($count)."\n",'li');} for (my $i=1;$i<$SIZE_CNT;$i++) { - formatprint($countlabel[$i].': '.get_count($count,$i)."\n",'li'); + formatprint($countdesc[$i].': '.get_count($count,$i)."\n",'li'); } if (get_count($count,$CNT_FILE)>1) { - formatprint($countlabel[$CNT_FILE].': '.get_count($count,$CNT_FILE)."\n",'li'); + formatprint($countdesc[$CNT_FILE].': '.get_count($count,$CNT_FILE)."\n",'li'); } my $subcounts=$count->{'subcounts'}; if ($showsubcounts && defined $subcounts && scalar(@{$subcounts})>=$showsubcounts) { @@ -2198,14 +3021,14 @@ sub _print_count_template { $template=~s/\\n/\n/g; if ($htmlstyle) {$template=~s/\n/<br>/g;} my ($subtemplate,$posttemplate); - while ($template=~/\{SUB\?((.*?)\|)?(.*?)(\|(.*?))?\?SUB\}/is) { - __print_count_using_template($count,$`); + while ($template=~/^(.*)\{SUB\?((.*?)\|)?(.*?)(\|(.*?))?\?SUB\}(.*)$/is) { + __print_count_using_template($count,$1); # $1 ~ ${^PREMATCH} if (number_of_subcounts($count)>1) { - if (defined $2) {print $2;} - __print_subcounts_using_template($count,$3); - if (defined $5) {print $5;} + if (defined $3) {print $3;} + __print_subcounts_using_template($count,$4); + if (defined $6) {print $6;} } - $template=$'; + $template=$7; # $7 ~ ${^POSTMATCH} } __print_count_using_template($count,$template); } @@ -2213,16 +3036,16 @@ sub _print_count_template { # Print counts using template sub __print_count_using_template { my ($count,$template)=@_; - for (my $i=0;$i<$SIZE_CNT;$i++) { - $template=__process_template($template,$i,get_count($count,$i)); - } - $template=~s/\{VER\}/$versionnumber/gi; - # TODO: Should base warnings and errors on TeXcode or Main object - $template=__process_template($template,"W|WARN|WARNING|WARNINGS",length(%warnings)); - $template=__process_template($template,"E|ERR|ERROR|ERRORS",$errorcount); - $template=__process_template($template,"S|SUM",get_sum_count($count)); - $template=__process_template($template,"T|TITLE",$count->{'title'}||""); - $template=__process_template($template,"SUB",number_of_subcounts($count)); + while (my ($key,$cnt)=each %key2cnt) { + $template=__process_template($template,$key,get_count($count,$cnt)); + } + $template=~s/\{VER\}/\Q$versionnumber\E/gi; + $template=__process_template($template,'WARN|WARNING|WARNINGS',number_of_distinct_warnings($count)); + $template=__process_template($template,'NWARN|NWARNING|NWARNINGS',number_of_warnings($count)); + $template=__process_template($template,'ERR|ERROR|ERRORS',$count->{'errorcount'}); + $template=__process_template($template,'SUM',get_sum_count($count)); + $template=__process_template($template,'TITLE',$count->{'title'}||''); + $template=__process_template($template,'SUB',number_of_subcounts($count)); print $template; } @@ -2246,7 +3069,7 @@ sub __process_template { $template=~s/\{($label)\?(.*?)\|(.*?)\?(\1)\}/$3/gis; $template=~s/\{($label)\?(.*?)\?(\1)\}//gis; } - if (!defined $value) {$value="";} + if (!defined $value) {$value='';} $template=~s/\{($label)\}/$value/gis; return $template; } @@ -2259,56 +3082,61 @@ sub __process_template { sub flush_next { my ($tex,$style)=@_; my $ret=undef; - if (defined $style && $style ne '') { - set_style($tex,$style); - } + if (defined $style) {set_style($tex,$style);} if (defined $tex->{'next'}) { $ret=print_style($tex->{'next'},$tex->{'style'},$tex->{'printstate'}); } $tex->{'printstate'}=undef; - $tex->{'style'}='-'; + $tex->{'style'}=$STYLE_BLOCK; return $ret; } -# Print next token and gobble following spaces +# Print next token and gobble following spaces (incl. line break and comments) sub flush_next_gobble_space { - my ($tex,$style,$status)=@_; + my ($tex,$style,$state)=@_; my $ret=flush_next($tex,$style); if (!defined $ret) {$ret=0;} - if (!defined $status) {$status=$STATUS_IGNORE;} - my $prt=($verbose>0); - if ($tex->{'line'}=~s/^([ \t\f]*)(\r\n?|\n)([ \t\f]*)//) { - if (!$prt) { - } elsif ($verbose>2 || $ret) { - print $1; - line_return(-1,$tex); - my $space=$3; - if ($htmlstyle) {$space=~s/ /\ /g;} - print $space; - } else { - line_return(0,$tex); + if (!defined $state) {$state=$STATE_IGNORE;} + my $prt=($printlevel>0); + while (1) { + if ($tex->{'line'}=~s/^([ \t\f]*)(\r\n|\r|\n)([ \t\f]*)//) { + if (!$prt) { + } elsif ($printlevel>1 || $ret) { + print $1; + line_return(-1,$tex); + my $space=$3; + if ($htmlstyle) {$space=~s/ /\ /g;} + print $space; + $ret=0; + } else { + line_return(0,$tex); + } + } elsif ($tex->{'line'}=~s/^([ \t\f]+)//) { + if ($prt) {print $1;} } - } elsif ($tex->{'line'}=~s/^([ \t\f]*)//) { - if ($prt) {print $1;} + if ($tex->{'line'}=~s/^(\%+[^\r\n]*)//) { + print_style($1,'comment'); + $ret=1; + } else {return;} } } # Set presentation style sub set_style { my ($tex,$style)=@_; - if (!(($tex->{'style'}) && ($tex->{'style'} eq '-')) && $style ne '') {$tex->{'style'}=$style;} + if (!(defined $tex->{'style'} && $tex->{'style'} eq $STYLE_BLOCK)) {$tex->{'style'}=$style;} } # Print text using the given style, and log state if given sub print_style { my ($text,$style,$state)=@_; - (($verbose>=0) && (defined $text) && (defined $style)) || return 0; + (($printlevel>=0) && (defined $text) && (defined $style)) || return 0; my $colour; ($colour=$STYLE{$style}) || return; - if (($colour) && !($colour eq '-')) { + if (defined $colour && $colour ne '-') { print_with_style($text,$style,$colour); - if ($state) {print_style($state,'state');} - if ($style ne "cumsum") {$blankline=-1;} + if (defined $state) {print_style($state,'state');} + if ($style ne 'cumsum') {$blankline=-1;} return 1; } else { return 0; @@ -2318,89 +3146,56 @@ sub print_style { # Conditional line return sub line_return { my ($blank,$tex)=@_; - if ($blank<0 && $verbose<3) {$blank=1;} + if ($blank<0 && $printlevel<2) {$blank=1;} if ($blank<0 || $blank>$blankline) { if ((defined $tex) && @sumweights) { my $num=get_sum_count($tex->{'subcount'}); - print_style(" [".$num."]","cumsum"); + print_style(" [$num]",'cumsum'); } linebreak(); $blankline++; } } -# Print error message -sub _print_error { - my $text=shift @_; - line_return(1); - print_style("!!! ".$text." !!!",'error'); - line_return(1); -} - -# Print errors in buffer and delete errorbuffer -sub flush_errorbuffer { - my $source=shift @_; - my $err=$source->{'errorbuffer'} || return; - foreach (@$err) {_print_error($_);} - $source->{'errorbuffer'}=undef; -} - ###### Print help on style/colour codes # Print output style codes if conditions are met -sub conditional_print_help_style { - if ($showcodes) {_print_help_style();} +sub conditional_print_style_list { + if ($showcodes) {_print_style_list();} return $showcodes; } # Print help on output styles -sub _print_help_style { - if ($verbose<=0) {return;} - if ($htmlstyle) {print "<div class='stylehelp'><p>";} - formatprint("Format/colour codes of verbose output:","h2"); +sub _print_style_list { + if ($printlevel<=0) {return;} + if ($htmlstyle) {print '<div class="stylehelp"><p>';} + formatprint('Format/colour codes of verbose output:','h2'); print "\n\n"; - _help_style_line('Text which is counted',"word1","counted as text words"); - _help_style_line('Header and title text',"word2","counted as header words"); - _help_style_line('Caption text and footnotes',"word3","counted as caption words"); - _help_style_line("Ignored text or code","ignore","excluded or ignored"); - _help_style_line('\documentclass',"document","document start, beginning of preamble"); - _help_style_line('\macro',"command","macro not counted, but parameters may be"); - _help_style_line('\macro',"exclcommand","macro in excluded region"); - _help_style_line("[Macro options]","option","not counted"); - _help_style_line('\begin{group} \end{group}',"grouping","begin/end group"); - _help_style_line('\begin{group} \end{group}',"exclgroup","begin/end group in excluded region"); - _help_style_line('$ $',"mathgroup","counted as one equation"); - _help_style_line('$ $',"exclmath","equation in excluded region"); - _help_style_line('% Comments',"comment","not counted"); - _help_style_line('%TC:TeXcount instructions',"tc","not counted"); - _help_style_line("File to include","fileinclude","not counted but file may be counted later"); - if ($showstates) { - _help_style_line('[state]',"state","internal TeXcount state"); - } - if (@sumweights) { - _help_style_line('[cumsum]',"cumsum","cumulative sum count"); - } - _help_style_line("ERROR","error","TeXcount error message"); - if ($htmlstyle) {print "</p></div>";} + foreach my $style (@STYLE_LIST) { + my $desc=$STYLE_DESC{$style}; + if ($desc=~/^(.*):\s+(.*)$/) { + _help_style_line($1,$style,$2); # $1~${^PREMATCH}, $2~${^POSTMATCH} + } else { + _help_style_line($desc,$style); + } + } + if ($htmlstyle) {print '</p></div>';} print "\n\n"; } # Print one line of help sub _help_style_line { my ($text,$style,$comment)=@_; - if ($htmlstyle) { - $comment=" .... ".text_to_print($comment); - } else { - $comment=" .... ".$comment; - } + if (!defined $comment) {$comment='';} + if ($htmlstyle) {$comment=' .... '.text_to_print($comment);} + else {$comment=' .... '.$comment;} if (print_style($text,$style)) { print $comment; linebreak(); } } - ###### Help routines @@ -2419,12 +3214,36 @@ sub print_license { wprintstringdata('License'); } -# Print TeXcount parameter list +# Print short TeXcount help +sub print_short_help { + wprintstringdata('ShortHelp'); +} + +# Print TeXcount options list sub print_syntax { wprintstringdata('OptionsHead'); wprintstringdata('Options','@ - :'); } +# Prinst TeXcount options containing substring +sub print_syntax_subset { + my $pattern=shift @_; + my $data=StringData('Options'); + if (!defined $data) { + error($Main,'No StringData Options.','BUG'); + return; + } + my @options; + foreach (@$data) { + if (/^\s*([^\s]+\s)*[^\s]*\Q$pattern\E/) {push @options,$_;} + } + if (scalar(@options)==0) {print "No options contained $pattern.\n";} + else { + print "Options containing \"$pattern\":\n\n"; + wprintlines('@ - :',@options); + } +} + # Print complete TeXcount help sub print_help { print_help_title(); @@ -2444,13 +3263,13 @@ sub print_help_text { wprintstringdata('TCinstructions'); } -# Print help on specific macro or group -sub print_help_on { +# Print help on specific macro or environment +sub print_help_on_rule { my $arg=shift @_; my $def; my %rules=( '\documentclass' => 'Initiates LaTeX document preamble.', - '\begin' => 'Treatmend depends on group handling rules.', + '\begin' => 'Treatmend depends on environment handling rules.', '\def' => 'Excluded from count.', '\verb' => 'Strong exclude for enclosed region.', '$' => 'Opens or closes inlined equation', @@ -2460,37 +3279,46 @@ sub print_help_on { '\[' => 'Opens displayed equation.', '\]' => 'Closes displayed equation initiated by \[.'); if (!defined $arg || $arg=~/^\s*$/) { - print "Specify macro or group name after the -h= option.\n"; + print "\nSpecify macro or environment name after the -h= option.\n"; return; } + if ($arg=~/^[\w\-\%]+:/) { + remove_all_rules(); + %rules=(); + while ($arg=~s/^([\w\-\%]+)://) {include_package($1);} + } if ($def=$rules{$arg}) { - print "Special rule (hard coded) for $arg\n"; + print "\nSpecial rule (hard coded) for $arg\n"; print $def."\n"; } elsif ($arg=~/^\\/) { + my $hasrule=0; + print "\nRule(s) for macro $arg\n"; + if ($def=$TeXmacrocount{$arg}) { + $hasrule=1; + } if ($def=$TeXfileinclude{$arg}) { - print "Rule for macro $arg\n"; + $hasrule=1; print "Takes file name as parameter which is included in document.\n"; - } elsif ($def=$TeXmacro{$arg}) { - print "Rule for macro $arg\n"; + } + if ($def=$TeXmacro{$arg}) { + $hasrule=1; _print_rule_macro($arg,$def); - if ($def=$TeXheader{$arg}) { - print "This macro is counted as a header\n"; - } - if ($def=$TeXfloatinc{$arg}) { - print "This macro is also counted inside floats as captions.\n"; - } - } elsif ($def=$TeXmacroword{$arg}) { - print "Rule for macro $arg\n"; - print "Count as ".$def." word(s).\n"; - } else { - print "No macro rule defined for $arg.\nParameters treated as surrounding text.\n"; + } + if ($def=$TeXfloatinc{$arg}) { + $hasrule=1; + print "\nIncluded inside floats:\n"; + _print_rule_macro($arg,$def); + } + if (!$hasrule) { + print "\nNo macro rule defined for $arg.\nParameters treated as surrounding text.\n"; } } else { - if ($def=$TeXgroup{$arg}) { - print "Rule for group $arg\n"; - _print_rule_group($arg,$def); + if ($def=$TeXenvir{$arg}) { + print "\nRule for environment $arg\n"; + _print_rule_macrocount($PREFIX_ENVIR.$arg); + _print_rule_envir($arg,$def); } else { - print "No default group rule defined for $arg.\nContent handled as surrounding text.\n"; + print "\nNo default environment rule defined for $arg.\nContent handled as surrounding text.\n"; } } } @@ -2498,60 +3326,119 @@ sub print_help_on { # Print macro handling rule sub _print_rule_macro { my ($arg,$def)=@_; - my %rules=( - 0 => 'Exclude from count', - 1 => 'Count as text', - 2 => 'Count as header', - 3 => 'Count as caption', - -1 => 'Exclude as float (i.e. include captions)', - -2 => 'Strong exclude (ignore begin-end groups)', - -3 => 'Stronger exclude (ignore all macros)', - -9 => 'Exclude as preamble'); if (ref($def) eq 'ARRAY') { - print "Takes ".scalar(@{$def})." parameter(s):\n"; - foreach my $i (@{$def}) { - print " - ".$rules{$i}."\n"; + my $optionflag=0; + print "Takes the following parameter(s):\n"; + foreach my $state (@{$def}) { + if ($state==$_STATE_OPTION) {$optionflag=1;} + elsif ($optionflag) { + $optionflag=0; + print " - Optional [] containing $state2desc{$state}\n"; + } else { + print " - $state2desc{$state}\n"; + } } } else { - print "Takes ".$def." parameter(s), not included in counts.\n"; + print "Takes $def parameter(s): content ignored, i.e. not included in counts.\n"; } } -# Print group handling rule -sub _print_rule_group { +# Print environment handling rule +sub _print_rule_envir { my ($arg,$def)=@_; - my %rules=( - 0 => 'Not included', - 1 => 'Text, words included in text count', - 2 => 'Header, words included in header count', - 3 => 'Float caption, words included in float caption count', - 6 => 'Inline mathematics, words not counted', - 7 => 'Displayed mathematics, words not counted', - -1 => 'Float, not included, but looks for captions'); - print "Rule used: ".$rules{$def}."\n"; - if ($def=$TeXmacro{'begin'.$arg}) { + print "Contents parsed as $state2desc{$def}\n"; + if ($def=$TeXmacro{$PREFIX_ENVIR.$arg}) { _print_rule_macro($def); } } +# Print macrocount rule, return rule +sub _print_rule_macrocount { + my $arg=shift @_; + my $def=$TeXmacrocount{$arg}; + if (!defined $def) {return undef;} + if (ref($def) eq 'ARRAY') { + my @wd=@$def; + foreach (@wd) {$_=$countdesc[$_];} + print 'Increments the following counters: ',join('; ',@wd),".\n"; + } else { + print "Counted as $def word(s).\n"; + } +} + +# Print style or style category summary +sub print_help_on_styles { + my $style=shift @_; + if (defined $style) { + if (my $def=$STYLES{$style}) { + _print_help_on_style_category($style); + print wrap('',' ',"$style = ".join(', ',grep(/^\w+$/,sort keys %$def))),".\n"; + } elsif ($def=$STYLE_DESC{$style}) { + print $style,' = ',$def,"\n"; + } else { + print "Unknown style or style category: $style.\n"; + } + } else { + print wrap('',' ','Styles: '.join(', ',@STYLE_LIST)),".\n\n"; + print wrap('',' ','Style categories: '.join(', ',sort grep(/^\w+$/,keys %STYLES))),".\n\n"; + print "Use -help-style={style} to get help on particular style or style category.\n"; + } +} + +# Print help on a particular style category (optional prefix and indentation) +sub _print_help_on_style_category { + my ($cat,$prefix,$indent)=@_; + if (!defined $prefix) {$prefix='';} + if (!defined $indent) {$indent=' ';} +} ###### HTML routines # Print HTML header sub html_head { - print "<html>\n<head>"; - print "\n<meta http-equiv='content-type' content='text/html; charset=$outputEncoding'>\n"; - _print_html_style(); - print "</head>\n\n<body>\n"; - print "\n<h1>LaTeX word count"; - if ($showVersion>0) {print " (version ",_html_version(),")"} - print "</h1>\n"; + if (defined $HTMLfile) { + if (open(FH,$HTMLfile)) { + my $text=join('',<FH>); + close FH; + if ( $text=~/^(.*)<!--+\s*texcount\s*--+>(.*)$/is + || $text=~/^(.*)\$\{texcount\}(.*)$/is + || $text=~/^(.*)(<\/body.*)$/is + ) { + $htmlopen=$1; + $htmlclose=$2; + } else { + error($Main,"Invalid HTML template format."); + } + } else { + error($Main,"HTML template file not found: $HTMLfile"); + } + } + if (defined $htmlopen) { + $htmlopen=~s/\$\{encoding\}/\Q$outputEncoding\E/g; + $htmlopen=~s/\$\{version\}/@{[_html_version()]}/g; + print $htmlopen; + $htmlopen=undef; + } else { + print "<html>\n<head>\n"; + print "<meta http-equiv='content-type' content='text/html; charset=$outputEncoding'>\n"; + _print_html_style(); + foreach (@htmlhead) {print $_,"\n";} + print "</head>\n\n<body>\n"; + print "\n<h1>LaTeX word count"; + if ($showVersion>0) {print ' (version ',_html_version(),')'} + print "</h1>\n"; + } } # Print HTML tail sub html_tail { - print "</body>\n\n</html>\n"; + if (defined $htmlclose) { + print $htmlclose; + $htmlclose=undef; + } else { + print "</body>\n\n</html>\n"; + } } # Return version number using HTML @@ -2563,27 +3450,45 @@ sub _html_version { # Print HTML STYLE element sub _print_html_style { +if (defined $CSShref) { + print "<link rel='stylesheet' href='$CSShref' type='text/css'>\n"; + return; +} +if (defined $CSSfile) { + if (open(FH,$CSSfile)) { + print "<style>\n<!---\n",<FH>,"-->\n</style>\n"; + close(FH); + return; + } else { + error($Main,"CSS file not found: $CSSfile"); + } +} print <<END <style> <!-- body {width:auto;padding:5;margin:5;} -.error {font-weight:bold;color:#f00;font-style:italic;} -.word1,.word2,.word3 {color: #009; border-left: 1px solid #CDF; border-bottom: 1px solid #CDF;} -.word2 {font-weight: 700;} -.word3 {font-style: italic;} -.command {color: #c00;} -.exclcommand {color: #f99;} +.error {font-weight:bold;color:#f00;font-style:italic; border: solid 1px red;} +.word,.hword,.oword,.altwd {border-left: 1px solid #CDF; border-bottom: 1px solid #CDF;} +.word {color: #009} +.hword {color: #009; font-weight: 700;} +.oword {color: #009; font-style: italic;} +.altwd {color: #00c; font-style: oblique;} +.cmd {color: #c00;} +.exclcmd {color: #f99;} .option {color: #cc0;} -.grouping, .document {color: #900; font-weight:bold;} +.optparm {color: #c90;} +.envir, .document {color: #900; font-weight:bold;} +.math {color: #6c0;} .mathgroup {color: #090;} -.exclmath {color: #6c6;} +.exclmath {color: #9f6;} +.mathcmd {color: #6c0;} .ignore {color: #999;} -.exclgroup {color:#c66;} +.exclenv {color:#c66;} .tc {color: #999; font-weight:bold;} .comment {color: #999; font-style: italic;} .state {color: #990; font-size: 70%;} .cumsum {color: #999; font-size: 80%;} -.fileinclude {color: #696; font-weight:bold;} +.fileinc {color: #696; font-weight:bold;} .warning {color: #c00; font-weight: 700;} div.filegroup, div.parse, div.stylehelp, div.count, div.sumcount, div.error { @@ -2603,7 +3508,7 @@ ul.count {list-style-type: none; margin: 4pt 0pt; padding: 0pt;} .fielddesc {font-style: italic;} .nb {color: #900;} -table.stat {border:2px solid #339; background:#666; border-collapse:collapse;} +table.stat {border:2px solid #339; margin:0 0 8pt 0; background:#666; border-collapse:collapse;} table.stat tr {border:1px solid #CCC;} table.stat th, table.stat td {padding:1pt 4pt;} table.stat col {padding:4pt;} @@ -2612,6 +3517,7 @@ table.stat tbody {border:1px solid #333;} table.stat tbody.sumstat {background:#FFC;} table.stat tbody.langstat {background:#FEE;} table.stat tbody.wordstat {background:#EEF;} +table.stat tbody.macrostat {background:#EFE;} table.stat .sum {font-weight:bold; font-style:italic;} --> </style> @@ -2655,6 +3561,7 @@ sub wprintstringdata { my $data=StringData($name); if (!defined $data) { error($Main,"No StringData $name.",'BUG'); + return; } wprintlines(@_,@$data); } @@ -2709,13 +3616,13 @@ sub wprintlines { my $firstindent=0; if ($line=~s/^(\t|\s{2,})(\S)/$2/) {$firstindent=$ind1;} my $indent=$firstindent; - if ($line=~/^(.*\S)(\t|\s{2,})/) { + if ($line=~/^(.*\S)(\t|\s{2,})(.*)$/) { $indent=$ind2; if ($1 eq '|') {$line=' ';} - else {$line=$1." ";} + else {$line=$1.' ';} $i=$indent-$firstindent-length($line); if ($i>0) {$line.=' ' x $i;} - $line.=$'; + $line.=$3; # $3~${^POSTMATCH} } print wrap(' ' x $firstindent,' ' x $indent,$line)."\n"; } @@ -2750,6 +3657,12 @@ which grants you, the user, the right to use, modify and distribute the script. The script has LPPL status "maintained" with ${maintainer} being the current maintainer. :::::::::::::::::::::::::::::::::::::::: +:::::::::: ShortHelp +Syntax: texcount.pl [options] files + +Use option -help (or just -h) to get help; -help-options (-hopt) to get list of command line options, or -help-options=substring for help on all options containing substring. + +:::::::::::::::::::::::::::::::::::::::: :::::::::: HelpTitle *************************************************************** * TeXcount version ${versionnumber}, ${versiondate} @@ -2760,7 +3673,7 @@ Count words in TeX and LaTeX files, ignoring macros, tables, formulae, etc. :::::::::::::::::::::::::::::::::::::::: :::::::::: OptionsHead -Syntax: TeXcount.pl [options] files +Syntax: texcount.pl [options] files Options: @@ -2775,6 +3688,7 @@ Options: -v2 More verbose: also print ignored text. -v3 Even more verbose: include comments and options. -v4 Same as -v3 -showstate. + -v=, v0=, ..., -v4= Set verbosity by adding/removing particular types of token (styles) to include in the verbose output. Use -help-style to get details of which tokens are included in each style, and of classes of tokens (style categories). -showstate Show internal states (with verbose). -brief Only prints a brief, one line summary of counts. -q, -quiet Quiet mode, no error messages. Use is discouraged! @@ -2789,7 +3703,11 @@ Options: -sep=, -separator= Separating character or string to be added after each word. -html Output in HTML format. -htmlcore Only HTML body contents. - -opt, -optionfile Read options/parameters from file. + -htmlfile= HTML template file to use with the -html option. Use <!-- TeXcode --> to indicate where the output from TeXcount should be inserted. + -tex Encode TeX special characters for output into TeX code + -cssfile= CSS file to include instead of default styles. + -css= CSS href to include instead of default styles. Can use -css=file:{filename} instead of -cssfile={filename}. + -opt=, -optionfile= Read options/parameters from file. - Read LaTeX code from STDIN. -inc Parse included TeX files (as separate file). -merge Merge included TeX files into code (in place). @@ -2800,7 +3718,8 @@ Options: -total Do not give sums per file, only total sum. -1 Same as -brief and -total. Ensures there is only one line of output. If used in conjunction with -sum, the output will only be the total number. (NB: Character is the number one, not the letter L.) -template= Speficy an output template. Use {1},...,{7}, {SUM} and {TITLE} to include values, {1?...?1} etc. to conditionally include sections, {1?....|...?1} etc. to specify an alternative text if zero. To include subcounts, use {SUB?...?SUB} where ... is replaced with the template to use per subcount. Line shift may be specified using \\n. - -dir, -dir= Specify the working directory using -dir=path. Remember that the path must end with \\ or /. If only -dir is used, the directory of the parent file is used. + -dir, -dir= Specify the working directory using -dir=path. Remember that the path must end with \\ or /. If only -dir is used, the directory of the provided file is used and paths (e.g. included files) are assumed to be absolute or relative to this. The default at startup is -dir=. which means that the directory from which TeXcount is run is the working directory with all paths absolute or relative to this. + -auxdir, -auxdir= Specify directory for auxilary files, e.g. the bibliograph (.bbl) file. If only -auxdir is used, the working directory (as determined by -dir or -dir=) is assumed. If -auxdir= is used with -dir=, it sets the path to the auxilary directiry. If -auxdir= is used with -dir, the working directory is determined from the location of the provided file, and the path to the auxilary directory is assumed to be absolute or relative to this. -enc=, -encoding= Specify encoding (default is to guess the encoding). -utf8, -unicode Selects Unicode (UTF-8) for input and output. This is automatic with -chinese, and is required to handle e.g. Korean text. Note that the TeX file must be save in UTF-8 format (not e.g. GB2312 or Big5), or the result will be unpredictable. -alpha=, -alphabets= List of Unicode character groups (or digit, alphabetic) permitted as letters. Names are separated by ',' or '+'. If list starts with '+', the alphabets will be added to those already included. The default is Digit+alphabetic. @@ -2813,12 +3732,18 @@ Options: -char, -character, -letters Counts letters/characters instead of words. Note that spaces and punctuation is not counted. -char-only, ..., -letters-only Like -letters, but counts alphabetic letters only. -countall, -count-all The default setting in which all characters are included as either alphabets og logograms. - -freq Produce individual word frequency table. - -stat Produce statistics on language/script usage. + -freq, -freq= Produce individual word frequency table. Optionally give minimal number of occurences to be listed. + -stat Produce statistics on language/script usage. + -macrostat, -macrofreq Print macro usage statistics. -codes Display output style code overview and explanation. This is on by default. -nocodes Do not display output style code overview. + -out= Write output to file, give filename as option value. -h, -?, -help, /? Help text. -h=, -?=, -help=, /?= Takes a macro or group name as option and returns a description of the rules for handling this if any are defined. If handling rule is package specific, use -incpackage=package name: -incpackage must come before -h= on the command line to take effect. + -help-options, -h-opt List all options. + -help-options=, -h-opt= List all options containing the provided string, e.g. -h-opt=dir or -h-opt=-v (the initial - in -v causes only options starting with v to be listed). + -help-style List the styles and style categories: i.e. those permitted used with -v={styles-list}. + -help-style= Give description of style or style category. -ver, -version Print version number. -lic, -license, -licence Licence information. @@ -2858,10 +3783,13 @@ and are used to control how TeXcount parses the document. The following instruct The [param.states] is used to indicate the number of parameters used by the macro and the rules of handling each of these: the format is [#,#,...,#] with # representing one number for each parameter giving the parsing state to use for that parameter, alternatively just a single number (#) indicating how many parameters to ignore (parsing state 0). The option [content-state] is used to give the parsing state to use for the contents of a begin-end group. The main parsing states are 0 to ignore and 1 to count as text. Parsing instructions which may be used anywhere are: -@ - : - %TC:ignore start block to ignore - %TC:endignore end block to ignore - %TC:break [title] add subcount break point here +@ - : + %TC:break [title] add subcount break point here + %TC:incbib include bibliography (same as running with -incbib) + %TC:ignore ignore region, end with %TC:endignore + %TC:insert [code] insert code for TeXcount to process as TeX code + %TC:newtemplate start a new template, ie delete the existing one + %TC:template [template] add another line to the template specification See the documentation for more details. Command line options and most %TC commands (prefixed by % rather than %TC:) may be placed in an options file. This is particularly useful for defining your own output templates and macro handling rules. diff --git a/Master/texmf-dist/doc/support/texcount/QuickReference.pdf b/Master/texmf-dist/doc/support/texcount/QuickReference.pdf Binary files differindex e3a8d1f7e9b..fdaa375a5c3 100644 --- a/Master/texmf-dist/doc/support/texcount/QuickReference.pdf +++ b/Master/texmf-dist/doc/support/texcount/QuickReference.pdf diff --git a/Master/texmf-dist/doc/support/texcount/QuickReference.tex b/Master/texmf-dist/doc/support/texcount/QuickReference.tex index 16726e808d2..7115a1bd944 100644 --- a/Master/texmf-dist/doc/support/texcount/QuickReference.tex +++ b/Master/texmf-dist/doc/support/texcount/QuickReference.tex @@ -30,15 +30,20 @@ Instructions to \TeXcount{} can be given from within the \LaTeX{} document using \LaTeX{} comments on the format \codeline{\%TC:\textit{instruction [name] parameters}} where the name is use for instructions providing macro handling rules to give the name of the macro or group for which the rule applies. - +% \input{sub_tc_other} -\subsection{Adding macro handling rules} - -The macro handling rules all take the format +Instructions for adding macro handling rules all take the format \codeline{\%TC:\textit{instruction name parameters}} -where the name indicates the macro (with backslash) or group name for which the rule applies. - +where the name indicates the macro (with backslash) or group name for which the rule applies: +% \input{sub_addrules} +The available parser rules for environment contents and macro parameters are \code{word}/\code{text}, \code{headerword}, \code{otherword}, \code{header}, \code{float}, \code{inlinemath}, \code{displaymath}, \code{ignore}, \code{xx} (strong exclude), \code{xxx} (stronger exclude), \code{xall} (exclude all) or any of their aliases. + +The available counters are \code{word}/\code{text}, \code{headerword}, \code{otherword}, \code{header}, \code{float}, \code{inlinemath}, \code{displaymath} or any of their aliases. + +Available file specifications contain one or more of \code{input} (for \code{\bs{input}}), \code{file} (file path), \code{texfile} (use with \code{\bs{include}}), \code{<bbl>} (to include the bibliography file), \code{dir} and \code{subdir}. The \code{dir} and \code{subdir} are used to modify the search path within the included document (used with the \code{import} package). + + \end{document} diff --git a/Master/texmf-dist/doc/support/texcount/TeXcount.pdf b/Master/texmf-dist/doc/support/texcount/TeXcount.pdf Binary files differindex c1fbb094c99..fe3802e435b 100644 --- a/Master/texmf-dist/doc/support/texcount/TeXcount.pdf +++ b/Master/texmf-dist/doc/support/texcount/TeXcount.pdf diff --git a/Master/texmf-dist/doc/support/texcount/TeXcount.tex b/Master/texmf-dist/doc/support/texcount/TeXcount.tex index 7d5e89110b1..16f90ededd4 100644 --- a/Master/texmf-dist/doc/support/texcount/TeXcount.tex +++ b/Master/texmf-dist/doc/support/texcount/TeXcount.tex @@ -5,6 +5,10 @@ \usepackage{listings} \usepackage{url} +%% TeXcount parsing rules +%TC:envir lstlisting [] xall +%TC:macro url [ignore] + \include{macros} %\parindent=0pt\parskip=8pt @@ -34,6 +38,8 @@ Version \version\copyrightfootnote \pagebreak +% --------------------------------------------------------------------------- + \section{What \TeXcount{} does} \TeXcount{} is a Perl script made for counting the words in a \LaTeX{} document. Since \LaTeX{} documents are formated using lots of macro instructions and often contain both mathematical formulae and floating tables and figures, this is no trivial task. @@ -62,6 +68,7 @@ Automatic parsing of included documents is possible, but is by default turned of Since \TeXcount{} relies on a relatively simple rules for handling the different macros and only performs limited checks on the validity of the \LaTeX{} document, it is your responsibility to make sure the document actually typesets in \LaTeX{} before running it through \TeXcount{}. Also, \TeXcount{} relies on handling macros taking parameters enclosed with \{ and \}, and on ignoring options enclosed by [ and ]: macros with significantly different syntax such as \code{\bs{vskip}} cannot be handled. There are also limitations on what may be contained in macro options enclosed in [], although this restriction may be relaxed by specifying the command line option \code{-relaxed}. + \subsection{What \TeXcount{} counts} Basically, \TeXcount{} has seven different counts plus an additional file count for use with total counts over a set of files. These and their indices (numbers used to identify them) are: @@ -71,13 +78,13 @@ Basically, \TeXcount{} has seven different counts plus an additional file count \item[2. Header words:] Words that occur in headers, e.g. \code{\bs{title}} and \code{\bs{section}}. \item[3. Caption words:] Words that occur in figure and table captions. \item[4. Header count:] This counts the number of headers, i.e. each \code{\bs{section}} counts as 1. -\item[5. Figure/float count:] This counts the number of floats and figures, e.g. \code{table} and \code{figure} groups. +\item[5. Figure/float count:] This counts the number of floats and figures, e.g. \code{table} and \code{figure} environments. \item[6. Inline formulae:] This counts the number of inline formulae, i.e. \code{\$\ldots\$}. -\item[7. Displayed formulae:] This counts the number of displayed formulae, e.g. \code{\bs{[}\ldots\bs{]}} or \code{equation} groups. +\item[7. Displayed formulae:] This counts the number of displayed formulae, e.g. \code{\bs{[}\ldots\bs{]}} or \code{equation} environments. \end{description} -These are stored in an array and often referenced by their index: e.g. in the option \code{-sum=} which takes parameter values corresponding to counts 1 to 7, or \code{-tempate=} in which the counts are referred to by the indices 0 to 7. +These are stored in an array and sometimes referenced by their index: e.g. in the option \code{-sum=} which takes parameter values corresponding to counts 1 to 7. In other contexts, however, like in the \code{-tempate=} or when incrementing specific counters through the \code{\%TC:macrocount} instruction, the counters may be referred to by keywords rather than the indices 0 to 7. -The primary role is to count the words. It is not entirely clear what should be considered words, so I have had to make some decisions. A sequence of letters is certainly a word. I also count acronyms like \textit{e.g.}, dashed words like \textit{over-all}, and \textit{it's} as one word. I have decided also to count numbers as words unless they are placed in a math environment. If \TeXcount{} breaks words that contain special characters, you may try the option \code{-relaxed} which extends the range of characters allowed as part of words. +The primary role is to count the words. It is not entirely clear what should be considered words, so I have had to make some decisions. A sequence of letters is certainly a word. \TeXcount{} also counts acronyms like \textit{e.g.}, dashed words like \textit{over-all}, and \textit{it's} as one word. It also counts numbers as words unless they are placed in a math group. If \TeXcount{} breaks words that contain special characters, you may try the option \code{-relaxed} which extends the range of characters allowed as part of words. Alternatively, \TeXcount{} may be asked to count the number of letters/characters (not including spaces). It may also be set to count Chinese or Japanese characters. @@ -85,33 +92,39 @@ Mathematical formulae are not counted as words: it would be difficult to define Text in headers (\code{\bs{title}}, \code{\bs{section}}, etc.) are counted separately: \TeXcount{} counts the number of headers as well as the number of words in headers. It may also provide subcounts for each of these by specifying the \code{-sub} option. -Floating environments (or potentially floating environments) such as tables and figures are not counted as text, even if the cells of a table may containt text. However, if they have captions, these will be counted separately much like headers were. Footnotes are included in this count. By default, begin--end environments do not modify the parsing state: i.e. environments within the text are counted as text, etc. Rules for the most common environments, at least those that require non-default treatment, should be predefined, but you may have to add more rules if you use environments defined in packages or by yourself. If you wish to be warned against any groups names you use that lack a defined rule, set the option \code{-strict}. +Floating environments (or potentially floating environments) such as tables and figures are not counted as text, even if the cells of a table may containt text. However, if they have captions, these will be counted separately much like headers were. Footnotes are included in this count. By default, environments do not modify the parsing state: i.e. environments within the text are counted as text, etc. Rules for the most common environments, at least those that require non-default treatment, should be predefined, but you may have to add more rules if you use environments defined in packages or by yourself. If you wish to be warned against any environments names you use that lack a defined rule, set the option \code{-strict}. Some macros are words by themselves: e.g. \code{\bs{LaTeX}}. These are counted as words provided the macro word rule has been defined for them, but you cannot expect \TeXcount{} to count something like \code{\bs{LaTeX}-word} or \code{\{\bs{TeX}\}count} as one word although the above explanation inicates that it should: \TeXcount{} will in both cases evaluate the macro and the following text separately and thus count them as separate entities. Since \TeXcount{} recognises \code{\bs{LaTeX}} and \code{\bs{TeX}} as single words, each of the two examples would end up being counted as two words. + \subsection{What \TeXcount{} does not do} -While an ideal solution should be able to expand the macro instructions, thus being able to handle new macros, that would at it's worst require reimplementing much of \TeX{}, something that is clearly unrealistic. Instead, I have opted for a simpler solution: to define rules stating which paramters to count and which to ignore and allowing for such rules to be added easily. Thus, \TeXcount{} cannot handle macros that may take a variable number of parameters. Nor can it handle macros that takes parameters on forms other than \code{\{parameter\}}. In particular, when interpreting macros, \TeXcount{} treats all \code{[\ldots]} blocks as macro options that should be excluded from the counts; \TeXcount{} has some restrictions on what it permits to go into an option in terms of size and characters. +While an ideal solution should be able to expand the macro instructions, thus being able to handle new macros, that would at it's worst require reimplementing much of \TeX{}, something that is clearly unrealistic. Instead, I have opted for a simpler solution: to define rules stating which paramters to count and which to ignore and allowing for such rules to be added easily. Thus, \TeXcount{} cannot handle macros that may take a variable number of parameters. Nor can it handle macros that takes parameters on forms other than \code{\{parameter\}}. However, support has now been added for macro options on the form \code{[\ldots]} to be parsed. In general, while \TeXcount{} does the parsing in some detail, it does not do it exacly as \TeX{} does it. In some respects there may therefore be critical differences: e.g. while \TeX{} reads one character at a time, \TeXcount{} reads one word at a time, so while \LaTeX{} would interpret \code{\bs{cite} me} as \code{\bs{}cite\{m\}e}, \TeXcount{} would interpret it like \code{\bs{cite}\{me\}}. Another issue is that, since \TeXcount{} does not know how to expand macros, it cannot handle macros like \code{\bs{maketitle}} that add text to the document. With respect to \code{\bs{maketitle}}, I have instead set the rule for \code{\bs{title}\{title text\}} to count this as a header although it does not itself produce any text. + \subsection{Problems to be aware of} In most large documents, there will be cases where \TeXcount{} does not give an exact count. Reasons may be macros \TeXcount{} does not recognise, words that \TeXcount{} split in two (or more) because of special characters not recognised as letters, or options and parameters not counted which actually produce text. Some problems may also arise because it is not always clear what should be counted and \TeXcount{} implements one particular choice: counting numbers as letters/words, not counting formulae as words, not to count tables as text, etc. However, hopefully these should either consist of individual, infrequent errors which should have limited effect on the total count, or entire regions that are included or excluded for which the user may change the parsing rule to produce the desired count. There are, however, problems that may arise which are more fundamental and result in counts which are simply wrong rather than just inaccurate, or even make \TeXcount{} fail entirely. -If \TeXcount{} fails to detect group endings properly, either closing \code{\{} or \code{\bs{end}}, it may end up ignoring major parts of the document. This should normally produce errors of some kind, although there may be cases when no errors are produced. However, by looking at the verbose output, it will be very clear that entire parts of the document has been excluded. Such problems may be cause by macros that allow unmatched group delimiters, and some effort has been made to minimise the risk of this at the cost of risking other but less critical errors: e.g. there are limits to what is permitted as macro options in order to ensure that a single unmatched \code{[} does not cause large parts of the document to be interpreted as a big option. +If \TeXcount{} fails to detect environment endings properly, either closing \code{\{} or \code{\bs{end}}, it may end up ignoring major parts of the document. This should normally produce errors of some kind, although there may be cases when no errors are produced. However, by looking at the verbose output, it will be very clear that entire parts of the document has been excluded. Such problems may be cause by macros that allow unmatched group delimiters, and some effort has been made to minimise the risk of this at the cost of risking other but less critical errors: e.g. there are limits to what is permitted as macro options in order to ensure that a single unmatched \code{[} does not cause large parts of the document to be interpreted as a big option. For users of languages containing letters other than the Latin letters A to Z, there is a risk that \TeXcount{} may have difficulty identifying words correctly. The script relies on Perl to recognise words as sequence of letters, and must therefore know which characters are considered to be letters. Words containing letters not recognised by \TeXcount{} will tend to be split into two or more words, which can dramatically inflate the word count. The first step is to ensure that the file is read using the correct encoding: I generally suggest using the UTF-8 Unicode encoding, and from version 2.3. this is the default encoding used by \TeXcount{}, although other encodings may also be used. Unicode has good annotation of which characters are letters, and starting with version 2.3, \TeXcount{} uses Unicode internally to represent the text. While non-Latin letters like \code{\aa} and \code{\"a} should be recognised as letters, \TeX/\LaTeX codes using macros or special characters, such as \code{\bs{aa}} and \code{\bs{"}a}, are not immediately understood as letters. I have added patterns aimed at recognising these as well, but depending on the code you are writing, these patterns may either not be flexible enough to recognise all letter codes, or may be too flexible and recognise things it should not. I have added a relaxed mode (\code{-relaxed}) and a more restricted mode (\code{-restricted}) in which these patterns are more general or more constrained, but you should check how this performs on you actual texts by viewing the verbose output. + +% --------------------------------------------------------------------------- + \section{Syntax and options} + \subsection{Running \TeXcount{}} The command to run \TeXcount{} may vary slightly depending on the operating system and the local settings. You may also wish to rename it or define an alias. @@ -130,6 +143,7 @@ where the options may be amongst the following: If more than one file is given, \TeXcount{} will perform the count on each of them printing the results for each, then print the total sum at the end. Note that files are parsed one by one in order of appearance and counts made per file; only afterwards are the totals computed. + \subsection{File encoding} If your \TeX/\LaTeX{} document consists entirely of ASCII characters, there should be no problems with file encoding. However, if it contains non-ASCII characters, e.g. non-Latin letters such as \o{}, there are different ways in which these may be encoded in the files. @@ -149,6 +163,7 @@ For convenience, if no encoding is specified, \TeXcount{} will try to guess whic Note that if no encoding is specified and \TeXcount{} left to guess the appropriate encoding, all output will be UTF-8. Thus, letting \TeXcount{} guess the encoding may not be suitable when using \TeXcount{} in a pipe since the UTF-8 output may not be compatible with the encoding of the input. If multiple files are parsed, \TeXcount{} will guess the encoding separately for each file even if they are included (\code{-inc} or \code{-merge}) in a file with an identified encoding, and may thus end up selecting different encodings for different files. + \subsection{Language scripts, alphabets and character sets} In additional to the traditional Latin letters, A-Z, a number of letters are recognised by Unicode as part of the extension of the Latin letters. Some languages, however, use entirely different character sets. @@ -173,18 +188,32 @@ The option \code{-stat} has been added to produce overall word counts per script \codeline{-alphabets=digit,Latin,Greek,Cyrillic} will count words containing the numbers 0--9, Latin letters (including the extended Latin character set), Greek letters and Cyrillic letters. Words may contain any combination of these: \TeXcount{} does not require that a word consist of only one type of script. Also, note that if \code{digit} had not been included, numbers would not be allowed to be part of or counted as words. The output statistics will then give the number of words containing each of these script classes (or combination). + \subsection{Parsing details} -By selecting one of the \code{-v} options, you can choose how much detail is printed. This is useful for checking what \TeXcount{} counts. The option \code{-showstate} shows the internal state and is for debugging purposes only. +By selecting one of the \code{-v} options, you can choose how much detail is printed. This is useful for checking what \TeXcount{} counts. Alternatives \code{-v0} to \code{-v4} control the amount of detail, with \code{-v} equal to \code{-v3}. The option \code{-showstate} shows the internal state and is for debugging purposes only: \code{-v4} switches this on. The output is colour coded with counted text coloured blue, other colours for other contexts. The colour coding is made using ANSI colour codes. These should work when printed directly to Linux xterm window, but need not work if piped through \code{more} or \code{less}: with \code{less} you need to use the option \code{-r} for the colours to be shown correcly. -Under Windows or other operating systems, ANSI colour don't work and are turned off by default. Instead I recommend using HTML output which can be viewed in a browser. +Under Windows or other operating systems, regular ANSI colour don't work, but there is a fix in place which adapts it for Windows, although this may not function exactly as desired. + +In general, however, I recommend using HTML output which can be viewed in a browser: in particular if the text output does not produce suitable colour coding. To print the details encoded as HTML document, use the option \code{-html}. Alternatively, \code{-htmlcore} only outputs the HTML body. I suggest using the options \code{-html -v} to get full detail, save this to a HTML file, e.g. using \codeline{texcount.pl -html -v -sum \textit{files} > details.html} where \code{-sum} computes the total count of words and formulae (or \code{-sum=1,1,1} to only count words) and adds the cumulative count at the end of each line of the parsing details, and \code{-sub} is on by default which produces subcounts per section. +\subsubsection{Control of details in verbose output} + +The verbosity option, \code{-v=\parm{styles-list}} or \code{-v\alt{0-4}\parm{styles-list}}, can be used to select exactly which elements to include or exclude from the verbose output. The styles list consists of a list of styles or style categories with \code{+\parm{style}} or \code{-\parm{style}} used to indicate if they should be added or removed. If the first style in the list is one of the categories 0 to 4, the \code{=} can be dropped. The option \code{-help-style} returns an overview of the available styles and style categories, while \code{-help-style=\parm{style}} may be used to get a description of a particular style or style category. + +Each token in the verbose output has a defined style: e.g. \code{word}, \code{hword} (header word), \code{ignore}, \code{option}, etc. If the style is included in the styles list, it will be printed in the verbose output; if not included in the styles list, it will not be printed. Thus, by setting which styles are included in the styles list, you can specify in detail which tokens are included in the verbose output. The included styles correspond to the list of colour codes listed at the start of the output when \code{-codes} is set. + +The style categories, which include \code{0} to \code{4}, are groups of related styles: e.g. \code{Words}, \code{Macros}, \code{Options}, etc. Note that apart from \code{0} to \code{4}, the style categories have capital initials, while the styles themselves are all lower case. + +For example, if you only want to output words (including those in headers and other contexts), you can set the option \code{-v=Words}; using \code{-v=Words+math}, the equation contents will be included (but not the enclosing \code{\$\ldots\$}). + + \subsection{Summary information} By default, \TeXcount{} outputs counts of text words, header words, caption words, number of headers, number of floats/figures, number of inlined formulae, and number of displayed formulae, and lists each of these counts. To shorten this to a one-line format per file, specify \code{-brief}. @@ -201,28 +230,40 @@ By adding the option \code{-freq}, \TeXcount{} will output the word frequencies A frequency table for each script type (alphabetic, Han, etc. or script classes like Greek, Hebrew etc. if specified in \code{-alphabets=}) is produced by the option \code{-stat}. + \subsection{Parsing options} \TeXcount{} uses regular expressions to identify words and macro options. By default, these have been set so as to fit most common usages. However, some users may find the default to be too strict, e.g. not recognise options that are long and contain less common symbols. More permissive patterns may be selected by using the option \code{-relaxed}. This allows more general document elements to be identified as words or macro options, which may sometimes be desired, but may also have undesirable effects, so check the verbose output to verify that \TeXcount{} has counted the appropriate elements. Conversely, if the default settings tends to combine words that should be counted as separate words, you may try the option \code{-restricted}. -Macro options, i.e. \code{[\ldots]} after macros and macro parameters are ignored. Since \TeXcount{} has no specific knowledge of which macros take options, this is a general rule. In order to avoid that uses of [\ldots] that are not macro options are mistaken as such, \TeXcount{} makes some restrictions on what may be contained in such an option. By default, this restriction is relatively strict under the assumption that it is better to count a few macro options as words than risk large fragments of text to be ignored. However, if your document contains macro options with more complicated values (e.g. certain special characters or macros), using \code{-relaxed} may help handle these correctly. +Macro options, i.e. \code{[\ldots]} after macros and macro parameters are ignored. Since \TeXcount{} has no specific knowledge of which macros take options, this is a general rule applied to all macros that take parameters\footnote{For macros that take no parameters, \code{[\ldots]} is not interpreted as a macro option. While slightly inconsisten, this avoids e.g. \code{\{\bs{bf}[bold text]\}} to be gobbled up as a macro option and ignored}. In order to avoid that uses of [\ldots] that are not macro options are mistaken as such, \TeXcount{} makes some restrictions on what may be contained in such an option. By default, this restriction is relatively strict under the assumption that it is better to count a few macro options as words than risk large fragments of text to be ignored. However, if your document contains macro options with more complicated values (e.g. certain special characters or macros), using \code{-relaxed} may help handle these correctly. By default, \TeXcount{} does not allow special characters or macros to be part of words. This may cause problems if character modifiers or some special characters are used which are entered as macros. The \code{-relaxed} option makes the word recognition regular expression somewhat more general. + \subsection{File inclusion} By specifying \code{-inc} or \code{-merge}, \TeXcount{} will automatically count documents that are included using \code{\bs{input}} or \code{\bs{include}}. The difference between the two is that \code{-inc} analyses the included files separately, while \code{-merge} merges the included documents into the parent document. Thus, \code{-inc} will result in one count per file and a total sum at the end, while \code{-merge} will treat the merged document as if it was one file. The default option is \code{-noinc} indicating that included documents are not counted. -By default, \TeXcount{} assumes paths are relative to the present working directory. Alternatively, an explicit path may be given using \code{-dir=path}. Note that the path must end with the path delimiter symbol. If only \code{-dir} is used, the path of the main file (the one given to \TeXcount{} on the command line) will be used. +Paths can absolute or relative. Relative paths are by default relative to the working directory, although e.g. the \code{import} package can cause files to be included from other directories. The working directory is by default set to be the current directory: i.e. the directory from which \TeXcount{} is executed. This default behaviour corresponds to the option \code{-dir=.}. -Note that when included documents are analysed as separate files, i.e. using \code{-inc}, the text of included documents is not included where the \code{\bs{input}} or \code{\bs{include}} is located. This has two consequences. First, since word counts are produced per file, subcounts, e.g. by chapter, will only include the text in the same file, not that of the included file. Secondly, if TC-instructions to \TeXcount{} are embedded in the \LaTeX{} document, e.g. defining additional macro handling rules, these take effect in the order they are parsed by \TeXcount{}. Since included documents are parsed after the parent document, definitions in the parent document will be in effect for the included documents; definitions made in the included documents will only be in effect for subsequently included documents, not in the parent or previously included documents. +The working directory can be specified explicitly by the \code{-dir=\parm{path}} option. The file names on the command line should still be relative to the current directory, i.e. the one from which \TeXcount{} is executed, while files included within these will be relative to the specified working directory. +Alternatively, if \code{-dir} is used without setting a path, the working directory is determined by the directory containing the top level \LaTeX{} documents, i.e. the document specified on the command line; if several files are provided on the command line, these may result in different working directories. Note that \code{-dir} and \code{-dir=} are fundamentally different: the first indicates that the working directory is determined by the top level \LaTeX{} documents, while the second fixes the working directory to be the current directory. + +Note that when included documents are parsed as separate files, i.e. using \code{-inc}, the text of included documents is not included where the \code{\bs{input}} or \code{\bs{include}} is located. This has two consequences. First, since word counts are produced per file, subcounts, e.g. by chapter, will only include the text in the same file, not that of the included file. Secondly, if TC-instructions to \TeXcount{} are embedded in the \LaTeX{} document, e.g. defining additional macro handling rules, these take effect in the order they are parsed by \TeXcount{}. Since included documents are parsed after the parent document, definitions in the parent document will be in effect for the included documents; definitions made in the included documents will only be in effect for subsequently included documents, not in the parent or previously included documents. + +In addition to the \code{-dir} option for setting the working directory, there is a similar option \code{-auxdir} for setting the path to the auxilary directory where e.g. the bibliography \code{.bbl} file should be read from. The default setting is \code{-auxdir} which means that working directory is used. However, \code{-auxdir=\parm{path}} can be used to overrule this and set an alternative path. If \code{-dir=\parm{path}} is used, the auxilary path should be relative to the current directory, not to the working directory specified with \code{-dir=\parm{path}}; if \code{-dir} is used, as is the default, the working directory will be the directory containing the top level \LaTeX{} dodcuments (the ones specified on the command line), and the auxilary path will be relative to this, unless an absolute path is specified. + + + +% --------------------------------------------------------------------------- \section{Macro handling rules} -A few special macro handling rules are hard-coded into the \TeXcount{} script: i.e. the handling of those can only be changed by editing the script. However, \TeXcount{} primarily relies on a few general rules and macro and group handling rules that follow a specific pattern. +A few special macro handling rules are hard-coded into the \TeXcount{} script: i.e. the handling of those can only be changed by editing the script. However, \TeXcount{} primarily relies on a few general rules and macro and environment handling rules that follow a specific pattern. + \subsection{General macro handling rules} @@ -230,22 +271,23 @@ The general macro handling rules fall into a few general categories: \begin{description} -\item[macro]In its simplest form, this type of rule just tells how many parameters to ignore following the macro. More generally, one may specify the number of parameters a macro takes and how each of these should be handled. Options enclosed in \code{[]} before, between and after parameters are also ignored; this also applies to macros not specified here, so for a macro with no rule, immediately following \code{[]}-options will be ignored. (This type of rule was called an exclude rule in older versions of \TeXcount{}, the reason being that the rule originally only gave the number of +\item[Macro]In its simplest form, this type of rule just tells how many parameters to ignore following the macro. More generally, one may specify the number of parameters a macro takes and how each of these should be handled. Options enclosed in \code{[]} before, between and after parameters are also ignored; this also applies to macros not specified here, so for a macro with no rule, immediately following \code{[]}-options will be ignored. (This type of rule was called an exclude rule in older versions of \TeXcount{}, the reason being that the rule originally only gave the number of parameters to ignore following a given macro.) -\item[header]Some macros are specified to be counted as headers. This initially only indicates that the macro should cause the number of headers to be increased by one, but an additional rule is added to the macro-rule to count the following parameter as header text. - -\item[group]For groups enclosed by \code{\bs{begin}\{\textit{name}\}} and \code{\bs{end}\{\textit{name}\}}, there are rules specifying how the contents should be interpreted. A macro rule is added for \code{begin\textit{name}} (without the backslash!) which is \TeXcount{}'s internal representation of \code{\bs{begin}\{\textit{name}\}}. Note that special characters like \code{*} may be part of the group name, e.g. as in \code{equation*} and rules for these need be specified\footnote{Previously, trailing \code{*} was supposed to be ignored so the same rule would apply to group \code{equation*} as to \code{equation}. However, due to a bug in a regular expression, this did not work as intended and I have decided not to follow that strategy and instead speficy these rules specifically.}. +\item[Environment]For environments enclosed by \code{\bs{begin}\{\textit{name}\}} and \code{\bs{end}\{\textit{name}\}}, there are rules specifying how the contents should be interpreted. A macro rule is added for \code{begin\textit{name}} (without the backslash!) which is \TeXcount{}'s internal representation of \code{\bs{begin}\{\textit{name}\}}. Note that special characters like \code{*} may be part of the environment name, e.g. as in \code{equation*} and rules for these need be specified\footnote{Previously, trailing \code{*} was supposed to be ignored so the same rule would apply to environment \code{equation*} as to \code{equation}. However, due to a bug in a regular expression, this did not work as intended and I have decided not to follow that strategy and instead speficy these rules explicitly.}. \emph{Previously, environment rules were referred to as 'group rules', but I have now renamed this both in the \TeXcount{} script and documentation, and replaced \code{group} by \code{envir} wherever appropriate.} -\item[macroword]This type of rule indicates that the macro itself represents one or more words. Initially, \code{\bs{LaTeX}} and \code{\bs{TeX}} are defined with values 1 indicating that each represents one word. +\item[Macroword]This type of rule indicates that the macro itself represents one or more words. Initially, \code{\bs{LaTeX}} and \code{\bs{TeX}} are defined with values 1 indicating that each represents one word. -\item[preamble]A few macros should be counted even if they are in the preamble. In particular, \code{\bs{title}\{title text\}} is counted as a header assuming it will later be used to produce a title. +\item[Preamble]A few macros should be counted even if they are in the preamble. In particular, \code{\bs{title}\{title text\}} is counted as a header assuming it will later be used to produce a title. -\item[float inclusion]Within floats (begin-end groups defined with parsing status $-1$) there may be texts that should still be counted: in particular captions. These are specified with the float inclusion rule. +\item[Float inclusion]Within floats (environments with the \code{float} parsing rule) there may be texts that should still be counted: in particular captions. These are specified with the float inclusion rule. \end{description} -A macro parameter is normally something on the form \code{\{something\}}; more generally it may be anything \TeXcount{} parses as a single unit (or token), e.g. a macro, but since \TeXcount{} parses word by word rather than character by character this may not always be correct if parameters are not \code{\{\}}-enclosed or macros. +Previously, there was also a separate header handling rule, but this is now incorporated into the more general macro handling rules. + +A macro parameter is normally on the form \code{\{\textit{something}\}}; more generally it may be anything \TeXcount{} parses as a single unit (or token), e.g. a macro, but since \TeXcount{} parses word by word rather than character by character this may not always be correct if parameters are not \code{\{\}}-enclosed or macros. In addition, some macros take optional parameters which are usually on the form \code{[\textit{option}]}, and \TeXcount{} can also (from version 4) count these. + \subsection{Special macro handling rules} @@ -253,7 +295,7 @@ Some macros do not follow the pattern used by \TeXcount{} to represent macro han \begin{description} -\item[file include]If \code{-inc} is specified, included files will also be parsed and the total presented at the end. Initially, \code{\bs{input}} and \code{\bs{include}} trigger file inclusion, but more file inclusion macros may be added to the \code{\%TeXfileinclude} hash. In addition to potentially triggering file inclusion, the syntax differs in that the file name need not be enclosed in \code{\{\ldots\}}. +\item[file include]If \code{-inc} is specified, included files will also be parsed and the total presented at the end. Initially, \code{\bs{input}} and \code{\bs{include}} trigger file inclusion, but more file inclusion macros may be added to the \code{\%TeXfileinclude} hash. In addition to potentially triggering file inclusion, the syntax may differ in that \code{input} does not require the file name to be enclosed in \code{\{\ldots\}}. \item[package include]When packages are included using \code{\bs{usepackage}{name}}, \TeXcount{} will check for package specific macro handling rules to include. Initially, only \code{\bs{usepackage}} triggers package inclusion, but more macros may be added to the \code{\%TeXpackageinc} hash. @@ -265,7 +307,8 @@ Rules for identifying \code{\$\ldots\$}, \code{\$\$\ldots\$\$}, \code{\bs(\ldots The macros \code{\bs{def}} and \code{\bs{verb}} have hard-coded rules since these do not follow the pattern for macro handling rules, but may contain \LaTeX{} code which could seriously disrupt the parsing, e.g. by containing unclosed \code{\bs{begin}}. Macros like \code{\bs{newcommand}}, however, are handled by ordinary macro rules. -The macro \code{\bs{biblography}} is handled to check if the bibliography file should be parsed. The \code{thebibliography} group is also handled differently, one difference being that a bibliography header is added to the count. +The macro \code{\bs{biblography}} is handled to check if the bibliography file should be parsed. The \code{thebibliography} environment is also handled differently, one difference being that a bibliography header is added to the count. + \subsection{Package specific macro handling rules} @@ -275,37 +318,42 @@ Note that \TeXcount{} is still doing the analyses sequentially. It is therefore As of now, the package support is sparse since most macro handling rules have been included in the main set of rules. + \subsection{Bibliography handling} By default, the bibliography is not included in the word count. If the \code{-incbib} option is specified, however, bibliography parsing is turned on. If the bibliography is included from the \code{bbl} file using the \code{\bs{bibliography}} macro, this will be parsed as if included with the \code{-inc} option. If \code{-merge} is specified together with \code{-incbib}, the bibliography will be merged into the document. Note that bibliography parsing may be non-trivial and depend on the bibliography style used, so the verbose output should be checked: some styles perform considerable formatting which may confuse \TeXcount{}. In addition, initials, page numbers, etc. will all be counted as words, which may result in a word count which is higher than intendet. + \subsection{Adding or modifying macro handling rules} There are basically two different ways in which you can add additional macro handling rules, e.g. for your own macros, or modify existing rules: by modifying the \TeXcount{} script, or by adding the rules through \TeXcount{} instructions embedded in the \LaTeX{} code. -The simplest method is to use \TeXcount{} instructions which are embedded in your \LaTeX{} document as \LaTeX{} comments on the format \code{\%TC:\textit{instruction}}. This approach is described in some detail in section \ref{sec:TC_addrule}. +The simplest method is to use \TeXcount{} instructions which are embedded in your \LaTeX{} document as \LaTeX{} comments on the format \code{\%TC:\textit{instruction}}. This approach is described in some detail in section \ref{subsec:TC_addrule}. + +It is also possible to modify the \TeXcount{} code. The macro handling rules are mostly defined in the hash tables named \code{TeXmacro}, \code{TeXenvir}, etc., and editing these definitions is simple and does not required in-depth knowledge of Perl. A brief overview of the \TeXcount{} code is provided in section \ref{sec:code}. -It is also possible to modify the \TeXcount{} code. The macro handling rules are mostly defined in the hash tables named \code{TeXmacro}, \code{TeXgroup}, etc., and editing these definitions is simple and does not required in-depth knowledge of Perl. A brief overview of the \TeXcount{} code is provided in section \ref{sec:code}. \subsection{Cautions!} Since the rules are of a relatively general nature, macros that have a great deal of flexibility are hard to deal with. In particular this applies to macros with a variable number of parameters or where the handling of the parameters are not constant. -Also, \code{[]}-options following macros and macro parameters are always ignored, and \TeXcount{} gives no flexibility in over-ruling that. Since options are, by definition of the term, meant to be optional, extending \TeXcount{} to handle them would require extensive reprogramming as well as require much more detailed macro definition rules than what is now possible. +By default, \TeXcount{} assumes that macro options, i.e. parameters on the form \code{[\ldots]}, should not be counted. From version 4.0, \TeXcount{} allows rules for optional parameters, but in most cases where optional parameters have not been specified in the macro handling rules, they will simply be ignored. There is some risk of misinterpreting text as an option: e.g. \code{\bs{bf}[text]}. This is not likely to be a frequent problem. However, if something like \code{\bs{bf}[a lot of text]} gets ignored because it is considered an option, it can influence the word count substantially. I have therefore been somewhat restrictive with what (and how much) may go into an option. The default restriction on what may be allowed as an option may sometimes be too restrictive, causing \TeXcount{} to interpret options as text or macro parameters; you may use the command line option \code{-relaxed} to relax this restriction and allow more general options. -More critically, since \TeXcount{} does not really know which macros take options or not, just assumes that options should never be included, there is some risk of misinterpreting as an option something that is not: e.g. \code{\bs{bf}[text]}. This is not likely to be a frequent problem. However, if something like \code{\bs{bf}[a lot of text]} gets ignored because it is considered an option, it can influence the word count substantially. I have therefore been somewhat restrictive with what (and how much) may go into an option. The default restriction on what may be allowed as an option may sometimes be too restrictive, causing \TeXcount{} to interpret options as text or macro parameters; you may use the command line option \code{-relaxed} to relax this restriction and allow more general options. +More advanced macros are not supported and can potentially confuse \TeXcount{}. In partcular, if you define macros that contain unbalanced \code{\bs{begin}}--\code{\bs{end}}, this will cause problems as \TeXcount{} needs to keep track of these to know where environments start and end. -More advanced macros are not supported and can potentially confuse \TeXcount{}. In partcular, if you define macros that contain unbalanced begin--end groups, this will cause problems as \TeXcount{} needs to keep track of these to know where different groups start and end. +% --------------------------------------------------------------------------- + \section{Output from \TeXcount{}} \TeXcount{} will by default provide a summary of the word and element counts. This may, however, be modified either by specifying \code{-brief} which reduces it to a one line summaryper file, \code{-total} to suppress per file summaries, or by providing an alternative template. If there are parsing errors, \TeXcount{} will print warnings about these. You may turn off this by specifying \code{-quiet} (\code{-q} for short), but there will still be an added comment about the number of errors in the final statistics to warn you of any errors. + \subsection{Count statistics} The summary output will by default provide a summary of all counts: i.e. word counts for text, header and captions, and the number of floats/tables, headers, inlined and displayed formulae. You may combine these into a summary count by using the \code{-sum} option which by default gives the total number of words and formulae. You may choose briefer output formats by using the \code{-brief} option which produces a one-line summary of the counts. The option \code{-1} is the same as specifying \code{-brief -total} and will give only one line of output for the total only. Combining \code{-brief} with \code{-sum} will cause only the sum to be printed rather than the full set of counts. @@ -314,39 +362,205 @@ If multiple files are processed in one run, \TeXcount{} will by default provide In order to only write the total summary, use the option \code{-total}. If there is only one file processed, the result will be similar except that subcounts (counts per section etc.) are not provided with the total count. -\subsection{Customising the output} + +\subsection{Customising the summary output} You may specify an output template to use instead of the default output formats. This will replace the output per file or for the total with output produced using this template. The template is a string with codes for inserting the count values and titles. To specify it, use the option \code{-template="\textit{template}"}. The encapsulating \code{"\ldots"} are required if the template contains spaces. You may insert line shifts by using \code{\bs{n}}. -The count numbers may be included by the codes \code{\{0\}} to \code{\{7\}} for the different counts: 0=number of files, 1=text words, etc. Codes \code{\{SUM\}}, \code{\{ERROR\}} and \code{\{TITLE\}} may be used to insert the count as specified by the \code{-sum} option, the number of parsing errors, the title (e.g. section name) and a header (same as title unless \TeXcount{} has replaced it). For the labels \code{SUM}, \code{\{ERROR\}} and \code{TITLE}, short forms \code{S}, \code{E} and \code{T} may be used. +The counts may be included by using the counter keywords: \code{word}, \code{headerword}, etc. Other codes that may be inserted are: \code{\{SUM\}} to insert the count as specified by the \code{-sum} option, \code{\{TITLE\}} for the title (e.g. section name) and a header (same as title unless \TeXcount{} has replaced it), \code{\{ERROR\}} for the number of parsing errors, \code{\{WARNINGS\}} for the number of distinct warnings, or \code{\{NWARNINGS\}} for the total number of warnings. Some of these also have shortened forms\footnote{Previously, one-letter versions of some of these codes were permitted, but that is no longer the case.} like \code{\{ERR\}} and \code{\{WARN\}}. -Conditional inclusion may be performed using the format \code{\{\textit{label}?\ldots?\textit{label}\}} where \code{\textit{label}} is one of \code{0} to \code{7}, \code{SUM}, \code{ERROR} or \code{TITLE} (or their alternative forms). The enclosed text will then be included only if the corresponding value exists and is non-zero. If you wish to include an alternative text when the value is non-existant or zero, use the format \code{\{\textit{label}?\textit{if non-zero}|\textit{if zero}?\textit{label}\}}. +Conditional inclusion may be performed using the format \code{\{\textit{label}?\ldots?\textit{label}\}} where \code{\textit{label}} is one of the counter keywords, \code{SUM}, \code{ERROR} or \code{TITLE} (or their alternative forms). The enclosed text will then be included only if the corresponding value exists and is non-zero. If you wish to include an alternative text when the value is non-existant or zero, use the format \code{\{\textit{label}?\textit{if non-zero}|\textit{if zero}?\textit{label}\}}. Subcounts, e.g. per section, may be included by using \code{\{SUB|\textit{template}|SUB\}} with a separate template text specified for the subcounts. This will only be included if there is more than one subcount, and in order to conditionally include prefix and suffix you may use \code{\{SUB?\textit{prefix}|\textit{template}|\textit{suffix}?SUB\}}. Note that you have to insert line shifts yourself. \TeXcount{} will only insert one line shift after each file count, and not after the total count: if you process only one file and want only to output the total sum without a line shift at the end, use \code{-sum -total -template="\{SUM\}"}, which should give the same output as \code{-1 -sum} when there are no parsing errors. + +% --------------------------------------------------------------------------- + \section{\TeXcount{} instructions in the \LaTeX{} document} It is possible to give some instructions to \TeXcount{} from within the -\LaTeX{} document. The general format of these instructions is -\codeline{\%TC:\textit{instruction name parameters}} -where \textit{name} is used with macro handling instructions to specify the macro or group name for which the rule applies and \textit{parameters} specify the details of how the macro and its parameters should be interpreted. In addition, there are some \TeXcount{} instructions that do not follow this syntax: +\LaTeX{} document. These can be used to control the parsing of the document and add custom made macro and environment handling rules directly from the \LaTeX{} document. The general format of these instructions is +\codeline{\%TC:\textit{instruction \alt{parameters}}} +which \LaTeX{} will interpret as a comment but \TeXcount{} will detect. + +Adding your own macro handling rules is relatively simple. While it is fairly easy to edit the script to add more rules, this has the disadvantage that the modifications will be lost if updating to a new version of \TeXcount. A better and more flexible solution is to include instructions to \TeXcount{} in the \LaTeX{} documents, alternatively to make a definition file in which new macro handling rules are defined. The \TeXcount{} instructions for doing this take the form +\codeline{\%TC:\textit{instruction name parameters \alt{option}}} +where \code{name} is either the macro name (including backslash) or environment name: % +\input{sub_addrules} + +Note that macro handling rules are added successively throughout the session: i.e. if more files are parsed, handling rules from previously parsed files still apply. This has advantages as well as disadvantages. If you give a list of files with the rules specified in the first file, these rules will be applied to all the documents. However, if you use the \code{-inc} option, included files will be parsed only after \TeXcount{} has finished parsing the file in which they are included, so any rules specified in these will not apply to the initial document. + +A few additional \TeXcount{} instructions exist to control the overall parsing and counting: + \input{sub_tc_other} -The inital implementation of \TeXcount{} instructions required that the format was strictly adhered to: the comments should be a separate line and the instruction name was case sensitive. In particular the \code{ignore} instruction was problematic. However, in more recent version, at least from version 2.3, these \TeXcount{} instructions should are more robust. -\subsection{Ignoring segments of the file} +\subsection{Parameter and content handling rules}\label{subsec:TC_addrule} + +There are a set of alternative rules that may be used for parsing macro parameters and environment contents. These rules, or \emph{parser states}, are identified by keywords: +% +\begin{description} +\def\option#1[#2]#3{\item[#1:] (key: \code{#3} formerly \code{#2})} +\option{Text}[1]{text, word, wd, w} + Count as text (i.e. count words). +\option{Header text}[2]{headertext, headerword, hword, hwd, hw} + Count as header text. +\option{Other text}[3]{otherword, other, oword, owd, ow} + Count as float/caption text. +\option{Displaymath}[7]{displaymath, dsmath, dmath, ds} + Count as displayed math formulae. +\option{Inline math}[6]{inlinemath, inline, imath, eq} + Count as inlined math formulae. +\option{To header}[4]{header, heading, head} + Count header, then count text as \code{headertext} (transition state). +\option{To float}[5]{float, table, figure} + Count float, then parse contents as \code{isfloat} (transition state). +\option{Preamble}[-9]{} + Parse as preamble, i.e. ignore text but look for \code{preambleinclude} macros. +\option{Ignore}[0]{ignore} + Ignore text, i.e. do not count, but will still parse the code. +\option{Float}[-1]{isfloat} + Float contents, ignore text but look for \code{floatinclude} macros. +\option{Strong exclude}[-2]{xx} + Strong ignore which ignore environments, e.g. to use in macro definitions where + \code{\bs{begin}}--\code{\bs{end}} need not be balanced. +\option{Stronger exclude}[-3]{xxx} + Stronger ignore, handles macros as isolated tokens without handling their parameters, + to use with macro definitions like \code{\bs{newcommand}} and \code{\bs{def}}. +\option{Exclude all}[-4]{xall} + Ignore all, including unbalanced braces (e.g. used by \code{\%TC:ignore} and the \code{verbatim} environment). This rule may be used for environment contents, but not for macro or environment parameters or options since the exclusion causes \{ and [ to be ignored. +\end{description} + +The keys are used to identify the rule. Environment content rules are simply specified by giving the desired key. Parameter rules are on the form \code{[\textit{rule},\textit{rule},\ldots]} with one rule provided per parameter, where each rule is either one of the above keywords or \code{option:\textit{rule}} to indicate an optional parameter on the form \code{[\ldots]}; alternatively, a single integer can be provided (not enclosed in \code{[]} to indicate that the indicated number of parameters should be ignored. + +The list of parser states is in order of increasing priority: i.e. when a parser state is specified as a rule for parsing a parameter or environment content, this will only take effect if it has higher priority than the current state. Thus, text within an ignored or excluded region will not be counted. + +The formerly used numberic codes are listed at the end of the keyword list for each state. Before version 4, these numeric codes were used to specify parsing rules, but although these should still work, using the named keywords is highly recommended. + +The transitional states indicates an incrementation of one of the counters and then change to another state: e.g. if the \code{header} rule is specified, this will first cause the header counter to be incremented, and then a change to the \code{headerword} state in which word counts are added to the header word counter. + +\subsubsection{Adding a macro handling rule} + +The \TeXcount{} instruction for adding (or changing) a rule for how \TeXcount{} handles a specific macro takes the form +\codeline{\%TC:macro \textit{macro-name parameter-rules}} +where the macro name includes the backslash and the parameter rules can be an integer or a \code{[]}-enclosed list as explained above. + +If a list, \code{[\textit{rule},\textit{rule},\ldots]}, of parsing rules is provided, the macro will be assumed to take this number of parameters. Each rule is either a keyword signifying the rule, or parser state, with which the parameter will be parsed, or \code{option:\textit{key}} for optional \code{[]}-enclosed parameters. Additional \code{[]}-enclosed options, between or after the macro and the parameters, will be ignored. + +Macro handling rules specified for macros \code{\bs{\textit{name}}} automatically apply to \code{\bs{\textit{name}*}}: i.e. a \code{*} is automatically gobbled up as a macro modifier. + +Here are some examples together with corresponding macro definitions: + +\begin{lstlisting} +%TC:macro \refnote[text,othertext] +\newcommand\refnote[2]{\textit{#1}\footnote{#2}} + +%TC:macro \newsection [header,ignore] +\newcommand\newsection[2]{\section{#1}\label{sec:#2}} + +%TC:macro \NB 1 +\newcommand\NB[1]{\marginpar{#1}} +\end{lstlisting} + +The predefined rules can easily be read off the script file: they are hash maps defined at the beginning of the script with names \code{TeXmacro}, \code{TeXenvir}, etc. + +\subsubsection{Adding an environment handling rule} + +Rules for environments may be added on the format +\codeline{\%TC:envir \parm{name} \parm{parameter-rules} \parm{content-rule}} +for parsing \code{\bs{begin}\{name\}\ldots\bs{end}\{name\}}. The parameter rules are specified as for the \code{macro} rule and is used to process the parameters that follow \code{\bs{begin}\{name\}}. The content rule is a single parsing rule to use on the environment content. + +\begin{lstlisting} +%TC:envir theorem [] text +\newtheorem{theorem}{Theorem} +\end{lstlisting} + +\subsubsection{Adding rules that apply to the preamble and float contents} + +Within the preamble (from \code{\bs{documentclass}} to \code{\bs{begin}\{document\}}) and within floating objects (tables, figures, etc. parsed using the \code{float}/\code{isfloat} states), texts and macros are generally ignored. However, it is possible to specify particular macro handling rules that apply within these regions by using the \code{preambleinclude} and \code{floatinclude} \TeXcount{} instructions. These take the same format as the \code{macro} instruction: +\codeline{\%TC:preambleinclude \textit{macro-name parameter-rules}} +\codeline{\%TC:floatinclude \textit{macro-name parameter-rules}} +It is possible for the same macro to specify different rules for preamble, floats and general use, although for most uses these should be expected to be the same. + +Preamble inclusion is typically used for macros like \code{\bs{title}} that define text that should be counted although it may be placed in the preamble. Another use is that macros that may occur in the preamble, like \code{\bs{newcommand}} and may contained unbalanced \code{\bs{begin}}--\code{\bs{end}} pairs, require a stronger exclusion than the regular \code{ignore} rule even in the preamble to ensure \TeXcount{} is not confused by these. + +Float inclusion is used e.g. for captions, and the parsing rules should normally be to count texts using the \code{otherword} parsing rule. + + +\subsection{Count macro, either as words or in other counters} + +Some macros, e.g. \code{\bs{LaTeX}}, generate words and should be counted as words. Other macros can generate other elements, e.g. headers or figures. Rules for counting macros can be specified as +\codeline{\%TC:macroword \textit{macro} \textit{number}} +where the parameter is the number of words produced by the macro, or +\codeline{\%TC:macroword \textit{macro} [\textit{countername},\ldots]} +which causes each of the counters in the list to be incremented by one (or more if listed multiple times). + +The counters for counting the number of files, text words, etc. are stored in an array. In some cases, e.g. when \code{-sum=} is specified, the order of the counters in this array is used to specify the rule. However, in most cases the counters should be specified by keywords. The counters, their index number and keywords are: +% +\begin{description} +\def\option[#1]#2{\item[#1.] (keys: \code{#2})} +\option[0]{file} Number of files. +\option[1]{text, word, wd, w} Number of words in text. +\option[2]{headerword, hword, hwd, hw} Number of words in headers. +\option[3]{otherword, oword, owd, ow} Words outside text, e.g. in floats/tables/figures. +\option[4]{header, heading, head} Number of headers. +\option[5]{float, table, figure} Number of floating environments, e.g. tables and figures. +\option[6]{inlinemath, inline, imath, eq} Number of inlined mathematics formulae. +\option[7]{displaymath, dsmath, dmath, ds} Number of displayed equations. +\end{description} + +Examples of uses: + +\begin{lstlisting} +%TC:macroword \TeXcount 1 +\newcommand\TeXcount{{\TeX}count} + +%TC:macroword acknowledge [header,hword] +\newcommand\acknowledge{\section*{Acknowledgements}} +\end{lstlisting} + + +\subsection{Specifying file inclusion macros} + +In addition to \code{\bs{input}} and \code{\bs{include}}, which are the standard \LaTeX{} macros for file inclusion, there are packages such as \code{import} intended to enable organising files into subfolders. \TeXcount{}, from version~3, adds some support for macros that change the path from which files are included. If the user needs to add additional file inclusion macros, the format is +\codeline{\%TC:fileinclusion \parm{macro} \parm{file-parameters}} +where the file parameters are a comma separated list of keywords, each corresponding to a macro parameter. Available parameters are: +% +\begin{description} + +\item[\code{input}:] This is a special keyword to use with \code{\bs{input}}. The handling of the parameter values is as \code{file}, but the parameter itself is not required to be enclosed in \code{\{\}}. + +\item[\code{file}:] This parameter simply gives the name of or path to a file. If the file is not found, \TeXcount{} will append \code{.tex} and try again. + +\item[\code{texfile}:] This parameter gives the name of or path to a file, but \code{.tex} will be appended, and is the rule used by \code{\bs{include}}. + +\item[\code{dir}:] This parameter provides the path of a directory relative to the \code{\$workdir}, and adds this to the search path before including any files. This is used with the \code{\bs{import}} macro of the \code{import} package. + +\item[\code{subdir}:] This parameter provides the path of a directory relative to the current directory, and adds this to the search path before including any files. This is used with the \code{\bs{subimport}} macro of the \code{import} package. + +\item[\code{<bbl>}:] This is a special keyword to use with \code{\bs{bibliography}} to specify inclusion of the bibliography file. It is different from the other keywords in that it does not take a macro parameter. + +\end{description} + +Examples showing how some existing macros, from basic \LaTeX{} and from the \code{import} package, are defined: + +\begin{lstlisting} +%TC:fileinclude \input input +\input macros.tex +%TC:fileinclude \include texfile +\include{intro} +%TC:fileinclude \import dir,file +\import{supplements/}{overview.tex} +%TC:fileinclude \subimport subdir,file +\subimport{tables/}{data.tex} +\end{lstlisting} -The TC-instruction \code{ignore}, later canceled by \code{endignore}, may be used to turn of all counting in a segment of the \LaTeX{} file. The ignored segment should thus be started by -\codeline{\%TC:ignore} -and ended by -\codeline{\%TC:endignore} -causing all text inbetween to be ignored.\footnote{In older versions, \TeXcount{} would still parse this text and might thus be affected by unbalanced brackets. As of version 2.3, however, this should be fixed to make the ignore instruction more robust.} \subsection{Adding subcount break points} @@ -358,65 +572,59 @@ If you define new section macros or macros you wish to cause a break point, thes \codeline{\%TC:breakmacro \textit{macro} \textit{label}} This defines the given macro to cause a break point, and uses the given label to indicate the type of break (e.g. Section, Chapter, etc.). -\subsection{Adding macro handling rules}\label{sec:TC_addrule} -Adding your own macro handling rules is relatively simple. Although editing the script is possible, and not too difficult, this has the disadvantage that the modifications will be lost if updating to a new version of \TeXcount. A better and more flexible solution is to include instructions to \TeXcount{} in the \LaTeX{} documents, alternatively to make a definition file in which new macro handling rules are defined. +\subsection{Ignoring segments of the file} -Comment lines on the form -\codeline{\%TC:\textit{instruction name parameters}} -encountered in the parsed document are used to add macro handling rules. The instruction states what kind of rule, the name specifies the macro or begin-end group, and parameters specify the rule. Be aware that these are not syntax checked and may produce either Perl errors or incorrect results if entered incorrectly. +The TC-instruction \code{ignore}, later canceled by \code{endignore}, may be used to turn of all counting in a segment of the \LaTeX{} file. The ignored segment should thus be started by +\codeline{\%TC:ignore} +and ended by +\codeline{\%TC:endignore} +causing all text inbetween to be ignored.\footnote{In older versions, \TeXcount{} would still parse this text and might thus be affected by unbalanced braces. As of version 2.3, however, this should be fixed to make the ignore instruction more robust.} -Macro names should be entered with their full name starting with backslash. Internally, begin-end groups are represented using macro names \code{begin\textit{name}} without backslash, but rules for begin-end groups are specified through a separate TC-instruction. -Note that macro handling rules are added successively throughout the session: i.e. if more files are parsed, handling rules from previously parsed files still apply. This has advantages as well as disadvantages. If you give a list of files with the rules specified in the first file, these rules will be applied to all the documents. However, if you use the \code{-inc} option, included files will be parsed only after \TeXcount{} has finished parsing the file in which they are included, so any rules specified in these will not apply to the initial document. +\subsection{Bibliography inclusion} -The instructions may be one of the following: -% -\input{sub_addrules} +In order to include the bibliography in the word counts, you can either specify \code{-incbib} on the command line, or use \TeXcount{} instruction +\codeline{\%TC:incbib} +which has the same effect: it specifies handling rules for the \code{\bs{bibliography}} macro and \code{thebibliography} environment that causes the bibliography to be included in the count, and if necessary the \code{.bbl} file to be included (without requiring \code{-inc} or \code{-merge}). -The parser status is used to dictate how each parameter should be parsed. E.g. if a macro has its parameter set defined by \code{[1,0,1]}, it means the first and third parameters are counted as text words (parser status $1$) whereas the second is ignored (parser status $0$). Another case is \code{\bs{renewcommand}} which is defined as \code{[-3,-2]}: the first parameter is to be read without interpreting the contents (which is going to be a macro name whose macro handling rules should not be applied here), and the second parameter should be ignored without requiring that begin--end groups be balanced. The different parsing states are: -% -\begin{description} -\def\optionhead#1{\item[\textbf{#1}]} -\def\option #1:{\item[{\makebox[7mm][r]{#1:}}]} -\optionhead{States for ignoring text} -\option 0: ignore text, i.e. do not count, but will still parse the code; -\option -1: float, ignore text but look for \code{floatinclude} macros; -\option -2: stronger ignore which ignore begin--end groups, e.g. to use in macro definitions where begin--end groups need not be balanced; -\option -3: even stronger ignore, handles macros as isolated tokens without handling their parameters, to use with macro definitions like \code{\bs{newcommand}} and \code{\bs{def}}; -\option -4: ignore all, including unbalanced brackets (used by \code{\%TC:ignore} and the \code{verbatim} environment); -\option -9: preamble, ignore text but look for \code{preambleinclude} macros. -\optionhead{States for counting words} -\option 1: count as text (i.e. count words); -\option 2: count as header text; -\option 3: count as float/caption text; -\optionhead{Transitional states (internal/limited use)} -\option 6: count as inlined math formulae; -\option 7: count as displayed math formulae; -\end{description} -The two transitional states may be used to speficy rules for begin--end group contents, but not for macro parameters. They are transitional in the sense that the contents is parsed using the 0 state which ignores the contents. +\subsection{Text substitution prior to parsing} -Here are some examples together with corresponding macro definitions: +There are cases where a macro needs to be substituted with a text prior to parsing. One such case is when a macro contains a file path which is later used by a file inclusion macro. Since \TeXcount{} does not actually expand the macros, it will not be able to generate the file path from the macro. Instead, one may perform an explicit substitution +\codeline{\%TC:subst \parm{macro} \parm{text}} +which will then cause all occurrences of the macro to be substituted by the provided text prior to parsing. Note that this substitution will therefore also be found in the verbose output. -%TC:Ignore \begin{lstlisting} -%TC:macroword \TeXcount 1 -\newcommand\TeXcount{{\TeX}count} +\newcommand\chappath{chapters} +%TC:subst \chappath chapters +\input \chappath/chapter1 +\end{lstlisting} -%TC:macro \NB 1 -\newcommand\NB[1]{\marginpar{#1}} +Note that the substitution is placed \emph{after} the \code{\bs{newcommand}} definition. Otherwise, the substitution would have taken effect, changing that line to \code{\bs{newcommand} chapters/chapters}. -%TC:header \newsection [2,0] -\newcommand\newsection[2]{\section{#1}\label{sec:#2}} -%TC:group theorem 0 1 -\newtheorem{theorem}{Theorem} + +\subsection{Adding a new counter} + +Initially, \TeXcount{} has eight different counters: file, text words, header words, other words, number of headers, number of floating objects, number of inlined formulae, and number of displayed formulae. However, it is possible to add more counters, e.g. to count footnotes separately. The syntax is +\codeline{\%TC:newcounter \parm{name} \opt{description}} +where the given name is used as keyword to refer to this counter. If no description is provided, the name will be used as description. A new counter is then added, and a parsing rule (parser state) with the same name is added which may be used in specifying macro and environment handling rules. + +The following example shows how two different counters are added: one to count the number of footnotes, and another to count the words in footnote. + +\begin{lstlisting} +%TC:newcounter fwords Words in footnotes +%TC:newcounter footnote Number of footnotes +%TC:macro \footnote [fwords] +%TC:macroword \footnote [footnote] +Each footnote\footnote{Words in footnotes will be counted separately.} will be counted. \end{lstlisting} -%TC:EndIgnore -The predefined rules can easily be read off the script file: they are hash maps defined at the beginning of the script with names \code{TeXmacro}, \code{TeXheader}, etc. +Note that we have to specify one rule for counting the words in footnotes, and another rule for counting the footnotes. Unlike headers and floating bodies, there are no transition states available that can do both. + +% --------------------------------------------------------------------------- \section{Using an option file} @@ -430,7 +638,6 @@ Blank lines and lines starting with \code{\#} are ignored and may thus be used t Here is an example which sets the total sum to be the number of words (not including formulae), subcounts by section, parses included files, and adds an output template. -%TC:Ignore \begin{lstlisting}[frame=single] ### Options to use with TeXcount @@ -441,7 +648,7 @@ Here is an example which sets the total sum to be the number of words (not inclu # Macro rules %macro \url 1 -%group sourcecode 0 0 +%envir sourcecode 0 0 %macroword \TeXcount 1 # Path used in file inclusion (\chapterpath filename) @@ -455,10 +662,11 @@ Here is an example which sets the total sum to be the number of words (not inclu \{5?Number of floats: {5}\n?5} \{SUB? - {sum} words in {title}\n?SUB} \end{lstlisting} -%TC:EndIgnore +% --------------------------------------------------------------------------- + \section{Customising \TeXcount{}} \TeXcount{} is a self-contained Perl script: no external packages or resources required except that you need to have Perl installed to run it. Unfortunately, as with much of Perl code since Perl does not itself encourage structured programing, after expanding somewhat in size, it is not the most readable of codes. However, there may still be cases where you might yourself want to modify the code. @@ -469,68 +677,43 @@ There are some things that may be modified quite easily even without knowing Per \item[Preset startup options]On one of the first lines of the code, the list \code{@StartupOptions} is defined. A list is simply a sequence of values (an array) on the form \code{(\textit{value},\textit{value},\ldots)}. As it stands, this list is empty, but you may add startup options to be included prior to command line options when you run \TeXcount{}. E.g. if you change this to \code{("-inc")} it will automatically add the \code{-inc} option so you don't have to do that yourself every time you run \TeXcount{}. -\item[Adding macro handling rules]While you may add macro handling rules using \code{\%TC:} commands either in the document or in a separate option file, this is inconvenient for large numbers of macros or if you want these rules always to be included. Also, you might want to add such rules for specific packages. In either case, it might be practical to add these directly to the \TeXcount{} code. \TeXcount{} stores the rules in hashes (maps from a key to a value) named \code{\%TeXmacro}, \code{\%TeXheader}, etc. There is more documentation on each of these in the code itself, and you may also inspect how rules have been defined for other macros and groups. +\item[Adding macro handling rules]While you may add macro handling rules using \code{\%TC:} commands either in the document or in a separate option file, this is inconvenient for large numbers of macros or if you want these rules always to be included. Also, you might want to add such rules for specific packages. In either case, it might be practical to add these directly to the \TeXcount{} code. \TeXcount{} stores the rules in hashes (maps from a key to a value) named \code{\%TeXmacro}, \code{\%TeXenvir}, etc. There is more documentation on each of these in the code itself, and you may also inspect how rules have been defined for other macros and environments. -\item[Output style]The ANSI colour codes for different levels of verbosity are encoded in the \code{\%STYLES} hashes and may be changed. The HTML style is encoded in the method \code{html\_head()} and is easily modified. +\item[Output style]The ANSI colour codes for different levels of verbosity are encoded in the \code{\%STYLES} hashes and may be changed. The HTML style is encoded in the method \code{html_head()} and is easily modified. \item[Character and word definitions]\TeXcount{} identifies words as those that match one of a given set of regular expressions (defined in \code{@WordPatterns}). Note that \code{@WordPatterns} is changed by options \code{-chinese}, \code{-japanese} and \code{-letters}. The pattern that is used within the word patterns to recognise letters is stored in \code{\$LetterPattern}. This is replaced if the \code{-relaxed} or \code{-restricted} option is set. Changing these definitions may be useful if you have special characters or wish to define words differently. \end{description} +% --------------------------------------------------------------------------- + \section{Modifying the \TeXcount{} script}\label{sec:code} -\TeXcount{} is written in Perl, and although hardly the best structured and documented code ever seen, I have tried to structure and document it somewhat. In particular, some parts of the code have been written with modifications in mind. +\TeXcount{} is written in Perl, and although hardly the best structured and documented code ever seen, I have tried to structure and document it somewhat. In particular, some parts of the code should be easily modifiable even without in-depth knowledge of Perl or the \TeXcount{} script: e.g. the macro handling rules. -Here's a quick walk-through of the code structure and comments on how easily the code may be modified. Some parts of the code are marked as \emph{CMD specific}. There are two version of the script: the CMD version intended for command line use, and the CGI version used with the web interface. The one you have is the CMD version. +For more aid on how the \TeXcount{} script is coded and organised, please consult the Technical Documentation. However, here is a very brief overview: \begin{description} -\item[\em INITIAL SETUP:] \textit{These set up global variables prior to execution.} - -\item[Settings and setup variables:] The start of the script sets of initial settings and variables. Many of these may be modified by command line options, but if you want to change the default behaviour these may be changed. However, note that there is a list \code{@StartupOptions} intended for this: initially, it is empty, but this is probably the simplest place the change startup options. - -\item[Internal states:] As of version 2.3, internal state identifiers (which are numerical codes) have been defined as \code{STATUS}, \code{TOKEN} and \code{CNT} variables, and these are also defined here. A few subroutines for interpreting these states have been included here, although most subroutines are defined after the main code, since they are intimately tied to the state's numerical values. None of these are intended to be modified. - -\item[Styles:] The style definitions basically define which elements to print for each of the verbosity levels. These map element names to ANSI colour codes. When used with HTML, the element names are used as tag classes. If you wish to change the ANSI colour scheme, or change which elements are written in each verbosity option, these may be changed. - -\item[Word pattern definitions:] This section contains regular expression patterns for identifying words and macro options. In addition, the additional character classes defined by \TeXcount{} are defined here. If you have special needs or wishes, modifying these definitions may be an option. - -\item[\TeXcount{} parsing rules:] This is the section in which the main rules for interpreting the \LaTeX{} code is specified: the exception is a few hard-coded rules that do not follow these general patterns. These are hashes that map the macro or group name to the macro handling rules. First, the default rules are defined, then packages specific rules are defined. - -\item[\em MAIN:] \textit{This is the top-level code which gets executed. All else is done through calls to subroutines.} - -\item[Main \TeXcount{} code:] This is the main code that is run. It is very simple: just a call to the method \code{MAIN} passing the command line options. - -\item[\em SUBROUTINES:] \textit{The subroutines are organised into blocks. Subroutines names use capital letters or initials if they are main routines (like public in other languages) to be used at the top-level, lower case if they may be used throughout but are considered to be lower-level subroutines, prefixed by one or two underscores (\_) if used only within the block.} +\item[Header and imports:] The shebang (\code{\#!}) and package imports (\code{use \parm{package}}). -\item[Main routines:] The \code{MAIN} routine gives the general processing flow. This in turn calls routines to parse to command line options, process/apply the options, parse the \TeX/\LaTeX{} files, and finally summarise the final results. The main routines are CMD specific. +\item[Global variables (and some methods related to these):] This defines and initialised global variables related to option settings, state variables used in parsing and counting (including functions for unterpreting these), variables and hashes for storing macro handling rules, and character class definitions (must be defined before use). -\item[CMD specific subroutines:] These are subroutine versions that are CMD specific, e.g. file inclusion and ANSI colours. Their location is somewhat illogical: logically, they might belong later together with related subroutines, but have been placed this early because they are specific to the CMD (or CGI) version. +\item[Main program:] This simply contains a call to the \code{MAIN} routine with the command line arguments. -\item[Option handling:] After parsing the options, the option values are processed using these subroutines. Some of the option handling operations call on global variables, whereas some are more hard-coded. Like the global variables, if you have special wishes or needs, there may be parts here that can be modified quite easily to change default settings or effects of specific options. +\item[Routines/functions/procedures:] The first procedure defined is \code{MAIN} which contains the program flow, then follows other subroutines. Routines with capitalised initial letters indicate high-level routines, while routines starting with underscores (\code{_}) are low-level routines. -\item[\TeX{} object:] The main role of the \code{TeX} object (which is technically not an object in the ordinary sense but just a hash) is to be a container object which links to the \TeX/\LaTeX{} code, the word count object, etc. The \code{TeX} object pertaining to any parsed \TeX/\LaTeX{} file is passed along from subroutine to subroutine, usually called \code{\$tex}. The \code{Main} object produced by \code{getMain} is a simple substitute for the \code{TeX} object for use when none is available, e.g. to catch errors not specific to any particular \code{TeX} object. - -\item[File reading routines:] These are used to read files and STDIN. - -\item[Parsing routines:] These contain the main routines for parsing the \TeX/\LaTeX{} code. The main worker method is the \code{\_{}parse\_{}unit} which parses a block of code: the \emph{unit}. A unit of code may be a begin-end group (environment), a \{\} separated group, a macro option or parameter, etc. The parsing of one unit is determined by the parsing status, which is passed to the parsing method, and the end marker which indicates which token marks the end of the unit. Different subroutines are then used to process the different types of code: macros, begin-end groups, TC instructions, etc. Amongst these routines are also routines for converting the parsed code into tokens, which is done one token at the time which is then removed from the start of the code. - -\item[Count object and routines:] The count object contains the counters as an array, plus titles and labels; in addition it can contain a list of subcounts which are themselves count objects. The count object is used for each file, but also to summarise multiple files, and region counts within files (e.g. per section). The \code{TeX} object contains an active count object to which newly counted words, equations, etc. get added. However, each \code{TeX} object also has a summary count object which will contain the final sum. - -\item[Output routines:] First, there are some routines for general output, i.e. independent of specific \code{TeX} objects. There are then some routines for formatting output, e.g. for the verbose output. There are also routines for printing count summaries in various formats. A special set of routines exist for printing the verbose output itself, and some of these are also involved in the parsing. - -\item[Help functions:] These routines are used to print help. - -\item[HTML functions:] These are routines for producing HTML output. In particular, the HTML style is defined here and may be easily modified. - -\item[Text data:] Some texts are not hard-coded into the script, but added as text data at the end. There are some routines defined to handle the text data, and then the text data itself. +\item[Text data:] At the end of the file is a \code{__DATA__} region containing text data used by the help routines. \end{description} Perl will first process the setup section which defines global variables, arrays and hashes. It then executes the main section (consisting of the call to \code{MAIN}), whereafter it exits. The subroutines and text data follow after the \code{exit}. + +% --------------------------------------------------------------------------- + \section{License} The \TeXcount{} package---script and accompanying documents---is distributed diff --git a/Master/texmf-dist/doc/support/texcount/TechDoc.pdf b/Master/texmf-dist/doc/support/texcount/TechDoc.pdf Binary files differnew file mode 100644 index 00000000000..02aa6690798 --- /dev/null +++ b/Master/texmf-dist/doc/support/texcount/TechDoc.pdf diff --git a/Master/texmf-dist/doc/support/texcount/TechDoc.tex b/Master/texmf-dist/doc/support/texcount/TechDoc.tex new file mode 100644 index 00000000000..af38a49d6f5 --- /dev/null +++ b/Master/texmf-dist/doc/support/texcount/TechDoc.tex @@ -0,0 +1,620 @@ +\documentclass{article} +\usepackage[T1]{fontenc} +\usepackage{a4wide} +\usepackage{times} +\usepackage{listings} +\usepackage{url} +\usepackage{color} + +\include{macros} +\newcommand\Obj[1]{\textsl{#1}} +\newcommand\wild{\ldots} +\newcommand\eqsim{$=\sim$} + +% CGI exclusion macro +%\def\CGI#1\CGIend{} % To exclude CGI text +\def\CGI#1\CGIend{#1} % To include CGI text +\def\CGIend{} + +%%% Title +\title{% +\LARGE \TeXcount\\ +\Large Technical documentation\\ +\Large Version \version\copyrightfootnote +} +\author{Einar Andreas R{\o}dland} + +\sloppy + +\begin{document} + +\maketitle + +{\abstract% +The aim of this document is to explain the implementation details of \TeXcount{} with the aim of aiding anyone who wishes to modify the Perl code (including the author). To be of practical use, it will require some knowledge of Perl and familiarity with \TeXcount{}: while full fledged development of \TeXcount{} requires a working knowledge of Perl, code modification may often be done with only limited experience with Perl. +} + +{\scriptsize\tableofcontents} + +\pagebreak + + + +\section{Introduction} + + +\subsection{\TeXcount{} versioning} + +The version number is on the form \code{\textit{major}.\textit{version}.\textit{subversion}.\textit{build}}. Main releases contain only the first to terms, implying that subversion and build number are both zero. Minor releases only contain the first three terms. Main as well as minor releases should be functional, tested versions. The subversion number can also be \code{alpha} ($\alpha=-2$) or \code{beta} ($\beta=-1$) for which the testing has been limited. The build number is used to keep track of changes and versions during development: they may be made available, but are purely for testing. + + +\subsection{Some things you need to know about Perl} + +\TeXcount{} is written in Perl. The entire script, including macro rules and help texts, is contained in one file. This makes the file somewhat big, and modularisation of the code is therefore not strictly enforced. I have however tried to structure the code somewhat. + +Perl has a few build-in data structures which are referenced in somewhat different manners. In this document, it will be important to recognize the difference between three different types of data: regular variables, arrays and hash maps. + +\begin{description} + +\item[\code{\$\textit{name}=\textit{value}}:] The \code{\$} at the start indicates that it is a Perl variable. The value can be numbers or strings, or it can be a reference which points to another data object (e.g. array or hash map). + +\item[\code{@\textit{name}=(\textit{value},\ldots)}:] The \code{@} at the start indicates that this is an array. The positions are indexed from 0 to $\textrm{length}-1$. + +\item[\code{\%\textit{name}=(\textit{key}=>\textit{value},\ldots)}:] The \code{\%} at the start indicates that this is a hash map that maps keys to values. The key will usually be a string, but can also be a number. + +\item[\code{sub \textit{name}} \{\ldots\}:] This defines a subroutine: function or procedure. Normally, these can be defined anywhere in the script, and I have generally placed them after the main program. + +\end{description} + +Note that \code{(\textit{value},\ldots)} is a list of values, not an array or a hash: it simply produces a list of values that are used to fill the defined array or hash. Arrays and hashes can also be produced directly by \code{[\textit{value},\ldots]} or \code{\{\textit{key}=>\textit{value},\ldots\}} respectively. Both these return a reference to the array/hash rather than the array/hash itself. + +In much of the code, hashes are passed by reference: e.g. if \code{\%hash} is a hash, \code{\$href=\bs{\%hash}} stores a reference to the hash, where the leading \code{\bs{}} causes a reference to be returned rather than the hash itself. Retreiving a value from the hash is done by \code{\$hash\{\textit{key}\}} or \code{\$href->\{\textit{key}\}} if the hash is accessed by reference. Note that individual values in array and hashes are prefixed by \code{\$}: i.e. \code{\$\textit{array}[\textit{index}]} and \code{\$\textit{hash}\{\textit{key}\}}. + +\TeXcount{} makes extensive use of regular expressions (regex): expressions on the form \code{\$\textit{string}\eqsim/\textit{pattern}/} and \code{\$\textit{string}\eqsim s/\textit{pattern}/\textit{replace}/}. In \TeXcount{}, the main use is to recognize (and remove) tokens (words, macros, spaces, etc.) at the start of a string of \LaTeX{} code. Some of these may be fairly simple to understand, while others may be more complex. + + + +\section{Overview} + + +\subsection{Code structure} + +\TeXcount{} is written in Perl, and although hardly the best structured and documented code ever seen, I have tried to structure and document it somewhat. In particular, some parts of the code have been written with modifications in mind so that users can make their own changes without in-depth knowledge of Perl or the \TeXcount{} script. + +Here's a quick walk-through of the code structure and comments on how easily the code may be modified. Some parts of the code are marked as \emph{CMD specific}. There are two version of the script: the CMD version intended for command line use, and the CGI version used with the web interface. The one you have is the CMD version. + +\begin{description} + +\item[\em HEADER AND IMPORTS:] \textit{The shebang (\code{\#!}) and \code{use} imports.} + +\item[\em INITIAL SETUP:] \textit{These set up global variables prior to execution.} + +\item[Settings and setup variables:] The start of the script sets of initial settings and variables. Many of these may be modified by command line options, but if you want to change the default behaviour these may be changed. However, note that there is a list \code{@StartupOptions} intended for this: initially, it is empty, but this is probably the simplest place the change startup options. + +\item[Internal states:] As of version 2.3, internal state identifiers (which are numerical codes) have been defined as \code{STATE}, \code{TOKEN} and \code{CNT} variables, and these are also defined here. A few subroutines for interpreting these states have been included here, although most subroutines are defined after the main code, since they are intimately tied to the state's numerical values. None of these are intended to be modified. + +\item[Styles:] The style definitions basically define which elements to print for each of the verbosity levels. These map element names to ANSI colour codes. When used with HTML, the element names are used as tag classes. If you wish to change the ANSI colour scheme, or change which elements are written in each verbosity option, these may be changed. + +\item[Word pattern definitions:] This section contains regular expression patterns for identifying words and macro options. In addition, the additional character classes defined by \TeXcount{} are defined here. If you have special needs or wishes, modifying these definitions may be an option. + +\item[\TeXcount{} parsing rules:] This is the section in which the main rules for interpreting the \LaTeX{} code is specified: the exception is a few hard-coded rules that do not follow these general patterns. These are hashes that map the macro or environment name to the macro handling rules. First, the default rules are defined, then packages specific rules are defined. + +\item[\em MAIN:] \textit{This is the top-level code which gets executed. All else is done through calls to subroutines.} + +\item[Main \TeXcount{} code:] This is the main code that is run. It is very simple: just a call to the method \code{MAIN} passing the command line options. + +\item[\em SUBROUTINES:] \textit{The subroutines are organised into blocks. Subroutines names use capital letters or initials if they are main routines (like public in other languages) to be used at the top-level, lower case if they may be used throughout but are considered to be lower-level subroutines, prefixed by one or two underscores (_) if used only within the block.} + +\item[Main routines:] The \code{MAIN} routine gives the general processing flow. This in turn calls routines to parse to command line options, process/apply the options, parse the \TeX/\LaTeX{} files, and finally summarise the final results. The main routines are CMD specific. + +\item[CMD specific subroutines:] These are subroutine versions that are CMD specific, e.g. file inclusion and ANSI colours. Their location is somewhat illogical: logically, they might belong later together with related subroutines, but have been placed this early because they are specific to the CMD (or CGI) version. + +\item[Option handling:] After parsing the options, the option values are processed using these subroutines. Some of the option handling operations call on global variables, whereas some are more hard-coded. Like the global variables, if you have special wishes or needs, there may be parts here that can be modified quite easily to change default settings or effects of specific options. + +\item[\TeX{} object:] The main role of the \code{TeX} object (which is technically not an object in the ordinary sense but just a hash) is to be a container object which links to the \TeX/\LaTeX{} code, the word count object, etc. The \code{TeX} object pertaining to any parsed \TeX/\LaTeX{} file is passed along from subroutine to subroutine, usually called \code{\$tex}. The \code{Main} object produced by \code{getMain} is a simple substitute for the \code{TeX} object for use when none is available, e.g. to catch errors not specific to any particular \code{TeX} object. + +\item[File reading routines:] These are used to read files and STDIN. + +\item[Parsing routines:] These contain the main routines for parsing the \TeX/\LaTeX{} code. The main worker method is the \code{_{}parse_{}unit} which parses a block of code: the \emph{unit}. A unit of code may be the contents of an environment, a \code{\{\ldots\}} group, a macro option or parameter, etc. The parsing of one unit is determined by the parsing state, which is passed to the parsing method, and the end marker which indicates which token marks the end of the unit. Different subroutines are then used to process the different types of code: macros, environments, TC instructions, etc. Amongst these routines are also routines for converting the parsed code into tokens, which is done one token at the time which is then removed from the start of the code. + +\item[Count object and routines:] The count object contains the counters as an array, plus titles and labels; in addition it can contain a list of subcounts which are themselves count objects. The count object is used for each file, but also to summarise multiple files, and region counts within files (e.g. per section). The \code{TeX} object contains an active count object to which newly counted words, equations, etc. get added. However, each \code{TeX} object also has a summary count object which will contain the final sum. + +\item[Output routines:] First, there are some routines for general output, i.e. independent of specific \code{TeX} objects. There are then some routines for formatting output, e.g. for the verbose output. There are also routines for printing count summaries in various formats. A special set of routines exist for printing the verbose output itself, and some of these are also involved in the parsing. + +\item[Help functions:] These routines are used to print help. + +\item[HTML functions:] These are routines for producing HTML output. In particular, the HTML style is defined here and may be easily modified. + +\item[Text data:] Some texts are not hard-coded into the script, but added as text data at the end. There are some routines defined to handle the text data, and then the text data itself. + +\end{description} + +Perl will first process the setup section which defines global variables, arrays and hashes. It then executes the main section (consisting of the call to \code{MAIN}), whereafter it exits. The subroutines and text data follow after the \code{exit}. + +\CGI +There is a separate CGI version of the \TeXcount{} script for the web service. While this is mostly the same as the regular command line version, there are some differences in how options are set and \LaTeX{} documents are read. Occasional differences in the CGI version will be commented on, but the main emphasis will be on the command line version of \TeXcount{}. +\CGIend + + +\subsection{Global variables} + +A number of globally defined variables, including constants, arrays and hashes, are defined at the start of the program. These fall into a few different categories. + +There are a number of variables defined for storing options and settings, many of which can be modified by command line options. In addition, there are few variables for global summaries and statistics, as well as a few for internal states during parsing. + +Global constants are defined to represent different states and counters. One set of constants,\code{\$CNT_\wild}, specify the position of the different counters in the counting array; parser states are defined as \code{\$STATE_\wild}; token types are named \code{\$TOKEN_\wild}. For example, the parsing state \code{\$STATE_TEXT} indicates that a block of \LaTeX{} code should be parsed and have words counted as text words. The constants simply take numerical values, but help make the code more readable. Together with some of these are defined functions for interpreting or transforming these + +Alternative settings for different options are defined in a number of hashes, e.g. \code{\%STYLES} indicating which tokens to print at different levels of verbosity, and \code{\%NamedLetterPattern} which stores alternative regex rules which may be used to recognize letters. + +A special set of global settings are the macro handling rules that are stored in a number of hashes: \code{\%TeXmacro}, \code{\%TeXenvir}, etc. as well as similar sets of hashes for package specific rules. + + +\subsection{\TeXcount{} objects} + +First, note that what is referred to as objects here are just hash maps with a predefined set of values. However, these serve the same purposes as objects. There are no explicit class specifications defining these, just a set of functions returning hashes that contain the required keys, some of which may even be optional. Still, it is useful to think of them as objects, and their main purpose is to encapsulate data so that they can conveniently be passed around. + +\begin{description} + +\item[The \Obj{Main} object] +Each \TeXcount{} session instantiates a singleton \Obj{Main} object. This is used as a replacement when no \Obj{TeXcode} object is available for capturing (counting and storing) error messages and warnings. + +\item[The \Obj{TeXcode} object] +The \Obj{TeXcode} object encapsulates the \LaTeX{} code that is to be parsed as well as counts and lists of reported errors. In the code, it is generally referred to using the \code{\$tex} variable. + +\item[The \Obj{count} object] +The \Obj{count} object is primarily a container for the array of counts: i.e. the array containing word counts and counts of headers, equations, etc. However, it also keeps track of subcounts from contained files, sections, etc. + +\end{description} + +A more detailed explanation of the different objects is provided in section \ref{sec:objects} + + +\subsection{Main program flow} + +The main program consists of a single call to the procedure \code{MAIN}. This does the following: + +\begin{description} + +\item[\code{Initialise}:] +Most of the initialisation is done when defining the global variables, but some initalisation required code execution: e.g. OS specific initialisation. + +\item[\code{Check_Arguments}:] +Runs an initial check of the command line arguments passed to \TeXcount{}, e.g. for \code{-help}, and may exit \TeXcount{}. + +\item[\code{Parse_Arguments}:] +Parses the command line arguments, setting option variables, and returns the list of \LaTeX{} files to be parsed. + +\item[\code{Apply_Options}:] +This applies the options set either in the initial setup or initialisation, or when parsing the arguments. While most options are set directly during the argument parsing, settings that may depend on multiple options or options that should be applied only once, e.g. initialising the output and writing the HTML header, are applied here. + +\item[Parse files (or write help or error message):] +The file parsing calls \code{Parse_file_list} with the list of files to be parsed, and this returns the total count object. Apart from this, help, summary output and error reports are produced if required. + +\item[\code{Close_Output}:] This just makes sure the output channel is properly closed, e.g. writing closing HTML code. + +\end{description} + +\CGI +In the CGI version of \TeXcount{}, \code{Initialise}, \code{Check_Arguments} and \code{Parse_Arguments} are replaced by a single call to \code{Set_Options}. Also, since the CGI version only processes one file, alternatives for parsing and reporting on multiple files are not required and is instead replaced by a single call of \code{parse}. +\CGIend + + +\subsection{How \TeXcount{} processes \LaTeX{} documents} + +The \code{parse} routine is the entry point for parsing \LaTeX{} code of a single file. It takes a \Obj{TeXcode} object, the container object of a \LaTeX{} document and its corresponding \Obj{count} object, performs the parsing of the entire document. The counts are stored in the counter in the \Obj{TeXcode} object. + +The hierarchy of delegation from \code{MAIN} down to \code{parse} is as follows: + +\begin{description} + +\item[\code{MAIN}] calls \code{Parse_file_list} with a list of files which return the total count (a count object) for \code{MAIN} to report. + +\item[\code{Parse_file_list}] calls \code{parse_file} for each file in the provided file list, and for STDIN (identified by \code{\$_STDIN_}) if the option to parse standard input has been set. It then aggregates the counts returned by \code{parse_file} into a total count which it returns. + +\item[\code{parse_file}] calls \code{_add_file}, first for the main file, and then again for each included file if file inclusion (\code{-inc}) has been set. The aggregation of counts is done by \code{_add_file} into a total count object provided by \code{parse_file}, and this total count object is then returned by \code{parse_file} upon completing the parsing of the main file as well as all included files. + +\item[\code{_add_file}] reads the file into memory, creates a \Obj{TeXcode} object which encapsulates the \LaTeX{} code and the counts, and calls \code{parse} to perform the parsing of this \Obj{TeXcode} object. The counts are added directly into the \Obj{TeXcode} object, so only the \Obj{TeXcode} object reference is being passed around. + +\end{description} + +\CGI +In the CGI version, \code{parse} is called directly from \code{MAIN} since only one document can be parsed and no file inclusion is possible. +\CGIend + + +\subsection{\LaTeX{} code parsing by \code{parse}} + +The \code{parse} routine takes a \Obj{TeXcode} object and parses this to the end. It is, however, only the entry point for parsing the \LaTeX{} code: other routines do the main parsing with \code{_parse_unit} being the main work horse. In fact, \code{parse} only initiates the parsing, calling \code{_parse_unit} repeatedly until the end of the file. + +The \code{_parse_unit} routine is used to parse one unit or block of \LaTeX{} code: a unit/block can be the a part of the document enclosed in e.g. \{\ldots\} or \code{\bs{begin}\ldots\bs{end}}, or based on context enclosed by e.g. \code{[\ldots]}, or the document at the top level. It is passed the \Obj{TeXcode} object to parse, a parsing state instructing it how the block should be parsed, and optinally a block-end token which tells \code{_parse_unit} when the block ends. The \code{_parse_unit} routine is then called recursively whenever a unit/block is encountered that requires a separate parsing state or closing token. + +The parsing state indicates if the block is part of the main text in which words should be counted, a header, equation contents, should be excluded, etc. and is the only state variable of the parser. In addition to the regular states with which the \LaTeX{} code is parsed, there are transition states. E.g. \code{\$STATE_TO_HEADER} indicates that the block should be counted as a header and the contents should then be parsed using \code{\$STATE_TEXT_HEADER} as specified in \code{\%transition2state}. + +The document is tokenized, and \code{_parse_unit} retrieves one token at the time by calling \code{next_token}. Depending on the active parsing state and token, different rules (most with their own subroutines) are applied. These rules add to the \Obj{count} object of the \Obj{TeXcode} object by calling \code{_inc_count} and set the presentation style of the verbose output included which tokens to print. The active token and its style is by default stored in the \Obj{TeXcode} object and printed to the verbose output by \code{next_token} upon retrieving the next token, although this is occasionally overrun by calls to e.g. \code{flush_next}. + +When \code{_parse_unit} encounters a new block/unit, it will determine the state with which this unit should be parsed based the present state and the context that defines the unit. + + +\subsection{Summary statistics} + +The counts are stored in the \Obj{TeXcode} object: subroutines performing the actual parsing increments the appropriate counter upon processing the parsed tokens. The \Obj{TeXcode} object contains a main \Obj{count} object from which summary output is generated. The \Obj{count} object can also contain a list of subcounts, themselves \Obj{count} objects, which may also be presented in the summary. + +Depending on options set and the number of files parsed, summary output can range from a single number of the total word count, to an extensive summary for each spcified file with separate summaries for each included file, as well as a total summary. + + + +\section{Global constants and variables} + +There are a number of global variables defined at the start of the script for storing options and settings as well as global counters. + +In addition, there are sets of global constants, as well as other globally defined variables, hashes and arrays. Here, we outline the main groups. + + +\subsection{Global constants} + +There are a few sets of global constants. The use of global constants makes the code more readable. The sets of global constants are: + +\begin{description} + +\item[\code{\$STATE_\wild}:] Parsing states, e.g. \code{\$STATE_TEXT} for parsing \LaTeX{} code as regular text and \code{\$STATE_IGNORE} for regious that should not be counted. + +\item[\code{\$CNT_\wild}:] Index pointing to the location in the counter array used for a specific count, e.g. \code{\$CNT_WORDS_TEXT=1} indicating that words in text are counted in position 1 of the array. + +\item[\code{\$TOKEN_\wild}:] Token types, e.g. \code{\$TOKEN_WORD} and \code{\$TOKEN_MACRO}. When a token is parsed, the \Obj{TeXcode} object stores the token type as well as the token, which can then be used to determine how the token should be interpreted. + +\end{description} + +\subsubsection{Counter indices: \code{\$CNT_\wild}} + +The \Obj{count} object contains an array with the following counts: number of files, number of words in text, number of words in headers, number of words in captions, number of headers, number of floating objects/tables/figures, number of inline equations, number of displayed equations. Storing these are the main purpose of the \Obj{count} object. + +Each count has a fixed position in the array, and the \code{\$CNT_\wild} constants provide the positions of each count: e.g. \code{\$CNT_WORDS_TEXT=1} indicates that the counter for words in the text is stored in position 1 of the array. Originally, these positions were hard-coded and directly related to the parsing states, but by using these constants, and keeping the counter indices distinct from the parsing states, the code becomes both more readable and more flexible in case of future changes. + +\subsubsection{Parsing states: \code{\$STATE_\wild}} + +The parsing states fall into two categories. + +First there are parsing states used during the parsing of a unit/block: e.g. \code{\$STATE_TEXT}, \code{\$STATE_MATH}, \code{\$STATE_IGNORE}, ldots. In some of the states, words are counted either as text words, header words or captions words; in other states, words are ignored and the state primarily influences how the parsed \LaTeX{} code is styled in the verbose output. + +Secondly, there are transitional states: e.g. \code{\$STATE_TO_HEADER} which indicates the start of a header which should first cause the header count to be incremented and then the contained text to be parsed as header text using the parsing state \code{\$STATE_TEXT_HEADER}. The handling of the transitional states are encoded in \code{\%transition2state} and performed by the \code{transition_to_content_state} routine which is called by \code{_parse_unit}. + +Macro handling rules specify how many parameters the macro takes and which parsing states are used to parse each parameter; for environments, it additionally specifies a parsing state for the contents of the environment. + +Originally, before implementing the \code{\$STATE_\wild} constants, fixed numerical values were hard coded into the Perl code, and these numerical codes were required for adding new rules. For the macro rules specified within the Perl code of \TeXcount{}, the original numerical codes remain in the initial rule specification. However, from version 2.3 of \TeXcount{}, the intention is that users should no longer use these numerical codes to specify new macro handling rules, but instead use a set of keywords: e.g. \code{text}, \code{header}, \code{ignore}, etc. For this purpose, a hash \code{\%key2state} is defined which maps keywords to parsing states. The original numerical codes are included in this map in part for backward compatibility, but also because this key-to-state map is applied to the macro handling rule hashes \code{\%TeXmacro}, \code{\%TeXenvir}, etc. The \code{\%key2state} has is set up e.g. with +\codeline{add_keys_to_hash(\bs{\%key2state},\$STATE_TEXT,1,'word','w','wd');} +which maps the keys \code{1}, \code{'word'}, \code{'w'} and \code{'wd'} all to the value \code{\$STATE_TEXT} (which need not be 1!). However, this specification, which is used to convert keywords to states during initialisation of the macro handling rules and later if adding new rules, ensures that the original numberical codes will be handled as before: \TeXcount{} will be backward compatibile with respect to using the numberical codes to add new macro handling rules through \code{\%TC} commands. + +Although in theory the parsing state numerical codes could be changed without any effect to the code, there are still a few places where the actual numerical values are used: e.g. the routine \code{state_to_text}. + +\subsubsection{Token types: \code{\$TOKEN_\wild}} + +When the \LaTeX{} code is tokenized, i.e. the string containing the \LaTeX{} code is converted to tokens like words or macros, not only is the token stored in the \Obj{TeXcode} object, but a token type is stored as well indicating if the object is a word, macro, space, symbol, bracket, etc. To make the Perl code more readable, these token type, although just integer values, are represented by constants \code{\$TOKEN_\wild}. + +When \code{_parse_unit} parses the \LaTeX{} code, it frequently uses the token type stored in the \Obj{TeXcode} object rather than the token itself to determine how to interpret the parsed tokens. + + +\subsection{Option alternatives} + +Some options result in choosing between a number of alternatives for parsing, counting or presentation. These alternatives tend to be defined in arrays or hashes. When an alternative is selected, the corresponding value(s) are copied to a variable, array or hash which may then later be applied or further processed. + +\begin{description} + +\item[\code{\%BreakPointOptions}:] For keywords like \code{section} or \code{chapter}, this defines which macros indicate a new break point (i.e. initiates a new subcount). + +\item[\code{\%STYLES}, \code{\%STYLE}:] The \code{\%STYLES} hash contains different sets of style definitions, used to define the style with which tokens are printed, and are used to set the \code{\%STYLE} hash by \code{Apply_Options} after the options have been processed. Each value of the \code{\%STYLES} is a hash mapping style name to ANSI colour styles. For a given style, only style names defined in the style are printed in the verbose output. If ANSI colour coded output is used, these are the colour codes; otherwise, the ANSI colour styles are not themselves used, but the style name must still be included in the hash to enable the token to be printed. + +\item[\code{\%NamedLetterPattern}, \code{\$LetterPattern}:] Named regex patterns are defined in \code{\%NamedLetterPattern} where the selected pattern is stored in \code{\$LetterPattern}. This regex pattern defines what is recognized as letters when parsing \LaTeX{} code. + +\item[\code{\%NamedWordPattern}, \code{@WordPatterns}, \code{\$WordPattern}:] Named word patterns are defined in \code{\%NamedWordPattern}. The selected patterns are stored in the array \code{@WordPatterns}. Letters are indicated by a special character, and when the options are applied replaced by \code{\$LetterPattern} and merged into a single regex stored in \code{\$WordPattern}. + +\item[\code{\%NamedMacroOptionPattern}, \code{\$MacroOptionPattern}:] Named regex patterns are stored in \code{\%NamedMacroOptionPattern}, and the selected pattern copied to \code{\$MacroOptionPattern}. This pattern is used to recognize macro options which should be excluded from word counts. + +\item[\code{\%NamedEncodingGuessOrder}:] For each named language, this gives an array of encodings to try if none is given. + +\end{description} + + + +\section{Details of the \TeXcount{} objects}\label{sec:objects} + +These objects are simply hashes that are created with a given set of keys. Some keys may, however, be optional. + + +\subsection{The \Obj{Main} object} + +The \Obj{Main} object is used instead of the \Obj{TeXcode} object to capture errors and warnings. It is created by the \code{getMain} routine. The values (keys) it contains are: + +\begin{description} + +\item[\code{errorcount}:] Numerical value, initialised to 0, used to count the number of errors reported. + +\item[\code{errorbuffer}:] Array, initialised to an empty array, used to buffer error messages reported before output is available: e.g. before the header or HTML header has been printed. + +\item[\code{warnings}:] Hash, initially empty, used to store warnings. + +\end{description} + +When errors are reported through calls to \code{error}, they will be stored in the \code{errorbuffer} if this exists, otherwise printed immediately. This is used to store errors reported before e.g. the HTML header has been written. After the appropriate headers have been written and the output channel is ready for writing, a call to \code{flush_errorbuffer} is made which prints all the errors in the errorbuffer and then deletes it so further errors will be printed immediately rather than buffered. + + +\subsection{The \Obj{TeXcode} object} + +The \Obj{TeXcode} object is used to encapsulate the \LaTeX{} code and corresponding counts. It is created by the \code{TeXcode} routine. The values it contains are: + +\begin{description} + +\item[\code{filename}, \code{filepath}:] The name and path of the parsed \LaTeX{} file. + +\item[\code{PATH}:] An array containing the paths to search for included documents. At creation, this is empty, but calls to \code{_add_file} will set it; the top level files, initiated from \code{parse_file}, will have this set to \code{\$workdir}. + +\item[\code{texcode}:] Initialised with the \LaTeX{} document as a single string. If included files are to be inserted into the document, they will be inserted into the \code{texcode} string. + +\item[\code{texlength}:] Counts the total length (in characters) of \LaTeX{} code. Initialised with the length of the \LaTeX{} document. If included documents are inserted, their length is added to \code{texlength}. + +\item[\code{line}:] Initialised to an empty string. During parsing, segment by segment (one paragraph at a time) is moved from \code{texcode} to \code{line}. Tokens are then read and subsequently removed from \code{line}. + +\item[\code{next}:] Initialised to \code{undef}, this stored the next token to be processed. Upon tokenization, the token is identified and removed from the start of \code{line} and moved to \code{next}. + +\item[\code{type}:] Initialised to \code{undef}, this contains the token type (\code{\$TOKEN_\wild}) of the \code{next} token. + +\item[\code{style}:] Initialised to \code{undef}, this is used to set the style with which the \code{next} token should be presented in the verbose output. + +\item[\code{printstate}:] Initialised to \code{undef}, this is used output the active parsing state for use with verbose output (if \code{\$showstates} is set). + +\item[\code{eof}:] Initialised to 0, this is set to 1 once the end of the document is reached. + +\item[\code{countsum}:] The contains the main \Obj{count} object. + +\item[\code{subcount}:] This contains the present subcount which is also a \Obj{count} object. These subcount are used to count e.g. section and chapters of the document. + +\item[\code{errorcount}:] Initialised to 0, used to count the number of errors reported during the parsing. + +\item[\code{errorbuffer}:] Undefined at initiation, indicating that errors should be printed instantly rather than stored for later printing. Can be defined as an array which is then used to store error messages so they can be printed later. + +\item[\code{warnings}:] Hash used to store warnings. + +\end{description} + +When the \Obj{TeXcode} object is initialised, the \LaTeX{} document is placed as a single big string in \code{texcode}. During parsing, \code{next_token} is called on to return the next token, which in turn it delegates to \code{_get_next_token}. Instead of operating on the whole document, which was done in older version of \TeXcount{} and was quite slow on large document, \code{more_texcode} is called on to move segments (i.e. paragraphs) of \LaTeX{} code from \code{texcode} to \code{line}, and then it grabs one token at a time from \code{line}. This is when \code{next} and \code{type} are set. + +When the tokens are interpreted and counted, \code{inc_count} is called which increments the appopriate counter in \code{subcount}. If a new subcount is initiated, a call to \code{next_subcount} adds \code{subcount} to \code{sumcount}, including appending the \code{subcount} object to the list of subcounts stored with \code{sumcount}, and then replaces \code{subcount} with a new \Obj{count} object. + + +\subsection{The \Obj{count} object} + +The \Obj{count} object is used to store the word and text element counters. It is created by \code{new_count}. The values it contains are: + +\begin{description} + +\item[\code{title}:] A string set upon creating to contain a descriptive title of the count. + +\item[\code{counts}:] An array, initialized with 0s, which is used to store the counts. The size of the array is determined by \code{\$SIZE_CNT} and should reflect the number of \code{\$CNT_\wild} indices defined. + +\item[\code{subcounts}:] This is an array, initialised to an empty array, used to store the subcounts. + +\end{description} + +In addition to the default fields, when used as the \code{sumcount} field of a \Obj{TeXcode} object, a few additional fields are added: + +\begin{description} + +\item[\code{TeXcode}:] This is a reference pointing back to the \Obj{TeXcode} object in which it is contained. + +\end{description} + + + +\section{\LaTeX{} code parsing and interpreting} + +The entry point for parsing a \LaTeX{} document is the \code{parse} routine. This simply calls \code{_parse_unit} repeatedly using parsing state \code{\$STATE_TEXT} until the end of the document is reached. Thus, \code{_parse_unit} is the main routine for performing the actual parsing. + +The \code{_parse_unit} routine is called with a \Obj{TeXcode} object, a parsing state, and optionally an unit-ending token as arguments. It then calls \code{next_token} on the \Obj{TeXcode} object until the unit-ending token is reached: if the file ends before this is found, an error is reported. If no unit-ending token is provided, only one unit will be parsed. If the unit-ending token is set to \code{\$_PARAM_}, indicating that the unit to be parsed is a macro parameter, the \code{\$simple_token} flag is set and passed to \code{next_token} to avoid combining letters into words, and only one token is parsed before returning. + +For each token, depending on the token, token type, and active parsing state, \code{_parse_unit} decides how the token should be interpreted. In some cases, the interpretation is done within \code{_parse_unit}, but in many cases the interpretation is delegated to subroutines like \code{_parse_macro}, \code{_parse_math}, etc. If new groups (\code{\{\ldots\}} or \code{\bs{begin}\ldots\bs{end}}) are encountered, this causes \code{_parse_unit} to be cause recursively with an unit-ending token passed to \code{_parse_unit} to identify the group end. + +Note that by default, even blocks that are to be ignored are parsed and required balanced units. Different exclude states exist to deal with cases in which the unit should not be completely parsed. + +Upon interpreting the parsed tokens, \code{_parse_unit} or the subroutines to which it delegates the interpretation control the counter incrementation as well as how the tokens are presented in the verbose output. The counter incrementation is done through calls to \code{inc_count} passing as arguments the \Obj{TeXcode} object, the appropriate count reference (\code{\$CNT_\wild}), and optionally a number if the counter should be increased by a number different from 1. Specifying how the token should be presented in the verbose output is done by deciding on the style, usually set using \code{set_style}: the styles are represented by strings that give the style name, which are the same as used as keys in \code{\%STYLE} and as styles in the HTML output. + +If a style for presenting a token is selected which is not in the \code{\%STYLE} hash, the token is not printed. Thus, the \code{\%STYLE} hash also filters which tokens are printed to the verbose output. + + +\subsection{Tokenization and token handling} + +The routine for retrieving the next token is \code{next_token}. This first makes sure that the previous token gets printed to the verbose output with the style specified by \code{set_style}. It then calls \code{_get_next_token} to retrive the next token: this will process comments and line breaks itself until a token is retrieved that it returns. + +The \code{_get_next_token} routine checks the \code{line} field of the \Obj{TeXcode} object to determine which is the next token in \code{line}. If the \code{line} field is empty, it calls \code{more_texcode} to move the next segment of \LaTeX{} code from the \code{texcode} field of the \Obj{TeXcode} object to \code{line}. When it has decided on the approriate kind of token, removing it from the start of the \code{line} field in the process, it sets the \code{next} and \code{type} fields of the \Obj{TeXcode} object through calls to \code{__set_token} or \code{__get_token} (for single character tokens). + +If the optional \code{\$simple_token} flag is set, only simple tokens will be returned: i.e. letters will not be combined into words. This is used for parsing macro parameters. + + +\subsection{Processing parameters and options} + +In \code{_parse_unit}, based on the parsing state and parsed token, it is decided how to interpret and process the token. In some cases, this processing is restricted to the parsed token itself: counting or ignoring it as well as deciding on the style with which it should be presented in the verbose output. + +In some cases, the token influences the parsing of subsequent text: e.g. macros can take parameters and options. Special subroutines exist to handle parsing of macro parameters, gobble up spaces or macro parameters, or handle ignored regions. + + +\subsection{Verbose output} + +By default, all parsed code is processed for printing to the verbose output. If it actually gets printed or not depends on whether the set style is included in the \code{\%STYLE} hash or not. + +Upon parsing a token, it is stored in the \code{next} field of the \Obj{TeXcode} object. If \code{set_style} is called during processing, this will set the \code{style} field of the \Obj{TeXcode} object, but will not itself print the token. The \code{flush_next} routine is used to print the \code{next} token using the style set in the \code{style} field, or provided in the call; this in turn calls \code{print_style} which is responsible for the printing. There is an automatic call to \code{flush_next} when the next token is retrieved, ensuring that all tokens are sent off for printing. When \code{flush_next} is called, the \code{style} field is set to \code{\$STYLE_BLOCK='-'} which blocks further printing (or change in style) of the token; the \code{style} field is then set to \code{undef} by \code{next_token} upon reading the next token. + +The tokens are passed to \code{print_style}, either directly from the parsing or via \code{next_token}, which looks up the style in the \code{\%STYLE} hash. Only tokens whose style is defined in the \code{\%STYLE} hash get printed. If colour coded output to text is set, the values \code{\%STYLE} are used with the \code{ansiprint} function to print the token using ANSI colour codes. If output to HTML is chosen, the token will be printed enclosed in a \code{<span>} tag using the style as class; the HTML style definitions are then used to determine how these elements will be displayed. + +Special style values are \code{\$STYLE_EMPTY=' '} which is used for spaces and must be defined in the \code{\%STYLE} for spaces to be printed, and the \code{\$STYLE_BLOCK='-'} style value which is not actually a style but a value used to mark that the token has already been printed and block further printing of it. + +In addition to the \code{\%STYLE} hash which specifies which tokens get printed, there is a global variable \code{\$printlevel} the value of which is taken from the \code{\%STYLE} which is used to control if verbose output is on ($1$ or $2$) or off ($0$ or $-1$). The $-1$ values indicates the quiet mode in which errors should not be printed; the value $1$, as opposed to $2$, indicates that multiple ignored lines should be collapsed to make the verbose output more compact, although this is only partially done. + +The routines for handling tokens, styles and verbose printing remain from the earliest version of \TeXcount{} and has not undergone much improvements or cleaning up and remains somewhat unstructured. Hence, there may be stray calls to e.g. \code{set_style} that no longer have any effect. + + + +\section{Regex patters: letters, words, macro options} + +One of the most important regex definitions in \TeXcount{} is that used to recognize words. This is done in two steps: first a regex for letters is produced, and then this is combined with patterns for words to generate one big pattern. + +Another regex defined is the one used to recognize macro options, i.e. \code{[\ldots]}, that appear together with macros and which should be ignored. + +One reason behind the desire to generate one big pattern rather than loop through alternative patterns is to enable Perl to compile each pattern just once. The pattern compilation typically takes longer than the pattern matching, so this can make a big difference. + + +\subsection{The word regex} + +First note that \TeXcount{} distinguishes between alphabetic words, i.e. words composed of letters, and logograms (e.g. Chinese characters) which are counted per character. When words (or letters) are counted, these are made from characters defined as alphabetic; characters defined as logographic are counted separately character by character. + +The regex pattern recognizing a letter is placed in \code{\$LetterPattern}. This is usually taken from one of the optional patterns in \code{\%NamedLetterPattern}, but can be modified elsewhere or replaced by \code{undef} to signify that no words or letters should be counted. + +A number of regex patterns which should be recognized as words are place in the array \code{\@WordPatterns}. This is usually set by using one of the named lists of word patterns defined in \code{\%NamedWordPattern}, but can be redefine or modified by options. In the word patterns, the character \code{\@} is used to represent a letter, and this is later replaced by \code{\$LetterPattern} when the options are applied. + +After parsing the command line arguments, the options and settings are applied. At this point, through \code{apply_language_options}, \code{\$LetterPattern} is applied to \code{\@WordPatterns}, which are then combined into a single regex: \code{\$WordPattern}. At this point, patterns for recognizing logograms are also added. + + +\subsection{The macro option regex} + +After macros and macro parameters, macro options on the form \code{[\ldots]} will be ignored. There is a single regex used to recognize and remove these macro options. + +For most uses, macro options tend to be short codes which are easily recognized. However, there are also cases where the macro options can be more complex. On the other hand, there are also cases where brackets are used without being macro options, and it is vital that these cases should not be mistaken for macro options: in particular if they contain text that should be counted. + +In order to capture most macro options as options without running a risk of ignoring actual text enclosed in brackets, restrictions are placed on what can go inside macro options. The default rule is moderately strict, but can be relaxed to allow more extensive and general macro options. + +The different macro option regex patterns are named in \code{\%NamedMacroOptionPattern} and copied to \code{\$MacroOptionPattern} when initialised or changed by options. + + +\subsection{Unicode character classes} + +The user can specify which character classes should be considered alphabetic (i.e. letters) and which should be considered logographic (i.e. counted as indicidual characters). Typical alphabetic characters are the Latin letters. Typical logograms are the Chinese characters. If any of the language options are used, these character classes will automatically be set. + +Specifications of alphabets and logograms are done by options \code{-alpha=} and \code{-logo=} using Unicode character classes. Unicode classes include Latin, Digit, Ideographic, Han, etc. Note that all Unicode character classes start with capital letters. + + +\subsection{Custom made character classes} + +Some of the Unicode character classes are not defined quite as desired by \TeXcount{}. In particular, the \code{Alphabetic} character class includes \code{Ideographic}, which would cause e.g. Chinese characters to be allowed as parts of words together with Latin characters rather than force them to be counted as individual characters. To resolve this problem, new character classes are defined in \TeXcount{} that fit our need. + +New character classes can be defined within \TeXcount{} through subroutines named \code{Is_\textit{name}}. Most notable is the \code{Is_alphabetic} character class from which the logographic characters have been excluded. This is now used as the default alphabetic character class. + +Presently defined characters classe are named \code{digit}, \code{alphabetic}, \code{alphanumeric}, \code{punctuation}, \code{cjk}, \code{cjkpunctuation}. Note that these are all lower case, and have the prefix \code{Is_} added when referred to in the code. + +When adding character classes to the set of alphabetic or logographic characters using \code{-alpha=} or \code{-logo=}, the names without the prefix \code{Is_} may be used: for character classes starting with a lower case letter, the prefix is added automatically. + +Note that the subroutines specifying the character classes must be defined prior in the code to any use: this is unlike other subroutines which may be defined anywhere in the code. Also, to be permitted as character classes by Perl, the subroutines must start with \code{Is_} (or \code{In_} although that is not used by \TeXcount{}), although different versions of Perl need not enforce this. + + + +\section{Macro handling rules} + +While some rules for handling macros are hard-coded into \TeXcount{}, most of the rules are stored in a number of hashes which \TeXcount{} look up whenever a macro is encountered. The general rule is that the keys are either macros (e.g. \code{'\bs{section}'}) or environment names (e.g. \code{'quote'}). + +\begin{description} + +\item[\code{\%TeXmacro}:] The keys are macros, or \code{'begin\textit{name}'} where name is an environment name, and the values specify how many parameters the macro (or environmemt) takes and how these should be processed. See the section on parameter handling rules further down. + +\item[\code{\%TeXenvir}:] The keys are environment names, and values are the parsing state with which the contents of the environment should be parsed. + +\item[\code{\%TeXpreamble}:] These are macro handling rules to be applied in the preamble, i.e. after \code{\bs{documentclass}} but before \code{\bs{begin}\{document\}}. The rules are specified as for \code{\%TeXmacro}. + +\item[\code{\%TeXfloatinc}:] These are macro handling rules to be applied within floating bodies, i.e. tables and figures. + +\item[\code{\%TeXmacroword}:] The keys are macros, and the values are numbers representing how many words the macro generates. This is used for macros like \code{\%LaTeX} which generates text. + +\item[\code{\%TeXpackageinc}:] The keys are macros used to include packages. Although included in \code{\%TeXmacro}, the processing of package inclusion is actually performed by \code{_parse_include_package} independent of the hash value. The value should therefore be \code{1} or \code{[\$STATE_IGNORE]} since this is how it will be processed by \code{_parse_include_package}. + +\item[\code{\%TeXfileinclude}:] The keys are macros used to include \LaTeX{} files into the document, the value a keyword or list of keywords telling how file names and paths should be interpreted. Processing of these macros is done by \code{_parse_include_file}. + +\end{description} + +Note that the definition of \code{\%TeXmacro} starts by including \code{\%TeXpreamble}, \code{\%TeXfloatinc} and \code{\%TeXpackageinc}. After that, the values of \code{\%TeXpackageinc} are never used. For \code{\%TeXpreamble} and \code{\%TeXfloatinc}, however, it is in principle possible to rules within the preamble and floats, respectively, that are different from those defined in \code{\%TeXmacro} and applied elsewhere in the document. + + +\subsection{Parameter handling rules} + +A macro can be specified to take a given number of parameters: this will typically be \code{\{\ldots\}} blocks following the macro. For each of these parameters, a separate parsing state can be specified. This is represented by an array with one element for each parameter, the elements being the parsing state (\code{\$STATE_\wild}) with which that parameter should be parsed. + +In addition to the \code{\$STATE_\wild} rules are some modifier/option states, \code{\$_STATE_\wild}. The \code{\$STATE_OPTION} states indicates that the next rule in the list is an optional parameter enclosed in \code{[]}. By default \code{[]} options are ignored, which can be swithed off by \code{\$STATE_NOOPTION} or on by \code{\$STATE_AUTOOPTION}. + +An alternative specification of a parameter handling rule is to give the number of parameters to ignore. \TeXcount{} will check if the specified rule is an array (as described above) or a number and interpret the rule accordingly. + +The hashes \code{\%TeXmacro}, \code{\%TeXpreamble} and \code{\%TeXfloatinc} all take values that are this kind of parameter handling rules, as are q\code{\%TeXpackageinc} since they are included in \code{\%TeXmacro}. + +Throughout the script, parsing states are referred to using the \code{\$STATE_\wild} constants. In previous versions, however, these codes were hard-coded into the script and used both to set up the hashes and to specify new rules through \%TC instructions. For backward compatibility, the old numerical state codes remain in the conversions from keywords to \code{\$STATE_\wild} constants as stored in \code{\%key2state} and applied through calls to \code{convert_hash} accompanied by \code{keyarray_to_state} or \code{key_to_state}. + + +\subsection{File inclusion and the \code{\%TeXfileinclude} hash} + +The main \LaTeX{} commands for file inclusion are \code{\bs{input}} and \code{\bs{include}}, while \code{\bs{bibliography}} includes the \code{.bbl} bibliography file. However, additional packages exist that can also modify the file search path, of which \TeXcount{} has support for the \code{import} package. + +File inclusion macro rule are stored in the \code{\%TeXfileinclude} hash. The values are strings which contain one or more keywords (separated by space or comma): +% +\begin{description} + +\item[\code{input}:] This is a special keyword to use with \code{\bs{input}}. The handling of the parameter values is as \code{file}, but the parameter itself is not required to be enclosed in \code{\{\}}. + +\item[\code{file}:] This parameter simply gives the name of or path to a file. If the file is not found, \TeXcount{} will append \code{.tex} and try again. + +\item[\code{texfile}:] This parameter gives the name of or path to a file, but \code{.tex} will be appended, and is the rule used by \code{\bs{include}}. + +\item[\code{dir}:] This parameter provides the path of a directory relative to the \code{\$workdir}, and adds this to the search path before including any files. This is used with the \code{\bs{import}} macro of the \code{import} package. + +\item[\code{subdir}:] This parameter provides the path of a directory relative to the current directory, and adds this to the search path before including any files. This is used with the \code{\bs{subimport}} macro of the \code{import} package. + +\item[\code{<bbl>}:] This is a special keyword to use with \code{\bs{bibliography}} to specify inclusion of the bibliography file. + +\end{description} + +The parsing of the macros and parameters is done by \code{_parse_include_file}. For each keyword it parses a parameter, unless the parameter is on the form \code{<\textit{keyword}>}. The parsing of the \code{input} parameter is handled differently from the rest since it need not be enclosed by \code{\{\}}. It then delegates the processing of macro inclusion rules to \code{include_file}. + +In \code{include_file}, the file is located (based on search path) and either appends the file to the \code{@filelist} array of files to be include, or merged immediately into the document by a calls to \code{read_binary} and \code{prepend_code}. + +The \code{@filelist} array contains elements which are themselves arrays on the form \code{[file,path,\ldots]} where the first element is the path to the file to be included, and the remaining elements are the search paths used to set the \code{PATH} values of the \Obj{TeXcode} object. For the top level files, i.e. the ones specified on the command line, the search path will contain only \code{\$workdir}: the directory from which \TeXcount{} is executed unless \code{-dir} is used to specify otherwise. If more directories are added to the path, \code{\$workdir} will remain the last directory of the search path, while the first directory of the search path will be considered the current directory. + +File inclusion macros can also take parameters that should be parsed using regular macro parsing rules. The \code{TeXmacro} hash will be checked for \code{@pre\bs{macroname}} and \code{@post\bs{macroname}} entries which will be applied before and after the file handling rules. + + +\subsection{Package and document class specific rules} + +Whenever \TeXcount{} encounters a package inclusion, it will check for package specific rules. These are defined in hashes names \code{\%PackageTeXmacro} etc. which maps the package name to the hash map of rules to be added to \code{\%TeXmacro} etc. There is an additional \code{\%PackageSubpackage} which for each package name in the set of keys maps to a list of packages whose rules should automatically be included. + +Similarly, rules specific to particular document classes may be implemented by using the key \code{class\%\parm{name}} instead of the package name, and these will then be added to the set of parsing rules if \code{\bs{documentclass}\{\parm{name}\}} is encountered. + +Note that rules for including the bibliography is also stored in these hashes under the key \code{\%incbib}. + + + +\section{Presentation of summary statistics} + +The counts (words, headers, etc.) from a \LaTeX{} document are stored as a \Obj{count} objects. The main routine for printing the summary statistics from a \Obj{count} object is \code{print_count}: the routine \code{conditional_print_total} which is called from \code{MAIN} delegates printing to \code{print_count} except if the brief output format is selected. The \code{print_count} routine then delegates the printing to one of a number of subroutines depending on the settings. + +Word frequencies are store globally in \code{\%WordFreq}. This gets incremented each time \code{_process_word} is called. Summary of word frequencies are produced and printed by \code{print_word_freq} which tries to combine words that differ only by capitalization, and also produces subcounts per character class. + +A global count of the number of errors reported is stored in \code{\$errorcount}, while warnings are stored globally in the \code{\%warnings} hash mapping when added through the \code{warning} routine with the warning as key and the number of occurrences as value to ensure each warning is only listed once no matter how many times it is reported. Both warnings and errors are also stored in their respective \Obj{Main} or \Obj{TeXcode} objects when reported through calls to \code{error} or \code{warning}. + +In \code{MAIN}, after processing of the \LaTeX{} documents, \code{Report_Errors} is called to give a total report on errors and warnings. The exact output depends on the settings. + +NB: Processing of errors and warnings requires some improvement. Now, parts of the code handle errors per file, others do so globally. + + + +\section{Encodings} + +The preferred encoding is Unicode UTF-8. From version 2.3 of \TeXcount{}, this is used internally to represent the \LaTeX{} code, and Unicode is relied upon to handle different character sets and classes. + +When files are read into \TeXcount{}, they may have to be decoded from whatever encoding they are saved in into UTF-8. The file encoding may be specified explicitly using the \code{-enc=} option, otherwise \TeXcount{} will try to guess the appropriate encoding. + +The output from \TeXcount{} is be default UTF-8. However, if a file encoding is specified using \code{-enc=} and output is text, not HTML, this encoding will also be applied to the output. This may be useful when using \TeXcount{} in a pipe, otherwise the documents will be converted to UTF-8. + + + +\section{Help routines and text data} + +A hash, \code{\%GLOBALDATA}, and hash reference \code{\$STRINGDATA} are is defined for storing strings used for various outputs. The \code{\%GLOBALDATA} is set up containing string constants for version number, maintainer name, etc., while \code{\$STRINGDATA} is initially undefined. + +The \code{\$STRINGDATA} hash is accessed through calls to \code{StringData} which initialises the hash if undefined. Initialisation, which is done by \code{STRINGDATA}, reads through the \code{__DATA__} section at the end of the script, identifies headers which are used as keys in the hash which maps to and array containing the subsequent text lines. References in the read text on the form \code{\$\{keyword\}} are replaced by the corresponding string in \code{\%GLOBALDATA}: this allows e.g. version information to be inserted into the text. + +Headers in the text data consists of three or more colons followed by space(s) and a keyword. Lines containing three or more colons but no keyword have no effect. + +Lines starting with \code{\@} are used to format output printed by \code{wprintlines}. The two characters \code{'-'} and \code{':'} can then be used to indicate indentation tabulators, and subsequent lines will be indented and wrapped: this is used for printing help on command line options. The \code{wprintlines} also wraps text: the page with is set by \code{\$Text::Wrap::columns}. + + +\end{document} diff --git a/Master/texmf-dist/doc/support/texcount/macros.tex b/Master/texmf-dist/doc/support/texcount/macros.tex index b7f853f39ad..a79e8f06144 100644 --- a/Master/texmf-dist/doc/support/texcount/macros.tex +++ b/Master/texmf-dist/doc/support/texcount/macros.tex @@ -1,10 +1,20 @@ %% LaTeX macros +\usepackage{relsize} +\usepackage[dvipsnames]{xcolor} + +% Define underscore to be a regular character in text mode +\begingroup + \catcode`\_=\active + \gdef_#1{\ensuremath{\sb{#1}}} +\endgroup +\mathcode`\_=\string"8000 +\catcode`\_=12 % Set version number -\newcommand\version{$2.3$} +\newcommand\version{$3.0$} \newcommand\copyrightfootnote{ -\footnotetext{Copyright (2008-2011) of Einar Andreas R{\o}dland, distributed +\footnotetext{Copyright (2008-2013) of Einar Andreas R{\o}dland, distributed under the \LaTeX{} Project Public License (LPPL).} } @@ -12,15 +22,43 @@ under the \LaTeX{} Project Public License (LPPL).} \newcommand\TeXcount{{\TeX}count} % Text formats -\newcommand\code[1]{\texttt{\small#1}} -\newcommand\bigcode[1]{\texttt{#1}} +\newcommand\codestyle[1]{\textsf{\color{Blue}#1}} +\newcommand\code[1]{{\smaller\codestyle{#1}}} +\newcommand\bigcode[1]{\codestyle{#1}} \newcommand\codeline[1]{\begin{quote}\code{#1}\end{quote}} \newcommand\bs[1]{\textbackslash#1} -\newcommand\URL[1]{\texttt{\small #1}} +\newcommand\URL[1]{\textsf{\small #1}} + +% Description items: options, parameters, optional parameters +% These are also interpreted by dos2html.pl +\def\option[#1]{\item[\bigcode{#1}]\hskip 0pt plus 10pt} +\def\parm#1{\textit{\color{OliveGreen}#1}} +\def\opt#1{\parm{[#1]}} +\def\alt#1{[#1]} +\def\optiontext#1{\textrm{\bfseries\color{black}#1}} + +% Mark off notification in contents for good visibility +\newcommand\ContentsNote[1]{\addtocontents{toc}{\string\marginpar{\textsf{\color{red}\tiny #1}}}} +% Notabene: margin note %TC:macro NB 1 -\newcommand\NB[1]{\marginpar{\textsf{\tiny#1}}} +\newcommand\NB[1]{\ContentsNote{NB}\marginpar{\textsf{\tiny#1}}} + +% Mark text as a notification +%TC:macro NOTE [text] +\newcommand\NOTE[1]{\textit{\color{red}#1}} + +% Mark text for update +%TC:macro UPDATE [text] +\newcommand\UPDATE[1]{\ContentsNote{UPDATE}\textit{\textbf{\color{red}This needs to be updated:} {\color{blue}#1}}} + +% Mark text as a notification +%TC:macro TODO [text] +\newcommand\TODO[1]{\ContentsNote{TODO}\textit{\textbf{\color{red}#1}}} +% BUG +%TC:macro BUG [text] +\newcommand\BUG[1]{{\color{red}#1}} \makeatletter @@ -34,13 +72,13 @@ under the \LaTeX{} Project Public License (LPPL).} \vskip 1.5em } -\renewcommand\abstractname{ABSTRACT} +\renewcommand\abstractname{Abstract} \renewenvironment{abstract}{% \begin{center}% - {\bfseries\large\abstractname\vspace{-.5em}\vspace{\z@}}% + {\slshape\bfseries\large\abstractname\vspace{-.5em}\vspace{\z@}}% \end{center}% \vskip 4pt - \bfseries + \slshape }{ \vskip 0.5em } diff --git a/Master/texmf-dist/doc/support/texcount/sub_addrules.tex b/Master/texmf-dist/doc/support/texcount/sub_addrules.tex index 6c702ea95c3..fb1d3adfec4 100644 --- a/Master/texmf-dist/doc/support/texcount/sub_addrules.tex +++ b/Master/texmf-dist/doc/support/texcount/sub_addrules.tex @@ -2,24 +2,28 @@ \begin{description} \sloppy -\def\option[#1]{\item[\bigcode{#1}]\hskip 0pt plus 10pt} -\option[macro]Defines macro handling rule for the specified macro. It takes one parameter which is either an integer or a \code{[]}-enclosed array of integers (e.g. \code{[0,1,0]}). An integer value $n$ indicates that the $n$ first parameters to the macro should be ignored. An array of length $n$ indicates that the first $n$ parameters should be handled by the rule, and the numbers in the array specifies the parsing status (see below) with which they should be parsed. Giving the number $n$ as parameter is equivalent to giving an array of $n$ zeroes (\code{[0,\ldots,0]}) as zero is the parsing status for ignoring text. For all macros, also those for which no rules have been defined, options enclosed in \code{[]} between or after macros and their parameters are ignored. +\option[macro \parm{macroname} \parm{parameter-rules}] +Defines macro handling rule for the specified macro. The parameter is on the form \code{[\textit{rule},\ldots]} where each rule is either a keyword indicating the parsing rule for a macro parameter or \code{option:\textit{rule}} for an optional \code{[]}-enclosed parameter. Alternatively, an integer value $n$ indicates that the $n$ first parameters to the macro should be ignored, equivalent to giving a list of $n$ \code{ignore} rules. -\option[macroword]This defines the given macro to be counted as a certain number of words, where the number is given as the parameter. +\option[envir \parm{envirname} \parm{parameter-rules} \parm{content-rule}] +(The previously used command, \code{group}, remains an alias for \code{envir}, but the name \code{envir} is more appropriate and therefore recommended.) +This specifies the handling of environments with the given name. The parameter handling rule, applied to parameters following \code{\bs{begin}\{\textit{name}\}}, are specified as in the \code{macro} instruction. The second parameter specifies the rule, i.e. parser state, with which the contents should be parsed. -\option[header]Define macro to be a header. This is specified as the macro rule, but has the added effect is that the header counter is increase by 1. Note, however, that you should specify a parameter array, otherwise none of the parameters will be parsed as header text. The parser status for header text is 2, so a standard header macro that uses the first parameter as header should be given the parameter \code{[2]}. +\option[macrocount \parm{macroname} \opt{count-spec.}] +(An alias for \code{macrocount} is \code{macroword}; the preferred name was changed to reflect that this can count any element, not just words.) +If a number is provided as the count parameter, this defines the given macro to be counted as the specified number of words; if no count is specified, it is assumed to be 1. Alternatively, a \code{[]}-enclosed list of counters can be specified (using the counter keywords), causing each of them to be incremented: counter are \code{word}/\code{text}, \code{headerword}, \code{otherword}, \code{header}, \code{float}, \code{inlinemath}, \code{displaymath} plus a number of aliases. -\option[breakmacro]Specify that the given macro should cause a break point. Defining it as a header macro does not do this, nor is it required of a break point macro that it be a header (although I suppose in most cases of interest it will be). +\option[breakmacro \parm{macroname}] +Specify that the given macro should cause a break point. -\option[group]This specifies a begin-end group with the given name (no backslash). It takes two further parameters. The first parameter speficies the macro rule following \code{\bs{begin}\{\textit{name}\}}. The second parameter specifies the parser status with which the contents should be parsed: e.g. $1$ for text (default rule), $0$ to ignore, $-1$ to specify a float (table, group, etc.) for which text should not be counted but captions should, $6$ and $7$ for inline or displated math. +\option[floatinclude \parm{macroname} \parm{parameter-rules}] +Specify macro handling rules used within float groups. The handling rules are specified as for \code{macro}. Most commonly, the parameter rule will be the \code{otherword}/\code{oword} to specify that words should be counted as \textit{other words}. -\option[floatinclude]This may be used to specify macros which should be counted when within float groups. The handling rules are spefified as for \code{macro}, but like with \code{header} an array parameter should be provided and parameters that should be counted as text in floats should be specified by parsing status 3. Thus, a macro that takes one parameter which should be counted as float/caption text should take the parameter \code{[3]}. +\option[preambleinclude \parm{macroname} \parm{parameter-rules}] +Specifies macro handling rules to be used in the preamble: the text between \code{\bs{documentclass}} and \code{\bs{begin}\{document\}}. The rule is specified like the \code{macro} rules. -\option[preambleinclude]The preamble, i.e. text between \code{\bs{documentclass}} and \code{\bs{begin}\{document\}}, if the document contains one, should generally not be included in the word count. However, there may be definitions, e.g. \code{\bs{title}\{title text\}}, that should still be counted. In order to be able to include these special cases, there is a preambleinclude rule in which one may speficy handling rules for macros within the preamble. Again, the rule is speficied like the \code{macro} rules, but since the default is to ignore text the only relevant rules to be specified require an array. - -\option[fileinclude]By default, \TeXcount{} does not automatically add files included in the document using \code{\bs{input}} or \code{\bs{include}}, but inclusion may be turned on by using the option \code{-inc}. If other macros are used to include files, these may be specifed by adding fileinclude rules for these macros. The specification takes one parameter: 0 if the file name should be used as provided, 1 if file type \code{.tex} should be added to files without a file type, and 2 if the file tyle \code{.tex} should always be added. - -\option[subst]This substitutes a macro (the first parameter) with any text (the remaining option). Substitution is performed only on the present file and on the text following the instruction. Note that substitution is performed directly on the \LaTeX{} code prior to parsing, and the verbose output will show the substituted text. E.g. \code{\%TC:subst \bs{test} TEST} will cause a following \code{\bs{newcommand}\bs{test}{TEST}} to be changed into \code{\bs{newcommand} TEST{TEST}}, which \TeXcount{} will interpret differently. Use with care! +\option[fileinclude \parm{macroname} \parm{file-path-spec.}] +Specifies macros that cause files to be included when \TeXcount{} is run with the \code{-inc} option. The parameters specify the format on which the file path is specified, and can also be used to modify the search path used within the included document. \end{description} diff --git a/Master/texmf-dist/doc/support/texcount/sub_options.tex b/Master/texmf-dist/doc/support/texcount/sub_options.tex index 75fdd7509ad..3efa05a0d80 100644 --- a/Master/texmf-dist/doc/support/texcount/sub_options.tex +++ b/Master/texmf-dist/doc/support/texcount/sub_options.tex @@ -17,6 +17,8 @@ \option[-v4]Same as \code{-v3 -showstate}. +\option[-v=\ldots, -v\alt{[0-4]}\ldots]Allows detailed control of which elements are included in the verbose output. The provided values is a list of styles or style categories separated by \code{+} or \code{-} to indicate if they should be added or removed from the list of included styles. Style categories start with capital letter and include \code{Words}, \code{Macros}, \code{Options}; the individual styles are in all lower case and include \code{word}, \code{hword}, \code{option}, \code{ignore}. + \option[-showstate]Show internal states (with verbose). \option[-brief]Only prints a one line summary of the counts for each file. @@ -31,13 +33,13 @@ \option[-0]Same as \code{-1}, i.e. \code{-brief} and \code{-total}, but does not put a line shift at the end. This may be useful when the one line output is to be used by another application, e.g. Emacs, for which the line shift would otherwise need to be stripped away. -\option[-template="\ldots"]Speficy an output template which is used to generate the summary output for each file and for the total count. Codes \code{\{\textit{label}\}} is used to include values, where \code{\textit{label}} is one of \code{0} to \code{7} (for the counts), \code{SUM}, \code{ERROR} or \code{TITLE} (first character of label is sufficient). Conditional inclusion is done using \code{\{\textit{label}?\textit{text}?\textit{label}\}} or \code{\{\textit{label}?\textit{if non-zero}|\textit{if zero}?\textit{label}\}}. If the count contains at least two subcounts, use \code{\{SUB|\textit{template}|SUB\}} with a separate template for the subcounts, or \code{\{SUB?\textit{prefix}|\textit{template}|\textit{suffix}?SUB\}}. +\option[-template="\ldots"]Specify an output template which is used to generate the summary output for each file and for the total count. Codes \code{\{\textit{label}\}} is used to include values, where \code{\textit{label}} is one of \code{0} to \code{7} (for the counts), \code{SUM}, \code{ERROR} or \code{TITLE} (first character of label is sufficient). Conditional inclusion is done using \code{\{\textit{label}?\textit{text}?\textit{label}\}} or \code{\{\textit{label}?\textit{if non-zero}|\textit{if zero}?\textit{label}\}}. If the count contains at least two subcounts, use \code{\{SUB|\textit{template}|SUB\}} with a separate template for the subcounts, or \code{\{SUB?\textit{prefix}|\textit{template}|\textit{suffix}?SUB\}}. \option[-sub\alt{=\ldots}, -subcount\alt{=\ldots}]Generate subcounts. Valid option values are \code{none}, \code{part}, \code{chapter}, \code{section} and \code{subsection} (default), indicating at which level subcounts are generated. (On by default.) \option[-nosub]Do not generate subcounts. -\option[-sum\alt{=n,n,\ldots}]Produces total sum, default being all words and formulae, but customizable to any weighted sum of the seven counts (list of weights for text words, header words, caption words, headers, floats, inlined formulae, displayed formulae). +\option[-sum\alt{=n,n,\ldots}]Produces total sum, default being all words and formulae, but customisable to any weighted sum of the seven counts (list of weights for text words, header words, caption words, headers, floats, inlined formulae, displayed formulae). \option[-nosum]Do not generate total sum. (Default choice.) @@ -45,6 +47,10 @@ \option[-nc, -nocol]No colours (colours require ANSI). Default under Windows. +\option[-nosep, -noseparator]No separating character/string added after each word in the verbose output (default). + +\option[-sep=, -separator=]Separating character or string to be added after each word in the verbose output. + \option[-relaxed]Relaxes the rules for matching words and macro options. \option[-restricted]Restricts the rules for matching words and macro options. @@ -63,7 +69,9 @@ \option[-incpackage=]Include rules for a given package. -\option[-dir\alt{=\ldots}]Specify working directory which will serve as root for all include files. Default (\code{-dir=""}) is to use the present directory. Use \code{-dir} to use path of the main \LaTeX{} document. +\option[-dir\alt{=\ldots}]Specify working directory which will serve as root for all include files. The default (\code{-dir=.}) is to use the current directory, i.e. from which \TeXcount{} is executed: the path can be absolute or relative to the current directory. Use \code{-dir} to use the path of the main \LaTeX{} document as working directory. + +\option[-auxdir\alt{=\ldots}]Specify the directory of the auxiliary files, e.g. the bibliography (\code{.bbl}) file. The default setting (\code{-auxdir} only) indicates that auxiliary files are in the working directory (from the \code{-dir} or \code{-dir=} option). If \code{-auxdir=} is used to provide a path and \code{-dir=} is used to specify the working directory, the path to the auxiliary directory is take to be relative to the current folder (from which \TeXcount{} is executed); if used with \code{-dir}, the working directory is taken from the path of the parsed file, and the auxiliary directory is taken to be relative to this (unless an absolute path is provided). \option[-enc=, -encoding=]Specify encoding to use in input (and text output). @@ -75,23 +83,35 @@ \option[-ch, -chinese, -zhongwen]Turn on Chinese mode in which Chinese characters are counted. I recommend using UTF-8, although \TeXcount{} will also test other encodings (GB2312, Big5, Hz) if UTF-8 fails, and other encodings may be specified by \code{-encoding=}. -\option[-jp, -japanese]Turn on Japanese mode in which Japanese characters (kanji and kana) are counted. I recommend using UTF-8, although \TeXcount{} will also test other encodings (e.g. EUC-JP) if UTF-8 fails, and other encodings may be specified by \code{-encoding=}. +\option[-jp, -japanese]Turn on Japanese mode in which Japanese characters (Kanji and Kana) are counted. I recommend using UTF-8, although \TeXcount{} will also test other encodings (e.g. EUC-JP) if UTF-8 fails, and other encodings may be specified by \code{-encoding=}. -\option[-kr, -korean]Turn on Korean mode in which Korean characters (hangul and han) are counted. I recommend using UTF-8, although \TeXcount{} will also test other encodings (e.g. EUC-KR) if UTF-8 fails, and other encodings may be specified by \code{-encoding=}. \emph{NB: Support for Korean is experimental.} +\option[-kr, -korean]Turn on Korean mode in which Korean characters (Hangul and Han) are counted. I recommend using UTF-8, although \TeXcount{} will also test other encodings (e.g. EUC-KR) if UTF-8 fails, and other encodings may be specified by \code{-encoding=}. -\option[-kr-words, -korean-words]Korean mode in which hangul words are counted (i.e. as words separated by spaces) rather than characters. Han characters are still counted as characters. See also \code{-korean}. \emph{NB: Support for Korean is experimental.} +\option[-kr-words, -korean-words]Korean mode in which Hangul words are counted (i.e. as words separated by spaces) rather than characters. Han characters are still counted as characters. See also \code{-korean}. \option[-chinese-only, ..., -korean-words-only]As options \code{-chinese}, ..., \code{-korean-words}, but also excludes other alphabets (e.g. letter-based words) and logographic characters. \option[-char, -letter]Count letters instead of words. This count does not include spaces. +\option[-out=]Send output to file. Takes file name as value. + \option[-html]Output in HTML format. \option[-htmlcore]Only HTML body contents. +\option[-htmlfile=]File containing a template HTML document with \code{<!-- TeXcount -->} included somewhere to indicate the location where the TeXcount output from the parsing should be inserted. + +\option[-tex]Encode \TeX{} special characters for output into \TeX{} code. + +\option[-css=]Reference to CSS to be included in the HTML output instead of including the style definition directly in the output. + +\option[-cssfile=, -css=file:]File containing style definitions to be included into the HTML output instead of the default styles. + \option[-freq\alt{=\#}]Count individual word frequencies. Optionally, give minimal frequency required to be included in output. -\option[-stat]Produce statistics on language usage, i.e. based on the alphabets and logograms included. +\option[-stat]Produce statistics on language usage, i.e. based on the alphabets and logograms included. + +\option[-macrostat, -macrofreq]Produce statistics on package, environment and macro usage. \option[-codes]Display an overview of the colour codes. Can be used as a separate option to only display the colour codes, or together with files to parse. @@ -105,7 +125,15 @@ \option[-h, -?, --help, /?]Help. -\option[-h=, -?=, --help=, /?=]Help on particular macro or group name: gives the parsing rule for that macro or group if defined. You may have to use \code{-incpackage=}package if the rule is defined for a specific package, and this option must be placed \emph{before} the \code{-h=} option on the command line. +\option[-h=, -?=, --help=, /?=]Help on particular macro or group name: gives the parsing rule for that macro or group if defined. If the the macro or environment is package specific, use \code{-h=\parm{package}:\parm{name}}; replace \code{\parm{package}} with \code{class\%\parm{name}} if it is specific to a document class. + +\option[-help-options, -h-opt]Lists all TeXcount options and help on them. + +\option[--help-option=, -h-opt=]Lists all TeXcount options containing the provided string: e.g. \code{-h-opt=inc} lists all options containing \code{inc}, while \code{-h-opt=-v} lists all options starting with \code{v}. + +\option[-help-style, -h-style]Lists all styles and style categories, i.e. those permitted used in -v={styles-list}. + +\option[-help-style=, -h-style=]Gives description of style or style category. \option[-ver, --version]Print version number. diff --git a/Master/texmf-dist/doc/support/texcount/sub_tc_other.tex b/Master/texmf-dist/doc/support/texcount/sub_tc_other.tex index 4c949ebdead..dd6c05fabcc 100644 --- a/Master/texmf-dist/doc/support/texcount/sub_tc_other.tex +++ b/Master/texmf-dist/doc/support/texcount/sub_tc_other.tex @@ -1,13 +1,27 @@ %% Other TC instructions \begin{description} -\def\option[#1]{\item[\bigcode{#1}]} -\option[ignore]Indicates start of a region to be ignored. End region with the TC-instruction \code{endignore}. +\option[break \parm{title}] +Break point which initiates a new subcount. The title is used to identify the following region in the summary output. -\option[break \textit{title}]Break point which initiates a new subcount. The title is used to identify the following region in the summary output. +\option[incbib \optiontext{or} includebibliography] +Sets bibliography inclusion, same as running \TeXcount{} with the option \code{-incbib}. -\option[incbib]Sets bibliography inclusion, same as running \TeXcount{} with the option \code{-incbib}. +\option[subst \parm{macro} \parm{text}] +This substitutes a macro with any text. The verbose output will show the substituted text: e.g. \code{\%TC:subst \bs{test} TEST} will cause a following \code{\bs{newcommand}\bs{test}\{TEST\}} to be changed into \code{\bs{newcommand} TEST\{TEST\}}, which \TeXcount{} will interpret differently. Use with care! + +\option[ignore] +Indicates start of a region to be ignored. End region with \code{\%TC:endignore}. + +\option[insert \parm{\TeX-code}] +Insert \TeX{} code for \TeXcount{} to process. + +\option[newcounter \parm{name} \opt{description}] +Define a new counter with the given name and description (optional). A corresponding parsing rule will also be added with the same name. + +\option[newtemplate \optiontext{and} template \opt{template-line}] +Specify a template for the summary output. The first line should just declare a new template using \code{\%TC:newtemplate}, while the subsequent lines use \code{\%TC:template} followed by text specifying the template. The line breaks in the template specification are not of importance: to specify a line break, use \code{\bs{n}}. \end{description} diff --git a/Master/texmf-dist/scripts/texcount/texcount.pl b/Master/texmf-dist/scripts/texcount/texcount.pl index 69fb4b5002f..5a53b3bfb4d 100755 --- a/Master/texmf-dist/scripts/texcount/texcount.pl +++ b/Master/texmf-dist/scripts/texcount/texcount.pl @@ -1,15 +1,22 @@ #! /usr/bin/env perl use strict; use warnings; -use Term::ANSIColor; -use Text::Wrap; -use Encode; use utf8; # Because the script itself is UTF-8 encoded +use Encode; +use Text::Wrap; +use Term::ANSIColor; + +BEGIN { + if ($^O=~/^MSWin/) { + require Win32::Console::ANSI; + Win32::Console::ANSI::->import(); + } +} ##### Version information -my $versionnumber="2.3"; -my $versiondate="2011 Jul 30"; +my $versionnumber="3.0"; +my $versiondate="2013 Jul 29"; ###### Set global settings and variables @@ -17,9 +24,9 @@ my $versiondate="2011 Jul 30"; my %GLOBALDATA= ('versionnumber' => $versionnumber ,'versiondate' => $versiondate - ,'maintainer' => "Einar Andreas Rodland" - ,'copyrightyears' => "2008-2011" - ,'website' => "http://app.uio.no/ifi/texcount/" + ,'maintainer' => 'Einar Andreas Rodland' + ,'copyrightyears' => '2008-2013' + ,'website' => 'http://app.uio.no/ifi/texcount/' ); ### Options and states @@ -29,28 +36,32 @@ my $Main=getMain(); # Global options and settings my $htmlstyle=0; # Flag to print HTML +my $texcodeoutput=0; # Flag to convert output to valid TeX text my $encoding=undef; # Selected input encoding (default will be guess) my @encodingGuessOrder=qw/ascii utf8 latin1/; # Encoding guessing order my $outputEncoding; # Encoding used for output my @AlphabetScripts=qw/Digit Is_alphabetic/; # Letters minus logograms: defined later my @LogogramScripts=qw/Ideographic Katakana Hiragana Thai Lao Hangul/; # Scripts counted as whole words -my $WordPattern; # Regex matching a word (defined in apply_language_options()) # Parsing rules options my $includeTeX=0; # Flag to parse included files my $includeBibliography=0; # Flag to include bibliography my %substitutions; # Substitutions to make globally +my %IncludedPackages; # List of included packages # Counting options my @sumweights; # Set count weights for computing sum my $optionWordFreq=0; # Count words of this frequency, or don't count if 0 my $optionWordClassFreq=0; # Count words per word class (language) if set +my $optionMacroStat=0; # Count macro, environment and package usage # Parsing details options -my $strictness=0; # Flag to check for undefined groups -my $verbose=0; # Level of verbosity -my $showcodes=1; # Flag to show overview of colour codes (2=force) +my $strictness=0; # Flag to check for undefined environments +my $defaultVerbosity='0'; # Specification of default verbose output style +my $defaultprintlevel=0; # Flag indicating default level of verbose output +my $printlevel=undef; # Flag indicating level of verbose output my $showstates=0; # Flag to show internal state in verbose log +my $showcodes=1; # Flag to show overview of colour codes (2=force) my $showsubcounts=0; # Write subcounts if #>this, or not (if 0) my $separatorstyleregex='^word'; # Styles (regex) after which separator should be added my $separator=''; # Separator to add after words/tokens @@ -70,29 +81,47 @@ my $blankline=0; # Number of blank lines printed my $errorcount=0; # Number of errors in parsing my %warnings=(); # Warnings my %WordFreq; # Hash for counting words +my %MacroUsage; # Hash for counting macros, environments and packages + +# External sources +my $HTMLfile; # HTML file to use as HTML output template +my $CSSfile; # CSS file to use with HTML output +my $CSShref; # CSS reference to use with HTML output +my @htmlhead; # Lines to add to the HTML header +my $htmlopen; # text used to open the HTML file +my $htmlclose; # text used to close the HTML file # String data storage my $STRINGDATA; +# Other constants +my $_PARAM_='<param>'; # to identify parameter to _parse_unit +my $STRING_PARAMETER='{_}'; # used to log macro parameters +my $STRING_OPTIONAL_PARAM='[_]'; # used to log optional parameter +my $STRING_GOBBLED_OPTION='[]'; # used to log gobbled macro option +my $STRING_ERROR='<error>'; # used to log errors causing parsing to stop +my $REGEX_NUMBER=qr/^\d+$/; # regex to recognize a number + ###### Set CMD specific settings and variables ## Preset command line options # List of options (stings) separated by comma, -# e.g. ("-inc","-v") to parse included files and +# e.g. ('-inc','-v') to parse included files and # give verbose output by default. my @StartupOptions=(); # CMD specific global variables my @filelist; # List of files to parse -my $workdir; # Root directory (taken from filename) -my $globalworkdir=""; # Overrules workdir (default=present root) +my $globalworkdir='./'; # Overrules workdir (default=present root) +my $workdir; # Current directory (default=taken from filename) +my $auxdir; # Directory for auxilary files, e.g. bbl (default=$workdir) my $fileFromSTDIN=0; # Flag to set input from STDIN -my $_STDIN_="<STDIN>"; # File name to represent STDIN +my $_STDIN_='<STDIN>'; # File name to represent STDIN (must be '<...>'!) # CMD specific settings $Text::Wrap::columns=76; # Page width for wrapped output -###### Set status identifiers and methods +###### Set state identifiers and methods ### Counter indices from 0 to $SIZE_CNT-1 @@ -105,125 +134,227 @@ $Text::Wrap::columns=76; # Page width for wrapped output # 6: Number of inlined math # 7: Number of displayed math my $SIZE_CNT=8; +my $SIZE_CNT_DEFAULT=8; my $CNT_FILE=0; my $CNT_WORDS_TEXT=1; my $CNT_WORDS_HEADER=2; -my $CNT_WORDS_CAPTION=3; +my $CNT_WORDS_OTHER=3; my $CNT_COUNT_HEADER=4; my $CNT_COUNT_FLOAT=5; my $CNT_COUNT_INLINEMATH=6; my $CNT_COUNT_DISPLAYMATH=7; # Labels used to describe the counts -my @countlabel=('Files','Words in text','Words in headers', - 'Words in float captions','Number of headers','Number of floats', +my @countkey=('file','word','hword','oword','header','float','inmath','dsmath'); +my @countdesc=('Files','Words in text','Words in headers', + 'Words outside text (captions, etc.)','Number of headers','Number of floats/tables/figures', 'Number of math inlines','Number of math displayed'); -### Parsing statuses -## Note that status numbers are used in rule definitions and should -## not be changed. +# Map keywords to counters +my %key2cnt; +add_keys_to_hash(\%key2cnt,$CNT_FILE,0,'file'); +add_keys_to_hash(\%key2cnt,$CNT_WORDS_TEXT,1,'text','word','w','wd'); +add_keys_to_hash(\%key2cnt,$CNT_WORDS_HEADER,2,'headerword','hword','hw','hwd'); +add_keys_to_hash(\%key2cnt,$CNT_WORDS_OTHER,3,'otherword','other','oword','ow','owd'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_HEADER,4,'header','heading','head'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_FLOAT,5,'float','table','figure'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_INLINEMATH,6,'inline','inlinemath','imath','eq'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_DISPLAYMATH,7,'displaymath','dsmath','dmath','ds'); + + +### Token types +# Set in $tex->{'type'} by call to _next_token +my $TOKEN_SPACE=-1; +my $TOKEN_COMMENT=0; +my $TOKEN_WORD=1; # word (or other form of text or text component) +my $TOKEN_SYMBOL=2; # symbol (not word, e.g. punctuation) +my $TOKEN_MACRO=3; # macro (\name) +my $TOKEN_BRACE=4; # curly braces: { } +my $TOKEN_BRACKET=5; # brackets: [ ] +my $TOKEN_MATH=6; +my $TOKEN_LINEBREAK=9; # line break in file +my $TOKEN_TC=666; # TeXcount instructions (%TC:instr) +my $TOKEN_END=999; # end of line or blank line + + +### Parsing states # -## Status for regions that should not be counted -# 0 = exclude from count -## Statuses for regions in which words should not be counted -# -1 = float (exclude, but include captions) -# -2 = strong exclude, ignore begin-end groups -# -3 = stronger exclude, do not parse macro parameters -# -4 = ignore everything except end marker: even { -# -9 = preamble (between \documentclass and \begin{document}) -## Statuses for regions in which words should be counted -# 1 = text -# 2 = header text -# 3 = float text -## Status change: not used in parsing, but to switch status then ignore contents -# 6 = switch to inlined math -# 7 = switch to displayed math -## Note that positive statuses must correspond to CNT codes! +## States for regions that should not be counted +# IGNORE = exclude from count +# FLOAT = float (exclude, but include captions) +# EXCLUDE_STRONG = strong exclude, ignore environments +# EXCLUDE_STRONGER = stronger exclude, do not parse macro parameters +# EXCLUDE_ALL = ignore everything except end marker: even { +# PREAMBLE = preamble (between \documentclass and \begin{document}) +## States for regions in which words should be counted +# TEXT = text +# TEXT_HEADER = header text +# TEXT_FLOAT = float text +## State change: not used in parsing, but to switch state then ignore contents +# TO_INLINEMATH = switch to inlined math +# TO_DISPLAYMATH = switch to displayed math +## Other states +# _NULL = default state to use if none other is defined +# _OPTION = state used to indicate that the next parameter is an option +# _EXCLUDE_ = cutoff, state <= this represents excluded text +## NB: Presently, it is assumed that additional states is added as 8,9,..., +## e.g. that states added through TC:newcounter correspond to the added counters. # -my $STATUS_IGNORE=0; -my $STATUS_FLOAT=-1; -my $STATUS_EXCLUDE_STRONG=-2; -my $STATUS_EXCLUDE_STRONGER=-3; -my $STATUS_EXCLUDE_ALL=-4; -my $STATUS_PREAMBLE=-9; -my $STATUS_TEXT=1; -my $STATUS_TEXT_HEADER=2; -my $STATUS_TEXT_FLOAT=3; -my $STATUS_TO_INLINEMATH=6; -my $STATUS_TO_DISPLAYMATH=7; - -# When combining two statuses, use the first one; list must be complete! -my @STATUS_PRIORITY_LIST=(-4,-3,-2,-1,0,-9,3,2,1); - -# Status: is a text status..."include status" is more correct -sub status_is_text { +my $STATE_IGNORE=-1; +my $STATE_MATH=-2; +my $STATE_FLOAT=-10; +my $STATE_EXCLUDE_STRONG=-20; +my $STATE_EXCLUDE_STRONGER=-30; +my $STATE_EXCLUDE_ALL=-40; +my $STATE_PREAMBLE=-99; +my $STATE_TEXT=1; +my $STATE_TEXT_HEADER=2; +my $STATE_TEXT_FLOAT=3; +my $STATE_TO_HEADER=4; +my $STATE_TO_FLOAT=5; +my $STATE_TO_INLINEMATH=6; +my $STATE_TO_DISPLAYMATH=7; +my $__STATE_EXCLUDE_=-10; +my $__STATE_NULL=1; +my $_STATE_OPTION=-1000; +my $_STATE_NOOPTION=-1001; +my $_STATE_AUTOOPTION=-1002; + +# Counter key mapped to STATE +my $PREFIX_PARAM_OPTION=' '; # Prefix for parameter options/modifiers +my %key2state; +add_keys_to_hash(\%key2state,$STATE_TEXT,1,'text','word','w','wd'); +add_keys_to_hash(\%key2state,$STATE_TEXT_HEADER,2,'headertext','headerword','hword','hw','hwd'); +add_keys_to_hash(\%key2state,$STATE_TEXT_FLOAT,3,'otherword','other','oword','ow','owd'); +add_keys_to_hash(\%key2state,$STATE_TO_HEADER,4,'header','heading','head'); +add_keys_to_hash(\%key2state,$STATE_TO_FLOAT,5,'float','table','figure'); +add_keys_to_hash(\%key2state,$STATE_TO_INLINEMATH,6,'inline','inlinemath','imath','eq'); +add_keys_to_hash(\%key2state,$STATE_TO_DISPLAYMATH,7,'displaymath','dsmath','dmath','ds'); +add_keys_to_hash(\%key2state,$STATE_IGNORE,0,'ignore','x'); +add_keys_to_hash(\%key2state,$STATE_MATH,'ismath'); +add_keys_to_hash(\%key2state,$STATE_FLOAT,-1,'isfloat'); +add_keys_to_hash(\%key2state,$STATE_EXCLUDE_STRONG,-2,'xx'); +add_keys_to_hash(\%key2state,$STATE_EXCLUDE_STRONGER,-3,'xxx'); +add_keys_to_hash(\%key2state,$STATE_EXCLUDE_ALL,-4,'xall'); +add_keys_to_hash(\%key2state,$_STATE_OPTION,'[',' option',' opt',' optional'); +add_keys_to_hash(\%key2state,$_STATE_NOOPTION,'nooption','nooptions','noopt','noopts'); +add_keys_to_hash(\%key2state,$_STATE_AUTOOPTION,'autooption','autooptions','autoopt','autoopts'); + +# When combining two states, use the first one; list must be complete! +my @STATE_FIRST_PRIORITY=( + $STATE_EXCLUDE_ALL, + $STATE_EXCLUDE_STRONGER, + $STATE_EXCLUDE_STRONG, + $STATE_FLOAT, + $STATE_MATH, + $STATE_IGNORE, + $STATE_PREAMBLE, + $STATE_TO_FLOAT, + $STATE_TO_HEADER, + $STATE_TO_INLINEMATH, + $STATE_TO_DISPLAYMATH); +my @STATE_MID_PRIORITY=(); +my @STATE_LAST_PRIORITY=( + $STATE_TEXT_FLOAT, + $STATE_TEXT_HEADER, + $STATE_TEXT); + +# Map state to corresponding word counter +my %state2cnt=( + $STATE_TEXT => $CNT_WORDS_TEXT, + $STATE_TEXT_HEADER => $CNT_WORDS_HEADER, + $STATE_TEXT_FLOAT => $CNT_WORDS_OTHER); + +# Transition state mapped to content state and counter +my %transition2state=( + $STATE_TO_HEADER => [$STATE_TEXT_HEADER,$CNT_COUNT_HEADER], + $STATE_TO_INLINEMATH => [$STATE_MATH ,$CNT_COUNT_INLINEMATH], + $STATE_TO_DISPLAYMATH => [$STATE_MATH ,$CNT_COUNT_DISPLAYMATH], + $STATE_TO_FLOAT => [$STATE_FLOAT ,$CNT_COUNT_FLOAT]); + +# Parsing state descriptions (used for macro rule help) +my %state2desc=( + $STATE_IGNORE => 'ignore: do not count', + $STATE_MATH => 'math/equation contents', + $STATE_FLOAT => 'float (figure, etc.): ignore all but special macros', + $STATE_EXCLUDE_STRONG => 'strong exclude: ignore environments', + $STATE_EXCLUDE_STRONGER => 'stronger exclude: ignore environments and macro paramters', + $STATE_EXCLUDE_ALL => 'exlude all: even {, only scan for end marker', + $STATE_PREAMBLE => 'preamble: from \documentclass to \begin{document}', + $STATE_TEXT => 'text: count words', + $STATE_TEXT_HEADER => 'header text: count words as header words', + $STATE_TEXT_FLOAT => 'float text: count words as float words (e.g. captions)', + $STATE_TO_HEADER => 'header: count header, then count words as header words', + $STATE_TO_FLOAT => 'float: count float, then count words as float/other words', + $STATE_TO_INLINEMATH => 'inline math: count as inline math/equation', + $STATE_TO_DISPLAYMATH => 'displayed math: count as displayed math/equation'); + +# Parsing state presentation style +my %state2style=( + $STATE_TEXT => 'word', + $STATE_TEXT_HEADER => 'hword', + $STATE_TEXT_FLOAT => 'oword', + ); + +# State: is a text state..."include state" is more correct +sub state_is_text { my $st=shift @_; - return ($st>0); + return ($st>=$STATE_TEXT); } -# Status: get CNT corresponding to text status (or undef) -sub status_text_cnt { +# State: is a parsed/included region, text or preamble +sub state_is_parsed { my $st=shift @_; - if ($st==$STATUS_TEXT) {return $CNT_WORDS_TEXT;} - if ($st==$STATUS_TEXT_HEADER) {return $CNT_WORDS_HEADER;} - if ($st==$STATUS_TEXT_FLOAT) {return $CNT_WORDS_CAPTION;} - return undef; + return ($st>=$STATE_TEXT || $st==$STATE_PREAMBLE); } -# Status: is an exclude status -sub status_is_exclude { +# State: get CNT corresponding to text state (or undef) +sub state_text_cnt { my $st=shift @_; - return ($st<0); + return $state2cnt{$st}; } -# Status: \begin and \end should be processed -sub status_inc_envir { +# State: is an exclude state +sub state_is_exclude { my $st=shift @_; - return ($st>-2); + return ($st<=$__STATE_EXCLUDE_); } -# Map status number to CNT value -sub status_to_cnt { +# State: \begin and \end should be processed +sub state_inc_envir { my $st=shift @_; - if ($st==$STATUS_TO_INLINEMATH) {return $CNT_COUNT_INLINEMATH;} - if ($st==$STATUS_TO_DISPLAYMATH) {return $CNT_COUNT_DISPLAYMATH;} - return undef; + return ($st>$STATE_EXCLUDE_STRONG); } -# Status as text -sub status_to_text { +# State as text (used with printstate) +# TODO: Should do a conversion based on STATE values. +sub state_to_text { my $st=shift @_; return $st; } -# Status as text -sub status_to_style { - return 'word'.status_to_text(@_); +# Style to use with text state +sub state_to_style { + return $state2style{shift @_}; } -### Token types -# -1: space -# 0: comment -# 1: word (or other forms of text or text components) -# 2: symbol (not word, e.g. punctuation) -# 3: macro -# 4: curly braces {} -# 5: brackets [] -# 6: maths -# 9: line break in file -# 999: end of line or blank line -# 666: TeXcount instruction (%TC:instruction) -my $TOKEN_SPACE=-1; -my $TOKEN_COMMENT=0; -my $TOKEN_WORD=1; -my $TOKEN_SYMBOL=2; -my $TOKEN_MACRO=3; -my $TOKEN_BRACE=4; -my $TOKEN_BRACKET=5; -my $TOKEN_MATH=6; -my $TOKEN_LINEBREAK=9; -my $TOKEN_TC=666; -my $TOKEN_END=999; +# Add new counter with the given key and description +sub add_new_counter { + my ($key,$desc,$like)=@_; + my $state=$SIZE_CNT; + my $cnt=$SIZE_CNT; + $key=lc($key); + if (!defined $like){$like=$CNT_WORDS_OTHER;} + $key2cnt{$key}=$cnt; + push @countkey,$key; + push @countdesc,$desc; + if (defined $sumweights[$like]) {$sumweights[$cnt]=$sumweights[$like];} + $key2state{$key}=$state; + $state2cnt{$state}=$cnt; + $state2style{$state}='altwd'; + push @STATE_MID_PRIORITY,$state; + $SIZE_CNT++; +} ###### Set global definitions @@ -243,20 +374,55 @@ my %BreakPoints=%{$BreakPointsOptions{'none'}}; ### Print styles # Definition of different print styles: maps of class labels # to ANSI codes. Class labels are as used by HTML styles. -my @STYLES=(); -my %STYLE; -$STYLES[0]={'error'=>'bold red'}; -$STYLES[1]={%{$STYLES[0]},''=>'normal',' '=>'normal', - 'word1'=>'blue','word2'=>'bold blue','word3'=>'blue', - 'grouping'=>'red','document'=>'red','mathgroup'=>'magenta', - 'state'=>'cyan underline','cumsum'=>'yellow'}; -$STYLES[2]={%{$STYLES[1]}, - 'command'=>'green','ignore'=>'cyan', - 'exclcommand'=>'yellow','exclgroup'=>'yellow','exclmath'=>'yellow'}; -$STYLES[3]={%{$STYLES[2]}, - 'tc'=>'bold yellow','comment'=>'yellow','option'=>'yellow', - 'fileinclude'=>'bold green'}; -$STYLES[4]={%{$STYLES[3]}}; +my %STYLES; +my $STYLE_EMPTY=' '; +my $STYLE_BLOCK='-'; +my $NOSTYLE=' '; +$STYLES{'Errors'}={'error'=>'bold red'}; +$STYLES{'Words'}={'word'=>'blue','hword'=>'bold blue','oword'=>'blue','altwd'=>'blue'}; +$STYLES{'Macros'}={'cmd'=>'green','fileinc'=>'bold green'}; +$STYLES{'Options'}={'option'=>'yellow','optparm'=>'green'}; +$STYLES{'Ignored'}={'ignore'=>'cyan','math'=>'magenta'}; +$STYLES{'Excluded'}={'exclcmd'=>'yellow','exclenv'=>'yellow','exclmath'=>'yellow','mathcmd'=>'yellow'}; +$STYLES{'Groups'}={'document'=>'red','envir'=>'red','mathgroup'=>'magenta'}; +$STYLES{'Comments'}={'tc'=>'bold yellow','comment'=>'yellow'}; +$STYLES{'Sums'}={'cumsum'=>'yellow'}; +$STYLES{'States'}={'state'=>'cyan underline'}; +$STYLES{'<core>'}={%{$STYLES{'Errors'}},$STYLE_EMPTY=>$NOSTYLE,'<printlevel>'=>1}; +$STYLES{0}={%{$STYLES{'Errors'}},'<printlevel>'=>0}; +$STYLES{1}={%{$STYLES{'<core>'}},%{$STYLES{'Words'}},%{$STYLES{'Groups'}},%{$STYLES{'Sums'}}}; +$STYLES{2}={%{$STYLES{1}},%{$STYLES{'Macros'}},%{$STYLES{'Ignored'}},%{$STYLES{'Excluded'}}}; +$STYLES{3}={%{$STYLES{2}},%{$STYLES{'Options'}},%{$STYLES{'Comments'}},'<printlevel>'=>2}; +$STYLES{4}={%{$STYLES{3}},%{$STYLES{'States'}}}; +$STYLES{'All'}=$STYLES{4}; +my %STYLE=%{$STYLES{$defaultVerbosity}}; + +my @STYLE_LIST=('error','word','hword','oword','altwd', + 'ignore','document','cmd','exclcmd','option','optparm','envir','exclenv', + 'mathgroup','exclmath','math','mathcmd','comment','tc','fileinc','state','cumsum'); +my %STYLE_DESC=( + 'error' => 'ERROR: TeXcount error message', + 'word' => 'Text which is counted: counted as text words', + 'hword' => 'Header and title text: counted as header words', + 'oword' => 'Caption text and footnotes: counted as caption words', + 'altwd' => 'Words in user specified counters: counted in separate counters', + 'ignore' => 'Ignored text or code: excluded or ignored', + 'document' => '\documentclass: document start, beginning of preamble', + 'cmd' => '\macro: macro not counted, but parameters may be', + 'exclcmd' => '\macro: macro in excluded region', + 'option' => '[Macro options]: not counted', + 'optparm' => '[Optional parameter]: content parsed and styled as counted', + 'envir' => '\begin{name} \end{name}: environment', + 'exclenv' => '\begin{name} \end{name}: environment in excluded region', + 'mathgroup' => '$ $: counted as one equation', + 'exclmath' => '$ $: equation in excluded region', + 'math' => '2+2=4: maths (inside $...$ etc.)', + 'mathcmd' => '$\macro$: macros inside maths', + 'comment' => '% Comments: not counted', + 'tc' => '%TC:TeXcount instructions: not counted', + 'fileinc' => 'File to include: not counted but file may be counted later', + 'state' => '[state]: internal TeXcount state', + 'cumsum' => '[cumsum]: cumulative sum count'); ###### Define what a word is and language options @@ -279,18 +445,22 @@ my $LetterPattern=$NamedLetterPattern{'default'}; # List of regexp patterns that should be analysed as words. # Use @ to represent a letter, will be substituted with $LetterPattern. # Named patterns may replace or be appended to the original patterns. +# Apply_Options() results in a call to apply_language_options() which +# constructs $WordPattern based on $LetterPattern, @WordPatterns and +# alphabet/logogram settings. my %NamedWordPattern; $NamedWordPattern{'letters'}='@'; -$NamedWordPattern{'words'}='(@+|\{@+\})([\-\'\.]?(@+|\{@+\}))*'; +$NamedWordPattern{'words'}='(@+|@+\{@+\}|\{@+\}@+)([\-\'\.]?(@+|\{@+\}))*'; my @WordPatterns=($NamedWordPattern{'words'}); +my $WordPattern; # Regex matching a word (defined in apply_language_options()) ### Macro option regexp list # List of regexp patterns to be gobbled as macro option in and after # a macro. my %NamedMacroOptionPattern; -$NamedMacroOptionPattern{'default'}='\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]'; -$NamedMacroOptionPattern{'relaxed'}='\[[^\[\]\n]*\]'; -$NamedMacroOptionPattern{'restricted'}=$NamedMacroOptionPattern{'default'}; +$NamedMacroOptionPattern{'default'}='\[[^\[\]\n]*\]'; +$NamedMacroOptionPattern{'relaxed'}='\[[^\[\]\n]*(\n[^\[\]\n]+)\n?\]'; +$NamedMacroOptionPattern{'restricted'}='\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]'; my $MacroOptionPattern=$NamedMacroOptionPattern{'default'}; ### Alternative language encodings @@ -367,133 +537,287 @@ END ###### Define core rules -### Macros for headers -# Macros that identify headers: i.e. following token or -# {...} is counted as header. The =>[2] indicates transition to -# state 2 which is used within headers (although the value is -# actually never used). This is copied to %TeXmacro and the -# only role of defining it here is that the counter for the number -# of headers is incremented by one. -my %TeXheader=('\title'=>[2],'\part'=>[2],'\chapter'=>[2], - '\section'=>[2],'\subsection'=>[2],'\subsubsection'=>[2], - '\paragraph'=>[2],'\subparagraph'=>[2]); - ### Macros indicating package inclusion # Will always be assumed to take one parameter (plus options). -my %TeXpackageinc=('\usepackage'=>1); +# Gets added to TeXmacro. After that, values are not used, only membership. +my %TeXpackageinc=('\usepackage'=>1,'\RequirePackage'=>1); ### Macros that are counted within the preamble # The preamble is the text between \documentclass and \begin{document}. # Text and macros in the preamble is ignored unless specified here. The -# value is the status (1=text, 2=header, etc.) they should be interpreted as. +# value is the states (1=text, 2=header, etc.) they should be interpreted as. # Note that only the first unit (token or {...} block) is counted. -my %TeXpreamble=( - '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3], - '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3], - '\title'=>[2]); +# Gets added to TeXmacro. Is used within preambles only. +my %TeXpreamble; +add_keys_to_hash(\%TeXpreamble,['header'],'\title'); +add_keys_to_hash(\%TeXpreamble,['other'],'\thanks'); +add_keys_to_hash(\%TeXpreamble,['xxx','xxx'],'\newcommand','\renewcommand'); +add_keys_to_hash(\%TeXpreamble,['xxx','xxx','xxx'],'\newenvironment','\renewenvironment'); ### In floats: include only specific macros # Macros used to identify caption text within floats. -my %TeXfloatinc=('\caption'=>[3]); +# Gets added to TeXmacro. Is used within floats only. +my %TeXfloatinc=('\caption'=>['otherword']); ### How many tokens to gobble after macro # Each macro is assumed to gobble up a given number of # tokens (or {...} groups), as well as options [...] before, within # and after. The %TeXmacro hash gives a link from a macro -# (or beginNAME for begin-end groups without the backslash) +# (or beginNAME for environment with no the backslash) # to either an integer giving the number of tokens to ignore -# or to an array (specified as [num,num,...]) of length N where -# N is the number of tokens to be read with the macro and the -# array values tell how each is to be interpreted (see the status -# values: 0=ignore, 1=count, etc.). Thus specifying a number N is -# equivalent to specifying an array [0,...,0] of N zeros. +# or to an array (specified as [rule,rule,...]) of length N where +# N is the number of parameters to be read with the macro. The +# array values tell how each is to be interpreted (see the parser state +# keywords for valid values). Thus specifying a number N is +# equivalent to specifying an array of N 'ignore' rules. # # For macros not specified here, the default value is 0: i.e. -# no tokens are excluded, but [...] options are. Header macros -# specified in %TeXheader are automatically included here. -my %TeXmacro=(%TeXheader,%TeXpreamble,%TeXfloatinc, - '\documentclass'=>1,'\documentstyle'=>1,'\usepackage'=>1, '\hyphenation'=>1, - '\pagestyle'=>1,'\thispagestyle'=>1, '\pagenumbering'=>1,'\markboth'=>1, '\markright'=>1, - '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3], - '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3], - '\newfont'=>2,'\newtheorem'=>2,'\bibliographystyle'=>1, '\bibliography'=>1, - '\parbox'=>1, '\marginpar'=>[3],'\makebox'=>0, '\raisebox'=>1, '\framebox'=>0, - '\newsavebox'=>1, '\sbox'=>1, '\savebox'=>2, '\usebox'=>1,'\rule'=>2, - '\footnote'=>[3],'\label'=>1, '\ref'=>1, '\pageref'=>1, '\bibitem'=>1, - '\cite'=>1, '\citep'=>1, '\citet'=>1, '\citeauthor'=>1, '\citealt'=>1, '\nocite'=>1, - '\eqlabel'=>1, '\eqref'=>1,'\hspace'=>1, '\vspace'=>1, '\addvspace'=>1, - '\input'=>1, '\include'=>1, '\includeonly'=>1,'\includegraphics'=>1, - '\newlength'=>1, '\setlength'=>2, '\addtolength'=>2,'\settodepth'=>2, - '\settoheight'=>2, '\settowidth'=>2,'\newcounter'=>1, '\setcounter'=>2, - '\addtocounter'=>2,'\stepcounter'=>1, '\refstepcounter'=>1, '\usecounter'=>1, - '\alph'=>1, '\arabic'=>1, '\fnsymbol'=>1, '\roman'=>1, '\value'=>1, - '\cline'=>1, '\multicolumn'=>3,'\typeout'=>1, '\typein'=>1, - 'beginlist'=>2, 'beginminipage'=>1, 'begintabular'=>1, - 'beginthebibliography'=>1,'beginlrbox'=>1, - '\begin'=>1,'\end'=>1,'\title'=>[2], - '\addtocontents'=>2,'\addcontentsline'=>3, - '\uppercase'=>0,'\lowercase'=>0); - -### Macros that should be counted as one or more words -# Macros that represent text may be declared here. The value gives -# the number of words the macro represents. -my %TeXmacroword=('\LaTeX'=>1,'\TeX'=>1); - -### Begin-End groups -# Identified as begin-end groups, and define =>state. The -# states used corresponds to the elements of the count array, and -# are: -# 0: Not included -# 1: Text, words included in text count -# 2: Header, words included in header count -# 3: Float caption, words included in float caption count -# 6: Inline mathematics, words not counted -# 7: Displayed mathematics, words not counted -# -1: Float, not included, but looks for captions +# no tokens are excluded, but [...] options are. +my %TeXmacro=(%TeXpreamble,%TeXfloatinc,%TeXpackageinc); +add_keys_to_hash(\%TeXmacro,['text'], + '\textnormal','\textrm','\textit','\textbf','\textsf','\texttt','\textsc','\textsl','\textup','\textmd', + '\makebox','\mbox','\framebox','\fbox','\uppercase','\lowercase','\textsuperscript','\textsubscript', + '\citetext'); +add_keys_to_hash(\%TeXmacro,['[','text'], + '\item'); +add_keys_to_hash(\%TeXmacro,['[','ignore'], + '\linebreak','\nolinebreak','\pagebreak','\nopagebreak'); +add_keys_to_hash(\%TeXmacro,0, + '\maketitle','\indent','\noindent', + '\centering','\raggedright','\raggedleft','\clearpage','\cleardoublepage','\newline','\newpage', + '\smallskip','\medskip','\bigskip','\vfill','\hfill','\hrulefill','\dotfill', + '\normalsize','\small','\footnotesize','\scriptsize','\tiny','\large','\Large','\LARGE','\huge','\Huge', + '\normalfont','\em','\rm','\it','\bf','\sf','\tt','\sc','\sl', + '\rmfamily','\sffamily','\ttfamily','\upshape','\itshape','\slshape','\scshape','\mdseries','\bfseries', + '\selectfont', + '\tableofcontents','\listoftables','\listoffigures'); +add_keys_to_hash(\%TeXmacro,1, + '\begin','\end', + '\documentclass','\documentstyle','\hyphenation','\pagestyle','\thispagestyle', + '\author','\date', + '\bibliographystyle','\bibliography','\pagenumbering','\markright', + '\includeonly','\includegraphics','\special', + '\label','\ref','\pageref','\bibitem', + '\eqlabel','\eqref','\hspace','\vspace','\addvspace', + '\newsavebox','\usebox', + '\newlength','\newcounter','\stepcounter','\refstepcounter','\usecounter', + '\fontfamily','\fontseries', + '\alph','\arabic','\fnsymbol','\roman','\value', + '\typeout', '\typein','\cline'); +add_keys_to_hash(\%TeXmacro,2, + '\newfont','\newtheorem','\sbox','\savebox','\rule','\markboth', + '\setlength','\addtolength','\settodepth','\settoheight','\settowidth','\setcounter', + '\addtocontents','\addtocounter', + '\fontsize'); +add_keys_to_hash(\%TeXmacro,3,'\multicolumn','\addcontentsline'); +add_keys_to_hash(\%TeXmacro,6,'\DeclareFontShape'); +add_keys_to_hash(\%TeXmacro,['[','text','ignore'], + '\cite','\nocite','\citep','\citet','\citeauthor','\citeyear','\citeyearpar', + '\citealp','\citealt','\Citep','\Citet','\Citealp','\Citealt','\Citeauthor'); +add_keys_to_hash(\%TeXmacro,['ignore','text'],'\parbox','\raisebox'); +add_keys_to_hash(\%TeXmacro,['otherword'],'\marginpar','\footnote','\footnotetext'); +add_keys_to_hash(\%TeXmacro,['header'], + '\title','\part','\chapter','\section','\subsection','\subsubsection','\paragraph','\subparagraph'); +add_keys_to_hash(\%TeXmacro,['xxx','xxx','text'],'\multicolumn'); +add_keys_to_hash(\%TeXmacro,['xxx','xxx'],'\newcommand','\renewcommand'); +add_keys_to_hash(\%TeXmacro,['xxx','xxx','xxx'],'\newenvironment','\renewenvironment'); + +### Environments +# The %TeXenvir hash provides content parsing rules (parser states). +# Environments that are not defined will be counted as the surrounding text. # -# 4 and 5 are used to count number of headers and floats -# and are not used as states. -# -# Groups that are not defined will be counted as the surrounding text. +# Parameters taken by the \begin{environment} are defined in %TeXmacro. # # Note that some environments may only exist within math-mode, and # therefore need not be defined here: in fact, they should not as it # is not clear if they will be in inlined or displayed math. -my %TeXgroup=('document'=>1,'letter'=>1,'titlepage'=>0, - 'center'=>1,'flushleft'=>1,'flushright'=>1, - 'abstract'=>1,'quote'=>1,'quotation'=>1,'verse'=>1,'minipage'=>1, - 'verbatim'=>-4,'tikzpicture'=>-4, - 'description'=>1,'enumerate'=>1,'itemize'=>1,'list'=>1, - 'theorem'=>1,'lemma'=>1,'definition'=>1,'corollary'=>1,'example'=>1, - 'math'=>6,'displaymath'=>7,'equation'=>7,'eqnarray'=>7,'align'=>7, - 'equation*'=>7,'eqnarray*'=>7,'align*'=>7, - 'figure'=>-1,'float'=>-1,'picture'=>-1,'table'=>-1, - 'tabbing'=>0,'tabular'=>0,'thebibliography'=>0,'lrbox'=>0); +my %TeXenvir; +add_keys_to_hash(\%TeXenvir,'ignore', + 'titlepage','tabbing','tabular','tabular*','thebibliography','lrbox'); +add_keys_to_hash(\%TeXenvir,'text', + 'document','letter','center','flushleft','flushright', + 'abstract','quote','quotation','verse','minipage', + 'description','enumerate','itemize','list', + 'theorem','thm','lemma','definition','corollary','example','proof','pf'); +add_keys_to_hash(\%TeXenvir,'inlinemath', + 'math'); +add_keys_to_hash(\%TeXenvir,'displaymath', + 'displaymath','equation','equation*','eqnarray','eqnarray*','align','align*',); +add_keys_to_hash(\%TeXenvir,'float', + 'float','picture','figure','figure*','table','table*'); +add_keys_to_hash(\%TeXenvir,'xall', + 'verbatim','tikzpicture'); + +# Environment parameters +my $PREFIX_ENVIR='begin'; # Prefix used for environment names +add_keys_to_hash(\%TeXmacro,1, + 'beginthebibliography','beginlrbox','beginminipage'); +add_keys_to_hash(\%TeXmacro,2, + 'beginlist'); +add_keys_to_hash(\%TeXmacro,['ignore'], + 'beginletter'); +add_keys_to_hash(\%TeXmacro,['xxx'], + 'begintabular'); +add_keys_to_hash(\%TeXmacro,['ignore','xxx'], + 'begintabular*'); +add_keys_to_hash(\%TeXmacro,['[','text'], + 'begintheorem','beginthm','beginlemma','begindefinition','begincorollary','beginexample','beginproof','beginpf'); +add_keys_to_hash(\%TeXmacro,['nooptions'], + 'beginverbatim'); + +### Macros that should be counted as one or more words +# Macros that represent text may be declared here. The value gives +# the number of words the macro represents. +my %TeXmacrocount=('\LaTeX'=>1,'\TeX'=>1,'beginabstract'=>['header','headerword']); ### Macros for including tex files # Allows \macro{file} or \macro file. If the value is 0, the filename will # be used as is; if it is 1, the filetype .tex will be added if the # filename is without filetype; if it is 2, the filetype .tex will be added. -my %TeXfileinclude=('\input'=>1,'\include'=>2); +my %TeXfileinclude=('\input'=>'input','\include'=>'texfile'); + +### Convert state keys to codes +convert_hash(\%TeXpreamble,\&keyarray_to_state); +convert_hash(\%TeXfloatinc,\&keyarray_to_state); +convert_hash(\%TeXmacro,\&keyarray_to_state); +convert_hash(\%TeXmacrocount,\&keyarray_to_cnt); +convert_hash(\%TeXenvir,\&key_to_state); ###### Define package specific rules ### Package rule definitions +my %PackageTeXpreamble=(); # TeXpreamble definitions per package +my %PackageTeXpackageinc=(); # TeXpackageinc definitions per package +my %PackageTeXfloatinc=(); # TeXfloatinc definitions per package my %PackageTeXmacro=(); # TeXmacro definitions per package -my %PackageTeXmacroword=(); # TeXmacroword definitions per package -my %PackageTeXheader=(); # TeXheader definitions per package -my %PackageTeXgroup=(); # TeXgroup definitions per package -my %PackageTeXfileinclude=(); # TeXgroup definitions per package +my %PackageTeXmacrocount=(); # TeXmacrocount definitions per package +my %PackageTeXenvir=(); # TeXenvir definitions per package +my %PackageTeXfileinclude=(); # TeXfileinclude definitions per package +my %PackageSubpackage=(); # Subpackages to include (listed in array [...]) -# Rules for package psfig -$PackageTeXmacro{'psfig'}={('\psfig'=>1)}; # Rules for bibliography inclusion -$PackageTeXmacroword{'%incbib'}={'thebibliography'=>1}; -$PackageTeXmacro{'%incbib'}={}; -$PackageTeXgroup{'%incbib'}={'thebibliography'=>1}; -$PackageTeXfileinclude{'%incbib'}={'\bibliography'=>'bbl'}; +$PackageTeXmacrocount{'%incbib'}={'beginthebibliography'=>['header','hword']}; +$PackageTeXmacro{'%incbib'}={'\bibliography'=>1}; +$PackageTeXenvir{'%incbib'}={'thebibliography'=>'text'}; +$PackageTeXfileinclude{'%incbib'}={'\bibliography'=>'<bbl>'}; + +# Rules for package alltt +$PackageTeXenvir{'alltt'}={ + 'alltt'=>'xall'}; + +# Rules for package babel +# NB: Only core macros implemented, those expected found in regular documents +$PackageTeXenvir{'babel'}={ + 'otherlanguage'=>'text','otherlanguage*'=>'text'}; +$PackageTeXmacro{'babel'}={ + '\selectlanguage'=>1,'\foreignlanguage'=>['ignore','text'], + 'beginotherlanguage'=>1,'beginotherlanguage*'=>1}; + +# Rules for package comment +$PackageTeXenvir{'comment'}={ + 'comment'=>'xxx'}; + +# Rules for package color +$PackageTeXmacro{'color'}={ + '\textcolor'=>['ignore','text'],'\color'=>1,'\pagecolor'=>1,'\normalcolor'=>0, + '\colorbox'=>['ignore','text'],'\fcolorbox'=>['ignore','ignore','text'], + '\definecolor'=>3,\'DefineNamedColor'=>4}; + +# Rules for package endnotes +$PackageTeXmacro{'endnotes'}={'\endnote'=>['oword'],'\endnotetext'=>['oword'],'\addtoendnotetext'=>['oword']}; + +# Rules for package fancyhdr +$PackageTeXmacro{'fancyhdr'}={ + '\fancyhf'=>1,'\lhead'=>1,'\chead'=>1,'\rhead'=>1,'\lfoot'=>1,'\cfoot'=>1,'\rfoot'=>1}; + +# Rules for package geometry +$PackageTeXmacro{'geometry'}={ + '\geometry'=>1,'\newgeometry'=>1,'\restoregeometry'=>0,,'\savegeometry'=>1,'\loadgeometry'=>1}; + +# Rules for package graphicx +$PackageTeXmacro{'graphicx'}={ + '\DeclareGraphicsExtensions'=>1,'\graphicspath'=>1, + '\includegraphics'=>['[','ignore','ignore'], + '\includegraphics*'=>['[','ignore','[','ignore','[','ignore','ignore'], + '\rotatebox'=>1,'\scalebox'=>1,'\reflectbox'=>1,'\resizebox'=>1}; + +# Rules for package hyperref (urls themselves counted as one word) +# NB: \hyperref[label]{text} preferred over \hyperref{url}{category}{name}{text} +# NB: Macros for use in Form environments not implemented +$PackageTeXmacro{'hyperref'}={ + '\hyperref'=>['[','ignore','text'], + '\url'=>1,'\nolinkurl'=>1,'\href'=>['ignore','text'], + '\hyperlink'=>['ignore','text'],'\hypertarget'=>['ignore','text'], + '\hyperbaseurl'=>1,'\hyperimage'=>['ignore','text'],'\hyperdef'=>['ignore','ignore','text'], + '\phantomsection'=>0,'\autoref'=>1,'\autopageref'=>1, + '\hypersetup'=>1,'\urlstyle'=>1, + '\pdfbookmark'=>2,'\currentpdfbookmark'=>2,'\subpdfbookmark'=>2,'\belowpdfbookmark'=>2, + '\pdfstringref'=>2,'\texorpdfstring'=>['text','ignore'], + '\hypercalcbp'=>1,'\Acrobatmenu'=>2}; +$PackageTeXmacrocount{'hyperref'}={ + '\url'=>1,'\nolinkurl'=>1}; + +# Rules for package import +$PackageTeXfileinclude{'import'}={ + '\import'=>'dir file','\subimport'=>'subdir file', + '\inputfrom'=>'dir file','\subinputfrom'=>'subdir file', + '\includefrom'=>'dir file','\subincludefrom'=>'subdir file'}; + +# Rules for package inputenc +$PackageTeXmacro{'inputenc'}={ + '\inputencoding'=>1}; + +# Rules for package listings +$PackageTeXenvir{'listings'}={'lstlisting'=>'xall'}; +$PackageTeXmacro{'listings'}={'\lstset'=>['ignore'],'\lstinputlisting'=>['ignore']}; + +# Rules for package psfig +$PackageTeXmacro{'psfig'}={'\psfig'=>1}; + +# Rules for package sectsty +$PackageTeXmacro{'sectsty'}={ + '\allsectionsfont'=>1,'\partfont'=>1,'\chapterfont'=>1,'\sectionfont'=>1, + '\subsectionfont'=>1,'\subsubsectionfont'=>1,'\paragraphfont'=>1,'\subparagraphfont'=>1, + '\minisecfont'=>1,'\partnumberfont'=>1,'\parttitlefont'=>1,'\chapternumberfont'=>1, + '\chaptertitlefont'=>1,'\nohang'=>0}; + +# Rules for package setspace +$PackageTeXenvir{'setspace'}={ + 'singlespace'=>'text','singlespace*'=>'text','onehalfspace'=>'text','doublespace'=>'text', + 'spacing'=>'text'}; +$PackageTeXmacro{'setspace'}={ + 'beginspacing'=>1, + '\singlespacing'=>0,'\onehalfspacing'=>0,'\doublespacing'=>0, + '\setstretch'=>1,'\SetSinglespace'=>1}; + +# Rules for package subfiles +$PackageTeXfileinclude{'subfiles'}={ + '\subfile'=>'file'}; + +# Rules for package url +# NB: \url|...| variant not implemented, only \url{...} +# NB: \urldef{macro}{url} will not be counted +$PackageTeXmacro{'url'}={ + '\url'=>1,'\urldef'=>2,'\urlstyle'=>1,'\DeclareUrlCommand'=>['ignore','xxx']}; +$PackageTeXmacro{'setspace'}={ + '\url'=>1}; + +# Rules for package wrapfig +$PackageTeXenvir{'wrapfig'}={ + 'wrapfigure'=>'float','wraptable'=>'float'}; +$PackageTeXmacro{'wrapfig'}={ + 'beginwrapfigure'=>2,'beginwraptable'=>2}; + +# Rules for package xcolor (reimplements the color package) +# NB: only main macros (mostly from package color) included +$PackageTeXmacro{'xcolor'}={ + '\textcolor'=>['ignore','text'],'\color'=>1,'\pagecolor'=>1,'\normalcolor'=>0, + '\colorbox'=>['ignore','text'],'\fcolorbox'=>['ignore','ignore','text'], + '\definecolor'=>3,\'DefineNamedColor'=>4, + '\colorlet'=>2}; + ###### Main script @@ -523,13 +847,14 @@ sub MAIN { if ($showVersion && !$htmlstyle && !($briefsum && $totalflag)) { print "\n=== LaTeX word count (TeXcount version $versionnumber) ===\n\n"; } - conditional_print_help_style(); + conditional_print_style_list(); my $totalcount=Parse_file_list(@toplevelfiles); conditional_print_total($totalcount); Report_Errors(); if ($optionWordFreq || $optionWordClassFreq) {print_word_freq();} + if ($optionMacroStat) {print_macro_stat();} } elsif ($showcodes>1) { - conditional_print_help_style(); + conditional_print_style_list(); } else { error($Main,'No files specified.'); } @@ -540,7 +865,9 @@ sub MAIN { sub Initialise { _option_subcount(); # Windows settings - if ($^O=~/^MSWin/) { + if ($^O eq 'MSWin32') { + } elsif ($^O=~/^MSWin/) { + #DELETE: do not overrule colour setting option_ansi_colours(0); } } @@ -548,21 +875,33 @@ sub Initialise { # Check arguments, exit on exit condition sub Check_Arguments { my @args=@_; + my $arg=$args[0]; if (!@args) { print_version(); - print_syntax(); - print_reference(); + print_short_help(); exit; - } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))$/) { + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))$/) { print_help(); exit; - } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))=/) { - print_help_on("$'"); + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))=(.*)$/) { + print_help_on_rule($4); + exit; + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(opt|options?)$/) { + print_syntax(); + exit; + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(opt|options?)=(.*)$/) { + print_syntax_subset($5); exit; - } elsif ($args[0]=~/^--?(ver|version)$/) { + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(styles?)$/) { + print_help_on_styles(); + exit; + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(styles?)=(\w+)$/) { + print_help_on_styles($5); + exit; + } elsif ($arg=~/^--?(ver|version)$/) { print_version(); exit; - } elsif ($args[0]=~/^--?(lic|license|licence)$/) { + } elsif ($arg=~/^--?(lic|license|licence)$/) { print_license(); exit; } @@ -576,8 +915,8 @@ sub Parse_Arguments { foreach my $arg (@args) { if (parse_option($arg)) {next;} if ($arg=~/^\-/) { - print 'Invalid opton '.$arg."\n"; - print_syntax(); + print "Invalid opton $arg \n\n"; + print_short_help(); exit; } $arg=~s/\\/\//g; @@ -600,8 +939,8 @@ sub parse_option { # Parse presetting options sub parse_options_preset { my $arg=shift @_; - if ($arg=~/^-(opt|option|options|optionfile)=/) { - _parse_optionfile($'); + if ($arg=~/^-(opt|option|options|optionfile)=(.*)$/) { + _parse_optionfile($2); } else {return 0;} return 1; @@ -622,6 +961,11 @@ sub parse_options_parsing { $globalworkdir=$1; $globalworkdir=~s:([^\/\\])$:$1\/:; } + elsif ($arg eq '-auxdir') {$auxdir=undef;} + elsif ($arg=~/^-auxdir=(.*)$/) { + $auxdir=$1; + $auxdir=~s:([^\/\\])$:$1\/:; + } elsif ($arg =~/^-(enc|encode|encoding)=(.+)$/) {$encoding=$2;} elsif ($arg =~/^-(utf8|unicode)$/) {$encoding='utf8';} elsif ($arg =~/^-(alpha(bets?)?)=(.*)$/) {set_script_options(\@AlphabetScripts,$3);} @@ -649,6 +993,7 @@ sub parse_options_counts { elsif ($arg eq '-freq') {$optionWordFreq=1;} elsif ($arg =~/^-freq=(\d+)$/) {$optionWordFreq=$1;} elsif ($arg eq '-stat') {$optionWordClassFreq=1;} + elsif ($arg =~/^-macro-?(stat|freq)$/) {$optionMacroStat=1;} else {return 0;} return 1; } @@ -657,11 +1002,11 @@ sub parse_options_counts { sub _option_sum { my $arg=shift @_; if (!defined $arg) { - @sumweights=(1,1,1,0,0,1,1); + @sumweights=(0,1,1,1,0,0,1,1); } elsif ($arg=~/^(\d+(\.\d*)?(,\d+(\.\d*)?){0,6})$/) { - @sumweights=split(',',$1); + @sumweights=(0,split(',',$1)); } else { - print STDERR "Warning: Option value ".$arg." not valid, ignoring option.\n"; + print STDERR "Warning: Option value $arg not valid, ignoring option.\n"; } } @@ -674,7 +1019,7 @@ sub _option_subcount { } elsif (my $option=$BreakPointsOptions{$arg}) { %BreakPoints=%{$option}; } else { - print STDERR "Warning: Option value ".$arg." not valid, using default instead.\n"; + print STDERR "Warning: Option value $arg not valid, using default instead.\n"; %BreakPoints=%{$BreakPointsOptions{'default'}}; } } @@ -683,20 +1028,18 @@ sub _option_subcount { sub parse_options_output { my $arg=shift @_; if ($arg eq '-strict') {$strictness=1;} - elsif ($arg eq '-v0') {$verbose=0;} - elsif ($arg eq '-v1') {$verbose=1;} - elsif ($arg eq '-v2') {$verbose=2;} - elsif ($arg eq '-v3' || $arg eq '-v') {$verbose=3;} - elsif ($arg eq '-v4') {$verbose=3; $showstates=1;} + elsif ($arg eq '-v') {set_verbosity_options('3');} + elsif ($arg =~/^-v([0-4=+\-].*)/) {set_verbosity_options($1);} elsif ($arg =~/^-showstates?$/ ) {$showstates=1;} - elsif ($arg =~/^-(q|quiet)$/ ) {$verbose=-1;} + elsif ($arg =~/^-(q|quiet)$/ ) {$printlevel=-1;} elsif ($arg =~/^-(template)=(.*)$/ ) {_set_output_template($2);} elsif ($arg eq '-split') {$optionFast=1;} elsif ($arg eq '-nosplit') {$optionFast=0;} elsif ($arg eq '-showver') {$showVersion=1;} elsif ($arg eq '-nover') {$showVersion=-1;} - elsif ($arg =~/^-nosep(s|arators?)?$/ ) {$separator='';} - elsif ($arg =~/^-sep(arators?)?=(.*)$/ ) {$separator=$2;} + elsif ($arg =~/^-nosep(arator)?s?$/ ) {$separator='';} + elsif ($arg =~/^-sep(arator)?s?=(.*)$/ ) {$separator=$2;} + elsif ($arg =~/^-out=(.*)/ ) {close STDOUT; open STDOUT,'>',$1;} else {return 0;} return 1; } @@ -705,8 +1048,8 @@ sub parse_options_output { sub _set_output_template { my $template=shift @_; $outputtemplate=$template; - if ($template=~/\{(S|SUM)[\?\}]/i && !@sumweights) { - @sumweights=(1,1,1,0,0,1,1); + if ($template=~/\{(SUM)[\?\}]/i && !@sumweights) { + @sumweights=(0,1,1,1,0,0,1,1); } if ($template=~/\{SUB\?/i && !$showsubcounts) { _option_subcount(); @@ -718,14 +1061,19 @@ sub parse_options_format { my $arg=shift @_; if ($arg eq '-brief') {$briefsum=1;} elsif ($arg eq '-total') {$totalflag=1;} - elsif ($arg eq '-0') {$briefsum=1;$totalflag=1;$verbose=-1;$finalLineBreak=0;} - elsif ($arg eq '-1') {$briefsum=1;$totalflag=1;$verbose=-1;} - elsif ($arg eq '-html' ) {option_ansi_colours(0);$htmlstyle = 2;} - elsif ($arg eq '-htmlcore' ) {option_ansi_colours(0);$htmlstyle = 1;} + elsif ($arg eq '-0') {$briefsum=1;$totalflag=1;$printlevel=-1;$finalLineBreak=0;} + elsif ($arg eq '-1') {$briefsum=1;$totalflag=1;$printlevel=-1;} + elsif ($arg eq '-htmlcore' ) {option_ansi_colours(0);$htmlstyle=1;} + elsif ($arg eq '-html' ) {option_ansi_colours(0);$htmlstyle=2;} + elsif ($arg eq '-tex' ) {option_ansi_colours(0);$texcodeoutput=1;} elsif ($arg =~/^\-(nocol|nc$)/) {option_ansi_colours(0);} elsif ($arg =~/^\-(col)$/) {option_ansi_colours(1);} elsif ($arg eq '-codes') {$showcodes=2;} elsif ($arg eq '-nocodes') {$showcodes=0;} + elsif ($arg =~/^-htmlfile=(.+)$/) {$HTMLfile=$1;} + elsif ($arg =~/^-cssfile=(.+)$/) {$CSSfile=$1;} + elsif ($arg =~/^-css=file:(.+)$/) {$CSSfile=$1;} + elsif ($arg =~/^-css=(.+)$/) {$CSShref=$1;} else {return 0;} return 1; } @@ -733,8 +1081,8 @@ sub parse_options_format { # Include options from option file sub _parse_optionfile { my $filename=shift @_; - open(FH,"<",$filename) - || die "Option file not found: ".$filename."\n"; + open(FH,'<',$filename) + || die "Option file not found: $filename\n"; my @options=<FH>; close(FH); s/^\s*(#.*|)//s for @options; @@ -744,7 +1092,7 @@ sub _parse_optionfile { foreach my $arg (@options) { __optionfile_tc($arg) || parse_option($arg) - || die "Invalid option ".$arg." in ".$filename."\n"; + || die "Invalid option $arg in $filename.\n"; } } @@ -752,12 +1100,12 @@ sub _parse_optionfile { sub __optionfile_tc { my $arg=shift @_; $arg=~s/^\%\s*// || return 0; - if ($arg=~/^subst\s+(\\\w+)\s+/i) { - $substitutions{$1}=$'; + if ($arg=~/^subst\s+(\\\w+)\s+(.*)$/i) { + $substitutions{$1}=$2; } elsif ($arg=~/^(\w+)\s+([\\]*\w+)\s+([^\s\n]+)(\s+([0-9]+))?/i) { - tc_macro_param_option($1,$2,$3,$5) || die "Invalid TC option: ".$arg."\n"; + tc_macro_param_option($Main,$1,$2,$3,$5) || die "Invalid TC option: $arg\n"; } else { - print "Invalid TC option format: ".$arg."\n"; + print "Invalid TC option format: $arg\n"; return 0; } return 1; @@ -766,10 +1114,10 @@ sub __optionfile_tc { # Parse file list and return total count sub Parse_file_list { my @files=@_; - my $listtotalcount=new_count("Total"); + my $listtotalcount=new_count('Total'); foreach (@files) {s/\\/\//g; s/ /\\ /g;} - if (@files) { - @files=<@files>; # For the sake of Windows: expand wildcards! + if (@files) { + @files=<@files>; # For the sake of Windows: expand wildcards! for my $file (@files) { my $filetotalcount=parse_file($file); add_to_total($listtotalcount,$filetotalcount); @@ -778,47 +1126,49 @@ sub Parse_file_list { if ($fileFromSTDIN) { my $filetotalcount=parse_file($_STDIN_); add_to_total($listtotalcount,$filetotalcount); - } + } return $listtotalcount; } # Parse file and included files, and return total count sub parse_file { my $file=shift @_; - $workdir=$globalworkdir; - if (!defined $workdir) { + if (defined ($workdir=$globalworkdir)) {} + elsif ($file eq $_STDIN_) {$workdir='';} + else { $workdir=$file; $workdir =~ s/^((.*[\\\/])?)[^\\\/]+$/$1/; } - if ($htmlstyle && ($verbose || !$totalflag)) {print "\n<div class='filegroup'>\n";} - my $filetotalcount=new_count("File(s) total: ".$file); + if ($htmlstyle && ($printlevel>0 || !$totalflag)) {print "\n<div class='filegroup'>\n";} + my $filetotalcount=new_count('File(s) total: '.$file); @filelist=(); - _add_file($filetotalcount,$file); + _add_file($filetotalcount,[$file,$workdir],'File: '.$file); foreach my $f (@filelist) { - _add_file($filetotalcount,$f,"Included file: ".$f); + _add_file($filetotalcount,$f,'Included file: '.$f->[0]); } if (!$totalflag && get_count($filetotalcount,$CNT_FILE)>1) { - if ($htmlstyle) {formatprint("Sum of files: ".$file."\n",'h2');} + if ($htmlstyle) {formatprint("Sum of files: $file\n",'h2');} print_count($filetotalcount,'sumcount'); } - if ($htmlstyle && ($verbose || !$totalflag)) {print "</div>\n\n";} + if ($htmlstyle && ($printlevel>0 || !$totalflag)) {print "</div>\n\n";} return $filetotalcount; } # Parse single file, included files will be appended to @filelist. sub _add_file { - my ($filetotalcount,$f,$title)=@_; - my $tex=TeXfile($f,$title); - my $fpath=$f; - $fpath=~s/^((.*[\\\/])?)[^\\\/]+$/$1/; + my ($filetotalcount,$pathfile,$title)=@_; + my @paths=@$pathfile; + my $file=shift @paths; + my $tex=TeXfile($file,$title); if (!defined $tex) { - error($Main,'File not found or not readable: '.$f); - } else { - parse($tex); - my $filecount=next_subcount($tex); - if (!$totalflag) {print_count($filecount);} - add_to_total($filetotalcount,$filecount); + error($Main,'File not found or not readable: '.$file); + return; } + $tex->{'PATH'}=\@paths; + parse($tex); + my $filecount=next_subcount($tex); + if (!$totalflag) {print_count($filecount);} + add_to_total($filetotalcount,$filecount); } ###### @@ -830,21 +1180,59 @@ sub _add_file { # Add file to list of files scheduled for parsing sub include_file { - my ($tex,$fname)=@_; - my $fpath=$workdir.$fname; + my ($tex,$state,$file,$refparam)=@_; + my %params=%$refparam; + my $type=$params{'<type>'}; + my @paths=@{$tex->{'PATH'}}; + my $filepath; + if ($type eq '<bbl>' && defined $auxdir) { + $auxdir=~s/([\\\/])*$/\//; + if (defined $globalworkdir) {@paths=[$auxdir];} + elsif ($auxdir=~/^(\w:)?[\\\/]/) {@paths=[$auxdir];} + else {@paths=[$workdir.$auxdir]} + } + foreach my $key (split(/\s+/,$type)) { + my $value=$params{$key}; + if ($key eq 'dir') {unshift @paths,$workdir.$value;} + elsif ($key eq 'subdir') {unshift @paths,$paths[0].$value;} + } + if ($file=~/^(\w:)?[\\\/]/) {$filepath=$file;} + else { + $filepath=_find_file_in_path($tex,$file,@paths) || BLOCK { + error($tex,'File '.$file.' not found in path ['.join(';',@paths).'].'); + return; + }; + } if ($includeTeX==2) { - my $bincode=read_binary($fpath) || BLOCK { - error($tex,"File $fpath not found."); + my $bincode=read_binary($filepath) || BLOCK { + error($tex,"File $filepath not readable."); return; }; flush_next($tex); line_return(0,$tex); - prepend_code($tex,$bincode,$fname); + # DELETE: prepend_code($tex,$bincode,$file); + my $texstate=texcode_insert_text($tex,$bincode,$file,@paths); + parse_all($tex,$state); + texcode_restore_state($tex,$texstate); } else { - push @filelist,$fpath; + push @filelist,[$filepath,@paths]; } } +# Seach path list to find file +sub _find_file_in_path { + my $tex=shift @_; + my $file=shift @_; + foreach my $path (@_) { + if (!$path=~/[\\\/]$/) {$path.='/';} + my $filepath=$path.$file; + if (-e $filepath) {return $filepath;} + elsif ($filepath=~/\.tex$/i) {} + elsif (-e $filepath.'.tex') {return $filepath.'.tex';} + } + return undef; +} + # Print count (total) if conditions are met sub conditional_print_total { my $sumcount=shift @_; @@ -852,7 +1240,7 @@ sub conditional_print_total { if ($totalflag && $briefsum && @sumweights) { print get_sum_count($sumcount),"\n"; } else { - if ($htmlstyle) {formatprint("Total word count",'h2');} + if ($htmlstyle) {formatprint('Total word count','h2');} print_count($sumcount,'sumcount'); } } @@ -875,24 +1263,49 @@ sub ansiprint { # Apply options to set values sub Apply_Options { - if ($showcodes>1 && $verbose<1) {$verbose=3;} - %STYLE=%{$STYLES[$verbose]}; - if (defined $encoding && $encoding eq 'guess') {$encoding=undef;} - if (!defined $encoding) { - } elsif (ref(find_encoding($encoding))) { - if (!$htmlstyle) {$outputEncoding=$encoding;} - } else { - error($Main,"Unknown encoding $encoding ignored."); - error_details($Main,'Valid encodings are: '.wrap('','',join(', ',Encode->encodings(':all')))); - $encoding=undef; - } - if (!defined $outputEncoding) {$outputEncoding='utf8';} - binmode STDIN; - binmode STDOUT,':encoding('.$outputEncoding.')'; + apply_encoding_options(); if ($htmlstyle>1) {html_head();} flush_errorbuffer($Main); apply_language_options(); if ($includeBibliography) {apply_include_bibliography();} + if ($showcodes>1 && !($STYLE{'<printlevel>'})) {%STYLE=%{$STYLES{'All'}};} + if ($showstates) {set_verbosity_options('+States');} + if (!@sumweights) {set_verbosity_options('-Sums');} + (defined $printlevel) + || (defined ($printlevel=$STYLE{'<printlevel>'})) + || ($printlevel=$defaultprintlevel); +} + +# Set or add styles included in the verbose output +sub set_verbosity_options { + my $verb=shift @_; + my $st; + if ($defaultprintlevel<1) {$defaultprintlevel=1;} + if ( $verb=~s/^([0-4])// || $verb=~s/^=([0-4])?// ) { + my $key=$1; + if (!defined $key) {%STYLE=(%{$STYLES{'<core>'}});} + elsif ($st=$STYLES{$key}) {%STYLE=(%$st);} + else { + error($Main,"Unknown verbosity option '$key'."); + %STYLE=(%{$STYLES{'<core>'}}); + } + } + while ($verb=~s/^([\+\-]?)(\w+)//) { + my $add=!($1 eq '-'); + my $key=$2; + if ($st=$STYLES{$key}) { + if ($add) {@STYLE{keys %$st}=values %$st;} + else {delete @STYLE{keys %$st};} + } elsif ($st=$STYLES{'All'}->{$key}) { + if ($add) {$STYLE{$key}=$st;} + elsif ($STYLE{$key}) {delete $STYLE{$key};} + } else { + error($Main,"Unknown verbosity option '$key'."); + } + } + if ($verb ne '') { + error($Main,"Unknown verbosity option format: $verb"); + } } # Set or add scripts to array of scripts @@ -907,13 +1320,14 @@ sub set_script_options { $scr='alphabetic'; } if ($scr=~/^[a-z]/) {$scr='Is_'.$scr;} - if (is_property_valid($scr)) {push @$scriptset,$scr;} + if (_is_property_valid($scr)) {push @$scriptset,$scr;} else {error($Main,"Unknown script $scr ignored.");} } } -sub is_property_valid { +sub _is_property_valid { my $script=shift @_; + if (! $script=~/^\w+$/) {return 0;} eval {' '=~/\p{$script}/}; if ($@) {return 0;} else {return 1;} } @@ -954,15 +1368,31 @@ sub set_language_option { } elsif ($language=~/^(char|character|letter)s?(-?only)?$/) { @WordPatterns=($NamedWordPattern{'letters'}); if (defined $2) {@LogogramScripts=();} - $countlabel[1]='Letters in text'; - $countlabel[2]='Letters in headers'; - $countlabel[3]='Letters in captions'; + $countdesc[1]='Letters in text'; + $countdesc[2]='Letters in headers'; + $countdesc[3]='Letters in captions'; return 'letter'; } else { return undef; } } +# Set encoding based on encoding options, guess encoding if not set +sub apply_encoding_options { + if (defined $encoding && $encoding eq 'guess') {$encoding=undef;} + if (!defined $encoding) { + } elsif (ref(find_encoding($encoding))) { + if (!$htmlstyle) {$outputEncoding=$encoding;} + } else { + error($Main,"Unknown encoding $encoding ignored."); + error_details($Main,'Valid encodings are: '.wrap('','',join(', ',Encode->encodings(':all')))); + $encoding=undef; + } + if (!defined $outputEncoding) {$outputEncoding='utf8';} + binmode STDIN; + binmode STDOUT,':encoding('.$outputEncoding.')'; +} + # Apply language options sub apply_language_options { my @tmp; @@ -993,49 +1423,239 @@ sub apply_include_bibliography { include_package('%incbib'); } +# Process TC instruction, return 1 if applied, 0 if not (e.g. error) +sub tc_macro_param_option { + my ($tex,$instr,$macro,$param,$option)=@_; + if (!defined $param) { + error($tex,'%TC:'.$instr.' requires parameter rule specification.'); + return 0; + } + elsif ($instr eq 'macro') { + if (!($macro=~/^\\/)) { + warning($tex,'%TC:macro '.$macro.': should perhaps be \\'.$macro.'?'); + } + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$macro,$param); + } + elsif ($instr eq 'exclude') { + warning($tex,'%TC:exclude is deprecated. Use %TC:macro instead.'); + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$macro,$param); + } + elsif ($instr eq 'header') { + warning($tex,'%TC:header is deprecated. Instead use e.g. %TC:macro \\name [header].'); + $TeXmacrocount{$macro}=[$CNT_COUNT_HEADER]; + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$macro,$param); + } + elsif ($instr eq 'preambleinclude') { + return _tc_macro_set_param($tex,\%TeXpreamble,$instr,$macro,$param); + } + elsif ($instr eq 'floatinclude') { + return _tc_macro_set_param($tex,\%TeXfloatinc,$instr,$macro,$param); + } + elsif ($instr=~/^(group|envir)$/) { + if (!defined $option) { + error($tex,'TC:'.$instr.' requires contents rule specification.'); + return 0; + } + defined ($option=key_to_state($option,$tex)) || return 0; + $TeXenvir{$macro}=$option; + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$PREFIX_ENVIR.$macro,$param); + } + elsif ($instr=~/^(macrocount|macroword)$/) { + return _tc_macro_set_param($tex,\%TeXmacrocount,$instr,$macro,$param,\&key_to_cnt,$REGEX_NUMBER); + } + elsif ($instr eq 'fileinclude') { + # No parameter checking here, just pass on; errors reported when used instead. + $param=~s/^\[(.*)\]$/$1/; + $param=~s/,/ /g; + assert($param=~/^\w+(\s\w+)*$/,$tex,'Invalid %TC:fileinclude parameter: '.$param) + || return 0; + $TeXfileinclude{$macro}=$param; + } + elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;} + else { + error($tex,'Unknown TC command: '.$instr); + return 0; + } + return 1; +} + +# Convert TC command parameters and add to hash +sub _tc_macro_set_param { + my ($tex,$hash,$instr,$macro,$param,$keymap,$validregex)=@_; + if (!defined $keymap) {$keymap=\&key_to_state; $validregex=$REGEX_NUMBER;} + $param=__interpret_tc_parameter($tex,$param,$keymap,$validregex); + if (!defined $param) {return 0;} + $hash->{$macro}=$param; + return 1; +} + +# Convert [...] into array of states, optionally return value if a valid pattern +sub __interpret_tc_parameter { + my ($tex,$param,$keymap,$validregex)=@_; + if ((defined $validregex) && ($param=~$validregex)) { + return $param; + } elsif ($param=~/^\[([0-9a-zA-Z,:]*)\]$/) { + my @params; + my $value; + foreach my $key (split(',',$1)) { + if ($key=~/^(\w*):(\w+)$/) { + defined ($value=$keymap->($PREFIX_PARAM_OPTION.$1,$tex)) || return undef; + push @params,$value; + $key=$2; + } + defined ($value=$keymap->($key,$tex)) || return undef; + push @params,$value; + } + return [@params]; + } + error($tex,'Invalid TC command parameter: '.$param); + return undef; +} + + +###### Macro rules handling + + +# Takes hash, value, list of keys and adds mappings to hash +sub add_keys_to_hash { + my $hash=shift @_; + my $value=shift @_; + foreach (@_) { + $hash->{$_}=$value; + } +} + +# Add source hash to target hash, optionally converting values +sub __add_to_hash { + my $target=shift @_; + my $source=shift @_; + my $convert=shift @_; + if (!defined $convert) {$convert=&__dummy;} + while (my ($key,$val)=each(%$source)) { + $target->{$key}=$convert->($val,@_); + } +} + +# Convert hash by applying function to all values +sub convert_hash { + my $hash=shift @_; + my $convert=shift @_; + while (my ($key,$val)=each(%$hash)) { + $hash->{$key}=$convert->($val,@_); + } +} + +# Convert key to parser state constant +sub key_to_state { + my ($key,$tex)=@_; + return key_convert(lc($key),$tex,\%key2state,'Unknown state key {key} replaced by IGNORE.',$STATE_IGNORE); +} + +# Convert key array to parser state constants +sub keyarray_to_state { + my ($array,$tex)=@_; + return keyarray_convert($array,$tex,\&key_to_state); +} + +# Convert key to counter constant +sub key_to_cnt { + my ($key,$tex)=@_; + return key_convert(lc($key),$tex,\%key2cnt,'Unknown counter key {key} ignored.'); +} + +# Convert key array to counter constants +sub keyarray_to_cnt { + my ($array,$tex)=@_; + return keyarray_convert($array,$tex,\&key_to_cnt); +} + +# Convert key to value using map +sub key_convert { + my ($key,$tex,$map,$error,$default)=@_; + my $value=$map->{$key}; + if (!defined $value) { + if (!defined $tex) {$tex=$Main;} + $error=~s/\{key\}/$key/g; + error($tex,$error); + return $default; + } + return $value; +} + +# Convert state keys to state nos for arrays +sub keyarray_convert { + my ($array,$tex,$conv)=@_; + if (ref($array) eq 'ARRAY') { + my @ar; + foreach my $key (@$array) { + if (defined (my $cnt=$conv->($key,$tex))) {push @ar,$cnt;} + } + $array=\@ar; + } + return $array; +} + +# Convert transition state to parsing state, and increment counter +sub transition_to_content_state { + my ($tex,$state)=@_; + my $tr=$transition2state{$state}; + if (defined $tr) { + if (my $cnt=$tr->[1]) {inc_count($tex,$tr->[1]);} + return $tr->[0]; + } + return $state; +} + +# Remove all rules +sub remove_all_rules { + %TeXpackageinc=(); + %TeXpreamble=(); + %TeXfloatinc=(); + %TeXmacro=(); + %TeXmacrocount=(); + %TeXenvir=(); + %TeXfileinclude=(); + %IncludedPackages=(); +} + # Process package inclusion sub include_package { - my $incpackage=shift @_; - my $sub; - _add_to_hash_if_exists(\%TeXmacro,\%PackageTeXmacro,$incpackage); - _add_to_hash_if_exists(\%TeXmacroword,\%PackageTeXmacroword,$incpackage); - _add_to_hash_if_exists(\%TeXheader,\%PackageTeXheader,$incpackage); - _add_to_hash_if_exists(\%TeXgroup,\%PackageTeXgroup,$incpackage); - _add_to_hash_if_exists(\%TeXfileinclude,\%PackageTeXfileinclude,$incpackage); + my ($incpackage,$tex)=@_; + if (defined $IncludedPackages{$incpackage}) {return;} + $IncludedPackages{$incpackage}=1; + # Add rules for package, preamble and float inclusion, and add to macro rules + _add_package(\%TeXpackageinc,\%PackageTeXpackageinc,$incpackage,\&__dummy,$tex); + _add_package(\%TeXpreamble,\%PackageTeXpreamble,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXfloatinc,\%PackageTeXfloatinc,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacro,\%PackageTeXpackageinc,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacro,\%PackageTeXpreamble,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacro,\%PackageTeXfloatinc,$incpackage,\&keyarray_to_state,$tex); + # Add regular rules + _add_package(\%TeXmacro,\%PackageTeXmacro,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacrocount,\%PackageTeXmacrocount,$incpackage,\&keyarray_to_cnt,$tex); + _add_package(\%TeXenvir,\%PackageTeXenvir,$incpackage,\&key_to_state,$tex); + _add_package(\%TeXfileinclude,\%PackageTeXfileinclude,$incpackage,\&__dummy,$tex); + if (my $subpackages=$PackageSubpackage{$incpackage}) { + foreach my $sub (@{$subpackages}) { + include_package($sub,$tex); + } + } } # Add package rules if defined -sub _add_to_hash_if_exists { - my ($target,$source,$name)=@_; +sub _add_package { + my ($target,$source,$name,$convert,$tex)=@_; my $sub; if ($sub=$source->{$name}) { - while (my ($key,$val)=each(%$sub)) { - $target->{$key}=$val; - } + __add_to_hash($target,$sub,$convert); } } -# Process TC instruction -sub tc_macro_param_option { - my ($instr,$macro,$param,$option)=@_; - if ($param=~/^\[([0-9,]+)\]$/) {$param=[split(',',$1)];} - if (!defined $option) {$option=1;} - if (($instr eq 'macro') || ($instr eq 'exclude')) {$TeXmacro{$macro}=$param;} - elsif ($instr eq 'header') {$TeXheader{$macro}=$param;$TeXmacro{$macro}=$param;} - elsif ($instr eq 'macroword') {$TeXmacroword{$macro}=$param;} - elsif ($instr eq 'preambleinclude') {$TeXpreamble{$macro}=$param;} - elsif ($instr eq 'group') { - $TeXmacro{'begin'.$macro}=$param; - $TeXgroup{$macro}=$option; - } - elsif ($instr eq 'floatinclude') {$TeXfloatinc{$macro}=$param;} - elsif ($instr eq 'fileinclude') {$TeXfileinclude{$macro}=$param;} - elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;} - else {return 0;} - return 1; +# Dummy conversion function: returns arguments unchanged +sub __dummy { + return shift @_; } - ###### TeX code handle @@ -1053,11 +1673,14 @@ sub getMain { ## Make TeX handle for LaTeX code: the main TeXcount object # The "TeX object" is a data containser: a hash containing -# - filename: name of LaTeX file being parsed -# - filepath: path to LaTeX file +# - filepath: path of LaTeX file being parsed +# - filename: file name of LaTeX file +# - dirpath: path to directory containing LaTeX file +# - PATH: array of paths to search for included files # - countsum: count object with total count (incl. subcounts) # - subcount: count object for subcount (to be added to countsum) # - subcounts: list of subcounts +# - errorcount: the number of errors reported # plus following elements used for the processing the LaTeX code # - line: the LaTeX paragraph being processed # - texcode: what remains of LaTeX code to process (after line) @@ -1074,8 +1697,8 @@ sub getMain { # prepared. Note that this requires that the last subcount object be added to # the countsum once the end of the document is reached. sub TeXcode { - my ($bincode,$filename,$title)=@_; - my $tex=_TeXcode_blank($filename,$title); + my ($bincode,$file,$title)=@_; + my $tex=_TeXcode_blank($file,$title); _TeXcode_setcode($tex,$bincode); more_texcode($tex); return $tex; @@ -1083,16 +1706,21 @@ sub TeXcode { # Return a blank TeXcode object sub _TeXcode_blank { - my ($filename,$title)=@_; + my ($file,$title)=@_; if (defined $title) {} - elsif (defined $filename) {$title="File: ".$filename;} - else {$title="Word count";} + elsif (defined $file) {$title='File: '.$file;} + else {$title='Word count';} my %TeX=(); $TeX{'errorcount'}=0; - $TeX{'filename'}=$filename; - if (!defined $filename) {$TeX{'filepath'}='';} - elsif ($filename=~/^(.*[\\\/])[^\\\/]+$/) {$TeX{'filepath'}=$1;} - else {$TeX{'filepath'}='';} + $TeX{'warnings'}={}; + $TeX{'filepath'}=$file; + if (!defined $file) {} + elsif ($file=~/^<.+>$/) {$TeX{'filename'}=$file;} + elsif ($file=~/^(.*[\\\/])([^\\\/]+)$/) { + $TeX{'dirpath'}=$1; $TeX{'filename'}=$2; + } + else {$TeX{'dirpath'}=''; $TeX{'filename'}=$file;} + $TeX{'PATH'}=[]; $TeX{'line'}=''; $TeX{'next'}=undef; $TeX{'type'}=undef; @@ -1102,11 +1730,9 @@ sub _TeXcode_blank { my $countsum=new_count($title); $TeX{'countsum'}=$countsum; $countsum->{'TeXcode'}=\%TeX; - my $count=new_count("_top_"); + my $count=new_count('_top_'); $TeX{'subcount'}=$count; inc_count(\%TeX,$CNT_FILE); - my @countlist=(); - $countsum->{'subcounts'}=\@countlist; return \%TeX; } @@ -1115,8 +1741,6 @@ sub _TeXcode_setcode { my ($tex,$bincode)=@_; $tex->{'texcode'}=_prepare_texcode($tex,$bincode); $tex->{'texlength'}=length($tex->{'texcode'}); - # TODO: Comment out this test - if ($tex->{'texcode'} =~ /([[:^ascii:]])/) {warning($tex,"Code contains non-ASCII characters.");} } # Decode and return TeX/LaTeX code @@ -1137,9 +1761,7 @@ sub _decode_texcode { my $decoder; if (defined $encoding) { $decoder=find_encoding($encoding); - eval { - $texcode=$decoder->decode($texcode); - }; + eval {$texcode=$decoder->decode($texcode);}; if ($@) { error($tex,'Decoding file/text using the '.$decoder->name.' encoding failed.'); } @@ -1200,26 +1822,51 @@ sub apply_substitution_rule { ## Get more TeX code from texcode buffer if possible, return 1 if done sub more_texcode { my ($tex)=@_; - if (!defined $tex->{'texcode'}) {return 0;} - if ( $optionFast && $tex->{'texcode'} =~ s/^.*?(\r{2,}|\n{2,}|(\r\n){2,})//s ) { - $tex->{'line'}.=$&; + if ($tex->{'texcode'} eq '') {return 0;} + if ( $optionFast && $tex->{'texcode'} =~ s/^(.*?(\r{2,}|\n{2,}|(\r\n){2,}))//s ) { + $tex->{'line'}.=$1; # $1 ~ ${^MATCH} return 1; } $tex->{'line'}.=$tex->{'texcode'}; - $tex->{'texcode'}=undef; + $tex->{'texcode'}=''; return 1; } ## Prepend LaTeX code to TeXcode object sub prepend_code { - my ($tex,$code,$fname)=@_; - my $prefix="\n% --- Start of included file ".$fname."\n"; - my $suffix="\n% --- End of included file ".$fname."\n"; + my ($tex,$code,$filename)=@_; + my $prefix="\n% --- Start of included file $filename\n"; + my $suffix="\n% --- End of included file $filename\n"; $code=_decode_texcode($tex,$code); $tex->{'length'}+=length($code); $tex->{'texcode'}=$prefix.$code.$suffix.$tex->{'line'}.$tex->{'texcode'}; $tex->{'line'}=''; - more_texcode(); + more_texcode($tex); +} + +# Save and return TeXcode state, and replace with new text +sub texcode_insert_text { + my ($tex,$code,$filename,@path)=@_; + my %texstate=( + 'texcode'=>$tex->{'line'}.$tex->{'texcode'}, + 'eof'=>$tex->{'eof'}, + 'PATH'=>$tex->{'PATH'}); + my $prefix="\n% --- Start of included file $filename\n"; + my $suffix="\n% --- End of included file $filename\n"; + $code=_decode_texcode($tex,$code); + $tex->{'length'}+=length($code); + $tex->{'texcode'}=$prefix.$code.$suffix; + $tex->{'line'}=''; + more_texcode($tex); + return \%texstate; +} + +# Restore TeXcode state +sub texcode_restore_state { + my ($tex,$texstate)=@_; + while (my ($key,$value)=each %$texstate) { + $tex->{$key}=$value; + } } ## Returns size of TeX code in bytes @@ -1235,20 +1882,20 @@ sub get_texsize { # Read LaTeX file into TeX object sub TeXfile { my ($filename,$title)=@_; + my ($header,$bincode); if ($filename eq $_STDIN_) { - if ($verbose>0) { - formatprint("File from STDIN\n",'h2'); - $blankline=0; - } - return TeXcode(_read_stdin(),'STDIN',$title); - } else { - my $bincode=read_binary($filename) || return undef; - if ($verbose>0) { - formatprint("File: ".$filename."\n",'h2'); - $blankline=0; - } - return TeXcode($bincode,$filename,$title); + $header='File from STDIN'; + $bincode=_read_stdin(); + } else { + defined ($bincode=read_binary($filename)) || return undef; + $header='File: '.$filename; + } + if ($printlevel>0) { + formatprint($header."\n",'h2'); + $blankline=0; } + my $tex=TeXcode($bincode,$filename,$title); + return $tex; } # Read file to string without regard for encoding @@ -1275,8 +1922,10 @@ sub _read_stdin { # Add warning to list of registered warnings (optionally to be reported at the end) sub warning { my ($tex,$text)=@_; - $tex->{'warnings'}->{$text}++; $warnings{$text}++; + # TODO: should only add warnings to subcount, not to TeXcode object + $tex->{'warnings'}->{$text}++; + if (my $count=$tex->{'subcount'}) {$count->{'warnings'}->{$text}++}; } # Register error and print error message @@ -1285,9 +1934,9 @@ sub error { if (defined $type) {$text=$type.': '.$text;} $errorcount++; $tex->{'errorcount'}++; - #print STDERR $text,"\n"; + if (my $count=$tex->{'subcount'}) {$count->{'errorcount'}++}; if (my $err=$tex->{'errorbuffer'}) {push @$err,$text;} - if ($verbose>=0) {_print_error($text);} + else {print_error($text);} } # Print error details @@ -1296,172 +1945,221 @@ sub error_details { print STDERR $text,"\n"; } +# Make assertion: i.e. report error if not true and return truthfullness +sub assert { + my $assertion=shift @_; + if ($assertion) {return 1;} + error(@_); + return 0; +} + +# Print error message +sub print_error { + my $text=shift @_; + line_return(1); + if ($printlevel<0) { + print STDERR 'ERROR: ',$text,"\n"; + } elsif ($htmlstyle) { + print STDERR 'ERROR: ',$text,"\n"; + print_style($text,'error'); + } else { + print_style("!!! $text !!!",'error'); + } + line_return(1); +} + +# Print errors in buffer and delete errorbuffer +sub flush_errorbuffer { + my $source=shift @_; + my $err=$source->{'errorbuffer'} || return; + foreach (@$err) {print_error($_);} + $source->{'errorbuffer'}=undef; +} + ###### Parsing routines # Parse LaTeX document sub parse { my ($tex)=@_; - if ($htmlstyle && $verbose) {print "<div class='parse'><p>\n";} + if ($htmlstyle && $printlevel>0) {print "<div class='parse'><p>\n";} + parse_all($tex); + if ($htmlstyle && $printlevel>0) {print "</p></div>\n";} +} + +# Parse until end +sub parse_all { + my ($tex,$state)=@_; + if (!defined $state) {$state=$STATE_TEXT;} while (!($tex->{'eof'})) { - _parse_unit($tex,$STATUS_TEXT); + _parse_unit($tex,$state); } - if ($htmlstyle && $verbose) {print "</p></div>\n";} } -# Parse one block or unit +# Parse one block/unit with given state until end token +# If end token is set to $_PARAM_, unit is assumed to be a parameter +# and words parsed as individual letters (simple_token is set) sub _parse_unit { - # Status: - # 0 = exclude from count - # 1 = text - # 2 = header text - # 3 = float text - # -1 = float (exclude) - # -2 = strong exclude, ignore begin-end groups - # -3 = stronger exclude, do not parse macro parameters - # -4 = ignore everything except end marker: even { - # -9 = preamble (between \documentclass and \begin{document}) - my ($tex,$status,$end)=@_; - if (!defined $status) { - error($tex,'Undefined parser status!','CRITICAL ERROR'); - exit; - } elsif (ref($status) eq 'ARRAY') { - error($tex,'Invalid parser status!','CRITICAL ERROR'); - exit; + my ($tex,$state,$end)=@_; + my $simple_token=0; + ( assert(defined $state,$tex,'Undefined parser state!','BUG') + && assert(!ref($state),$tex,'Invalid parser state of type '.ref($state).'!','BUG') + ) || exit; + $state=transition_to_content_state($tex,$state); + _set_printstate($tex,$state,$end); + if (defined $end && $end eq $_PARAM_) { + $end=undef; + $simple_token=1; } - if ($showstates) { - if (defined $end) { - $tex->{'printstate'}=':'.status_to_text($status).':'.$end.':'; - } else { - $tex->{'printstate'}=':'.status_to_text($status).':'; - } - flush_next($tex); - } - while (defined (my $next=_next_token($tex))) { + my $next; + while (defined ($next=next_token($tex,$simple_token))) { # Parse next token until token matches $end - set_style($tex,"ignore"); + set_style($tex,'ignore'); + if ($state==$STATE_MATH) {set_style($tex,'math');} if ((defined $end) && ($end eq $next)) {return;} # Determine how token should be interpreted - if ($status==$STATUS_PREAMBLE && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) { + if ($state==$STATE_PREAMBLE && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) { # \begin{document} - $status=$STATUS_TEXT; + $state=$STATE_TEXT; } - if ($status==$STATUS_EXCLUDE_ALL) { - # Ignore everything + # Pick the first matching interpretation: + if ($state==$STATE_EXCLUDE_ALL) { + # ignore everything } elsif ($tex->{'type'}==$TOKEN_SPACE) { # space or other code that should be passed through without styling flush_next($tex,' '); - } elsif ($next eq '\documentclass') { - # starts preamble - set_style($tex,'document'); - __gobble_option($tex); - __gobble_macro_parms($tex,1); - while (!($tex->{'eof'})) { - _parse_unit($tex,$STATUS_PREAMBLE); - } } elsif ($tex->{'type'}==$TOKEN_TC) { # parse TC instructions _parse_tc($tex,$next); } elsif ($tex->{'type'}==$TOKEN_WORD) { # word - if (my $cnt=status_text_cnt($status)) { - _process_word($tex,$next,$status); + if (my $cnt=state_text_cnt($state)) { + _process_word($tex,$next,$state); inc_count($tex,$cnt); - set_style($tex,status_to_style($status)); + set_style($tex,state_to_style($state)); } } elsif ($next eq '{') { - # {...} - _parse_unit($tex,$status,'}'); + # {...} group + set_style($tex,'ignore'); + _parse_unit($tex,$state,'}'); + set_style($tex,'ignore'); } elsif ($next eq '}') { error($tex,'Encountered } without corresponding {.'); - } elsif ($tex->{'type'}==$TOKEN_MACRO && $status==$STATUS_EXCLUDE_STRONGER) { - # ignore macro call + } elsif ($state==$STATE_EXCLUDE_STRONGER) { + # ignore remaining tokens set_style($tex,'ignore'); + } elsif ($next eq '\documentclass') { + # defines document class and starts preamble + set_style($tex,'document'); + _parse_documentclass_params($tex); + while (!($tex->{'eof'})) { + _parse_unit($tex,$STATE_PREAMBLE); + } } elsif ($tex->{'type'}==$TOKEN_MACRO) { # macro call - _parse_macro($tex,$next,$status); + _parse_macro($tex,$next,$state); } elsif ($next eq '$') { # math inline - _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'$'); + _parse_math($tex,$state,$CNT_COUNT_INLINEMATH,'$'); } elsif ($next eq '$$') { # math display (unless already in inlined math) if (!(defined $end && $end eq '$')) { - _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'$$'); + _parse_math($tex,$state,$CNT_COUNT_DISPLAYMATH,'$$'); } + } elsif ($simple_token) { + #DELETE: simple tokens should be handled properly + #print STDERR '<',$next,'>',"\n"; + # handle as parameter that should not be counted + set_style($tex,'ignore'); } if (!defined $end) {return;} } - if (defined $end) { - error($tex,'Reached end of file while waiting for '.$end.'.'); - } + defined $end && error($tex,'Reached end of file while waiting for '.$end.'.'); } -# Process word with a given status (>0, i.e. counted) +# Print state +sub _set_printstate { + my ($tex,$state,$end)=@_; + $tex->{'printstate'}=':'.state_to_text($state).':'.(defined $end?$end.':':''); + flush_next($tex); +} + +# Process word with a given state (>0, i.e. counted) sub _process_word { - my ($tex,$word,$status)=@_; + my ($tex,$word,$state)=@_; $WordFreq{$word}++; } # Parse unit when next token is a macro sub _parse_macro { - my ($tex,$next,$status)=@_; - # substat = parameter status settings + my ($tex,$next,$state)=@_; my $substat; + my @macro=($next); if (my $label=$BreakPoints{$next}) { if ($tex->{'line'}=~ /^[*]?(\s*\[.*?\])*\s*\{((.|\{.*\})*)\}/ ) { $label=$label.': '.$2; } next_subcount($tex,$label); } - set_style($tex,status_is_text($status)?'command':'exclcommand'); - if ($next eq '\begin' && status_inc_envir($status)) { - _parse_begin_end($tex,$status); - } elsif ($next eq '\end' && status_inc_envir($status)) { + if ($state==$STATE_MATH) {set_style($tex,'mathcmd');} + else {set_style($tex,state_is_text($state)?'cmd':'exclcmd');} + if ($next eq '\begin' && state_inc_envir($state)) { + _parse_envir($tex,$state); + push @macro,'<envir>'; + } elsif ($next eq '\end' && state_inc_envir($state)) { error($tex,'Encountered \end without corresponding \begin'); + push @macro,$STRING_ERROR; } elsif ($next eq '\verb') { - _parse_verb_region($tex,$status); - } elsif ( (status_is_text($status) || $status==$STATUS_PREAMBLE) - && defined ($substat=$TeXpackageinc{$next}) ) { - _parse_include_package($tex,$substat); - } elsif (($status==$STATUS_FLOAT) && ($substat=$TeXfloatinc{$next})) { + _parse_verb_region($tex,$state); + } elsif (state_is_parsed($state) && defined $TeXpackageinc{$next} ) { + _parse_include_package($tex); + push @macro,'<package>'; + } elsif (state_is_parsed($state) && defined (my $def=$TeXfileinclude{$next})) { + # include file (merge in or queue up for parsing) + set_style($tex,'cmd'); + if (defined ($substat=$TeXmacro{'@pre'.$next})) {__gobble_macro_parms($tex,$substat,$state);} + __count_macrocount($tex,$next,$state); + _parse_include_file($tex,$state,$def); + push @macro,'<filespec>'; + if (defined ($substat=$TeXmacro{'@post'.$next})) {__gobble_macro_parms($tex,$substat,$state);} + } elsif (($state==$STATE_FLOAT) && ($substat=$TeXfloatinc{$next})) { # text included from float - set_style($tex,'command'); - __gobble_macro_parms($tex,$substat); - } elsif ($status==$STATUS_PREAMBLE && defined ($substat=$TeXpreamble{$next})) { - # parse preamble include macros - set_style($tex,'command'); - if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);} - __gobble_macro_parms($tex,$substat,1); - } elsif (status_is_exclude($status)) { + set_style($tex,'cmd'); + push @macro,__gobble_macro_parms($tex,$substat,$__STATE_NULL); + } elsif ($state==$STATE_PREAMBLE && defined ($substat=$TeXpreamble{$next})) { + # parse preamble include macros + set_style($tex,'cmd'); + __count_macrocount($tex,$next,$STATE_TEXT); + push @macro,__gobble_macro_parms($tex,$substat,$__STATE_NULL); + } elsif (state_is_exclude($state)) { # ignore - __gobble_option($tex); + push @macro,__gobble_options($tex),'/*ignored*/'; } elsif ($next eq '\(') { # math inline - _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'\)'); + _parse_math($tex,$state,$CNT_COUNT_INLINEMATH,'\)'); } elsif ($next eq '\[') { # math display - _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'\]'); - } elsif ($next eq '\def') { + _parse_math($tex,$state,$CNT_COUNT_DISPLAYMATH,'\]'); + } elsif ($next=~/^\\(def|edef|gdef|xdef)$/) { # ignore \def... $tex->{'line'} =~ s/^([^\{]*)\{/\{/; flush_next($tex); print_style($1,'ignore'); - _parse_unit($tex,$STATUS_EXCLUDE_STRONG); - } elsif (defined (my $addsuffix=$TeXfileinclude{$next})) { - # include file: queue up for parsing - _parse_include_file($tex,$status,$addsuffix); + _parse_unit($tex,$STATE_EXCLUDE_STRONG); + push @macro,'<macro>',$STRING_PARAMETER; } elsif (defined ($substat=$TeXmacro{$next})) { # macro: exclude options - if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);} - __count_macroword($tex,$next,$status); - __gobble_macro_parms($tex,$substat,$status); - } elsif (defined __count_macroword($tex,$next,$status)) { + __count_macrocount($tex,$next,$state); + push @macro,__gobble_macro_parms($tex,$substat,$state); + } elsif (defined __count_macrocount($tex,$next,$state)) { # count macro as word (or a given number of words) - set_style($tex,'word'.$status); - } elsif ($next =~ /^\\[^\w\_]/) { + } elsif ($next =~ /^\\[^\w\_]$/) { + # handle \<symbol> as single symbol macro + push @macro,__gobble_options($tex),'/*symbol*/'; } else { - __gobble_option($tex); + push @macro,__gobble_options($tex),'/*defaultrule*/'; } + $MacroUsage{join('',@macro)}++; } # Parse TC instruction @@ -1469,190 +2167,196 @@ sub _parse_tc { my ($tex,$next)=@_; set_style($tex,'tc'); flush_next($tex); - $next=~s/^\%+TC:\s*(\w+)\s*//i || BLOCK { - error($tex,'TC command should have format %TC:instruction [macro] [parameters]'); - return; - }; + assert($next=~s/^\%+TC:\s*(\w+)\s*//i + ,$tex,'TC command should have format %TC:instruction [[parameters]]') + || return; my $instr=$1; $instr=~tr/[A-Z]/[a-z]/; - if ($instr=~/^(break)$/) { - if ($instr eq 'break') {next_subcount($tex,$next);} + if ($instr eq 'break') {next_subcount($tex,$next); } elsif ($instr=~/^(incbib|includebibliography)$/) { $includeBibliography=1; apply_include_bibliography(); - } elsif ($instr eq 'ignore') { - __gobble_tc_ignore($tex); - } elsif ($instr eq 'endignore') { - error($tex,'TC:endignore without corresponding TC:ignore.'); + } elsif ($instr eq 'ignore') {__gobble_tc_ignore($tex); + } elsif ($instr eq 'endignore') {error($tex,'%TC:endignore without corresponding %TC:ignore.'); } elsif ($instr eq 'newtemplate') {$outputtemplate=''; } elsif ($instr eq 'template') {$outputtemplate.=$next; + } elsif ($instr eq 'insert') { + $tex->{'line'}="\n".$next.$tex->{'line'}; } elsif ($instr eq 'subst') { - if ($next=~/^(\\\S+)\s+/) { + if ($next=~/^(\\\S+)\s+(.*)$/) { my $from=$1; - my $to=$'; + my $to=$2; $substitutions{$from}=$to; apply_substitution_rule($tex,$from,$to); } else { - error($tex,'Invalid TC:subst format.'); + error($tex,'Invalid %TC:subst format.'); } - } elsif ($next=~/^([\\]*\S+)\s+([^\s\n]+)(\s+(-?[0-9]+))?/) { - # Format = TC:word macro + } elsif ($instr eq 'newcounter') { + assert($next=~s/^(\w+)(=(\w+))?\s*//,$tex,'Should have format %TC:newcounter {key}[={like-key}] {description}') + || return; + my $key=$1; + my $like=$3; + if ($next eq '') {$next=$key;} + add_new_counter($key,$next,$like); + } elsif ($next=~/^([\\]*\S+)\s+([^\s]+)(\s+(-?\w+))?/) { + # %TC:instr macro param option my $macro=$1; my $param=$2; my $option=$4; - if (tc_macro_param_option($instr,$macro,$param,$option)) {} - else {error($tex,'Unknown TC command: '.$instr);} + tc_macro_param_option($tex,$instr,$macro,$param,$option); } else { - error($tex,'Invalid TC command format: '.$instr); + error($tex,'Invalid TC command: '.$instr); } } -# Parse through ignored LaTeX code -sub __gobble_tc_ignore { - my ($tex)=@_; - set_style($tex,'ignore'); - _parse_unit($tex,$STATUS_EXCLUDE_ALL,'%TC:endignore'); - set_style($tex,'tc'); - flush_next($tex); -} - # Parse math formulae sub _parse_math { - my ($tex,$status,$cnt,$end)=@_; + my ($tex,$state,$cnt,$end)=@_; my $localstyle; - if (status_is_text($status)) { + if (state_is_text($state)) { $localstyle='mathgroup'; inc_count($tex,$cnt); } else { $localstyle='exclmath'; } set_style($tex,$localstyle); - _parse_unit($tex,$STATUS_IGNORE,$end); + _parse_unit($tex,$STATE_MATH,$end); set_style($tex,$localstyle); } # Parse \verb region sub _parse_verb_region { - my ($tex,$status)=@_; - flush_next($tex); + my ($tex,$state)=@_; + #flush_next($tex); + __gobble_macro_modifier($tex); set_style($tex,'ignore'); - if (!($tex->{'line'} =~ s/^[^\s]// )) { - error($tex,'Invalid \verb: delimiter required.'); - } - my $dlm=$&; - print_style($dlm,'command'); - if (!($tex->{'line'} =~ s/^([^$dlm]*)($dlm)// )) { - error($tex,'Invalid \verb: could not find ending delimiter ('.$dlm.').'); - } + assert($tex->{'line'} =~ s/^([^\s])//,$tex,'Invalid \verb: delimiter required.') + || return; + my $dlm=$1; + print_style($dlm,'cmd'); + assert($tex->{'line'}=~s/^([^\Q$dlm\E]*)(\Q$dlm\E)// + ,$tex,'Invalid \verb: could not find ending delimiter ('.$dlm.').') + || return; print_style($1,'ignore'); - print_style($2,'command'); + print_style($2,'cmd'); } -# Parse begin-end group -sub _parse_begin_end { - my ($tex,$status)=@_; - my $localstyle=status_is_text($status) ? 'grouping' : 'exclgroup'; - flush_next_gobble_space($tex,$localstyle,$status); - #__gobble_option($tex); # no option before group name - my ($groupname,$next); +# Parse environment +sub _parse_envir { + my ($tex,$state)=@_; + my $localstyle=state_is_text($state) ? 'envir' : 'exclenv'; + flush_next_gobble_space($tex,$localstyle,$state); + my ($envirname,$next); if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) { - # gobble group type - $groupname=$1; + # gobble environment name + $envirname=$1; + my @macro=('<envir:'.$envirname.'>'); print_style('{'.$1.'}',$localstyle); - $next='begin'.$groupname; - if (defined (my $substat=$TeXmacro{$next})) { - __gobble_macro_parms($tex,$substat); + $next=$PREFIX_ENVIR.$envirname; + __count_macrocount($tex,$next,$state); + if (defined (my $def=$TeXmacro{$next})) { + push @macro,__gobble_macro_parms($tex,$def,$__STATE_NULL); + } else { + push @macro,__gobble_options($tex); } + $MacroUsage{join('',@macro)}++; } else { - $groupname='???'; $next='???'; + $envirname='???'; $next='???'; error($tex,'Encountered \begin without environment name provided.'); } - # find group status (or leave unchanged) - my $substat=$TeXgroup{$1}; + # find new parsing state (or leave unchanged) + my $substat=$TeXenvir{$1}; if (!defined $substat) { - $substat=$status; + $substat=$state; if ($strictness>=1) { - warning($tex,"Using default rule for group ".$groupname); + warning($tex,'Using default rule for environment '.$envirname); } - } elsif (!status_is_text($status)) { - # Do not raise status - $substat=__new_status($substat,$status); - } elsif ($substat==$STATUS_FLOAT) { - # Count float - inc_count($tex,$CNT_COUNT_FLOAT); - } elsif (my $cnt=status_to_cnt($substat)) { - # Count as given type (was: if $substat>3) - inc_count($tex,$cnt); - $substat=0; } else { - # Use $substat value as given - } - if ($includeBibliography && $groupname eq 'thebibliography' && $substat>0) { - # Add bibliography header - inc_count($tex,$CNT_COUNT_HEADER); - __count_macroword($tex,$groupname,$STATUS_TEXT_HEADER); + $substat=__new_state($substat,$state); } - if (!status_inc_envir($substat)) { - # Keep parsing until appropriate end group arrives - while (!$tex->{'eof'}) { - _parse_unit($tex,$substat,'\end'); - if ($tex->{'line'} =~ s/^\s*\{$groupname\}[ \t\r\f]*//) { - # gobble end group parameter - flush_next($tex,$localstyle,$status); - print_style($&,$localstyle); - return; - } - } - } else { + if (state_inc_envir($substat)) { # Parse until \end arrives, and check that it matches _parse_unit($tex,$substat,'\end'); - flush_next_gobble_space($tex,$localstyle,$status); + flush_next_gobble_space($tex,$localstyle,$state); if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) { - # gobble group type + # gobble environment name print_style('{'.$1.'}',$localstyle); - if ($groupname ne $1) { - error($tex,'Group \begin{'.$groupname.'} ended with end{'.$1.'}.'); - } + assert($envirname eq $1,$tex,"Environment \\begin{$envirname} ended with \\end{$1}."); } else { - error($tex,'Group ended while waiting for \end{'.$groupname.'}.'); + error($tex,"Environment ended while waiting for \end{$envirname}."); + } + } else { + # Keep parsing until appropriate \end arrives ignoring all else + while (!$tex->{'eof'}) { + _parse_unit($tex,$substat,'\end'); + if ($tex->{'line'} =~ s/^(\s*)(\{\Q$envirname\E\})[ \t\r\f]*//) { + # gobble \end parameter and end environment + flush_next($tex,$localstyle,$state); + print_style($1,$localstyle); + print_style($2,$localstyle); + return; + } } } } # Parse and process file inclusion sub _parse_include_file { - my ($tex,$status,$addsuffix)=@_; - if ($addsuffix eq 'bbl') { - _parse_include_bbl($tex,$status); - return; - } + my ($tex,$state,$includetype)=@_; + my $include=state_is_parsed($state); + my $style=$include?'fileinc':'ignore'; + my $file; + my %params; flush_next($tex); - $tex->{'line'} =~ s/^\s*\{([^\{\}\s]+)\}// - || $tex->{'line'} =~ s/^\s*([^\{\}\%\\\s]+)// - || $tex->{'line'} =~ s/^\s*\{(.+?)\}// - || BLOCK { - error($tex,'Failed to read or interpret file name for inclusion.'); + $params{'<type>'}=$includetype; + if ($includetype eq '<bbl>') { + _parse_include_bbl($tex,$state,\%params); return; - }; - if (status_is_text($status)) { - print_style($&,'fileinclude'); - my $fname=$1; - if ($addsuffix==2) {$fname.='.tex';} - elsif ($addsuffix==1 && ($fname=~/^[^\.]+$/)) {$fname.='.tex';} - if ($includeTeX) {include_file($tex,$fname);} - } else { - print_style($&,'ignored'); + } + elsif ($includetype eq 'input') { + $tex->{'line'} =~ s/^(\s*\{([^\{\}\s]+)\})// + || $tex->{'line'} =~ s/^(\s*([^\{\}\%\\\s]+))// + || $tex->{'line'} =~ s/^(\s*\{(.+?)\})// + || BLOCK { + error($tex,'Failed to identify file name.'); + return; + }; + print_style($1,$style); + $file=$2; + } + else { + foreach my $param (split(/\s+|,/,$includetype)) { + if ($param=~/^<.*>$/) {next;} + $tex->{'line'} =~ s/(\s*\{)([^\{\}\%]*)(\})//|| BLOCK { + error($tex,"Failed to identify file parameter {$param}."); + return; + }; + print_style($1,'ignore'); + print_style($2,$style); + print_style($3,'ignore'); + if ($param eq 'file') {$file=$2;} + elsif ($param eq 'texfile') { + $file=$2; + if (!$file=~/\.tex$/i) {$file.='.tex';} + } + else {$params{$param}=$2;} + } + } + if (!defined $file) { + error($tex,'Undefined file name.'); + } + elsif ($includeTeX && $include) { + include_file($tex,$state,$file,\%params); } } # Parse and process bibliography file sub _parse_include_bbl { - my ($tex,$status)=@_; - __gobble_macro_parms($tex,1,$status); - if (status_is_text($status) && $includeBibliography) { - my $fname=$tex->{'filename'}; - $fname=~s/\.\w+$/\.bbl/; - include_file($tex,$fname); + my ($tex,$state,$params)=@_; + if ($includeBibliography && state_is_text($state)) { + my $filename=$tex->{'filename'}; + $filename=~s/(\.tex)?$/\.bbl/i; + include_file($tex,$state,$filename,$params); } } @@ -1661,26 +2365,53 @@ sub _parse_include_package { my ($tex)=@_; set_style($tex,'document'); __gobble_option($tex); - if ( $tex->{'line'}=~s/^\{((\w+)(\s*,\s*\w+)*)\}// ) { - print_style('{'.$1.'}','document'); + if ( $tex->{'line'}=~s/^\{(([\w\-]+)(\s*,\s*[\w\-]+)*)\}// ) { + print_style("{$1}",'document'); foreach (split(/\s*,\s*/,$1)) { - include_package($_); + $MacroUsage{"<package:$_>"}++; + include_package($_,$tex); } } else { - _parse_unit($tex,$STATUS_IGNORE); - error($tex,"Could not recognise package list, ignoring it instead."); + _parse_unit($tex,$STATE_IGNORE); + error($tex,'Could not recognise package list, ignoring it instead.'); } - __gobble_options($tex); } -# Count macroword using given status -sub __count_macroword { - my ($tex,$next,$status)=@_; - my $n=$TeXmacroword{$next}; - if (defined $n && (my $cnt=status_text_cnt($status))) { - inc_count($tex,$cnt,$n); +# Parse \documentclass parameters and include rules +sub _parse_documentclass_params { + my ($tex)=@_; + my $options=__gobble_option($tex); + if ( $tex->{'line'}=~s/^(\{\s*([^\{\}\s]+)\s*\})// ) { + print_style("$1",'document'); + $MacroUsage{"<documentclass:$2>"}++; + include_package("class%$2",$tex); + } else { + _parse_unit($tex,$STATE_IGNORE); + error($tex,'Could not identify document class.'); } - return $n; +} + +# Count macrocount using given state +sub __count_macrocount { + my ($tex,$next,$state)=@_; + my $def=$TeXmacrocount{$next}; + my $cnt; + if (!defined $def) {return undef;} + elsif (ref($def) eq 'ARRAY') { + # TODO: Is this an appropriate style to use? + set_style($tex,state_to_style($state)); + flush_next($tex); + foreach $cnt (@{$def}) { + if ($cnt==$CNT_WORDS_TEXT) {$cnt=state_text_cnt($state);} + inc_count($tex,$cnt); + } + } + elsif ($cnt=state_text_cnt($state)) { + set_style($tex,state_to_style($state)); + flush_next($tex); + inc_count($tex,$cnt,$def); + } + return $def; } # Gobble next option, return option or undef if none @@ -1694,64 +2425,104 @@ sub __gobble_option { return undef; } -# Gobble all options +# Gobble all options, return the number of gobble options sub __gobble_options { - while (__gobble_option(@_)) {} + my $n=0; + while (__gobble_option(@_)) {$n++} + return ($STRING_GOBBLED_OPTION)x$n; } # Gobble macro modifyer (e.g. following *) sub __gobble_macro_modifier { my $tex=shift @_; flush_next($tex); - if ($tex->{'line'} =~ s/^\*//) { + if ($tex->{'line'} =~ s/^(\*)//) { print_style($1,'option'); return $1; } - return undef; + return; } # Gobble macro parameters as specified in parm plus options sub __gobble_macro_parms { my ($tex,$parm,$oldstat)=@_; my $n; + my @ret; if (ref($parm) eq 'ARRAY') { $n=scalar @{$parm}; } else { $n=$parm; - $parm=[($STATUS_IGNORE)x$n]; + $parm=[($STATE_IGNORE)x$n]; } - if ($n>0) {__gobble_macro_modifier($tex);} - __gobble_options($tex); + # TODO: Optionally gobble macro modifier? + if ($n>0) {push @ret,__gobble_macro_modifier($tex);} + my $auto_gobble_options=1; for (my $j=0;$j<$n;$j++) { - _parse_unit($tex,__new_status($parm->[$j],$oldstat)); - __gobble_options($tex); + my $p=$parm->[$j]; + if ($p==$_STATE_OPTION) { + # Parse macro option + $p=$parm->[++$j]; + if ($tex->{'line'}=~s/^(\s*\[)//) { + flush_next_gobble_space($tex); + print_style($1,'optparm'); + push @ret,$STRING_OPTIONAL_PARAM; + _parse_unit($tex,__new_state($p,$oldstat),']'); + set_style($tex,'optparm'); + } + } elsif ($p==$_STATE_NOOPTION) { + $auto_gobble_options=0; + } elsif ($p==$_STATE_AUTOOPTION) { + $auto_gobble_options=1; + } else { + # Parse macro parameter + if ($auto_gobble_options) {push @ret,__gobble_options($tex);} + push @ret,$STRING_PARAMETER; + _parse_unit($tex,__new_state($p,$oldstat),$_PARAM_); + } } + #TODO: Drop default gobbling of option at end? + if ($auto_gobble_options) {push @ret,__gobble_options($tex);} + return @ret; } -# Return new parsing status given old and substatus -sub __new_status { +# Parse through ignored LaTeX code +sub __gobble_tc_ignore { + my ($tex)=@_; + set_style($tex,'ignore'); + _parse_unit($tex,$STATE_EXCLUDE_ALL,'%TC:endignore'); + set_style($tex,'tc'); + flush_next($tex); +} + +# Return new parsing state given old and substate +sub __new_state { my ($substat,$oldstat)=@_; if (!defined $oldstat) {return $substat;} - foreach my $st (@STATUS_PRIORITY_LIST) { + foreach my $st (@STATE_FIRST_PRIORITY,@STATE_MID_PRIORITY,@STATE_LAST_PRIORITY) { if ($oldstat==$st || $substat==$st) {return $st;} } - error($Main,'Could not determine new status in __new_status!','BUG'); + error($Main,'Could not determine new state in __new_state!','BUG'); return $oldstat; } + +###### Tokenisation routines + + # Get next token skipping comments and flushing output buffer -sub _next_token { - my $tex=shift @_; +sub next_token { + my ($tex,$simple_token)=@_; + if (!defined $simple_token) {$simple_token=0;} my ($next,$type); my $style=$tex->{'style'}; if (defined $tex->{'next'}) {print_style($tex->{'next'},$tex->{'style'});} $tex->{'style'}=undef; - while (defined ($next=__get_next_token($tex))) { + while (defined ($next=_get_next_token($tex,$simple_token))) { $type=$tex->{'type'}; if ($type==$TOKEN_COMMENT) { print_style($next,'comment'); } elsif ($type==$TOKEN_LINEBREAK) { - if ($verbose>0) {line_return(-1,$tex);} + if ($printlevel>0) {line_return(-1,$tex);} } else { return $next; } @@ -1760,20 +2531,9 @@ sub _next_token { } # Read, interpret and return next token -sub __get_next_token { - # Token (or token group) category: - # -1: space - # 0: comment - # 1: word (or other forms of text or text components) - # 2: symbol (not word, e.g. punctuation) - # 3: macro - # 4: curly braces {} - # 5: brackets [] - # 6: maths - # 9: line break in file - # 999: end of line or blank line - # 666: TeXcount instruction (%TC:instruction) - my $tex=shift @_; +# If simple_token is set, words cannot form tokens +sub _get_next_token { + my ($tex,$simple_token)=@_; my $next; my $ch; while (!$tex->{'eof'}) { @@ -1787,8 +2547,18 @@ sub __get_next_token { } elsif ($ch eq "\n" || $ch eq "\r") { $tex->{'line'}=~s/^(\r\n?|\n)//; return __set_token($tex,$1,$TOKEN_LINEBREAK); - } elsif ($tex->{'line'}=~s/^($WordPattern)//) { + } elsif (!$simple_token && $tex->{'line'}=~s/^($WordPattern)//o) { return __set_token($tex,$1,$TOKEN_WORD); + } elsif ($simple_token && $tex->{'line'}=~s/^(\w)//) { + # When parsing simple tokens (not words), only include single characters + # TODO: the handling of simple tokens should be improved + my $match=$1; + if ($match=~/^$WordPattern$/) {return __set_token($tex,$match,$TOKEN_WORD);} + else {return __set_token($tex,$match,$TOKEN_SYMBOL);} + } elsif ($ch eq '\\') { + if ($tex->{'line'}=~s/^(\\[{}%])//) {return __set_token($tex,$1,$TOKEN_SYMBOL);} + if ($tex->{'line'}=~s/^(\\([a-zA-Z@]+|[^a-zA-Z@[:^print:]]))//) {return __set_token($tex,$1,$TOKEN_MACRO);} + return __get_chtoken($tex,$ch,$TOKEN_END); } elsif ($ch eq '$') { $tex->{'line'}=~s/^(\$\$?)//; return __set_token($tex,$1,$TOKEN_MATH); @@ -1806,10 +2576,6 @@ sub __get_next_token { if ($tex->{'line'}=~s/^(\%+[tT][cC]:[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_TC);} if ($tex->{'line'}=~s/^(\%+[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_COMMENT);} return __get_chtoken($tex,$ch,$TOKEN_COMMENT); - } elsif ($ch eq '\\') { - if ($tex->{'line'}=~s/^(\\[{}%])//) {return __set_token($tex,$1,$TOKEN_SYMBOL);} - if ($tex->{'line'}=~s/^(\\([a-zA-Z_]+|[^a-zA-Z_]))//) {return __set_token($tex,$1,$TOKEN_MACRO);} - return __get_chtoken($tex,$ch,$TOKEN_END); } else { return __get_chtoken($tex,$ch,$TOKEN_END); } @@ -1840,30 +2606,26 @@ sub __get_chtoken { ## Make new count object -# The "count object" is a hash containing +# The "count object" is a hash containing: # - title: the title of the count (name of file, section, ...) # - counts: a list of numbers (the counts: files, text words, ...) -# upon creation, but where the element -# - subcounts: list of count objects (added by the TeX object) -# may exist if the count contains subcounts. The elements of the -# count are (by their index): -# 0 = #files: counts the number of files -# 1 = text words: counts the number of words in the text -# 2 = header words: number of words in headers -# 3 = caption words: number of words in float captions -# 4 = #headers: number of headers -# 5 = #floats: number of tables, figures, floats, etc. -# 6 = #inline formulae: number of math elements in text ($...$) -# 7 = #displayed formulae: number of displayed equations +# - subcounts: list of count objects (used by the TeX object) +# - warnings: list of warnings produced +# The elements are specified by the $CONT_* constants. sub new_count { my ($title)=@_; my @cnt=(0) x $SIZE_CNT; - my %count=('counts'=>\@cnt,'title'=>$title); - # files, text words, header words, float words, - # headers, floats, math-inline, math-display; + my %count=('counts'=>\@cnt,'title'=>$title,'subcounts'=>[],'errorcount'=>0,'warnings'=>{}); return \%count; } +# Ensure the count array is of the right size +sub update_count_size { + my $count=shift @_; + my $counts=$count->{'counts'}; + while (scalar @{$counts}<$SIZE_CNT) {push @$counts,0;} +} + # Increment TeX count for a given count type sub inc_count { my ($tex,$cnt,$value)=@_; @@ -1875,15 +2637,17 @@ sub inc_count { # Get count value for a given count type sub get_count { my ($count,$cnt)=@_; - return ${$count->{'counts'}}[$cnt]; + my $counts=$count->{'counts'}; + if ($cnt<scalar @{$counts}) {return ${$counts}[$cnt];} + else {return 0;} } # Compute sum count for a count object sub get_sum_count { my $count=shift @_; my $sum=0; - for (my $i=scalar(@sumweights);$i-->0;) { - $sum+=get_count($count,$i+1)*$sumweights[$i]; + for (my $i=scalar(@sumweights);$i-->1;) { + $sum+=get_count($count,$i)*$sumweights[$i]; } return $sum; } @@ -1898,6 +2662,20 @@ sub number_of_subcounts { } } +# Returns the number of warnings (not distinct) +sub number_of_warnings { + my $count=shift @_; + my $n=0; + foreach my $m (values %{$count->{'warnings'}}) {$n+=$m;} + return $n; +} + +# Returns the number of distinct warnings +sub number_of_distinct_warnings { + my $count=shift @_; + return scalar keys %{$count->{'warnings'}}; +} + # Is a null count? (counts 1-7 zero, title starts with _) sub _count_is_null { my $count=shift @_; @@ -1905,15 +2683,23 @@ sub _count_is_null { for (my $i=1;$i<$SIZE_CNT;$i++) { if (get_count($count,$i)>0) {return 0;} } + if (scalar keys %{$count->{'warnings'}}) {return 0;} + if ($count->{'errorcount'}) {return 0;} return 1; } # Add one count to another sub _add_to_count { my ($a,$b)=@_; + update_count_size($a); + update_count_size($b); for (my $i=0;$i<$SIZE_CNT;$i++) { ${$a->{'counts'}}[$i]+=${$b->{'counts'}}[$i]; } + while (my ($text,$n)=each %{$a->{'warnings'}}) { + $a->{'warnings'}->{$text}+=$n; + } + $a->{'errorcount'}+=$b->{'errorcount'}; } # Add subcount to sum count and prepare new subcount @@ -1928,39 +2714,38 @@ sub next_subcount { sub add_to_total { my ($total,$count)=@_; _add_to_count($total,$count); - if (!_count_is_null($count)) { - push @{$total->{'subcounts'}},$count; - $count->{'parentcount'}=$total; - } + if (_count_is_null($count)) {return;} + push @{$total->{'subcounts'}},$count; + $count->{'parentcount'}=$total; } - ###### Result output routines -# Close the output, e.g. adding HTML tail +# Close the output (STDOUT), e.g. adding HTML tail sub Close_Output { if ($htmlstyle>1) { html_tail(); } + close STDOUT; } # Report if there were any errors occurring during parsing sub Report_Errors { if (defined $outputtemplate) {return;} - if ( !$briefsum && !$totalflag && $verbose>=0 ) { - foreach (keys %warnings) {formatprint($_,"p","nb");print "\n";} + if ( !$briefsum && !$totalflag && $printlevel>=0 ) { + foreach (keys %warnings) {formatprint($_,'p','nb');print "\n";} } if ($errorcount==0) {return;} - if ($briefsum && $totalflag) {print " ";} + if ($briefsum && $totalflag) {print ' ';} if ($htmlstyle) { print "<div class='error'><p>\n"; - print "There were ".$errorcount." error(s) reported!\n"; + print "There were $errorcount error(s) reported!\n"; print "</p></div>\n"; } elsif ($briefsum && $totalflag) { - print "(errors:".$errorcount.")"; + print "(errors:$errorcount)"; } else { - print "(errors:".$errorcount.")\n"; + print "(errors:$errorcount)\n"; } } @@ -1979,14 +2764,14 @@ sub print_word_freq { $Word{$wd}=__lc_merge($word,$Word{$wd}); } if ($htmlstyle) {print "<table class='stat'>\n<thead>\n";} - __print_word_freq("Word","Freq","th"); + __print_word_freq('Word','Freq','th'); if ($htmlstyle) {print "</thead>\n";} if ($optionWordClassFreq>0) { for $word (keys %Freq) {$Class{__word_class($word,\%regs)}+=$Freq{$word};} __print_sorted_freqs('langstat',\%Class); } if ($htmlstyle) {print "<tbody class='sumstat'>\n";} - __print_word_freq("All words",$sum,,'td','sum'); + __print_word_freq('All words',$sum,'td','sum'); if ($htmlstyle) {print "</tbody>\n";} if ($optionWordFreq>0) {__print_sorted_freqs('wordstat',\%Freq,\%Word,$optionWordFreq);} if ($htmlstyle) {print "</table>\n";} @@ -2026,7 +2811,7 @@ sub __print_sorted_freqs { my $sum=0; my ($word,$wd,$freq); if (!defined $min) {$min=0;} - if ($htmlstyle) {print "<tbody class='",$class,"'>\n";} + if ($htmlstyle) {print "<tbody class='$class'>\n";} else {print "---\n";} for $wd (sort {$Freq->{$b} <=> $Freq->{$a}} keys %{$Freq}) { if (defined $Word) {$word=$Word->{$wd};} else {$word=$wd;} @@ -2036,7 +2821,7 @@ sub __print_sorted_freqs { __print_word_freq($word,$freq); } } - if ($min>0) {__print_word_freq("Sum of subset",$sum,'td','sum');} + if ($min>0) {__print_word_freq('Sum of subset',$sum,'td','sum');} if ($htmlstyle) {print "</tbody>\n";} } @@ -2044,27 +2829,45 @@ sub __print_sorted_freqs { sub __print_word_freq { my ($word,$freq,$tag,$class)=@_; if (!defined $tag) {$tag='td';} - if (defined $class) {$class=' class=\''.$class.'\'';} else {$class='';} + if (defined $class) {$class=" class='$class'";} else {$class='';} if ($htmlstyle) { - print "<tr",$class,"><",$tag,">",$word,"</",$tag,"><",$tag,">",$freq,"</",$tag,"></tr>\n"; + print "<tr$class><$tag>$word</$tag><$tag>$freq</$tag></tr>\n"; } else { - print $word,": ",$freq,"\n"; + print $word,': ',$freq,"\n"; + } +} + +# Print macro usage statistics +sub print_macro_stat { + if ($htmlstyle) {print "<table class='stat'>\n<thead>\n";} + __print_word_freq('Macro/envir','Freq','th'); + if ($htmlstyle) {print "</thead>\n";} + if ($htmlstyle) {print "<tbody class='macrostat'>\n";} + foreach my $name (sort keys %MacroUsage) { + my $freq=$MacroUsage{$name}; + $name=text_to_printable($name); + $name=text_to_print($name); + if ($htmlstyle) { + print "<tr><td>$name</td><td>$freq</td></tr>\n"; + } else { + print $name,': ',$freq,"\n"; + } } + if ($htmlstyle) {print "</tbody>\n";} + if ($htmlstyle) {print "</table>\n";} } + ###### Printing routines # Print text using given style/colour sub print_with_style { my ($text,$style,$colour)=@_; - if ($style eq ' ') { - print text_to_print($text); - } elsif ($style eq '') { + if ($style eq $NOSTYLE) { print text_to_print($text); - error($Main,'Empty style should not occur in print_with_style!','BUG'); } elsif ($htmlstyle) { - print "<span class='$style'>".text_to_print($text).'</span>'; + print "<span class='$style'>",text_to_print($text),'</span>'; } else { ansiprint(text_to_print($text),$colour); if ($style=~/$separatorstyleregex/) {print $separator;} @@ -2079,18 +2882,35 @@ sub text_to_print { $text=~s/</</g; $text=~s/>/>/g; $text=~s/[ \t]{2}/\ /g; + } elsif ($texcodeoutput) { + $text=~s/\\/\\textbackslash¤/g; + $text=~s/([%{}])/\\$1/g; + $text=~s/¤/{}/g; } return $text; } +# Return &#xxx; code for character ; +sub convert_to_charcode { + my $ch=shift @_; + return '&#'.ord($ch).';'; +} + +# Convert special characters to charcodes to make it printable ; +sub text_to_printable { + my $text=shift @_; + $text=~s/([^[:print:]])/convert_to_charcode($1)/gse; + return $text; +} + # Print text, using appropriate tags for HTML sub formatprint { my ($text,$tag,$class)=@_; my $break=($text=~s/(\r\n?|\n)$//); if ($htmlstyle && defined $tag) { print '<'.$tag; - if ($class) {print " class='".$class."'";} - print '>'.text_to_print($text).'</'.$tag.'>'; + if ($class) {print " class='$class'";} + print '>'.text_to_print($text)."</$tag>"; } else { print text_to_print($text); } @@ -2130,7 +2950,7 @@ sub __count_and_header { return $count,$header,@_; } -# Return count title or "" if missing +# Return count title or '' if missing sub __count_header { my $count=shift @_; return $count->{'title'}||''; @@ -2150,16 +2970,19 @@ sub _print_sum_count { sub _print_count_brief { my ($count,$header,$tag1,$tag2)=__count_and_header(@_); my @cnt=@{$count->{'counts'}}; - if ($htmlstyle && $tag1) {print "<".$tag1.">";} - print $cnt[$CNT_WORDS_TEXT]."+".$cnt[$CNT_WORDS_HEADER]."+".$cnt[$CNT_WORDS_CAPTION]. - " (".$cnt[$CNT_COUNT_HEADER]."/".$cnt[$CNT_COUNT_FLOAT]. - "/".$cnt[$CNT_COUNT_INLINEMATH]."/".$cnt[$CNT_COUNT_DISPLAYMATH].")"; + if ($htmlstyle && $tag1) {print '<',$tag1,'>';} + print $cnt[$CNT_WORDS_TEXT],'+',$cnt[$CNT_WORDS_HEADER],'+',$cnt[$CNT_WORDS_OTHER]; + for (my $i=$SIZE_CNT_DEFAULT;$i<$SIZE_CNT;$i++) { + if ($cnt[$i]) {print '+',$cnt[$i],$countkey[$i];} + } + print ' (',$cnt[$CNT_COUNT_HEADER],'/',$cnt[$CNT_COUNT_FLOAT], + '/',$cnt[$CNT_COUNT_INLINEMATH],'/',$cnt[$CNT_COUNT_DISPLAYMATH],')'; if ($htmlstyle && $tag2) { - print "</".$tag1."><".$tag2.">"; + print '</',$tag1,'><',$tag2,'>'; $tag1=$tag2; - } else {print " ";} + } else {print ' ';} print text_to_print($header); - if ($htmlstyle && $tag1) {print "</".$tag1.">";} + if ($htmlstyle && $tag1) {print '</',$tag1,'>';} if ($finalLineBreak) {print "\n";} } @@ -2173,10 +2996,10 @@ sub _print_count_details { } if (@sumweights) {formatprint('Sum count: '.get_sum_count($count)."\n",'li');} for (my $i=1;$i<$SIZE_CNT;$i++) { - formatprint($countlabel[$i].': '.get_count($count,$i)."\n",'li'); + formatprint($countdesc[$i].': '.get_count($count,$i)."\n",'li'); } if (get_count($count,$CNT_FILE)>1) { - formatprint($countlabel[$CNT_FILE].': '.get_count($count,$CNT_FILE)."\n",'li'); + formatprint($countdesc[$CNT_FILE].': '.get_count($count,$CNT_FILE)."\n",'li'); } my $subcounts=$count->{'subcounts'}; if ($showsubcounts && defined $subcounts && scalar(@{$subcounts})>=$showsubcounts) { @@ -2198,14 +3021,14 @@ sub _print_count_template { $template=~s/\\n/\n/g; if ($htmlstyle) {$template=~s/\n/<br>/g;} my ($subtemplate,$posttemplate); - while ($template=~/\{SUB\?((.*?)\|)?(.*?)(\|(.*?))?\?SUB\}/is) { - __print_count_using_template($count,$`); + while ($template=~/^(.*)\{SUB\?((.*?)\|)?(.*?)(\|(.*?))?\?SUB\}(.*)$/is) { + __print_count_using_template($count,$1); # $1 ~ ${^PREMATCH} if (number_of_subcounts($count)>1) { - if (defined $2) {print $2;} - __print_subcounts_using_template($count,$3); - if (defined $5) {print $5;} + if (defined $3) {print $3;} + __print_subcounts_using_template($count,$4); + if (defined $6) {print $6;} } - $template=$'; + $template=$7; # $7 ~ ${^POSTMATCH} } __print_count_using_template($count,$template); } @@ -2213,16 +3036,16 @@ sub _print_count_template { # Print counts using template sub __print_count_using_template { my ($count,$template)=@_; - for (my $i=0;$i<$SIZE_CNT;$i++) { - $template=__process_template($template,$i,get_count($count,$i)); - } - $template=~s/\{VER\}/$versionnumber/gi; - # TODO: Should base warnings and errors on TeXcode or Main object - $template=__process_template($template,"W|WARN|WARNING|WARNINGS",length(%warnings)); - $template=__process_template($template,"E|ERR|ERROR|ERRORS",$errorcount); - $template=__process_template($template,"S|SUM",get_sum_count($count)); - $template=__process_template($template,"T|TITLE",$count->{'title'}||""); - $template=__process_template($template,"SUB",number_of_subcounts($count)); + while (my ($key,$cnt)=each %key2cnt) { + $template=__process_template($template,$key,get_count($count,$cnt)); + } + $template=~s/\{VER\}/\Q$versionnumber\E/gi; + $template=__process_template($template,'WARN|WARNING|WARNINGS',number_of_distinct_warnings($count)); + $template=__process_template($template,'NWARN|NWARNING|NWARNINGS',number_of_warnings($count)); + $template=__process_template($template,'ERR|ERROR|ERRORS',$count->{'errorcount'}); + $template=__process_template($template,'SUM',get_sum_count($count)); + $template=__process_template($template,'TITLE',$count->{'title'}||''); + $template=__process_template($template,'SUB',number_of_subcounts($count)); print $template; } @@ -2246,7 +3069,7 @@ sub __process_template { $template=~s/\{($label)\?(.*?)\|(.*?)\?(\1)\}/$3/gis; $template=~s/\{($label)\?(.*?)\?(\1)\}//gis; } - if (!defined $value) {$value="";} + if (!defined $value) {$value='';} $template=~s/\{($label)\}/$value/gis; return $template; } @@ -2259,56 +3082,61 @@ sub __process_template { sub flush_next { my ($tex,$style)=@_; my $ret=undef; - if (defined $style && $style ne '') { - set_style($tex,$style); - } + if (defined $style) {set_style($tex,$style);} if (defined $tex->{'next'}) { $ret=print_style($tex->{'next'},$tex->{'style'},$tex->{'printstate'}); } $tex->{'printstate'}=undef; - $tex->{'style'}='-'; + $tex->{'style'}=$STYLE_BLOCK; return $ret; } -# Print next token and gobble following spaces +# Print next token and gobble following spaces (incl. line break and comments) sub flush_next_gobble_space { - my ($tex,$style,$status)=@_; + my ($tex,$style,$state)=@_; my $ret=flush_next($tex,$style); if (!defined $ret) {$ret=0;} - if (!defined $status) {$status=$STATUS_IGNORE;} - my $prt=($verbose>0); - if ($tex->{'line'}=~s/^([ \t\f]*)(\r\n?|\n)([ \t\f]*)//) { - if (!$prt) { - } elsif ($verbose>2 || $ret) { - print $1; - line_return(-1,$tex); - my $space=$3; - if ($htmlstyle) {$space=~s/ /\ /g;} - print $space; - } else { - line_return(0,$tex); + if (!defined $state) {$state=$STATE_IGNORE;} + my $prt=($printlevel>0); + while (1) { + if ($tex->{'line'}=~s/^([ \t\f]*)(\r\n|\r|\n)([ \t\f]*)//) { + if (!$prt) { + } elsif ($printlevel>1 || $ret) { + print $1; + line_return(-1,$tex); + my $space=$3; + if ($htmlstyle) {$space=~s/ /\ /g;} + print $space; + $ret=0; + } else { + line_return(0,$tex); + } + } elsif ($tex->{'line'}=~s/^([ \t\f]+)//) { + if ($prt) {print $1;} } - } elsif ($tex->{'line'}=~s/^([ \t\f]*)//) { - if ($prt) {print $1;} + if ($tex->{'line'}=~s/^(\%+[^\r\n]*)//) { + print_style($1,'comment'); + $ret=1; + } else {return;} } } # Set presentation style sub set_style { my ($tex,$style)=@_; - if (!(($tex->{'style'}) && ($tex->{'style'} eq '-')) && $style ne '') {$tex->{'style'}=$style;} + if (!(defined $tex->{'style'} && $tex->{'style'} eq $STYLE_BLOCK)) {$tex->{'style'}=$style;} } # Print text using the given style, and log state if given sub print_style { my ($text,$style,$state)=@_; - (($verbose>=0) && (defined $text) && (defined $style)) || return 0; + (($printlevel>=0) && (defined $text) && (defined $style)) || return 0; my $colour; ($colour=$STYLE{$style}) || return; - if (($colour) && !($colour eq '-')) { + if (defined $colour && $colour ne '-') { print_with_style($text,$style,$colour); - if ($state) {print_style($state,'state');} - if ($style ne "cumsum") {$blankline=-1;} + if (defined $state) {print_style($state,'state');} + if ($style ne 'cumsum') {$blankline=-1;} return 1; } else { return 0; @@ -2318,89 +3146,56 @@ sub print_style { # Conditional line return sub line_return { my ($blank,$tex)=@_; - if ($blank<0 && $verbose<3) {$blank=1;} + if ($blank<0 && $printlevel<2) {$blank=1;} if ($blank<0 || $blank>$blankline) { if ((defined $tex) && @sumweights) { my $num=get_sum_count($tex->{'subcount'}); - print_style(" [".$num."]","cumsum"); + print_style(" [$num]",'cumsum'); } linebreak(); $blankline++; } } -# Print error message -sub _print_error { - my $text=shift @_; - line_return(1); - print_style("!!! ".$text." !!!",'error'); - line_return(1); -} - -# Print errors in buffer and delete errorbuffer -sub flush_errorbuffer { - my $source=shift @_; - my $err=$source->{'errorbuffer'} || return; - foreach (@$err) {_print_error($_);} - $source->{'errorbuffer'}=undef; -} - ###### Print help on style/colour codes # Print output style codes if conditions are met -sub conditional_print_help_style { - if ($showcodes) {_print_help_style();} +sub conditional_print_style_list { + if ($showcodes) {_print_style_list();} return $showcodes; } # Print help on output styles -sub _print_help_style { - if ($verbose<=0) {return;} - if ($htmlstyle) {print "<div class='stylehelp'><p>";} - formatprint("Format/colour codes of verbose output:","h2"); +sub _print_style_list { + if ($printlevel<=0) {return;} + if ($htmlstyle) {print '<div class="stylehelp"><p>';} + formatprint('Format/colour codes of verbose output:','h2'); print "\n\n"; - _help_style_line('Text which is counted',"word1","counted as text words"); - _help_style_line('Header and title text',"word2","counted as header words"); - _help_style_line('Caption text and footnotes',"word3","counted as caption words"); - _help_style_line("Ignored text or code","ignore","excluded or ignored"); - _help_style_line('\documentclass',"document","document start, beginning of preamble"); - _help_style_line('\macro',"command","macro not counted, but parameters may be"); - _help_style_line('\macro',"exclcommand","macro in excluded region"); - _help_style_line("[Macro options]","option","not counted"); - _help_style_line('\begin{group} \end{group}',"grouping","begin/end group"); - _help_style_line('\begin{group} \end{group}',"exclgroup","begin/end group in excluded region"); - _help_style_line('$ $',"mathgroup","counted as one equation"); - _help_style_line('$ $',"exclmath","equation in excluded region"); - _help_style_line('% Comments',"comment","not counted"); - _help_style_line('%TC:TeXcount instructions',"tc","not counted"); - _help_style_line("File to include","fileinclude","not counted but file may be counted later"); - if ($showstates) { - _help_style_line('[state]',"state","internal TeXcount state"); - } - if (@sumweights) { - _help_style_line('[cumsum]',"cumsum","cumulative sum count"); - } - _help_style_line("ERROR","error","TeXcount error message"); - if ($htmlstyle) {print "</p></div>";} + foreach my $style (@STYLE_LIST) { + my $desc=$STYLE_DESC{$style}; + if ($desc=~/^(.*):\s+(.*)$/) { + _help_style_line($1,$style,$2); # $1~${^PREMATCH}, $2~${^POSTMATCH} + } else { + _help_style_line($desc,$style); + } + } + if ($htmlstyle) {print '</p></div>';} print "\n\n"; } # Print one line of help sub _help_style_line { my ($text,$style,$comment)=@_; - if ($htmlstyle) { - $comment=" .... ".text_to_print($comment); - } else { - $comment=" .... ".$comment; - } + if (!defined $comment) {$comment='';} + if ($htmlstyle) {$comment=' .... '.text_to_print($comment);} + else {$comment=' .... '.$comment;} if (print_style($text,$style)) { print $comment; linebreak(); } } - ###### Help routines @@ -2419,12 +3214,36 @@ sub print_license { wprintstringdata('License'); } -# Print TeXcount parameter list +# Print short TeXcount help +sub print_short_help { + wprintstringdata('ShortHelp'); +} + +# Print TeXcount options list sub print_syntax { wprintstringdata('OptionsHead'); wprintstringdata('Options','@ - :'); } +# Prinst TeXcount options containing substring +sub print_syntax_subset { + my $pattern=shift @_; + my $data=StringData('Options'); + if (!defined $data) { + error($Main,'No StringData Options.','BUG'); + return; + } + my @options; + foreach (@$data) { + if (/^\s*([^\s]+\s)*[^\s]*\Q$pattern\E/) {push @options,$_;} + } + if (scalar(@options)==0) {print "No options contained $pattern.\n";} + else { + print "Options containing \"$pattern\":\n\n"; + wprintlines('@ - :',@options); + } +} + # Print complete TeXcount help sub print_help { print_help_title(); @@ -2444,13 +3263,13 @@ sub print_help_text { wprintstringdata('TCinstructions'); } -# Print help on specific macro or group -sub print_help_on { +# Print help on specific macro or environment +sub print_help_on_rule { my $arg=shift @_; my $def; my %rules=( '\documentclass' => 'Initiates LaTeX document preamble.', - '\begin' => 'Treatmend depends on group handling rules.', + '\begin' => 'Treatmend depends on environment handling rules.', '\def' => 'Excluded from count.', '\verb' => 'Strong exclude for enclosed region.', '$' => 'Opens or closes inlined equation', @@ -2460,37 +3279,46 @@ sub print_help_on { '\[' => 'Opens displayed equation.', '\]' => 'Closes displayed equation initiated by \[.'); if (!defined $arg || $arg=~/^\s*$/) { - print "Specify macro or group name after the -h= option.\n"; + print "\nSpecify macro or environment name after the -h= option.\n"; return; } + if ($arg=~/^[\w\-\%]+:/) { + remove_all_rules(); + %rules=(); + while ($arg=~s/^([\w\-\%]+)://) {include_package($1);} + } if ($def=$rules{$arg}) { - print "Special rule (hard coded) for $arg\n"; + print "\nSpecial rule (hard coded) for $arg\n"; print $def."\n"; } elsif ($arg=~/^\\/) { + my $hasrule=0; + print "\nRule(s) for macro $arg\n"; + if ($def=$TeXmacrocount{$arg}) { + $hasrule=1; + } if ($def=$TeXfileinclude{$arg}) { - print "Rule for macro $arg\n"; + $hasrule=1; print "Takes file name as parameter which is included in document.\n"; - } elsif ($def=$TeXmacro{$arg}) { - print "Rule for macro $arg\n"; + } + if ($def=$TeXmacro{$arg}) { + $hasrule=1; _print_rule_macro($arg,$def); - if ($def=$TeXheader{$arg}) { - print "This macro is counted as a header\n"; - } - if ($def=$TeXfloatinc{$arg}) { - print "This macro is also counted inside floats as captions.\n"; - } - } elsif ($def=$TeXmacroword{$arg}) { - print "Rule for macro $arg\n"; - print "Count as ".$def." word(s).\n"; - } else { - print "No macro rule defined for $arg.\nParameters treated as surrounding text.\n"; + } + if ($def=$TeXfloatinc{$arg}) { + $hasrule=1; + print "\nIncluded inside floats:\n"; + _print_rule_macro($arg,$def); + } + if (!$hasrule) { + print "\nNo macro rule defined for $arg.\nParameters treated as surrounding text.\n"; } } else { - if ($def=$TeXgroup{$arg}) { - print "Rule for group $arg\n"; - _print_rule_group($arg,$def); + if ($def=$TeXenvir{$arg}) { + print "\nRule for environment $arg\n"; + _print_rule_macrocount($PREFIX_ENVIR.$arg); + _print_rule_envir($arg,$def); } else { - print "No default group rule defined for $arg.\nContent handled as surrounding text.\n"; + print "\nNo default environment rule defined for $arg.\nContent handled as surrounding text.\n"; } } } @@ -2498,60 +3326,119 @@ sub print_help_on { # Print macro handling rule sub _print_rule_macro { my ($arg,$def)=@_; - my %rules=( - 0 => 'Exclude from count', - 1 => 'Count as text', - 2 => 'Count as header', - 3 => 'Count as caption', - -1 => 'Exclude as float (i.e. include captions)', - -2 => 'Strong exclude (ignore begin-end groups)', - -3 => 'Stronger exclude (ignore all macros)', - -9 => 'Exclude as preamble'); if (ref($def) eq 'ARRAY') { - print "Takes ".scalar(@{$def})." parameter(s):\n"; - foreach my $i (@{$def}) { - print " - ".$rules{$i}."\n"; + my $optionflag=0; + print "Takes the following parameter(s):\n"; + foreach my $state (@{$def}) { + if ($state==$_STATE_OPTION) {$optionflag=1;} + elsif ($optionflag) { + $optionflag=0; + print " - Optional [] containing $state2desc{$state}\n"; + } else { + print " - $state2desc{$state}\n"; + } } } else { - print "Takes ".$def." parameter(s), not included in counts.\n"; + print "Takes $def parameter(s): content ignored, i.e. not included in counts.\n"; } } -# Print group handling rule -sub _print_rule_group { +# Print environment handling rule +sub _print_rule_envir { my ($arg,$def)=@_; - my %rules=( - 0 => 'Not included', - 1 => 'Text, words included in text count', - 2 => 'Header, words included in header count', - 3 => 'Float caption, words included in float caption count', - 6 => 'Inline mathematics, words not counted', - 7 => 'Displayed mathematics, words not counted', - -1 => 'Float, not included, but looks for captions'); - print "Rule used: ".$rules{$def}."\n"; - if ($def=$TeXmacro{'begin'.$arg}) { + print "Contents parsed as $state2desc{$def}\n"; + if ($def=$TeXmacro{$PREFIX_ENVIR.$arg}) { _print_rule_macro($def); } } +# Print macrocount rule, return rule +sub _print_rule_macrocount { + my $arg=shift @_; + my $def=$TeXmacrocount{$arg}; + if (!defined $def) {return undef;} + if (ref($def) eq 'ARRAY') { + my @wd=@$def; + foreach (@wd) {$_=$countdesc[$_];} + print 'Increments the following counters: ',join('; ',@wd),".\n"; + } else { + print "Counted as $def word(s).\n"; + } +} + +# Print style or style category summary +sub print_help_on_styles { + my $style=shift @_; + if (defined $style) { + if (my $def=$STYLES{$style}) { + _print_help_on_style_category($style); + print wrap('',' ',"$style = ".join(', ',grep(/^\w+$/,sort keys %$def))),".\n"; + } elsif ($def=$STYLE_DESC{$style}) { + print $style,' = ',$def,"\n"; + } else { + print "Unknown style or style category: $style.\n"; + } + } else { + print wrap('',' ','Styles: '.join(', ',@STYLE_LIST)),".\n\n"; + print wrap('',' ','Style categories: '.join(', ',sort grep(/^\w+$/,keys %STYLES))),".\n\n"; + print "Use -help-style={style} to get help on particular style or style category.\n"; + } +} + +# Print help on a particular style category (optional prefix and indentation) +sub _print_help_on_style_category { + my ($cat,$prefix,$indent)=@_; + if (!defined $prefix) {$prefix='';} + if (!defined $indent) {$indent=' ';} +} ###### HTML routines # Print HTML header sub html_head { - print "<html>\n<head>"; - print "\n<meta http-equiv='content-type' content='text/html; charset=$outputEncoding'>\n"; - _print_html_style(); - print "</head>\n\n<body>\n"; - print "\n<h1>LaTeX word count"; - if ($showVersion>0) {print " (version ",_html_version(),")"} - print "</h1>\n"; + if (defined $HTMLfile) { + if (open(FH,$HTMLfile)) { + my $text=join('',<FH>); + close FH; + if ( $text=~/^(.*)<!--+\s*texcount\s*--+>(.*)$/is + || $text=~/^(.*)\$\{texcount\}(.*)$/is + || $text=~/^(.*)(<\/body.*)$/is + ) { + $htmlopen=$1; + $htmlclose=$2; + } else { + error($Main,"Invalid HTML template format."); + } + } else { + error($Main,"HTML template file not found: $HTMLfile"); + } + } + if (defined $htmlopen) { + $htmlopen=~s/\$\{encoding\}/\Q$outputEncoding\E/g; + $htmlopen=~s/\$\{version\}/@{[_html_version()]}/g; + print $htmlopen; + $htmlopen=undef; + } else { + print "<html>\n<head>\n"; + print "<meta http-equiv='content-type' content='text/html; charset=$outputEncoding'>\n"; + _print_html_style(); + foreach (@htmlhead) {print $_,"\n";} + print "</head>\n\n<body>\n"; + print "\n<h1>LaTeX word count"; + if ($showVersion>0) {print ' (version ',_html_version(),')'} + print "</h1>\n"; + } } # Print HTML tail sub html_tail { - print "</body>\n\n</html>\n"; + if (defined $htmlclose) { + print $htmlclose; + $htmlclose=undef; + } else { + print "</body>\n\n</html>\n"; + } } # Return version number using HTML @@ -2563,27 +3450,45 @@ sub _html_version { # Print HTML STYLE element sub _print_html_style { +if (defined $CSShref) { + print "<link rel='stylesheet' href='$CSShref' type='text/css'>\n"; + return; +} +if (defined $CSSfile) { + if (open(FH,$CSSfile)) { + print "<style>\n<!---\n",<FH>,"-->\n</style>\n"; + close(FH); + return; + } else { + error($Main,"CSS file not found: $CSSfile"); + } +} print <<END <style> <!-- body {width:auto;padding:5;margin:5;} -.error {font-weight:bold;color:#f00;font-style:italic;} -.word1,.word2,.word3 {color: #009; border-left: 1px solid #CDF; border-bottom: 1px solid #CDF;} -.word2 {font-weight: 700;} -.word3 {font-style: italic;} -.command {color: #c00;} -.exclcommand {color: #f99;} +.error {font-weight:bold;color:#f00;font-style:italic; border: solid 1px red;} +.word,.hword,.oword,.altwd {border-left: 1px solid #CDF; border-bottom: 1px solid #CDF;} +.word {color: #009} +.hword {color: #009; font-weight: 700;} +.oword {color: #009; font-style: italic;} +.altwd {color: #00c; font-style: oblique;} +.cmd {color: #c00;} +.exclcmd {color: #f99;} .option {color: #cc0;} -.grouping, .document {color: #900; font-weight:bold;} +.optparm {color: #c90;} +.envir, .document {color: #900; font-weight:bold;} +.math {color: #6c0;} .mathgroup {color: #090;} -.exclmath {color: #6c6;} +.exclmath {color: #9f6;} +.mathcmd {color: #6c0;} .ignore {color: #999;} -.exclgroup {color:#c66;} +.exclenv {color:#c66;} .tc {color: #999; font-weight:bold;} .comment {color: #999; font-style: italic;} .state {color: #990; font-size: 70%;} .cumsum {color: #999; font-size: 80%;} -.fileinclude {color: #696; font-weight:bold;} +.fileinc {color: #696; font-weight:bold;} .warning {color: #c00; font-weight: 700;} div.filegroup, div.parse, div.stylehelp, div.count, div.sumcount, div.error { @@ -2603,7 +3508,7 @@ ul.count {list-style-type: none; margin: 4pt 0pt; padding: 0pt;} .fielddesc {font-style: italic;} .nb {color: #900;} -table.stat {border:2px solid #339; background:#666; border-collapse:collapse;} +table.stat {border:2px solid #339; margin:0 0 8pt 0; background:#666; border-collapse:collapse;} table.stat tr {border:1px solid #CCC;} table.stat th, table.stat td {padding:1pt 4pt;} table.stat col {padding:4pt;} @@ -2612,6 +3517,7 @@ table.stat tbody {border:1px solid #333;} table.stat tbody.sumstat {background:#FFC;} table.stat tbody.langstat {background:#FEE;} table.stat tbody.wordstat {background:#EEF;} +table.stat tbody.macrostat {background:#EFE;} table.stat .sum {font-weight:bold; font-style:italic;} --> </style> @@ -2655,6 +3561,7 @@ sub wprintstringdata { my $data=StringData($name); if (!defined $data) { error($Main,"No StringData $name.",'BUG'); + return; } wprintlines(@_,@$data); } @@ -2709,13 +3616,13 @@ sub wprintlines { my $firstindent=0; if ($line=~s/^(\t|\s{2,})(\S)/$2/) {$firstindent=$ind1;} my $indent=$firstindent; - if ($line=~/^(.*\S)(\t|\s{2,})/) { + if ($line=~/^(.*\S)(\t|\s{2,})(.*)$/) { $indent=$ind2; if ($1 eq '|') {$line=' ';} - else {$line=$1." ";} + else {$line=$1.' ';} $i=$indent-$firstindent-length($line); if ($i>0) {$line.=' ' x $i;} - $line.=$'; + $line.=$3; # $3~${^POSTMATCH} } print wrap(' ' x $firstindent,' ' x $indent,$line)."\n"; } @@ -2750,6 +3657,12 @@ which grants you, the user, the right to use, modify and distribute the script. The script has LPPL status "maintained" with ${maintainer} being the current maintainer. :::::::::::::::::::::::::::::::::::::::: +:::::::::: ShortHelp +Syntax: texcount.pl [options] files + +Use option -help (or just -h) to get help; -help-options (-hopt) to get list of command line options, or -help-options=substring for help on all options containing substring. + +:::::::::::::::::::::::::::::::::::::::: :::::::::: HelpTitle *************************************************************** * TeXcount version ${versionnumber}, ${versiondate} @@ -2760,7 +3673,7 @@ Count words in TeX and LaTeX files, ignoring macros, tables, formulae, etc. :::::::::::::::::::::::::::::::::::::::: :::::::::: OptionsHead -Syntax: TeXcount.pl [options] files +Syntax: texcount.pl [options] files Options: @@ -2775,6 +3688,7 @@ Options: -v2 More verbose: also print ignored text. -v3 Even more verbose: include comments and options. -v4 Same as -v3 -showstate. + -v=, v0=, ..., -v4= Set verbosity by adding/removing particular types of token (styles) to include in the verbose output. Use -help-style to get details of which tokens are included in each style, and of classes of tokens (style categories). -showstate Show internal states (with verbose). -brief Only prints a brief, one line summary of counts. -q, -quiet Quiet mode, no error messages. Use is discouraged! @@ -2789,7 +3703,11 @@ Options: -sep=, -separator= Separating character or string to be added after each word. -html Output in HTML format. -htmlcore Only HTML body contents. - -opt, -optionfile Read options/parameters from file. + -htmlfile= HTML template file to use with the -html option. Use <!-- TeXcode --> to indicate where the output from TeXcount should be inserted. + -tex Encode TeX special characters for output into TeX code + -cssfile= CSS file to include instead of default styles. + -css= CSS href to include instead of default styles. Can use -css=file:{filename} instead of -cssfile={filename}. + -opt=, -optionfile= Read options/parameters from file. - Read LaTeX code from STDIN. -inc Parse included TeX files (as separate file). -merge Merge included TeX files into code (in place). @@ -2800,7 +3718,8 @@ Options: -total Do not give sums per file, only total sum. -1 Same as -brief and -total. Ensures there is only one line of output. If used in conjunction with -sum, the output will only be the total number. (NB: Character is the number one, not the letter L.) -template= Speficy an output template. Use {1},...,{7}, {SUM} and {TITLE} to include values, {1?...?1} etc. to conditionally include sections, {1?....|...?1} etc. to specify an alternative text if zero. To include subcounts, use {SUB?...?SUB} where ... is replaced with the template to use per subcount. Line shift may be specified using \\n. - -dir, -dir= Specify the working directory using -dir=path. Remember that the path must end with \\ or /. If only -dir is used, the directory of the parent file is used. + -dir, -dir= Specify the working directory using -dir=path. Remember that the path must end with \\ or /. If only -dir is used, the directory of the provided file is used and paths (e.g. included files) are assumed to be absolute or relative to this. The default at startup is -dir=. which means that the directory from which TeXcount is run is the working directory with all paths absolute or relative to this. + -auxdir, -auxdir= Specify directory for auxilary files, e.g. the bibliograph (.bbl) file. If only -auxdir is used, the working directory (as determined by -dir or -dir=) is assumed. If -auxdir= is used with -dir=, it sets the path to the auxilary directiry. If -auxdir= is used with -dir, the working directory is determined from the location of the provided file, and the path to the auxilary directory is assumed to be absolute or relative to this. -enc=, -encoding= Specify encoding (default is to guess the encoding). -utf8, -unicode Selects Unicode (UTF-8) for input and output. This is automatic with -chinese, and is required to handle e.g. Korean text. Note that the TeX file must be save in UTF-8 format (not e.g. GB2312 or Big5), or the result will be unpredictable. -alpha=, -alphabets= List of Unicode character groups (or digit, alphabetic) permitted as letters. Names are separated by ',' or '+'. If list starts with '+', the alphabets will be added to those already included. The default is Digit+alphabetic. @@ -2813,12 +3732,18 @@ Options: -char, -character, -letters Counts letters/characters instead of words. Note that spaces and punctuation is not counted. -char-only, ..., -letters-only Like -letters, but counts alphabetic letters only. -countall, -count-all The default setting in which all characters are included as either alphabets og logograms. - -freq Produce individual word frequency table. - -stat Produce statistics on language/script usage. + -freq, -freq= Produce individual word frequency table. Optionally give minimal number of occurences to be listed. + -stat Produce statistics on language/script usage. + -macrostat, -macrofreq Print macro usage statistics. -codes Display output style code overview and explanation. This is on by default. -nocodes Do not display output style code overview. + -out= Write output to file, give filename as option value. -h, -?, -help, /? Help text. -h=, -?=, -help=, /?= Takes a macro or group name as option and returns a description of the rules for handling this if any are defined. If handling rule is package specific, use -incpackage=package name: -incpackage must come before -h= on the command line to take effect. + -help-options, -h-opt List all options. + -help-options=, -h-opt= List all options containing the provided string, e.g. -h-opt=dir or -h-opt=-v (the initial - in -v causes only options starting with v to be listed). + -help-style List the styles and style categories: i.e. those permitted used with -v={styles-list}. + -help-style= Give description of style or style category. -ver, -version Print version number. -lic, -license, -licence Licence information. @@ -2858,10 +3783,13 @@ and are used to control how TeXcount parses the document. The following instruct The [param.states] is used to indicate the number of parameters used by the macro and the rules of handling each of these: the format is [#,#,...,#] with # representing one number for each parameter giving the parsing state to use for that parameter, alternatively just a single number (#) indicating how many parameters to ignore (parsing state 0). The option [content-state] is used to give the parsing state to use for the contents of a begin-end group. The main parsing states are 0 to ignore and 1 to count as text. Parsing instructions which may be used anywhere are: -@ - : - %TC:ignore start block to ignore - %TC:endignore end block to ignore - %TC:break [title] add subcount break point here +@ - : + %TC:break [title] add subcount break point here + %TC:incbib include bibliography (same as running with -incbib) + %TC:ignore ignore region, end with %TC:endignore + %TC:insert [code] insert code for TeXcount to process as TeX code + %TC:newtemplate start a new template, ie delete the existing one + %TC:template [template] add another line to the template specification See the documentation for more details. Command line options and most %TC commands (prefixed by % rather than %TC:) may be placed in an options file. This is particularly useful for defining your own output templates and macro handling rules. |