diff options
author | Karl Berry <karl@freefriends.org> | 2013-07-31 21:49:49 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2013-07-31 21:49:49 +0000 |
commit | feaf302cdb400ff155ccc77f71c218b56ef948ce (patch) | |
tree | c3a1c4dd1f79db4f08faca0140ed0430731a1326 /Master/texmf-dist/scripts | |
parent | b77a0dba8e53ef2aed2f09b95f5f6569de88e8d9 (diff) |
texcount
git-svn-id: svn://tug.org/texlive/trunk@31323 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts')
-rwxr-xr-x | Master/texmf-dist/scripts/texcount/texcount.pl | 2644 |
1 files changed, 1786 insertions, 858 deletions
diff --git a/Master/texmf-dist/scripts/texcount/texcount.pl b/Master/texmf-dist/scripts/texcount/texcount.pl index 69fb4b5002f..5a53b3bfb4d 100755 --- a/Master/texmf-dist/scripts/texcount/texcount.pl +++ b/Master/texmf-dist/scripts/texcount/texcount.pl @@ -1,15 +1,22 @@ #! /usr/bin/env perl use strict; use warnings; -use Term::ANSIColor; -use Text::Wrap; -use Encode; use utf8; # Because the script itself is UTF-8 encoded +use Encode; +use Text::Wrap; +use Term::ANSIColor; + +BEGIN { + if ($^O=~/^MSWin/) { + require Win32::Console::ANSI; + Win32::Console::ANSI::->import(); + } +} ##### Version information -my $versionnumber="2.3"; -my $versiondate="2011 Jul 30"; +my $versionnumber="3.0"; +my $versiondate="2013 Jul 29"; ###### Set global settings and variables @@ -17,9 +24,9 @@ my $versiondate="2011 Jul 30"; my %GLOBALDATA= ('versionnumber' => $versionnumber ,'versiondate' => $versiondate - ,'maintainer' => "Einar Andreas Rodland" - ,'copyrightyears' => "2008-2011" - ,'website' => "http://app.uio.no/ifi/texcount/" + ,'maintainer' => 'Einar Andreas Rodland' + ,'copyrightyears' => '2008-2013' + ,'website' => 'http://app.uio.no/ifi/texcount/' ); ### Options and states @@ -29,28 +36,32 @@ my $Main=getMain(); # Global options and settings my $htmlstyle=0; # Flag to print HTML +my $texcodeoutput=0; # Flag to convert output to valid TeX text my $encoding=undef; # Selected input encoding (default will be guess) my @encodingGuessOrder=qw/ascii utf8 latin1/; # Encoding guessing order my $outputEncoding; # Encoding used for output my @AlphabetScripts=qw/Digit Is_alphabetic/; # Letters minus logograms: defined later my @LogogramScripts=qw/Ideographic Katakana Hiragana Thai Lao Hangul/; # Scripts counted as whole words -my $WordPattern; # Regex matching a word (defined in apply_language_options()) # Parsing rules options my $includeTeX=0; # Flag to parse included files my $includeBibliography=0; # Flag to include bibliography my %substitutions; # Substitutions to make globally +my %IncludedPackages; # List of included packages # Counting options my @sumweights; # Set count weights for computing sum my $optionWordFreq=0; # Count words of this frequency, or don't count if 0 my $optionWordClassFreq=0; # Count words per word class (language) if set +my $optionMacroStat=0; # Count macro, environment and package usage # Parsing details options -my $strictness=0; # Flag to check for undefined groups -my $verbose=0; # Level of verbosity -my $showcodes=1; # Flag to show overview of colour codes (2=force) +my $strictness=0; # Flag to check for undefined environments +my $defaultVerbosity='0'; # Specification of default verbose output style +my $defaultprintlevel=0; # Flag indicating default level of verbose output +my $printlevel=undef; # Flag indicating level of verbose output my $showstates=0; # Flag to show internal state in verbose log +my $showcodes=1; # Flag to show overview of colour codes (2=force) my $showsubcounts=0; # Write subcounts if #>this, or not (if 0) my $separatorstyleregex='^word'; # Styles (regex) after which separator should be added my $separator=''; # Separator to add after words/tokens @@ -70,29 +81,47 @@ my $blankline=0; # Number of blank lines printed my $errorcount=0; # Number of errors in parsing my %warnings=(); # Warnings my %WordFreq; # Hash for counting words +my %MacroUsage; # Hash for counting macros, environments and packages + +# External sources +my $HTMLfile; # HTML file to use as HTML output template +my $CSSfile; # CSS file to use with HTML output +my $CSShref; # CSS reference to use with HTML output +my @htmlhead; # Lines to add to the HTML header +my $htmlopen; # text used to open the HTML file +my $htmlclose; # text used to close the HTML file # String data storage my $STRINGDATA; +# Other constants +my $_PARAM_='<param>'; # to identify parameter to _parse_unit +my $STRING_PARAMETER='{_}'; # used to log macro parameters +my $STRING_OPTIONAL_PARAM='[_]'; # used to log optional parameter +my $STRING_GOBBLED_OPTION='[]'; # used to log gobbled macro option +my $STRING_ERROR='<error>'; # used to log errors causing parsing to stop +my $REGEX_NUMBER=qr/^\d+$/; # regex to recognize a number + ###### Set CMD specific settings and variables ## Preset command line options # List of options (stings) separated by comma, -# e.g. ("-inc","-v") to parse included files and +# e.g. ('-inc','-v') to parse included files and # give verbose output by default. my @StartupOptions=(); # CMD specific global variables my @filelist; # List of files to parse -my $workdir; # Root directory (taken from filename) -my $globalworkdir=""; # Overrules workdir (default=present root) +my $globalworkdir='./'; # Overrules workdir (default=present root) +my $workdir; # Current directory (default=taken from filename) +my $auxdir; # Directory for auxilary files, e.g. bbl (default=$workdir) my $fileFromSTDIN=0; # Flag to set input from STDIN -my $_STDIN_="<STDIN>"; # File name to represent STDIN +my $_STDIN_='<STDIN>'; # File name to represent STDIN (must be '<...>'!) # CMD specific settings $Text::Wrap::columns=76; # Page width for wrapped output -###### Set status identifiers and methods +###### Set state identifiers and methods ### Counter indices from 0 to $SIZE_CNT-1 @@ -105,125 +134,227 @@ $Text::Wrap::columns=76; # Page width for wrapped output # 6: Number of inlined math # 7: Number of displayed math my $SIZE_CNT=8; +my $SIZE_CNT_DEFAULT=8; my $CNT_FILE=0; my $CNT_WORDS_TEXT=1; my $CNT_WORDS_HEADER=2; -my $CNT_WORDS_CAPTION=3; +my $CNT_WORDS_OTHER=3; my $CNT_COUNT_HEADER=4; my $CNT_COUNT_FLOAT=5; my $CNT_COUNT_INLINEMATH=6; my $CNT_COUNT_DISPLAYMATH=7; # Labels used to describe the counts -my @countlabel=('Files','Words in text','Words in headers', - 'Words in float captions','Number of headers','Number of floats', +my @countkey=('file','word','hword','oword','header','float','inmath','dsmath'); +my @countdesc=('Files','Words in text','Words in headers', + 'Words outside text (captions, etc.)','Number of headers','Number of floats/tables/figures', 'Number of math inlines','Number of math displayed'); -### Parsing statuses -## Note that status numbers are used in rule definitions and should -## not be changed. +# Map keywords to counters +my %key2cnt; +add_keys_to_hash(\%key2cnt,$CNT_FILE,0,'file'); +add_keys_to_hash(\%key2cnt,$CNT_WORDS_TEXT,1,'text','word','w','wd'); +add_keys_to_hash(\%key2cnt,$CNT_WORDS_HEADER,2,'headerword','hword','hw','hwd'); +add_keys_to_hash(\%key2cnt,$CNT_WORDS_OTHER,3,'otherword','other','oword','ow','owd'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_HEADER,4,'header','heading','head'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_FLOAT,5,'float','table','figure'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_INLINEMATH,6,'inline','inlinemath','imath','eq'); +add_keys_to_hash(\%key2cnt,$CNT_COUNT_DISPLAYMATH,7,'displaymath','dsmath','dmath','ds'); + + +### Token types +# Set in $tex->{'type'} by call to _next_token +my $TOKEN_SPACE=-1; +my $TOKEN_COMMENT=0; +my $TOKEN_WORD=1; # word (or other form of text or text component) +my $TOKEN_SYMBOL=2; # symbol (not word, e.g. punctuation) +my $TOKEN_MACRO=3; # macro (\name) +my $TOKEN_BRACE=4; # curly braces: { } +my $TOKEN_BRACKET=5; # brackets: [ ] +my $TOKEN_MATH=6; +my $TOKEN_LINEBREAK=9; # line break in file +my $TOKEN_TC=666; # TeXcount instructions (%TC:instr) +my $TOKEN_END=999; # end of line or blank line + + +### Parsing states # -## Status for regions that should not be counted -# 0 = exclude from count -## Statuses for regions in which words should not be counted -# -1 = float (exclude, but include captions) -# -2 = strong exclude, ignore begin-end groups -# -3 = stronger exclude, do not parse macro parameters -# -4 = ignore everything except end marker: even { -# -9 = preamble (between \documentclass and \begin{document}) -## Statuses for regions in which words should be counted -# 1 = text -# 2 = header text -# 3 = float text -## Status change: not used in parsing, but to switch status then ignore contents -# 6 = switch to inlined math -# 7 = switch to displayed math -## Note that positive statuses must correspond to CNT codes! +## States for regions that should not be counted +# IGNORE = exclude from count +# FLOAT = float (exclude, but include captions) +# EXCLUDE_STRONG = strong exclude, ignore environments +# EXCLUDE_STRONGER = stronger exclude, do not parse macro parameters +# EXCLUDE_ALL = ignore everything except end marker: even { +# PREAMBLE = preamble (between \documentclass and \begin{document}) +## States for regions in which words should be counted +# TEXT = text +# TEXT_HEADER = header text +# TEXT_FLOAT = float text +## State change: not used in parsing, but to switch state then ignore contents +# TO_INLINEMATH = switch to inlined math +# TO_DISPLAYMATH = switch to displayed math +## Other states +# _NULL = default state to use if none other is defined +# _OPTION = state used to indicate that the next parameter is an option +# _EXCLUDE_ = cutoff, state <= this represents excluded text +## NB: Presently, it is assumed that additional states is added as 8,9,..., +## e.g. that states added through TC:newcounter correspond to the added counters. # -my $STATUS_IGNORE=0; -my $STATUS_FLOAT=-1; -my $STATUS_EXCLUDE_STRONG=-2; -my $STATUS_EXCLUDE_STRONGER=-3; -my $STATUS_EXCLUDE_ALL=-4; -my $STATUS_PREAMBLE=-9; -my $STATUS_TEXT=1; -my $STATUS_TEXT_HEADER=2; -my $STATUS_TEXT_FLOAT=3; -my $STATUS_TO_INLINEMATH=6; -my $STATUS_TO_DISPLAYMATH=7; - -# When combining two statuses, use the first one; list must be complete! -my @STATUS_PRIORITY_LIST=(-4,-3,-2,-1,0,-9,3,2,1); - -# Status: is a text status..."include status" is more correct -sub status_is_text { +my $STATE_IGNORE=-1; +my $STATE_MATH=-2; +my $STATE_FLOAT=-10; +my $STATE_EXCLUDE_STRONG=-20; +my $STATE_EXCLUDE_STRONGER=-30; +my $STATE_EXCLUDE_ALL=-40; +my $STATE_PREAMBLE=-99; +my $STATE_TEXT=1; +my $STATE_TEXT_HEADER=2; +my $STATE_TEXT_FLOAT=3; +my $STATE_TO_HEADER=4; +my $STATE_TO_FLOAT=5; +my $STATE_TO_INLINEMATH=6; +my $STATE_TO_DISPLAYMATH=7; +my $__STATE_EXCLUDE_=-10; +my $__STATE_NULL=1; +my $_STATE_OPTION=-1000; +my $_STATE_NOOPTION=-1001; +my $_STATE_AUTOOPTION=-1002; + +# Counter key mapped to STATE +my $PREFIX_PARAM_OPTION=' '; # Prefix for parameter options/modifiers +my %key2state; +add_keys_to_hash(\%key2state,$STATE_TEXT,1,'text','word','w','wd'); +add_keys_to_hash(\%key2state,$STATE_TEXT_HEADER,2,'headertext','headerword','hword','hw','hwd'); +add_keys_to_hash(\%key2state,$STATE_TEXT_FLOAT,3,'otherword','other','oword','ow','owd'); +add_keys_to_hash(\%key2state,$STATE_TO_HEADER,4,'header','heading','head'); +add_keys_to_hash(\%key2state,$STATE_TO_FLOAT,5,'float','table','figure'); +add_keys_to_hash(\%key2state,$STATE_TO_INLINEMATH,6,'inline','inlinemath','imath','eq'); +add_keys_to_hash(\%key2state,$STATE_TO_DISPLAYMATH,7,'displaymath','dsmath','dmath','ds'); +add_keys_to_hash(\%key2state,$STATE_IGNORE,0,'ignore','x'); +add_keys_to_hash(\%key2state,$STATE_MATH,'ismath'); +add_keys_to_hash(\%key2state,$STATE_FLOAT,-1,'isfloat'); +add_keys_to_hash(\%key2state,$STATE_EXCLUDE_STRONG,-2,'xx'); +add_keys_to_hash(\%key2state,$STATE_EXCLUDE_STRONGER,-3,'xxx'); +add_keys_to_hash(\%key2state,$STATE_EXCLUDE_ALL,-4,'xall'); +add_keys_to_hash(\%key2state,$_STATE_OPTION,'[',' option',' opt',' optional'); +add_keys_to_hash(\%key2state,$_STATE_NOOPTION,'nooption','nooptions','noopt','noopts'); +add_keys_to_hash(\%key2state,$_STATE_AUTOOPTION,'autooption','autooptions','autoopt','autoopts'); + +# When combining two states, use the first one; list must be complete! +my @STATE_FIRST_PRIORITY=( + $STATE_EXCLUDE_ALL, + $STATE_EXCLUDE_STRONGER, + $STATE_EXCLUDE_STRONG, + $STATE_FLOAT, + $STATE_MATH, + $STATE_IGNORE, + $STATE_PREAMBLE, + $STATE_TO_FLOAT, + $STATE_TO_HEADER, + $STATE_TO_INLINEMATH, + $STATE_TO_DISPLAYMATH); +my @STATE_MID_PRIORITY=(); +my @STATE_LAST_PRIORITY=( + $STATE_TEXT_FLOAT, + $STATE_TEXT_HEADER, + $STATE_TEXT); + +# Map state to corresponding word counter +my %state2cnt=( + $STATE_TEXT => $CNT_WORDS_TEXT, + $STATE_TEXT_HEADER => $CNT_WORDS_HEADER, + $STATE_TEXT_FLOAT => $CNT_WORDS_OTHER); + +# Transition state mapped to content state and counter +my %transition2state=( + $STATE_TO_HEADER => [$STATE_TEXT_HEADER,$CNT_COUNT_HEADER], + $STATE_TO_INLINEMATH => [$STATE_MATH ,$CNT_COUNT_INLINEMATH], + $STATE_TO_DISPLAYMATH => [$STATE_MATH ,$CNT_COUNT_DISPLAYMATH], + $STATE_TO_FLOAT => [$STATE_FLOAT ,$CNT_COUNT_FLOAT]); + +# Parsing state descriptions (used for macro rule help) +my %state2desc=( + $STATE_IGNORE => 'ignore: do not count', + $STATE_MATH => 'math/equation contents', + $STATE_FLOAT => 'float (figure, etc.): ignore all but special macros', + $STATE_EXCLUDE_STRONG => 'strong exclude: ignore environments', + $STATE_EXCLUDE_STRONGER => 'stronger exclude: ignore environments and macro paramters', + $STATE_EXCLUDE_ALL => 'exlude all: even {, only scan for end marker', + $STATE_PREAMBLE => 'preamble: from \documentclass to \begin{document}', + $STATE_TEXT => 'text: count words', + $STATE_TEXT_HEADER => 'header text: count words as header words', + $STATE_TEXT_FLOAT => 'float text: count words as float words (e.g. captions)', + $STATE_TO_HEADER => 'header: count header, then count words as header words', + $STATE_TO_FLOAT => 'float: count float, then count words as float/other words', + $STATE_TO_INLINEMATH => 'inline math: count as inline math/equation', + $STATE_TO_DISPLAYMATH => 'displayed math: count as displayed math/equation'); + +# Parsing state presentation style +my %state2style=( + $STATE_TEXT => 'word', + $STATE_TEXT_HEADER => 'hword', + $STATE_TEXT_FLOAT => 'oword', + ); + +# State: is a text state..."include state" is more correct +sub state_is_text { my $st=shift @_; - return ($st>0); + return ($st>=$STATE_TEXT); } -# Status: get CNT corresponding to text status (or undef) -sub status_text_cnt { +# State: is a parsed/included region, text or preamble +sub state_is_parsed { my $st=shift @_; - if ($st==$STATUS_TEXT) {return $CNT_WORDS_TEXT;} - if ($st==$STATUS_TEXT_HEADER) {return $CNT_WORDS_HEADER;} - if ($st==$STATUS_TEXT_FLOAT) {return $CNT_WORDS_CAPTION;} - return undef; + return ($st>=$STATE_TEXT || $st==$STATE_PREAMBLE); } -# Status: is an exclude status -sub status_is_exclude { +# State: get CNT corresponding to text state (or undef) +sub state_text_cnt { my $st=shift @_; - return ($st<0); + return $state2cnt{$st}; } -# Status: \begin and \end should be processed -sub status_inc_envir { +# State: is an exclude state +sub state_is_exclude { my $st=shift @_; - return ($st>-2); + return ($st<=$__STATE_EXCLUDE_); } -# Map status number to CNT value -sub status_to_cnt { +# State: \begin and \end should be processed +sub state_inc_envir { my $st=shift @_; - if ($st==$STATUS_TO_INLINEMATH) {return $CNT_COUNT_INLINEMATH;} - if ($st==$STATUS_TO_DISPLAYMATH) {return $CNT_COUNT_DISPLAYMATH;} - return undef; + return ($st>$STATE_EXCLUDE_STRONG); } -# Status as text -sub status_to_text { +# State as text (used with printstate) +# TODO: Should do a conversion based on STATE values. +sub state_to_text { my $st=shift @_; return $st; } -# Status as text -sub status_to_style { - return 'word'.status_to_text(@_); +# Style to use with text state +sub state_to_style { + return $state2style{shift @_}; } -### Token types -# -1: space -# 0: comment -# 1: word (or other forms of text or text components) -# 2: symbol (not word, e.g. punctuation) -# 3: macro -# 4: curly braces {} -# 5: brackets [] -# 6: maths -# 9: line break in file -# 999: end of line or blank line -# 666: TeXcount instruction (%TC:instruction) -my $TOKEN_SPACE=-1; -my $TOKEN_COMMENT=0; -my $TOKEN_WORD=1; -my $TOKEN_SYMBOL=2; -my $TOKEN_MACRO=3; -my $TOKEN_BRACE=4; -my $TOKEN_BRACKET=5; -my $TOKEN_MATH=6; -my $TOKEN_LINEBREAK=9; -my $TOKEN_TC=666; -my $TOKEN_END=999; +# Add new counter with the given key and description +sub add_new_counter { + my ($key,$desc,$like)=@_; + my $state=$SIZE_CNT; + my $cnt=$SIZE_CNT; + $key=lc($key); + if (!defined $like){$like=$CNT_WORDS_OTHER;} + $key2cnt{$key}=$cnt; + push @countkey,$key; + push @countdesc,$desc; + if (defined $sumweights[$like]) {$sumweights[$cnt]=$sumweights[$like];} + $key2state{$key}=$state; + $state2cnt{$state}=$cnt; + $state2style{$state}='altwd'; + push @STATE_MID_PRIORITY,$state; + $SIZE_CNT++; +} ###### Set global definitions @@ -243,20 +374,55 @@ my %BreakPoints=%{$BreakPointsOptions{'none'}}; ### Print styles # Definition of different print styles: maps of class labels # to ANSI codes. Class labels are as used by HTML styles. -my @STYLES=(); -my %STYLE; -$STYLES[0]={'error'=>'bold red'}; -$STYLES[1]={%{$STYLES[0]},''=>'normal',' '=>'normal', - 'word1'=>'blue','word2'=>'bold blue','word3'=>'blue', - 'grouping'=>'red','document'=>'red','mathgroup'=>'magenta', - 'state'=>'cyan underline','cumsum'=>'yellow'}; -$STYLES[2]={%{$STYLES[1]}, - 'command'=>'green','ignore'=>'cyan', - 'exclcommand'=>'yellow','exclgroup'=>'yellow','exclmath'=>'yellow'}; -$STYLES[3]={%{$STYLES[2]}, - 'tc'=>'bold yellow','comment'=>'yellow','option'=>'yellow', - 'fileinclude'=>'bold green'}; -$STYLES[4]={%{$STYLES[3]}}; +my %STYLES; +my $STYLE_EMPTY=' '; +my $STYLE_BLOCK='-'; +my $NOSTYLE=' '; +$STYLES{'Errors'}={'error'=>'bold red'}; +$STYLES{'Words'}={'word'=>'blue','hword'=>'bold blue','oword'=>'blue','altwd'=>'blue'}; +$STYLES{'Macros'}={'cmd'=>'green','fileinc'=>'bold green'}; +$STYLES{'Options'}={'option'=>'yellow','optparm'=>'green'}; +$STYLES{'Ignored'}={'ignore'=>'cyan','math'=>'magenta'}; +$STYLES{'Excluded'}={'exclcmd'=>'yellow','exclenv'=>'yellow','exclmath'=>'yellow','mathcmd'=>'yellow'}; +$STYLES{'Groups'}={'document'=>'red','envir'=>'red','mathgroup'=>'magenta'}; +$STYLES{'Comments'}={'tc'=>'bold yellow','comment'=>'yellow'}; +$STYLES{'Sums'}={'cumsum'=>'yellow'}; +$STYLES{'States'}={'state'=>'cyan underline'}; +$STYLES{'<core>'}={%{$STYLES{'Errors'}},$STYLE_EMPTY=>$NOSTYLE,'<printlevel>'=>1}; +$STYLES{0}={%{$STYLES{'Errors'}},'<printlevel>'=>0}; +$STYLES{1}={%{$STYLES{'<core>'}},%{$STYLES{'Words'}},%{$STYLES{'Groups'}},%{$STYLES{'Sums'}}}; +$STYLES{2}={%{$STYLES{1}},%{$STYLES{'Macros'}},%{$STYLES{'Ignored'}},%{$STYLES{'Excluded'}}}; +$STYLES{3}={%{$STYLES{2}},%{$STYLES{'Options'}},%{$STYLES{'Comments'}},'<printlevel>'=>2}; +$STYLES{4}={%{$STYLES{3}},%{$STYLES{'States'}}}; +$STYLES{'All'}=$STYLES{4}; +my %STYLE=%{$STYLES{$defaultVerbosity}}; + +my @STYLE_LIST=('error','word','hword','oword','altwd', + 'ignore','document','cmd','exclcmd','option','optparm','envir','exclenv', + 'mathgroup','exclmath','math','mathcmd','comment','tc','fileinc','state','cumsum'); +my %STYLE_DESC=( + 'error' => 'ERROR: TeXcount error message', + 'word' => 'Text which is counted: counted as text words', + 'hword' => 'Header and title text: counted as header words', + 'oword' => 'Caption text and footnotes: counted as caption words', + 'altwd' => 'Words in user specified counters: counted in separate counters', + 'ignore' => 'Ignored text or code: excluded or ignored', + 'document' => '\documentclass: document start, beginning of preamble', + 'cmd' => '\macro: macro not counted, but parameters may be', + 'exclcmd' => '\macro: macro in excluded region', + 'option' => '[Macro options]: not counted', + 'optparm' => '[Optional parameter]: content parsed and styled as counted', + 'envir' => '\begin{name} \end{name}: environment', + 'exclenv' => '\begin{name} \end{name}: environment in excluded region', + 'mathgroup' => '$ $: counted as one equation', + 'exclmath' => '$ $: equation in excluded region', + 'math' => '2+2=4: maths (inside $...$ etc.)', + 'mathcmd' => '$\macro$: macros inside maths', + 'comment' => '% Comments: not counted', + 'tc' => '%TC:TeXcount instructions: not counted', + 'fileinc' => 'File to include: not counted but file may be counted later', + 'state' => '[state]: internal TeXcount state', + 'cumsum' => '[cumsum]: cumulative sum count'); ###### Define what a word is and language options @@ -279,18 +445,22 @@ my $LetterPattern=$NamedLetterPattern{'default'}; # List of regexp patterns that should be analysed as words. # Use @ to represent a letter, will be substituted with $LetterPattern. # Named patterns may replace or be appended to the original patterns. +# Apply_Options() results in a call to apply_language_options() which +# constructs $WordPattern based on $LetterPattern, @WordPatterns and +# alphabet/logogram settings. my %NamedWordPattern; $NamedWordPattern{'letters'}='@'; -$NamedWordPattern{'words'}='(@+|\{@+\})([\-\'\.]?(@+|\{@+\}))*'; +$NamedWordPattern{'words'}='(@+|@+\{@+\}|\{@+\}@+)([\-\'\.]?(@+|\{@+\}))*'; my @WordPatterns=($NamedWordPattern{'words'}); +my $WordPattern; # Regex matching a word (defined in apply_language_options()) ### Macro option regexp list # List of regexp patterns to be gobbled as macro option in and after # a macro. my %NamedMacroOptionPattern; -$NamedMacroOptionPattern{'default'}='\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]'; -$NamedMacroOptionPattern{'relaxed'}='\[[^\[\]\n]*\]'; -$NamedMacroOptionPattern{'restricted'}=$NamedMacroOptionPattern{'default'}; +$NamedMacroOptionPattern{'default'}='\[[^\[\]\n]*\]'; +$NamedMacroOptionPattern{'relaxed'}='\[[^\[\]\n]*(\n[^\[\]\n]+)\n?\]'; +$NamedMacroOptionPattern{'restricted'}='\[(\w|[,\-\s\~\.\:\;\+\?\*\_\=])*\]'; my $MacroOptionPattern=$NamedMacroOptionPattern{'default'}; ### Alternative language encodings @@ -367,133 +537,287 @@ END ###### Define core rules -### Macros for headers -# Macros that identify headers: i.e. following token or -# {...} is counted as header. The =>[2] indicates transition to -# state 2 which is used within headers (although the value is -# actually never used). This is copied to %TeXmacro and the -# only role of defining it here is that the counter for the number -# of headers is incremented by one. -my %TeXheader=('\title'=>[2],'\part'=>[2],'\chapter'=>[2], - '\section'=>[2],'\subsection'=>[2],'\subsubsection'=>[2], - '\paragraph'=>[2],'\subparagraph'=>[2]); - ### Macros indicating package inclusion # Will always be assumed to take one parameter (plus options). -my %TeXpackageinc=('\usepackage'=>1); +# Gets added to TeXmacro. After that, values are not used, only membership. +my %TeXpackageinc=('\usepackage'=>1,'\RequirePackage'=>1); ### Macros that are counted within the preamble # The preamble is the text between \documentclass and \begin{document}. # Text and macros in the preamble is ignored unless specified here. The -# value is the status (1=text, 2=header, etc.) they should be interpreted as. +# value is the states (1=text, 2=header, etc.) they should be interpreted as. # Note that only the first unit (token or {...} block) is counted. -my %TeXpreamble=( - '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3], - '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3], - '\title'=>[2]); +# Gets added to TeXmacro. Is used within preambles only. +my %TeXpreamble; +add_keys_to_hash(\%TeXpreamble,['header'],'\title'); +add_keys_to_hash(\%TeXpreamble,['other'],'\thanks'); +add_keys_to_hash(\%TeXpreamble,['xxx','xxx'],'\newcommand','\renewcommand'); +add_keys_to_hash(\%TeXpreamble,['xxx','xxx','xxx'],'\newenvironment','\renewenvironment'); ### In floats: include only specific macros # Macros used to identify caption text within floats. -my %TeXfloatinc=('\caption'=>[3]); +# Gets added to TeXmacro. Is used within floats only. +my %TeXfloatinc=('\caption'=>['otherword']); ### How many tokens to gobble after macro # Each macro is assumed to gobble up a given number of # tokens (or {...} groups), as well as options [...] before, within # and after. The %TeXmacro hash gives a link from a macro -# (or beginNAME for begin-end groups without the backslash) +# (or beginNAME for environment with no the backslash) # to either an integer giving the number of tokens to ignore -# or to an array (specified as [num,num,...]) of length N where -# N is the number of tokens to be read with the macro and the -# array values tell how each is to be interpreted (see the status -# values: 0=ignore, 1=count, etc.). Thus specifying a number N is -# equivalent to specifying an array [0,...,0] of N zeros. +# or to an array (specified as [rule,rule,...]) of length N where +# N is the number of parameters to be read with the macro. The +# array values tell how each is to be interpreted (see the parser state +# keywords for valid values). Thus specifying a number N is +# equivalent to specifying an array of N 'ignore' rules. # # For macros not specified here, the default value is 0: i.e. -# no tokens are excluded, but [...] options are. Header macros -# specified in %TeXheader are automatically included here. -my %TeXmacro=(%TeXheader,%TeXpreamble,%TeXfloatinc, - '\documentclass'=>1,'\documentstyle'=>1,'\usepackage'=>1, '\hyphenation'=>1, - '\pagestyle'=>1,'\thispagestyle'=>1, '\pagenumbering'=>1,'\markboth'=>1, '\markright'=>1, - '\newcommand'=>[-3,-3],'\renewcommand'=>[-3,-3], - '\newenvironment'=>[-3,-3,-3], '\renewenvironment'=>[-3,-3,-3], - '\newfont'=>2,'\newtheorem'=>2,'\bibliographystyle'=>1, '\bibliography'=>1, - '\parbox'=>1, '\marginpar'=>[3],'\makebox'=>0, '\raisebox'=>1, '\framebox'=>0, - '\newsavebox'=>1, '\sbox'=>1, '\savebox'=>2, '\usebox'=>1,'\rule'=>2, - '\footnote'=>[3],'\label'=>1, '\ref'=>1, '\pageref'=>1, '\bibitem'=>1, - '\cite'=>1, '\citep'=>1, '\citet'=>1, '\citeauthor'=>1, '\citealt'=>1, '\nocite'=>1, - '\eqlabel'=>1, '\eqref'=>1,'\hspace'=>1, '\vspace'=>1, '\addvspace'=>1, - '\input'=>1, '\include'=>1, '\includeonly'=>1,'\includegraphics'=>1, - '\newlength'=>1, '\setlength'=>2, '\addtolength'=>2,'\settodepth'=>2, - '\settoheight'=>2, '\settowidth'=>2,'\newcounter'=>1, '\setcounter'=>2, - '\addtocounter'=>2,'\stepcounter'=>1, '\refstepcounter'=>1, '\usecounter'=>1, - '\alph'=>1, '\arabic'=>1, '\fnsymbol'=>1, '\roman'=>1, '\value'=>1, - '\cline'=>1, '\multicolumn'=>3,'\typeout'=>1, '\typein'=>1, - 'beginlist'=>2, 'beginminipage'=>1, 'begintabular'=>1, - 'beginthebibliography'=>1,'beginlrbox'=>1, - '\begin'=>1,'\end'=>1,'\title'=>[2], - '\addtocontents'=>2,'\addcontentsline'=>3, - '\uppercase'=>0,'\lowercase'=>0); - -### Macros that should be counted as one or more words -# Macros that represent text may be declared here. The value gives -# the number of words the macro represents. -my %TeXmacroword=('\LaTeX'=>1,'\TeX'=>1); - -### Begin-End groups -# Identified as begin-end groups, and define =>state. The -# states used corresponds to the elements of the count array, and -# are: -# 0: Not included -# 1: Text, words included in text count -# 2: Header, words included in header count -# 3: Float caption, words included in float caption count -# 6: Inline mathematics, words not counted -# 7: Displayed mathematics, words not counted -# -1: Float, not included, but looks for captions +# no tokens are excluded, but [...] options are. +my %TeXmacro=(%TeXpreamble,%TeXfloatinc,%TeXpackageinc); +add_keys_to_hash(\%TeXmacro,['text'], + '\textnormal','\textrm','\textit','\textbf','\textsf','\texttt','\textsc','\textsl','\textup','\textmd', + '\makebox','\mbox','\framebox','\fbox','\uppercase','\lowercase','\textsuperscript','\textsubscript', + '\citetext'); +add_keys_to_hash(\%TeXmacro,['[','text'], + '\item'); +add_keys_to_hash(\%TeXmacro,['[','ignore'], + '\linebreak','\nolinebreak','\pagebreak','\nopagebreak'); +add_keys_to_hash(\%TeXmacro,0, + '\maketitle','\indent','\noindent', + '\centering','\raggedright','\raggedleft','\clearpage','\cleardoublepage','\newline','\newpage', + '\smallskip','\medskip','\bigskip','\vfill','\hfill','\hrulefill','\dotfill', + '\normalsize','\small','\footnotesize','\scriptsize','\tiny','\large','\Large','\LARGE','\huge','\Huge', + '\normalfont','\em','\rm','\it','\bf','\sf','\tt','\sc','\sl', + '\rmfamily','\sffamily','\ttfamily','\upshape','\itshape','\slshape','\scshape','\mdseries','\bfseries', + '\selectfont', + '\tableofcontents','\listoftables','\listoffigures'); +add_keys_to_hash(\%TeXmacro,1, + '\begin','\end', + '\documentclass','\documentstyle','\hyphenation','\pagestyle','\thispagestyle', + '\author','\date', + '\bibliographystyle','\bibliography','\pagenumbering','\markright', + '\includeonly','\includegraphics','\special', + '\label','\ref','\pageref','\bibitem', + '\eqlabel','\eqref','\hspace','\vspace','\addvspace', + '\newsavebox','\usebox', + '\newlength','\newcounter','\stepcounter','\refstepcounter','\usecounter', + '\fontfamily','\fontseries', + '\alph','\arabic','\fnsymbol','\roman','\value', + '\typeout', '\typein','\cline'); +add_keys_to_hash(\%TeXmacro,2, + '\newfont','\newtheorem','\sbox','\savebox','\rule','\markboth', + '\setlength','\addtolength','\settodepth','\settoheight','\settowidth','\setcounter', + '\addtocontents','\addtocounter', + '\fontsize'); +add_keys_to_hash(\%TeXmacro,3,'\multicolumn','\addcontentsline'); +add_keys_to_hash(\%TeXmacro,6,'\DeclareFontShape'); +add_keys_to_hash(\%TeXmacro,['[','text','ignore'], + '\cite','\nocite','\citep','\citet','\citeauthor','\citeyear','\citeyearpar', + '\citealp','\citealt','\Citep','\Citet','\Citealp','\Citealt','\Citeauthor'); +add_keys_to_hash(\%TeXmacro,['ignore','text'],'\parbox','\raisebox'); +add_keys_to_hash(\%TeXmacro,['otherword'],'\marginpar','\footnote','\footnotetext'); +add_keys_to_hash(\%TeXmacro,['header'], + '\title','\part','\chapter','\section','\subsection','\subsubsection','\paragraph','\subparagraph'); +add_keys_to_hash(\%TeXmacro,['xxx','xxx','text'],'\multicolumn'); +add_keys_to_hash(\%TeXmacro,['xxx','xxx'],'\newcommand','\renewcommand'); +add_keys_to_hash(\%TeXmacro,['xxx','xxx','xxx'],'\newenvironment','\renewenvironment'); + +### Environments +# The %TeXenvir hash provides content parsing rules (parser states). +# Environments that are not defined will be counted as the surrounding text. # -# 4 and 5 are used to count number of headers and floats -# and are not used as states. -# -# Groups that are not defined will be counted as the surrounding text. +# Parameters taken by the \begin{environment} are defined in %TeXmacro. # # Note that some environments may only exist within math-mode, and # therefore need not be defined here: in fact, they should not as it # is not clear if they will be in inlined or displayed math. -my %TeXgroup=('document'=>1,'letter'=>1,'titlepage'=>0, - 'center'=>1,'flushleft'=>1,'flushright'=>1, - 'abstract'=>1,'quote'=>1,'quotation'=>1,'verse'=>1,'minipage'=>1, - 'verbatim'=>-4,'tikzpicture'=>-4, - 'description'=>1,'enumerate'=>1,'itemize'=>1,'list'=>1, - 'theorem'=>1,'lemma'=>1,'definition'=>1,'corollary'=>1,'example'=>1, - 'math'=>6,'displaymath'=>7,'equation'=>7,'eqnarray'=>7,'align'=>7, - 'equation*'=>7,'eqnarray*'=>7,'align*'=>7, - 'figure'=>-1,'float'=>-1,'picture'=>-1,'table'=>-1, - 'tabbing'=>0,'tabular'=>0,'thebibliography'=>0,'lrbox'=>0); +my %TeXenvir; +add_keys_to_hash(\%TeXenvir,'ignore', + 'titlepage','tabbing','tabular','tabular*','thebibliography','lrbox'); +add_keys_to_hash(\%TeXenvir,'text', + 'document','letter','center','flushleft','flushright', + 'abstract','quote','quotation','verse','minipage', + 'description','enumerate','itemize','list', + 'theorem','thm','lemma','definition','corollary','example','proof','pf'); +add_keys_to_hash(\%TeXenvir,'inlinemath', + 'math'); +add_keys_to_hash(\%TeXenvir,'displaymath', + 'displaymath','equation','equation*','eqnarray','eqnarray*','align','align*',); +add_keys_to_hash(\%TeXenvir,'float', + 'float','picture','figure','figure*','table','table*'); +add_keys_to_hash(\%TeXenvir,'xall', + 'verbatim','tikzpicture'); + +# Environment parameters +my $PREFIX_ENVIR='begin'; # Prefix used for environment names +add_keys_to_hash(\%TeXmacro,1, + 'beginthebibliography','beginlrbox','beginminipage'); +add_keys_to_hash(\%TeXmacro,2, + 'beginlist'); +add_keys_to_hash(\%TeXmacro,['ignore'], + 'beginletter'); +add_keys_to_hash(\%TeXmacro,['xxx'], + 'begintabular'); +add_keys_to_hash(\%TeXmacro,['ignore','xxx'], + 'begintabular*'); +add_keys_to_hash(\%TeXmacro,['[','text'], + 'begintheorem','beginthm','beginlemma','begindefinition','begincorollary','beginexample','beginproof','beginpf'); +add_keys_to_hash(\%TeXmacro,['nooptions'], + 'beginverbatim'); + +### Macros that should be counted as one or more words +# Macros that represent text may be declared here. The value gives +# the number of words the macro represents. +my %TeXmacrocount=('\LaTeX'=>1,'\TeX'=>1,'beginabstract'=>['header','headerword']); ### Macros for including tex files # Allows \macro{file} or \macro file. If the value is 0, the filename will # be used as is; if it is 1, the filetype .tex will be added if the # filename is without filetype; if it is 2, the filetype .tex will be added. -my %TeXfileinclude=('\input'=>1,'\include'=>2); +my %TeXfileinclude=('\input'=>'input','\include'=>'texfile'); + +### Convert state keys to codes +convert_hash(\%TeXpreamble,\&keyarray_to_state); +convert_hash(\%TeXfloatinc,\&keyarray_to_state); +convert_hash(\%TeXmacro,\&keyarray_to_state); +convert_hash(\%TeXmacrocount,\&keyarray_to_cnt); +convert_hash(\%TeXenvir,\&key_to_state); ###### Define package specific rules ### Package rule definitions +my %PackageTeXpreamble=(); # TeXpreamble definitions per package +my %PackageTeXpackageinc=(); # TeXpackageinc definitions per package +my %PackageTeXfloatinc=(); # TeXfloatinc definitions per package my %PackageTeXmacro=(); # TeXmacro definitions per package -my %PackageTeXmacroword=(); # TeXmacroword definitions per package -my %PackageTeXheader=(); # TeXheader definitions per package -my %PackageTeXgroup=(); # TeXgroup definitions per package -my %PackageTeXfileinclude=(); # TeXgroup definitions per package +my %PackageTeXmacrocount=(); # TeXmacrocount definitions per package +my %PackageTeXenvir=(); # TeXenvir definitions per package +my %PackageTeXfileinclude=(); # TeXfileinclude definitions per package +my %PackageSubpackage=(); # Subpackages to include (listed in array [...]) -# Rules for package psfig -$PackageTeXmacro{'psfig'}={('\psfig'=>1)}; # Rules for bibliography inclusion -$PackageTeXmacroword{'%incbib'}={'thebibliography'=>1}; -$PackageTeXmacro{'%incbib'}={}; -$PackageTeXgroup{'%incbib'}={'thebibliography'=>1}; -$PackageTeXfileinclude{'%incbib'}={'\bibliography'=>'bbl'}; +$PackageTeXmacrocount{'%incbib'}={'beginthebibliography'=>['header','hword']}; +$PackageTeXmacro{'%incbib'}={'\bibliography'=>1}; +$PackageTeXenvir{'%incbib'}={'thebibliography'=>'text'}; +$PackageTeXfileinclude{'%incbib'}={'\bibliography'=>'<bbl>'}; + +# Rules for package alltt +$PackageTeXenvir{'alltt'}={ + 'alltt'=>'xall'}; + +# Rules for package babel +# NB: Only core macros implemented, those expected found in regular documents +$PackageTeXenvir{'babel'}={ + 'otherlanguage'=>'text','otherlanguage*'=>'text'}; +$PackageTeXmacro{'babel'}={ + '\selectlanguage'=>1,'\foreignlanguage'=>['ignore','text'], + 'beginotherlanguage'=>1,'beginotherlanguage*'=>1}; + +# Rules for package comment +$PackageTeXenvir{'comment'}={ + 'comment'=>'xxx'}; + +# Rules for package color +$PackageTeXmacro{'color'}={ + '\textcolor'=>['ignore','text'],'\color'=>1,'\pagecolor'=>1,'\normalcolor'=>0, + '\colorbox'=>['ignore','text'],'\fcolorbox'=>['ignore','ignore','text'], + '\definecolor'=>3,\'DefineNamedColor'=>4}; + +# Rules for package endnotes +$PackageTeXmacro{'endnotes'}={'\endnote'=>['oword'],'\endnotetext'=>['oword'],'\addtoendnotetext'=>['oword']}; + +# Rules for package fancyhdr +$PackageTeXmacro{'fancyhdr'}={ + '\fancyhf'=>1,'\lhead'=>1,'\chead'=>1,'\rhead'=>1,'\lfoot'=>1,'\cfoot'=>1,'\rfoot'=>1}; + +# Rules for package geometry +$PackageTeXmacro{'geometry'}={ + '\geometry'=>1,'\newgeometry'=>1,'\restoregeometry'=>0,,'\savegeometry'=>1,'\loadgeometry'=>1}; + +# Rules for package graphicx +$PackageTeXmacro{'graphicx'}={ + '\DeclareGraphicsExtensions'=>1,'\graphicspath'=>1, + '\includegraphics'=>['[','ignore','ignore'], + '\includegraphics*'=>['[','ignore','[','ignore','[','ignore','ignore'], + '\rotatebox'=>1,'\scalebox'=>1,'\reflectbox'=>1,'\resizebox'=>1}; + +# Rules for package hyperref (urls themselves counted as one word) +# NB: \hyperref[label]{text} preferred over \hyperref{url}{category}{name}{text} +# NB: Macros for use in Form environments not implemented +$PackageTeXmacro{'hyperref'}={ + '\hyperref'=>['[','ignore','text'], + '\url'=>1,'\nolinkurl'=>1,'\href'=>['ignore','text'], + '\hyperlink'=>['ignore','text'],'\hypertarget'=>['ignore','text'], + '\hyperbaseurl'=>1,'\hyperimage'=>['ignore','text'],'\hyperdef'=>['ignore','ignore','text'], + '\phantomsection'=>0,'\autoref'=>1,'\autopageref'=>1, + '\hypersetup'=>1,'\urlstyle'=>1, + '\pdfbookmark'=>2,'\currentpdfbookmark'=>2,'\subpdfbookmark'=>2,'\belowpdfbookmark'=>2, + '\pdfstringref'=>2,'\texorpdfstring'=>['text','ignore'], + '\hypercalcbp'=>1,'\Acrobatmenu'=>2}; +$PackageTeXmacrocount{'hyperref'}={ + '\url'=>1,'\nolinkurl'=>1}; + +# Rules for package import +$PackageTeXfileinclude{'import'}={ + '\import'=>'dir file','\subimport'=>'subdir file', + '\inputfrom'=>'dir file','\subinputfrom'=>'subdir file', + '\includefrom'=>'dir file','\subincludefrom'=>'subdir file'}; + +# Rules for package inputenc +$PackageTeXmacro{'inputenc'}={ + '\inputencoding'=>1}; + +# Rules for package listings +$PackageTeXenvir{'listings'}={'lstlisting'=>'xall'}; +$PackageTeXmacro{'listings'}={'\lstset'=>['ignore'],'\lstinputlisting'=>['ignore']}; + +# Rules for package psfig +$PackageTeXmacro{'psfig'}={'\psfig'=>1}; + +# Rules for package sectsty +$PackageTeXmacro{'sectsty'}={ + '\allsectionsfont'=>1,'\partfont'=>1,'\chapterfont'=>1,'\sectionfont'=>1, + '\subsectionfont'=>1,'\subsubsectionfont'=>1,'\paragraphfont'=>1,'\subparagraphfont'=>1, + '\minisecfont'=>1,'\partnumberfont'=>1,'\parttitlefont'=>1,'\chapternumberfont'=>1, + '\chaptertitlefont'=>1,'\nohang'=>0}; + +# Rules for package setspace +$PackageTeXenvir{'setspace'}={ + 'singlespace'=>'text','singlespace*'=>'text','onehalfspace'=>'text','doublespace'=>'text', + 'spacing'=>'text'}; +$PackageTeXmacro{'setspace'}={ + 'beginspacing'=>1, + '\singlespacing'=>0,'\onehalfspacing'=>0,'\doublespacing'=>0, + '\setstretch'=>1,'\SetSinglespace'=>1}; + +# Rules for package subfiles +$PackageTeXfileinclude{'subfiles'}={ + '\subfile'=>'file'}; + +# Rules for package url +# NB: \url|...| variant not implemented, only \url{...} +# NB: \urldef{macro}{url} will not be counted +$PackageTeXmacro{'url'}={ + '\url'=>1,'\urldef'=>2,'\urlstyle'=>1,'\DeclareUrlCommand'=>['ignore','xxx']}; +$PackageTeXmacro{'setspace'}={ + '\url'=>1}; + +# Rules for package wrapfig +$PackageTeXenvir{'wrapfig'}={ + 'wrapfigure'=>'float','wraptable'=>'float'}; +$PackageTeXmacro{'wrapfig'}={ + 'beginwrapfigure'=>2,'beginwraptable'=>2}; + +# Rules for package xcolor (reimplements the color package) +# NB: only main macros (mostly from package color) included +$PackageTeXmacro{'xcolor'}={ + '\textcolor'=>['ignore','text'],'\color'=>1,'\pagecolor'=>1,'\normalcolor'=>0, + '\colorbox'=>['ignore','text'],'\fcolorbox'=>['ignore','ignore','text'], + '\definecolor'=>3,\'DefineNamedColor'=>4, + '\colorlet'=>2}; + ###### Main script @@ -523,13 +847,14 @@ sub MAIN { if ($showVersion && !$htmlstyle && !($briefsum && $totalflag)) { print "\n=== LaTeX word count (TeXcount version $versionnumber) ===\n\n"; } - conditional_print_help_style(); + conditional_print_style_list(); my $totalcount=Parse_file_list(@toplevelfiles); conditional_print_total($totalcount); Report_Errors(); if ($optionWordFreq || $optionWordClassFreq) {print_word_freq();} + if ($optionMacroStat) {print_macro_stat();} } elsif ($showcodes>1) { - conditional_print_help_style(); + conditional_print_style_list(); } else { error($Main,'No files specified.'); } @@ -540,7 +865,9 @@ sub MAIN { sub Initialise { _option_subcount(); # Windows settings - if ($^O=~/^MSWin/) { + if ($^O eq 'MSWin32') { + } elsif ($^O=~/^MSWin/) { + #DELETE: do not overrule colour setting option_ansi_colours(0); } } @@ -548,21 +875,33 @@ sub Initialise { # Check arguments, exit on exit condition sub Check_Arguments { my @args=@_; + my $arg=$args[0]; if (!@args) { print_version(); - print_syntax(); - print_reference(); + print_short_help(); exit; - } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))$/) { + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))$/) { print_help(); exit; - } elsif ($args[0]=~/^(--?(h|\?|help)|\/(\?|h))=/) { - print_help_on("$'"); + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))=(.*)$/) { + print_help_on_rule($4); + exit; + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(opt|options?)$/) { + print_syntax(); + exit; + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(opt|options?)=(.*)$/) { + print_syntax_subset($5); exit; - } elsif ($args[0]=~/^--?(ver|version)$/) { + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(styles?)$/) { + print_help_on_styles(); + exit; + } elsif ($arg=~/^(--?(h|\?|help)|\/(\?|h))-?(styles?)=(\w+)$/) { + print_help_on_styles($5); + exit; + } elsif ($arg=~/^--?(ver|version)$/) { print_version(); exit; - } elsif ($args[0]=~/^--?(lic|license|licence)$/) { + } elsif ($arg=~/^--?(lic|license|licence)$/) { print_license(); exit; } @@ -576,8 +915,8 @@ sub Parse_Arguments { foreach my $arg (@args) { if (parse_option($arg)) {next;} if ($arg=~/^\-/) { - print 'Invalid opton '.$arg."\n"; - print_syntax(); + print "Invalid opton $arg \n\n"; + print_short_help(); exit; } $arg=~s/\\/\//g; @@ -600,8 +939,8 @@ sub parse_option { # Parse presetting options sub parse_options_preset { my $arg=shift @_; - if ($arg=~/^-(opt|option|options|optionfile)=/) { - _parse_optionfile($'); + if ($arg=~/^-(opt|option|options|optionfile)=(.*)$/) { + _parse_optionfile($2); } else {return 0;} return 1; @@ -622,6 +961,11 @@ sub parse_options_parsing { $globalworkdir=$1; $globalworkdir=~s:([^\/\\])$:$1\/:; } + elsif ($arg eq '-auxdir') {$auxdir=undef;} + elsif ($arg=~/^-auxdir=(.*)$/) { + $auxdir=$1; + $auxdir=~s:([^\/\\])$:$1\/:; + } elsif ($arg =~/^-(enc|encode|encoding)=(.+)$/) {$encoding=$2;} elsif ($arg =~/^-(utf8|unicode)$/) {$encoding='utf8';} elsif ($arg =~/^-(alpha(bets?)?)=(.*)$/) {set_script_options(\@AlphabetScripts,$3);} @@ -649,6 +993,7 @@ sub parse_options_counts { elsif ($arg eq '-freq') {$optionWordFreq=1;} elsif ($arg =~/^-freq=(\d+)$/) {$optionWordFreq=$1;} elsif ($arg eq '-stat') {$optionWordClassFreq=1;} + elsif ($arg =~/^-macro-?(stat|freq)$/) {$optionMacroStat=1;} else {return 0;} return 1; } @@ -657,11 +1002,11 @@ sub parse_options_counts { sub _option_sum { my $arg=shift @_; if (!defined $arg) { - @sumweights=(1,1,1,0,0,1,1); + @sumweights=(0,1,1,1,0,0,1,1); } elsif ($arg=~/^(\d+(\.\d*)?(,\d+(\.\d*)?){0,6})$/) { - @sumweights=split(',',$1); + @sumweights=(0,split(',',$1)); } else { - print STDERR "Warning: Option value ".$arg." not valid, ignoring option.\n"; + print STDERR "Warning: Option value $arg not valid, ignoring option.\n"; } } @@ -674,7 +1019,7 @@ sub _option_subcount { } elsif (my $option=$BreakPointsOptions{$arg}) { %BreakPoints=%{$option}; } else { - print STDERR "Warning: Option value ".$arg." not valid, using default instead.\n"; + print STDERR "Warning: Option value $arg not valid, using default instead.\n"; %BreakPoints=%{$BreakPointsOptions{'default'}}; } } @@ -683,20 +1028,18 @@ sub _option_subcount { sub parse_options_output { my $arg=shift @_; if ($arg eq '-strict') {$strictness=1;} - elsif ($arg eq '-v0') {$verbose=0;} - elsif ($arg eq '-v1') {$verbose=1;} - elsif ($arg eq '-v2') {$verbose=2;} - elsif ($arg eq '-v3' || $arg eq '-v') {$verbose=3;} - elsif ($arg eq '-v4') {$verbose=3; $showstates=1;} + elsif ($arg eq '-v') {set_verbosity_options('3');} + elsif ($arg =~/^-v([0-4=+\-].*)/) {set_verbosity_options($1);} elsif ($arg =~/^-showstates?$/ ) {$showstates=1;} - elsif ($arg =~/^-(q|quiet)$/ ) {$verbose=-1;} + elsif ($arg =~/^-(q|quiet)$/ ) {$printlevel=-1;} elsif ($arg =~/^-(template)=(.*)$/ ) {_set_output_template($2);} elsif ($arg eq '-split') {$optionFast=1;} elsif ($arg eq '-nosplit') {$optionFast=0;} elsif ($arg eq '-showver') {$showVersion=1;} elsif ($arg eq '-nover') {$showVersion=-1;} - elsif ($arg =~/^-nosep(s|arators?)?$/ ) {$separator='';} - elsif ($arg =~/^-sep(arators?)?=(.*)$/ ) {$separator=$2;} + elsif ($arg =~/^-nosep(arator)?s?$/ ) {$separator='';} + elsif ($arg =~/^-sep(arator)?s?=(.*)$/ ) {$separator=$2;} + elsif ($arg =~/^-out=(.*)/ ) {close STDOUT; open STDOUT,'>',$1;} else {return 0;} return 1; } @@ -705,8 +1048,8 @@ sub parse_options_output { sub _set_output_template { my $template=shift @_; $outputtemplate=$template; - if ($template=~/\{(S|SUM)[\?\}]/i && !@sumweights) { - @sumweights=(1,1,1,0,0,1,1); + if ($template=~/\{(SUM)[\?\}]/i && !@sumweights) { + @sumweights=(0,1,1,1,0,0,1,1); } if ($template=~/\{SUB\?/i && !$showsubcounts) { _option_subcount(); @@ -718,14 +1061,19 @@ sub parse_options_format { my $arg=shift @_; if ($arg eq '-brief') {$briefsum=1;} elsif ($arg eq '-total') {$totalflag=1;} - elsif ($arg eq '-0') {$briefsum=1;$totalflag=1;$verbose=-1;$finalLineBreak=0;} - elsif ($arg eq '-1') {$briefsum=1;$totalflag=1;$verbose=-1;} - elsif ($arg eq '-html' ) {option_ansi_colours(0);$htmlstyle = 2;} - elsif ($arg eq '-htmlcore' ) {option_ansi_colours(0);$htmlstyle = 1;} + elsif ($arg eq '-0') {$briefsum=1;$totalflag=1;$printlevel=-1;$finalLineBreak=0;} + elsif ($arg eq '-1') {$briefsum=1;$totalflag=1;$printlevel=-1;} + elsif ($arg eq '-htmlcore' ) {option_ansi_colours(0);$htmlstyle=1;} + elsif ($arg eq '-html' ) {option_ansi_colours(0);$htmlstyle=2;} + elsif ($arg eq '-tex' ) {option_ansi_colours(0);$texcodeoutput=1;} elsif ($arg =~/^\-(nocol|nc$)/) {option_ansi_colours(0);} elsif ($arg =~/^\-(col)$/) {option_ansi_colours(1);} elsif ($arg eq '-codes') {$showcodes=2;} elsif ($arg eq '-nocodes') {$showcodes=0;} + elsif ($arg =~/^-htmlfile=(.+)$/) {$HTMLfile=$1;} + elsif ($arg =~/^-cssfile=(.+)$/) {$CSSfile=$1;} + elsif ($arg =~/^-css=file:(.+)$/) {$CSSfile=$1;} + elsif ($arg =~/^-css=(.+)$/) {$CSShref=$1;} else {return 0;} return 1; } @@ -733,8 +1081,8 @@ sub parse_options_format { # Include options from option file sub _parse_optionfile { my $filename=shift @_; - open(FH,"<",$filename) - || die "Option file not found: ".$filename."\n"; + open(FH,'<',$filename) + || die "Option file not found: $filename\n"; my @options=<FH>; close(FH); s/^\s*(#.*|)//s for @options; @@ -744,7 +1092,7 @@ sub _parse_optionfile { foreach my $arg (@options) { __optionfile_tc($arg) || parse_option($arg) - || die "Invalid option ".$arg." in ".$filename."\n"; + || die "Invalid option $arg in $filename.\n"; } } @@ -752,12 +1100,12 @@ sub _parse_optionfile { sub __optionfile_tc { my $arg=shift @_; $arg=~s/^\%\s*// || return 0; - if ($arg=~/^subst\s+(\\\w+)\s+/i) { - $substitutions{$1}=$'; + if ($arg=~/^subst\s+(\\\w+)\s+(.*)$/i) { + $substitutions{$1}=$2; } elsif ($arg=~/^(\w+)\s+([\\]*\w+)\s+([^\s\n]+)(\s+([0-9]+))?/i) { - tc_macro_param_option($1,$2,$3,$5) || die "Invalid TC option: ".$arg."\n"; + tc_macro_param_option($Main,$1,$2,$3,$5) || die "Invalid TC option: $arg\n"; } else { - print "Invalid TC option format: ".$arg."\n"; + print "Invalid TC option format: $arg\n"; return 0; } return 1; @@ -766,10 +1114,10 @@ sub __optionfile_tc { # Parse file list and return total count sub Parse_file_list { my @files=@_; - my $listtotalcount=new_count("Total"); + my $listtotalcount=new_count('Total'); foreach (@files) {s/\\/\//g; s/ /\\ /g;} - if (@files) { - @files=<@files>; # For the sake of Windows: expand wildcards! + if (@files) { + @files=<@files>; # For the sake of Windows: expand wildcards! for my $file (@files) { my $filetotalcount=parse_file($file); add_to_total($listtotalcount,$filetotalcount); @@ -778,47 +1126,49 @@ sub Parse_file_list { if ($fileFromSTDIN) { my $filetotalcount=parse_file($_STDIN_); add_to_total($listtotalcount,$filetotalcount); - } + } return $listtotalcount; } # Parse file and included files, and return total count sub parse_file { my $file=shift @_; - $workdir=$globalworkdir; - if (!defined $workdir) { + if (defined ($workdir=$globalworkdir)) {} + elsif ($file eq $_STDIN_) {$workdir='';} + else { $workdir=$file; $workdir =~ s/^((.*[\\\/])?)[^\\\/]+$/$1/; } - if ($htmlstyle && ($verbose || !$totalflag)) {print "\n<div class='filegroup'>\n";} - my $filetotalcount=new_count("File(s) total: ".$file); + if ($htmlstyle && ($printlevel>0 || !$totalflag)) {print "\n<div class='filegroup'>\n";} + my $filetotalcount=new_count('File(s) total: '.$file); @filelist=(); - _add_file($filetotalcount,$file); + _add_file($filetotalcount,[$file,$workdir],'File: '.$file); foreach my $f (@filelist) { - _add_file($filetotalcount,$f,"Included file: ".$f); + _add_file($filetotalcount,$f,'Included file: '.$f->[0]); } if (!$totalflag && get_count($filetotalcount,$CNT_FILE)>1) { - if ($htmlstyle) {formatprint("Sum of files: ".$file."\n",'h2');} + if ($htmlstyle) {formatprint("Sum of files: $file\n",'h2');} print_count($filetotalcount,'sumcount'); } - if ($htmlstyle && ($verbose || !$totalflag)) {print "</div>\n\n";} + if ($htmlstyle && ($printlevel>0 || !$totalflag)) {print "</div>\n\n";} return $filetotalcount; } # Parse single file, included files will be appended to @filelist. sub _add_file { - my ($filetotalcount,$f,$title)=@_; - my $tex=TeXfile($f,$title); - my $fpath=$f; - $fpath=~s/^((.*[\\\/])?)[^\\\/]+$/$1/; + my ($filetotalcount,$pathfile,$title)=@_; + my @paths=@$pathfile; + my $file=shift @paths; + my $tex=TeXfile($file,$title); if (!defined $tex) { - error($Main,'File not found or not readable: '.$f); - } else { - parse($tex); - my $filecount=next_subcount($tex); - if (!$totalflag) {print_count($filecount);} - add_to_total($filetotalcount,$filecount); + error($Main,'File not found or not readable: '.$file); + return; } + $tex->{'PATH'}=\@paths; + parse($tex); + my $filecount=next_subcount($tex); + if (!$totalflag) {print_count($filecount);} + add_to_total($filetotalcount,$filecount); } ###### @@ -830,21 +1180,59 @@ sub _add_file { # Add file to list of files scheduled for parsing sub include_file { - my ($tex,$fname)=@_; - my $fpath=$workdir.$fname; + my ($tex,$state,$file,$refparam)=@_; + my %params=%$refparam; + my $type=$params{'<type>'}; + my @paths=@{$tex->{'PATH'}}; + my $filepath; + if ($type eq '<bbl>' && defined $auxdir) { + $auxdir=~s/([\\\/])*$/\//; + if (defined $globalworkdir) {@paths=[$auxdir];} + elsif ($auxdir=~/^(\w:)?[\\\/]/) {@paths=[$auxdir];} + else {@paths=[$workdir.$auxdir]} + } + foreach my $key (split(/\s+/,$type)) { + my $value=$params{$key}; + if ($key eq 'dir') {unshift @paths,$workdir.$value;} + elsif ($key eq 'subdir') {unshift @paths,$paths[0].$value;} + } + if ($file=~/^(\w:)?[\\\/]/) {$filepath=$file;} + else { + $filepath=_find_file_in_path($tex,$file,@paths) || BLOCK { + error($tex,'File '.$file.' not found in path ['.join(';',@paths).'].'); + return; + }; + } if ($includeTeX==2) { - my $bincode=read_binary($fpath) || BLOCK { - error($tex,"File $fpath not found."); + my $bincode=read_binary($filepath) || BLOCK { + error($tex,"File $filepath not readable."); return; }; flush_next($tex); line_return(0,$tex); - prepend_code($tex,$bincode,$fname); + # DELETE: prepend_code($tex,$bincode,$file); + my $texstate=texcode_insert_text($tex,$bincode,$file,@paths); + parse_all($tex,$state); + texcode_restore_state($tex,$texstate); } else { - push @filelist,$fpath; + push @filelist,[$filepath,@paths]; } } +# Seach path list to find file +sub _find_file_in_path { + my $tex=shift @_; + my $file=shift @_; + foreach my $path (@_) { + if (!$path=~/[\\\/]$/) {$path.='/';} + my $filepath=$path.$file; + if (-e $filepath) {return $filepath;} + elsif ($filepath=~/\.tex$/i) {} + elsif (-e $filepath.'.tex') {return $filepath.'.tex';} + } + return undef; +} + # Print count (total) if conditions are met sub conditional_print_total { my $sumcount=shift @_; @@ -852,7 +1240,7 @@ sub conditional_print_total { if ($totalflag && $briefsum && @sumweights) { print get_sum_count($sumcount),"\n"; } else { - if ($htmlstyle) {formatprint("Total word count",'h2');} + if ($htmlstyle) {formatprint('Total word count','h2');} print_count($sumcount,'sumcount'); } } @@ -875,24 +1263,49 @@ sub ansiprint { # Apply options to set values sub Apply_Options { - if ($showcodes>1 && $verbose<1) {$verbose=3;} - %STYLE=%{$STYLES[$verbose]}; - if (defined $encoding && $encoding eq 'guess') {$encoding=undef;} - if (!defined $encoding) { - } elsif (ref(find_encoding($encoding))) { - if (!$htmlstyle) {$outputEncoding=$encoding;} - } else { - error($Main,"Unknown encoding $encoding ignored."); - error_details($Main,'Valid encodings are: '.wrap('','',join(', ',Encode->encodings(':all')))); - $encoding=undef; - } - if (!defined $outputEncoding) {$outputEncoding='utf8';} - binmode STDIN; - binmode STDOUT,':encoding('.$outputEncoding.')'; + apply_encoding_options(); if ($htmlstyle>1) {html_head();} flush_errorbuffer($Main); apply_language_options(); if ($includeBibliography) {apply_include_bibliography();} + if ($showcodes>1 && !($STYLE{'<printlevel>'})) {%STYLE=%{$STYLES{'All'}};} + if ($showstates) {set_verbosity_options('+States');} + if (!@sumweights) {set_verbosity_options('-Sums');} + (defined $printlevel) + || (defined ($printlevel=$STYLE{'<printlevel>'})) + || ($printlevel=$defaultprintlevel); +} + +# Set or add styles included in the verbose output +sub set_verbosity_options { + my $verb=shift @_; + my $st; + if ($defaultprintlevel<1) {$defaultprintlevel=1;} + if ( $verb=~s/^([0-4])// || $verb=~s/^=([0-4])?// ) { + my $key=$1; + if (!defined $key) {%STYLE=(%{$STYLES{'<core>'}});} + elsif ($st=$STYLES{$key}) {%STYLE=(%$st);} + else { + error($Main,"Unknown verbosity option '$key'."); + %STYLE=(%{$STYLES{'<core>'}}); + } + } + while ($verb=~s/^([\+\-]?)(\w+)//) { + my $add=!($1 eq '-'); + my $key=$2; + if ($st=$STYLES{$key}) { + if ($add) {@STYLE{keys %$st}=values %$st;} + else {delete @STYLE{keys %$st};} + } elsif ($st=$STYLES{'All'}->{$key}) { + if ($add) {$STYLE{$key}=$st;} + elsif ($STYLE{$key}) {delete $STYLE{$key};} + } else { + error($Main,"Unknown verbosity option '$key'."); + } + } + if ($verb ne '') { + error($Main,"Unknown verbosity option format: $verb"); + } } # Set or add scripts to array of scripts @@ -907,13 +1320,14 @@ sub set_script_options { $scr='alphabetic'; } if ($scr=~/^[a-z]/) {$scr='Is_'.$scr;} - if (is_property_valid($scr)) {push @$scriptset,$scr;} + if (_is_property_valid($scr)) {push @$scriptset,$scr;} else {error($Main,"Unknown script $scr ignored.");} } } -sub is_property_valid { +sub _is_property_valid { my $script=shift @_; + if (! $script=~/^\w+$/) {return 0;} eval {' '=~/\p{$script}/}; if ($@) {return 0;} else {return 1;} } @@ -954,15 +1368,31 @@ sub set_language_option { } elsif ($language=~/^(char|character|letter)s?(-?only)?$/) { @WordPatterns=($NamedWordPattern{'letters'}); if (defined $2) {@LogogramScripts=();} - $countlabel[1]='Letters in text'; - $countlabel[2]='Letters in headers'; - $countlabel[3]='Letters in captions'; + $countdesc[1]='Letters in text'; + $countdesc[2]='Letters in headers'; + $countdesc[3]='Letters in captions'; return 'letter'; } else { return undef; } } +# Set encoding based on encoding options, guess encoding if not set +sub apply_encoding_options { + if (defined $encoding && $encoding eq 'guess') {$encoding=undef;} + if (!defined $encoding) { + } elsif (ref(find_encoding($encoding))) { + if (!$htmlstyle) {$outputEncoding=$encoding;} + } else { + error($Main,"Unknown encoding $encoding ignored."); + error_details($Main,'Valid encodings are: '.wrap('','',join(', ',Encode->encodings(':all')))); + $encoding=undef; + } + if (!defined $outputEncoding) {$outputEncoding='utf8';} + binmode STDIN; + binmode STDOUT,':encoding('.$outputEncoding.')'; +} + # Apply language options sub apply_language_options { my @tmp; @@ -993,49 +1423,239 @@ sub apply_include_bibliography { include_package('%incbib'); } +# Process TC instruction, return 1 if applied, 0 if not (e.g. error) +sub tc_macro_param_option { + my ($tex,$instr,$macro,$param,$option)=@_; + if (!defined $param) { + error($tex,'%TC:'.$instr.' requires parameter rule specification.'); + return 0; + } + elsif ($instr eq 'macro') { + if (!($macro=~/^\\/)) { + warning($tex,'%TC:macro '.$macro.': should perhaps be \\'.$macro.'?'); + } + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$macro,$param); + } + elsif ($instr eq 'exclude') { + warning($tex,'%TC:exclude is deprecated. Use %TC:macro instead.'); + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$macro,$param); + } + elsif ($instr eq 'header') { + warning($tex,'%TC:header is deprecated. Instead use e.g. %TC:macro \\name [header].'); + $TeXmacrocount{$macro}=[$CNT_COUNT_HEADER]; + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$macro,$param); + } + elsif ($instr eq 'preambleinclude') { + return _tc_macro_set_param($tex,\%TeXpreamble,$instr,$macro,$param); + } + elsif ($instr eq 'floatinclude') { + return _tc_macro_set_param($tex,\%TeXfloatinc,$instr,$macro,$param); + } + elsif ($instr=~/^(group|envir)$/) { + if (!defined $option) { + error($tex,'TC:'.$instr.' requires contents rule specification.'); + return 0; + } + defined ($option=key_to_state($option,$tex)) || return 0; + $TeXenvir{$macro}=$option; + return _tc_macro_set_param($tex,\%TeXmacro,$instr,$PREFIX_ENVIR.$macro,$param); + } + elsif ($instr=~/^(macrocount|macroword)$/) { + return _tc_macro_set_param($tex,\%TeXmacrocount,$instr,$macro,$param,\&key_to_cnt,$REGEX_NUMBER); + } + elsif ($instr eq 'fileinclude') { + # No parameter checking here, just pass on; errors reported when used instead. + $param=~s/^\[(.*)\]$/$1/; + $param=~s/,/ /g; + assert($param=~/^\w+(\s\w+)*$/,$tex,'Invalid %TC:fileinclude parameter: '.$param) + || return 0; + $TeXfileinclude{$macro}=$param; + } + elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;} + else { + error($tex,'Unknown TC command: '.$instr); + return 0; + } + return 1; +} + +# Convert TC command parameters and add to hash +sub _tc_macro_set_param { + my ($tex,$hash,$instr,$macro,$param,$keymap,$validregex)=@_; + if (!defined $keymap) {$keymap=\&key_to_state; $validregex=$REGEX_NUMBER;} + $param=__interpret_tc_parameter($tex,$param,$keymap,$validregex); + if (!defined $param) {return 0;} + $hash->{$macro}=$param; + return 1; +} + +# Convert [...] into array of states, optionally return value if a valid pattern +sub __interpret_tc_parameter { + my ($tex,$param,$keymap,$validregex)=@_; + if ((defined $validregex) && ($param=~$validregex)) { + return $param; + } elsif ($param=~/^\[([0-9a-zA-Z,:]*)\]$/) { + my @params; + my $value; + foreach my $key (split(',',$1)) { + if ($key=~/^(\w*):(\w+)$/) { + defined ($value=$keymap->($PREFIX_PARAM_OPTION.$1,$tex)) || return undef; + push @params,$value; + $key=$2; + } + defined ($value=$keymap->($key,$tex)) || return undef; + push @params,$value; + } + return [@params]; + } + error($tex,'Invalid TC command parameter: '.$param); + return undef; +} + + +###### Macro rules handling + + +# Takes hash, value, list of keys and adds mappings to hash +sub add_keys_to_hash { + my $hash=shift @_; + my $value=shift @_; + foreach (@_) { + $hash->{$_}=$value; + } +} + +# Add source hash to target hash, optionally converting values +sub __add_to_hash { + my $target=shift @_; + my $source=shift @_; + my $convert=shift @_; + if (!defined $convert) {$convert=&__dummy;} + while (my ($key,$val)=each(%$source)) { + $target->{$key}=$convert->($val,@_); + } +} + +# Convert hash by applying function to all values +sub convert_hash { + my $hash=shift @_; + my $convert=shift @_; + while (my ($key,$val)=each(%$hash)) { + $hash->{$key}=$convert->($val,@_); + } +} + +# Convert key to parser state constant +sub key_to_state { + my ($key,$tex)=@_; + return key_convert(lc($key),$tex,\%key2state,'Unknown state key {key} replaced by IGNORE.',$STATE_IGNORE); +} + +# Convert key array to parser state constants +sub keyarray_to_state { + my ($array,$tex)=@_; + return keyarray_convert($array,$tex,\&key_to_state); +} + +# Convert key to counter constant +sub key_to_cnt { + my ($key,$tex)=@_; + return key_convert(lc($key),$tex,\%key2cnt,'Unknown counter key {key} ignored.'); +} + +# Convert key array to counter constants +sub keyarray_to_cnt { + my ($array,$tex)=@_; + return keyarray_convert($array,$tex,\&key_to_cnt); +} + +# Convert key to value using map +sub key_convert { + my ($key,$tex,$map,$error,$default)=@_; + my $value=$map->{$key}; + if (!defined $value) { + if (!defined $tex) {$tex=$Main;} + $error=~s/\{key\}/$key/g; + error($tex,$error); + return $default; + } + return $value; +} + +# Convert state keys to state nos for arrays +sub keyarray_convert { + my ($array,$tex,$conv)=@_; + if (ref($array) eq 'ARRAY') { + my @ar; + foreach my $key (@$array) { + if (defined (my $cnt=$conv->($key,$tex))) {push @ar,$cnt;} + } + $array=\@ar; + } + return $array; +} + +# Convert transition state to parsing state, and increment counter +sub transition_to_content_state { + my ($tex,$state)=@_; + my $tr=$transition2state{$state}; + if (defined $tr) { + if (my $cnt=$tr->[1]) {inc_count($tex,$tr->[1]);} + return $tr->[0]; + } + return $state; +} + +# Remove all rules +sub remove_all_rules { + %TeXpackageinc=(); + %TeXpreamble=(); + %TeXfloatinc=(); + %TeXmacro=(); + %TeXmacrocount=(); + %TeXenvir=(); + %TeXfileinclude=(); + %IncludedPackages=(); +} + # Process package inclusion sub include_package { - my $incpackage=shift @_; - my $sub; - _add_to_hash_if_exists(\%TeXmacro,\%PackageTeXmacro,$incpackage); - _add_to_hash_if_exists(\%TeXmacroword,\%PackageTeXmacroword,$incpackage); - _add_to_hash_if_exists(\%TeXheader,\%PackageTeXheader,$incpackage); - _add_to_hash_if_exists(\%TeXgroup,\%PackageTeXgroup,$incpackage); - _add_to_hash_if_exists(\%TeXfileinclude,\%PackageTeXfileinclude,$incpackage); + my ($incpackage,$tex)=@_; + if (defined $IncludedPackages{$incpackage}) {return;} + $IncludedPackages{$incpackage}=1; + # Add rules for package, preamble and float inclusion, and add to macro rules + _add_package(\%TeXpackageinc,\%PackageTeXpackageinc,$incpackage,\&__dummy,$tex); + _add_package(\%TeXpreamble,\%PackageTeXpreamble,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXfloatinc,\%PackageTeXfloatinc,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacro,\%PackageTeXpackageinc,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacro,\%PackageTeXpreamble,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacro,\%PackageTeXfloatinc,$incpackage,\&keyarray_to_state,$tex); + # Add regular rules + _add_package(\%TeXmacro,\%PackageTeXmacro,$incpackage,\&keyarray_to_state,$tex); + _add_package(\%TeXmacrocount,\%PackageTeXmacrocount,$incpackage,\&keyarray_to_cnt,$tex); + _add_package(\%TeXenvir,\%PackageTeXenvir,$incpackage,\&key_to_state,$tex); + _add_package(\%TeXfileinclude,\%PackageTeXfileinclude,$incpackage,\&__dummy,$tex); + if (my $subpackages=$PackageSubpackage{$incpackage}) { + foreach my $sub (@{$subpackages}) { + include_package($sub,$tex); + } + } } # Add package rules if defined -sub _add_to_hash_if_exists { - my ($target,$source,$name)=@_; +sub _add_package { + my ($target,$source,$name,$convert,$tex)=@_; my $sub; if ($sub=$source->{$name}) { - while (my ($key,$val)=each(%$sub)) { - $target->{$key}=$val; - } + __add_to_hash($target,$sub,$convert); } } -# Process TC instruction -sub tc_macro_param_option { - my ($instr,$macro,$param,$option)=@_; - if ($param=~/^\[([0-9,]+)\]$/) {$param=[split(',',$1)];} - if (!defined $option) {$option=1;} - if (($instr eq 'macro') || ($instr eq 'exclude')) {$TeXmacro{$macro}=$param;} - elsif ($instr eq 'header') {$TeXheader{$macro}=$param;$TeXmacro{$macro}=$param;} - elsif ($instr eq 'macroword') {$TeXmacroword{$macro}=$param;} - elsif ($instr eq 'preambleinclude') {$TeXpreamble{$macro}=$param;} - elsif ($instr eq 'group') { - $TeXmacro{'begin'.$macro}=$param; - $TeXgroup{$macro}=$option; - } - elsif ($instr eq 'floatinclude') {$TeXfloatinc{$macro}=$param;} - elsif ($instr eq 'fileinclude') {$TeXfileinclude{$macro}=$param;} - elsif ($instr eq 'breakmacro') {$BreakPoints{$macro}=$param;} - else {return 0;} - return 1; +# Dummy conversion function: returns arguments unchanged +sub __dummy { + return shift @_; } - ###### TeX code handle @@ -1053,11 +1673,14 @@ sub getMain { ## Make TeX handle for LaTeX code: the main TeXcount object # The "TeX object" is a data containser: a hash containing -# - filename: name of LaTeX file being parsed -# - filepath: path to LaTeX file +# - filepath: path of LaTeX file being parsed +# - filename: file name of LaTeX file +# - dirpath: path to directory containing LaTeX file +# - PATH: array of paths to search for included files # - countsum: count object with total count (incl. subcounts) # - subcount: count object for subcount (to be added to countsum) # - subcounts: list of subcounts +# - errorcount: the number of errors reported # plus following elements used for the processing the LaTeX code # - line: the LaTeX paragraph being processed # - texcode: what remains of LaTeX code to process (after line) @@ -1074,8 +1697,8 @@ sub getMain { # prepared. Note that this requires that the last subcount object be added to # the countsum once the end of the document is reached. sub TeXcode { - my ($bincode,$filename,$title)=@_; - my $tex=_TeXcode_blank($filename,$title); + my ($bincode,$file,$title)=@_; + my $tex=_TeXcode_blank($file,$title); _TeXcode_setcode($tex,$bincode); more_texcode($tex); return $tex; @@ -1083,16 +1706,21 @@ sub TeXcode { # Return a blank TeXcode object sub _TeXcode_blank { - my ($filename,$title)=@_; + my ($file,$title)=@_; if (defined $title) {} - elsif (defined $filename) {$title="File: ".$filename;} - else {$title="Word count";} + elsif (defined $file) {$title='File: '.$file;} + else {$title='Word count';} my %TeX=(); $TeX{'errorcount'}=0; - $TeX{'filename'}=$filename; - if (!defined $filename) {$TeX{'filepath'}='';} - elsif ($filename=~/^(.*[\\\/])[^\\\/]+$/) {$TeX{'filepath'}=$1;} - else {$TeX{'filepath'}='';} + $TeX{'warnings'}={}; + $TeX{'filepath'}=$file; + if (!defined $file) {} + elsif ($file=~/^<.+>$/) {$TeX{'filename'}=$file;} + elsif ($file=~/^(.*[\\\/])([^\\\/]+)$/) { + $TeX{'dirpath'}=$1; $TeX{'filename'}=$2; + } + else {$TeX{'dirpath'}=''; $TeX{'filename'}=$file;} + $TeX{'PATH'}=[]; $TeX{'line'}=''; $TeX{'next'}=undef; $TeX{'type'}=undef; @@ -1102,11 +1730,9 @@ sub _TeXcode_blank { my $countsum=new_count($title); $TeX{'countsum'}=$countsum; $countsum->{'TeXcode'}=\%TeX; - my $count=new_count("_top_"); + my $count=new_count('_top_'); $TeX{'subcount'}=$count; inc_count(\%TeX,$CNT_FILE); - my @countlist=(); - $countsum->{'subcounts'}=\@countlist; return \%TeX; } @@ -1115,8 +1741,6 @@ sub _TeXcode_setcode { my ($tex,$bincode)=@_; $tex->{'texcode'}=_prepare_texcode($tex,$bincode); $tex->{'texlength'}=length($tex->{'texcode'}); - # TODO: Comment out this test - if ($tex->{'texcode'} =~ /([[:^ascii:]])/) {warning($tex,"Code contains non-ASCII characters.");} } # Decode and return TeX/LaTeX code @@ -1137,9 +1761,7 @@ sub _decode_texcode { my $decoder; if (defined $encoding) { $decoder=find_encoding($encoding); - eval { - $texcode=$decoder->decode($texcode); - }; + eval {$texcode=$decoder->decode($texcode);}; if ($@) { error($tex,'Decoding file/text using the '.$decoder->name.' encoding failed.'); } @@ -1200,26 +1822,51 @@ sub apply_substitution_rule { ## Get more TeX code from texcode buffer if possible, return 1 if done sub more_texcode { my ($tex)=@_; - if (!defined $tex->{'texcode'}) {return 0;} - if ( $optionFast && $tex->{'texcode'} =~ s/^.*?(\r{2,}|\n{2,}|(\r\n){2,})//s ) { - $tex->{'line'}.=$&; + if ($tex->{'texcode'} eq '') {return 0;} + if ( $optionFast && $tex->{'texcode'} =~ s/^(.*?(\r{2,}|\n{2,}|(\r\n){2,}))//s ) { + $tex->{'line'}.=$1; # $1 ~ ${^MATCH} return 1; } $tex->{'line'}.=$tex->{'texcode'}; - $tex->{'texcode'}=undef; + $tex->{'texcode'}=''; return 1; } ## Prepend LaTeX code to TeXcode object sub prepend_code { - my ($tex,$code,$fname)=@_; - my $prefix="\n% --- Start of included file ".$fname."\n"; - my $suffix="\n% --- End of included file ".$fname."\n"; + my ($tex,$code,$filename)=@_; + my $prefix="\n% --- Start of included file $filename\n"; + my $suffix="\n% --- End of included file $filename\n"; $code=_decode_texcode($tex,$code); $tex->{'length'}+=length($code); $tex->{'texcode'}=$prefix.$code.$suffix.$tex->{'line'}.$tex->{'texcode'}; $tex->{'line'}=''; - more_texcode(); + more_texcode($tex); +} + +# Save and return TeXcode state, and replace with new text +sub texcode_insert_text { + my ($tex,$code,$filename,@path)=@_; + my %texstate=( + 'texcode'=>$tex->{'line'}.$tex->{'texcode'}, + 'eof'=>$tex->{'eof'}, + 'PATH'=>$tex->{'PATH'}); + my $prefix="\n% --- Start of included file $filename\n"; + my $suffix="\n% --- End of included file $filename\n"; + $code=_decode_texcode($tex,$code); + $tex->{'length'}+=length($code); + $tex->{'texcode'}=$prefix.$code.$suffix; + $tex->{'line'}=''; + more_texcode($tex); + return \%texstate; +} + +# Restore TeXcode state +sub texcode_restore_state { + my ($tex,$texstate)=@_; + while (my ($key,$value)=each %$texstate) { + $tex->{$key}=$value; + } } ## Returns size of TeX code in bytes @@ -1235,20 +1882,20 @@ sub get_texsize { # Read LaTeX file into TeX object sub TeXfile { my ($filename,$title)=@_; + my ($header,$bincode); if ($filename eq $_STDIN_) { - if ($verbose>0) { - formatprint("File from STDIN\n",'h2'); - $blankline=0; - } - return TeXcode(_read_stdin(),'STDIN',$title); - } else { - my $bincode=read_binary($filename) || return undef; - if ($verbose>0) { - formatprint("File: ".$filename."\n",'h2'); - $blankline=0; - } - return TeXcode($bincode,$filename,$title); + $header='File from STDIN'; + $bincode=_read_stdin(); + } else { + defined ($bincode=read_binary($filename)) || return undef; + $header='File: '.$filename; + } + if ($printlevel>0) { + formatprint($header."\n",'h2'); + $blankline=0; } + my $tex=TeXcode($bincode,$filename,$title); + return $tex; } # Read file to string without regard for encoding @@ -1275,8 +1922,10 @@ sub _read_stdin { # Add warning to list of registered warnings (optionally to be reported at the end) sub warning { my ($tex,$text)=@_; - $tex->{'warnings'}->{$text}++; $warnings{$text}++; + # TODO: should only add warnings to subcount, not to TeXcode object + $tex->{'warnings'}->{$text}++; + if (my $count=$tex->{'subcount'}) {$count->{'warnings'}->{$text}++}; } # Register error and print error message @@ -1285,9 +1934,9 @@ sub error { if (defined $type) {$text=$type.': '.$text;} $errorcount++; $tex->{'errorcount'}++; - #print STDERR $text,"\n"; + if (my $count=$tex->{'subcount'}) {$count->{'errorcount'}++}; if (my $err=$tex->{'errorbuffer'}) {push @$err,$text;} - if ($verbose>=0) {_print_error($text);} + else {print_error($text);} } # Print error details @@ -1296,172 +1945,221 @@ sub error_details { print STDERR $text,"\n"; } +# Make assertion: i.e. report error if not true and return truthfullness +sub assert { + my $assertion=shift @_; + if ($assertion) {return 1;} + error(@_); + return 0; +} + +# Print error message +sub print_error { + my $text=shift @_; + line_return(1); + if ($printlevel<0) { + print STDERR 'ERROR: ',$text,"\n"; + } elsif ($htmlstyle) { + print STDERR 'ERROR: ',$text,"\n"; + print_style($text,'error'); + } else { + print_style("!!! $text !!!",'error'); + } + line_return(1); +} + +# Print errors in buffer and delete errorbuffer +sub flush_errorbuffer { + my $source=shift @_; + my $err=$source->{'errorbuffer'} || return; + foreach (@$err) {print_error($_);} + $source->{'errorbuffer'}=undef; +} + ###### Parsing routines # Parse LaTeX document sub parse { my ($tex)=@_; - if ($htmlstyle && $verbose) {print "<div class='parse'><p>\n";} + if ($htmlstyle && $printlevel>0) {print "<div class='parse'><p>\n";} + parse_all($tex); + if ($htmlstyle && $printlevel>0) {print "</p></div>\n";} +} + +# Parse until end +sub parse_all { + my ($tex,$state)=@_; + if (!defined $state) {$state=$STATE_TEXT;} while (!($tex->{'eof'})) { - _parse_unit($tex,$STATUS_TEXT); + _parse_unit($tex,$state); } - if ($htmlstyle && $verbose) {print "</p></div>\n";} } -# Parse one block or unit +# Parse one block/unit with given state until end token +# If end token is set to $_PARAM_, unit is assumed to be a parameter +# and words parsed as individual letters (simple_token is set) sub _parse_unit { - # Status: - # 0 = exclude from count - # 1 = text - # 2 = header text - # 3 = float text - # -1 = float (exclude) - # -2 = strong exclude, ignore begin-end groups - # -3 = stronger exclude, do not parse macro parameters - # -4 = ignore everything except end marker: even { - # -9 = preamble (between \documentclass and \begin{document}) - my ($tex,$status,$end)=@_; - if (!defined $status) { - error($tex,'Undefined parser status!','CRITICAL ERROR'); - exit; - } elsif (ref($status) eq 'ARRAY') { - error($tex,'Invalid parser status!','CRITICAL ERROR'); - exit; + my ($tex,$state,$end)=@_; + my $simple_token=0; + ( assert(defined $state,$tex,'Undefined parser state!','BUG') + && assert(!ref($state),$tex,'Invalid parser state of type '.ref($state).'!','BUG') + ) || exit; + $state=transition_to_content_state($tex,$state); + _set_printstate($tex,$state,$end); + if (defined $end && $end eq $_PARAM_) { + $end=undef; + $simple_token=1; } - if ($showstates) { - if (defined $end) { - $tex->{'printstate'}=':'.status_to_text($status).':'.$end.':'; - } else { - $tex->{'printstate'}=':'.status_to_text($status).':'; - } - flush_next($tex); - } - while (defined (my $next=_next_token($tex))) { + my $next; + while (defined ($next=next_token($tex,$simple_token))) { # Parse next token until token matches $end - set_style($tex,"ignore"); + set_style($tex,'ignore'); + if ($state==$STATE_MATH) {set_style($tex,'math');} if ((defined $end) && ($end eq $next)) {return;} # Determine how token should be interpreted - if ($status==$STATUS_PREAMBLE && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) { + if ($state==$STATE_PREAMBLE && $next eq '\begin' && $tex->{'line'}=~/^\{\s*document\s*\}/) { # \begin{document} - $status=$STATUS_TEXT; + $state=$STATE_TEXT; } - if ($status==$STATUS_EXCLUDE_ALL) { - # Ignore everything + # Pick the first matching interpretation: + if ($state==$STATE_EXCLUDE_ALL) { + # ignore everything } elsif ($tex->{'type'}==$TOKEN_SPACE) { # space or other code that should be passed through without styling flush_next($tex,' '); - } elsif ($next eq '\documentclass') { - # starts preamble - set_style($tex,'document'); - __gobble_option($tex); - __gobble_macro_parms($tex,1); - while (!($tex->{'eof'})) { - _parse_unit($tex,$STATUS_PREAMBLE); - } } elsif ($tex->{'type'}==$TOKEN_TC) { # parse TC instructions _parse_tc($tex,$next); } elsif ($tex->{'type'}==$TOKEN_WORD) { # word - if (my $cnt=status_text_cnt($status)) { - _process_word($tex,$next,$status); + if (my $cnt=state_text_cnt($state)) { + _process_word($tex,$next,$state); inc_count($tex,$cnt); - set_style($tex,status_to_style($status)); + set_style($tex,state_to_style($state)); } } elsif ($next eq '{') { - # {...} - _parse_unit($tex,$status,'}'); + # {...} group + set_style($tex,'ignore'); + _parse_unit($tex,$state,'}'); + set_style($tex,'ignore'); } elsif ($next eq '}') { error($tex,'Encountered } without corresponding {.'); - } elsif ($tex->{'type'}==$TOKEN_MACRO && $status==$STATUS_EXCLUDE_STRONGER) { - # ignore macro call + } elsif ($state==$STATE_EXCLUDE_STRONGER) { + # ignore remaining tokens set_style($tex,'ignore'); + } elsif ($next eq '\documentclass') { + # defines document class and starts preamble + set_style($tex,'document'); + _parse_documentclass_params($tex); + while (!($tex->{'eof'})) { + _parse_unit($tex,$STATE_PREAMBLE); + } } elsif ($tex->{'type'}==$TOKEN_MACRO) { # macro call - _parse_macro($tex,$next,$status); + _parse_macro($tex,$next,$state); } elsif ($next eq '$') { # math inline - _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'$'); + _parse_math($tex,$state,$CNT_COUNT_INLINEMATH,'$'); } elsif ($next eq '$$') { # math display (unless already in inlined math) if (!(defined $end && $end eq '$')) { - _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'$$'); + _parse_math($tex,$state,$CNT_COUNT_DISPLAYMATH,'$$'); } + } elsif ($simple_token) { + #DELETE: simple tokens should be handled properly + #print STDERR '<',$next,'>',"\n"; + # handle as parameter that should not be counted + set_style($tex,'ignore'); } if (!defined $end) {return;} } - if (defined $end) { - error($tex,'Reached end of file while waiting for '.$end.'.'); - } + defined $end && error($tex,'Reached end of file while waiting for '.$end.'.'); } -# Process word with a given status (>0, i.e. counted) +# Print state +sub _set_printstate { + my ($tex,$state,$end)=@_; + $tex->{'printstate'}=':'.state_to_text($state).':'.(defined $end?$end.':':''); + flush_next($tex); +} + +# Process word with a given state (>0, i.e. counted) sub _process_word { - my ($tex,$word,$status)=@_; + my ($tex,$word,$state)=@_; $WordFreq{$word}++; } # Parse unit when next token is a macro sub _parse_macro { - my ($tex,$next,$status)=@_; - # substat = parameter status settings + my ($tex,$next,$state)=@_; my $substat; + my @macro=($next); if (my $label=$BreakPoints{$next}) { if ($tex->{'line'}=~ /^[*]?(\s*\[.*?\])*\s*\{((.|\{.*\})*)\}/ ) { $label=$label.': '.$2; } next_subcount($tex,$label); } - set_style($tex,status_is_text($status)?'command':'exclcommand'); - if ($next eq '\begin' && status_inc_envir($status)) { - _parse_begin_end($tex,$status); - } elsif ($next eq '\end' && status_inc_envir($status)) { + if ($state==$STATE_MATH) {set_style($tex,'mathcmd');} + else {set_style($tex,state_is_text($state)?'cmd':'exclcmd');} + if ($next eq '\begin' && state_inc_envir($state)) { + _parse_envir($tex,$state); + push @macro,'<envir>'; + } elsif ($next eq '\end' && state_inc_envir($state)) { error($tex,'Encountered \end without corresponding \begin'); + push @macro,$STRING_ERROR; } elsif ($next eq '\verb') { - _parse_verb_region($tex,$status); - } elsif ( (status_is_text($status) || $status==$STATUS_PREAMBLE) - && defined ($substat=$TeXpackageinc{$next}) ) { - _parse_include_package($tex,$substat); - } elsif (($status==$STATUS_FLOAT) && ($substat=$TeXfloatinc{$next})) { + _parse_verb_region($tex,$state); + } elsif (state_is_parsed($state) && defined $TeXpackageinc{$next} ) { + _parse_include_package($tex); + push @macro,'<package>'; + } elsif (state_is_parsed($state) && defined (my $def=$TeXfileinclude{$next})) { + # include file (merge in or queue up for parsing) + set_style($tex,'cmd'); + if (defined ($substat=$TeXmacro{'@pre'.$next})) {__gobble_macro_parms($tex,$substat,$state);} + __count_macrocount($tex,$next,$state); + _parse_include_file($tex,$state,$def); + push @macro,'<filespec>'; + if (defined ($substat=$TeXmacro{'@post'.$next})) {__gobble_macro_parms($tex,$substat,$state);} + } elsif (($state==$STATE_FLOAT) && ($substat=$TeXfloatinc{$next})) { # text included from float - set_style($tex,'command'); - __gobble_macro_parms($tex,$substat); - } elsif ($status==$STATUS_PREAMBLE && defined ($substat=$TeXpreamble{$next})) { - # parse preamble include macros - set_style($tex,'command'); - if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);} - __gobble_macro_parms($tex,$substat,1); - } elsif (status_is_exclude($status)) { + set_style($tex,'cmd'); + push @macro,__gobble_macro_parms($tex,$substat,$__STATE_NULL); + } elsif ($state==$STATE_PREAMBLE && defined ($substat=$TeXpreamble{$next})) { + # parse preamble include macros + set_style($tex,'cmd'); + __count_macrocount($tex,$next,$STATE_TEXT); + push @macro,__gobble_macro_parms($tex,$substat,$__STATE_NULL); + } elsif (state_is_exclude($state)) { # ignore - __gobble_option($tex); + push @macro,__gobble_options($tex),'/*ignored*/'; } elsif ($next eq '\(') { # math inline - _parse_math($tex,$status,$CNT_COUNT_INLINEMATH,'\)'); + _parse_math($tex,$state,$CNT_COUNT_INLINEMATH,'\)'); } elsif ($next eq '\[') { # math display - _parse_math($tex,$status,$CNT_COUNT_DISPLAYMATH,'\]'); - } elsif ($next eq '\def') { + _parse_math($tex,$state,$CNT_COUNT_DISPLAYMATH,'\]'); + } elsif ($next=~/^\\(def|edef|gdef|xdef)$/) { # ignore \def... $tex->{'line'} =~ s/^([^\{]*)\{/\{/; flush_next($tex); print_style($1,'ignore'); - _parse_unit($tex,$STATUS_EXCLUDE_STRONG); - } elsif (defined (my $addsuffix=$TeXfileinclude{$next})) { - # include file: queue up for parsing - _parse_include_file($tex,$status,$addsuffix); + _parse_unit($tex,$STATE_EXCLUDE_STRONG); + push @macro,'<macro>',$STRING_PARAMETER; } elsif (defined ($substat=$TeXmacro{$next})) { # macro: exclude options - if (defined $TeXheader{$next}) {inc_count($tex,$CNT_COUNT_HEADER);} - __count_macroword($tex,$next,$status); - __gobble_macro_parms($tex,$substat,$status); - } elsif (defined __count_macroword($tex,$next,$status)) { + __count_macrocount($tex,$next,$state); + push @macro,__gobble_macro_parms($tex,$substat,$state); + } elsif (defined __count_macrocount($tex,$next,$state)) { # count macro as word (or a given number of words) - set_style($tex,'word'.$status); - } elsif ($next =~ /^\\[^\w\_]/) { + } elsif ($next =~ /^\\[^\w\_]$/) { + # handle \<symbol> as single symbol macro + push @macro,__gobble_options($tex),'/*symbol*/'; } else { - __gobble_option($tex); + push @macro,__gobble_options($tex),'/*defaultrule*/'; } + $MacroUsage{join('',@macro)}++; } # Parse TC instruction @@ -1469,190 +2167,196 @@ sub _parse_tc { my ($tex,$next)=@_; set_style($tex,'tc'); flush_next($tex); - $next=~s/^\%+TC:\s*(\w+)\s*//i || BLOCK { - error($tex,'TC command should have format %TC:instruction [macro] [parameters]'); - return; - }; + assert($next=~s/^\%+TC:\s*(\w+)\s*//i + ,$tex,'TC command should have format %TC:instruction [[parameters]]') + || return; my $instr=$1; $instr=~tr/[A-Z]/[a-z]/; - if ($instr=~/^(break)$/) { - if ($instr eq 'break') {next_subcount($tex,$next);} + if ($instr eq 'break') {next_subcount($tex,$next); } elsif ($instr=~/^(incbib|includebibliography)$/) { $includeBibliography=1; apply_include_bibliography(); - } elsif ($instr eq 'ignore') { - __gobble_tc_ignore($tex); - } elsif ($instr eq 'endignore') { - error($tex,'TC:endignore without corresponding TC:ignore.'); + } elsif ($instr eq 'ignore') {__gobble_tc_ignore($tex); + } elsif ($instr eq 'endignore') {error($tex,'%TC:endignore without corresponding %TC:ignore.'); } elsif ($instr eq 'newtemplate') {$outputtemplate=''; } elsif ($instr eq 'template') {$outputtemplate.=$next; + } elsif ($instr eq 'insert') { + $tex->{'line'}="\n".$next.$tex->{'line'}; } elsif ($instr eq 'subst') { - if ($next=~/^(\\\S+)\s+/) { + if ($next=~/^(\\\S+)\s+(.*)$/) { my $from=$1; - my $to=$'; + my $to=$2; $substitutions{$from}=$to; apply_substitution_rule($tex,$from,$to); } else { - error($tex,'Invalid TC:subst format.'); + error($tex,'Invalid %TC:subst format.'); } - } elsif ($next=~/^([\\]*\S+)\s+([^\s\n]+)(\s+(-?[0-9]+))?/) { - # Format = TC:word macro + } elsif ($instr eq 'newcounter') { + assert($next=~s/^(\w+)(=(\w+))?\s*//,$tex,'Should have format %TC:newcounter {key}[={like-key}] {description}') + || return; + my $key=$1; + my $like=$3; + if ($next eq '') {$next=$key;} + add_new_counter($key,$next,$like); + } elsif ($next=~/^([\\]*\S+)\s+([^\s]+)(\s+(-?\w+))?/) { + # %TC:instr macro param option my $macro=$1; my $param=$2; my $option=$4; - if (tc_macro_param_option($instr,$macro,$param,$option)) {} - else {error($tex,'Unknown TC command: '.$instr);} + tc_macro_param_option($tex,$instr,$macro,$param,$option); } else { - error($tex,'Invalid TC command format: '.$instr); + error($tex,'Invalid TC command: '.$instr); } } -# Parse through ignored LaTeX code -sub __gobble_tc_ignore { - my ($tex)=@_; - set_style($tex,'ignore'); - _parse_unit($tex,$STATUS_EXCLUDE_ALL,'%TC:endignore'); - set_style($tex,'tc'); - flush_next($tex); -} - # Parse math formulae sub _parse_math { - my ($tex,$status,$cnt,$end)=@_; + my ($tex,$state,$cnt,$end)=@_; my $localstyle; - if (status_is_text($status)) { + if (state_is_text($state)) { $localstyle='mathgroup'; inc_count($tex,$cnt); } else { $localstyle='exclmath'; } set_style($tex,$localstyle); - _parse_unit($tex,$STATUS_IGNORE,$end); + _parse_unit($tex,$STATE_MATH,$end); set_style($tex,$localstyle); } # Parse \verb region sub _parse_verb_region { - my ($tex,$status)=@_; - flush_next($tex); + my ($tex,$state)=@_; + #flush_next($tex); + __gobble_macro_modifier($tex); set_style($tex,'ignore'); - if (!($tex->{'line'} =~ s/^[^\s]// )) { - error($tex,'Invalid \verb: delimiter required.'); - } - my $dlm=$&; - print_style($dlm,'command'); - if (!($tex->{'line'} =~ s/^([^$dlm]*)($dlm)// )) { - error($tex,'Invalid \verb: could not find ending delimiter ('.$dlm.').'); - } + assert($tex->{'line'} =~ s/^([^\s])//,$tex,'Invalid \verb: delimiter required.') + || return; + my $dlm=$1; + print_style($dlm,'cmd'); + assert($tex->{'line'}=~s/^([^\Q$dlm\E]*)(\Q$dlm\E)// + ,$tex,'Invalid \verb: could not find ending delimiter ('.$dlm.').') + || return; print_style($1,'ignore'); - print_style($2,'command'); + print_style($2,'cmd'); } -# Parse begin-end group -sub _parse_begin_end { - my ($tex,$status)=@_; - my $localstyle=status_is_text($status) ? 'grouping' : 'exclgroup'; - flush_next_gobble_space($tex,$localstyle,$status); - #__gobble_option($tex); # no option before group name - my ($groupname,$next); +# Parse environment +sub _parse_envir { + my ($tex,$state)=@_; + my $localstyle=state_is_text($state) ? 'envir' : 'exclenv'; + flush_next_gobble_space($tex,$localstyle,$state); + my ($envirname,$next); if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) { - # gobble group type - $groupname=$1; + # gobble environment name + $envirname=$1; + my @macro=('<envir:'.$envirname.'>'); print_style('{'.$1.'}',$localstyle); - $next='begin'.$groupname; - if (defined (my $substat=$TeXmacro{$next})) { - __gobble_macro_parms($tex,$substat); + $next=$PREFIX_ENVIR.$envirname; + __count_macrocount($tex,$next,$state); + if (defined (my $def=$TeXmacro{$next})) { + push @macro,__gobble_macro_parms($tex,$def,$__STATE_NULL); + } else { + push @macro,__gobble_options($tex); } + $MacroUsage{join('',@macro)}++; } else { - $groupname='???'; $next='???'; + $envirname='???'; $next='???'; error($tex,'Encountered \begin without environment name provided.'); } - # find group status (or leave unchanged) - my $substat=$TeXgroup{$1}; + # find new parsing state (or leave unchanged) + my $substat=$TeXenvir{$1}; if (!defined $substat) { - $substat=$status; + $substat=$state; if ($strictness>=1) { - warning($tex,"Using default rule for group ".$groupname); + warning($tex,'Using default rule for environment '.$envirname); } - } elsif (!status_is_text($status)) { - # Do not raise status - $substat=__new_status($substat,$status); - } elsif ($substat==$STATUS_FLOAT) { - # Count float - inc_count($tex,$CNT_COUNT_FLOAT); - } elsif (my $cnt=status_to_cnt($substat)) { - # Count as given type (was: if $substat>3) - inc_count($tex,$cnt); - $substat=0; } else { - # Use $substat value as given - } - if ($includeBibliography && $groupname eq 'thebibliography' && $substat>0) { - # Add bibliography header - inc_count($tex,$CNT_COUNT_HEADER); - __count_macroword($tex,$groupname,$STATUS_TEXT_HEADER); + $substat=__new_state($substat,$state); } - if (!status_inc_envir($substat)) { - # Keep parsing until appropriate end group arrives - while (!$tex->{'eof'}) { - _parse_unit($tex,$substat,'\end'); - if ($tex->{'line'} =~ s/^\s*\{$groupname\}[ \t\r\f]*//) { - # gobble end group parameter - flush_next($tex,$localstyle,$status); - print_style($&,$localstyle); - return; - } - } - } else { + if (state_inc_envir($substat)) { # Parse until \end arrives, and check that it matches _parse_unit($tex,$substat,'\end'); - flush_next_gobble_space($tex,$localstyle,$status); + flush_next_gobble_space($tex,$localstyle,$state); if ($tex->{'line'} =~ s/^\{([^\{\}\s]+)\}[ \t\r\f]*//) { - # gobble group type + # gobble environment name print_style('{'.$1.'}',$localstyle); - if ($groupname ne $1) { - error($tex,'Group \begin{'.$groupname.'} ended with end{'.$1.'}.'); - } + assert($envirname eq $1,$tex,"Environment \\begin{$envirname} ended with \\end{$1}."); } else { - error($tex,'Group ended while waiting for \end{'.$groupname.'}.'); + error($tex,"Environment ended while waiting for \end{$envirname}."); + } + } else { + # Keep parsing until appropriate \end arrives ignoring all else + while (!$tex->{'eof'}) { + _parse_unit($tex,$substat,'\end'); + if ($tex->{'line'} =~ s/^(\s*)(\{\Q$envirname\E\})[ \t\r\f]*//) { + # gobble \end parameter and end environment + flush_next($tex,$localstyle,$state); + print_style($1,$localstyle); + print_style($2,$localstyle); + return; + } } } } # Parse and process file inclusion sub _parse_include_file { - my ($tex,$status,$addsuffix)=@_; - if ($addsuffix eq 'bbl') { - _parse_include_bbl($tex,$status); - return; - } + my ($tex,$state,$includetype)=@_; + my $include=state_is_parsed($state); + my $style=$include?'fileinc':'ignore'; + my $file; + my %params; flush_next($tex); - $tex->{'line'} =~ s/^\s*\{([^\{\}\s]+)\}// - || $tex->{'line'} =~ s/^\s*([^\{\}\%\\\s]+)// - || $tex->{'line'} =~ s/^\s*\{(.+?)\}// - || BLOCK { - error($tex,'Failed to read or interpret file name for inclusion.'); + $params{'<type>'}=$includetype; + if ($includetype eq '<bbl>') { + _parse_include_bbl($tex,$state,\%params); return; - }; - if (status_is_text($status)) { - print_style($&,'fileinclude'); - my $fname=$1; - if ($addsuffix==2) {$fname.='.tex';} - elsif ($addsuffix==1 && ($fname=~/^[^\.]+$/)) {$fname.='.tex';} - if ($includeTeX) {include_file($tex,$fname);} - } else { - print_style($&,'ignored'); + } + elsif ($includetype eq 'input') { + $tex->{'line'} =~ s/^(\s*\{([^\{\}\s]+)\})// + || $tex->{'line'} =~ s/^(\s*([^\{\}\%\\\s]+))// + || $tex->{'line'} =~ s/^(\s*\{(.+?)\})// + || BLOCK { + error($tex,'Failed to identify file name.'); + return; + }; + print_style($1,$style); + $file=$2; + } + else { + foreach my $param (split(/\s+|,/,$includetype)) { + if ($param=~/^<.*>$/) {next;} + $tex->{'line'} =~ s/(\s*\{)([^\{\}\%]*)(\})//|| BLOCK { + error($tex,"Failed to identify file parameter {$param}."); + return; + }; + print_style($1,'ignore'); + print_style($2,$style); + print_style($3,'ignore'); + if ($param eq 'file') {$file=$2;} + elsif ($param eq 'texfile') { + $file=$2; + if (!$file=~/\.tex$/i) {$file.='.tex';} + } + else {$params{$param}=$2;} + } + } + if (!defined $file) { + error($tex,'Undefined file name.'); + } + elsif ($includeTeX && $include) { + include_file($tex,$state,$file,\%params); } } # Parse and process bibliography file sub _parse_include_bbl { - my ($tex,$status)=@_; - __gobble_macro_parms($tex,1,$status); - if (status_is_text($status) && $includeBibliography) { - my $fname=$tex->{'filename'}; - $fname=~s/\.\w+$/\.bbl/; - include_file($tex,$fname); + my ($tex,$state,$params)=@_; + if ($includeBibliography && state_is_text($state)) { + my $filename=$tex->{'filename'}; + $filename=~s/(\.tex)?$/\.bbl/i; + include_file($tex,$state,$filename,$params); } } @@ -1661,26 +2365,53 @@ sub _parse_include_package { my ($tex)=@_; set_style($tex,'document'); __gobble_option($tex); - if ( $tex->{'line'}=~s/^\{((\w+)(\s*,\s*\w+)*)\}// ) { - print_style('{'.$1.'}','document'); + if ( $tex->{'line'}=~s/^\{(([\w\-]+)(\s*,\s*[\w\-]+)*)\}// ) { + print_style("{$1}",'document'); foreach (split(/\s*,\s*/,$1)) { - include_package($_); + $MacroUsage{"<package:$_>"}++; + include_package($_,$tex); } } else { - _parse_unit($tex,$STATUS_IGNORE); - error($tex,"Could not recognise package list, ignoring it instead."); + _parse_unit($tex,$STATE_IGNORE); + error($tex,'Could not recognise package list, ignoring it instead.'); } - __gobble_options($tex); } -# Count macroword using given status -sub __count_macroword { - my ($tex,$next,$status)=@_; - my $n=$TeXmacroword{$next}; - if (defined $n && (my $cnt=status_text_cnt($status))) { - inc_count($tex,$cnt,$n); +# Parse \documentclass parameters and include rules +sub _parse_documentclass_params { + my ($tex)=@_; + my $options=__gobble_option($tex); + if ( $tex->{'line'}=~s/^(\{\s*([^\{\}\s]+)\s*\})// ) { + print_style("$1",'document'); + $MacroUsage{"<documentclass:$2>"}++; + include_package("class%$2",$tex); + } else { + _parse_unit($tex,$STATE_IGNORE); + error($tex,'Could not identify document class.'); } - return $n; +} + +# Count macrocount using given state +sub __count_macrocount { + my ($tex,$next,$state)=@_; + my $def=$TeXmacrocount{$next}; + my $cnt; + if (!defined $def) {return undef;} + elsif (ref($def) eq 'ARRAY') { + # TODO: Is this an appropriate style to use? + set_style($tex,state_to_style($state)); + flush_next($tex); + foreach $cnt (@{$def}) { + if ($cnt==$CNT_WORDS_TEXT) {$cnt=state_text_cnt($state);} + inc_count($tex,$cnt); + } + } + elsif ($cnt=state_text_cnt($state)) { + set_style($tex,state_to_style($state)); + flush_next($tex); + inc_count($tex,$cnt,$def); + } + return $def; } # Gobble next option, return option or undef if none @@ -1694,64 +2425,104 @@ sub __gobble_option { return undef; } -# Gobble all options +# Gobble all options, return the number of gobble options sub __gobble_options { - while (__gobble_option(@_)) {} + my $n=0; + while (__gobble_option(@_)) {$n++} + return ($STRING_GOBBLED_OPTION)x$n; } # Gobble macro modifyer (e.g. following *) sub __gobble_macro_modifier { my $tex=shift @_; flush_next($tex); - if ($tex->{'line'} =~ s/^\*//) { + if ($tex->{'line'} =~ s/^(\*)//) { print_style($1,'option'); return $1; } - return undef; + return; } # Gobble macro parameters as specified in parm plus options sub __gobble_macro_parms { my ($tex,$parm,$oldstat)=@_; my $n; + my @ret; if (ref($parm) eq 'ARRAY') { $n=scalar @{$parm}; } else { $n=$parm; - $parm=[($STATUS_IGNORE)x$n]; + $parm=[($STATE_IGNORE)x$n]; } - if ($n>0) {__gobble_macro_modifier($tex);} - __gobble_options($tex); + # TODO: Optionally gobble macro modifier? + if ($n>0) {push @ret,__gobble_macro_modifier($tex);} + my $auto_gobble_options=1; for (my $j=0;$j<$n;$j++) { - _parse_unit($tex,__new_status($parm->[$j],$oldstat)); - __gobble_options($tex); + my $p=$parm->[$j]; + if ($p==$_STATE_OPTION) { + # Parse macro option + $p=$parm->[++$j]; + if ($tex->{'line'}=~s/^(\s*\[)//) { + flush_next_gobble_space($tex); + print_style($1,'optparm'); + push @ret,$STRING_OPTIONAL_PARAM; + _parse_unit($tex,__new_state($p,$oldstat),']'); + set_style($tex,'optparm'); + } + } elsif ($p==$_STATE_NOOPTION) { + $auto_gobble_options=0; + } elsif ($p==$_STATE_AUTOOPTION) { + $auto_gobble_options=1; + } else { + # Parse macro parameter + if ($auto_gobble_options) {push @ret,__gobble_options($tex);} + push @ret,$STRING_PARAMETER; + _parse_unit($tex,__new_state($p,$oldstat),$_PARAM_); + } } + #TODO: Drop default gobbling of option at end? + if ($auto_gobble_options) {push @ret,__gobble_options($tex);} + return @ret; } -# Return new parsing status given old and substatus -sub __new_status { +# Parse through ignored LaTeX code +sub __gobble_tc_ignore { + my ($tex)=@_; + set_style($tex,'ignore'); + _parse_unit($tex,$STATE_EXCLUDE_ALL,'%TC:endignore'); + set_style($tex,'tc'); + flush_next($tex); +} + +# Return new parsing state given old and substate +sub __new_state { my ($substat,$oldstat)=@_; if (!defined $oldstat) {return $substat;} - foreach my $st (@STATUS_PRIORITY_LIST) { + foreach my $st (@STATE_FIRST_PRIORITY,@STATE_MID_PRIORITY,@STATE_LAST_PRIORITY) { if ($oldstat==$st || $substat==$st) {return $st;} } - error($Main,'Could not determine new status in __new_status!','BUG'); + error($Main,'Could not determine new state in __new_state!','BUG'); return $oldstat; } + +###### Tokenisation routines + + # Get next token skipping comments and flushing output buffer -sub _next_token { - my $tex=shift @_; +sub next_token { + my ($tex,$simple_token)=@_; + if (!defined $simple_token) {$simple_token=0;} my ($next,$type); my $style=$tex->{'style'}; if (defined $tex->{'next'}) {print_style($tex->{'next'},$tex->{'style'});} $tex->{'style'}=undef; - while (defined ($next=__get_next_token($tex))) { + while (defined ($next=_get_next_token($tex,$simple_token))) { $type=$tex->{'type'}; if ($type==$TOKEN_COMMENT) { print_style($next,'comment'); } elsif ($type==$TOKEN_LINEBREAK) { - if ($verbose>0) {line_return(-1,$tex);} + if ($printlevel>0) {line_return(-1,$tex);} } else { return $next; } @@ -1760,20 +2531,9 @@ sub _next_token { } # Read, interpret and return next token -sub __get_next_token { - # Token (or token group) category: - # -1: space - # 0: comment - # 1: word (or other forms of text or text components) - # 2: symbol (not word, e.g. punctuation) - # 3: macro - # 4: curly braces {} - # 5: brackets [] - # 6: maths - # 9: line break in file - # 999: end of line or blank line - # 666: TeXcount instruction (%TC:instruction) - my $tex=shift @_; +# If simple_token is set, words cannot form tokens +sub _get_next_token { + my ($tex,$simple_token)=@_; my $next; my $ch; while (!$tex->{'eof'}) { @@ -1787,8 +2547,18 @@ sub __get_next_token { } elsif ($ch eq "\n" || $ch eq "\r") { $tex->{'line'}=~s/^(\r\n?|\n)//; return __set_token($tex,$1,$TOKEN_LINEBREAK); - } elsif ($tex->{'line'}=~s/^($WordPattern)//) { + } elsif (!$simple_token && $tex->{'line'}=~s/^($WordPattern)//o) { return __set_token($tex,$1,$TOKEN_WORD); + } elsif ($simple_token && $tex->{'line'}=~s/^(\w)//) { + # When parsing simple tokens (not words), only include single characters + # TODO: the handling of simple tokens should be improved + my $match=$1; + if ($match=~/^$WordPattern$/) {return __set_token($tex,$match,$TOKEN_WORD);} + else {return __set_token($tex,$match,$TOKEN_SYMBOL);} + } elsif ($ch eq '\\') { + if ($tex->{'line'}=~s/^(\\[{}%])//) {return __set_token($tex,$1,$TOKEN_SYMBOL);} + if ($tex->{'line'}=~s/^(\\([a-zA-Z@]+|[^a-zA-Z@[:^print:]]))//) {return __set_token($tex,$1,$TOKEN_MACRO);} + return __get_chtoken($tex,$ch,$TOKEN_END); } elsif ($ch eq '$') { $tex->{'line'}=~s/^(\$\$?)//; return __set_token($tex,$1,$TOKEN_MATH); @@ -1806,10 +2576,6 @@ sub __get_next_token { if ($tex->{'line'}=~s/^(\%+[tT][cC]:[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_TC);} if ($tex->{'line'}=~s/^(\%+[^\r\n]*)//) {return __set_token($tex,$1,$TOKEN_COMMENT);} return __get_chtoken($tex,$ch,$TOKEN_COMMENT); - } elsif ($ch eq '\\') { - if ($tex->{'line'}=~s/^(\\[{}%])//) {return __set_token($tex,$1,$TOKEN_SYMBOL);} - if ($tex->{'line'}=~s/^(\\([a-zA-Z_]+|[^a-zA-Z_]))//) {return __set_token($tex,$1,$TOKEN_MACRO);} - return __get_chtoken($tex,$ch,$TOKEN_END); } else { return __get_chtoken($tex,$ch,$TOKEN_END); } @@ -1840,30 +2606,26 @@ sub __get_chtoken { ## Make new count object -# The "count object" is a hash containing +# The "count object" is a hash containing: # - title: the title of the count (name of file, section, ...) # - counts: a list of numbers (the counts: files, text words, ...) -# upon creation, but where the element -# - subcounts: list of count objects (added by the TeX object) -# may exist if the count contains subcounts. The elements of the -# count are (by their index): -# 0 = #files: counts the number of files -# 1 = text words: counts the number of words in the text -# 2 = header words: number of words in headers -# 3 = caption words: number of words in float captions -# 4 = #headers: number of headers -# 5 = #floats: number of tables, figures, floats, etc. -# 6 = #inline formulae: number of math elements in text ($...$) -# 7 = #displayed formulae: number of displayed equations +# - subcounts: list of count objects (used by the TeX object) +# - warnings: list of warnings produced +# The elements are specified by the $CONT_* constants. sub new_count { my ($title)=@_; my @cnt=(0) x $SIZE_CNT; - my %count=('counts'=>\@cnt,'title'=>$title); - # files, text words, header words, float words, - # headers, floats, math-inline, math-display; + my %count=('counts'=>\@cnt,'title'=>$title,'subcounts'=>[],'errorcount'=>0,'warnings'=>{}); return \%count; } +# Ensure the count array is of the right size +sub update_count_size { + my $count=shift @_; + my $counts=$count->{'counts'}; + while (scalar @{$counts}<$SIZE_CNT) {push @$counts,0;} +} + # Increment TeX count for a given count type sub inc_count { my ($tex,$cnt,$value)=@_; @@ -1875,15 +2637,17 @@ sub inc_count { # Get count value for a given count type sub get_count { my ($count,$cnt)=@_; - return ${$count->{'counts'}}[$cnt]; + my $counts=$count->{'counts'}; + if ($cnt<scalar @{$counts}) {return ${$counts}[$cnt];} + else {return 0;} } # Compute sum count for a count object sub get_sum_count { my $count=shift @_; my $sum=0; - for (my $i=scalar(@sumweights);$i-->0;) { - $sum+=get_count($count,$i+1)*$sumweights[$i]; + for (my $i=scalar(@sumweights);$i-->1;) { + $sum+=get_count($count,$i)*$sumweights[$i]; } return $sum; } @@ -1898,6 +2662,20 @@ sub number_of_subcounts { } } +# Returns the number of warnings (not distinct) +sub number_of_warnings { + my $count=shift @_; + my $n=0; + foreach my $m (values %{$count->{'warnings'}}) {$n+=$m;} + return $n; +} + +# Returns the number of distinct warnings +sub number_of_distinct_warnings { + my $count=shift @_; + return scalar keys %{$count->{'warnings'}}; +} + # Is a null count? (counts 1-7 zero, title starts with _) sub _count_is_null { my $count=shift @_; @@ -1905,15 +2683,23 @@ sub _count_is_null { for (my $i=1;$i<$SIZE_CNT;$i++) { if (get_count($count,$i)>0) {return 0;} } + if (scalar keys %{$count->{'warnings'}}) {return 0;} + if ($count->{'errorcount'}) {return 0;} return 1; } # Add one count to another sub _add_to_count { my ($a,$b)=@_; + update_count_size($a); + update_count_size($b); for (my $i=0;$i<$SIZE_CNT;$i++) { ${$a->{'counts'}}[$i]+=${$b->{'counts'}}[$i]; } + while (my ($text,$n)=each %{$a->{'warnings'}}) { + $a->{'warnings'}->{$text}+=$n; + } + $a->{'errorcount'}+=$b->{'errorcount'}; } # Add subcount to sum count and prepare new subcount @@ -1928,39 +2714,38 @@ sub next_subcount { sub add_to_total { my ($total,$count)=@_; _add_to_count($total,$count); - if (!_count_is_null($count)) { - push @{$total->{'subcounts'}},$count; - $count->{'parentcount'}=$total; - } + if (_count_is_null($count)) {return;} + push @{$total->{'subcounts'}},$count; + $count->{'parentcount'}=$total; } - ###### Result output routines -# Close the output, e.g. adding HTML tail +# Close the output (STDOUT), e.g. adding HTML tail sub Close_Output { if ($htmlstyle>1) { html_tail(); } + close STDOUT; } # Report if there were any errors occurring during parsing sub Report_Errors { if (defined $outputtemplate) {return;} - if ( !$briefsum && !$totalflag && $verbose>=0 ) { - foreach (keys %warnings) {formatprint($_,"p","nb");print "\n";} + if ( !$briefsum && !$totalflag && $printlevel>=0 ) { + foreach (keys %warnings) {formatprint($_,'p','nb');print "\n";} } if ($errorcount==0) {return;} - if ($briefsum && $totalflag) {print " ";} + if ($briefsum && $totalflag) {print ' ';} if ($htmlstyle) { print "<div class='error'><p>\n"; - print "There were ".$errorcount." error(s) reported!\n"; + print "There were $errorcount error(s) reported!\n"; print "</p></div>\n"; } elsif ($briefsum && $totalflag) { - print "(errors:".$errorcount.")"; + print "(errors:$errorcount)"; } else { - print "(errors:".$errorcount.")\n"; + print "(errors:$errorcount)\n"; } } @@ -1979,14 +2764,14 @@ sub print_word_freq { $Word{$wd}=__lc_merge($word,$Word{$wd}); } if ($htmlstyle) {print "<table class='stat'>\n<thead>\n";} - __print_word_freq("Word","Freq","th"); + __print_word_freq('Word','Freq','th'); if ($htmlstyle) {print "</thead>\n";} if ($optionWordClassFreq>0) { for $word (keys %Freq) {$Class{__word_class($word,\%regs)}+=$Freq{$word};} __print_sorted_freqs('langstat',\%Class); } if ($htmlstyle) {print "<tbody class='sumstat'>\n";} - __print_word_freq("All words",$sum,,'td','sum'); + __print_word_freq('All words',$sum,'td','sum'); if ($htmlstyle) {print "</tbody>\n";} if ($optionWordFreq>0) {__print_sorted_freqs('wordstat',\%Freq,\%Word,$optionWordFreq);} if ($htmlstyle) {print "</table>\n";} @@ -2026,7 +2811,7 @@ sub __print_sorted_freqs { my $sum=0; my ($word,$wd,$freq); if (!defined $min) {$min=0;} - if ($htmlstyle) {print "<tbody class='",$class,"'>\n";} + if ($htmlstyle) {print "<tbody class='$class'>\n";} else {print "---\n";} for $wd (sort {$Freq->{$b} <=> $Freq->{$a}} keys %{$Freq}) { if (defined $Word) {$word=$Word->{$wd};} else {$word=$wd;} @@ -2036,7 +2821,7 @@ sub __print_sorted_freqs { __print_word_freq($word,$freq); } } - if ($min>0) {__print_word_freq("Sum of subset",$sum,'td','sum');} + if ($min>0) {__print_word_freq('Sum of subset',$sum,'td','sum');} if ($htmlstyle) {print "</tbody>\n";} } @@ -2044,27 +2829,45 @@ sub __print_sorted_freqs { sub __print_word_freq { my ($word,$freq,$tag,$class)=@_; if (!defined $tag) {$tag='td';} - if (defined $class) {$class=' class=\''.$class.'\'';} else {$class='';} + if (defined $class) {$class=" class='$class'";} else {$class='';} if ($htmlstyle) { - print "<tr",$class,"><",$tag,">",$word,"</",$tag,"><",$tag,">",$freq,"</",$tag,"></tr>\n"; + print "<tr$class><$tag>$word</$tag><$tag>$freq</$tag></tr>\n"; } else { - print $word,": ",$freq,"\n"; + print $word,': ',$freq,"\n"; + } +} + +# Print macro usage statistics +sub print_macro_stat { + if ($htmlstyle) {print "<table class='stat'>\n<thead>\n";} + __print_word_freq('Macro/envir','Freq','th'); + if ($htmlstyle) {print "</thead>\n";} + if ($htmlstyle) {print "<tbody class='macrostat'>\n";} + foreach my $name (sort keys %MacroUsage) { + my $freq=$MacroUsage{$name}; + $name=text_to_printable($name); + $name=text_to_print($name); + if ($htmlstyle) { + print "<tr><td>$name</td><td>$freq</td></tr>\n"; + } else { + print $name,': ',$freq,"\n"; + } } + if ($htmlstyle) {print "</tbody>\n";} + if ($htmlstyle) {print "</table>\n";} } + ###### Printing routines # Print text using given style/colour sub print_with_style { my ($text,$style,$colour)=@_; - if ($style eq ' ') { - print text_to_print($text); - } elsif ($style eq '') { + if ($style eq $NOSTYLE) { print text_to_print($text); - error($Main,'Empty style should not occur in print_with_style!','BUG'); } elsif ($htmlstyle) { - print "<span class='$style'>".text_to_print($text).'</span>'; + print "<span class='$style'>",text_to_print($text),'</span>'; } else { ansiprint(text_to_print($text),$colour); if ($style=~/$separatorstyleregex/) {print $separator;} @@ -2079,18 +2882,35 @@ sub text_to_print { $text=~s/</</g; $text=~s/>/>/g; $text=~s/[ \t]{2}/\ /g; + } elsif ($texcodeoutput) { + $text=~s/\\/\\textbackslash¤/g; + $text=~s/([%{}])/\\$1/g; + $text=~s/¤/{}/g; } return $text; } +# Return &#xxx; code for character ; +sub convert_to_charcode { + my $ch=shift @_; + return '&#'.ord($ch).';'; +} + +# Convert special characters to charcodes to make it printable ; +sub text_to_printable { + my $text=shift @_; + $text=~s/([^[:print:]])/convert_to_charcode($1)/gse; + return $text; +} + # Print text, using appropriate tags for HTML sub formatprint { my ($text,$tag,$class)=@_; my $break=($text=~s/(\r\n?|\n)$//); if ($htmlstyle && defined $tag) { print '<'.$tag; - if ($class) {print " class='".$class."'";} - print '>'.text_to_print($text).'</'.$tag.'>'; + if ($class) {print " class='$class'";} + print '>'.text_to_print($text)."</$tag>"; } else { print text_to_print($text); } @@ -2130,7 +2950,7 @@ sub __count_and_header { return $count,$header,@_; } -# Return count title or "" if missing +# Return count title or '' if missing sub __count_header { my $count=shift @_; return $count->{'title'}||''; @@ -2150,16 +2970,19 @@ sub _print_sum_count { sub _print_count_brief { my ($count,$header,$tag1,$tag2)=__count_and_header(@_); my @cnt=@{$count->{'counts'}}; - if ($htmlstyle && $tag1) {print "<".$tag1.">";} - print $cnt[$CNT_WORDS_TEXT]."+".$cnt[$CNT_WORDS_HEADER]."+".$cnt[$CNT_WORDS_CAPTION]. - " (".$cnt[$CNT_COUNT_HEADER]."/".$cnt[$CNT_COUNT_FLOAT]. - "/".$cnt[$CNT_COUNT_INLINEMATH]."/".$cnt[$CNT_COUNT_DISPLAYMATH].")"; + if ($htmlstyle && $tag1) {print '<',$tag1,'>';} + print $cnt[$CNT_WORDS_TEXT],'+',$cnt[$CNT_WORDS_HEADER],'+',$cnt[$CNT_WORDS_OTHER]; + for (my $i=$SIZE_CNT_DEFAULT;$i<$SIZE_CNT;$i++) { + if ($cnt[$i]) {print '+',$cnt[$i],$countkey[$i];} + } + print ' (',$cnt[$CNT_COUNT_HEADER],'/',$cnt[$CNT_COUNT_FLOAT], + '/',$cnt[$CNT_COUNT_INLINEMATH],'/',$cnt[$CNT_COUNT_DISPLAYMATH],')'; if ($htmlstyle && $tag2) { - print "</".$tag1."><".$tag2.">"; + print '</',$tag1,'><',$tag2,'>'; $tag1=$tag2; - } else {print " ";} + } else {print ' ';} print text_to_print($header); - if ($htmlstyle && $tag1) {print "</".$tag1.">";} + if ($htmlstyle && $tag1) {print '</',$tag1,'>';} if ($finalLineBreak) {print "\n";} } @@ -2173,10 +2996,10 @@ sub _print_count_details { } if (@sumweights) {formatprint('Sum count: '.get_sum_count($count)."\n",'li');} for (my $i=1;$i<$SIZE_CNT;$i++) { - formatprint($countlabel[$i].': '.get_count($count,$i)."\n",'li'); + formatprint($countdesc[$i].': '.get_count($count,$i)."\n",'li'); } if (get_count($count,$CNT_FILE)>1) { - formatprint($countlabel[$CNT_FILE].': '.get_count($count,$CNT_FILE)."\n",'li'); + formatprint($countdesc[$CNT_FILE].': '.get_count($count,$CNT_FILE)."\n",'li'); } my $subcounts=$count->{'subcounts'}; if ($showsubcounts && defined $subcounts && scalar(@{$subcounts})>=$showsubcounts) { @@ -2198,14 +3021,14 @@ sub _print_count_template { $template=~s/\\n/\n/g; if ($htmlstyle) {$template=~s/\n/<br>/g;} my ($subtemplate,$posttemplate); - while ($template=~/\{SUB\?((.*?)\|)?(.*?)(\|(.*?))?\?SUB\}/is) { - __print_count_using_template($count,$`); + while ($template=~/^(.*)\{SUB\?((.*?)\|)?(.*?)(\|(.*?))?\?SUB\}(.*)$/is) { + __print_count_using_template($count,$1); # $1 ~ ${^PREMATCH} if (number_of_subcounts($count)>1) { - if (defined $2) {print $2;} - __print_subcounts_using_template($count,$3); - if (defined $5) {print $5;} + if (defined $3) {print $3;} + __print_subcounts_using_template($count,$4); + if (defined $6) {print $6;} } - $template=$'; + $template=$7; # $7 ~ ${^POSTMATCH} } __print_count_using_template($count,$template); } @@ -2213,16 +3036,16 @@ sub _print_count_template { # Print counts using template sub __print_count_using_template { my ($count,$template)=@_; - for (my $i=0;$i<$SIZE_CNT;$i++) { - $template=__process_template($template,$i,get_count($count,$i)); - } - $template=~s/\{VER\}/$versionnumber/gi; - # TODO: Should base warnings and errors on TeXcode or Main object - $template=__process_template($template,"W|WARN|WARNING|WARNINGS",length(%warnings)); - $template=__process_template($template,"E|ERR|ERROR|ERRORS",$errorcount); - $template=__process_template($template,"S|SUM",get_sum_count($count)); - $template=__process_template($template,"T|TITLE",$count->{'title'}||""); - $template=__process_template($template,"SUB",number_of_subcounts($count)); + while (my ($key,$cnt)=each %key2cnt) { + $template=__process_template($template,$key,get_count($count,$cnt)); + } + $template=~s/\{VER\}/\Q$versionnumber\E/gi; + $template=__process_template($template,'WARN|WARNING|WARNINGS',number_of_distinct_warnings($count)); + $template=__process_template($template,'NWARN|NWARNING|NWARNINGS',number_of_warnings($count)); + $template=__process_template($template,'ERR|ERROR|ERRORS',$count->{'errorcount'}); + $template=__process_template($template,'SUM',get_sum_count($count)); + $template=__process_template($template,'TITLE',$count->{'title'}||''); + $template=__process_template($template,'SUB',number_of_subcounts($count)); print $template; } @@ -2246,7 +3069,7 @@ sub __process_template { $template=~s/\{($label)\?(.*?)\|(.*?)\?(\1)\}/$3/gis; $template=~s/\{($label)\?(.*?)\?(\1)\}//gis; } - if (!defined $value) {$value="";} + if (!defined $value) {$value='';} $template=~s/\{($label)\}/$value/gis; return $template; } @@ -2259,56 +3082,61 @@ sub __process_template { sub flush_next { my ($tex,$style)=@_; my $ret=undef; - if (defined $style && $style ne '') { - set_style($tex,$style); - } + if (defined $style) {set_style($tex,$style);} if (defined $tex->{'next'}) { $ret=print_style($tex->{'next'},$tex->{'style'},$tex->{'printstate'}); } $tex->{'printstate'}=undef; - $tex->{'style'}='-'; + $tex->{'style'}=$STYLE_BLOCK; return $ret; } -# Print next token and gobble following spaces +# Print next token and gobble following spaces (incl. line break and comments) sub flush_next_gobble_space { - my ($tex,$style,$status)=@_; + my ($tex,$style,$state)=@_; my $ret=flush_next($tex,$style); if (!defined $ret) {$ret=0;} - if (!defined $status) {$status=$STATUS_IGNORE;} - my $prt=($verbose>0); - if ($tex->{'line'}=~s/^([ \t\f]*)(\r\n?|\n)([ \t\f]*)//) { - if (!$prt) { - } elsif ($verbose>2 || $ret) { - print $1; - line_return(-1,$tex); - my $space=$3; - if ($htmlstyle) {$space=~s/ /\ /g;} - print $space; - } else { - line_return(0,$tex); + if (!defined $state) {$state=$STATE_IGNORE;} + my $prt=($printlevel>0); + while (1) { + if ($tex->{'line'}=~s/^([ \t\f]*)(\r\n|\r|\n)([ \t\f]*)//) { + if (!$prt) { + } elsif ($printlevel>1 || $ret) { + print $1; + line_return(-1,$tex); + my $space=$3; + if ($htmlstyle) {$space=~s/ /\ /g;} + print $space; + $ret=0; + } else { + line_return(0,$tex); + } + } elsif ($tex->{'line'}=~s/^([ \t\f]+)//) { + if ($prt) {print $1;} } - } elsif ($tex->{'line'}=~s/^([ \t\f]*)//) { - if ($prt) {print $1;} + if ($tex->{'line'}=~s/^(\%+[^\r\n]*)//) { + print_style($1,'comment'); + $ret=1; + } else {return;} } } # Set presentation style sub set_style { my ($tex,$style)=@_; - if (!(($tex->{'style'}) && ($tex->{'style'} eq '-')) && $style ne '') {$tex->{'style'}=$style;} + if (!(defined $tex->{'style'} && $tex->{'style'} eq $STYLE_BLOCK)) {$tex->{'style'}=$style;} } # Print text using the given style, and log state if given sub print_style { my ($text,$style,$state)=@_; - (($verbose>=0) && (defined $text) && (defined $style)) || return 0; + (($printlevel>=0) && (defined $text) && (defined $style)) || return 0; my $colour; ($colour=$STYLE{$style}) || return; - if (($colour) && !($colour eq '-')) { + if (defined $colour && $colour ne '-') { print_with_style($text,$style,$colour); - if ($state) {print_style($state,'state');} - if ($style ne "cumsum") {$blankline=-1;} + if (defined $state) {print_style($state,'state');} + if ($style ne 'cumsum') {$blankline=-1;} return 1; } else { return 0; @@ -2318,89 +3146,56 @@ sub print_style { # Conditional line return sub line_return { my ($blank,$tex)=@_; - if ($blank<0 && $verbose<3) {$blank=1;} + if ($blank<0 && $printlevel<2) {$blank=1;} if ($blank<0 || $blank>$blankline) { if ((defined $tex) && @sumweights) { my $num=get_sum_count($tex->{'subcount'}); - print_style(" [".$num."]","cumsum"); + print_style(" [$num]",'cumsum'); } linebreak(); $blankline++; } } -# Print error message -sub _print_error { - my $text=shift @_; - line_return(1); - print_style("!!! ".$text." !!!",'error'); - line_return(1); -} - -# Print errors in buffer and delete errorbuffer -sub flush_errorbuffer { - my $source=shift @_; - my $err=$source->{'errorbuffer'} || return; - foreach (@$err) {_print_error($_);} - $source->{'errorbuffer'}=undef; -} - ###### Print help on style/colour codes # Print output style codes if conditions are met -sub conditional_print_help_style { - if ($showcodes) {_print_help_style();} +sub conditional_print_style_list { + if ($showcodes) {_print_style_list();} return $showcodes; } # Print help on output styles -sub _print_help_style { - if ($verbose<=0) {return;} - if ($htmlstyle) {print "<div class='stylehelp'><p>";} - formatprint("Format/colour codes of verbose output:","h2"); +sub _print_style_list { + if ($printlevel<=0) {return;} + if ($htmlstyle) {print '<div class="stylehelp"><p>';} + formatprint('Format/colour codes of verbose output:','h2'); print "\n\n"; - _help_style_line('Text which is counted',"word1","counted as text words"); - _help_style_line('Header and title text',"word2","counted as header words"); - _help_style_line('Caption text and footnotes',"word3","counted as caption words"); - _help_style_line("Ignored text or code","ignore","excluded or ignored"); - _help_style_line('\documentclass',"document","document start, beginning of preamble"); - _help_style_line('\macro',"command","macro not counted, but parameters may be"); - _help_style_line('\macro',"exclcommand","macro in excluded region"); - _help_style_line("[Macro options]","option","not counted"); - _help_style_line('\begin{group} \end{group}',"grouping","begin/end group"); - _help_style_line('\begin{group} \end{group}',"exclgroup","begin/end group in excluded region"); - _help_style_line('$ $',"mathgroup","counted as one equation"); - _help_style_line('$ $',"exclmath","equation in excluded region"); - _help_style_line('% Comments',"comment","not counted"); - _help_style_line('%TC:TeXcount instructions',"tc","not counted"); - _help_style_line("File to include","fileinclude","not counted but file may be counted later"); - if ($showstates) { - _help_style_line('[state]',"state","internal TeXcount state"); - } - if (@sumweights) { - _help_style_line('[cumsum]',"cumsum","cumulative sum count"); - } - _help_style_line("ERROR","error","TeXcount error message"); - if ($htmlstyle) {print "</p></div>";} + foreach my $style (@STYLE_LIST) { + my $desc=$STYLE_DESC{$style}; + if ($desc=~/^(.*):\s+(.*)$/) { + _help_style_line($1,$style,$2); # $1~${^PREMATCH}, $2~${^POSTMATCH} + } else { + _help_style_line($desc,$style); + } + } + if ($htmlstyle) {print '</p></div>';} print "\n\n"; } # Print one line of help sub _help_style_line { my ($text,$style,$comment)=@_; - if ($htmlstyle) { - $comment=" .... ".text_to_print($comment); - } else { - $comment=" .... ".$comment; - } + if (!defined $comment) {$comment='';} + if ($htmlstyle) {$comment=' .... '.text_to_print($comment);} + else {$comment=' .... '.$comment;} if (print_style($text,$style)) { print $comment; linebreak(); } } - ###### Help routines @@ -2419,12 +3214,36 @@ sub print_license { wprintstringdata('License'); } -# Print TeXcount parameter list +# Print short TeXcount help +sub print_short_help { + wprintstringdata('ShortHelp'); +} + +# Print TeXcount options list sub print_syntax { wprintstringdata('OptionsHead'); wprintstringdata('Options','@ - :'); } +# Prinst TeXcount options containing substring +sub print_syntax_subset { + my $pattern=shift @_; + my $data=StringData('Options'); + if (!defined $data) { + error($Main,'No StringData Options.','BUG'); + return; + } + my @options; + foreach (@$data) { + if (/^\s*([^\s]+\s)*[^\s]*\Q$pattern\E/) {push @options,$_;} + } + if (scalar(@options)==0) {print "No options contained $pattern.\n";} + else { + print "Options containing \"$pattern\":\n\n"; + wprintlines('@ - :',@options); + } +} + # Print complete TeXcount help sub print_help { print_help_title(); @@ -2444,13 +3263,13 @@ sub print_help_text { wprintstringdata('TCinstructions'); } -# Print help on specific macro or group -sub print_help_on { +# Print help on specific macro or environment +sub print_help_on_rule { my $arg=shift @_; my $def; my %rules=( '\documentclass' => 'Initiates LaTeX document preamble.', - '\begin' => 'Treatmend depends on group handling rules.', + '\begin' => 'Treatmend depends on environment handling rules.', '\def' => 'Excluded from count.', '\verb' => 'Strong exclude for enclosed region.', '$' => 'Opens or closes inlined equation', @@ -2460,37 +3279,46 @@ sub print_help_on { '\[' => 'Opens displayed equation.', '\]' => 'Closes displayed equation initiated by \[.'); if (!defined $arg || $arg=~/^\s*$/) { - print "Specify macro or group name after the -h= option.\n"; + print "\nSpecify macro or environment name after the -h= option.\n"; return; } + if ($arg=~/^[\w\-\%]+:/) { + remove_all_rules(); + %rules=(); + while ($arg=~s/^([\w\-\%]+)://) {include_package($1);} + } if ($def=$rules{$arg}) { - print "Special rule (hard coded) for $arg\n"; + print "\nSpecial rule (hard coded) for $arg\n"; print $def."\n"; } elsif ($arg=~/^\\/) { + my $hasrule=0; + print "\nRule(s) for macro $arg\n"; + if ($def=$TeXmacrocount{$arg}) { + $hasrule=1; + } if ($def=$TeXfileinclude{$arg}) { - print "Rule for macro $arg\n"; + $hasrule=1; print "Takes file name as parameter which is included in document.\n"; - } elsif ($def=$TeXmacro{$arg}) { - print "Rule for macro $arg\n"; + } + if ($def=$TeXmacro{$arg}) { + $hasrule=1; _print_rule_macro($arg,$def); - if ($def=$TeXheader{$arg}) { - print "This macro is counted as a header\n"; - } - if ($def=$TeXfloatinc{$arg}) { - print "This macro is also counted inside floats as captions.\n"; - } - } elsif ($def=$TeXmacroword{$arg}) { - print "Rule for macro $arg\n"; - print "Count as ".$def." word(s).\n"; - } else { - print "No macro rule defined for $arg.\nParameters treated as surrounding text.\n"; + } + if ($def=$TeXfloatinc{$arg}) { + $hasrule=1; + print "\nIncluded inside floats:\n"; + _print_rule_macro($arg,$def); + } + if (!$hasrule) { + print "\nNo macro rule defined for $arg.\nParameters treated as surrounding text.\n"; } } else { - if ($def=$TeXgroup{$arg}) { - print "Rule for group $arg\n"; - _print_rule_group($arg,$def); + if ($def=$TeXenvir{$arg}) { + print "\nRule for environment $arg\n"; + _print_rule_macrocount($PREFIX_ENVIR.$arg); + _print_rule_envir($arg,$def); } else { - print "No default group rule defined for $arg.\nContent handled as surrounding text.\n"; + print "\nNo default environment rule defined for $arg.\nContent handled as surrounding text.\n"; } } } @@ -2498,60 +3326,119 @@ sub print_help_on { # Print macro handling rule sub _print_rule_macro { my ($arg,$def)=@_; - my %rules=( - 0 => 'Exclude from count', - 1 => 'Count as text', - 2 => 'Count as header', - 3 => 'Count as caption', - -1 => 'Exclude as float (i.e. include captions)', - -2 => 'Strong exclude (ignore begin-end groups)', - -3 => 'Stronger exclude (ignore all macros)', - -9 => 'Exclude as preamble'); if (ref($def) eq 'ARRAY') { - print "Takes ".scalar(@{$def})." parameter(s):\n"; - foreach my $i (@{$def}) { - print " - ".$rules{$i}."\n"; + my $optionflag=0; + print "Takes the following parameter(s):\n"; + foreach my $state (@{$def}) { + if ($state==$_STATE_OPTION) {$optionflag=1;} + elsif ($optionflag) { + $optionflag=0; + print " - Optional [] containing $state2desc{$state}\n"; + } else { + print " - $state2desc{$state}\n"; + } } } else { - print "Takes ".$def." parameter(s), not included in counts.\n"; + print "Takes $def parameter(s): content ignored, i.e. not included in counts.\n"; } } -# Print group handling rule -sub _print_rule_group { +# Print environment handling rule +sub _print_rule_envir { my ($arg,$def)=@_; - my %rules=( - 0 => 'Not included', - 1 => 'Text, words included in text count', - 2 => 'Header, words included in header count', - 3 => 'Float caption, words included in float caption count', - 6 => 'Inline mathematics, words not counted', - 7 => 'Displayed mathematics, words not counted', - -1 => 'Float, not included, but looks for captions'); - print "Rule used: ".$rules{$def}."\n"; - if ($def=$TeXmacro{'begin'.$arg}) { + print "Contents parsed as $state2desc{$def}\n"; + if ($def=$TeXmacro{$PREFIX_ENVIR.$arg}) { _print_rule_macro($def); } } +# Print macrocount rule, return rule +sub _print_rule_macrocount { + my $arg=shift @_; + my $def=$TeXmacrocount{$arg}; + if (!defined $def) {return undef;} + if (ref($def) eq 'ARRAY') { + my @wd=@$def; + foreach (@wd) {$_=$countdesc[$_];} + print 'Increments the following counters: ',join('; ',@wd),".\n"; + } else { + print "Counted as $def word(s).\n"; + } +} + +# Print style or style category summary +sub print_help_on_styles { + my $style=shift @_; + if (defined $style) { + if (my $def=$STYLES{$style}) { + _print_help_on_style_category($style); + print wrap('',' ',"$style = ".join(', ',grep(/^\w+$/,sort keys %$def))),".\n"; + } elsif ($def=$STYLE_DESC{$style}) { + print $style,' = ',$def,"\n"; + } else { + print "Unknown style or style category: $style.\n"; + } + } else { + print wrap('',' ','Styles: '.join(', ',@STYLE_LIST)),".\n\n"; + print wrap('',' ','Style categories: '.join(', ',sort grep(/^\w+$/,keys %STYLES))),".\n\n"; + print "Use -help-style={style} to get help on particular style or style category.\n"; + } +} + +# Print help on a particular style category (optional prefix and indentation) +sub _print_help_on_style_category { + my ($cat,$prefix,$indent)=@_; + if (!defined $prefix) {$prefix='';} + if (!defined $indent) {$indent=' ';} +} ###### HTML routines # Print HTML header sub html_head { - print "<html>\n<head>"; - print "\n<meta http-equiv='content-type' content='text/html; charset=$outputEncoding'>\n"; - _print_html_style(); - print "</head>\n\n<body>\n"; - print "\n<h1>LaTeX word count"; - if ($showVersion>0) {print " (version ",_html_version(),")"} - print "</h1>\n"; + if (defined $HTMLfile) { + if (open(FH,$HTMLfile)) { + my $text=join('',<FH>); + close FH; + if ( $text=~/^(.*)<!--+\s*texcount\s*--+>(.*)$/is + || $text=~/^(.*)\$\{texcount\}(.*)$/is + || $text=~/^(.*)(<\/body.*)$/is + ) { + $htmlopen=$1; + $htmlclose=$2; + } else { + error($Main,"Invalid HTML template format."); + } + } else { + error($Main,"HTML template file not found: $HTMLfile"); + } + } + if (defined $htmlopen) { + $htmlopen=~s/\$\{encoding\}/\Q$outputEncoding\E/g; + $htmlopen=~s/\$\{version\}/@{[_html_version()]}/g; + print $htmlopen; + $htmlopen=undef; + } else { + print "<html>\n<head>\n"; + print "<meta http-equiv='content-type' content='text/html; charset=$outputEncoding'>\n"; + _print_html_style(); + foreach (@htmlhead) {print $_,"\n";} + print "</head>\n\n<body>\n"; + print "\n<h1>LaTeX word count"; + if ($showVersion>0) {print ' (version ',_html_version(),')'} + print "</h1>\n"; + } } # Print HTML tail sub html_tail { - print "</body>\n\n</html>\n"; + if (defined $htmlclose) { + print $htmlclose; + $htmlclose=undef; + } else { + print "</body>\n\n</html>\n"; + } } # Return version number using HTML @@ -2563,27 +3450,45 @@ sub _html_version { # Print HTML STYLE element sub _print_html_style { +if (defined $CSShref) { + print "<link rel='stylesheet' href='$CSShref' type='text/css'>\n"; + return; +} +if (defined $CSSfile) { + if (open(FH,$CSSfile)) { + print "<style>\n<!---\n",<FH>,"-->\n</style>\n"; + close(FH); + return; + } else { + error($Main,"CSS file not found: $CSSfile"); + } +} print <<END <style> <!-- body {width:auto;padding:5;margin:5;} -.error {font-weight:bold;color:#f00;font-style:italic;} -.word1,.word2,.word3 {color: #009; border-left: 1px solid #CDF; border-bottom: 1px solid #CDF;} -.word2 {font-weight: 700;} -.word3 {font-style: italic;} -.command {color: #c00;} -.exclcommand {color: #f99;} +.error {font-weight:bold;color:#f00;font-style:italic; border: solid 1px red;} +.word,.hword,.oword,.altwd {border-left: 1px solid #CDF; border-bottom: 1px solid #CDF;} +.word {color: #009} +.hword {color: #009; font-weight: 700;} +.oword {color: #009; font-style: italic;} +.altwd {color: #00c; font-style: oblique;} +.cmd {color: #c00;} +.exclcmd {color: #f99;} .option {color: #cc0;} -.grouping, .document {color: #900; font-weight:bold;} +.optparm {color: #c90;} +.envir, .document {color: #900; font-weight:bold;} +.math {color: #6c0;} .mathgroup {color: #090;} -.exclmath {color: #6c6;} +.exclmath {color: #9f6;} +.mathcmd {color: #6c0;} .ignore {color: #999;} -.exclgroup {color:#c66;} +.exclenv {color:#c66;} .tc {color: #999; font-weight:bold;} .comment {color: #999; font-style: italic;} .state {color: #990; font-size: 70%;} .cumsum {color: #999; font-size: 80%;} -.fileinclude {color: #696; font-weight:bold;} +.fileinc {color: #696; font-weight:bold;} .warning {color: #c00; font-weight: 700;} div.filegroup, div.parse, div.stylehelp, div.count, div.sumcount, div.error { @@ -2603,7 +3508,7 @@ ul.count {list-style-type: none; margin: 4pt 0pt; padding: 0pt;} .fielddesc {font-style: italic;} .nb {color: #900;} -table.stat {border:2px solid #339; background:#666; border-collapse:collapse;} +table.stat {border:2px solid #339; margin:0 0 8pt 0; background:#666; border-collapse:collapse;} table.stat tr {border:1px solid #CCC;} table.stat th, table.stat td {padding:1pt 4pt;} table.stat col {padding:4pt;} @@ -2612,6 +3517,7 @@ table.stat tbody {border:1px solid #333;} table.stat tbody.sumstat {background:#FFC;} table.stat tbody.langstat {background:#FEE;} table.stat tbody.wordstat {background:#EEF;} +table.stat tbody.macrostat {background:#EFE;} table.stat .sum {font-weight:bold; font-style:italic;} --> </style> @@ -2655,6 +3561,7 @@ sub wprintstringdata { my $data=StringData($name); if (!defined $data) { error($Main,"No StringData $name.",'BUG'); + return; } wprintlines(@_,@$data); } @@ -2709,13 +3616,13 @@ sub wprintlines { my $firstindent=0; if ($line=~s/^(\t|\s{2,})(\S)/$2/) {$firstindent=$ind1;} my $indent=$firstindent; - if ($line=~/^(.*\S)(\t|\s{2,})/) { + if ($line=~/^(.*\S)(\t|\s{2,})(.*)$/) { $indent=$ind2; if ($1 eq '|') {$line=' ';} - else {$line=$1." ";} + else {$line=$1.' ';} $i=$indent-$firstindent-length($line); if ($i>0) {$line.=' ' x $i;} - $line.=$'; + $line.=$3; # $3~${^POSTMATCH} } print wrap(' ' x $firstindent,' ' x $indent,$line)."\n"; } @@ -2750,6 +3657,12 @@ which grants you, the user, the right to use, modify and distribute the script. The script has LPPL status "maintained" with ${maintainer} being the current maintainer. :::::::::::::::::::::::::::::::::::::::: +:::::::::: ShortHelp +Syntax: texcount.pl [options] files + +Use option -help (or just -h) to get help; -help-options (-hopt) to get list of command line options, or -help-options=substring for help on all options containing substring. + +:::::::::::::::::::::::::::::::::::::::: :::::::::: HelpTitle *************************************************************** * TeXcount version ${versionnumber}, ${versiondate} @@ -2760,7 +3673,7 @@ Count words in TeX and LaTeX files, ignoring macros, tables, formulae, etc. :::::::::::::::::::::::::::::::::::::::: :::::::::: OptionsHead -Syntax: TeXcount.pl [options] files +Syntax: texcount.pl [options] files Options: @@ -2775,6 +3688,7 @@ Options: -v2 More verbose: also print ignored text. -v3 Even more verbose: include comments and options. -v4 Same as -v3 -showstate. + -v=, v0=, ..., -v4= Set verbosity by adding/removing particular types of token (styles) to include in the verbose output. Use -help-style to get details of which tokens are included in each style, and of classes of tokens (style categories). -showstate Show internal states (with verbose). -brief Only prints a brief, one line summary of counts. -q, -quiet Quiet mode, no error messages. Use is discouraged! @@ -2789,7 +3703,11 @@ Options: -sep=, -separator= Separating character or string to be added after each word. -html Output in HTML format. -htmlcore Only HTML body contents. - -opt, -optionfile Read options/parameters from file. + -htmlfile= HTML template file to use with the -html option. Use <!-- TeXcode --> to indicate where the output from TeXcount should be inserted. + -tex Encode TeX special characters for output into TeX code + -cssfile= CSS file to include instead of default styles. + -css= CSS href to include instead of default styles. Can use -css=file:{filename} instead of -cssfile={filename}. + -opt=, -optionfile= Read options/parameters from file. - Read LaTeX code from STDIN. -inc Parse included TeX files (as separate file). -merge Merge included TeX files into code (in place). @@ -2800,7 +3718,8 @@ Options: -total Do not give sums per file, only total sum. -1 Same as -brief and -total. Ensures there is only one line of output. If used in conjunction with -sum, the output will only be the total number. (NB: Character is the number one, not the letter L.) -template= Speficy an output template. Use {1},...,{7}, {SUM} and {TITLE} to include values, {1?...?1} etc. to conditionally include sections, {1?....|...?1} etc. to specify an alternative text if zero. To include subcounts, use {SUB?...?SUB} where ... is replaced with the template to use per subcount. Line shift may be specified using \\n. - -dir, -dir= Specify the working directory using -dir=path. Remember that the path must end with \\ or /. If only -dir is used, the directory of the parent file is used. + -dir, -dir= Specify the working directory using -dir=path. Remember that the path must end with \\ or /. If only -dir is used, the directory of the provided file is used and paths (e.g. included files) are assumed to be absolute or relative to this. The default at startup is -dir=. which means that the directory from which TeXcount is run is the working directory with all paths absolute or relative to this. + -auxdir, -auxdir= Specify directory for auxilary files, e.g. the bibliograph (.bbl) file. If only -auxdir is used, the working directory (as determined by -dir or -dir=) is assumed. If -auxdir= is used with -dir=, it sets the path to the auxilary directiry. If -auxdir= is used with -dir, the working directory is determined from the location of the provided file, and the path to the auxilary directory is assumed to be absolute or relative to this. -enc=, -encoding= Specify encoding (default is to guess the encoding). -utf8, -unicode Selects Unicode (UTF-8) for input and output. This is automatic with -chinese, and is required to handle e.g. Korean text. Note that the TeX file must be save in UTF-8 format (not e.g. GB2312 or Big5), or the result will be unpredictable. -alpha=, -alphabets= List of Unicode character groups (or digit, alphabetic) permitted as letters. Names are separated by ',' or '+'. If list starts with '+', the alphabets will be added to those already included. The default is Digit+alphabetic. @@ -2813,12 +3732,18 @@ Options: -char, -character, -letters Counts letters/characters instead of words. Note that spaces and punctuation is not counted. -char-only, ..., -letters-only Like -letters, but counts alphabetic letters only. -countall, -count-all The default setting in which all characters are included as either alphabets og logograms. - -freq Produce individual word frequency table. - -stat Produce statistics on language/script usage. + -freq, -freq= Produce individual word frequency table. Optionally give minimal number of occurences to be listed. + -stat Produce statistics on language/script usage. + -macrostat, -macrofreq Print macro usage statistics. -codes Display output style code overview and explanation. This is on by default. -nocodes Do not display output style code overview. + -out= Write output to file, give filename as option value. -h, -?, -help, /? Help text. -h=, -?=, -help=, /?= Takes a macro or group name as option and returns a description of the rules for handling this if any are defined. If handling rule is package specific, use -incpackage=package name: -incpackage must come before -h= on the command line to take effect. + -help-options, -h-opt List all options. + -help-options=, -h-opt= List all options containing the provided string, e.g. -h-opt=dir or -h-opt=-v (the initial - in -v causes only options starting with v to be listed). + -help-style List the styles and style categories: i.e. those permitted used with -v={styles-list}. + -help-style= Give description of style or style category. -ver, -version Print version number. -lic, -license, -licence Licence information. @@ -2858,10 +3783,13 @@ and are used to control how TeXcount parses the document. The following instruct The [param.states] is used to indicate the number of parameters used by the macro and the rules of handling each of these: the format is [#,#,...,#] with # representing one number for each parameter giving the parsing state to use for that parameter, alternatively just a single number (#) indicating how many parameters to ignore (parsing state 0). The option [content-state] is used to give the parsing state to use for the contents of a begin-end group. The main parsing states are 0 to ignore and 1 to count as text. Parsing instructions which may be used anywhere are: -@ - : - %TC:ignore start block to ignore - %TC:endignore end block to ignore - %TC:break [title] add subcount break point here +@ - : + %TC:break [title] add subcount break point here + %TC:incbib include bibliography (same as running with -incbib) + %TC:ignore ignore region, end with %TC:endignore + %TC:insert [code] insert code for TeXcount to process as TeX code + %TC:newtemplate start a new template, ie delete the existing one + %TC:template [template] add another line to the template specification See the documentation for more details. Command line options and most %TC commands (prefixed by % rather than %TC:) may be placed in an options file. This is particularly useful for defining your own output templates and macro handling rules. |