From 80bf3d55d794628d8b084ac36b60ec3bcc3cc642 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Sun, 24 Oct 2010 16:34:32 +0000 Subject: move the parse-texcatalogue-keywords file into tlpkg/bin, and add option handling git-svn-id: svn://tug.org/texlive/trunk@20165 c570f23f-e606-0410-a88d-b1316a301751 --- Master/tlpkg/bin/parse-texcatalogue-keywords | 208 +++++++++++++++++++++ .../dev/keyword-search/parse-texcatalogue-keywords | 176 ----------------- 2 files changed, 208 insertions(+), 176 deletions(-) create mode 100755 Master/tlpkg/bin/parse-texcatalogue-keywords delete mode 100755 Master/tlpkg/dev/keyword-search/parse-texcatalogue-keywords (limited to 'Master') diff --git a/Master/tlpkg/bin/parse-texcatalogue-keywords b/Master/tlpkg/bin/parse-texcatalogue-keywords new file mode 100755 index 00000000000..ab15297b196 --- /dev/null +++ b/Master/tlpkg/bin/parse-texcatalogue-keywords @@ -0,0 +1,208 @@ +#!/usr/bin/env perl +# $Id$ +# parse a TeXCatalogue v2 .xml dump containing keywords and +# characterizations +# +# Copyright 2010 Norbert Preining +# This file is licensed under the GNU General Public License version 2 +# or any later version. + +BEGIN { + $^W = 1; + chomp ($mydir = `dirname $0`); + unshift (@INC, "$mydir/.."); +} + +use strict; + +use XML::Parser; +use XML::XPath; +use XML::XPath::XMLParser; +use Text::Unidecode; +use Data::Dumper; +use Getopt::Long; +use TeXLive::TLUtils; + +my %taxonomy; + +my $tcfile; +my $output; +my $help = 0; + +TeXLive::TLUtils::process_logging_options(); +GetOptions( + "input|catalogue=s" => \$tcfile, + "output=s" => \$output, + "help|?" => \$help) or pod2usage(1); +pod2usage(-exitstatus => 0, -verbose => 2) if $help; + +my $progname = TeXLive::TLUtils::basename($0); + +exit (&main()); + + +sub main +{ + my $io; + if (!defined($tcfile)) { + $io = \*STDIN; + } else { + if (! -f $tcfile) { + die "$progname: argument to --input is not a file: $!.\n"; + } + open($io, "<", $tcfile) or die "Cannot open input file $tcfile: $!"; + } + parse_texcatalogue($io); + $Data::Dumper::Indent = 1; + + my $out; + if (!defined($output)) { + $out = \*STDOUT; + } else { + open($out, ">$output") or die "Cannot open $output for writing: $!"; + } + print $out Data::Dumper->Dump([\%taxonomy], ["taxonomy"]), $/; +} + + + +# how to read in!!! +#my $taxonomy; +#my $foo = `cat tc-dump`; +# the no strict "vars" is *ABSOLUT* necessary otherwise the file is not +# evaluated, no idea why! +#no strict "vars"; +#eval "$foo"; +#use strict "vars"; +# +#print_keywords($taxonomy->{'by-taxonomy'}{'keywords'}); +#print "\n===================\nprimary characterizations\n"; +#walk_cz_tree($taxonomy->{'by-taxonomy'}{'primary'}, "PRIM"); +#print "\n===================\nsecondary characterizations\n"; +#walk_cz_tree($taxonomy->{'by-taxonomy'}{'secondary'}, "SEC"); +#print "\n===================\nfunctionality characterizations\n"; +#walk_cz_tree($cz_pkg->{'by-taxonomy'}{'functionality'}, "BFUNC"); + +##### only subs from here ########## + +sub parse_texcatalogue { + my $io = shift; + my $_parser = XML::Parser->new( + ErrorContext => 2, + ParseParamEnt => 1, + NoLWP => 1 + ); + my $parser = new XML::XPath->new(ioref => $io, parser => $_parser) || + die "Failed to parse input file $tcfile: $!"; + + # format of the data + # $taxonomy{'by-package'}{'keyword'}{$package} = [ $kwi, $kwii,...]; + # $taxonomy{'by-package'}{'primary'}{$package} = $primchar; + # $taxonomy{'by-package'}{'secondary'}{$package'} = $secchar; + # $taxonomy{'by-package'}{'functionality'}{$package} = [ $bfi, $bfii, ...]; + # + # post processing gives + # $taxonomy{'by-taxonomy'}{'keyword'}{$keyword} = [ $pkg1, $pkg2, ...]; + # $taxonomy{'by-taxonomy'}{'primary'}{$level1}{$level2}...{'_packages_'} = [ $pkg1, $pkg2, ...]; + # $taxonomy{'by-taxonomy'}{'secondary'}{$level1}{$level2}...{'_packages_'} = [ $pkg1, $pkg2, ...]; + # $taxonomy{'by-taxonomy'}{'functionality'}{$level1}{$level2}...{'_packages_'} = [ $pkg1, $pkg2, ...]; + + + foreach my $e ($parser->find('/fullcat/catalogue/entry')->get_nodelist) { + my $pkg = $parser->findvalue('./name',$e)->value(); + #print "FOUND: $pkg\n"; + my $n = $parser->find('./keyword',$e); + for my $kw ($n->get_nodelist) { + my $kwval = $parser->find('./@keyword',$kw)->string_value(); + push @{$taxonomy{'by-package'}{'keyword'}{$pkg}}, $kwval; + #print "keyword = $kwval\n"; + } + $n = $parser->find('./characterization',$e); + for my $cz ($n->get_nodelist) { + my $czdimnl = $parser->find('./@dimension',$cz); + my $czdim = $czdimnl->string_value(); + my $czvalnl = $parser->findvalue('.',$cz); + my $czval = $czvalnl->value(); + if (($czdim eq "primary") || ($czdim eq "secondary")) { + # assume only one primary and one secondary function + $taxonomy{'by-package'}{$czdim}{$pkg} = $czval; + } else { + # assume that it is always "functionality" + push @{$taxonomy{'by-package'}{'functionality'}{$pkg}}, $czval; + } + #print "char dim = $czdim val=$czval\n"; + } + } + + + # + # do the keyword reshuffling + for my $pkg (keys %{$taxonomy{'by-package'}{'keyword'}}) { + for my $kw (@{$taxonomy{'by-package'}{'keyword'}{$pkg}}) { + push @{$taxonomy{'by-taxonomy'}{'keyword'}{$kw}}, $pkg; + } + } + + parse_characterizations('primary'); + parse_characterizations('secondary'); + parse_characterizations('functionality'); +} + +sub parse_characterizations { + my $what = shift; + $taxonomy{'by-taxonomy'}{$what} = {}; + for my $pkg (keys %{$taxonomy{'by-package'}{$what}}) { + my $value = $taxonomy{'by-package'}{$what}{$pkg}; + my @charlist; + if (!ref($value)) { + @charlist = ($value); + } else { + @charlist = @$value; + } + for my $prim (@charlist) { + # split the primary into levels sep by > + my @levels = split(' > ', $prim); + my $currentpointer; + $currentpointer = $taxonomy{'by-taxonomy'}{$what}; + for my $l (@levels) { + if (!defined($currentpointer->{$l})) { + $currentpointer->{$l} = {}; + } + $currentpointer = $currentpointer->{$l} + } + push @{$currentpointer->{'_packages_'}}, $pkg; + } + } +} + + +sub print_keywords { + my $kw_pkg = shift; + for my $k (keys %$kw_pkg) { + my @pkgl = @{$kw_pkg->{$k}}; + if (@pkgl) { + print "keyword = $k\n package = @pkgl\n"; + } else { + print "keyword = $k\n package = NO PACKAGE FOUND!\n"; + } + } +} + + +sub walk_cz_tree { + my $cp = shift; + my $prestring = shift; + if (defined($cp->{'_packages_'})) { + my @pkgs = sort @{$cp->{'_packages_'}}; + print "$prestring\n"; + print "--> @pkgs\n"; + } + for my $cz (keys %$cp) { + if ($cz ne '_packages_') { + my $nextstring = "$prestring > $cz"; + my $np = $cp->{$cz}; + &walk_cz_tree($np,$nextstring); + } + } +} + diff --git a/Master/tlpkg/dev/keyword-search/parse-texcatalogue-keywords b/Master/tlpkg/dev/keyword-search/parse-texcatalogue-keywords deleted file mode 100755 index 25cb5617a45..00000000000 --- a/Master/tlpkg/dev/keyword-search/parse-texcatalogue-keywords +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env perl -# $Id$ -# parse a TeXCatalogue v2 .xml dump containing keywords and -# characterizations -# -# Copyright 2010 Norbert Preining -# This file is licensed under the GNU General Public License version 2 -# or any later version. - -$^W = 1; -use strict; - - -use XML::Parser; -use XML::XPath; -use XML::XPath::XMLParser; -use Text::Unidecode; -use Data::Dumper; - -my $tcfile; -if (@ARGV) { - $tcfile = shift @ARGV; -} else { - $tcfile = "texcatalogue.xml"; -} - -my %taxonomy; - -parse_texcatalogue($tcfile); - -$Data::Dumper::Indent = 1; -print Data::Dumper->Dump([\%taxonomy], ["taxonomy"]), $/; - - -# how to read in!!! -#my $taxonomy; -#my $foo = `cat tc-dump`; -# the no strict "vars" is *ABSOLUT* necessary otherwise the file is not -# evaluated, no idea why! -#no strict "vars"; -#eval "$foo"; -#use strict "vars"; -# -#print_keywords($taxonomy->{'by-taxonomy'}{'keywords'}); -#print "\n===================\nprimary characterizations\n"; -#walk_cz_tree($taxonomy->{'by-taxonomy'}{'primary'}, "PRIM"); -#print "\n===================\nsecondary characterizations\n"; -#walk_cz_tree($taxonomy->{'by-taxonomy'}{'secondary'}, "SEC"); -#print "\n===================\nfunctionality characterizations\n"; -#walk_cz_tree($cz_pkg->{'by-taxonomy'}{'functionality'}, "BFUNC"); - -##### only subs from here ########## - -sub parse_texcatalogue { - my $tcfile = shift; - my $_parser = XML::Parser->new( - ErrorContext => 2, - ParseParamEnt => 1, - NoLWP => 1 - ); - my $io; - open($io, "<", $tcfile) or die "Cannot open texcatalogue.xml: $!"; - my $parser = new XML::XPath->new(ioref => $io, parser => $_parser) || - die "Failed to parse texcatalogue.xml: $!"; - - # format of the data - # $taxonomy{'by-package'}{'keyword'}{$package} = [ $kwi, $kwii,...]; - # $taxonomy{'by-package'}{'primary'}{$package} = $primchar; - # $taxonomy{'by-package'}{'secondary'}{$package'} = $secchar; - # $taxonomy{'by-package'}{'functionality'}{$package} = [ $bfi, $bfii, ...]; - # - # post processing gives - # $taxonomy{'by-taxonomy'}{'keyword'}{$keyword} = [ $pkg1, $pkg2, ...]; - # $taxonomy{'by-taxonomy'}{'primary'}{$level1}{$level2}...{'_packages_'} = [ $pkg1, $pkg2, ...]; - # $taxonomy{'by-taxonomy'}{'secondary'}{$level1}{$level2}...{'_packages_'} = [ $pkg1, $pkg2, ...]; - # $taxonomy{'by-taxonomy'}{'functionality'}{$level1}{$level2}...{'_packages_'} = [ $pkg1, $pkg2, ...]; - - - foreach my $e ($parser->find('/fullcat/catalogue/entry')->get_nodelist) { - my $pkg = $parser->findvalue('./name',$e)->value(); - #print "FOUND: $pkg\n"; - my $n = $parser->find('./keyword',$e); - for my $kw ($n->get_nodelist) { - my $kwval = $parser->find('./@keyword',$kw)->string_value(); - push @{$taxonomy{'by-package'}{'keyword'}{$pkg}}, $kwval; - #print "keyword = $kwval\n"; - } - $n = $parser->find('./characterization',$e); - for my $cz ($n->get_nodelist) { - my $czdimnl = $parser->find('./@dimension',$cz); - my $czdim = $czdimnl->string_value(); - my $czvalnl = $parser->findvalue('.',$cz); - my $czval = $czvalnl->value(); - if (($czdim eq "primary") || ($czdim eq "secondary")) { - # assume only one primary and one secondary function - $taxonomy{'by-package'}{$czdim}{$pkg} = $czval; - } else { - # assume that it is always "functionality" - push @{$taxonomy{'by-package'}{'functionality'}{$pkg}}, $czval; - } - #print "char dim = $czdim val=$czval\n"; - } - } - - - # - # do the keyword reshuffling - for my $pkg (keys %{$taxonomy{'by-package'}{'keyword'}}) { - for my $kw (@{$taxonomy{'by-package'}{'keyword'}{$pkg}}) { - push @{$taxonomy{'by-taxonomy'}{'keyword'}{$kw}}, $pkg; - } - } - - parse_characterizations('primary'); - parse_characterizations('secondary'); - parse_characterizations('functionality'); -} - -sub parse_characterizations { - my $what = shift; - $taxonomy{'by-taxonomy'}{$what} = {}; - for my $pkg (keys %{$taxonomy{'by-package'}{$what}}) { - my $value = $taxonomy{'by-package'}{$what}{$pkg}; - my @charlist; - if (!ref($value)) { - @charlist = ($value); - } else { - @charlist = @$value; - } - for my $prim (@charlist) { - # split the primary into levels sep by > - my @levels = split(' > ', $prim); - my $currentpointer; - $currentpointer = $taxonomy{'by-taxonomy'}{$what}; - for my $l (@levels) { - if (!defined($currentpointer->{$l})) { - $currentpointer->{$l} = {}; - } - $currentpointer = $currentpointer->{$l} - } - push @{$currentpointer->{'_packages_'}}, $pkg; - } - } -} - - -sub print_keywords { - my $kw_pkg = shift; - for my $k (keys %$kw_pkg) { - my @pkgl = @{$kw_pkg->{$k}}; - if (defined(@pkgl)) { - print "keyword = $k\n package = @pkgl\n"; - } else { - print "keyword = $k\n package = NO PACKAGE FOUND!\n"; - } - } -} - - -sub walk_cz_tree { - my $cp = shift; - my $prestring = shift; - if (defined($cp->{'_packages_'})) { - my @pkgs = sort @{$cp->{'_packages_'}}; - print "$prestring\n"; - print "--> @pkgs\n"; - } - for my $cz (keys %$cp) { - if ($cz ne '_packages_') { - my $nextstring = "$prestring > $cz"; - my $np = $cp->{$cz}; - &walk_cz_tree($np,$nextstring); - } - } -} - -- cgit v1.2.3