From 24579180da51beb0b66f8014929adcc247b8f945 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Mon, 12 Sep 2011 23:55:56 +0000 Subject: first attempt at ctan mirror extraction git-svn-id: svn://tug.org/texlive/trunk@23925 c570f23f-e606-0410-a88d-b1316a301751 --- Master/tlpkg/bin/tl-ctan-mirror-to-perl-hash | 104 -------------------- Master/tlpkg/bin/tl-update-auto | 46 ++++++--- Master/tlpkg/bin/tl-update-ctan-mirrors | 104 ++++++++++++++++++++ Master/tlpkg/doc/packages.txt | 8 ++ Master/tlpkg/installer/ctan-mirrors.pl | 141 +++++++++++++++++++++++++++ Master/tlpkg/installer/mirrors.pl | 141 --------------------------- 6 files changed, 288 insertions(+), 256 deletions(-) delete mode 100755 Master/tlpkg/bin/tl-ctan-mirror-to-perl-hash create mode 100755 Master/tlpkg/bin/tl-update-ctan-mirrors create mode 100644 Master/tlpkg/installer/ctan-mirrors.pl delete mode 100644 Master/tlpkg/installer/mirrors.pl diff --git a/Master/tlpkg/bin/tl-ctan-mirror-to-perl-hash b/Master/tlpkg/bin/tl-ctan-mirror-to-perl-hash deleted file mode 100755 index a7404222337..00000000000 --- a/Master/tlpkg/bin/tl-ctan-mirror-to-perl-hash +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/perl - -# needed file: -# http://ctan.org/mirrors (CTAN.sites, README.mirrors) -# rsync://comedy.dante.de/MirMon/mirmon.state - -use strict; -$^W = 1; -use Data::Dumper; - -if (!defined($ARGV[0])) { - die ("Need the location of README.mirror as argument\n"); -} -if (!defined($ARGV[1])) { - die ("Need the location of mirmon.state file as argument\n"); -} - -open FOO, "<$ARGV[0]" || die("Cannot open $ARGV[0]: $!"); -open MSTATE, "<$ARGV[1]" || die("Cannot open $ARGV[1]: $!"); - -my %good_urls; - -read_mstate(); -my %foo = read_readme_mirror(); -$Data::Dumper::Indent = 1; -print Data::Dumper->Dump([\%foo], [qw(mirrors)]); - -sub read_mstate { - my %mstate; - while () { - my ($m, $age, $status_last_probe, $time_last_succesful_probe, - $probe_history, $state_history, $last_probe) = split ' '; - if ($status_last_probe eq "ok") { - $good_urls{$m} = 1; - } else { - $good_urls{$m} = 0; - } - } - close(MSTATE); -} - -sub add_mirror { - my ($mirref, $continent, $country, $mirror, $p, $ppath) = @_; - my $url = "$p://$ppath"; - if (defined($good_urls{$url})) { - if ($good_urls{$url}) { - #$mirref->{$continent}{$country}{$mirror}{'protocols_path'}{$p} = $ppath; - $mirref->{$continent}{$country}{$url} = 1; - } else { - printf STDERR "mirror seems to be dead, skipped: $url\n"; - } - } else { - printf STDERR "mirror not in mirmon file, skipped: $url\n"; - } -} - -sub read_readme_mirror { - my %mirrors; - my $continent; - my $mirror; - my $country; - my %protocols; - while () { - chomp; - if (m/^ (Africa|Asia|Australasia|Europe|North America|South America)/) { - my $save_continent = $1; - if (defined($mirror)) { - for my $p (keys %protocols) { - add_mirror(\%mirrors, $continent, $country, $mirror, $p, $protocols{$p}); - } - } - $continent = $save_continent; - $mirror = undef; - $country = undef; - %protocols = (); - next; - } - next if (!defined($continent)); - if (m/^ ([-a-zA-Z0-9.]+) \((.*)\)\s*$/) { - my $save_mirror = $1; - my $save_country = $2; - # make country names more reasonable - $save_country =~ s/^The //; - if (defined($mirror)) { - for my $p (keys %protocols) { - add_mirror(\%mirrors, $continent, $country, $mirror, $p, $protocols{$p}); - } - } - $mirror = $save_mirror; - $country = $save_country; - %protocols = (); - next; - } - next if (!defined($mirror)); - if (m!^ URL: (ftp|http|rsync)://([-a-zA-Z0-9.]+)/([-\w/]*)!) { - $protocols{$1} = "$2/$3"; - next; - } - #print "ignored >>$_<<\n"; - } - return %mirrors; -} - - diff --git a/Master/tlpkg/bin/tl-update-auto b/Master/tlpkg/bin/tl-update-auto index 68eb87a50f7..a48be70c32d 100755 --- a/Master/tlpkg/bin/tl-update-auto +++ b/Master/tlpkg/bin/tl-update-auto @@ -7,30 +7,33 @@ PATH=/usr/local/gnu/bin:/usr/local/bin:$PATH; export PATH umask 0 +# maybe someday we'll make real options. chicken=echo chicken= # verbose=echo verbose=false # +config_scripts_only=false + +TMPDIR=/tmp/tlua; export TMPDIR +test -d $TMPDIR || mkdir $TMPDIR +test -d $TMPDIR || { echo "$0: no directory $TMPDIR, goodbye." >&2; exit 1; } + cp="cp -fv" mv="mv -fv" diff="diff -U 2" -# maybe someday we'll make real options. -config_scripts_only=false - mydir=`cd \`dirname $0\` && /bin/pwd` Master=`cd $mydir/../.. && /bin/pwd` cd $Master || exit 1 -temp=/tmp/ua$$ +temp=$TMPDIR/ua$$ trap "rm -f $temp*" 0 1 2 15 update_list= add_list= - # autogenerate README.EN from readme.html. # if $config_scripts_only; then :; else @@ -166,10 +169,10 @@ if $config_scripts_only; then :; else # (we need to ignore the timestamps in the generation lines.) same_except_for_th () { - rm -f /tmp/seft1 /tmp/seft2 - sed '/^\.TH/d' <"$1" >/tmp/seft1 - sed '/^\.TH/d' <"$2" >/tmp/seft2 - cmp -s /tmp/seft1 /tmp/seft2 + rm -f $TMPDIR/seft1 $TMPDIR/seft2 + sed '/^\.TH/d' <"$1" >$TMPDIR/seft1 + sed '/^\.TH/d' <"$2" >$TMPDIR/seft2 + cmp -s $TMPDIR/seft1 $TMPDIR/seft2 } mandir=texmf/doc/man @@ -192,7 +195,7 @@ for script in texmf/scripts/texlive/tlmgr.pl install-tl; do # Update the version on the web site; since these aren't checked in, we # can do it unconditionally. # - pod2html="$chicken pod2html --cachedir=/tmp" + pod2html="$chicken pod2html --cachedir=$TMPDIR" $pod2html $script >$webdir/$basescript.html fi done @@ -224,7 +227,7 @@ fi # !config_scripts_only # texcatalogue.keywords # if $config_scripts_only; then :; else -cat_dump=/tmp/tc.xml +cat_dump=$TMPDIR/tc.xml rm -f $cat_dump.new wget --quiet ftp://tug.ctan.org/ftpmaint/az/texcatalogue.xml -O $cat_dump.new \ && test -s $cat_dump.new && mv $cat_dump.new $cat_dump @@ -246,6 +249,27 @@ fi fi # !config_scripts_only +# ctan mirror list from ctan. +if $config_scripts_only; then :; else +ctan_mirrors=$TMPDIR/mirrors # incoming data from CTAN +ctan_mirmon=$TMPDIR/mirmon.state +mirrors4tl=tlpkg/installer/ctan_mirrors.pl # massaged for TL + +if wget --quiet http://ctan.org/mirrors -O $ctan_mirrors \ + && rsync rsync://comedy.dante.de/MirMon/mirmon.state $ctan_mirmon; then + $mydir/tl-update-ctan-mirrors $ctan_mirrors $ctan_mirmon >$mirrors_4tl.new + if $diff $mirrors4tl.new $mirrors4tl; then + $verbose " $mirrors4tl ok." + else + $chicken $cp $mirrors4tl.new $mirrors4tl + update_list="$update_list $mirrors4tl" + fi +else + echo "$0: ctan mirror data not retrievable, skipping." >&2 +fi +fi # !config_scripts_only + + # svn commits. status=0 if test -z "$update_list"; then diff --git a/Master/tlpkg/bin/tl-update-ctan-mirrors b/Master/tlpkg/bin/tl-update-ctan-mirrors new file mode 100755 index 00000000000..a7404222337 --- /dev/null +++ b/Master/tlpkg/bin/tl-update-ctan-mirrors @@ -0,0 +1,104 @@ +#!/usr/bin/perl + +# needed file: +# http://ctan.org/mirrors (CTAN.sites, README.mirrors) +# rsync://comedy.dante.de/MirMon/mirmon.state + +use strict; +$^W = 1; +use Data::Dumper; + +if (!defined($ARGV[0])) { + die ("Need the location of README.mirror as argument\n"); +} +if (!defined($ARGV[1])) { + die ("Need the location of mirmon.state file as argument\n"); +} + +open FOO, "<$ARGV[0]" || die("Cannot open $ARGV[0]: $!"); +open MSTATE, "<$ARGV[1]" || die("Cannot open $ARGV[1]: $!"); + +my %good_urls; + +read_mstate(); +my %foo = read_readme_mirror(); +$Data::Dumper::Indent = 1; +print Data::Dumper->Dump([\%foo], [qw(mirrors)]); + +sub read_mstate { + my %mstate; + while () { + my ($m, $age, $status_last_probe, $time_last_succesful_probe, + $probe_history, $state_history, $last_probe) = split ' '; + if ($status_last_probe eq "ok") { + $good_urls{$m} = 1; + } else { + $good_urls{$m} = 0; + } + } + close(MSTATE); +} + +sub add_mirror { + my ($mirref, $continent, $country, $mirror, $p, $ppath) = @_; + my $url = "$p://$ppath"; + if (defined($good_urls{$url})) { + if ($good_urls{$url}) { + #$mirref->{$continent}{$country}{$mirror}{'protocols_path'}{$p} = $ppath; + $mirref->{$continent}{$country}{$url} = 1; + } else { + printf STDERR "mirror seems to be dead, skipped: $url\n"; + } + } else { + printf STDERR "mirror not in mirmon file, skipped: $url\n"; + } +} + +sub read_readme_mirror { + my %mirrors; + my $continent; + my $mirror; + my $country; + my %protocols; + while () { + chomp; + if (m/^ (Africa|Asia|Australasia|Europe|North America|South America)/) { + my $save_continent = $1; + if (defined($mirror)) { + for my $p (keys %protocols) { + add_mirror(\%mirrors, $continent, $country, $mirror, $p, $protocols{$p}); + } + } + $continent = $save_continent; + $mirror = undef; + $country = undef; + %protocols = (); + next; + } + next if (!defined($continent)); + if (m/^ ([-a-zA-Z0-9.]+) \((.*)\)\s*$/) { + my $save_mirror = $1; + my $save_country = $2; + # make country names more reasonable + $save_country =~ s/^The //; + if (defined($mirror)) { + for my $p (keys %protocols) { + add_mirror(\%mirrors, $continent, $country, $mirror, $p, $protocols{$p}); + } + } + $mirror = $save_mirror; + $country = $save_country; + %protocols = (); + next; + } + next if (!defined($mirror)); + if (m!^ URL: (ftp|http|rsync)://([-a-zA-Z0-9.]+)/([-\w/]*)!) { + $protocols{$1} = "$2/$3"; + next; + } + #print "ignored >>$_<<\n"; + } + return %mirrors; +} + + diff --git a/Master/tlpkg/doc/packages.txt b/Master/tlpkg/doc/packages.txt index 10fc7ae2c16..090af62d586 100644 --- a/Master/tlpkg/doc/packages.txt +++ b/Master/tlpkg/doc/packages.txt @@ -127,6 +127,14 @@ svn copy -r 23181 -m'branch2011 branch based on r23181' svn://tug.org/texlive/tr Do both a branch and a tag so that it's possible to easily compare the current state of the branch with its starting point (the tag). + Then, to check out the branch, so critical source fixes can be applied + for the distros: +cd /home/texlive +rm -rf branch # if starting a new year on tug +mkdir branch; cd branch +svn co file:///home/svn/texlive/branches/branch2011/Build/source . >&out + + - update ~karl/bin/cron.tl, and reenable when feel ready. critical= pretest= diff --git a/Master/tlpkg/installer/ctan-mirrors.pl b/Master/tlpkg/installer/ctan-mirrors.pl new file mode 100644 index 00000000000..9c9f9cf3ae1 --- /dev/null +++ b/Master/tlpkg/installer/ctan-mirrors.pl @@ -0,0 +1,141 @@ +$mirrors = { + 'Europe' => { + 'UK' => { + 'http://ctan.sqsol.co.uk/' => 1, + 'http://anorien.csc.warwick.ac.uk/mirrors/CTAN/' => 1 + }, + 'Italy' => { + 'http://ftp.uniRoma2.it/TeX/' => 1, + 'http://ctan.mirror.garr.it/mirrors/CTAN/' => 1 + }, + 'Germany' => { + 'http://sunsite.informatik.rwth-aachen.de/ftp/pub/mirror/ctan/' => 1, + 'http://ftp.uni-erlangen.de/mirrors/CTAN/' => 1, + 'http://mirror.informatik.uni-mannheim.de/pub/mirrors/tex-archive/' => 1, + 'http://ctan.larsko.net/' => 1, + 'ftp://ftp.rrzn.uni-hannover.de/pub/mirror/tex-archive/' => 1 + }, + 'Serbia and Montenegro' => { + 'http://rpm.scl.rs/CTAN/' => 1 + }, + 'Lithuania' => { + 'http://ctan.vgtu.lt/' => 1 + }, + 'Norway' => { + 'http://ftp.fagskolen.gjovik.no/pub/tex-archive/' => 1, + 'http://ctan.uib.no/' => 1 + }, + 'Netherlands' => { + 'http://ftp.snt.utwente.nl/pub/software/tex/' => 1 + }, + 'France' => { + 'http://ctan.mines-albi.fr/' => 1, + 'http://distrib-coffee.ipsl.jussieu.fr/pub/mirrors/ctan/' => 1, + 'http://mirror.ibcp.fr/pub/CTAN/' => 1, + 'http://ftp.oleane.net/pub/CTAN/' => 1, + 'http://mirrors.ircam.fr/pub/CTAN/' => 1, + 'ftp://ftp.inria.fr/pub/TeX/CTAN/' => 1 + }, + 'Poland' => { + 'ftp://ftp.tpnet.pl/pub/CTAN/' => 1 + }, + 'Russia' => { + 'http://mirror.macomnet.net/pub/CTAN/' => 1, + 'http://ctan.uni-altai.ru/' => 1, + 'ftp://ftp.vsu.ru/pub/CTAN/' => 1 + }, + 'Slovenia' => { + 'http://ctan.ijs.si/tex-archive/' => 1 + }, + 'Portugal' => { + 'http://ftp.eq.uc.pt/software/TeX/' => 1 + }, + 'Denmark' => { + 'http://mirrors.dotsrc.org/ctan/' => 1 + }, + 'Spain' => { + 'http://ftp.udc.es/CTAN/' => 1 + }, + 'Greece' => { + 'http://ftp.cc.uoc.gr/mirrors/CTAN/' => 1 + }, + 'Czech Republic' => { + 'http://ftp.cvut.cz/tex-archive/' => 1 + } + }, + 'North America' => { + 'Canada' => { + 'http://mirror.its.dal.ca/ctan/' => 1, + 'http://mirror.csclub.uwaterloo.ca/CTAN/' => 1, + 'http://ctan.mirror.rafal.ca/' => 1 + }, + 'USA' => { + 'http://mirrors.ibiblio.org/pub/mirrors/CTAN/' => 1, + 'http://ctan.mackichan.com/' => 1, + 'http://mirror.math.ku.edu/tex-archive/' => 1, + 'http://mirror.unl.edu/ctan/' => 1, + 'http://mirrors.med.harvard.edu/ctan/' => 1, + 'http://mirror.hmc.edu/ctan/' => 1, + 'http://mirror.its.uidaho.edu/pub/tex-archive/' => 1 + } + }, + 'Australasia' => { + 'Australia' => { + 'http://ctan.unsw.edu.au/' => 1 + } + }, + 'South America' => { + 'Chile' => { + 'ftp://ftp.inf.utfsm.cl/pub/tex-archive/' => 1 + }, + 'Colombia' => { + 'http://www.laqee.unal.edu.co/tex-archive/' => 1 + }, + 'Brazil' => { + 'http://ceres.cisc.usp.br/CTAN/' => 1, + 'http://ctan.tche.br/' => 1, + 'http://linorg.usp.br/CTAN/' => 1, + 'http://mirror.softwarelivre.ufsc.br/pub/ctan/' => 1 + } + }, + 'Asia' => { + 'Taiwan' => { + 'http://ctan.cdpa.nsysu.edu.tw/' => 1, + 'http://shadow.ind.ntou.edu.tw/ctan/' => 1 + }, + 'Iran' => { + 'http://ctan.um.ac.ir/' => 1 + }, + 'Indonesia' => { + 'http://kebo.vlsm.org/CTAN/' => 1, + 'http://kambing.ui.ac.id/ctan/' => 1, + 'http://ctan.cermin.lipi.go.id/' => 1, + 'http://ftp.itb.ac.id/pub/CTAN/' => 1 + }, + 'Pakistan' => { + 'http://mirror-cybernet.lums.edu.pk/pub/ctan/' => 1 + }, + 'Malaysia' => { + 'ftp://mirror.upm.edu.my/ctan/' => 1 + }, + 'Korea' => { + 'http://ftp.kaist.ac.kr/pub/tex-archive/' => 1 + }, + 'China' => { + 'http://mirror.bjtu.edu.cn/CTAN/' => 1, + 'http://mirror.osqdu.org/CTAN/' => 1, + 'http://mirrors.xmu.edu.cn/CTAN/' => 1, + 'http://mirror.neu.edu.cn/CTAN/' => 1, + 'http://ftp.ctex.org/mirrors/CTAN/' => 1, + 'http://mirrors.ustc.edu.cn/CTAN/' => 1 + }, + 'Japan' => { + 'http://ftp.jaist.ac.jp/pub/CTAN/' => 1 + } + }, + 'Africa' => { + 'South Africa' => { + 'http://ftp.leg.uct.ac.za/pub/packages/ctan/' => 1 + } + } +}; diff --git a/Master/tlpkg/installer/mirrors.pl b/Master/tlpkg/installer/mirrors.pl deleted file mode 100644 index 9c9f9cf3ae1..00000000000 --- a/Master/tlpkg/installer/mirrors.pl +++ /dev/null @@ -1,141 +0,0 @@ -$mirrors = { - 'Europe' => { - 'UK' => { - 'http://ctan.sqsol.co.uk/' => 1, - 'http://anorien.csc.warwick.ac.uk/mirrors/CTAN/' => 1 - }, - 'Italy' => { - 'http://ftp.uniRoma2.it/TeX/' => 1, - 'http://ctan.mirror.garr.it/mirrors/CTAN/' => 1 - }, - 'Germany' => { - 'http://sunsite.informatik.rwth-aachen.de/ftp/pub/mirror/ctan/' => 1, - 'http://ftp.uni-erlangen.de/mirrors/CTAN/' => 1, - 'http://mirror.informatik.uni-mannheim.de/pub/mirrors/tex-archive/' => 1, - 'http://ctan.larsko.net/' => 1, - 'ftp://ftp.rrzn.uni-hannover.de/pub/mirror/tex-archive/' => 1 - }, - 'Serbia and Montenegro' => { - 'http://rpm.scl.rs/CTAN/' => 1 - }, - 'Lithuania' => { - 'http://ctan.vgtu.lt/' => 1 - }, - 'Norway' => { - 'http://ftp.fagskolen.gjovik.no/pub/tex-archive/' => 1, - 'http://ctan.uib.no/' => 1 - }, - 'Netherlands' => { - 'http://ftp.snt.utwente.nl/pub/software/tex/' => 1 - }, - 'France' => { - 'http://ctan.mines-albi.fr/' => 1, - 'http://distrib-coffee.ipsl.jussieu.fr/pub/mirrors/ctan/' => 1, - 'http://mirror.ibcp.fr/pub/CTAN/' => 1, - 'http://ftp.oleane.net/pub/CTAN/' => 1, - 'http://mirrors.ircam.fr/pub/CTAN/' => 1, - 'ftp://ftp.inria.fr/pub/TeX/CTAN/' => 1 - }, - 'Poland' => { - 'ftp://ftp.tpnet.pl/pub/CTAN/' => 1 - }, - 'Russia' => { - 'http://mirror.macomnet.net/pub/CTAN/' => 1, - 'http://ctan.uni-altai.ru/' => 1, - 'ftp://ftp.vsu.ru/pub/CTAN/' => 1 - }, - 'Slovenia' => { - 'http://ctan.ijs.si/tex-archive/' => 1 - }, - 'Portugal' => { - 'http://ftp.eq.uc.pt/software/TeX/' => 1 - }, - 'Denmark' => { - 'http://mirrors.dotsrc.org/ctan/' => 1 - }, - 'Spain' => { - 'http://ftp.udc.es/CTAN/' => 1 - }, - 'Greece' => { - 'http://ftp.cc.uoc.gr/mirrors/CTAN/' => 1 - }, - 'Czech Republic' => { - 'http://ftp.cvut.cz/tex-archive/' => 1 - } - }, - 'North America' => { - 'Canada' => { - 'http://mirror.its.dal.ca/ctan/' => 1, - 'http://mirror.csclub.uwaterloo.ca/CTAN/' => 1, - 'http://ctan.mirror.rafal.ca/' => 1 - }, - 'USA' => { - 'http://mirrors.ibiblio.org/pub/mirrors/CTAN/' => 1, - 'http://ctan.mackichan.com/' => 1, - 'http://mirror.math.ku.edu/tex-archive/' => 1, - 'http://mirror.unl.edu/ctan/' => 1, - 'http://mirrors.med.harvard.edu/ctan/' => 1, - 'http://mirror.hmc.edu/ctan/' => 1, - 'http://mirror.its.uidaho.edu/pub/tex-archive/' => 1 - } - }, - 'Australasia' => { - 'Australia' => { - 'http://ctan.unsw.edu.au/' => 1 - } - }, - 'South America' => { - 'Chile' => { - 'ftp://ftp.inf.utfsm.cl/pub/tex-archive/' => 1 - }, - 'Colombia' => { - 'http://www.laqee.unal.edu.co/tex-archive/' => 1 - }, - 'Brazil' => { - 'http://ceres.cisc.usp.br/CTAN/' => 1, - 'http://ctan.tche.br/' => 1, - 'http://linorg.usp.br/CTAN/' => 1, - 'http://mirror.softwarelivre.ufsc.br/pub/ctan/' => 1 - } - }, - 'Asia' => { - 'Taiwan' => { - 'http://ctan.cdpa.nsysu.edu.tw/' => 1, - 'http://shadow.ind.ntou.edu.tw/ctan/' => 1 - }, - 'Iran' => { - 'http://ctan.um.ac.ir/' => 1 - }, - 'Indonesia' => { - 'http://kebo.vlsm.org/CTAN/' => 1, - 'http://kambing.ui.ac.id/ctan/' => 1, - 'http://ctan.cermin.lipi.go.id/' => 1, - 'http://ftp.itb.ac.id/pub/CTAN/' => 1 - }, - 'Pakistan' => { - 'http://mirror-cybernet.lums.edu.pk/pub/ctan/' => 1 - }, - 'Malaysia' => { - 'ftp://mirror.upm.edu.my/ctan/' => 1 - }, - 'Korea' => { - 'http://ftp.kaist.ac.kr/pub/tex-archive/' => 1 - }, - 'China' => { - 'http://mirror.bjtu.edu.cn/CTAN/' => 1, - 'http://mirror.osqdu.org/CTAN/' => 1, - 'http://mirrors.xmu.edu.cn/CTAN/' => 1, - 'http://mirror.neu.edu.cn/CTAN/' => 1, - 'http://ftp.ctex.org/mirrors/CTAN/' => 1, - 'http://mirrors.ustc.edu.cn/CTAN/' => 1 - }, - 'Japan' => { - 'http://ftp.jaist.ac.jp/pub/CTAN/' => 1 - } - }, - 'Africa' => { - 'South Africa' => { - 'http://ftp.leg.uct.ac.za/pub/packages/ctan/' => 1 - } - } -}; -- cgit v1.2.3