You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/08/25 23:46:59 UTC
lucene-solr:branch_6x: LUCENE-7425: change from Perl to Python for
our script to polling mirrors during the release process
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 6242717a2 -> 07ebc563f
LUCENE-7425: change from Perl to Python for our script to polling mirrors during the release process
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/07ebc563
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/07ebc563
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/07ebc563
Branch: refs/heads/branch_6x
Commit: 07ebc563f0e4006a198723536662ac605635815d
Parents: 6242717
Author: Mike McCandless <mi...@apache.org>
Authored: Thu Aug 25 19:46:25 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Thu Aug 25 19:46:25 2016 -0400
----------------------------------------------------------------------
dev-tools/scripts/poll-mirrors.pl | 155 ---------------------------------
dev-tools/scripts/poll-mirrors.py | 153 ++++++++++++++++++++++++++++++++
2 files changed, 153 insertions(+), 155 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07ebc563/dev-tools/scripts/poll-mirrors.pl
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/poll-mirrors.pl b/dev-tools/scripts/poll-mirrors.pl
deleted file mode 100755
index cdbd3bb..0000000
--- a/dev-tools/scripts/poll-mirrors.pl
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/perl
-#
-# poll-mirrors.pl
-#
-# This script is designed to poll download sites after posting a release
-# and print out notice as each becomes available. The RM can use this
-# script to delay the release announcement until the release can be
-# downloaded.
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-use strict;
-use warnings;
-use Getopt::Long;
-use POSIX qw/strftime/;
-use LWP::UserAgent;
-
-my $rel_path;
-my $version;
-my $interval = 300;
-my $details = 0;
-
-my $result = GetOptions ("version=s" => \$version,
- "details!" => \$details,
- "path=s" => \$rel_path,
- "interval=i" => \$interval);
-
-my $usage = ""
- . "$0 -v version [ -i interval (seconds; default: 300) ] [ -details ]\n"
- . "$0 -p some/explicit/path [ -i interval (seconds; default: 300) ] [ -details ]\n"
- ;
-
-unless ($result) {
- print STDERR $usage;
- exit(1);
-}
-
-unless (defined($version) xor defined($rel_path)) {
- print STDERR "You must specify either -version or -path but not both\n$usage";
- exit(1);
-}
-
-my $label;
-my $apache_url_suffix;
-my $maven_url;
-
-if (defined($version)) {
- if ($version !~ /^\d+(?:\.\d+)+/) {
- print STDERR "You must specify the release version as a number.\n$usage";
- exit(1);
- }
- $label = $version;
- $apache_url_suffix = "lucene/java/$version/changes/Changes.html";
- $maven_url = "http://repo1.maven.org/maven2/org/apache/lucene/lucene-core/$version/lucene-core-$version.pom.asc";
-} else {
- # path based
- $apache_url_suffix = $label = $rel_path;
-}
-my $previously_selected = select STDOUT;
-$| = 1; # turn off buffering of STDOUT, so status is printed immediately
-select $previously_selected;
-
-my $apache_mirrors_list_url = "http://www.apache.org/mirrors/";
-
-my $agent = LWP::UserAgent->new();
-$agent->timeout(2);
-
-my $maven_available = defined($maven_url) ? 0 : -999;
-
-my @apache_mirrors = ();
-
-my $apache_mirrors_list_page = $agent->get($apache_mirrors_list_url)->decoded_content;
-if (defined($apache_mirrors_list_page)) {
- #<TR>
- # <TD ALIGN=RIGHT><A HREF="http://apache.dattatec.com/">apache.dattatec.com</A> <A HREF="http://apache.dattatec.com/">@</A></TD>
- #
- # <TD>http</TD>
- # <TD ALIGN=RIGHT>8 hours<BR><IMG BORDER=1 SRC="icons/mms14.gif" ALT=""></TD>
- # <TD ALIGN=RIGHT>5 hours<BR><IMG BORDER=1 SRC="icons/mms14.gif" ALT=""></TD>
- # <TD>ok</TD>
- #</TR>
- while ($apache_mirrors_list_page =~ m~<TR>(.*?)</TR>~gis) {
- my $mirror_entry = $1;
- next unless ($mirror_entry =~ m~<TD>\s*ok\s*</TD>\s*$~i); # skip mirrors with problems
- if ($mirror_entry =~ m~<A\s+HREF\s*=\s*"([^"]+)"\s*>~i) {
- my $mirror_url = $1;
- push @apache_mirrors, "${mirror_url}${apache_url_suffix}";
- }
- }
-} else {
- print STDERR "Error fetching Apache mirrors list $apache_mirrors_list_url";
- exit(1);
-}
-
-my $num_apache_mirrors = $#apache_mirrors;
-
-my $sleep_interval = 0;
-while (1) {
- print "\n", strftime('%d-%b-%Y %H:%M:%S', localtime);
- print "\nPolling $#apache_mirrors Apache Mirrors";
- print " and Maven Central" unless ($maven_available);
- print "...\n";
-
- my $start = time();
- $maven_available = (200 == $agent->head($maven_url)->code)
- unless ($maven_available);
- @apache_mirrors = &check_mirrors;
- my $stop = time();
- $sleep_interval = $interval - ($stop - $start);
-
- my $num_downloadable_apache_mirrors = $num_apache_mirrors - $#apache_mirrors;
- print "$label is ", ($maven_available ? "" : "not "),
- "downloadable from Maven Central.\n" if defined($maven_url);
- printf "$label is downloadable from %d/%d Apache Mirrors (%0.1f%%)\n",
- $num_downloadable_apache_mirrors, $num_apache_mirrors,
- ($num_downloadable_apache_mirrors*100/$num_apache_mirrors);
-
- last if ($maven_available && 0 == $#apache_mirrors);
-
- if ($sleep_interval > 0) {
- print "Sleeping for $sleep_interval seconds...\n";
- sleep($sleep_interval)
- }
-}
-
-sub check_mirrors {
- my @not_yet_downloadable_apache_mirrors;
- for my $mirror (@apache_mirrors) {
-
- ### print "\n$mirror\n";
- if (200 != $agent->head($mirror)->code) {
- push @not_yet_downloadable_apache_mirrors, $mirror;
- print $details ? "\nFAIL: $mirror\n" : "X";
- } else {
- print ".";
- }
- }
- print "\n";
- return @not_yet_downloadable_apache_mirrors;
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07ebc563/dev-tools/scripts/poll-mirrors.py
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/poll-mirrors.py b/dev-tools/scripts/poll-mirrors.py
new file mode 100644
index 0000000..1ff7e54
--- /dev/null
+++ b/dev-tools/scripts/poll-mirrors.py
@@ -0,0 +1,153 @@
+#!/usr/bin/python
+#
+# vim: softtabstop=2 shiftwidth=2 expandtab
+#
+# Python port of poll-mirrors.pl
+#
+# This script is designed to poll download sites after posting a release
+# and print out notice as each becomes available. The RM can use this
+# script to delay the release announcement until the release can be
+# downloaded.
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import argparse
+import datetime
+import ftplib
+import re
+import sys
+import time
+
+try:
+ from urllib.parse import urlparse
+except:
+ from urlparse import urlparse
+
+try:
+ import http.client as http
+except ImportError:
+ import httplib as http
+
+def p(s):
+ sys.stdout.write(s)
+ sys.stdout.flush()
+
+def mirror_contains_file(url):
+ url = urlparse(url)
+
+ if url.scheme == 'http':
+ return http_file_exists(url)
+ elif url.scheme == 'ftp':
+ return ftp_file_exists(url)
+
+def http_file_exists(url):
+ exists = False
+
+ try:
+ conn = http.HTTPConnection(url.netloc)
+ conn.request('HEAD', url.path)
+ response = conn.getresponse()
+
+ exists = response.status == 200
+ except:
+ pass
+
+ return exists
+
+def ftp_file_exists(url):
+ listing = []
+ try:
+ conn = ftplib.FTP(url.netloc)
+ conn.login()
+ listing = conn.nlst(url.path)
+ conn.quit()
+ except Exception as e:
+ pass
+
+ return len(listing) > 0
+
+def check_url_list(lst):
+ ret = []
+ for url in lst:
+ if mirror_contains_file(url):
+ p('.')
+ else:
+ p('X')
+ ret.append(url)
+
+ return ret
+
+parser = argparse.ArgumentParser(description='Checks that all Lucene mirrors contain a copy of a release')
+parser.add_argument('-version', '-v', help='Lucene version to check', required=True)
+parser.add_argument('-interval', '-i', help='seconds to wait to query again pending mirrors', type=int, default=300)
+args = parser.parse_args()
+
+try:
+ conn = http.HTTPConnection('www.apache.org')
+ conn.request('GET', '/mirrors/')
+ response = conn.getresponse()
+ html = response.read()
+except Exception as e:
+ p('Unable to fetch the Apache mirrors list!\n')
+ sys.exit(1)
+
+apache_path = 'lucene/java/{}/changes/Changes.html'.format(args.version);
+maven_url = 'http://repo1.maven.org/maven2/' \
+ 'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version)
+maven_available = False
+
+pending_mirrors = []
+for match in re.finditer('<TR>(.*?)</TR>', str(html), re.MULTILINE | re.IGNORECASE | re.DOTALL):
+ row = match.group(1)
+ if not '<TD>ok</TD>' in row:
+ # skip bad mirrors
+ continue
+
+ match = re.search('<A\s+HREF\s*=\s*"([^"]+)"\s*>', row, re.MULTILINE | re.IGNORECASE)
+ if match:
+ pending_mirrors.append(match.group(1) + apache_path)
+
+total_mirrors = len(pending_mirrors)
+
+while True:
+ p('\n' + str(datetime.datetime.now()))
+ p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors)))
+ if not maven_available:
+ p(' and Maven Central')
+ p('...\n')
+
+ if not maven_available:
+ maven_available = mirror_contains_file(maven_url)
+
+ start = time.time()
+ pending_mirrors = check_url_list(pending_mirrors)
+ stop = time.time()
+ remaining = args.interval - (stop - start)
+
+ available_mirrors = total_mirrors - len(pending_mirrors)
+
+ p('\n\n{} is{}downloadable from Maven Central\n'.format(args.version, maven_available and ' ' or ' not '))
+ p('{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'.format(args.version, available_mirrors,
+ total_mirrors,
+ available_mirrors * 100 / total_mirrors))
+ if len(pending_mirrors) == 0:
+ break
+
+ if remaining > 0:
+ p('Sleeping for {} seconds...\n'.format(remaining))
+ time.sleep(remaining)
+