You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ja...@apache.org on 2021/10/17 17:55:05 UTC
[solr] branch main updated: SOLR-15698 No longer check for release
status on mirrors (#348)
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 3437977 SOLR-15698 No longer check for release status on mirrors (#348)
3437977 is described below
commit 3437977069fa9f4ae18febf3822952772f658485
Author: Jan Høydahl <ja...@users.noreply.github.com>
AuthorDate: Sun Oct 17 19:55:01 2021 +0200
SOLR-15698 No longer check for release status on mirrors (#348)
Port from lucene-solr/branch_8x to solr/main
---
dev-tools/scripts/README.md | 19 ----
dev-tools/scripts/poll-mirrors.py | 175 -----------------------------------
dev-tools/scripts/releaseWizard.py | 19 ++++
dev-tools/scripts/releaseWizard.yaml | 48 +++++-----
4 files changed, 43 insertions(+), 218 deletions(-)
diff --git a/dev-tools/scripts/README.md b/dev-tools/scripts/README.md
index 8eb3169..f2024b7 100644
--- a/dev-tools/scripts/README.md
+++ b/dev-tools/scripts/README.md
@@ -144,25 +144,6 @@ and prints a regular expression that will match all of them
--no-git Do not run "git" at all
--iters N Number of iterations per test suite (default: 5)
-### poll-mirrors.py
-
- usage: poll-mirrors.py [-h] [-version VERSION] [-path PATH]
- [-interval INTERVAL] [-details] [-once]
-
- Periodically checks that all Lucene/Solr mirrors contain either a copy of a
- release or a specified path
-
- optional arguments:
- -h, --help show this help message and exit
- -version VERSION, -v VERSION
- Lucene/Solr version to check
- -path PATH, -p PATH instead of a versioned release, check for
- some/explicit/path
- -interval INTERVAL, -i INTERVAL
- seconds to wait before re-querying mirrors
- -details, -d print missing mirror URLs
- -once, -o run only once
-
### githubPRs.py
usage: githubPRs.py [-h] [--json] [--token TOKEN]
diff --git a/dev-tools/scripts/poll-mirrors.py b/dev-tools/scripts/poll-mirrors.py
deleted file mode 100755
index 86fb7ec..0000000
--- a/dev-tools/scripts/poll-mirrors.py
+++ /dev/null
@@ -1,175 +0,0 @@
-#!/usr/bin/env python3
-#
-# vim: softtabstop=2 shiftwidth=2 expandtab
-#
-# Python port of poll-mirrors.pl
-#
-# This script is designed to poll download sites after posting a release
-# and print out notice as each becomes available. The RM can use this
-# script to delay the release announcement until the release can be
-# downloaded.
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import argparse
-import datetime
-import ftplib
-import re
-import sys
-import time
-
-from urllib.parse import urlparse
-from multiprocessing import Pool
-import http.client as http
-
-
-def p(s):
- sys.stdout.write(s)
- sys.stdout.flush()
-
-
-def mirror_contains_file(url):
- url = urlparse(url)
-
- if url.scheme == 'https':
- return https_file_exists(url)
- elif url.scheme == 'http':
- return http_file_exists(url)
- elif url.scheme == 'ftp':
- return ftp_file_exists(url)
-
-
-def http_file_exists(url):
- exists = False
-
- try:
- conn = http.HTTPConnection(url.netloc)
- conn.request('HEAD', url.path)
- response = conn.getresponse()
-
- exists = response.status == 200
- except:
- pass
-
- return exists
-
-def https_file_exists(url):
- exists = False
-
- try:
- conn = http.HTTPSConnection(url.netloc)
- conn.request('HEAD', url.path)
- response = conn.getresponse()
- exists = response.status == 200
- except:
- pass
-
- return exists
-
-def ftp_file_exists(url):
- listing = []
- try:
- conn = ftplib.FTP(url.netloc)
- conn.login()
- listing = conn.nlst(url.path)
- conn.quit()
- except Exception as e:
- pass
-
- return len(listing) > 0
-
-
-def check_mirror(url):
- if mirror_contains_file(url):
- p('.')
- return None
- else:
- p('\nFAIL: ' + url + '\n' if args.details else 'X')
- return url
-
-if __name__ == '__main__':
- desc = 'Periodically checks that all Lucene/Solr mirrors contain either a copy of a release or a specified path'
- parser = argparse.ArgumentParser(description=desc)
- parser.add_argument('-version', '-v', help='Lucene/Solr version to check')
- parser.add_argument('-path', '-p', help='instead of a versioned release, check for some/explicit/path')
- parser.add_argument('-interval', '-i', help='seconds to wait before re-querying mirrors', type=int, default=300)
- parser.add_argument('-details', '-d', help='print missing mirror URLs', action='store_true', default=False)
- parser.add_argument('-once', '-o', help='run only once', action='store_true', default=False)
- args = parser.parse_args()
-
- if (args.version is None and args.path is None) \
- or (args.version is not None and args.path is not None):
- p('You must specify either -version or -path but not both!\n')
- sys.exit(1)
-
- try:
- conn = http.HTTPConnection('www.apache.org')
- conn.request('GET', '/mirrors/')
- response = conn.getresponse()
- html = response.read()
- except Exception as e:
- p('Unable to fetch the Apache mirrors list!\n')
- sys.exit(1)
-
- mirror_path = args.path if args.path is not None else 'lucene/java/{}/changes/Changes.html'.format(args.version)
- maven_url = None if args.version is None else 'https://repo1.maven.org/maven2/' \
- 'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version)
- maven_available = False
-
- pending_mirrors = []
- for match in re.finditer('<TR>(.*?)</TR>', str(html), re.MULTILINE | re.IGNORECASE | re.DOTALL):
- row = match.group(1)
- if not '<TD>ok</TD>' in row:
- # skip bad mirrors
- continue
-
- match = re.search('<A\s+HREF\s*=\s*"([^"]+)"\s*>', row, re.MULTILINE | re.IGNORECASE)
- if match:
- pending_mirrors.append(match.group(1) + mirror_path)
-
- total_mirrors = len(pending_mirrors)
-
- label = args.version if args.version is not None else args.path
- while True:
- p('\n{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))
- p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors)))
- if maven_url is not None and not maven_available:
- p(' and Maven Central')
- p('...\n')
-
- if maven_url is not None and not maven_available:
- maven_available = mirror_contains_file(maven_url)
-
- start = time.time()
- with Pool(processes=5) as pool:
- pending_mirrors = list(filter(lambda x: x is not None, pool.map(check_mirror, pending_mirrors)))
- stop = time.time()
- remaining = args.interval - (stop - start)
-
- available_mirrors = total_mirrors - len(pending_mirrors)
-
- if maven_url is not None:
- p('\n\n{} is{}downloadable from Maven Central'.format(label, ' ' if maven_available else ' not '))
- p('\n{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'
- .format(label, available_mirrors, total_mirrors, available_mirrors * 100 / total_mirrors))
- if len(pending_mirrors) == 0 or args.once == True:
- break
-
- if remaining > 0:
- p('Sleeping for {:d} seconds...\n'.format(int(remaining + 0.5)))
- time.sleep(remaining)
-
diff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.py
index b10fada..c05a542 100755
--- a/dev-tools/scripts/releaseWizard.py
+++ b/dev-tools/scripts/releaseWizard.py
@@ -1973,6 +1973,25 @@ def prepare_announce_solr(todo):
return True
+def check_artifacts_available(todo):
+ try:
+ cdnUrl = expand_jinja("https://dlcdn.apache.org/solr/{{ release_version }}/solr-{{ release_version }}-src.tgz.asc")
+ load(cdnUrl)
+ print("Found %s" % cdnUrl)
+ except Exception as e:
+ print("Could not fetch %s (%s)" % (cdnUrl, e))
+ return False
+
+ try:
+ mavenUrl = expand_jinja("https://repo1.maven.org/maven2/org/apache/solr/solr-core/{{ release_version }}/solr-core-{{ release_version }}.jar.asc")
+ load(mavenUrl)
+ print("Found %s" % mavenUrl)
+ except Exception as e:
+ print("Could not fetch %s (%s)" % (mavenUrl, e))
+ return False
+
+ return True
+
def set_java_home(version):
os.environ['JAVA_HOME'] = state.get_java_home_for_version(version)
os.environ['JAVACMD'] = state.get_java_cmd_for_version(version)
diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml
index 4b6d31b..91436c2 100644
--- a/dev-tools/scripts/releaseWizard.yaml
+++ b/dev-tools/scripts/releaseWizard.yaml
@@ -907,8 +907,8 @@ groups:
- https://www.apache.org/foundation/voting.html
- !TodoGroup
id: publish
- title: Publishing to the ASF Mirrors
- description: Once the vote has passed, the release may be published to the ASF Mirrors and to Maven Central.
+ title: Publishing to the ASF Distribution Directory
+ description: Once the vote has passed, the release may be published to the ASF Distribution Directory and to Maven Central.
todos:
- !Todo
id: tag_release
@@ -937,7 +937,7 @@ groups:
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
confirm_each_command: false
- commands_text: This will remove maven artifacts so they do not end up in the mirrors
+ commands_text: This will remove maven artifacts so they do not end up in the Distribution Directory
commands:
- !Command
cmd: svn rm -m "Delete the lucene maven artifacts" {{ dist_stage_url }}/lucene/maven
@@ -1010,22 +1010,22 @@ groups:
. Wait and keep clicking refresh until the "Release" button becomes available
. Click on the "Release" button above the repository list, then enter a description when prompted, e.g. "Lucene/Solr {{ release_version }}".
- Maven central should show the release after a short while, but you need to
- wait 24 hours to give the Apache mirrors a chance to copy the new release.
+ Maven central should show the release after a short while
links:
- https://wiki.apache.org/lucene-java/PublishMavenArtifacts
- https://repository.apache.org/index.html
- !Todo
- id: check_mirroring
- title: Check state of mirroring so far
- description: Mark this as complete once a good spread is confirmed
- commands: !Commands
- root_folder: '{{ git_checkout_folder }}'
- commands_text: Run this script to check the number and percentage of mirrors (and Maven Central) that have the release
- commands:
- - !Command
- cmd: python3 -u dev-tools/scripts/poll-mirrors.py -version {{ release_version }} -o
- live: true
+ id: check_distribution_directory
+ depends: publish_maven
+ title: Check that artifacts are available
+ function: check_artifacts_available
+ description: |
+ The task will attempt to fetch https://dlcdn.apache.org/solr/{{ release_version }}/solr-{{ release_version }}-src.tgz.asc
+ to validate ASF repo, and https://repo1.maven.org/maven2/org/apache/solr/solr-core/{{ release_version }}/solr-core-{{ release_version }}.jar.asc
+ to validate Maven repo.
+
+ If the check fails, please re-run the task, until it succeeds.
+
- !TodoGroup
id: website
title: Update the website
@@ -1607,36 +1607,36 @@ groups:
- https://issues.apache.org/jira/plugins/servlet/project-config/LUCENE/versions
- https://issues.apache.org/jira/plugins/servlet/project-config/SOLR/versions
- !Todo
- id: stop_mirroring
- title: Stop mirroring old releases
+ id: stop_promoting_old
+ title: Stop promoting old releases
description: |
- Shortly after new releases are first mirrored, they are automatically copied to the archives.
+ Shortly after new releases are first published, they are automatically copied to the archives.
Only the latest point release from each active branch should be kept under the Lucene PMC
svnpubsub areas `dist/releases/lucene/` and `dist/releases/solr/`. Older releases can be
safely deleted, since they are already backed up in the archives.
- Currenlty these versions are in the mirrors:
+ Currenlty these versions exist in the distribution directory:
*{{ mirrored_versions|join(', ') }}*
The commands below will remove old versions automatically. If this suggestion is wrong,
please do *not* execute the commands automatically, but edit the command and run manually.
- Versions to be deleted from the mirrors are:
+ Versions to be deleted from the distribution directory are:
*{{ mirrored_versions_to_delete|join(', ') }}*
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
commands_text: |
- Run these commands to delete proposed versions from mirrors.
+ Run these commands to delete proposed versions from distribution directory.
WARNING: Validate that the proposal is correct!
commands:
- !Command
cmd: |
- svn rm -m "Stop mirroring old Lucene releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/java/{{ ver }}{% endfor %}
+ svn rm -m "Stop publishing old Lucene releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/java/{{ ver }}{% endfor %}
logfile: svn-rm-lucene.log
- !Command
cmd: |
- svn rm -m "Stop mirroring old Solr releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/solr/{{ ver }}{% endfor %}
- logfile: svn-rm-lucene.log
+ svn rm -m "Stop publishing old Solr releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/solr/{{ ver }}{% endfor %}
+ logfile: svn-rm-solr.log