You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ho...@apache.org on 2021/11/10 16:51:22 UTC
[solr-operator] branch main updated: Remove mirrors from release
wizard (#371)
This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-operator.git
The following commit(s) were added to refs/heads/main by this push:
new f3075c0 Remove mirrors from release wizard (#371)
f3075c0 is described below
commit f3075c0e9458f07af8e8ef8d8051ba8435bf0984
Author: Houston Putman <ho...@apache.org>
AuthorDate: Wed Nov 10 11:51:15 2021 -0500
Remove mirrors from release wizard (#371)
---
hack/release/wizard/poll-mirrors.py | 165 ---------------------------------
hack/release/wizard/releaseWizard.py | 39 ++++----
hack/release/wizard/releaseWizard.yaml | 70 ++++----------
hack/release/wizard/scriptutil.py | 2 +-
4 files changed, 43 insertions(+), 233 deletions(-)
diff --git a/hack/release/wizard/poll-mirrors.py b/hack/release/wizard/poll-mirrors.py
deleted file mode 100755
index f7011e3..0000000
--- a/hack/release/wizard/poll-mirrors.py
+++ /dev/null
@@ -1,165 +0,0 @@
-#!/usr/bin/env python3
-#
-# vim: softtabstop=2 shiftwidth=2 expandtab
-#
-# Python port of poll-mirrors.pl
-#
-# This script is designed to poll download sites after posting a release
-# and print out notice as each becomes available. The RM can use this
-# script to delay the release announcement until the release can be
-# downloaded.
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import argparse
-import datetime
-import ftplib
-import re
-import sys
-import time
-
-from urllib.parse import urlparse
-from multiprocessing import Pool
-import http.client as http
-
-
-def p(s):
- sys.stdout.write(s)
- sys.stdout.flush()
-
-
-def mirror_contains_file(url):
- url = urlparse(url)
-
- if url.scheme == 'https':
- return https_file_exists(url)
- elif url.scheme == 'http':
- return http_file_exists(url)
- elif url.scheme == 'ftp':
- return ftp_file_exists(url)
-
-
-def http_file_exists(url):
- exists = False
-
- try:
- conn = http.HTTPConnection(url.netloc)
- conn.request('HEAD', url.path)
- response = conn.getresponse()
-
- exists = response.status == 200
- except:
- pass
-
- return exists
-
-def https_file_exists(url):
- exists = False
-
- try:
- conn = http.HTTPSConnection(url.netloc)
- conn.request('HEAD', url.path)
- response = conn.getresponse()
- exists = response.status == 200
- except:
- pass
-
- return exists
-
-def ftp_file_exists(url):
- listing = []
- try:
- conn = ftplib.FTP(url.netloc)
- conn.login()
- listing = conn.nlst(url.path)
- conn.quit()
- except Exception as e:
- pass
-
- return len(listing) > 0
-
-
-def check_mirror(url):
- if mirror_contains_file(url):
- p('.')
- return None
- else:
- # p('\nFAIL: ' + url + '\n')
- p('X')
- return url
-
-if __name__ == '__main__':
- desc = 'Periodically checks that all Solr mirrors contain either a copy of a release or a specified path'
- parser = argparse.ArgumentParser(description=desc)
- parser.add_argument('-version', '-v', help='Solr Operator version to check')
- parser.add_argument('-path', '-p', help='instead of a versioned release, check for some/explicit/path')
- parser.add_argument('-interval', '-i', help='seconds to wait before re-querying mirrors', type=int, default=300)
- parser.add_argument('-once', '-o', help='run only once', action='store_true', default=False)
- args = parser.parse_args()
-
- if (args.version is None and args.path is None) \
- or (args.version is not None and args.path is not None):
- p('You must specify either -version or -path but not both!\n')
- sys.exit(1)
-
- try:
- conn = http.HTTPSConnection('www.apache.org')
- conn.request('GET', '/mirrors/')
- response = conn.getresponse()
- html = response.read()
- except Exception as e:
- p('Unable to fetch the Apache mirrors list!\n')
- sys.exit(1)
-
- mirror_path = args.path if args.path is not None else 'solr/solr-operator/{}/solr-operator-{}.tgz'.format(args.version, args.version)
-
- pending_mirrors = []
- for match in re.finditer('<TR>(.*?)</TR>', str(html), re.MULTILINE | re.IGNORECASE | re.DOTALL):
- row = match.group(1)
- if not '<TD>ok</TD>' in row:
- # skip bad mirrors
- continue
-
- match = re.search('<A\s+HREF\s*=\s*"([^"]+)"\s*>', row, re.MULTILINE | re.IGNORECASE)
- if match:
- pending_mirrors.append(match.group(1) + mirror_path)
-
- total_mirrors = len(pending_mirrors)
-
- label = args.version if args.version is not None else args.path
- while True:
- p('\n{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))
- p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors)))
- p('...\n')
-
- start = time.time()
- with Pool(processes=5) as pool:
- pending_mirrors = list(filter(lambda x: x is not None, pool.map(check_mirror, pending_mirrors)))
- stop = time.time()
- remaining = args.interval - (stop - start)
-
- available_mirrors = total_mirrors - len(pending_mirrors)
-
- p('\n{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'
- .format(label, available_mirrors, total_mirrors, available_mirrors * 100 / (1 if total_mirrors == 0 else total_mirrors) ))
- if len(pending_mirrors) == 0 or args.once == True:
- break
-
- if remaining > 0:
- p('Sleeping for {:d} seconds...\n'.format(int(remaining + 0.5)))
- time.sleep(remaining)
-
diff --git a/hack/release/wizard/releaseWizard.py b/hack/release/wizard/releaseWizard.py
index 765163d..62ee37b 100755
--- a/hack/release/wizard/releaseWizard.py
+++ b/hack/release/wizard/releaseWizard.py
@@ -116,8 +116,8 @@ def expand_jinja(text, vars=None):
'latest_version': state.get_latest_version(),
'latest_lts_version': state.get_latest_lts_version(),
'main_version': state.get_main_version(),
- 'mirrored_versions': state.get_mirrored_versions(),
- 'mirrored_versions_to_delete': state.get_mirrored_versions_to_delete(),
+ 'released_versions': state.get_released_versions(),
+ 'released_versions_to_delete': state.get_released_versions_to_delete(),
'home': os.path.expanduser("~")
})
global_vars.update(state.get_todo_states())
@@ -285,7 +285,7 @@ class ReleaseState:
self.rc_number = 1
self.start_date = unix_time_millis(datetime.utcnow())
self.script_branch = run("git rev-parse --abbrev-ref HEAD").strip()
- self.mirrored_versions = None
+ self.released_versions = None
try:
self.script_branch_type = scriptutil.find_branch_type()
except:
@@ -341,11 +341,8 @@ class ReleaseState:
def get_latest_version(self):
if self.latest_version is None:
- #TODO: Remove when first release is made
- #versions = self.get_mirrored_versions()
- #latest = versions[0]
- versions = []
- latest = "v0.2.8"
+ versions = self.get_released_versions()
+ latest = versions[0]
for ver in versions:
if Version.parse(ver).gt(Version.parse(latest)):
latest = ver
@@ -353,18 +350,18 @@ class ReleaseState:
self.save()
return state.latest_version
- def get_mirrored_versions(self):
- if state.mirrored_versions is None:
+ def get_released_versions(self):
+ if state.released_versions is None:
releases_str = load("https://projects.apache.org/json/foundation/releases.json", "utf-8")
releases = json.loads(releases_str)
- state.mirrored_versions = []
+ state.released_versions = []
if 'solr' in releases.keys():
releases = releases['solr']
- state.mirrored_versions = [ r for r in list(map(lambda y: y[14:], filter(lambda x: x.startswith('solr-operator-v'), list(releases.keys())))) ]
- return state.mirrored_versions
+ state.released_versions = [ r for r in list(map(lambda y: y[14:], filter(lambda x: x.startswith('solr-operator-v'), list(releases.keys())))) ]
+ return state.released_versions
- def get_mirrored_versions_to_delete(self):
- versions = self.get_mirrored_versions()
+ def get_released_versions_to_delete(self):
+ versions = self.get_released_versions()
to_keep = versions
if state.release_type == 'major':
to_keep = [self.release_version, self.get_latest_version()]
@@ -385,7 +382,7 @@ class ReleaseState:
def get_latest_lts_version(self):
return None
- versions = self.get_mirrored_versions()
+ versions = self.get_released_versions()
latest = self.get_latest_version()
lts_prefix = "%s." % (Version.parse(latest).major - 1)
lts_versions = list(filter(lambda x: x.startswith(lts_prefix), versions))
@@ -1993,6 +1990,16 @@ def prepare_announce(todo):
print("Draft already exist, not re-generating")
return True
+def check_artifacts_available(todo):
+ try:
+ cdnUrl = expand_jinja("https://dlcdn.apache.org/solr/solr-operator/{{ release_version }}/solr-operator-{{ release_version }}.tgz.asc")
+ load(cdnUrl)
+ print("Found %s" % cdnUrl)
+ except Exception as e:
+ print("Could not fetch %s (%s)" % (cdnUrl, e))
+ return False
+
+ return True
def load_lines(file, from_line=0):
if os.path.exists(file):
diff --git a/hack/release/wizard/releaseWizard.yaml b/hack/release/wizard/releaseWizard.yaml
index 19e8bb0..8429414 100644
--- a/hack/release/wizard/releaseWizard.yaml
+++ b/hack/release/wizard/releaseWizard.yaml
@@ -131,11 +131,6 @@ templates:
{% for line in load_lines(solr_operator_news_file, 4) -%}
{{ line }}
{%- endfor %}
-
-
- Note: The Apache Software Foundation uses an extensive mirroring network for
- distributing releases. It is possible that the mirror you are using may not have
- replicated the release yet. If that is the case, please try another mirror.
# TODOs belong to groups for easy navigation in menus. Todo objects may contain asciidoc
# descriptions, a number of commands to execute, some links to display, user input to gather
# etc. Here is the documentation of each type of object. For further details, please consult
@@ -997,8 +992,8 @@ groups:
- https://www.apache.org/foundation/voting.html
- !TodoGroup
id: publish
- title: Publishing to the ASF Mirrors
- description: Once the vote has passed, the release may be published to the ASF Mirrors.
+ title: Publishing to the ASF
+ description: Once the vote has passed, the release may be published to the ASF.
todos:
- !Todo
id: tag_release
@@ -1149,16 +1144,14 @@ groups:
cmd: docker buildx build --pull --push --platform "linux/amd64,linux/arm64" --build-arg GIT_SHA={{ build_rc.git_sha | default("<git_sha>", True) }} . --tag apache/solr-operator:{{ release_version }} -f ./build/Dockerfile
logfile: docker_upload.log
- !Todo
- id: check_mirroring
- title: Check state of mirroring so far
- description: Mark this as complete once a good spread is confirmed
- commands: !Commands
- root_folder: '{{ git_checkout_folder }}'
- commands_text: Run this script to check the number and percentage of mirrors that have the release
- commands:
- - !Command
- cmd: python3 -u hack/release/wizard/poll-mirrors.py -version {{ release_version }} -o
- live: true
+ id: check_distribution_directory
+ depends: svn_release_mv
+ title: Check that artifacts are available
+ function: check_artifacts_available
+ description: |
+ The task will attempt to fetch https://dlcdn.apache.org/solr/solr-operator/{{ release_version }}/solr-operator-{{ release_version }}.tgz.asc
+ to validate the ASF repo.
+ If the check fails, please re-run the task, until it succeeds.
- !Todo
id: check_artifact_hub
title: Check state of ArtifactHub entries
@@ -1464,15 +1457,6 @@ groups:
title: Announce the Solr Operator release (announce@a.o)
description: |
(( template=announce_solr_operator_sign_mail ))
-# - !Todo
-# id: add_to_wikipedia
-# title: Add the new version to Wikipedia
-# description: |
-# Go to Wikipedia and edit the page to include the new release.
-# Major versions should have a small new paragraph under 'History'.
-# If you know other languages than English, edit those as well.
-# links:
-# - https://en.wikipedia.org/wiki/Apache_Solr
- !Todo
id: add_to_apache_reporter
title: Add the new version to the Apache Release Reporter
@@ -1514,22 +1498,6 @@ groups:
links:
- https://github.com/apache/solr-operator/milestones?state=open
-# - !Todo
-# id: jira_clear_security
-# title: Clear Security Level of Public Solr JIRA Issues
-# description: |-
-# ASF JIRA has a deficiency in which issues that have a security level of "Public" are nonetheless not searchable.
-# As a maintenance task, we'll clear the security flag for all public Solr JIRAs, even if it is not a task directly
-# related to the release:
-#
-# . Open in browser: https://issues.apache.org/jira/issues/?jql=project+=+SOLR+AND+level+=+%22Public%22
-# . In the `Tools` menu, start a bulk change, select all issues and click `Next`
-# . Select operation="Edit issues" and click `Next`
-# . Click the checkbox next to `Change security level` and choose `None` in the dropdown.
-# . On the bottom of the form, uncheck the box that says `Send mail for this update`
-# . Click `Next`, review the changes and click `Confirm`
-# links:
-# - https://issues.apache.org/jira/issues/?jql=project+=+SOLR+AND+level+=+%22Public%22
- !Todo
id: new_github_milestone_version_bugfix_minor
title: Add a new milestone in Github for the next releases
@@ -1543,32 +1511,32 @@ groups:
links:
- https://github.com/apache/solr-operator/milestones
- !Todo
- id: stop_mirroring
- title: Stop mirroring old releases
+ id: archive_old_releases
+ title: Move old releases to archive
description: |
- Shortly after new releases are first mirrored, they are automatically copied to the archives.
+ Shortly after new releases are uploaded, they are automatically copied to the archives.
Only the latest point release from each active branch should be kept under the Solr PMC
svnpubsub areas `dist/releases/solr/solr-operator`. Older releases can be
safely deleted, since they are already backed up in the archives.
- Currenlty these versions are in the mirrors:
+ Currently these versions are in the ASF releases:
- *{{ mirrored_versions|join(', ') }}*
+ *{{ released_versions|join(', ') }}*
The commands below will remove old versions automatically. If this suggestion is wrong,
please do *not* execute the commands automatically, but edit the command and run manually.
- Versions to be deleted from the mirrors are:
+ Versions to be deleted from the distribution directory repo are:
- *{{ mirrored_versions_to_delete|join(', ') }}*
+ *{{ released_versions_to_delete|join(', ') }}*
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
commands_text: |
- Run these commands to delete proposed versions from mirrors.
+ Run these commands to delete proposed versions from the distribution directory repo.
WARNING: Validate that the proposal is correct!
commands:
- !Command
cmd: |
- svn rm -m "Stop mirroring old Solr Operator releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/solr/solr-operator/{{ ver }}{% endfor %}
+ svn rm -m "Move old Solr Operator releases to archive"{% for ver in released_versions_to_delete %} https://dist.apache.org/repos/dist/release/solr/solr-operator/{{ ver }}{% endfor %}
logfile: svn-rm-solr-operator.log
diff --git a/hack/release/wizard/scriptutil.py b/hack/release/wizard/scriptutil.py
index d74f681..ac39d13 100644
--- a/hack/release/wizard/scriptutil.py
+++ b/hack/release/wizard/scriptutil.py
@@ -179,7 +179,7 @@ def attemptDownload(urlString, fileName):
if not success:
os.remove(fileName)
-version_prop_re = re.compile(r'Version\s*string\s*=\s*([\'"])(.*)\1')
+version_prop_re = re.compile(r'Version\s*=\s*([\'"])(.*)\1')
def find_current_version():
script_path = os.path.dirname(os.path.realpath(__file__))
top_level_dir = os.path.join(os.path.abspath("%s/" % script_path), os.path.pardir, os.path.pardir, os.path.pardir)