You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ja...@apache.org on 2022/01/14 15:49:29 UTC
[solr] branch main updated: SOLR-15852 Update dev-tools/scripts for 9.0 (#497)
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new ca08f37 SOLR-15852 Update dev-tools/scripts for 9.0 (#497)
ca08f37 is described below
commit ca08f3797cc729d5aff9765d9e254ed1fb5201fb
Author: Jan Høydahl <ja...@users.noreply.github.com>
AuthorDate: Fri Jan 14 16:49:23 2022 +0100
SOLR-15852 Update dev-tools/scripts for 9.0 (#497)
---
dev-tools/scripts/buildAndPushRelease.py | 259 +++++---
dev-tools/scripts/releaseWizard.py | 134 ++--
dev-tools/scripts/releaseWizard.yaml | 328 +++-------
dev-tools/scripts/requirements.txt | 2 +-
dev-tools/scripts/scriptutil.py | 1 -
dev-tools/scripts/smokeTestRelease.py | 1044 +++++++++++-------------------
solr/CHANGES.txt | 2 +
7 files changed, 690 insertions(+), 1080 deletions(-)
diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py
index 21861cb..22ca14e 100755
--- a/dev-tools/scripts/buildAndPushRelease.py
+++ b/dev-tools/scripts/buildAndPushRelease.py
@@ -30,6 +30,7 @@ import xml.etree.ElementTree as ET
import scriptutil
LOG = '/tmp/release.log'
+dev_mode = False
def log(msg):
f = open(LOG, mode='ab')
@@ -43,6 +44,7 @@ def run(command):
print(msg)
raise RuntimeError(msg)
+
def runAndSendGPGPassword(command, password):
p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
f = open(LOG, 'ab')
@@ -77,30 +79,30 @@ def load(urlString, encoding="utf-8"):
return content
def getGitRev():
- status = os.popen('git status').read().strip()
- if 'nothing to commit, working directory clean' not in status and 'nothing to commit, working tree clean' not in status:
- raise RuntimeError('git clone is dirty:\n\n%s' % status)
- branch = os.popen('git rev-parse --abbrev-ref HEAD').read().strip()
- command = 'git log origin/%s..' % branch
- p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = p.communicate()
- if len(stdout.strip()) > 0:
- raise RuntimeError('There are unpushed commits - "%s" output is:\n\n%s' % (command, stdout.decode('utf-8')))
- if len(stderr.strip()) > 0:
- raise RuntimeError('Command "%s" failed:\n\n%s' % (command, stderr.decode('utf-8')))
-
- print(' git clone is clean')
+ if not dev_mode:
+ status = os.popen('git status').read().strip()
+ if 'nothing to commit, working directory clean' not in status and 'nothing to commit, working tree clean' not in status:
+ raise RuntimeError('git clone is dirty:\n\n%s' % status)
+ if 'Your branch is ahead of' in status:
+ raise RuntimeError('Your local branch is ahead of the remote? git status says:\n%s' % status)
+ print(' git clone is clean')
+ else:
+ print(' Ignoring dirty git clone due to dev-mode')
return os.popen('git rev-parse HEAD').read().strip()
-def prepare(root, version, gpgKeyID, gpgPassword):
+
+def prepare(root, version, gpg_key_id, gpg_password, gpg_home=None, sign_gradle=False):
print()
print('Prepare release...')
if os.path.exists(LOG):
os.remove(LOG)
- os.chdir(root)
- print(' git pull...')
- run('git pull')
+ if not dev_mode:
+ os.chdir(root)
+ print(' git pull...')
+ run('git pull')
+ else:
+ print(' Development mode, not running git pull')
rev = getGitRev()
print(' git rev: %s' % rev)
@@ -109,24 +111,39 @@ def prepare(root, version, gpgKeyID, gpgPassword):
print(' Check DOAP files')
checkDOAPfiles(version)
- print(' ./gradlew -Dtests.badapples=false clean check')
- run('./gradlew -Dtests.badapples=false clean check')
+ if not dev_mode:
+ print(' ./gradlew --no-daemon -Dtests.badapples=false clean check')
+ run('./gradlew --no-daemon -Dtests.badapples=false clean check')
+ else:
+ print(' skipping precommit check due to dev-mode')
- open('rev.txt', mode='wb').write(rev.encode('UTF-8'))
-
print(' prepare-release')
- cmd = './gradlew -Dversion.release=%s clean assembleDist' % version
- if gpgKeyID is not None:
- # TODO sign
- # cmd += ' -Psigning.keyId=%s publishSignedPublicationToMavenLocal' % gpgKeyID
- pass
- cmd += ' mavenToLocalFolder'
-
- if gpgPassword is not None:
- runAndSendGPGPassword(cmd, gpgPassword)
+ cmd = './gradlew --no-daemon assembleRelease' \
+ ' -Dversion.release=%s' % version
+ if dev_mode:
+ cmd += ' -Pvalidation.git.failOnModified=false'
+ if gpg_key_id is not None:
+ cmd += ' -Psign --max-workers 2'
+ if sign_gradle:
+ print(" Signing method is gradle java-plugin")
+ cmd += ' -Psigning.keyId="%s"' % gpg_key_id
+ if gpg_home is not None:
+ cmd += ' -Psigning.secretKeyRingFile="%s"' % os.path.join(gpg_home, 'secring.gpg')
+ if gpg_password is not None:
+ # Pass gpg passphrase as env.var to gradle rather than as plaintext argument
+ os.environ['ORG_GRADLE_PROJECT_signingPassword'] = gpg_password
+ else:
+ print(" Signing method is gpg tool")
+ cmd += ' -PuseGpg -Psigning.gnupg.keyName="%s"' % gpg_key_id
+ if gpg_home is not None:
+ cmd += ' -Psigning.gnupg.homeDir="%s"' % gpg_home
+
+ print(" Running: %s" % cmd)
+ if gpg_password is not None:
+ runAndSendGPGPassword(cmd, gpg_password)
else:
run(cmd)
-
+
print(' done!')
print()
return rev
@@ -135,35 +152,38 @@ reVersion1 = re.compile(r'\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<', re.IGNORECAS
reVersion2 = re.compile(r'-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<', re.IGNORECASE)
reDoapRevision = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?', re.IGNORECASE)
def checkDOAPfiles(version):
- # In Lucene and Solr DOAP files, verify presence of all releases less than the one being produced.
+ # In Solr DOAP file, verify presence of all releases less than the one being produced.
errorMessages = []
- for product in 'lucene', 'solr':
- url = 'https://archive.apache.org/dist/lucene/%s' % ('java' if product == 'lucene' else product)
- distpage = load(url)
- releases = set()
- for regex in reVersion1, reVersion2:
- for tup in regex.findall(distpage):
- if tup[0] in ('1', '2'): # Ignore 1.X and 2.X releases
- continue
- releases.add(normalizeVersion(tup))
- doapNS = '{http://usefulinc.com/ns/doap#}'
- xpathRevision = '{0}Project/{0}release/{0}Version/{0}revision'.format(doapNS)
- doapFile = "dev-tools/doap/%s.rdf" % product
- treeRoot = ET.parse(doapFile).getroot()
- doapRevisions = set()
- for revision in treeRoot.findall(xpathRevision):
- match = reDoapRevision.match(revision.text)
- if (match is not None):
- if (match.group(1) not in ('0', '1', '2')): # Ignore 0.X, 1.X and 2.X revisions
- doapRevisions.add(normalizeVersion(match.groups()))
- else:
- errorMessages.append('ERROR: Failed to parse revision: %s in %s' % (revision.text, doapFile))
- missingDoapRevisions = set()
- for release in releases:
- if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced
- missingDoapRevisions.add(release)
- if len(missingDoapRevisions) > 0:
- errorMessages.append('ERROR: Missing revision(s) in %s: %s' % (doapFile, ', '.join(sorted(missingDoapRevisions))))
+ distpage_lucene_solr = load('https://archive.apache.org/dist/lucene/solr')
+ distpage_solr = load('https://archive.apache.org/dist/solr')
+ releases = set()
+ for regex in reVersion1, reVersion2:
+ for tup in regex.findall(distpage_lucene_solr):
+ if tup[0] in ('1', '2'): # Ignore 1.X and 2.X releases
+ continue
+ releases.add(normalizeVersion(tup))
+ for tup in regex.findall(distpage_solr):
+ if tup[0] in ('1', '2'): # Ignore 1.X and 2.X releases
+ continue
+ releases.add(normalizeVersion(tup))
+ doapNS = '{http://usefulinc.com/ns/doap#}'
+ xpathRevision = '{0}Project/{0}release/{0}Version/{0}revision'.format(doapNS)
+ doapFile = "dev-tools/doap/solr.rdf"
+ treeRoot = ET.parse(doapFile).getroot()
+ doapRevisions = set()
+ for revision in treeRoot.findall(xpathRevision):
+ match = reDoapRevision.match(revision.text)
+ if (match is not None):
+ if (match.group(1) not in ('0', '1', '2')): # Ignore 0.X, 1.X and 2.X revisions
+ doapRevisions.add(normalizeVersion(match.groups()))
+ else:
+ errorMessages.append('ERROR: Failed to parse revision: %s in %s' % (revision.text, doapFile))
+ missingDoapRevisions = set()
+ for release in releases:
+ if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced
+ missingDoapRevisions.add(release)
+ if len(missingDoapRevisions) > 0:
+ errorMessages.append('ERROR: Missing revision(s) in %s: %s' % (doapFile, ', '.join(sorted(missingDoapRevisions))))
if (len(errorMessages) > 0):
raise RuntimeError('\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)'
% '\n'.join(errorMessages))
@@ -182,48 +202,40 @@ def normalizeVersion(tup):
tup = tup + ('0',)
return '.'.join(tup) + suffix
-def pushLocal(version, root, rev, rcNum, localDir):
+
+def pushLocal(version, root, rcNum, localDir):
print('Push local [%s]...' % localDir)
os.makedirs(localDir)
- dir = 'lucene-solr-%s-RC%d-rev%s' % (version, rcNum, rev)
- os.makedirs('%s/%s/lucene' % (localDir, dir))
- os.makedirs('%s/%s/solr' % (localDir, dir))
- print(' Lucene')
- lucene_dist_dir = '%s/lucene/packaging/build/distributions' % root
- os.chdir(lucene_dist_dir)
- print(' zip...')
- if os.path.exists('lucene.tar.bz2'):
- os.remove('lucene.tar.bz2')
- run('tar cjf lucene.tar.bz2 *')
-
- os.chdir('%s/%s/lucene' % (localDir, dir))
- print(' unzip...')
- run('tar xjf "%s/lucene.tar.bz2"' % lucene_dist_dir)
- os.remove('%s/lucene.tar.bz2' % lucene_dist_dir)
+ rev = open('%s/solr/distribution/build/release/.gitrev' % root, encoding='UTF-8').read()
+ dir = 'solr-%s-RC%d-rev-%s' % (version, rcNum, rev)
+ os.makedirs('%s/%s/solr' % (localDir, dir))
print(' Solr')
- solr_dist_dir = '%s/solr/packaging/build/distributions' % root
+ solr_dist_dir = '%s/solr/distribution/build/release' % root
os.chdir(solr_dist_dir)
- print(' zip...')
- if os.path.exists('solr.tar.bz2'):
- os.remove('solr.tar.bz2')
- run('tar cjf solr.tar.bz2 *')
- print(' unzip...')
+ print(' archive...')
+ if os.path.exists('solr.tar'):
+ os.remove('solr.tar')
+ run('tar cf solr.tar *')
+
+ print(' extract...')
os.chdir('%s/%s/solr' % (localDir, dir))
- run('tar xjf "%s/solr.tar.bz2"' % solr_dist_dir)
- os.remove('%s/solr.tar.bz2' % solr_dist_dir)
+ run('tar xf "%s/solr.tar"' % solr_dist_dir)
+ os.remove('%s/solr.tar' % solr_dist_dir)
+ os.chdir('..')
print(' chmod...')
- os.chdir('..')
run('chmod -R a+rX-w .')
print(' done!')
return 'file://%s/%s' % (os.path.abspath(localDir), dir)
-def read_version(path):
+
+def read_version(path): # pylint: disable=unused-argument
return scriptutil.find_current_version()
+
def parse_config():
epilogue = textwrap.dedent('''
Example usage for a Release Manager:
@@ -240,12 +252,24 @@ def parse_config():
help='Push the release to the local path')
parser.add_argument('--sign', metavar='KEYID',
help='Sign the release with the given gpg key')
+ parser.add_argument('--sign-method-gradle', dest='sign_method_gradle', default=False, action='store_true',
+ help='Use Gradle built-in GPG signing instead of gpg command for signing artifacts. '
+ ' This may require --gpg-secring argument if your keychain cannot be resolved automatically.')
+ parser.add_argument('--gpg-pass-noprompt', dest='gpg_pass_noprompt', default=False, action='store_true',
+ help='Do not prompt for gpg passphrase. For the default gnupg method, this means your gpg-agent'
+ ' needs a non-TTY pin-entry program. For gradle signing method, passphrase must be provided'
+ ' in gradle.properties or by env.var/sysprop. See ./gradlew helpPublishing for more info')
+ parser.add_argument('--gpg-home', metavar='PATH',
+ help='Path to gpg home containing your secring.gpg'
+ ' Optional, will use $HOME/.gnupg/secring.gpg by default')
parser.add_argument('--rc-num', metavar='NUM', type=int, default=1,
help='Release Candidate number. Default: 1')
parser.add_argument('--root', metavar='PATH', default='.',
- help='Root of Git working tree for lucene-solr. Default: "." (the current directory)')
+ help='Root of Git working tree for solr. Default: "." (the current directory)')
parser.add_argument('--logfile', metavar='PATH',
help='Specify log file path (default /tmp/release.log)')
+ parser.add_argument('--dev-mode', default=False, action='store_true',
+ help='Enable development mode, which disables some strict checks')
config = parser.parse_args()
if not config.prepare and config.sign:
@@ -258,15 +282,22 @@ def parse_config():
parser.error('Root path "%s" is not a directory' % config.root)
if config.local_keys is not None and not os.path.exists(config.local_keys):
parser.error('Local KEYS file "%s" not found' % config.local_keys)
+ if config.gpg_home and not os.path.exists(os.path.join(config.gpg_home, 'secring.gpg')):
+ parser.error('Specified gpg home %s does not exist or does not contain a secring.gpg' % config.gpg_home)
+ global dev_mode
+ if config.dev_mode:
+ print("Enabling development mode - DO NOT USE FOR ACTUAL RELEASE!")
+ dev_mode = True
cwd = os.getcwd()
os.chdir(config.root)
config.root = os.getcwd() # Absolutize root dir
- if os.system('git rev-parse') or 3 != len([d for d in ('dev-tools','lucene','solr') if os.path.isdir(d)]):
- parser.error('Root path "%s" is not a valid lucene-solr checkout' % config.root)
+ if os.system('git rev-parse') or 2 != len([d for d in ('dev-tools','solr') if os.path.isdir(d)]):
+ parser.error('Root path "%s" is not a valid solr checkout' % config.root)
os.chdir(cwd)
global LOG
if config.logfile:
LOG = config.logfile
+ print("Logfile is: %s" % LOG)
config.version = read_version(config.root)
print('Building version: %s' % config.version)
@@ -285,7 +316,7 @@ def check_key_in_keys(gpgKeyID, local_keys):
keysFileText = open(local_keys, encoding='iso-8859-1').read()
keysFileLocation = local_keys
else:
- keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
+ keysFileURL = "https://archive.apache.org/dist/solr/KEYS"
keysFileLocation = keysFileURL
print(" Using online KEYS file %s" % keysFileURL)
keysFileText = load(keysFileURL, encoding='iso-8859-1')
@@ -313,29 +344,60 @@ def check_key_in_keys(gpgKeyID, local_keys):
exit(2)
+def resolve_gpghome():
+ for p in [
+ # Linux, macos
+ os.path.join(os.path.expanduser("~"), '.gnupg'),
+ # Windows 10
+ os.path.expandvars(r'%APPDATA%\GnuPG')
+ # TODO: Should we support Cygwin?
+ ]:
+ if os.path.exists(os.path.join(p, 'secring.gpg')):
+ return p
+ return None
+
+
def main():
check_cmdline_tools()
c = parse_config()
+ gpg_home = None
if c.sign:
sys.stdout.flush()
c.key_id = c.sign
check_key_in_keys(c.key_id, c.local_keys)
- import getpass
- c.key_password = getpass.getpass('Enter GPG keystore password: ')
+ if c.gpg_home is not None:
+ print("Using custom gpg-home: %s" % c.gpg_home)
+ gpg_home = c.gpg_home
+ if c.sign_method_gradle:
+ if gpg_home is None:
+ resolved_gpg_home = resolve_gpghome()
+ if resolved_gpg_home is not None:
+ print("Resolved gpg home to %s" % resolved_gpg_home)
+ gpg_home = resolved_gpg_home
+ else:
+ print("WARN: Could not locate your gpg secret keyring, and --gpg-home not specified.")
+ print(" Falling back to location configured in gradle.properties.")
+ print(" See 'gradlew helpPublishing' for details.")
+ gpg_home = None
+ if c.gpg_pass_noprompt:
+ print("Will not prompt for gpg password. Make sure your signing setup supports this.")
+ c.key_password = None
+ else:
+ import getpass
+ c.key_password = getpass.getpass('Enter GPG keystore password: ')
else:
c.key_id = None
c.key_password = None
-
+
if c.prepare:
- rev = prepare(c.root, c.version, c.key_id, c.key_password)
+ prepare(c.root, c.version, c.key_id, c.key_password, gpg_home=gpg_home, sign_gradle=c.sign_method_gradle)
else:
os.chdir(c.root)
- rev = open('rev.txt', encoding='UTF-8').read()
if c.push_local:
- url = pushLocal(c.version, c.root, rev, c.rc_num, c.push_local)
+ url = pushLocal(c.version, c.root, c.rc_num, c.push_local)
else:
url = None
@@ -344,11 +406,14 @@ def main():
print('Next run the smoker tester:')
p = re.compile(".*/")
m = p.match(sys.argv[0])
- print('%s -u %ssmokeTestRelease.py %s' % (sys.executable, m.group(), url))
+ if not c.sign:
+ signed = "--not-signed"
+ else:
+ signed = ""
+ print('%s -u %ssmokeTestRelease.py %s %s' % (sys.executable, m.group(), signed, url))
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('Keyboard interrupt...exiting')
-
diff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.py
index be412c4..c3ec106 100755
--- a/dev-tools/scripts/releaseWizard.py
+++ b/dev-tools/scripts/releaseWizard.py
@@ -30,7 +30,7 @@
# -h, --help show this help message and exit
# --dry-run Do not execute any commands, but echo them instead. Display
# extra debug info
-# --root PATH Specify different root folder than ~/.lucene-releases
+# --root PATH Specify different root folder than ~/.solr-releases
import argparse
import copy
@@ -67,8 +67,11 @@ from consolemenu.screen import Screen
from scriptutil import BranchType, Version, download, run
# Solr-to-Java version mapping
-java_versions = {6: 8, 7: 8, 8: 8, 9: 11}
+java_versions = {6: 8, 7: 8, 8: 8, 9: 11, 10: 11}
editor = None
+state = None
+templates = None
+solr_news_file = None
# Edit this to add other global jinja2 variables or filters
def expand_jinja(text, vars=None):
@@ -85,7 +88,7 @@ def expand_jinja(text, vars=None):
'git_checkout_folder': state.get_git_checkout_folder(),
'ref_guide_svn_folder': state.get_ref_guide_svn_folder(),
'git_website_folder': state.get_website_git_folder(),
- 'dist_url_base': 'https://dist.apache.org/repos/dist/dev/lucene',
+ 'dist_url_base': 'https://dist.apache.org/repos/dist/dev/solr',
'm2_repository_url': 'https://repository.apache.org/service/local/staging/deploy/maven2',
'dist_file_path': state.get_dist_folder(),
'rc_folder': state.get_rc_folder(),
@@ -111,7 +114,6 @@ def expand_jinja(text, vars=None):
'vote_close_72h': vote_close_72h_date().strftime("%Y-%m-%d %H:00 UTC"),
'vote_close_72h_epoch': unix_time_millis(vote_close_72h_date()),
'vote_close_72h_holidays': vote_close_72h_holidays(),
- 'lucene_news_file': lucene_news_file,
'solr_news_file': solr_news_file,
'load_lines': load_lines,
'set_java_home': set_java_home,
@@ -192,7 +194,7 @@ def check_prerequisites(todo=None):
except:
sys.exit("You will need git installed")
try:
- svn_ver = run("svn --version").splitlines()[0]
+ run("svn --version").splitlines()[0]
except:
sys.exit("You will need svn installed")
if not 'EDITOR' in os.environ:
@@ -239,11 +241,11 @@ def maybe_remove_rc_from_svn():
Commands(state.get_git_checkout_folder(),
"""Looks like you uploaded artifacts for {{ build_rc.git_rev | default("<git_rev>", True) }} to svn which needs to be removed.""",
[Command(
- """svn -m "Remove cancelled Lucene/Solr {{ release_version }} RC{{ rc_number }}" rm {{ dist_url }}""",
+ """svn -m "Remove cancelled Solr {{ release_version }} RC{{ rc_number }}" rm {{ dist_url }}""",
logfile="svn_rm.log",
tee=True,
vars={
- 'dist_folder': """lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}""",
+ 'dist_folder': """solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}""",
'dist_url': "{{ dist_url_base }}/{{ dist_folder }}"
}
)],
@@ -309,7 +311,7 @@ class ReleaseState:
self.release_type = 'bugfix'
def is_released(self):
- return self.get_todo_by_id('announce_lucene').is_done()
+ return self.get_todo_by_id('announce_solr').is_done()
def get_gpg_key(self):
gpg_task = self.get_todo_by_id('gpg')
@@ -345,9 +347,13 @@ class ReleaseState:
def get_mirrored_versions(self):
if state.mirrored_versions is None:
+ # Add the solr release versions from lucene and solr projects
releases_str = load("https://projects.apache.org/json/foundation/releases.json", "utf-8")
- releases = json.loads(releases_str)['lucene']
- state.mirrored_versions = [ r for r in list(map(lambda y: y[7:], filter(lambda x: x.startswith('lucene-'), list(releases.keys())))) ]
+ releases_lucene_solr = json.loads(releases_str)['lucene']
+ releases_solr = json.loads(releases_str)['solr']
+ versions_l_s = [ r for r in list(map(lambda y: y[5:], filter(lambda x: x.startswith('solr-'), list(releases_lucene_solr.keys())))) ]
+ versions_s = [ r for r in list(map(lambda y: y[5:], filter(lambda x: re.match(r'^solr-(9|1\d)\.', x), list(releases_solr.keys())))) ]
+ state.mirrored_versions = versions_l_s + versions_s
return state.mirrored_versions
def get_mirrored_versions_to_delete(self):
@@ -417,7 +423,6 @@ class ReleaseState:
def clear_rc(self):
if ask_yes_no("Are you sure? This will clear and restart RC%s" % self.rc_number):
maybe_remove_rc_from_svn()
- dict = {}
for g in list(filter(lambda x: x.in_rc_loop(), self.todo_groups)):
for t in g.get_todos():
t.clear()
@@ -425,7 +430,7 @@ class ReleaseState:
try:
shutil.rmtree(self.get_rc_folder())
print("Cleared folder %s" % self.get_rc_folder())
- except Exception as e:
+ except Exception:
print("WARN: Failed to clear %s, please do it manually with higher privileges" % self.get_rc_folder())
self.save()
@@ -557,7 +562,7 @@ class ReleaseState:
return folder
def get_git_checkout_folder(self):
- folder = os.path.join(self.get_release_folder(), "lucene-solr")
+ folder = os.path.join(self.get_release_folder(), "solr")
return folder
def get_ref_guide_svn_folder(self):
@@ -565,7 +570,7 @@ class ReleaseState:
return folder
def get_website_git_folder(self):
- folder = os.path.join(self.get_release_folder(), "lucene-site")
+ folder = os.path.join(self.get_release_folder(), "solr-site")
return folder
def get_minor_branch_name(self):
@@ -590,6 +595,7 @@ class ReleaseState:
return "%s.%s.0" % (self.release_version_major, self.release_version_minor + 1)
if self.release_type == 'bugfix':
return "%s.%s.%s" % (self.release_version_major, self.release_version_minor, self.release_version_bugfix + 1)
+ return None
def get_refguide_release(self):
return "%s_%s" % (self.release_version_major, self.release_version_minor)
@@ -777,7 +783,6 @@ class Todo(SecretYamlObject):
def display_and_confirm(self):
try:
if self.depends:
- ret_str = ""
for dep in ensure_list(self.depends):
g = state.get_group_by_id(dep)
if not g:
@@ -896,7 +901,7 @@ def get_todo_menuitem_title():
def get_releasing_text():
- return "Releasing Lucene/Solr %s RC%d" % (state.release_version, state.rc_number)
+ return "Releasing Solr %s RC%d" % (state.release_version, state.rc_number)
def get_start_new_rc_menu_title():
@@ -952,14 +957,14 @@ def unix_to_datetime(unix_stamp):
def generate_asciidoc():
base_filename = os.path.join(state.get_release_folder(),
- "lucene_solr_release_%s"
+ "solr_release_%s"
% (state.release_version.replace("\.", "_")))
filename_adoc = "%s.adoc" % base_filename
filename_html = "%s.html" % base_filename
fh = open(filename_adoc, "w")
- fh.write("= Lucene/Solr Release %s\n\n" % state.release_version)
+ fh.write("= Solr Release %s\n\n" % state.release_version)
fh.write("(_Generated by releaseWizard.py v%s at %s_)\n\n"
% (getScriptVersion(), datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC")))
fh.write(":numbered:\n\n")
@@ -1038,21 +1043,21 @@ def generate_asciidoc():
def load_rc():
- lucenerc = os.path.expanduser("~/.lucenerc")
+ solrrc = os.path.expanduser("~/.solrrc")
try:
- with open(lucenerc, 'r') as fp:
+ with open(solrrc, 'r') as fp:
return json.load(fp)
except:
return None
def store_rc(release_root, release_version=None):
- lucenerc = os.path.expanduser("~/.lucenerc")
+ solrrc = os.path.expanduser("~/.solrrc")
dict = {}
dict['root'] = release_root
if release_version:
dict['release_version'] = release_version
- with open(lucenerc, "w") as fp:
+ with open(solrrc, "w") as fp:
json.dump(dict, fp, indent=2)
@@ -1068,7 +1073,7 @@ def file_to_string(filename):
return f.read().strip()
def download_keys():
- download('KEYS', "https://archive.apache.org/dist/lucene/KEYS", state.config_path)
+ download('KEYS', "https://archive.apache.org/dist/solr/KEYS", state.config_path)
def keys_downloaded():
return os.path.exists(os.path.join(state.config_path, "KEYS"))
@@ -1210,13 +1215,29 @@ def configure_pgp(gpg_todo):
else:
print(textwrap.dedent("""\
Could not find your key ID in official KEYS file.
- Please make sure it is added to https://dist.apache.org/repos/dist/release/lucene/KEYS
+ Please make sure it is added to https://dist.apache.org/repos/dist/release/solr/KEYS
and committed to svn. Then re-try this initialization"""))
if not ask_yes_no("Do you want to continue without fixing KEYS file? (not recommended) "):
return False
gpg_state['apache_id'] = id
gpg_state['gpg_key'] = gpg_id
+
+ print(textwrap.dedent("""\
+ You can choose between signing the release with the gpg program or with
+ the gradle signing plugin. Read about the difference by running
+ ./gradlew helpPublishing"""))
+
+ gpg_state['use_gradle'] = ask_yes_no("Do you want to sign the release with gradle plugin? No means gpg")
+
+ print(textwrap.dedent("""\
+ You need the passphrase to sign the release.
+ This script can prompt you securely for your passphrase (will not be stored) and pass it on to
+ buildAndPushRelease in a secure way. However, you can also configure your passphrase in advance
+ and avoid having to type it in the terminal. This can be done with either a gpg-agent (for gpg tool)
+ or in gradle.properties or an ENV.var (for gradle), See ./gradlew helpPublishing for details."""))
+ gpg_state['prompt_pass'] = ask_yes_no("Do you want this wizard to prompt you for your gpg password? ")
+
return True
@@ -1261,9 +1282,8 @@ class UpdatableSubmenuItem(SubmenuItem):
"""
:ivar ConsoleMenu self.submenu: The submenu to be opened when this item is selected
"""
- super(SubmenuItem, self).__init__(text=text, menu=menu, should_exit=should_exit)
+ super(UpdatableSubmenuItem, self).__init__(text=text, menu=menu, should_exit=should_exit, submenu=submenu)
- self.submenu = submenu
if menu:
self.get_submenu().parent = menu
@@ -1327,7 +1347,7 @@ class MyScreen(Screen):
class CustomExitItem(ExitItem):
def show(self, index):
- return super(ExitItem, self).show(index)
+ return super(CustomExitItem, self).show(index)
def get_return(self):
return ""
@@ -1338,18 +1358,17 @@ def main():
global dry_run
global templates
- print("Lucene/Solr releaseWizard v%s" % getScriptVersion())
+ print("Solr releaseWizard v%s" % getScriptVersion())
c = parse_config()
if c.dry:
print("Entering dry-run mode where all commands will be echoed instead of executed")
dry_run = True
- release_root = os.path.expanduser("~/.lucene-releases")
+ release_root = os.path.expanduser("~/.solr-releases")
if not load_rc() or c.init:
print("Initializing")
- dir_ok = False
- root = str(input("Choose root folder: [~/.lucene-releases] "))
+ root = str(input("Choose root folder: [~/.solr-releases] "))
if os.path.exists(root) and (not os.path.isdir(root) or not os.access(root, os.W_OK)):
sys.exit("Root %s exists but is not a directory or is not writable" % root)
if not root == '':
@@ -1391,19 +1410,15 @@ def main():
os.environ['JAVA_HOME'] = state.get_java_home()
os.environ['JAVACMD'] = state.get_java_cmd()
- global lucene_news_file
global solr_news_file
- lucene_news_file = os.path.join(state.get_website_git_folder(), 'content', 'core', 'core_news',
- "%s-%s-available.md" % (state.get_release_date_iso(), state.release_version.replace(".", "-")))
solr_news_file = os.path.join(state.get_website_git_folder(), 'content', 'solr', 'solr_news',
"%s-%s-available.md" % (state.get_release_date_iso(), state.release_version.replace(".", "-")))
- website_folder = state.get_website_git_folder()
- main_menu = UpdatableConsoleMenu(title="Lucene/Solr ReleaseWizard",
+ main_menu = UpdatableConsoleMenu(title="Solr ReleaseWizard",
subtitle=get_releasing_text,
prologue_text="Welcome to the release wizard. From here you can manage the process including creating new RCs. "
"All changes are persisted, so you can exit any time and continue later. Make sure to read the Help section.",
- epilogue_text="® 2020 The Lucene/Solr project. Licensed under the Apache License 2.0\nScript version v%s)" % getScriptVersion(),
+ epilogue_text="® 2022 The Solr project. Licensed under the Apache License 2.0\nScript version v%s)" % getScriptVersion(),
screen=MyScreen())
todo_menu = UpdatableConsoleMenu(title=get_releasing_text,
@@ -1550,7 +1565,7 @@ def run_follow(command, cwd=None, fh=sys.stdout, tee=False, live=False, shell=No
lines_written += 1
print_line_cr(line, lines_written, stdout=(fh == sys.stdout), tee=tee)
- except Exception as ioe:
+ except Exception:
pass
if not endstderr:
try:
@@ -1571,7 +1586,7 @@ def run_follow(command, cwd=None, fh=sys.stdout, tee=False, live=False, shell=No
errlines.append("%s\n" % line.rstrip())
lines_written += 1
print_line_cr(line, lines_written, stdout=(fh == sys.stdout), tee=tee)
- except Exception as e:
+ except Exception:
pass
if not lines_written > lines_before:
@@ -1621,7 +1636,7 @@ class Commands(SecretYamlObject):
fields = loader.construct_mapping(node, deep = True)
return Commands(**fields)
- def run(self):
+ def run(self): # pylint: disable=inconsistent-return-statements # TODO
root = self.get_root_folder()
if self.commands_text:
@@ -1640,15 +1655,15 @@ class Commands(SecretYamlObject):
for line in cmd.display_cmd():
print(" %s" % line)
print()
+ confirm_each = (not self.confirm_each_command is False) and len(commands) > 1
if not self.enable_execute is False:
if self.run_text:
print("\n%s\n" % self.get_run_text())
- if len(commands) > 1:
- if not self.confirm_each_command is False:
- print("You will get prompted before running each individual command.")
- else:
- print(
- "You will not be prompted for each command but will see the ouput of each. If one command fails the execution will stop.")
+ if confirm_each:
+ print("You will get prompted before running each individual command.")
+ else:
+ print(
+ "You will not be prompted for each command but will see the output of each. If one command fails the execution will stop.")
success = True
if ask_yes_no("Do you want me to run these commands now?"):
if self.remove_files:
@@ -1677,8 +1692,10 @@ class Commands(SecretYamlObject):
folder_prefix = ''
if cmd.cwd:
folder_prefix = cmd.cwd + "_"
- if self.confirm_each_command is False or len(commands) == 1 or ask_yes_no("Shall I run '%s' in folder '%s'" % (cmd, cwd)):
- if self.confirm_each_command is False:
+ if confirm_each and cmd.comment:
+ print("# %s\n" % cmd.get_comment())
+ if not confirm_each or ask_yes_no("Shall I run '%s' in folder '%s'" % (cmd, cwd)):
+ if not confirm_each:
print("------------\nRunning '%s' in folder '%s'" % (cmd, cwd))
logfilename = cmd.logfile
logfile = None
@@ -1784,6 +1801,8 @@ def abbreviate_homedir(line):
return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1%HOME%", line)
elif 'USERPROFILE' in os.environ:
return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1%USERPROFILE%", line)
+ else:
+ return None
else:
return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1~", line)
@@ -1864,7 +1883,6 @@ class Command(SecretYamlObject):
def display_cmd(self):
lines = []
- pre = post = ''
if self.comment:
if is_windows():
lines.append("REM %s" % self.get_comment())
@@ -1910,11 +1928,11 @@ class UserInput(SecretYamlObject):
return result
-def create_ical(todo):
+def create_ical(todo): # pylint: disable=unused-argument
if ask_yes_no("Do you want to add a Calendar reminder for the close vote time?"):
c = Calendar()
e = Event()
- e.name = "Lucene/Solr %s vote ends" % state.release_version
+ e.name = "Solr %s vote ends" % state.release_version
e.begin = vote_close_72h_date()
e.description = "Remember to sum up votes and continue release :)"
c.events.add(e)
@@ -1957,17 +1975,7 @@ def vote_close_72h_holidays():
return holidays if len(holidays) > 0 else None
-def prepare_announce_lucene(todo):
- if not os.path.exists(lucene_news_file):
- lucene_text = expand_jinja("(( template=announce_lucene ))")
- with open(lucene_news_file, 'w') as fp:
- fp.write(lucene_text)
- # print("Wrote Lucene announce draft to %s" % lucene_news_file)
- else:
- print("Draft already exist, not re-generating")
- return True
-
-def prepare_announce_solr(todo):
+def prepare_announce_solr(todo): # pylint: disable=unused-argument
if not os.path.exists(solr_news_file):
solr_text = expand_jinja("(( template=announce_solr ))")
with open(solr_news_file, 'w') as fp:
@@ -1978,7 +1986,7 @@ def prepare_announce_solr(todo):
return True
-def check_artifacts_available(todo):
+def check_artifacts_available(todo): # pylint: disable=unused-argument
try:
cdnUrl = expand_jinja("https://dlcdn.apache.org/solr/{{ release_version }}/solr-{{ release_version }}-src.tgz.asc")
load(cdnUrl)
diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml
index b3db1a7..ead7d9d 100644
--- a/dev-tools/scripts/releaseWizard.yaml
+++ b/dev-tools/scripts/releaseWizard.yaml
@@ -32,7 +32,7 @@
#
templates:
help: |
- Welcome to the role as Release Manager for Lucene/Solr, and the releaseWizard!
+ Welcome to the role as Release Manager for Solr, and the releaseWizard!
The Release Wizard aims to walk you through the whole release process step by step,
helping you to to run the right commands in the right order, generating
@@ -42,9 +42,9 @@ templates:
As you complete each step the tool will ask you if the task is complete, making
it easy for you to know what is done and what is left to do. If you need to
- re-spin a Release Candidata (RC) the Wizard will also help.
+ re-spin a Release Candidate (RC) the Wizard will also help.
- The Lucene project has automated much of the release process with various scripts,
+ The Solr project has automated much of the release process with various scripts,
and this wizard is the glue that binds it all together.
In the first TODO step in the checklist you will be asked to read up on the
@@ -65,7 +65,7 @@ templates:
.Mail template {% if passed %}successful{% else %}failed{% endif %} vote
----
To: dev@solr.apache.org
- Subject: [{% if passed %}RESULT{% else %}FAILED{% endif %}] [VOTE] Release Lucene/Solr {{ release_version }} RC{{ rc_number }}
+ Subject: [{% if passed %}RESULT{% else %}FAILED{% endif %}] [VOTE] Release Solr {{ release_version }} RC{{ rc_number }}
It's been >72h since the vote was initiated and the result is:
@@ -80,36 +80,14 @@ templates:
This vote has {% if passed %}PASSED{% else %}FAILED{% endif %}
----
{%- endmacro %}
- announce_lucene: |
- Title: Apache Lucene™ {{ release_version }} available
- category: core/news
- URL:
- save_as:
-
- The Lucene PMC is pleased to announce the release of Apache Lucene {{ release_version }}.
-
- Apache Lucene is a high-performance, full-featured text search engine library written entirely in Java. It is a technology suitable for nearly any application that requires full-text search, especially cross-platform.
-
- This release contains numerous bug fixes, optimizations, and improvements, some of which are highlighted below. The release is available for immediate download at:
-
- <https://lucene.apache.org/core/downloads.html>
-
- ### Lucene {{ release_version }} Release Highlights:
-
- * Feature 1 pasted from WIKI release notes
- * Feature 2 ...
-
- Please read CHANGES.txt for a full list of {% if is_feature_release %}new features and {% endif %}changes:
-
- <https://lucene.apache.org/core/{{ release_version_underscore }}/changes/Changes.html>
announce_solr: |
Title: Apache Solr™ {{ release_version }} available
category: solr/news
save_as:
- The Lucene PMC is pleased to announce the release of Apache Solr {{ release_version }}.
+ The Solr PMC is pleased to announce the release of Apache Solr {{ release_version }}.
- Solr is the popular, blazing fast, open source NoSQL search platform from the Apache Lucene project. Its major features include powerful full-text search, hit highlighting, faceted search, dynamic clustering, database integration, rich document handling, and geospatial search. Solr is highly scalable, providing fault tolerant distributed search and indexing, and powers the search and navigation features of many of the world's largest internet sites.
+ Solr is the popular, blazing fast, open source NoSQL search platform from the Apache Solr project. Its major features include powerful full-text search, hit highlighting, faceted search, dynamic clustering, database integration, rich document handling, and geospatial search. Solr is highly scalable, providing fault tolerant distributed search and indexing, and powers the search and navigation features of many of the world's largest internet sites.
Solr {{ release_version }} is available for immediate download at:
@@ -127,44 +105,6 @@ templates:
Please read CHANGES.txt for a full list of {% if is_feature_release %}new features, changes and {% endif %}bugfixes:
<https://solr.apache.org/{{ release_version_underscore }}/changes/Changes.html>
-
- Solr {{ release_version }} also includes {% if is_feature_release %}features, optimizations and {% endif %}bugfixes in the corresponding Apache Lucene release:
-
- <https://lucene.apache.org/core/{{ release_version_underscore }}/changes/Changes.html>
- announce_lucene_mail: |
- The template below can be used to announce the Lucene release to the
- internal mailing lists.
-
- .Mail template
- ----
- To: dev@solr.apache.org, users@solr.apache.org
- Subject: [ANNOUNCE] Apache Lucene {{ release_version }} released
-
- (( template=announce_lucene_mail_body ))
- ----
- announce_lucene_sign_mail: |
- The template below can be used to announce the Lucene release to the
- `announce@apache.org` mailing list. The mail *should be signed with PGP.*
- and sent *from your `@apache.org` account*.
-
- .Mail template
- ----
- From: {{ gpg.apache_id }}@apache.org
- To: announce@apache.org
- Subject: [ANNOUNCE] Apache Lucene {{ release_version }} released
-
- (( template=announce_lucene_mail_body ))
- ----
- announce_lucene_mail_body: |
- {% for line in load_lines(lucene_news_file, 5) -%}
- {{ line }}
- {%- endfor %}
-
-
- Note: The Apache Software Foundation uses an extensive mirroring network for
- distributing releases. It is possible that the mirror you are using may not have
- replicated the release yet. If that is the case, please try another mirror.
- This also applies to Maven access.
announce_solr_mail: |
The template below can be used to announce the Solr release to the
internal mailing lists.
@@ -195,10 +135,6 @@ templates:
{%- endfor %}
- Note: The Apache Software Foundation uses an extensive mirroring network for
- distributing releases. It is possible that the mirror you are using may not have
- replicated the release yet. If that is the case, please try another mirror.
- This also applies to Maven access.
# TODOs belong to groups for easy navigation in menus. Todo objects may contain asciidoc
# descriptions, a number of commands to execute, some links to display, user input to gather
# etc. Here is the documentation of each type of object. For further details, please consult
@@ -219,7 +155,7 @@ templates:
# you should introduce the task in more detail. You can use {{ jinja_var }} to
# reference variables. See `releaseWizard.py` for list of global vars supported.
# You can reference state saved from earlier TODO items using syntax
-# {{ todi_id.var_name }}
+# {{ todo_id.var_name }}
# with `var_name` being either fetched from user_input or persist_vars
# depends: # One or more dependencies which will bar execution
# - todo_id1
@@ -283,7 +219,6 @@ groups:
- https://infra.apache.org/release-publishing.html
- https://www.apache.org/legal/release-policy.html
- https://infra.apache.org/release-signing.html
- - https://cwiki.apache.org/confluence/display/LUCENE/ReleaseTodo
- !Todo
id: tools
title: Necessary tools are installed
@@ -315,25 +250,34 @@ groups:
* Upload your key to the MIT key server, pgp.mit.edu
* Put you GPG key's fingerprint in the `OpenPGP Public Key Primary Fingerprint`
field in your id.apache.org profile
- * The tests will complain if your GPG key has not been signed by another Lucene
+ * The tests will complain if your GPG key has not been signed by another Solr
committer. This makes you a part of the GPG "web of trust" (WoT). Ask a committer
that you know personally to sign your key for you, providing them with the
fingerprint for the key.
+
+ You can choose between signing the release with the gpg program or with the
+ gradle signing plugin. Read about the difference in https://github.com/apache/solr/blob/main/help/publishing.txt
+
+ This wizard can prompt you securely for your passphrase (will not be stored) and pass it on to
+ buildAndPushRelease in a secure way. However, you can also configure your passphrase in advance
+ and avoid having to type it in the terminal. This can be done with either a gpg-agent (for gpg tool)
+ or in `gradle.properties` or an ENV.var (for gradle), See `./gradlew helpPublishing` for details.
function: configure_pgp
links:
- https://infra.apache.org/release-signing.html
- https://infra.apache.org/openpgp.html#apache-wot
- https://id.apache.org
- - https://dist.apache.org/repos/dist/release/lucene/KEYS
+ - https://dist.apache.org/repos/dist/release/solr/KEYS
+ - https://github.com/apache/solr/blob/main/help/publishing.txt
- !Todo
id: jira_permissions
- title: Obtain the neccessary permissions for Apache Jira
+ title: Obtain the necessary permissions for Apache Jira
description: |-
- If you are not a PMC member and this is your first time as RM, please ask to be granted extra permissions in Jira in order to complete all steps of the release.
+ If you are not a PMC member and this is your first time as RM, please ask to be granted extra permissions
+ in Jira in order to complete all steps of the release.
If you are a PMC member, you will already have the necessary permissions.
links:
- - https://issues.apache.org/jira/projects/LUCENE
- https://issues.apache.org/jira/projects/SOLR
- !TodoGroup
id: preparation
@@ -378,7 +322,7 @@ groups:
- '{{ git_checkout_folder }}'
commands:
- !Command
- cmd: git clone --progress https://gitbox.apache.org/repos/asf/lucene-solr.git lucene-solr
+ cmd: git clone --progress https://gitbox.apache.org/repos/asf/solr.git solr
logfile: git_clone.log
- !Todo
id: gradle_precommit
@@ -434,7 +378,7 @@ groups:
cmd: git checkout -b {{ stable_branch }}
tee: true
- !Command
- cmd: git push origin {{ stable_branch }}
+ cmd: git push --set-upstream origin {{ stable_branch }}
tee: true
- !Todo
id: create_minor_branch
@@ -463,7 +407,7 @@ groups:
cmd: git checkout -b {{ release_branch }}
tee: true
- !Command
- cmd: git push origin {{ release_branch }}
+ cmd: git push --set-upstream origin {{ release_branch }}
tee: true
- !Todo
id: add_version_major
@@ -488,12 +432,7 @@ groups:
cmd: git add -u . && git commit -m "Add next major version {{ next_version }}" && git push
logfile: commit-stable.log
post_description: |
- Make sure to follow the manual instructions printed by the script:
-
- * Move backcompat oldIndexes to unsupportedIndexes in TestBackwardsCompatibility
- * Update IndexFormatTooOldException throw cases
-
- There may be other steps needed as well
+ Make sure to follow the manual instructions printed by the script.
- !Todo
id: add_version_minor
title: Add a new minor version on stable branch
@@ -535,7 +474,7 @@ groups:
- major
- minor
links:
- - https://wiki.apache.org/lucene-java/JenkinsReleaseBuilds
+ - https://cwiki.apache.org/confluence/display/LUCENEJAVA/JenkinsReleaseBuilds
- !Todo
id: inform_devs
title: Inform Devs of the new Release Branch
@@ -549,7 +488,7 @@ groups:
.Mail template
----
To: dev@solr.apache.org
- Subject: New branch and feature freeze for Lucene/Solr {{ release_version }}
+ Subject: New branch and feature freeze for Solr {{ release_version }}
NOTICE:
@@ -589,7 +528,7 @@ groups:
.Mail template
----
To: dev@solr.apache.org
- Subject: Bugfix release Lucene/Solr {{ release_version }}
+ Subject: Bugfix release Solr {{ release_version }}
NOTICE:
@@ -616,14 +555,12 @@ groups:
These are typically edited on the Wiki
Clone a page for a previous version as a starting point for your release notes.
- You will need two pages, one for Lucene and another for Solr, see links.
Edit the contents of `CHANGES.txt` into a more concise format for public consumption.
Ask on dev@ for input. Ideally the timing of this request mostly coincides with the
release branch creation. It's a good idea to remind the devs of this later in the release too.
NOTE: Do not add every single JIRA issue, but distill the Release note into important changes!
links:
- - https://cwiki.apache.org/confluence/display/LUCENE/Release+Notes
- https://cwiki.apache.org/confluence/display/SOLR/Release+Notes
- !Todo
id: new_jira_versions
@@ -635,13 +572,10 @@ groups:
. Change name of version `main ({{ release_version_major }}.0)` into `{{ release_version_major }}.0`
{%- endif %}
. Create a new (unreleased) version `{{ get_next_version }}`
-
- This needs to be done both for Lucene and Solr JIRAs, see links.
types:
- major
- minor
links:
- - https://issues.apache.org/jira/plugins/servlet/project-config/LUCENE/versions
- https://issues.apache.org/jira/plugins/servlet/project-config/SOLR/versions
- !Todo
id: jira_blocker_refguide
@@ -689,7 +623,7 @@ groups:
description: |
It is recommended to clean your Gradle cache before building the artifacts.
This ensures that all Gradle dependencies are freshly downloaded,
- so we emulate a user that never used the Lucene build system before.
+ so we emulate a user that never used the Solr build system before.
commands: !Commands
root_folder: '{{ home }}'
remove_files: .gradle/caches/modules-2/files-2.1_bak
@@ -708,7 +642,7 @@ groups:
vars:
logfile: '{{ [rc_folder, ''logs'', ''buildAndPushRelease.log''] | path_join }}'
git_rev: '{{ current_git_rev }}' # Note, git_rev will be recorded in todo state AFTER completion of commands
- local_keys: '{% if keys_downloaded %} --local-keys "{{ [config_path, ''KEYS''] | path_join }}"{% endif %}'
+ local_keys: '{% if keys_downloaded %} --local-keys "{{ [config_path, ''KEYS''] | path_join }}"{% endif %}'
persist_vars:
- git_rev
commands: !Commands
@@ -736,8 +670,8 @@ groups:
cmd: git pull --ff-only
tee: true
- !Command
- cmd: python3 -u dev-tools/scripts/buildAndPushRelease.py {{ local_keys }} --logfile {{ logfile }} --push-local "{{ dist_file_path }}" --rc-num {{ rc_number }} --sign {{ gpg_key | default("<gpg_key_id>", True) }}
- comment: "NOTE: Remember to type your GPG pass-phrase at the prompt!"
+ cmd: python3 -u dev-tools/scripts/buildAndPushRelease.py {{ local_keys }} --logfile {{ logfile }} --push-local "{{ dist_file_path }}" --rc-num {{ rc_number }} --sign {{ gpg_key | default("<gpg_key_id>", True) }}{% if gpg.use_gradle %} --sign-method-gradle{% endif %}{% if not gpg.prompt_pass %} --gpg-pass-noprompt{% endif %}
+ comment: "Using {% if gpg.use_gradle %}gradle{% else %}gpg command{% endif %} for signing.{% if gpg.prompt_pass %} Remember to type your GPG pass-phrase at the prompt!{% endif %}"
logfile: build_rc.log
tee: true
- !Todo
@@ -745,7 +679,7 @@ groups:
title: Run the smoke tester
depends: build_rc
vars:
- dist_folder: lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
+ dist_folder: solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
dist_path: '{{ [dist_file_path, dist_folder] | path_join }}'
tmp_dir: '{{ [rc_folder, ''smoketest''] | path_join }}'
local_keys: '{% if keys_downloaded %} --local-keys "{{ [config_path, ''KEYS''] | path_join }}"{% endif %}'
@@ -765,15 +699,15 @@ groups:
Here we'll import the artifacts into Subversion.
depends: smoke_tester
vars:
- dist_folder: lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
+ dist_folder: solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
dist_path: '{{ [dist_file_path, dist_folder] | path_join }}'
- dist_url: https://dist.apache.org/repos/dist/dev/lucene/{{ dist_folder}}
+ dist_url: '{{ dist_url_base }}/{{ dist_folder}}'
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
commands_text: Have your Apache credentials handy, you'll be prompted for your password
commands:
- !Command
- cmd: svn -m "Lucene/Solr {{ release_version }} RC{{ rc_number }}" import {{ dist_path }} {{ dist_url }}
+ cmd: svn -m "Solr {{ release_version }} RC{{ rc_number }}" import {{ dist_path }} {{ dist_url }}
logfile: import_svn.log
tee: true
- !Todo
@@ -784,8 +718,8 @@ groups:
area and checks hash and signatures, but does not re-run all tests.
depends: import_svn
vars:
- dist_folder: lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
- dist_url: https://dist.apache.org/repos/dist/dev/lucene/{{ dist_folder}}
+ dist_folder: solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
+ dist_url: '{{ dist_url_base }}/{{ dist_folder}}'
tmp_dir: '{{ [rc_folder, ''smoketest_staged''] | path_join }}'
local_keys: '{% if keys_downloaded %} --local-keys "{{ [config_path, ''KEYS''] | path_join }}"{% endif %}'
commands: !Commands
@@ -813,17 +747,17 @@ groups:
.Mail template
----
To: dev@solr.apache.org
- Subject: [VOTE] Release Lucene/Solr {{ release_version }} RC{{ rc_number }}
+ Subject: [VOTE] Release Solr {{ release_version }} RC{{ rc_number }}
- Please vote for release candidate {{ rc_number }} for Lucene/Solr {{ release_version }}
+ Please vote for release candidate {{ rc_number }} for Solr {{ release_version }}
The artifacts can be downloaded from:
- https://dist.apache.org/repos/dist/dev/lucene/lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
+ {{ dist_url_base }}/solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
You can run the smoke tester directly with this command:
python3 -u dev-tools/scripts/smokeTestRelease.py \
- https://dist.apache.org/repos/dist/dev/lucene/lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
+ {{ dist_url_base }}/solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
The vote will be open for at least 72 hours i.e. until {{ vote_close }}.
@@ -933,30 +867,26 @@ groups:
logs_prefix: tag_release
commands:
- !Command
- cmd: git tag -a releases/lucene-solr/{{ release_version }} -m "Lucene/Solr {{ release_version }} release" {{ build_rc.git_rev | default("<git_rev>", True) }}
+ cmd: git tag -a releases/solr/{{ release_version }} -m "Solr {{ release_version }} release" {{ build_rc.git_rev | default("<git_rev>", True) }}
logfile: git_tag.log
tee: true
- !Command
- cmd: git push origin releases/lucene-solr/{{ release_version }}
+ cmd: git push origin releases/solr/{{ release_version }}
logfile: git_push_tag.log
tee: true
- !Todo
id: rm_staged_mvn
title: Delete mvn artifacts from staging repo
vars:
- dist_folder: lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
+ dist_folder: solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
dist_path: '{{ [dist_file_path, dist_folder] | path_join }}'
- dist_stage_url: https://dist.apache.org/repos/dist/dev/lucene/{{ dist_folder}}
+ dist_stage_url: '{{ dist_url_base }}/{{ dist_folder}}'
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
confirm_each_command: false
commands_text: This will remove maven artifacts so they do not end up in the Distribution Directory
commands:
- !Command
- cmd: svn rm -m "Delete the lucene maven artifacts" {{ dist_stage_url }}/lucene/maven
- logfile: svn_rm_mvn_lucene.log
- tee: true
- - !Command
cmd: svn rm -m "Delete the solr maven artifacts" {{ dist_stage_url }}/solr/maven
logfile: svn_rm_mvn_solr.log
tee: true
@@ -964,47 +894,38 @@ groups:
id: mv_to_release
title: Move release artifacts to release repo
vars:
- dist_folder: lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
- dist_stage_url: https://dist.apache.org/repos/dist/dev/lucene/{{ dist_folder}}
- dist_release_url: https://dist.apache.org/repos/dist/release/lucene
+ dist_folder: solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
+ dist_stage_url: '{{ dist_url_base }}/{{ dist_folder}}'
+ dist_release_url: https://dist.apache.org/repos/dist/release/solr
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
confirm_each_command: false
commands_text: This will move the new release artifacts from staging repo to the release repo
commands:
- !Command
- cmd: svn move -m "Move Lucene {{ release_version }} RC{{ rc_number }} to release repo" {{ dist_stage_url }}/lucene {{ dist_release_url }}/java/{{ release_version }}
- logfile: svn_mv_lucene.log
- tee: true
- - !Command
cmd: svn move -m "Move Solr {{ release_version }} RC{{ rc_number }} to release repo" {{ dist_stage_url }}/solr {{ dist_release_url }}/solr/{{ release_version }}
logfile: svn_mv_solr.log
tee: true
- !Command
- cmd: svn rm -m "Clean up the RC folder for {{ release_version }} RC{{ rc_number }}" https://dist.apache.org/repos/dist/dev/lucene/lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
+ cmd: svn rm -m "Clean up the RC folder for {{ release_version }} RC{{ rc_number }}" {{ dist_url_base }}/solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
logfile: svn_rm_containing.log
comment: Clean up containing folder on the staging repo
tee: true
- post_description: 'Note at this point you will see the Jenkins job "Lucene-Solr-SmokeRelease-main" begin to fail, until you run the "Generate Backcompat Indexes" '
+ post_description: 'Note at this point you will see the Jenkins job "Solr-SmokeRelease-main" begin to fail, until you run the "Generate Backcompat Indexes" '
- !Todo
id: stage_maven
title: Stage the maven artifacts for publishing
vars:
- dist_folder: lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
+ dist_folder: solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
confirm_each_command: true
commands_text: In the source checkout do the following (note that this step will prompt you for your Apache LDAP credentials)
commands:
- !Command
- cmd: ant clean stage-maven-artifacts -Dmaven.dist.dir={{ [dist_file_path, dist_folder, 'lucene', 'maven'] | path_join }} -Dm2.repository.id=apache.releases.https -Dm2.repository.url={{ m2_repository_url }}
- logfile: publish_lucene_maven.log
- - !Command
cmd: ant clean stage-maven-artifacts -Dmaven.dist.dir={{ [dist_file_path, dist_folder, 'solr', 'maven'] | path_join }} -Dm2.repository.id=apache.releases.https -Dm2.repository.url={{ m2_repository_url }}
logfile: publish_solr_maven.log
post_description: The artifacts are not published yet, please proceed with the next step to actually publish!
- links:
- - https://wiki.apache.org/lucene-java/PublishMavenArtifacts
- !Todo
id: publish_maven
depends: stage_maven
@@ -1016,12 +937,12 @@ groups:
* Close the staging repository
. Log in to https://repository.apache.org/ with your ASF credentials
. Select "Staging Repositories" under "Build Promotion" from the navigation bar on the left
- . Select the staging repository containing the Lucene artifacts
- . Click on the "Close" button above the repository list, then enter a description when prompted, e.g. "Lucene/Solr {{ release_version }} RC{{ rc_number }}"
+ . Select the staging repository containing the Solr artifacts
+ . Click on the "Close" button above the repository list, then enter a description when prompted, e.g. "Solr {{ release_version }} RC{{ rc_number }}"
* The system will now spend some time validating the artifacts. Grab a coke and come back.
- * Release the Lucene and/or Solr artifacts
+ * Release the Solr artifacts
. Wait and keep clicking refresh until the "Release" button becomes available
- . Click on the "Release" button above the repository list, then enter a description when prompted, e.g. "Lucene/Solr {{ release_version }}".
+ . Click on the "Release" button above the repository list, then enter a description when prompted, e.g. "Solr {{ release_version }}".
Maven central should show the release after a short while
links:
@@ -1044,7 +965,7 @@ groups:
title: Update the website
description: |
For every release, we publish docs on the website, we need to update the
- download pages etc. The website is hosted in https://github.com/apache/lucene-site
+ download pages etc. The website is hosted in https://github.com/apache/solr-site
but the Javadocs and Solr Reference Guide are pushed to SVN and then included
in the main site through links.
todos:
@@ -1056,9 +977,9 @@ groups:
depending on your upload bandwidth. We'll publish this directly to the production tree.
At the end of the task, the link below shall work.
links:
- - https://solr.apache.org/{{ version }}
+ - https://solr.apache.org/docs/{{ version }}
vars:
- release_tag: releases/lucene-solr/{{ release_version }}
+ release_tag: releases/solr/{{ release_version }}
version: "{{ release_version_major }}_{{ release_version_minor }}_{{ release_version_bugfix }}"
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
@@ -1073,10 +994,6 @@ groups:
cmd: "{{ gradle_cmd }} documentation -Dversion.release={{ release_version }}"
comment: Build documentation
- !Command
- cmd: svn -m "Add docs, changes and javadocs for Lucene {{ release_version }}" import {{ git_checkout_folder }}/lucene/build/docs https://svn.apache.org/repos/infra/sites/lucene/core/{{ version }}
- logfile: add-docs-lucene.log
- comment: Add docs for Lucene
- - !Command
cmd: svn -m "Add docs, changes and javadocs for Solr {{ release_version }}" import {{ git_checkout_folder }}/solr/build/docs https://svn.apache.org/repos/infra/sites/solr/docs/{{ version }}
logfile: add-docs-solr.log
comment: Add docs for Solr
@@ -1181,22 +1098,22 @@ groups:
- '{{ git_website_folder }}'
commands:
- !Command
- cmd: git clone --progress https://gitbox.apache.org/repos/asf/lucene-site.git lucene-site
+ cmd: git clone --progress https://gitbox.apache.org/repos/asf/solr-site.git solr-site
logfile: website_git_clone.log
- !Todo
id: website_update_versions
title: Update website versions
depends: website_git_clone
vars:
- release_tag: releases/lucene-solr/{{ release_version }}
+ release_tag: releases/solr/{{ release_version }}
description: |
We need to update the website so that the download pages list the new release, and the
"latest" javadoc links point to the new release.
Fortunately the only thing you need to change is a few variables in `pelicanconf.py`.
- If you release a current latest release, change the `LUCENE_LATEST_RELEASE` and `LUCENE_LATEST_RELEASE_DATE`
+ If you release a current latest release, change the `SOLR_LATEST_RELEASE` and `SOLR_LATEST_RELEASE_DATE`
variables.
- If you relese a bugfix release for previos version, then change the `LUCENE_PREVIOUS_MAJOR_RELEASE` variable.
+ If you relese a bugfix release for previos version, then change the `SOLR_PREVIOUS_MAJOR_RELEASE` variable.
commands: !Commands
root_folder: '{{ git_website_folder }}'
commands_text: Edit pelicanconf.py to update version numbers
@@ -1212,31 +1129,6 @@ groups:
post_description: |
You will push and verify all changes in a later step
- !Todo
- id: prepare_announce_lucene
- title: Author the Lucene release news
- depends: website_git_clone
- description: |
- Edit a news text for the Lucene website. This text will be the basis for the release announcement email later.
- This step will open an editor with a template. You will need to copy/paste the final release announcement text
- from the Wiki page and format it as Markdown.
- function: prepare_announce_lucene
- commands: !Commands
- root_folder: '{{ git_website_folder }}'
- commands_text: |
- Copy the Lucene announcement from https://cwiki.apache.org/confluence/display/LUCENE/Release+Notes
- You have to exit the editor after edit to continue.
- commands:
- - !Command
- cmd: "{{ editor }} {{ lucene_news_file }}"
- comment: Edit the for Lucene announcement news
- stdout: true
- - !Command
- cmd: git add . && git commit -m "Adding Lucene news for release {{ release_version }}"
- logfile: commit.log
- stdout: true
- post_description: |
- You will push and verify all changes in a later step
- - !Todo
id: prepare_announce_solr
title: Author the Solr release news
depends: website_git_clone
@@ -1320,14 +1212,14 @@ groups:
logfile: push-website.log
post_description: |
Wait a few minutes for the build to happen. You can follow the site build at https://ci2.apache.org/#/builders/3
- and view the staged site at https://lucene.staged.apache.org
+ and view the staged site at https://solr.staged.apache.org
Verify that correct links and versions are mentioned in download pages, download buttons etc.
If you find anything wrong, then commit and push any changes and check again.
Next step is to merge the changes to branch 'production' in order to publish the site.
links:
- https://ci2.apache.org/#/builders/3
- - https://lucene.staged.apache.org
+ - https://solr.staged.apache.org
- !Todo
id: publish_website
title: Publish the website changes
@@ -1361,11 +1253,11 @@ groups:
- https://solr.apache.org/api/solr-core/
- !Todo
id: update_doap
- title: Update the DOAP files
+ title: Update the DOAP file
description: |
- Update the Core & Solr DOAP RDF files on the unstable, stable and release branches to
+ Update the Solr DOAP RDF file on the unstable, stable and release branches to
reflect the new versions (note that the website .htaccess file redirects from their
- canonical URLs to their locations in the Lucene/Solr Git source repository - see
+ canonical URLs to their locations in the Solr Git source repository - see
dev-tools/doap/README.txt for more info)
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
@@ -1376,15 +1268,11 @@ groups:
stdout: true
comment: Goto main branch
- !Command
- cmd: "{{ editor }} dev-tools/doap/lucene.rdf"
- comment: Edit Lucene DOAP, add version {{ release_version }}
- stdout: true
- - !Command
cmd: "{{ editor }} dev-tools/doap/solr.rdf"
comment: Edit Solr DOAP, add version {{ release_version }}
stdout: true
- !Command
- cmd: git add dev-tools/doap/lucene.rdf dev-tools/doap/solr.rdf && git commit -m "DOAP changes for release {{ release_version }}"
+ cmd: git add dev-tools/doap/solr.rdf && git commit -m "DOAP changes for release {{ release_version }}"
logfile: commit.log
stdout: true
- !Command
@@ -1439,13 +1327,8 @@ groups:
Do any last-minute necessary edits to the text as you copy it over to the email.
todos:
- !Todo
- id: announce_lucene
- title: Announce the Lucene release (@l.a.o)
- description: |
- (( template=announce_lucene_mail ))
- - !Todo
id: announce_solr
- title: Announce the Solr release (@l.a.o)
+ title: Announce the Solr release (@s.a.o)
description: |
(( template=announce_solr_mail ))
- !Todo
@@ -1460,7 +1343,7 @@ groups:
If you prefer to sign the announcements manually rather than using a plugin,
you can do so from the command line and then copy the output into your mail program.
- gpg --output - --clearsign lucene_announce.txt
+ gpg --output - --clearsign solr_announce.txt
links:
- https://www.openpgp.org/software/
- https://ssd.eff.org/en/module/how-use-pgp-mac-os-x
@@ -1468,11 +1351,6 @@ groups:
- https://ssd.eff.org/en/module/how-use-pgp-windows
- https://www.openpgp.org/software/mailvelope/
- !Todo
- id: announce_lucene_sig
- title: Announce the Lucene release (announce@a.o)
- description: |
- (( template=announce_lucene_sign_mail ))
- - !Todo
id: announce_solr_sig
title: Announce the Solr release (announce@a.o)
description: |
@@ -1485,18 +1363,17 @@ groups:
Major versions should have a small new paragraph under 'History'.
If you know other languages than English, edit those as well.
links:
- - https://en.wikipedia.org/wiki/Apache_Lucene
- https://en.wikipedia.org/wiki/Apache_Solr
- !Todo
id: add_to_apache_reporter
title: Add the new version to the Apache Release Reporter
description: |
- Go to the Apache Release Reporter and add a release for lucene.
+ Go to the Apache Release Reporter and add a release for Solr.
Fill in the same date that you used for the release in previous steps.
- Do not use a product name prefix for the version, as this is the main release of the lucene PMC.
+ Do not use a product name prefix for the version, as this is the main release of the Solr PMC.
Just use the version of this release: {{ release_version }}
links:
- - https://reporter.apache.org/addrelease.html?lucene
+ - https://reporter.apache.org/addrelease.html?solr
- !TodoGroup
id: post_release
title: Tasks to do after release.
@@ -1564,7 +1441,7 @@ groups:
logfile: checkout-release.log
stdout: true
- !Command
- cmd: python3 -u -B dev-tools/scripts/releasedJirasRegex.py {{ release_version }} lucene/CHANGES.txt && python3 -u -B dev-tools/scripts/releasedJirasRegex.py {{ release_version }} solr/CHANGES.txt
+ cmd: python3 -u -B dev-tools/scripts/releasedJirasRegex.py {{ release_version }} solr/CHANGES.txt
tee: true
comment: Find version regexes
- !Command
@@ -1573,11 +1450,7 @@ groups:
logfile: checkout-main.log
- !Command
cmd: "{{ editor }} solr/CHANGES.txt"
- comment: Edit Solr CHANGES, do necessary changes
- stdout: true
- - !Command
- cmd: "{{ editor }} lucene/CHANGES.txt"
- comment: Edit Lucene CHANGES, do necessary changes
+ comment: Edit CHANGES.txt for main branch, do necessary changes
stdout: true
- !Command
cmd: git add -u . && git commit -m "Sync CHANGES for {{ release_version }}" && git push
@@ -1588,11 +1461,7 @@ groups:
logfile: checkout-stable.log
- !Command
cmd: "{{ editor }} solr/CHANGES.txt"
- comment: Edit Solr CHANGES, do necessary changes
- stdout: true
- - !Command
- cmd: "{{ editor }} lucene/CHANGES.txt"
- comment: Edit Lucene CHANGES, do necessary changes
+ comment: Edit CHANGES.txt for stable branch, do necessary changes
stdout: true
- !Command
cmd: git add -u . && git commit -m "Sync CHANGES for {{ release_version }}" && git push
@@ -1632,43 +1501,26 @@ groups:
. Fill in the release date ({{ release_date | formatdate }})
. It will give the option of transitioning issues marked fix-for the released version to the
next version, but do not do this as it will send an email for each issue :)
-
- This needs to be done both for Lucene and Solr JIRAs, see links.
links:
- - https://issues.apache.org/jira/plugins/servlet/project-config/LUCENE/versions
- https://issues.apache.org/jira/plugins/servlet/project-config/SOLR/versions
- !Todo
id: jira_close_resolved
title: Close all issues resolved in the release
description: |-
- Go to JIRA search in both Solr and Lucene and find all issues that were fixed in the release
+ Go to JIRA search to find all issues that were fixed in the release
you just made, whose Status is Resolved.
- . Go to https://issues.apache.org/jira/issues/?jql=project+in+(LUCENE,SOLR)+AND+status=Resolved+AND+fixVersion={{ release_version }}
+ . Go to https://issues.apache.org/jira/issues/?jql=project+in+(SOLR)+AND+status=Resolved+AND+fixVersion={{ release_version }}
. Do a bulk change (Under Tools... menu) to close all of these issues. This is a workflow transition task
. In the 'Comment' box type `Closing after the {{ release_version }} release`
. *Uncheck* the box that says `Send mail for this update`
-
- This needs to be done both for Lucene and Solr JIRAs, see links.
links:
- - https://issues.apache.org/jira/issues/?jql=project+in+(LUCENE,SOLR)+AND+status=Resolved+AND+fixVersion={{ release_version }}
+ - https://issues.apache.org/jira/issues/?jql=project+in+(SOLR)+AND+status=Resolved+AND+fixVersion={{ release_version }}
- !Todo
id: jira_change_unresolved
title: Remove fixVersion for unresolved
description: |-
Do another JIRA search to find all issues with Resolution=_Unresolved_ and fixVersion=_{{ release_version }}_.
- You need to do this separately for Lucene and Solr.
-
- *Lucene*
-
- . Open https://issues.apache.org/jira/issues/?jql=project+=+LUCENE+AND+resolution=Unresolved+AND+fixVersion={{ release_version }}
- . In the `Tools` menu, start a bulk change - operation="Edit issues"
- . Identify issues that *are included* in the release, but are unresolved e.g. due to being REOPENED. These shall *not* be bulk changed!
- . Check the box next to `Change Fix Version/s` and in the dropdown `Find and remove these`, selecting v {{ release_version }}
- . On the bottom of the form, uncheck the box that says `Send mail for this update`
- . Click `Next`, review the changes and click `Confirm`
-
- *Solr*
. Open https://issues.apache.org/jira/issues/?jql=project+=+SOLR+AND+resolution=Unresolved+AND+fixVersion={{ release_version }}
. In the `Tools` menu, start a bulk change - operation="Edit issues"
@@ -1677,7 +1529,6 @@ groups:
. On the bottom of the form, uncheck the box that says `Send mail for this update`
. Click `Next`, review the changes and click `Confirm`
links:
- - https://issues.apache.org/jira/issues/?jql=project+=+LUCENE+AND+resolution=Unresolved+AND+fixVersion={{ release_version }}
- https://issues.apache.org/jira/issues/?jql=project+=+SOLR+AND+resolution=Unresolved+AND+fixVersion={{ release_version }}
- !Todo
id: jira_clear_security
@@ -1702,23 +1553,20 @@ groups:
Go to the JIRA "Manage Versions" Administration pages and add the new version:
. Create a new (unreleased) version `{{ get_next_version }}`
-
- This needs to be done both for Lucene and Solr JIRAs, see links.
types:
- bugfix
links:
- - https://issues.apache.org/jira/plugins/servlet/project-config/LUCENE/versions
- https://issues.apache.org/jira/plugins/servlet/project-config/SOLR/versions
- !Todo
id: stop_promoting_old
title: Stop promoting old releases
description: |
Shortly after new releases are first published, they are automatically copied to the archives.
- Only the latest point release from each active branch should be kept under the Lucene PMC
- svnpubsub areas `dist/releases/lucene/` and `dist/releases/solr/`. Older releases can be
+ Only the latest point release from each active branch should be kept under the Solr PMC
+ svnpubsub area `dist/releases/solr/`. Older releases can be
safely deleted, since they are already backed up in the archives.
- Currenlty these versions exist in the distribution directory:
+ Currently these versions exist in the distribution directory:
*{{ mirrored_versions|join(', ') }}*
@@ -1732,14 +1580,18 @@ groups:
root_folder: '{{ git_checkout_folder }}'
commands_text: |
Run these commands to delete proposed versions from distribution directory.
+ Note, as long as we have some releases (7.x, 8.x) in Lucene dist repo and other
+ releases (9.0 ->) in the Solr dist repo, we may need to delete two places.
WARNING: Validate that the proposal is correct!
commands:
- !Command
cmd: |
- svn rm -m "Stop publishing old Lucene releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/java/{{ ver }}{% endfor %}
- logfile: svn-rm-lucene.log
+ svn rm -m "Stop publishing old Solr releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/solr/{{ ver }}{% endfor %}
+ logfile: svn-rm-solr.log
+ comment: Delete from Lucene dist area
- !Command
cmd: |
- svn rm -m "Stop publishing old Solr releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/solr/{{ ver }}{% endfor %}
+ svn rm -m "Stop publishing old Solr releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/solr/{{ ver }}{% endfor %}
logfile: svn-rm-solr.log
+ comment: Delete from new Solr dist area
diff --git a/dev-tools/scripts/requirements.txt b/dev-tools/scripts/requirements.txt
index b8a124b..20715e7 100644
--- a/dev-tools/scripts/requirements.txt
+++ b/dev-tools/scripts/requirements.txt
@@ -3,6 +3,6 @@ Jinja2>=2.10.1
PyYAML>=5.1
holidays>=0.9.10
ics>=0.4
-console-menu>=0.5.1
+console-menu==0.5.1
PyGithub
jira
\ No newline at end of file
diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py
index 93d062f..fd12cea 100644
--- a/dev-tools/scripts/scriptutil.py
+++ b/dev-tools/scripts/scriptutil.py
@@ -21,7 +21,6 @@ import os
from enum import Enum
import time
import urllib.request, urllib.error, urllib.parse
-import urllib.parse
class Version(object):
def __init__(self, major, minor, bugfix, prerelease):
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
index eca6e65..bb235dc 100755
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -31,23 +31,20 @@ import textwrap
import traceback
import urllib.error
import urllib.parse
-import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
import zipfile
-from collections import defaultdict
from collections import namedtuple
import scriptutil
-import checkJavadocLinks
-
-# This tool expects to find /lucene and /solr off the base URL. You
+# This tool expects to find /solr off the base URL. You
# must have a working gpg, tar, unzip in your path. This has been
# tested on Linux and on Cygwin under Windows 7.
cygwin = platform.system().lower().startswith('cygwin')
cygwinWindowsRoot = os.popen('cygpath -w /').read().strip().replace('\\','/') if cygwin else ''
+
def unshortenURL(url):
parsed = urllib.parse.urlparse(url)
if parsed[0] in ('http', 'https'):
@@ -56,26 +53,18 @@ def unshortenURL(url):
response = h.getresponse()
if int(response.status/100) == 3 and response.getheader('Location'):
return response.getheader('Location')
- return url
+ return url
# TODO
-# + verify KEYS contains key that signed the release
-# + make sure changes HTML looks ok
-# - verify license/notice of all dep jars
-# - check maven
-# - check JAR manifest version
-# - check license/notice exist
-# - check no "extra" files
# - make sure jars exist inside bin release
-# - run "ant test"
# - make sure docs exist
-# - use java5 for lucene/modules
reHREF = re.compile('<a href="(.*?)">(.*?)</a>')
# Set to False to avoid re-downloading the packages...
FORCE_CLEAN = True
+
def getHREFs(urlString):
# Deref any redirects
@@ -102,7 +91,7 @@ def getHREFs(urlString):
print('\nFAILED to open url %s' % urlString)
traceback.print_exc()
raise
-
+
for subUrl, text in reHREF.findall(html):
fullURL = urllib.parse.urljoin(urlString, subUrl)
links.append((text, fullURL))
@@ -117,19 +106,23 @@ def load(urlString):
content = urllib.request.urlopen(urlString).read().decode('utf-8')
return content
+
def noJavaPackageClasses(desc, file):
with zipfile.ZipFile(file) as z2:
for name2 in z2.namelist():
if name2.endswith('.class') and (name2.startswith('java/') or name2.startswith('javax/')):
raise RuntimeError('%s contains sheisty class "%s"' % (desc, name2))
+
def decodeUTF8(bytes):
return codecs.getdecoder('UTF-8')(bytes)[0]
+
MANIFEST_FILE_NAME = 'META-INF/MANIFEST.MF'
NOTICE_FILE_NAME = 'META-INF/NOTICE.txt'
LICENSE_FILE_NAME = 'META-INF/LICENSE.txt'
+
def checkJARMetaData(desc, jarFile, gitRevision, version):
with zipfile.ZipFile(jarFile, 'r') as z:
@@ -141,21 +134,19 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
raise RuntimeError('%s is missing %s' % (desc, name))
except KeyError:
raise RuntimeError('%s is missing %s' % (desc, name))
-
+
s = decodeUTF8(z.read(MANIFEST_FILE_NAME))
-
+
for verify in (
'Specification-Vendor: The Apache Software Foundation',
'Implementation-Vendor: The Apache Software Foundation',
- # Make sure 1.8 compiler was used to build release bits:
+ 'Specification-Title: Solr Search Engine:',
+ 'Implementation-Title: org.apache.solr',
'X-Compile-Source-JDK: 11',
- # Make sure 1.8, 1.9 or 1.10 ant was used to build release bits: (this will match 1.8.x, 1.9.x, 1.10.x)
- ('Ant-Version: Apache Ant 1.8', 'Ant-Version: Apache Ant 1.9', 'Ant-Version: Apache Ant 1.10'),
- # Make sure .class files are 1.8 format:
'X-Compile-Target-JDK: 11',
'Specification-Version: %s' % version,
- # Make sure the release was compiled with 1.8:
- 'Created-By: 11'):
+ 'X-Build-JDK: 11.',
+ 'Extension-Name: org.apache.solr'):
if type(verify) is not tuple:
verify = (verify,)
for x in verify:
@@ -169,119 +160,90 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
if gitRevision != 'skip':
# Make sure this matches the version and git revision we think we are releasing:
- # TODO: LUCENE-7023: is it OK that Implementation-Version's value now spans two lines?
- verifyRevision = 'Implementation-Version: %s %s' % (version, gitRevision)
- if s.find(verifyRevision) == -1:
- raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF (wrong git revision?)' % \
+ match = re.search("Implementation-Version: (.+\r\n .+)", s, re.MULTILINE)
+ if match:
+ implLine = match.group(1).replace("\r\n ", "")
+ verifyRevision = '%s %s' % (version, gitRevision)
+ if implLine.find(verifyRevision) == -1:
+ raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF (wrong git revision?)' % \
(desc, verifyRevision))
+ else:
+ raise RuntimeError('%s is missing Implementation-Version inside its META-INF/MANIFEST.MF' % desc)
notice = decodeUTF8(z.read(NOTICE_FILE_NAME))
license = decodeUTF8(z.read(LICENSE_FILE_NAME))
- justFileName = os.path.split(desc)[1]
-
- if justFileName.lower().find('solr') != -1:
- if SOLR_LICENSE is None:
- raise RuntimeError('BUG in smokeTestRelease!')
- if SOLR_NOTICE is None:
- raise RuntimeError('BUG in smokeTestRelease!')
- if notice != SOLR_NOTICE:
- raise RuntimeError('%s: %s contents doesn\'t match main NOTICE.txt' % \
- (desc, NOTICE_FILE_NAME))
- if license != SOLR_LICENSE:
- raise RuntimeError('%s: %s contents doesn\'t match main LICENSE.txt' % \
- (desc, LICENSE_FILE_NAME))
- else:
- if LUCENE_LICENSE is None:
- raise RuntimeError('BUG in smokeTestRelease!')
- if LUCENE_NOTICE is None:
- raise RuntimeError('BUG in smokeTestRelease!')
- if notice != LUCENE_NOTICE:
- raise RuntimeError('%s: %s contents doesn\'t match main NOTICE.txt' % \
- (desc, NOTICE_FILE_NAME))
- if license != LUCENE_LICENSE:
- raise RuntimeError('%s: %s contents doesn\'t match main LICENSE.txt' % \
- (desc, LICENSE_FILE_NAME))
+ if SOLR_LICENSE is None or SOLR_NOTICE is None:
+ raise RuntimeError('BUG in smokeTestRelease!')
+ if notice != SOLR_NOTICE:
+ raise RuntimeError('%s: %s contents doesn\'t match main NOTICE.txt' % \
+ (desc, NOTICE_FILE_NAME))
+ if license != SOLR_LICENSE:
+ raise RuntimeError('%s: %s contents doesn\'t match main LICENSE.txt' % \
+ (desc, LICENSE_FILE_NAME))
+
def normSlashes(path):
return path.replace(os.sep, '/')
-
-def checkAllJARs(topDir, project, gitRevision, version, tmpDir, baseURL):
+
+def checkAllJARs(topDir, gitRevision, version):
print(' verify JAR metadata/identity/no javax.* or java.* classes...')
- if project == 'solr':
- luceneDistFilenames = dict()
- for file in getBinaryDistFiles('lucene', tmpDir, version, baseURL):
- luceneDistFilenames[os.path.basename(file)] = file
- for root, dirs, files in os.walk(topDir):
+ for root, dirs, files in os.walk(topDir): # pylint: disable=unused-variable
normRoot = normSlashes(root)
- if project == 'solr' and normRoot.endswith('/server/lib'):
+ if normRoot.endswith('/server/lib'):
# Solr's example intentionally ships servlet JAR:
continue
for file in files:
if file.lower().endswith('.jar'):
- if project == 'solr':
- if ((normRoot.endswith('/test-framework/lib') and file.startswith('jersey-'))
- or (normRoot.endswith('/contrib/extraction/lib') and file.startswith('xml-apis-'))):
- print(' **WARNING**: skipping check of %s/%s: it has javax.* classes' % (root, file))
- continue
- else:
- if normRoot.endswith('/replicator/lib') and file.startswith('javax.servlet'):
- continue
+ if ((normRoot.endswith('/test-framework/lib') and file.startswith('jersey-'))
+ or (normRoot.endswith('/contrib/extraction/lib') and file.startswith('xml-apis-'))):
+ print(' **WARNING**: skipping check of %s/%s: it has javax.* classes' % (root, file))
+ continue
fullPath = '%s/%s' % (root, file)
noJavaPackageClasses('JAR file "%s"' % fullPath, fullPath)
- if file.lower().find('lucene') != -1 or file.lower().find('solr') != -1:
+ if file.lower().find('solr') != -1:
checkJARMetaData('JAR file "%s"' % fullPath, fullPath, gitRevision, version)
- if project == 'solr' and file.lower().find('lucene') != -1:
- jarFilename = os.path.basename(file)
- if jarFilename not in luceneDistFilenames:
- raise RuntimeError('Artifact %s is not present in Lucene binary distribution' % fullPath)
- identical = filecmp.cmp(fullPath, luceneDistFilenames[jarFilename], shallow=False)
- if not identical:
- raise RuntimeError('Artifact %s is not identical to %s in Lucene binary distribution'
- % (fullPath, luceneDistFilenames[jarFilename]))
-def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
-
+def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
print(' test basics...')
ents = getDirEntries(urlString)
artifact = None
changesURL = None
mavenURL = None
+ artifactURL = None
expectedSigs = []
if isSigned:
expectedSigs.append('asc')
expectedSigs.extend(['sha512'])
-
+ sigs = []
artifacts = []
+
for text, subURL in ents:
if text == 'KEYS':
- raise RuntimeError('%s: release dir should not contain a KEYS file - only toplevel /dist/lucene/KEYS is used' % project)
+ raise RuntimeError('solr: release dir should not contain a KEYS file - only toplevel /dist/solr/KEYS is used')
elif text == 'maven/':
mavenURL = subURL
elif text.startswith('changes'):
if text not in ('changes/', 'changes-%s/' % version):
- raise RuntimeError('%s: found %s vs expected changes-%s/' % (project, text, version))
+ raise RuntimeError('solr: found %s vs expected changes-%s/' % (text, version))
changesURL = subURL
- elif artifact == None:
+ elif artifact is None:
artifact = text
artifactURL = subURL
- if project == 'solr':
- expected = 'solr-%s' % version
- else:
- expected = 'lucene-%s' % version
+ expected = 'solr-%s' % version
if not artifact.startswith(expected):
- raise RuntimeError('%s: unknown artifact %s: expected prefix %s' % (project, text, expected))
+ raise RuntimeError('solr: unknown artifact %s: expected prefix %s' % (text, expected))
sigs = []
elif text.startswith(artifact + '.'):
sigs.append(text[len(artifact)+1:])
else:
if sigs != expectedSigs:
- raise RuntimeError('%s: artifact %s has wrong sigs: expected %s but got %s' % (project, artifact, expectedSigs, sigs))
+ raise RuntimeError('solr: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, sigs))
artifacts.append((artifact, artifactURL))
artifact = text
artifactURL = subURL
@@ -290,37 +252,32 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
if sigs != []:
artifacts.append((artifact, artifactURL))
if sigs != expectedSigs:
- raise RuntimeError('%s: artifact %s has wrong sigs: expected %s but got %s' % (project, artifact, expectedSigs, sigs))
+ raise RuntimeError('solr: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, sigs))
- if project == 'lucene':
- expected = ['lucene-%s-src.tgz' % version,
- 'lucene-%s.tgz' % version,
- 'lucene-%s.zip' % version]
- else:
- expected = ['solr-%s-src.tgz' % version,
- 'solr-%s.tgz' % version,
- 'solr-%s.zip' % version]
+ expected = ['solr-%s-src.tgz' % version,
+ 'solr-%s.tgz' % version,
+ 'solr-%s.zip' % version]
actual = [x[0] for x in artifacts]
if expected != actual:
- raise RuntimeError('%s: wrong artifacts: expected %s but got %s' % (project, expected, actual))
-
+ raise RuntimeError('solr: wrong artifacts: expected %s but got %s' % (expected, actual))
+
# Set up clean gpg world; import keys file:
- gpgHomeDir = '%s/%s.gpg' % (tmpDir, project)
+ gpgHomeDir = '%s/solr.gpg' % tmpDir
if os.path.exists(gpgHomeDir):
shutil.rmtree(gpgHomeDir)
os.makedirs(gpgHomeDir, 0o700)
run('gpg --homedir %s --import %s' % (gpgHomeDir, keysFile),
- '%s/%s.gpg.import.log' % (tmpDir, project))
+ '%s/solr.gpg.import.log' % tmpDir)
if mavenURL is None:
- raise RuntimeError('%s is missing maven' % project)
+ raise RuntimeError('solr is missing maven')
if changesURL is None:
- raise RuntimeError('%s is missing changes-%s' % (project, version))
- testChanges(project, version, changesURL)
+ raise RuntimeError('solr is missing changes-%s' % version)
+ testChanges(version, changesURL)
- for artifact, urlString in artifacts:
+ for artifact, urlString in artifacts: # pylint: disable=redefined-argument-from-local
print(' download %s...' % artifact)
scriptutil.download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN)
verifyDigests(artifact, urlString, tmpDir)
@@ -331,31 +288,31 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
scriptutil.download(artifact + '.asc', urlString + '.asc', tmpDir, force_clean=FORCE_CLEAN)
sigFile = '%s/%s.asc' % (tmpDir, artifact)
artifactFile = '%s/%s' % (tmpDir, artifact)
- logFile = '%s/%s.%s.gpg.verify.log' % (tmpDir, project, artifact)
- run('gpg --homedir %s --verify %s %s' % (gpgHomeDir, sigFile, artifactFile),
+ logFile = '%s/solr.%s.gpg.verify.log' % (tmpDir, artifact)
+ run('gpg --homedir %s --display-charset utf-8 --verify %s %s' % (gpgHomeDir, sigFile, artifactFile),
logFile)
# Forward any GPG warnings, except the expected one (since it's a clean world)
- f = open(logFile)
- for line in f.readlines():
- if line.lower().find('warning') != -1 \
- and line.find('WARNING: This key is not certified with a trusted signature') == -1:
- print(' GPG: %s' % line.strip())
- f.close()
+ with open(logFile) as f:
+ print("File: %s" % logFile)
+ for line in f.readlines():
+ if line.lower().find('warning') != -1 \
+ and line.find('WARNING: This key is not certified with a trusted signature') == -1:
+ print(' GPG: %s' % line.strip())
# Test trust (this is done with the real users config)
run('gpg --import %s' % (keysFile),
- '%s/%s.gpg.trust.import.log' % (tmpDir, project))
+ '%s/solr.gpg.trust.import.log' % tmpDir)
print(' verify trust')
- logFile = '%s/%s.%s.gpg.trust.log' % (tmpDir, project, artifact)
- run('gpg --verify %s %s' % (sigFile, artifactFile), logFile)
+ logFile = '%s/solr.%s.gpg.trust.log' % (tmpDir, artifact)
+ run('gpg --display-charset utf-8 --verify %s %s' % (sigFile, artifactFile), logFile)
# Forward any GPG warnings:
- f = open(logFile)
- for line in f.readlines():
- if line.lower().find('warning') != -1:
- print(' GPG: %s' % line.strip())
- f.close()
+ with open(logFile) as f:
+ for line in f.readlines():
+ if line.lower().find('warning') != -1:
+ print(' GPG: %s' % line.strip())
-def testChanges(project, version, changesURLString):
+
+def testChanges(version, changesURLString):
print(' check changes HTML...')
changesURL = None
for text, subURL in getDirEntries(changesURLString):
@@ -366,22 +323,25 @@ def testChanges(project, version, changesURLString):
raise RuntimeError('did not see Changes.html link from %s' % changesURLString)
s = load(changesURL)
- checkChangesContent(s, version, changesURL, project, True)
+ checkChangesContent(s, version, changesURL, True)
+
-def testChangesText(dir, version, project):
+def testChangesText(dir, version):
"Checks all CHANGES.txt under this dir."
- for root, dirs, files in os.walk(dir):
+ for root, dirs, files in os.walk(dir): # pylint: disable=unused-variable
# NOTE: O(N) but N should be smallish:
if 'CHANGES.txt' in files:
fullPath = '%s/CHANGES.txt' % root
#print 'CHECK %s' % fullPath
- checkChangesContent(open(fullPath, encoding='UTF-8').read(), version, fullPath, project, False)
+ checkChangesContent(open(fullPath, encoding='UTF-8').read(), version, fullPath, False)
reChangesSectionHREF = re.compile('<a id="(.*?)".*?>(.*?)</a>', re.IGNORECASE)
-reUnderbarNotDashHTML = re.compile(r'<li>(\s*(LUCENE|SOLR)_\d\d\d\d+)')
-reUnderbarNotDashTXT = re.compile(r'\s+((LUCENE|SOLR)_\d\d\d\d+)', re.MULTILINE)
-def checkChangesContent(s, version, name, project, isHTML):
+reUnderbarNotDashHTML = re.compile(r'<li>(\s*(SOLR)_\d\d\d\d+)')
+reUnderbarNotDashTXT = re.compile(r'\s+((SOLR)_\d\d\d\d+)', re.MULTILINE)
+
+
+def checkChangesContent(s, version, name, isHTML):
currentVersionTuple = versionToTuple(version, name)
if isHTML and s.find('Release %s' % version) == -1:
@@ -395,16 +355,12 @@ def checkChangesContent(s, version, name, project, isHTML):
m = r.search(s)
if m is not None:
raise RuntimeError('incorrect issue (_ instead of -) in %s: %s' % (name, m.group(1)))
-
+
if s.lower().find('not yet released') != -1:
raise RuntimeError('saw "not yet released" in %s' % name)
if not isHTML:
- if project == 'lucene':
- sub = 'Lucene %s' % version
- else:
- sub = version
-
+ sub = version
if s.find(sub) == -1:
# benchmark never seems to include release info:
if name.find('/benchmark/') == -1:
@@ -433,6 +389,8 @@ def checkChangesContent(s, version, name, project, isHTML):
reVersion = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?\s*(-alpha|-beta|final|RC\d+)?\s*(?:\[.*\])?', re.IGNORECASE)
+
+
def versionToTuple(version, name):
versionMatch = reVersion.match(version)
if versionMatch is None:
@@ -456,22 +414,24 @@ reUnixPath = re.compile(r'\b[a-zA-Z_]+=(?:"(?:\\"|[^"])*"' + '|(?:\\\\.|[^"\'\\s
+ r'|("/(?:\\.|[^"])*")' \
+ r"|('/(?:\\.|[^'])*')")
+
def unix2win(matchobj):
if matchobj.group(1) is not None: return cygwinWindowsRoot + matchobj.group()
if matchobj.group(2) is not None: return '"%s%s' % (cygwinWindowsRoot, matchobj.group().lstrip('"'))
if matchobj.group(3) is not None: return "'%s%s" % (cygwinWindowsRoot, matchobj.group().lstrip("'"))
return matchobj.group()
+
def cygwinifyPaths(command):
- # The problem: Native Windows applications running under Cygwin
- # (e.g. Ant, which isn't available as a Cygwin package) can't
+ # The problem: Native Windows applications running under Cygwin can't
# handle Cygwin's Unix-style paths. However, environment variable
# values are automatically converted, so only paths outside of
# environment variable values should be converted to Windows paths.
# Assumption: all paths will be absolute.
- if '; ant ' in command: command = reUnixPath.sub(unix2win, command)
+ if '; gradlew ' in command: command = reUnixPath.sub(unix2win, command)
return command
+
def printFileContents(fileName):
# Assume log file was written in system's default encoding, but
@@ -489,6 +449,7 @@ def printFileContents(fileName):
print(codecs.getdecoder(sys.stdout.encoding)(bytes)[0])
print()
+
def run(command, logFile):
if cygwin: command = cygwinifyPaths(command)
if os.system('%s > %s 2>&1' % (command, logFile)):
@@ -496,13 +457,14 @@ def run(command, logFile):
print('\ncommand "%s" failed:' % command)
printFileContents(logFile)
raise RuntimeError('command "%s" failed; see log file %s' % (command, logPath))
-
+
+
def verifyDigests(artifact, urlString, tmpDir):
print(' verify sha512 digest')
sha512Expected, t = load(urlString + '.sha512').strip().split()
if t != '*'+artifact:
raise RuntimeError('SHA512 %s.sha512 lists artifact %s but expected *%s' % (urlString, t, artifact))
-
+
s512 = hashlib.sha512()
f = open('%s/%s' % (tmpDir, artifact), 'rb')
while True:
@@ -515,6 +477,7 @@ def verifyDigests(artifact, urlString, tmpDir):
if sha512Actual != sha512Expected:
raise RuntimeError('SHA512 digest mismatch for %s: expected %s but got %s' % (artifact, sha512Expected, sha512Actual))
+
def getDirEntries(urlString):
if urlString.startswith('file:/') and not urlString.startswith('file://'):
# stupid bogus ant URI
@@ -537,18 +500,20 @@ def getDirEntries(urlString):
return l
else:
links = getHREFs(urlString)
- for i, (text, subURL) in enumerate(links):
+ for i, (text, subURL) in enumerate(links): # pylint: disable=unused-variable
if text == 'Parent Directory' or text == '..':
return links[(i+1):]
+ return None
-def unpackAndVerify(java, project, tmpDir, artifact, gitRevision, version, testArgs, baseURL):
+
+def unpackAndVerify(java, tmpDir, artifact, gitRevision, version, testArgs):
destDir = '%s/unpack' % tmpDir
if os.path.exists(destDir):
shutil.rmtree(destDir)
os.makedirs(destDir)
os.chdir(destDir)
print(' unpack %s...' % artifact)
- unpackLogFile = '%s/%s-unpack-%s.log' % (tmpDir, project, artifact)
+ unpackLogFile = '%s/solr-unpack-%s.log' % (tmpDir, artifact)
if artifact.endswith('.tar.gz') or artifact.endswith('.tgz'):
run('tar xzf %s/%s' % (tmpDir, artifact), unpackLogFile)
elif artifact.endswith('.zip'):
@@ -556,87 +521,79 @@ def unpackAndVerify(java, project, tmpDir, artifact, gitRevision, version, testA
# make sure it unpacks to proper subdir
l = os.listdir(destDir)
- expected = '%s-%s' % (project, version)
+ expected = 'solr-%s' % version
if l != [expected]:
raise RuntimeError('unpack produced entries %s; expected only %s' % (l, expected))
unpackPath = '%s/%s' % (destDir, expected)
- verifyUnpacked(java, project, artifact, unpackPath, gitRevision, version, testArgs, tmpDir, baseURL)
+ verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs)
return unpackPath
-LUCENE_NOTICE = None
-LUCENE_LICENSE = None
SOLR_NOTICE = None
SOLR_LICENSE = None
-def verifyUnpacked(java, project, artifact, unpackPath, gitRevision, version, testArgs, tmpDir, baseURL):
- global LUCENE_NOTICE
- global LUCENE_LICENSE
+def is_in_list(in_folder, files, indent=4):
+ for file_name in files:
+ print("%sChecking %s" % (" "*indent, file_name))
+ found = False
+ for f in [file_name, file_name + '.txt', file_name + '.md']:
+ if f in in_folder:
+ in_folder.remove(f)
+ found = True
+ if not found:
+ raise RuntimeError('file "%s" is missing' % file_name)
+
+
+def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
global SOLR_NOTICE
global SOLR_LICENSE
os.chdir(unpackPath)
isSrc = artifact.find('-src') != -1
-
- l = os.listdir(unpackPath)
- textFiles = ['LICENSE', 'NOTICE', 'README']
- if project == 'lucene':
- textFiles.extend(('JRE_VERSION_MIGRATION', 'CHANGES', 'MIGRATE', 'SYSTEM_REQUIREMENTS'))
- if isSrc:
- textFiles.append('BUILD')
-
- for fileName in textFiles:
- fileNameTxt = fileName + '.txt'
- fileNameMd = fileName + '.md'
- if fileNameTxt in l:
- l.remove(fileNameTxt)
- elif fileNameMd in l:
- l.remove(fileNameMd)
- else:
- raise RuntimeError('file "%s".[txt|md] is missing from artifact %s' % (fileName, artifact))
- if project == 'lucene':
- if LUCENE_NOTICE is None:
- LUCENE_NOTICE = open('%s/NOTICE.txt' % unpackPath, encoding='UTF-8').read()
- if LUCENE_LICENSE is None:
- LUCENE_LICENSE = open('%s/LICENSE.txt' % unpackPath, encoding='UTF-8').read()
+ # Check text files in release
+ print(" %s" % artifact)
+ in_root_folder = list(filter(lambda x: x[0] != '.', os.listdir(unpackPath)))
+ in_solr_folder = []
+ if isSrc:
+ in_solr_folder.extend(os.listdir(os.path.join(unpackPath, 'solr')))
+ is_in_list(in_root_folder, ['LICENSE', 'NOTICE', 'README'])
+ is_in_list(in_solr_folder, ['JRE_VERSION_MIGRATION', 'CHANGES', 'MIGRATE', 'SYSTEM_REQUIREMENTS'])
else:
- if SOLR_NOTICE is None:
- SOLR_NOTICE = open('%s/NOTICE.txt' % unpackPath, encoding='UTF-8').read()
- if SOLR_LICENSE is None:
- SOLR_LICENSE = open('%s/LICENSE.txt' % unpackPath, encoding='UTF-8').read()
-
- if not isSrc:
- # TODO: we should add verifyModule/verifySubmodule (e.g. analysis) here and recurse through
- if project == 'lucene':
- expectedJARs = ()
- else:
- expectedJARs = ()
-
- for fileName in expectedJARs:
- fileName += '.jar'
- if fileName not in l:
- raise RuntimeError('%s: file "%s" is missing from artifact %s' % (project, fileName, artifact))
- l.remove(fileName)
+ is_in_list(in_root_folder, ['LICENSE', 'NOTICE', 'README', 'JRE_VERSION_MIGRATION', 'CHANGES',
+ 'MIGRATE', 'SYSTEM_REQUIREMENTS'])
+
+ if SOLR_NOTICE is None:
+ SOLR_NOTICE = open('%s/NOTICE.txt' % unpackPath, encoding='UTF-8').read()
+ if SOLR_LICENSE is None:
+ SOLR_LICENSE = open('%s/LICENSE.txt' % unpackPath, encoding='UTF-8').read()
+
+ # if not isSrc:
+ # # TODO: we should add verifyModule/verifySubmodule (e.g. analysis) here and recurse through
+ # expectedJARs = ()
+ #
+ # for fileName in expectedJARs:
+ # fileName += '.jar'
+ # if fileName not in l:
+ # raise RuntimeError('solr: file "%s" is missing from artifact %s' % (fileName, artifact))
+ # in_root_folder.remove(fileName)
- if project == 'lucene':
- # TODO: clean this up to not be a list of modules that we must maintain
- extras = ('analysis', 'backward-codecs', 'benchmark', 'classification', 'codecs', 'core', 'demo', 'docs', 'expressions', 'facet', 'grouping', 'highlighter', 'join', 'luke', 'memory', 'misc', 'monitor', 'queries', 'queryparser', 'replicator', 'sandbox', 'spatial-extras', 'spatial3d', 'suggest', 'test-framework', 'licenses')
- if isSrc:
- extras += ('build.gradle', 'build.xml', 'common-build.xml', 'module-build.xml', 'top-level-ivy-settings.xml', 'default-nested-ivy-settings.xml', 'ivy-versions.properties', 'ivy-ignore-conflicts.properties', 'tools', 'site', 'dev-docs')
+ if isSrc:
+ expected_src_root_folders = ['buildSrc', 'dev-docs', 'dev-tools', 'gradle', 'help', 'solr']
+ expected_src_root_files = ['build.gradle', 'gradlew', 'gradlew.bat', 'settings.gradle', 'versions.lock', 'versions.props']
+ expected_src_solr_files = ['build.gradle']
+ expected_src_solr_folders = ['benchmark', 'bin', 'bin-test', 'build', 'contrib', 'core', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'server', 'solr-ref-guide', 'solrj', 'test-framework', 'webapp']
+ is_in_list(in_root_folder, expected_src_root_folders)
+ is_in_list(in_root_folder, expected_src_root_files)
+ is_in_list(in_solr_folder, expected_src_solr_folders)
+ is_in_list(in_solr_folder, expected_src_solr_files)
+ if len(in_solr_folder) > 0:
+ raise RuntimeError('solr: unexpected files/dirs in artifact %s solr/ folder: %s' % (artifact, in_solr_folder))
else:
- extras = ()
+ is_in_list(in_root_folder, ['bin', 'contrib', 'dist', 'docs', 'example', 'licenses', 'server'])
- # TODO: if solr, verify lucene/licenses, solr/licenses are present
-
- for e in extras:
- if e not in l:
- raise RuntimeError('%s: %s missing from artifact %s' % (project, e, artifact))
- l.remove(e)
-
- if project == 'lucene':
- if len(l) > 0:
- raise RuntimeError('%s: unexpected files/dirs in artifact %s: %s' % (project, artifact, l))
+ if len(in_root_folder) > 0:
+ raise RuntimeError('solr: unexpected files/dirs in artifact %s: %s' % (artifact, in_root_folder))
if isSrc:
print(' make sure no JARs/WARs in src dist...')
@@ -653,115 +610,49 @@ def verifyUnpacked(java, project, artifact, unpackPath, gitRevision, version, te
print(' %s' % line.strip())
raise RuntimeError('source release has WARs...')
- # Can't run documentation-lint in lucene src, because dev-tools is missing
- validateCmd = 'ant validate' if project == 'lucene' else 'ant validate documentation-lint';
+ validateCmd = './gradlew --no-daemon check -p solr/documentation'
print(' run "%s"' % validateCmd)
java.run_java11(validateCmd, '%s/validate.log' % unpackPath)
- if project == 'lucene':
- print(" run tests w/ Java 11 and testArgs='%s'..." % testArgs)
- java.run_java11('ant clean test %s' % testArgs, '%s/test.log' % unpackPath)
- java.run_java11('ant jar', '%s/compile.log' % unpackPath)
- testDemo(java.run_java11, isSrc, version, '11')
-
- print(' generate javadocs w/ Java 11...')
- java.run_java11('ant javadocs', '%s/javadocs.log' % unpackPath)
- checkBrokenLinks('%s/build/docs' % unpackPath)
-
- if java.run_java12:
- print(" run tests w/ Java 12 and testArgs='%s'..." % testArgs)
- java.run_java12('ant clean test %s' % testArgs, '%s/test.log' % unpackPath)
- java.run_java12('ant jar', '%s/compile.log' % unpackPath)
- testDemo(java.run_java12, isSrc, version, '12')
-
- #print(' generate javadocs w/ Java 12...')
- #java.run_java12('ant javadocs', '%s/javadocs.log' % unpackPath)
- #checkBrokenLinks('%s/build/docs' % unpackPath)
-
- else:
- os.chdir('solr')
-
- print(" run tests w/ Java 11 and testArgs='%s'..." % testArgs)
- java.run_java11('ant clean test -Dtests.slow=false %s' % testArgs, '%s/test.log' % unpackPath)
-
- # test javadocs
- print(' generate javadocs w/ Java 11...')
- java.run_java11('ant clean javadocs', '%s/javadocs.log' % unpackPath)
- checkBrokenLinks('%s/solr/build/docs')
-
- print(' test solr example w/ Java 11...')
- java.run_java11('ant clean server', '%s/antexample.log' % unpackPath)
- testSolrExample(unpackPath, java.java11_home, True)
-
- if java.run_java12:
- print(" run tests w/ Java 12 and testArgs='%s'..." % testArgs)
- java.run_java12('ant clean test -Dtests.slow=false %s' % testArgs, '%s/test.log' % unpackPath)
-
- #print(' generate javadocs w/ Java 12...')
- #java.run_java12('ant clean javadocs', '%s/javadocs.log' % unpackPath)
- #checkBrokenLinks('%s/solr/build/docs' % unpackPath)
+ print(" run tests w/ Java 11 and testArgs='%s'..." % testArgs)
+ java.run_java11('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
+ print(" compile jars w/ Java 11")
+ java.run_java11('./gradlew --no-daemon jar -Dversion.release=%s' % version, '%s/compile.log' % unpackPath)
+ testSolrExample(unpackPath, java.java11_home, True)
- print(' test solr example w/ Java 12...')
- java.run_java12('ant clean server', '%s/antexample.log' % unpackPath)
- testSolrExample(unpackPath, java.java12_home, True)
-
- os.chdir('..')
- print(' check NOTICE')
- testNotice(unpackPath)
+ if java.run_java17:
+ print(" run tests w/ Java 17 and testArgs='%s'..." % testArgs)
+ java.run_java17('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
+ print(" compile jars w/ Java 17")
+ java.run_java17('./gradlew --no-daemon jar -Dversion.release=%s' % version, '%s/compile.log' % unpackPath)
else:
+ checkAllJARs(os.getcwd(), gitRevision, version)
+
+ print(' copying unpacked distribution for Java 11 ...')
+ java11UnpackPath = '%s-java11' % unpackPath
+ if os.path.exists(java11UnpackPath):
+ shutil.rmtree(java11UnpackPath)
+ shutil.copytree(unpackPath, java11UnpackPath)
+ os.chdir(java11UnpackPath)
+ print(' test solr example w/ Java 11...')
+ testSolrExample(java11UnpackPath, java.java11_home, False)
+
+ if java.run_java17:
+ print(' copying unpacked distribution for Java 17 ...')
+ java17UnpackPath = '%s-java17' % unpackPath
+ if os.path.exists(java17UnpackPath):
+ shutil.rmtree(java17UnpackPath)
+ shutil.copytree(unpackPath, java17UnpackPath)
+ os.chdir(java17UnpackPath)
+ print(' test solr example w/ Java 17...')
+ testSolrExample(java17UnpackPath, java.java17_home, False)
- checkAllJARs(os.getcwd(), project, gitRevision, version, tmpDir, baseURL)
-
- if project == 'lucene':
- testDemo(java.run_java11, isSrc, version, '11')
- if java.run_java12:
- testDemo(java.run_java12, isSrc, version, '12')
-
- else:
- print(' copying unpacked distribution for Java 11 ...')
- java11UnpackPath = '%s-java11' % unpackPath
- if os.path.exists(java11UnpackPath):
- shutil.rmtree(java11UnpackPath)
- shutil.copytree(unpackPath, java11UnpackPath)
- os.chdir(java11UnpackPath)
- print(' test solr example w/ Java 11...')
- testSolrExample(java11UnpackPath, java.java11_home, False)
-
- if java.run_java12:
- print(' copying unpacked distribution for Java 12 ...')
- java12UnpackPath = '%s-java12' % unpackPath
- if os.path.exists(java12UnpackPath):
- shutil.rmtree(java12UnpackPath)
- shutil.copytree(unpackPath, java12UnpackPath)
- os.chdir(java12UnpackPath)
- print(' test solr example w/ Java 12...')
- testSolrExample(java12UnpackPath, java.java12_home, False)
-
- os.chdir(unpackPath)
+ os.chdir(unpackPath)
- testChangesText('.', version, project)
+ testChangesText('.', version)
- if project == 'lucene' and isSrc:
- print(' confirm all releases have coverage in TestBackwardsCompatibility')
- confirmAllReleasesAreTestedForBackCompat(version, unpackPath)
-
-def testNotice(unpackPath):
- solrNotice = open('%s/NOTICE.txt' % unpackPath, encoding='UTF-8').read()
- luceneNotice = open('%s/lucene/NOTICE.txt' % unpackPath, encoding='UTF-8').read()
-
- expected = """
-=========================================================================
-== Apache Lucene Notice ==
-=========================================================================
-
-""" + luceneNotice + """---
-"""
-
- if solrNotice.find(expected) == -1:
- raise RuntimeError('Solr\'s NOTICE.txt does not have the verbatim copy, plus header/footer, of Lucene\'s NOTICE.txt')
-
def readSolrOutput(p, startupEvent, failureEvent, logFile):
f = open(logFile, 'wb')
try:
@@ -861,88 +752,40 @@ def testSolrExample(unpackPath, javaPath, isSrc):
else:
os.chdir(unpackPath)
-# check for broken links
-def checkBrokenLinks(path):
- # also validate html/check for broken links
- if checkJavadocLinks.checkAll(path):
- raise RuntimeError('broken javadocs links found!')
-
-def testDemo(run_java, isSrc, version, jdk):
- if os.path.exists('index'):
- shutil.rmtree('index') # nuke any index from any previous iteration
-
- print(' test demo with %s...' % jdk)
- sep = ';' if cygwin else ':'
- if isSrc:
- cp = 'build/core/classes/java{0}build/demo/classes/java{0}build/analysis/common/classes/java{0}build/queryparser/classes/java'.format(sep)
- docsDir = 'core/src'
- else:
- cp = 'core/lucene-core-{0}.jar{1}demo/lucene-demo-{0}.jar{1}analysis/common/lucene-analyzers-common-{0}.jar{1}queryparser/lucene-queryparser-{0}.jar'.format(version, sep)
- docsDir = 'docs'
- run_java('java -cp "%s" org.apache.lucene.demo.IndexFiles -index index -docs %s' % (cp, docsDir), 'index.log')
- run_java('java -cp "%s" org.apache.lucene.demo.SearchFiles -index index -query lucene' % cp, 'search.log')
- reMatchingDocs = re.compile('(\d+) total matching documents')
- m = reMatchingDocs.search(open('search.log', encoding='UTF-8').read())
- if m is None:
- raise RuntimeError('lucene demo\'s SearchFiles found no results')
- else:
- numHits = int(m.group(1))
- if numHits < 100:
- raise RuntimeError('lucene demo\'s SearchFiles found too few results: %s' % numHits)
- print(' got %d hits for query "lucene"' % numHits)
- print(' checkindex with %s...' % jdk)
- run_java('java -ea -cp "%s" org.apache.lucene.index.CheckIndex index' % cp, 'checkindex.log')
- s = open('checkindex.log').read()
- m = re.search(r'^\s+version=(.*?)$', s, re.MULTILINE)
- if m is None:
- raise RuntimeError('unable to locate version=NNN output from CheckIndex; see checkindex.log')
- actualVersion = m.group(1)
- if removeTrailingZeros(actualVersion) != removeTrailingZeros(version):
- raise RuntimeError('wrong version from CheckIndex: got "%s" but expected "%s"' % (actualVersion, version))
def removeTrailingZeros(version):
return re.sub(r'(\.0)*$', '', version)
-def checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile):
- POMtemplates = defaultdict()
- getPOMtemplates(solrSrcUnpackPath, POMtemplates, tmpDir)
+
+def checkMaven(baseURL, tmpDir, gitRevision, version, isSigned, keysFile):
print(' download artifacts')
- artifacts = {'lucene': [], 'solr': []}
- for project in ('lucene', 'solr'):
- artifactsURL = '%s/%s/maven/org/apache/%s/' % (baseURL, project, project)
- targetDir = '%s/maven/org/apache/%s' % (tmpDir, project)
- if not os.path.exists(targetDir):
- os.makedirs(targetDir)
- crawl(artifacts[project], artifactsURL, targetDir)
+ artifacts = []
+ artifactsURL = '%s/solr/maven/org/apache/solr/' % baseURL
+ targetDir = '%s/maven/org/apache/solr' % tmpDir
+ if not os.path.exists(targetDir):
+ os.makedirs(targetDir)
+ crawl(artifacts, artifactsURL, targetDir)
print()
verifyPOMperBinaryArtifact(artifacts, version)
- verifyArtifactPerPOMtemplate(POMtemplates, artifacts, tmpDir, version)
verifyMavenDigests(artifacts)
checkJavadocAndSourceArtifacts(artifacts, version)
verifyDeployedPOMsCoordinates(artifacts, version)
if isSigned:
- verifyMavenSigs(baseURL, tmpDir, artifacts, keysFile)
+ verifyMavenSigs(tmpDir, artifacts, keysFile)
- distFiles = getBinaryDistFilesForMavenChecks(tmpDir, version, baseURL)
+ distFiles = getBinaryDistFiles(tmpDir, version, baseURL)
checkIdenticalMavenArtifacts(distFiles, artifacts, version)
- checkAllJARs('%s/maven/org/apache/lucene' % tmpDir, 'lucene', gitRevision, version, tmpDir, baseURL)
- checkAllJARs('%s/maven/org/apache/solr' % tmpDir, 'solr', gitRevision, version, tmpDir, baseURL)
+ checkAllJARs('%s/maven/org/apache/solr' % tmpDir, gitRevision, version)
-def getBinaryDistFilesForMavenChecks(tmpDir, version, baseURL):
- # TODO: refactor distribution unpacking so that it only happens once per distribution per smoker run
- distFiles = defaultdict()
- for project in ('lucene', 'solr'):
- distFiles[project] = getBinaryDistFiles(project, tmpDir, version, baseURL)
- return distFiles
-
-def getBinaryDistFiles(project, tmpDir, version, baseURL):
- distribution = '%s-%s.tgz' % (project, version)
+
+def getBinaryDistFiles(tmpDir, version, baseURL):
+ distribution = 'solr-%s.tgz' % version
if not os.path.exists('%s/%s' % (tmpDir, distribution)):
- distURL = '%s/%s/%s' % (baseURL, project, distribution)
+ distURL = '%s/solr/%s' % (baseURL, distribution)
print(' download %s...' % distribution, end=' ')
scriptutil.download(distribution, distURL, tmpDir, force_clean=FORCE_CLEAN)
- destDir = '%s/unpack-%s-getBinaryDistFiles' % (tmpDir, project)
+ destDir = '%s/unpack-solr-getBinaryDistFiles' % tmpDir
if os.path.exists(destDir):
shutil.rmtree(destDir)
os.makedirs(destDir)
@@ -951,21 +794,22 @@ def getBinaryDistFiles(project, tmpDir, version, baseURL):
unpackLogFile = '%s/unpack-%s-getBinaryDistFiles.log' % (tmpDir, distribution)
run('tar xzf %s/%s' % (tmpDir, distribution), unpackLogFile)
distributionFiles = []
- for root, dirs, files in os.walk(destDir):
+ for root, dirs, files in os.walk(destDir): # pylint: disable=unused-variable
distributionFiles.extend([os.path.join(root, file) for file in files])
return distributionFiles
+
def checkJavadocAndSourceArtifacts(artifacts, version):
print(' check for javadoc and sources artifacts...')
- for project in ('lucene', 'solr'):
- for artifact in artifacts[project]:
- if artifact.endswith(version + '.jar'):
- javadocJar = artifact[:-4] + '-javadoc.jar'
- if javadocJar not in artifacts[project]:
- raise RuntimeError('missing: %s' % javadocJar)
- sourcesJar = artifact[:-4] + '-sources.jar'
- if sourcesJar not in artifacts[project]:
- raise RuntimeError('missing: %s' % sourcesJar)
+ for artifact in artifacts:
+ if artifact.endswith(version + '.jar'):
+ javadocJar = artifact[:-4] + '-javadoc.jar'
+ if javadocJar not in artifacts:
+ raise RuntimeError('missing: %s' % javadocJar)
+ sourcesJar = artifact[:-4] + '-sources.jar'
+ if sourcesJar not in artifacts:
+ raise RuntimeError('missing: %s' % sourcesJar)
+
def getZipFileEntries(fileName):
entries = []
@@ -976,57 +820,57 @@ def getZipFileEntries(fileName):
entries.sort()
return entries
+
def checkIdenticalMavenArtifacts(distFiles, artifacts, version):
print(' verify that Maven artifacts are same as in the binary distribution...')
- reJarWar = re.compile(r'%s\.[wj]ar$' % version) # exclude *-javadoc.jar and *-sources.jar
- for project in ('lucene', 'solr'):
- distFilenames = dict()
- for file in distFiles[project]:
- baseName = os.path.basename(file)
- distFilenames[baseName] = file
- for artifact in artifacts[project]:
- if reJarWar.search(artifact):
- artifactFilename = os.path.basename(artifact)
- if artifactFilename not in distFilenames:
- raise RuntimeError('Maven artifact %s is not present in %s binary distribution'
- % (artifact, project))
- else:
- identical = filecmp.cmp(artifact, distFilenames[artifactFilename], shallow=False)
- if not identical:
- raise RuntimeError('Maven artifact %s is not identical to %s in %s binary distribution'
- % (artifact, distFilenames[artifactFilename], project))
+ reJarWar = re.compile(r'%s\.[wj]ar$' % version) # exclude *-javadoc.jar and *-sources.jar
+ distFilenames = dict()
+ for file in distFiles:
+ baseName = os.path.basename(file)
+ distFilenames[baseName] = file
+ for artifact in artifacts:
+ if reJarWar.search(artifact):
+ artifactFilename = os.path.basename(artifact)
+ if artifactFilename not in distFilenames:
+ raise RuntimeError('Maven artifact %s is not present in solr binary distribution' % artifact)
+ else:
+ identical = filecmp.cmp(artifact, distFilenames[artifactFilename], shallow=False)
+ if not identical:
+ raise RuntimeError('Maven artifact %s is not identical to %s in solr binary distribution'
+ % (artifact, distFilenames[artifactFilename]))
+
def verifyMavenDigests(artifacts):
print(" verify Maven artifacts' md5/sha1 digests...")
reJarWarPom = re.compile(r'\.(?:[wj]ar|pom)$')
- for project in ('lucene', 'solr'):
- for artifactFile in [a for a in artifacts[project] if reJarWarPom.search(a)]:
- if artifactFile + '.md5' not in artifacts[project]:
- raise RuntimeError('missing: MD5 digest for %s' % artifactFile)
- if artifactFile + '.sha1' not in artifacts[project]:
- raise RuntimeError('missing: SHA1 digest for %s' % artifactFile)
- with open(artifactFile + '.md5', encoding='UTF-8') as md5File:
- md5Expected = md5File.read().strip()
- with open(artifactFile + '.sha1', encoding='UTF-8') as sha1File:
- sha1Expected = sha1File.read().strip()
- md5 = hashlib.md5()
- sha1 = hashlib.sha1()
- inputFile = open(artifactFile, 'rb')
- while True:
- bytes = inputFile.read(65536)
- if len(bytes) == 0:
- break
- md5.update(bytes)
- sha1.update(bytes)
- inputFile.close()
- md5Actual = md5.hexdigest()
- sha1Actual = sha1.hexdigest()
- if md5Actual != md5Expected:
- raise RuntimeError('MD5 digest mismatch for %s: expected %s but got %s'
- % (artifactFile, md5Expected, md5Actual))
- if sha1Actual != sha1Expected:
- raise RuntimeError('SHA1 digest mismatch for %s: expected %s but got %s'
- % (artifactFile, sha1Expected, sha1Actual))
+ for artifactFile in [a for a in artifacts if reJarWarPom.search(a)]:
+ if artifactFile + '.md5' not in artifacts:
+ raise RuntimeError('missing: MD5 digest for %s' % artifactFile)
+ if artifactFile + '.sha1' not in artifacts:
+ raise RuntimeError('missing: SHA1 digest for %s' % artifactFile)
+ with open(artifactFile + '.md5', encoding='UTF-8') as md5File:
+ md5Expected = md5File.read().strip()
+ with open(artifactFile + '.sha1', encoding='UTF-8') as sha1File:
+ sha1Expected = sha1File.read().strip()
+ md5 = hashlib.md5()
+ sha1 = hashlib.sha1()
+ inputFile = open(artifactFile, 'rb')
+ while True:
+ bytes = inputFile.read(65536)
+ if len(bytes) == 0:
+ break
+ md5.update(bytes)
+ sha1.update(bytes)
+ inputFile.close()
+ md5Actual = md5.hexdigest()
+ sha1Actual = sha1.hexdigest()
+ if md5Actual != md5Expected:
+ raise RuntimeError('MD5 digest mismatch for %s: expected %s but got %s'
+ % (artifactFile, md5Expected, md5Actual))
+ if sha1Actual != sha1Expected:
+ raise RuntimeError('SHA1 digest mismatch for %s: expected %s but got %s'
+ % (artifactFile, sha1Expected, sha1Actual))
+
def getPOMcoordinate(treeRoot):
namespace = '{http://maven.apache.org/POM/4.0.0}'
@@ -1043,60 +887,59 @@ def getPOMcoordinate(treeRoot):
packaging = 'jar' if packaging is None else packaging.text.strip()
return groupId, artifactId, packaging, version
-def verifyMavenSigs(baseURL, tmpDir, artifacts, keysFile):
+
+def verifyMavenSigs(tmpDir, artifacts, keysFile):
print(' verify maven artifact sigs', end=' ')
- for project in ('lucene', 'solr'):
-
- # Set up clean gpg world; import keys file:
- gpgHomeDir = '%s/%s.gpg' % (tmpDir, project)
- if os.path.exists(gpgHomeDir):
- shutil.rmtree(gpgHomeDir)
- os.makedirs(gpgHomeDir, 0o700)
- run('gpg --homedir %s --import %s' % (gpgHomeDir, keysFile),
- '%s/%s.gpg.import.log' % (tmpDir, project))
-
- reArtifacts = re.compile(r'\.(?:pom|[jw]ar)$')
- for artifactFile in [a for a in artifacts[project] if reArtifacts.search(a)]:
- artifact = os.path.basename(artifactFile)
- sigFile = '%s.asc' % artifactFile
- # Test sig (this is done with a clean brand-new GPG world)
- logFile = '%s/%s.%s.gpg.verify.log' % (tmpDir, project, artifact)
- run('gpg --homedir %s --verify %s %s' % (gpgHomeDir, sigFile, artifactFile),
- logFile)
- # Forward any GPG warnings, except the expected one (since it's a clean world)
- f = open(logFile)
- for line in f.readlines():
- if line.lower().find('warning') != -1 \
- and line.find('WARNING: This key is not certified with a trusted signature') == -1 \
- and line.find('WARNING: using insecure memory') == -1:
- print(' GPG: %s' % line.strip())
- f.close()
- # Test trust (this is done with the real users config)
- run('gpg --import %s' % keysFile,
- '%s/%s.gpg.trust.import.log' % (tmpDir, project))
- logFile = '%s/%s.%s.gpg.trust.log' % (tmpDir, project, artifact)
- run('gpg --verify %s %s' % (sigFile, artifactFile), logFile)
- # Forward any GPG warnings:
- f = open(logFile)
- for line in f.readlines():
- if line.lower().find('warning') != -1 \
- and line.find('WARNING: This key is not certified with a trusted signature') == -1 \
- and line.find('WARNING: using insecure memory') == -1:
- print(' GPG: %s' % line.strip())
- f.close()
-
- sys.stdout.write('.')
+ # Set up clean gpg world; import keys file:
+ gpgHomeDir = '%s/solr.gpg' % tmpDir
+ if os.path.exists(gpgHomeDir):
+ shutil.rmtree(gpgHomeDir)
+ os.makedirs(gpgHomeDir, 0o700)
+ run('gpg --homedir %s --import %s' % (gpgHomeDir, keysFile),
+ '%s/solr.gpg.import.log' % tmpDir)
+
+ reArtifacts = re.compile(r'\.(?:pom|[jw]ar)$')
+ for artifactFile in [a for a in artifacts if reArtifacts.search(a)]:
+ artifact = os.path.basename(artifactFile)
+ sigFile = '%s.asc' % artifactFile
+ # Test sig (this is done with a clean brand-new GPG world)
+ logFile = '%s/solr.%s.gpg.verify.log' % (tmpDir, artifact)
+ run('gpg --display-charset utf-8 --homedir %s --verify %s %s' % (gpgHomeDir, sigFile, artifactFile),
+ logFile)
+
+ # Forward any GPG warnings, except the expected one (since it's a clean world)
+ print_warnings_in_file(logFile)
+
+ # Test trust (this is done with the real users config)
+ run('gpg --import %s' % keysFile,
+ '%s/solr.gpg.trust.import.log' % tmpDir)
+ logFile = '%s/solr.%s.gpg.trust.log' % (tmpDir, artifact)
+ run('gpg --display-charset utf-8 --verify %s %s' % (sigFile, artifactFile), logFile)
+ # Forward any GPG warnings:
+ print_warnings_in_file(logFile)
+
+ sys.stdout.write('.')
print()
+
+def print_warnings_in_file(file):
+ with open(file) as f:
+ for line in f.readlines():
+ if line.lower().find('warning') != -1 \
+ and line.find('WARNING: This key is not certified with a trusted signature') == -1 \
+ and line.find('WARNING: using insecure memory') == -1:
+ print(' GPG: %s' % line.strip())
+
+
def verifyPOMperBinaryArtifact(artifacts, version):
print(' verify that each binary artifact has a deployed POM...')
reBinaryJarWar = re.compile(r'%s\.[jw]ar$' % re.escape(version))
- for project in ('lucene', 'solr'):
- for artifact in [a for a in artifacts[project] if reBinaryJarWar.search(a)]:
- POM = artifact[:-4] + '.pom'
- if POM not in artifacts[project]:
- raise RuntimeError('missing: POM for %s' % artifact)
+ for artifact in [a for a in artifacts if reBinaryJarWar.search(a)]:
+ POM = artifact[:-4] + '.pom'
+ if POM not in artifacts:
+ raise RuntimeError('missing: POM for %s' % artifact)
+
def verifyDeployedPOMsCoordinates(artifacts, version):
"""
@@ -1104,61 +947,19 @@ def verifyDeployedPOMsCoordinates(artifacts, version):
its filepath, and verify that the corresponding artifact exists.
"""
print(" verify deployed POMs' coordinates...")
- for project in ('lucene', 'solr'):
- for POM in [a for a in artifacts[project] if a.endswith('.pom')]:
- treeRoot = ET.parse(POM).getroot()
- groupId, artifactId, packaging, POMversion = getPOMcoordinate(treeRoot)
- POMpath = '%s/%s/%s/%s-%s.pom' \
- % (groupId.replace('.', '/'), artifactId, version, artifactId, version)
- if not POM.endswith(POMpath):
- raise RuntimeError("Mismatch between POM coordinate %s:%s:%s and filepath: %s"
- % (groupId, artifactId, POMversion, POM))
- # Verify that the corresponding artifact exists
- artifact = POM[:-3] + packaging
- if artifact not in artifacts[project]:
- raise RuntimeError('Missing corresponding .%s artifact for POM %s' % (packaging, POM))
-
-def verifyArtifactPerPOMtemplate(POMtemplates, artifacts, tmpDir, version):
- print(' verify that there is an artifact for each POM template...')
- namespace = '{http://maven.apache.org/POM/4.0.0}'
- xpathPlugin = '{0}build/{0}plugins/{0}plugin'.format(namespace)
- xpathSkipConfiguration = '{0}configuration/{0}skip'.format(namespace)
- for project in ('lucene', 'solr'):
- for POMtemplate in POMtemplates[project]:
- treeRoot = ET.parse(POMtemplate).getroot()
- skipDeploy = False
- for plugin in treeRoot.findall(xpathPlugin):
- artifactId = plugin.find('%sartifactId' % namespace).text.strip()
- if artifactId == 'maven-deploy-plugin':
- skip = plugin.find(xpathSkipConfiguration)
- if skip is not None: skipDeploy = (skip.text.strip().lower() == 'true')
- if not skipDeploy:
- groupId, artifactId, packaging, POMversion = getPOMcoordinate(treeRoot)
- # Ignore POMversion, since its value will not have been interpolated
- artifact = '%s/maven/%s/%s/%s/%s-%s.%s' \
- % (tmpDir, groupId.replace('.', '/'), artifactId,
- version, artifactId, version, packaging)
- if artifact not in artifacts['lucene'] and artifact not in artifacts['solr']:
- raise RuntimeError('Missing artifact %s' % artifact)
-
-def getPOMtemplates(solrSrcUnpackPath, POMtemplates, tmpDir):
- print(' find pom.xml.template files in the unpacked Solr source distribution')
- allPOMtemplates = []
- rePOMtemplate = re.compile(r'^pom\.xml\.template$')
- for root, dirs, files in os.walk(solrSrcUnpackPath):
- allPOMtemplates.extend([os.path.join(root, f) for f in files if rePOMtemplate.search(f)])
-
- reLucenePOMtemplate = re.compile(r'.*/maven/lucene.*/pom\.xml\.template$')
- POMtemplates['lucene'] = [p for p in allPOMtemplates if reLucenePOMtemplate.search(p)]
- if POMtemplates['lucene'] is None:
- raise RuntimeError('No Lucene POMs found at %s' % solrSrcUnpackPath)
- reSolrPOMtemplate = re.compile(r'.*/maven/solr.*/pom\.xml\.template$')
- POMtemplates['solr'] = [p for p in allPOMtemplates if reSolrPOMtemplate.search(p)]
- if POMtemplates['solr'] is None:
- raise RuntimeError('No Solr POMs found at %s' % solrSrcUnpackPath)
- POMtemplates['grandfather'] = [p for p in allPOMtemplates if '/maven/pom.xml.template' in p]
- if len(POMtemplates['grandfather']) == 0:
- raise RuntimeError('No Lucene/Solr grandfather POM found at %s' % solrSrcUnpackPath)
+ for POM in [a for a in artifacts if a.endswith('.pom')]:
+ treeRoot = ET.parse(POM).getroot()
+ groupId, artifactId, packaging, POMversion = getPOMcoordinate(treeRoot)
+ POMpath = '%s/%s/%s/%s-%s.pom' \
+ % (groupId.replace('.', '/'), artifactId, version, artifactId, version)
+ if not POM.endswith(POMpath):
+ raise RuntimeError("Mismatch between POM coordinate %s:%s:%s and filepath: %s"
+ % (groupId, artifactId, POMversion, POM))
+ # Verify that the corresponding artifact exists
+ artifact = POM[:-3] + packaging
+ if artifact not in artifacts:
+ raise RuntimeError('Missing corresponding .%s artifact for POM %s' % (packaging, POM))
+
def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
for text, subURL in getDirEntries(urlString):
@@ -1174,7 +975,8 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
downloadedFiles.append(path)
sys.stdout.write('.')
-def make_java_config(parser, java12_home):
+
+def make_java_config(parser, java17_home):
def _make_runner(java_home, version):
print('Java %s JAVA_HOME=%s' % (version, java_home))
if cygwin:
@@ -1184,7 +986,7 @@ def make_java_config(parser, java12_home):
s = subprocess.check_output('%s; java -version' % cmd_prefix,
shell=True, stderr=subprocess.STDOUT).decode('utf-8')
if s.find(' version "%s' % version) == -1:
- parser.error('got wrong version for java %s:\n%s' % (version, s))
+ parser.error('got wrong version for java %s:\n%s' % (version, s))
def run_java(cmd, logfile):
run('%s; %s' % (cmd_prefix, cmd), logfile)
return run_java
@@ -1192,40 +994,40 @@ def make_java_config(parser, java12_home):
if java11_home is None:
parser.error('JAVA_HOME must be set')
run_java11 = _make_runner(java11_home, '11')
- run_java12 = None
- if java12_home is not None:
- run_java12 = _make_runner(java12_home, '12')
+ run_java17 = None
+ if java17_home is not None:
+ run_java17 = _make_runner(java17_home, '17')
- jc = namedtuple('JavaConfig', 'run_java11 java11_home run_java12 java12_home')
- return jc(run_java11, java11_home, run_java12, java12_home)
+ jc = namedtuple('JavaConfig', 'run_java11 java11_home run_java17 java17_home')
+ return jc(run_java11, java11_home, run_java17, java17_home)
version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)')
-revision_re = re.compile(r'rev([a-f\d]+)')
+revision_re = re.compile(r'rev-([a-f\d]+)')
def parse_config():
epilogue = textwrap.dedent('''
Example usage:
- python3 -u dev-tools/scripts/smokeTestRelease.py https://dist.apache.org/repos/dist/dev/lucene/lucene-solr-6.0.1-RC2-revc7510a0...
+ python3 -u dev-tools/scripts/smokeTestRelease.py https://dist.apache.org/repos/dist/dev/solr/solr-9.0.0-RC1-rev-c7510a0...
''')
description = 'Utility to test a release.'
parser = argparse.ArgumentParser(description=description, epilog=epilogue,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--tmp-dir', metavar='PATH',
- help='Temporary directory to test inside, defaults to /tmp/smoke_lucene_$version_$revision')
+ help='Temporary directory to test inside, defaults to /tmp/smoke_solr_$version_$revision')
parser.add_argument('--not-signed', dest='is_signed', action='store_false', default=True,
help='Indicates the release is not signed')
parser.add_argument('--local-keys', metavar='PATH',
- help='Uses local KEYS file instead of fetching from https://archive.apache.org/dist/lucene/KEYS')
+ help='Uses local KEYS file instead of fetching from https://archive.apache.org/dist/solr/KEYS')
parser.add_argument('--revision',
help='GIT revision number that release was built with, defaults to that in URL')
parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
help='Version of the release, defaults to that in URL')
- parser.add_argument('--test-java12', metavar='JAVA12_HOME',
- help='Path to Java12 home directory, to run tests with if specified')
+ parser.add_argument('--test-java17', metavar='java17_home',
+ help='Path to Java17 home directory, to run tests with if specified')
parser.add_argument('--download-only', action='store_true', default=False,
help='Only perform download and sha hash check steps')
parser.add_argument('url', help='Url pointing to release to test')
parser.add_argument('test_args', nargs=argparse.REMAINDER,
- help='Arguments to pass to ant for testing, e.g. -Dwhat=ever.')
+ help='Arguments to pass to gradle for testing, e.g. -Dwhat=ever.')
c = parser.parse_args()
if c.version is not None:
@@ -1247,12 +1049,12 @@ def parse_config():
if c.local_keys is not None and not os.path.exists(c.local_keys):
parser.error('Local KEYS file "%s" not found' % c.local_keys)
- c.java = make_java_config(parser, c.test_java12)
+ c.java = make_java_config(parser, c.test_java17)
if c.tmp_dir:
c.tmp_dir = os.path.abspath(c.tmp_dir)
else:
- tmp = '/tmp/smoke_lucene_%s_%s' % (c.version, c.revision)
+ tmp = '/tmp/smoke_solr_%s_%s' % (c.version, c.revision)
c.tmp_dir = tmp
i = 1
while os.path.exists(c.tmp_dir):
@@ -1264,118 +1066,12 @@ def parse_config():
reVersion1 = re.compile(r'\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<', re.IGNORECASE)
reVersion2 = re.compile(r'-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.', re.IGNORECASE)
-def getAllLuceneReleases():
- s = load('https://archive.apache.org/dist/lucene/java')
-
- releases = set()
- for r in reVersion1, reVersion2:
- for tup in r.findall(s):
- if tup[-1].lower() == '-alpha':
- tup = tup[:3] + ('0',)
- elif tup[-1].lower() == '-beta':
- tup = tup[:3] + ('1',)
- elif tup[-1] == '':
- tup = tup[:3]
- else:
- raise RuntimeError('failed to parse version: %s' % tup[-1])
- releases.add(tuple(int(x) for x in tup))
-
- l = list(releases)
- l.sort()
- return l
-
-def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
-
- print(' find all past Lucene releases...')
- allReleases = getAllLuceneReleases()
- #for tup in allReleases:
- # print(' %s' % '.'.join(str(x) for x in tup))
-
- testedIndices = set()
-
- os.chdir(unpackPath)
-
- print(' run TestBackwardsCompatibility..')
- command = 'ant test -Dtestcase=TestBackwardsCompatibility -Dtests.verbose=true'
- p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- stdout, stderr = p.communicate()
- if p.returncode != 0:
- # Not good: the test failed!
- raise RuntimeError('%s failed:\n%s' % (command, stdout))
- stdout = stdout.decode('utf-8',errors='replace').replace('\r\n','\n')
-
- if stderr is not None:
- # Should not happen since we redirected stderr to stdout:
- raise RuntimeError('stderr non-empty')
-
- reIndexName = re.compile(r'TEST: index[\s*=\s*](.*?)(-cfs|-nocfs)$', re.MULTILINE)
- for name, cfsPart in reIndexName.findall(stdout):
- # Fragile: decode the inconsistent naming schemes we've used in TestBWC's indices:
- #print('parse name %s' % name)
- tup = tuple(name.split('.'))
- if len(tup) == 3:
- # ok
- tup = tuple(int(x) for x in tup)
- elif tup == ('4', '0', '0', '1'):
- # CONFUSING: this is the 4.0.0-alpha index??
- tup = 4, 0, 0, 0
- elif tup == ('4', '0', '0', '2'):
- # CONFUSING: this is the 4.0.0-beta index??
- tup = 4, 0, 0, 1
- elif name == '5x-with-4x-segments':
- # Mixed version test case; ignore it for our purposes because we only
- # tally up the "tests single Lucene version" indices
- continue
- elif name == '5.0.0.singlesegment':
- tup = 5, 0, 0
- else:
- raise RuntimeError('could not parse version %s' % name)
-
- testedIndices.add(tup)
-
- l = list(testedIndices)
- l.sort()
- if False:
- for release in l:
- print(' %s' % '.'.join(str(x) for x in release))
-
- allReleases = set(allReleases)
-
- for x in testedIndices:
- if x not in allReleases:
- # Curious: we test 1.9.0 index but it's not in the releases (I think it was pulled because of nasty bug?)
- if x != (1, 9, 0):
- raise RuntimeError('tested version=%s but it was not released?' % '.'.join(str(y) for y in x))
-
- notTested = []
- for x in allReleases:
- if x not in testedIndices:
- releaseVersion = '.'.join(str(y) for y in x)
- if releaseVersion in ('1.4.3', '1.9.1', '2.3.1', '2.3.2'):
- # Exempt the dark ages indices
- continue
- if x >= tuple(int(y) for y in smokeVersion.split('.')):
- # Exempt versions not less than the one being smoke tested
- print(' Backcompat testing not required for release %s because it\'s not less than %s'
- % (releaseVersion, smokeVersion))
- continue
- notTested.append(x)
-
- if len(notTested) > 0:
- notTested.sort()
- print('Releases that don\'t seem to be tested:')
- failed = True
- for x in notTested:
- print(' %s' % '.'.join(str(y) for y in x))
- raise RuntimeError('some releases are not tested by TestBackwardsCompatibility?')
- else:
- print(' success!')
-
def main():
c = parse_config()
- scriptVersion = scriptutil.find_current_version()
+ # Pick <major>.<minor> part of version and require script to be from same branch
+ scriptVersion = re.search(r'((\d+).(\d+)).(\d+)', scriptutil.find_current_version()).group(1).strip()
if not c.version.startswith(scriptVersion + '.'):
raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))
@@ -1383,20 +1079,23 @@ def main():
smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args),
downloadOnly=c.download_only)
+
def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs, downloadOnly=False):
startTime = datetime.datetime.now()
- # disable flakey tests for smoke-tester runs:
- testArgs = '-Dtests.badapples=false %s' % testArgs
-
+ # Tests annotated @Nightly are more resource-intensive but often cover
+ # important code paths. They're disabled by default to preserve a good
+ # developer experience, but we enable them for smoke tests where we want good
+ # coverage. Still we disable @BadApple tests
+ testArgs = '-Dtests.nightly=true -Dtests.badapples=false %s' % testArgs
+
if FORCE_CLEAN:
if os.path.exists(tmpDir):
raise RuntimeError('temp dir %s exists; please remove first' % tmpDir)
if not os.path.exists(tmpDir):
os.makedirs(tmpDir)
-
- lucenePath = None
+
solrPath = None
print()
print('Load release URL "%s"...' % baseURL)
@@ -1404,15 +1103,11 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,
if newBaseURL != baseURL:
print(' unshortened: %s' % newBaseURL)
baseURL = newBaseURL
-
+
for text, subURL in getDirEntries(baseURL):
- if text.lower().find('lucene') != -1:
- lucenePath = subURL
- elif text.lower().find('solr') != -1:
+ if text.lower().find('solr') != -1:
solrPath = subURL
- if lucenePath is None:
- raise RuntimeError('could not find lucene subdir')
if solrPath is None:
raise RuntimeError('could not find solr subdir')
@@ -1422,7 +1117,7 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,
print(" Using local KEYS file %s" % local_keys)
keysFile = local_keys
else:
- keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
+ keysFileURL = "https://archive.apache.org/dist/solr/KEYS"
print(" Downloading online KEYS file %s" % keysFileURL)
scriptutil.download('KEYS', keysFileURL, tmpDir, force_clean=FORCE_CLEAN)
keysFile = '%s/KEYS' % (tmpDir)
@@ -1431,34 +1126,23 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,
raise RuntimeError('Port 8983 is already in use. The smoketester needs it to test Solr')
print()
- print('Test Lucene...')
- checkSigs('lucene', lucenePath, version, tmpDir, isSigned, keysFile)
- if not downloadOnly:
- for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
- unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
- unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
- else:
- print("\nLucene test done (--download-only specified)")
-
- print()
print('Test Solr...')
- checkSigs('solr', solrPath, version, tmpDir, isSigned, keysFile)
+ checkSigs(solrPath, version, tmpDir, isSigned, keysFile)
if not downloadOnly:
- for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
- unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
- solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
- gitRevision, version, testArgs, baseURL)
+ unpackAndVerify(java, tmpDir, 'solr-%s.tgz' % version, gitRevision, version, testArgs)
+ unpackAndVerify(java, tmpDir, 'solr-%s.zip' % version, gitRevision, version, testArgs)
+ unpackAndVerify(java, tmpDir, 'solr-%s-src.tgz' % version, gitRevision, version, testArgs)
print()
- print('Test Maven artifacts for Lucene and Solr...')
- checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
+ print('Test Maven artifacts...')
+ checkMaven(baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
else:
print("Solr test done (--download-only specified)")
print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))
+
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('Keyboard interrupt...exiting')
-
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 480f5db..ac19b1a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -243,6 +243,8 @@ Build
* SOLR-15867: Make the gradle build ready for 9.0 release (janhoy, Houston Putman, Dawid Weiss)
+* SOLR-15852: Update dev-tools/scripts for the 9.0 release, including releaseWizard (janhoy)
+
* SOLR-15891: The Solr Docker image uses the default file permissions from the tarball artifact (Houston Putman, janhoy)
Other Changes