You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2019/06/11 08:17:16 UTC
[lucene-solr] branch master updated: LUCENE-8837
smokeTestRelease.py option --download-only (#702)
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 44287d4 LUCENE-8837 smokeTestRelease.py option --download-only (#702)
44287d4 is described below
commit 44287d420624fe669a319f95f4712486bd57ae38
Author: Jan Høydahl <ja...@users.noreply.github.com>
AuthorDate: Tue Jun 11 10:17:10 2019 +0200
LUCENE-8837 smokeTestRelease.py option --download-only (#702)
* LUCENE-8837: smokeTestRelease.py option --download-only
Move download() and check_and() functions to scriptutil
Add cwd param to run() function in scriptutil
Move the check_ant function from buildAndPushRelease into scriptutil.py, and let it return the version.
---
dev-tools/scripts/buildAndPushRelease.py | 11 +--
dev-tools/scripts/scriptutil.py | 68 ++++++++++++++++-
dev-tools/scripts/smokeTestRelease.py | 121 +++++++++++--------------------
3 files changed, 111 insertions(+), 89 deletions(-)
diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py
index 66fbd3f..0667be9 100644
--- a/dev-tools/scripts/buildAndPushRelease.py
+++ b/dev-tools/scripts/buildAndPushRelease.py
@@ -21,6 +21,7 @@ import os
import sys
import subprocess
from subprocess import TimeoutExpired
+from scriptutil import check_ant
import textwrap
import urllib.request, urllib.error, urllib.parse
import xml.etree.ElementTree as ET
@@ -286,16 +287,6 @@ def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
raise RuntimeError('"git --version" returned a non-zero exit code.')
check_ant()
-def check_ant():
- antVersion = os.popen('ant -version').read().strip()
- if (antVersion.startswith('Apache Ant(TM) version 1.8')):
- return
- if (antVersion.startswith('Apache Ant(TM) version 1.9')):
- return
- if (antVersion.startswith('Apache Ant(TM) version 1.10')):
- return
- raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion)
-
def check_key_in_keys(gpgKeyID, local_keys):
if gpgKeyID is not None:
print(' Verify your gpg key is in the main KEYS file')
diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py
index 9b26413..6a30508 100644
--- a/dev-tools/scripts/scriptutil.py
+++ b/dev-tools/scripts/scriptutil.py
@@ -17,7 +17,11 @@ import argparse
import re
import subprocess
import sys
+import os
from enum import Enum
+import time
+import urllib.request, urllib.error, urllib.parse
+import urllib.parse
class Version(object):
def __init__(self, major, minor, bugfix, prerelease):
@@ -66,14 +70,19 @@ class Version(object):
(self.bugfix > other.bugfix or self.bugfix == other.bugfix and
self.prerelease >= other.prerelease)))
+ def gt(self, other):
+ return (self.major > other.major or
+ (self.major == other.major and self.minor > other.minor) or
+ (self.major == other.major and self.minor == other.minor and self.bugfix > other.bugfix))
+
def is_back_compat_with(self, other):
if not self.on_or_after(other):
raise Exception('Back compat check disallowed for newer version: %s < %s' % (self, other))
return other.major + 1 >= self.major
-def run(cmd):
+def run(cmd, cwd=None):
try:
- output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+ output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, cwd=cwd)
except subprocess.CalledProcessError as e:
print(e.output.decode('utf-8'))
raise e
@@ -99,6 +108,18 @@ def update_file(filename, line_re, edit):
f.write(''.join(buffer))
return True
+
+def check_ant():
+ antVersion = os.popen('ant -version').read().strip()
+ if (antVersion.startswith('Apache Ant(TM) version 1.8')):
+ return antVersion.split(" ")[3]
+ if (antVersion.startswith('Apache Ant(TM) version 1.9')):
+ return antVersion.split(" ")[3]
+ if (antVersion.startswith('Apache Ant(TM) version 1.10')):
+ return antVersion.split(" ")[3]
+ raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion)
+
+
# branch types are "release", "stable" and "unstable"
class BranchType(Enum):
unstable = 1
@@ -122,6 +143,49 @@ def find_branch_type():
return BranchType.release
raise Exception('Cannot run %s on feature branch' % sys.argv[0].rsplit('/', 1)[-1])
+
+def download(name, urlString, tmpDir, quiet=False, force_clean=True):
+ if not quiet:
+ print("Downloading %s" % urlString)
+ startTime = time.time()
+ fileName = '%s/%s' % (tmpDir, name)
+ if not force_clean and os.path.exists(fileName):
+ if not quiet and fileName.find('.asc') == -1:
+ print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.))
+ return
+ try:
+ attemptDownload(urlString, fileName)
+ except Exception as e:
+ print('Retrying download of url %s after exception: %s' % (urlString, e))
+ try:
+ attemptDownload(urlString, fileName)
+ except Exception as e:
+ raise RuntimeError('failed to download url "%s"' % urlString) from e
+ if not quiet and fileName.find('.asc') == -1:
+ t = time.time()-startTime
+ sizeMB = os.path.getsize(fileName)/1024./1024.
+ print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t))
+
+
+def attemptDownload(urlString, fileName):
+ fIn = urllib.request.urlopen(urlString)
+ fOut = open(fileName, 'wb')
+ success = False
+ try:
+ while True:
+ s = fIn.read(65536)
+ if s == b'':
+ break
+ fOut.write(s)
+ fOut.close()
+ fIn.close()
+ success = True
+ finally:
+ fIn.close()
+ fOut.close()
+ if not success:
+ os.remove(fileName)
+
version_prop_re = re.compile('version\.base=(.*)')
def find_current_version():
return version_prop_re.search(open('lucene/version.properties').read()).group(1).strip()
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
old mode 100644
new mode 100755
index 617ef4b..aa7e0ca
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -14,35 +16,31 @@
# limitations under the License.
import argparse
-import os
-import zipfile
import codecs
-import tarfile
-import zipfile
-import threading
-import traceback
import datetime
-import time
-import subprocess
-import signal
-import shutil
+import filecmp
import hashlib
import http.client
+import os
+import platform
import re
-import urllib.request, urllib.error, urllib.parse
-import urllib.parse
+import shutil
+import subprocess
import sys
-import html.parser
-from collections import defaultdict
+import textwrap
+import traceback
+import urllib.error
+import urllib.parse
+import urllib.parse
+import urllib.request
import xml.etree.ElementTree as ET
-import filecmp
-import platform
+import zipfile
+from collections import defaultdict
+from collections import namedtuple
+from scriptutil import download
+
import checkJavaDocs
import checkJavadocLinks
-import io
-import codecs
-import textwrap
-from collections import namedtuple
# This tool expects to find /lucene and /solr off the base URL. You
# must have a working gpg, tar, unzip in your path. This has been
@@ -111,44 +109,6 @@ def getHREFs(urlString):
links.append((text, fullURL))
return links
-def download(name, urlString, tmpDir, quiet=False):
- startTime = time.time()
- fileName = '%s/%s' % (tmpDir, name)
- if not FORCE_CLEAN and os.path.exists(fileName):
- if not quiet and fileName.find('.asc') == -1:
- print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.))
- return
- try:
- attemptDownload(urlString, fileName)
- except Exception as e:
- print('Retrying download of url %s after exception: %s' % (urlString, e))
- try:
- attemptDownload(urlString, fileName)
- except Exception as e:
- raise RuntimeError('failed to download url "%s"' % urlString) from e
- if not quiet and fileName.find('.asc') == -1:
- t = time.time()-startTime
- sizeMB = os.path.getsize(fileName)/1024./1024.
- print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t))
-
-def attemptDownload(urlString, fileName):
- fIn = urllib.request.urlopen(urlString)
- fOut = open(fileName, 'wb')
- success = False
- try:
- while True:
- s = fIn.read(65536)
- if s == b'':
- break
- fOut.write(s)
- fOut.close()
- fIn.close()
- success = True
- finally:
- fIn.close()
- fOut.close()
- if not success:
- os.remove(fileName)
def load(urlString):
try:
@@ -362,13 +322,13 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
for artifact, urlString in artifacts:
print(' download %s...' % artifact)
- download(artifact, urlString, tmpDir)
+ download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN)
verifyDigests(artifact, urlString, tmpDir)
if isSigned:
print(' verify sig')
# Test sig (this is done with a clean brand-new GPG world)
- download(artifact + '.asc', urlString + '.asc', tmpDir)
+ download(artifact + '.asc', urlString + '.asc', tmpDir, force_clean=FORCE_CLEAN)
sigFile = '%s/%s.asc' % (tmpDir, artifact)
artifactFile = '%s/%s' % (tmpDir, artifact)
logFile = '%s/%s.%s.gpg.verify.log' % (tmpDir, project, artifact)
@@ -997,7 +957,7 @@ def getBinaryDistFiles(project, tmpDir, version, baseURL):
if not os.path.exists('%s/%s' % (tmpDir, distribution)):
distURL = '%s/%s/%s' % (baseURL, project, distribution)
print(' download %s...' % distribution, end=' ')
- download(distribution, distURL, tmpDir)
+ download(distribution, distURL, tmpDir, force_clean=FORCE_CLEAN)
destDir = '%s/unpack-%s-getBinaryDistFiles' % (tmpDir, project)
if os.path.exists(destDir):
shutil.rmtree(destDir)
@@ -1226,7 +1186,7 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
crawl(downloadedFiles, subURL, path, exclusions)
else:
if not os.path.exists(path) or FORCE_CLEAN:
- download(text, subURL, targetDir, quiet=True)
+ download(text, subURL, targetDir, quiet=True, force_clean=FORCE_CLEAN)
downloadedFiles.append(path)
sys.stdout.write('.')
@@ -1277,6 +1237,8 @@ def parse_config():
help='Version of the release, defaults to that in URL')
parser.add_argument('--test-java12', metavar='JAVA12_HOME',
help='Path to Java12 home directory, to run tests with if specified')
+ parser.add_argument('--download-only', action='store_true', default=False,
+ help='Only perform download and sha hash check steps')
parser.add_argument('url', help='Url pointing to release to test')
parser.add_argument('test_args', nargs=argparse.REMAINDER,
help='Arguments to pass to ant for testing, e.g. -Dwhat=ever.')
@@ -1445,10 +1407,10 @@ def main():
raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))
print('NOTE: output encoding is %s' % sys.stdout.encoding)
- smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args))
-
-def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs):
+ smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args),
+ downloadOnly=c.download_only)
+def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs, downloadOnly=False):
startTime = datetime.datetime.now()
# disable flakey tests for smoke-tester runs:
@@ -1489,27 +1451,32 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,
else:
keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
print(" Downloading online KEYS file %s" % keysFileURL)
- download('KEYS', keysFileURL, tmpDir)
+ download('KEYS', keysFileURL, tmpDir, force_clean=FORCE_CLEAN)
keysFile = '%s/KEYS' % (tmpDir)
print()
print('Test Lucene...')
checkSigs('lucene', lucenePath, version, tmpDir, isSigned, keysFile)
- for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
- unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
- unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
+ if not downloadOnly:
+ for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
+ unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
+ unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
+ else:
+ print("\nLucene test done (--download-only specified)")
print()
print('Test Solr...')
checkSigs('solr', solrPath, version, tmpDir, isSigned, keysFile)
- for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
- unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
- solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
- gitRevision, version, testArgs, baseURL)
-
- print()
- print('Test Maven artifacts for Lucene and Solr...')
- checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
+ if not downloadOnly:
+ for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
+ unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
+ solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
+ gitRevision, version, testArgs, baseURL)
+ print()
+ print('Test Maven artifacts for Lucene and Solr...')
+ checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
+ else:
+ print("Solr test done (--download-only specified)")
print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))