You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2019/06/12 14:05:38 UTC
[lucene-solr] branch branch_8_1 updated (9a63222 -> 23e5bda)
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a change to branch branch_8_1
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 9a63222 SOLR-13526: Backport the fix introduced by SOLR-13434, basically set REQ_PRINCIPAL_KEY before calling onQueued() listener
new 0fd5d5f LUCENE-8818: Fix smokeTestRelease.py encoding bug
new 23e5bda LUCENE-8837 smokeTestRelease.py option --download-only (#702)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-tools/scripts/buildAndPushRelease.py | 11 +--
dev-tools/scripts/scriptutil.py | 68 +++++++++++++++-
dev-tools/scripts/smokeTestRelease.py | 129 ++++++++++++-------------------
lucene/CHANGES.txt | 2 +
4 files changed, 117 insertions(+), 93 deletions(-)
mode change 100644 => 100755 dev-tools/scripts/smokeTestRelease.py
[lucene-solr] 01/02: LUCENE-8818: Fix smokeTestRelease.py encoding
bug
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch branch_8_1
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 0fd5d5f437069895ad30c7a792dcc9de3ecf473e
Author: Jan Høydahl <ja...@apache.org>
AuthorDate: Thu Jun 6 21:42:24 2019 +0200
LUCENE-8818: Fix smokeTestRelease.py encoding bug
(cherry picked from commit 8d6fd7298fe480a627db333eafeadc7b6a8fdcad)
---
dev-tools/scripts/smokeTestRelease.py | 8 ++++----
lucene/CHANGES.txt | 2 ++
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
index a6f0990..614a941 100644
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -375,7 +375,7 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
run('gpg --homedir %s --verify %s %s' % (gpgHomeDir, sigFile, artifactFile),
logFile)
# Forward any GPG warnings, except the expected one (since it's a clean world)
- f = open(logFile, encoding='UTF-8')
+ f = open(logFile)
for line in f.readlines():
if line.lower().find('warning') != -1 \
and line.find('WARNING: This key is not certified with a trusted signature') == -1:
@@ -389,7 +389,7 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
logFile = '%s/%s.%s.gpg.trust.log' % (tmpDir, project, artifact)
run('gpg --verify %s %s' % (sigFile, artifactFile), logFile)
# Forward any GPG warnings:
- f = open(logFile, encoding='UTF-8')
+ f = open(logFile)
for line in f.readlines():
if line.lower().find('warning') != -1:
print(' GPG: %s' % line.strip())
@@ -1120,7 +1120,7 @@ def verifyMavenSigs(baseURL, tmpDir, artifacts, keysFile):
run('gpg --homedir %s --verify %s %s' % (gpgHomeDir, sigFile, artifactFile),
logFile)
# Forward any GPG warnings, except the expected one (since it's a clean world)
- f = open(logFile, encoding='UTF-8')
+ f = open(logFile)
for line in f.readlines():
if line.lower().find('warning') != -1 \
and line.find('WARNING: This key is not certified with a trusted signature') == -1 \
@@ -1134,7 +1134,7 @@ def verifyMavenSigs(baseURL, tmpDir, artifacts, keysFile):
logFile = '%s/%s.%s.gpg.trust.log' % (tmpDir, project, artifact)
run('gpg --verify %s %s' % (sigFile, artifactFile), logFile)
# Forward any GPG warnings:
- f = open(logFile, encoding='UTF-8')
+ f = open(logFile)
for line in f.readlines():
if line.lower().find('warning') != -1 \
and line.find('WARNING: This key is not certified with a trusted signature') == -1 \
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c51b631..8c6abc7 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -10,6 +10,8 @@ Bug fixes
* LUCENE-8809: Fix concurrent refreshes and rollback can leave
segment states unclosed (Nhat Nguyen)
+* LUCENE-8818: Fix smokeTestRelease.py encoding bug (janhoy)
+
======================= Lucene 8.1.1 =======================
(No Changes)
[lucene-solr] 02/02: LUCENE-8837 smokeTestRelease.py option
--download-only (#702)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch branch_8_1
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 23e5bdabbf2046ca0a9d4a6aa7e302baf6196417
Author: Jan Høydahl <ja...@apache.org>
AuthorDate: Tue Jun 11 10:17:10 2019 +0200
LUCENE-8837 smokeTestRelease.py option --download-only (#702)
* LUCENE-8837: smokeTestRelease.py option --download-only
Move download() and check_and() functions to scriptutil
Add cwd param to run() function in scriptutil
Move the check_ant function from buildAndPushRelease into scriptutil.py, and let it return the version.
(cherry picked from commit 44287d420624fe669a319f95f4712486bd57ae38)
---
dev-tools/scripts/buildAndPushRelease.py | 11 +--
dev-tools/scripts/scriptutil.py | 68 ++++++++++++++++-
dev-tools/scripts/smokeTestRelease.py | 121 +++++++++++--------------------
3 files changed, 111 insertions(+), 89 deletions(-)
diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py
index 66fbd3f..0667be9 100644
--- a/dev-tools/scripts/buildAndPushRelease.py
+++ b/dev-tools/scripts/buildAndPushRelease.py
@@ -21,6 +21,7 @@ import os
import sys
import subprocess
from subprocess import TimeoutExpired
+from scriptutil import check_ant
import textwrap
import urllib.request, urllib.error, urllib.parse
import xml.etree.ElementTree as ET
@@ -286,16 +287,6 @@ def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
raise RuntimeError('"git --version" returned a non-zero exit code.')
check_ant()
-def check_ant():
- antVersion = os.popen('ant -version').read().strip()
- if (antVersion.startswith('Apache Ant(TM) version 1.8')):
- return
- if (antVersion.startswith('Apache Ant(TM) version 1.9')):
- return
- if (antVersion.startswith('Apache Ant(TM) version 1.10')):
- return
- raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion)
-
def check_key_in_keys(gpgKeyID, local_keys):
if gpgKeyID is not None:
print(' Verify your gpg key is in the main KEYS file')
diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py
index 9b26413..6a30508 100644
--- a/dev-tools/scripts/scriptutil.py
+++ b/dev-tools/scripts/scriptutil.py
@@ -17,7 +17,11 @@ import argparse
import re
import subprocess
import sys
+import os
from enum import Enum
+import time
+import urllib.request, urllib.error, urllib.parse
+import urllib.parse
class Version(object):
def __init__(self, major, minor, bugfix, prerelease):
@@ -66,14 +70,19 @@ class Version(object):
(self.bugfix > other.bugfix or self.bugfix == other.bugfix and
self.prerelease >= other.prerelease)))
+ def gt(self, other):
+ return (self.major > other.major or
+ (self.major == other.major and self.minor > other.minor) or
+ (self.major == other.major and self.minor == other.minor and self.bugfix > other.bugfix))
+
def is_back_compat_with(self, other):
if not self.on_or_after(other):
raise Exception('Back compat check disallowed for newer version: %s < %s' % (self, other))
return other.major + 1 >= self.major
-def run(cmd):
+def run(cmd, cwd=None):
try:
- output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+ output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, cwd=cwd)
except subprocess.CalledProcessError as e:
print(e.output.decode('utf-8'))
raise e
@@ -99,6 +108,18 @@ def update_file(filename, line_re, edit):
f.write(''.join(buffer))
return True
+
+def check_ant():
+ antVersion = os.popen('ant -version').read().strip()
+ if (antVersion.startswith('Apache Ant(TM) version 1.8')):
+ return antVersion.split(" ")[3]
+ if (antVersion.startswith('Apache Ant(TM) version 1.9')):
+ return antVersion.split(" ")[3]
+ if (antVersion.startswith('Apache Ant(TM) version 1.10')):
+ return antVersion.split(" ")[3]
+ raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion)
+
+
# branch types are "release", "stable" and "unstable"
class BranchType(Enum):
unstable = 1
@@ -122,6 +143,49 @@ def find_branch_type():
return BranchType.release
raise Exception('Cannot run %s on feature branch' % sys.argv[0].rsplit('/', 1)[-1])
+
+def download(name, urlString, tmpDir, quiet=False, force_clean=True):
+ if not quiet:
+ print("Downloading %s" % urlString)
+ startTime = time.time()
+ fileName = '%s/%s' % (tmpDir, name)
+ if not force_clean and os.path.exists(fileName):
+ if not quiet and fileName.find('.asc') == -1:
+ print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.))
+ return
+ try:
+ attemptDownload(urlString, fileName)
+ except Exception as e:
+ print('Retrying download of url %s after exception: %s' % (urlString, e))
+ try:
+ attemptDownload(urlString, fileName)
+ except Exception as e:
+ raise RuntimeError('failed to download url "%s"' % urlString) from e
+ if not quiet and fileName.find('.asc') == -1:
+ t = time.time()-startTime
+ sizeMB = os.path.getsize(fileName)/1024./1024.
+ print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t))
+
+
+def attemptDownload(urlString, fileName):
+ fIn = urllib.request.urlopen(urlString)
+ fOut = open(fileName, 'wb')
+ success = False
+ try:
+ while True:
+ s = fIn.read(65536)
+ if s == b'':
+ break
+ fOut.write(s)
+ fOut.close()
+ fIn.close()
+ success = True
+ finally:
+ fIn.close()
+ fOut.close()
+ if not success:
+ os.remove(fileName)
+
version_prop_re = re.compile('version\.base=(.*)')
def find_current_version():
return version_prop_re.search(open('lucene/version.properties').read()).group(1).strip()
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
old mode 100644
new mode 100755
index 614a941..8aa237b
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -14,35 +16,31 @@
# limitations under the License.
import argparse
-import os
-import zipfile
import codecs
-import tarfile
-import zipfile
-import threading
-import traceback
import datetime
-import time
-import subprocess
-import signal
-import shutil
+import filecmp
import hashlib
import http.client
+import os
+import platform
import re
-import urllib.request, urllib.error, urllib.parse
-import urllib.parse
+import shutil
+import subprocess
import sys
-import html.parser
-from collections import defaultdict
+import textwrap
+import traceback
+import urllib.error
+import urllib.parse
+import urllib.parse
+import urllib.request
import xml.etree.ElementTree as ET
-import filecmp
-import platform
+import zipfile
+from collections import defaultdict
+from collections import namedtuple
+from scriptutil import download
+
import checkJavaDocs
import checkJavadocLinks
-import io
-import codecs
-import textwrap
-from collections import namedtuple
# This tool expects to find /lucene and /solr off the base URL. You
# must have a working gpg, tar, unzip in your path. This has been
@@ -111,44 +109,6 @@ def getHREFs(urlString):
links.append((text, fullURL))
return links
-def download(name, urlString, tmpDir, quiet=False):
- startTime = time.time()
- fileName = '%s/%s' % (tmpDir, name)
- if not FORCE_CLEAN and os.path.exists(fileName):
- if not quiet and fileName.find('.asc') == -1:
- print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.))
- return
- try:
- attemptDownload(urlString, fileName)
- except Exception as e:
- print('Retrying download of url %s after exception: %s' % (urlString, e))
- try:
- attemptDownload(urlString, fileName)
- except Exception as e:
- raise RuntimeError('failed to download url "%s"' % urlString) from e
- if not quiet and fileName.find('.asc') == -1:
- t = time.time()-startTime
- sizeMB = os.path.getsize(fileName)/1024./1024.
- print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t))
-
-def attemptDownload(urlString, fileName):
- fIn = urllib.request.urlopen(urlString)
- fOut = open(fileName, 'wb')
- success = False
- try:
- while True:
- s = fIn.read(65536)
- if s == b'':
- break
- fOut.write(s)
- fOut.close()
- fIn.close()
- success = True
- finally:
- fIn.close()
- fOut.close()
- if not success:
- os.remove(fileName)
def load(urlString):
try:
@@ -362,13 +322,13 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
for artifact, urlString in artifacts:
print(' download %s...' % artifact)
- download(artifact, urlString, tmpDir)
+ download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN)
verifyDigests(artifact, urlString, tmpDir)
if isSigned:
print(' verify sig')
# Test sig (this is done with a clean brand-new GPG world)
- download(artifact + '.asc', urlString + '.asc', tmpDir)
+ download(artifact + '.asc', urlString + '.asc', tmpDir, force_clean=FORCE_CLEAN)
sigFile = '%s/%s.asc' % (tmpDir, artifact)
artifactFile = '%s/%s' % (tmpDir, artifact)
logFile = '%s/%s.%s.gpg.verify.log' % (tmpDir, project, artifact)
@@ -997,7 +957,7 @@ def getBinaryDistFiles(project, tmpDir, version, baseURL):
if not os.path.exists('%s/%s' % (tmpDir, distribution)):
distURL = '%s/%s/%s' % (baseURL, project, distribution)
print(' download %s...' % distribution, end=' ')
- download(distribution, distURL, tmpDir)
+ download(distribution, distURL, tmpDir, force_clean=FORCE_CLEAN)
destDir = '%s/unpack-%s-getBinaryDistFiles' % (tmpDir, project)
if os.path.exists(destDir):
shutil.rmtree(destDir)
@@ -1226,7 +1186,7 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
crawl(downloadedFiles, subURL, path, exclusions)
else:
if not os.path.exists(path) or FORCE_CLEAN:
- download(text, subURL, targetDir, quiet=True)
+ download(text, subURL, targetDir, quiet=True, force_clean=FORCE_CLEAN)
downloadedFiles.append(path)
sys.stdout.write('.')
@@ -1277,6 +1237,8 @@ def parse_config():
help='Version of the release, defaults to that in URL')
parser.add_argument('--test-java9', metavar='JAVA9_HOME',
help='Path to Java9 home directory, to run tests with if specified')
+ parser.add_argument('--download-only', action='store_true', default=False,
+ help='Only perform download and sha hash check steps')
parser.add_argument('url', help='Url pointing to release to test')
parser.add_argument('test_args', nargs=argparse.REMAINDER,
help='Arguments to pass to ant for testing, e.g. -Dwhat=ever.')
@@ -1445,10 +1407,10 @@ def main():
raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))
print('NOTE: output encoding is %s' % sys.stdout.encoding)
- smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args))
-
-def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs):
+ smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args),
+ downloadOnly=c.download_only)
+def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs, downloadOnly=False):
startTime = datetime.datetime.now()
# disable flakey tests for smoke-tester runs:
@@ -1489,27 +1451,32 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,
else:
keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
print(" Downloading online KEYS file %s" % keysFileURL)
- download('KEYS', keysFileURL, tmpDir)
+ download('KEYS', keysFileURL, tmpDir, force_clean=FORCE_CLEAN)
keysFile = '%s/KEYS' % (tmpDir)
print()
print('Test Lucene...')
checkSigs('lucene', lucenePath, version, tmpDir, isSigned, keysFile)
- for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
- unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
- unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
+ if not downloadOnly:
+ for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
+ unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
+ unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
+ else:
+ print("\nLucene test done (--download-only specified)")
print()
print('Test Solr...')
checkSigs('solr', solrPath, version, tmpDir, isSigned, keysFile)
- for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
- unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
- solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
- gitRevision, version, testArgs, baseURL)
-
- print()
- print('Test Maven artifacts for Lucene and Solr...')
- checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
+ if not downloadOnly:
+ for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
+ unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
+ solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
+ gitRevision, version, testArgs, baseURL)
+ print()
+ print('Test Maven artifacts for Lucene and Solr...')
+ checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
+ else:
+ print("Solr test done (--download-only specified)")
print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))