You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2018/09/11 21:36:58 UTC
lucene-solr:branch_7x: LUCENE-5143: Fix smoketester,
fix RM PGP key check, fix solr DOAP file,
add CHANGES entry Remove unused/stale 'copy-to-stage' and '-dist-keys' targets
from ant build
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 1c493552a -> 63a95ea9e
LUCENE-5143: Fix smoketester, fix RM PGP key check, fix solr DOAP file, add CHANGES entry
Remove unused/stale 'copy-to-stage' and '-dist-keys' targets from ant build
(cherry picked from commit 5b96f89)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/63a95ea9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/63a95ea9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/63a95ea9
Branch: refs/heads/branch_7x
Commit: 63a95ea9e95de5584a3debc7d860a33495380f42
Parents: 1c49355
Author: Jan Høydahl <ja...@apache.org>
Authored: Tue Sep 11 22:27:14 2018 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Tue Sep 11 23:18:37 2018 +0200
----------------------------------------------------------------------
dev-tools/doap/solr.rdf | 2 +-
dev-tools/scripts/buildAndPushRelease.py | 35 ++++++++++++++------------
dev-tools/scripts/smokeTestRelease.py | 36 ++++++++++++++++-----------
lucene/CHANGES.txt | 6 +++++
lucene/build.xml | 15 +----------
lucene/common-build.xml | 26 -------------------
solr/build.xml | 12 +--------
7 files changed, 49 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/63a95ea9/dev-tools/doap/solr.rdf
----------------------------------------------------------------------
diff --git a/dev-tools/doap/solr.rdf b/dev-tools/doap/solr.rdf
index 108eee3..c739fb7 100644
--- a/dev-tools/doap/solr.rdf
+++ b/dev-tools/doap/solr.rdf
@@ -71,7 +71,7 @@
<Version>
<name>solr-7.4.0</name>
<created>2018-06-27</created>
- <revision>7.4.1</revision>
+ <revision>7.4.0</revision>
</Version>
</release>
<release>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/63a95ea9/dev-tools/scripts/buildAndPushRelease.py
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py
index 3694f1a..5a8f5cc 100644
--- a/dev-tools/scripts/buildAndPushRelease.py
+++ b/dev-tools/scripts/buildAndPushRelease.py
@@ -78,9 +78,12 @@ def getGitRev():
raise RuntimeError('git clone is dirty:\n\n%s' % status)
branch = os.popen('git rev-parse --abbrev-ref HEAD').read().strip()
command = 'git log origin/%s..' % branch
- unpushedCommits = os.popen(command).read().strip()
- if len(unpushedCommits) > 0:
- raise RuntimeError('There are unpushed commits - "%s" output is:\n\n%s' % (command, unpushedCommits))
+ p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ if len(stdout.strip()) > 0:
+ raise RuntimeError('There are unpushed commits - "%s" output is:\n\n%s' % (command, stdout.decode('utf-8')))
+ if len(stderr.strip()) > 0:
+ raise RuntimeError('Command "%s" failed:\n\n%s' % (command, stderr.decode('utf-8')))
print(' git clone is clean')
return os.popen('git rev-parse HEAD').read().strip()
@@ -271,14 +274,6 @@ def parse_config():
config.version = read_version(config.root)
print('Building version: %s' % config.version)
- if config.sign:
- sys.stdout.flush()
- import getpass
- config.key_id = config.sign
- config.key_password = getpass.getpass('Enter GPG keystore password: ')
- else:
- config.gpg_password = None
-
return config
def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
@@ -313,15 +308,15 @@ def check_key_in_keys(gpgKeyID, local_keys):
if len(gpgKeyID) > 40:
gpgKeyID = gpgKeyID.replace(" ", "")
if len(gpgKeyID) == 8:
- re_to_match = r"^pub\s+\d+[DR]/%s " % gpgKeyID
+ gpgKeyID8Char = "%s %s" % (gpgKeyID[0:4], gpgKeyID[4:8])
+ re_to_match = r"^pub .*\n\s+\w{4} \w{4} \w{4} \w{4} \w{4} \w{4} \w{4} \w{4} %s" % gpgKeyID8Char
elif len(gpgKeyID) == 40:
gpgKeyID40Char = "%s %s %s %s %s %s %s %s %s %s" % \
(gpgKeyID[0:4], gpgKeyID[4:8], gpgKeyID[8:12], gpgKeyID[12:16], gpgKeyID[16:20],
gpgKeyID[20:24], gpgKeyID[24:28], gpgKeyID[28:32], gpgKeyID[32:36], gpgKeyID[36:])
- print("Generated id string %s" % gpgKeyID40Char)
- re_to_match = r"^\s+Key fingerprint = %s$" % gpgKeyID40Char
+ re_to_match = r"^pub .*\n\s+%s" % gpgKeyID40Char
else:
- print('Invalid gpg key id format. Must be 8 byte short ID or 40 byte fingerprint, with or without 0x prefix.')
+ print('Invalid gpg key id format. Must be 8 byte short ID or 40 byte fingerprint, with or without 0x prefix, no spaces.')
exit(2)
if re.search(re_to_match, keysFileText, re.MULTILINE):
print(' Found key %s in KEYS file at %s' % (gpgKeyID, keysFileLocation))
@@ -337,7 +332,15 @@ def main():
c = parse_config()
- check_key_in_keys(c.key_id, c.local_keys)
+ if c.sign:
+ sys.stdout.flush()
+ c.key_id = c.sign
+ check_key_in_keys(c.key_id, c.local_keys)
+ import getpass
+ c.key_password = getpass.getpass('Enter GPG keystore password: ')
+ else:
+ c.key_id = None
+ c.key_password = None
if c.prepare:
rev = prepare(c.root, c.version, c.key_id, c.key_password)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/63a95ea9/dev-tools/scripts/smokeTestRelease.py
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
index f68a9b1..af26ae1 100644
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -285,23 +285,22 @@ def checkAllJARs(topDir, project, gitRevision, version, tmpDir, baseURL):
% (fullPath, luceneDistFilenames[jarFilename]))
-def checkSigs(project, urlString, version, tmpDir, isSigned):
+def checkSigs(project, urlString, version, tmpDir, isSigned, local_keys):
print(' test basics...')
ents = getDirEntries(urlString)
artifact = None
- keysURL = None
changesURL = None
mavenURL = None
expectedSigs = []
if isSigned:
expectedSigs.append('asc')
expectedSigs.extend(['sha1', 'sha512'])
-
+
artifacts = []
for text, subURL in ents:
if text == 'KEYS':
- keysURL = subURL
+ raise RuntimeError('%s: release dir should not contain a KEYS file - only toplevel /dist/lucene/KEYS is used' % project)
elif text == 'maven/':
mavenURL = subURL
elif text.startswith('changes'):
@@ -346,14 +345,16 @@ def checkSigs(project, urlString, version, tmpDir, isSigned):
if expected != actual:
raise RuntimeError('%s: wrong artifacts: expected %s but got %s' % (project, expected, actual))
- if keysURL is None:
- raise RuntimeError('%s is missing KEYS' % project)
-
print(' get KEYS')
- download('%s.KEYS' % project, keysURL, tmpDir)
-
- keysFile = '%s/%s.KEYS' % (tmpDir, project)
-
+ if local_keys is not None:
+ print(" Using local KEYS file %s" % local_keys)
+ keysFile = local_keys
+ else:
+ keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
+ print(" Downloading online KEYS file %s" % keysFileURL)
+ download('KEYS', keysFileURL, tmpDir)
+ keysFile = '%s/KEYS' % (tmpDir)
+
# Set up clean gpg world; import keys file:
gpgHomeDir = '%s/%s.gpg' % (tmpDir, project)
if os.path.exists(gpgHomeDir):
@@ -1291,6 +1292,8 @@ def parse_config():
help='Temporary directory to test inside, defaults to /tmp/smoke_lucene_$version_$revision')
parser.add_argument('--not-signed', dest='is_signed', action='store_false', default=True,
help='Indicates the release is not signed')
+ parser.add_argument('--local-keys', metavar='PATH',
+ help='Uses local KEYS file instead of fetching from https://archive.apache.org/dist/lucene/KEYS')
parser.add_argument('--revision',
help='GIT revision number that release was built with, defaults to that in URL')
parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
@@ -1318,6 +1321,9 @@ def parse_config():
c.revision = revision_match.group(1)
print('Revision: %s' % c.revision)
+ if c.local_keys is not None and not os.path.exists(c.local_keys):
+ parser.error('Local KEYS file "%s" not found' % c.local_keys)
+
c.java = make_java_config(parser, c.test_java9)
if c.tmp_dir:
@@ -1462,9 +1468,9 @@ def main():
raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))
print('NOTE: output encoding is %s' % sys.stdout.encoding)
- smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, ' '.join(c.test_args))
+ smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args))
-def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, testArgs):
+def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs):
startTime = datetime.datetime.now()
@@ -1500,14 +1506,14 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, testArgs):
print()
print('Test Lucene...')
- checkSigs('lucene', lucenePath, version, tmpDir, isSigned)
+ checkSigs('lucene', lucenePath, version, tmpDir, isSigned, local_keys)
for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
print()
print('Test Solr...')
- checkSigs('solr', solrPath, version, tmpDir, isSigned)
+ checkSigs('solr', solrPath, version, tmpDir, isSigned, local_keys)
for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/63a95ea9/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index e46b017..42cff5a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -150,6 +150,12 @@ Improvements
* LUCENE-8422: IntervalQuery now returns useful Matches (Alan Woodward)
+Build
+
+* LUCENE-5143: Stop publishing KEYS file with each version, use topmost lucene/KEYS file only.
+ The buildAndPushRelease.py script validates that RM's PGP key is in the KEYS file.
+ Remove unused 'copy-to-stage' and '-dist-keys' targets from ant build. (janhoy)
+
Other:
* LUCENE-8485: Update randomizedtesting to version 2.6.4. (Dawid Weiss)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/63a95ea9/lucene/build.xml
----------------------------------------------------------------------
diff --git a/lucene/build.xml b/lucene/build.xml
index b6131af..3c1439c 100644
--- a/lucene/build.xml
+++ b/lucene/build.xml
@@ -387,7 +387,7 @@
<!-- ================================================================== -->
<target name="dist-src" depends="package-tgz-src"/>
- <target name="dist-all" depends="dist, dist-src, -dist-changes, -dist-keys"/>
+ <target name="dist-all" depends="dist, dist-src, -dist-changes"/>
<!-- copy changes/ to the release folder -->
<target name="-dist-changes">
@@ -396,21 +396,8 @@
</copy>
</target>
- <!-- copy KEYS to the release folder -->
- <target name="-dist-keys">
- <get src="http://home.apache.org/keys/group/lucene.asc"
- dest="${dist.dir}/KEYS"/>
- </target>
-
- <target name="copy-to-stage">
- <copy-to-stage-macro artifacts.dir="${dist.dir}"/>
- </target>
-
<target name="prepare-release-no-sign" depends="clean, dist-all, generate-maven-artifacts"/>
<target name="prepare-release" depends="prepare-release-no-sign, sign-artifacts"/>
- <target name="stage" depends="prepare-release, copy-to-stage">
-
- </target>
<target name="-dist-maven" depends="install-maven-tasks">
<sequential>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/63a95ea9/lucene/common-build.xml
----------------------------------------------------------------------
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index 68e17da..9fbddc3 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -2380,32 +2380,6 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
</sequential>
</macrodef>
- <property name="rc" value="rc0"/>
- <property name="remote.staging.dir" value="public_html/staging_area/${rc}/${version}"/>
- <property name="keyfile" value="${user.home}/.ssh/id_rsa"/>
- <property name="scp.user" value="${user.name}"/>
- <!--keys.dir is the location of the https://svn.apache.org/repos/asf/lucene/java/dist/ directory-->
- <property name="keys.dir" value="${common.dir}/../../dist"/>
- <macrodef name="copy-to-stage-macro">
- <attribute name="artifacts.dir"/>
- <sequential>
- <sshexec host="home.apache.org"
- username="${scp.user}"
- keyfile="${keyfile}"
- command="mkdir -p ${remote.staging.dir}"/>
- <echo>Uploading artifacts to ${scp.user}@home.apache.org:${remote.staging.dir}</echo>
- <scp todir="${scp.user}@home.apache.org:${remote.staging.dir}"
- username="${scp.user}"
- keyfile="${keyfile}"
- verbose="true">
- <fileset dir="${artifacts.dir}"/>
- <fileset dir="${keys.dir}">
- <include name="KEYS"/>
- </fileset>
- </scp>
- </sequential>
- </macrodef>
-
<!-- JFlex task -->
<target name="-install-jflex" unless="jflex.loaded" depends="ivy-availability-check,ivy-configure">
<ivy:cachepath organisation="de.jflex" module="jflex" revision="1.6.0"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/63a95ea9/solr/build.xml
----------------------------------------------------------------------
diff --git a/solr/build.xml b/solr/build.xml
index cb4ed75..7ed40bb 100644
--- a/solr/build.xml
+++ b/solr/build.xml
@@ -443,10 +443,6 @@
<!-- ===================== DISTRIBUTION-RELATED TASKS ======================== -->
<!-- ========================================================================= -->
- <target name="copy-to-stage">
- <copy-to-stage-macro artifacts.dir="${package.dir}"/>
- </target>
-
<target name="dist"
description="Creates the Solr distribution files."
depends="dist-solrj, dist-core, dist-test-framework, dist-contrib" />
@@ -467,7 +463,7 @@
<target name="prepare-release" depends="prepare-release-no-sign, sign-artifacts"/>
<!-- make a distribution -->
- <target name="package" depends="package-src-tgz,create-package,documentation,-dist-changes,-dist-keys"/>
+ <target name="package" depends="package-src-tgz,create-package,documentation,-dist-changes"/>
<!-- copy changes/ to the release folder -->
<target name="-dist-changes">
@@ -476,12 +472,6 @@
</copy>
</target>
- <!-- copy KEYS to the release folder -->
- <target name="-dist-keys">
- <get src="http://home.apache.org/keys/group/lucene.asc"
- dest="${package.dir}/KEYS"/>
- </target>
-
<!-- Makes a tarball of the source. -->
<!-- Copies NOTICE.txt and LICENSE.txt from solr/ to the root level. -->
<target name="package-src-tgz" depends="init-dist"