You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Uwe Schindler <uw...@thetaphi.de> on 2014/08/28 10:38:37 UTC
RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts: buildAndPushRelease.py smokeTestRelease.py
Hi,
did the parameter names change by that? If yes, you need to fix the build.xml in the root folder to pass right params in "nigtly-smoke".
Uwe
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: rjernst@apache.org [mailto:rjernst@apache.org]
> Sent: Thursday, August 28, 2014 10:02 AM
> To: commits@lucene.apache.org
> Subject: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> buildAndPushRelease.py smokeTestRelease.py
>
> Author: rjernst
> Date: Thu Aug 28 08:01:50 2014
> New Revision: 1621085
>
> URL: http://svn.apache.org/r1621085
> Log:
> Change buildAndPushRelease and smokeTestRelease scripts to use argparse
>
> Modified:
> lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
>
> Modified: lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> tools/scripts/buildAndPushRelease.py?rev=1621085&r1=1621084&r2=162108
> 5&view=diff
> ==========================================================
> ====================
> --- lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py (original)
> +++ lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py Thu Aug
> 28
> +++ 08:01:50 2014
> @@ -13,6 +13,7 @@
> # See the License for the specific language governing permissions and #
> limitations under the License.
>
> +import argparse
> import datetime
> import re
> import time
> @@ -20,16 +21,7 @@ import shutil
> import os
> import sys
> import subprocess
> -
> -# Usage: python3.2 -u buildAndPushRelease.py [-sign gpgKey(eg:
> 6E68DA61)] [-prepare] [-push userName] [-pushLocal dirName] [-smoke
> tmpDir] /path/to/checkout version(eg: 3.4.0) rcNum(eg: 0) -# -# EG:
> python3.2 -u buildAndPushRelease.py -prepare -push mikemccand -sign
> 6E68DA61 /path/to/lucene_solr_4_7 4.7.0 0
> -
> -# NOTE: if you specify -sign, you have to type in your gpg password at -#
> some point while this runs; it's VERY confusing because the output -# is
> directed to /tmp/release.log, so, you have to tail that and when -# GPG
> wants your password, type it! Also sometimes you have to type -# it twice in
> a row!
> +import textwrap
>
> LOG = '/tmp/release.log'
>
> @@ -226,94 +218,89 @@ def pushLocal(version, root, rev, rcNum,
>
> print(' done!')
> return 'file://%s/%s' % (os.path.abspath(localDir), dir)
> -
> -def main():
> - doPrepare = '-prepare' in sys.argv
> - if doPrepare:
> - sys.argv.remove('-prepare')
>
> - try:
> - idx = sys.argv.index('-push')
> - except ValueError:
> - doPushRemote = False
> - else:
> - doPushRemote = True
> - username = sys.argv[idx+1]
> - del sys.argv[idx:idx+2]
> +def read_version(path):
> + version_props_file = os.path.join(path, 'lucene',
> +'version.properties')
> + return re.search(r'version\.base=(.*)',
> +open(version_props_file).read()).group(1)
> +
> +def parse_config():
> + epilogue = textwrap.dedent('''
> + Example usage for a Release Manager:
> + python3.2 -u buildAndPushRelease.py --push-remote mikemccand --sign
> +6E68DA61 --rc-num 1 --version 4.7.0 /path/to/lucene_solr_4_7
> + ''')
> + description = 'Utility to build, push, and test a release.'
> + parser = argparse.ArgumentParser(description=description,
> epilog=epilogue,
> +
> +formatter_class=argparse.RawDescriptionHelpFormatter)
> + parser.add_argument('--no-prepare', dest='prepare', default=True,
> action='store_false',
> + help='Use the already built release in the
> +provided checkout')
> + parser.add_argument('--push-remote', metavar='USERNAME',
> + help='Push the release to people.apache.org for
> +the given user')
> + parser.add_argument('--push-local', metavar='PATH',
> + help='Push the release to the local path')
> + parser.add_argument('--sign', metavar='KEYID',
> + help='Sign the release with the given gpg key')
> + parser.add_argument('--rc-num', metavar='NUM', type=int, default=1,
> + help='Release Candidate number, required')
> + parser.add_argument('--smoke-test', metavar='PATH',
> + help='Run the smoker tester on the release in the
> +given directory')
> + parser.add_argument('root', metavar='checkout_path',
> + help='Root of SVN checkout for lucene-solr')
> + config = parser.parse_args()
> +
> + if config.push_remote is not None and config.push_local is not None:
> + parser.error('Cannot specify --push-remote and --push-local
> + together') if not config.prepare and config.sign:
> + parser.error('Cannot sign already built release') if
> + config.push_local is not None and os.path.exists(config.push_local):
> + parser.error('Cannot push to local path that already exists') if
> + config.rc_num <= 0:
> + parser.error('Release Candidate number must be a positive integer')
> + if not os.path.isdir(config.root):
> + # TODO: add additional svn check to ensure dir is a real lucene-solr
> checkout
> + parser.error('Root path is not a valid lucene-solr checkout') if
> + config.smoke_test is not None and os.path.exists(config.smoke_test):
> + parser.error('Smoke test path already exists')
>
> - try:
> - idx = sys.argv.index('-smoke')
> - except ValueError:
> - smokeTmpDir = None
> - else:
> - smokeTmpDir = sys.argv[idx+1]
> - del sys.argv[idx:idx+2]
> - if os.path.exists(smokeTmpDir):
> - print()
> - print('ERROR: smoke tmpDir "%s" exists; please remove first' %
> smokeTmpDir)
> - print()
> - sys.exit(1)
> -
> - try:
> - idx = sys.argv.index('-pushLocal')
> - except ValueError:
> - doPushLocal = False
> - else:
> - doPushLocal = True
> - localStagingDir = sys.argv[idx+1]
> - del sys.argv[idx:idx+2]
> - if os.path.exists(localStagingDir):
> - print()
> - print('ERROR: pushLocal dir "%s" exists; please remove first' %
> localStagingDir)
> - print()
> - sys.exit(1)
> -
> - if doPushRemote and doPushLocal:
> - print()
> - print('ERROR: specify at most one of -push or -pushLocal (got both)')
> - print()
> - sys.exit(1)
> -
> - try:
> - idx = sys.argv.index('-sign')
> - except ValueError:
> - gpgKeyID = None
> - else:
> - gpgKeyID = sys.argv[idx+1]
> - del sys.argv[idx:idx+2]
> + config.version = read_version(config.root) print('Building version:
> + %s' % config.version)
>
> + if config.sign:
> sys.stdout.flush()
> import getpass
> - gpgPassword = getpass.getpass('Enter GPG keystore password: ')
> + config.key_id = config.sign
> + config.key_password = getpass.getpass('Enter GPG keystore password:
> + ')
> + else:
> + config.gpg_password = None
>
> - root = os.path.abspath(sys.argv[1])
> - version = sys.argv[2]
> - rcNum = int(sys.argv[3])
> + return config
> +
> +def main():
> + c = parse_config()
>
> - if doPrepare:
> - rev = prepare(root, version, gpgKeyID, gpgPassword, smokeTmpDir is
> None)
> + if c.prepare:
> + rev = prepare(c.root, c.version, c.key_id, c.key_password,
> + key.smoke_test is None)
> else:
> os.chdir(root)
> rev = open('rev.txt', encoding='UTF-8').read()
>
> - if doPushRemote:
> + if c.push_remote:
> url = push(version, root, rev, rcNum, username)
> - elif doPushLocal:
> - url = pushLocal(version, root, rev, rcNum, localStagingDir)
> + elif c.push_local:
> + url = pushLocal(version, root, rev, c.rc_num, c.push_local)
> else:
> url = None
>
> if url is not None:
> print(' URL: %s' % url)
>
> - if smokeTmpDir is not None:
> + if c.smoke_test is not None:
> import smokeTestRelease
> smokeTestRelease.DEBUG = False
> - smokeTestRelease.smokeTest(url, rev, version, smokeTmpDir, gpgKeyID
> is not None, '')
> + smokeTestRelease.smokeTest(url, rev, c.version, c.smoke_test,
> + c.sign is not None, '')
>
> if __name__ == '__main__':
> try:
> main()
> - except:
> - import traceback
> - traceback.print_exc()
> + except KeyboardInterrupt:
> + print('Keyboard interrupt...exiting')
> +
>
> Modified: lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> tools/scripts/smokeTestRelease.py?rev=1621085&r1=1621084&r2=1621085&
> view=diff
> ==========================================================
> ====================
> --- lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py (original)
> +++ lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py Thu Aug 28
> +++ 08:01:50 2014
> @@ -13,6 +13,7 @@
> # See the License for the specific language governing permissions and #
> limitations under the License.
>
> +import argparse
> import os
> import zipfile
> import codecs
> @@ -40,6 +41,7 @@ import checkJavaDocs
> import checkJavadocLinks
> import io
> import codecs
> +import textwrap
>
> # This tool expects to find /lucene and /solr off the base URL. You # must
> have a working gpg, tar, unzip in your path. This has been @@ -1248,55
> +1250,60 @@ def crawl(downloadedFiles, urlString, ta
> downloadedFiles.append(path)
> sys.stdout.write('.')
>
> -reAllowedVersion = re.compile(r'^\d+\.\d+\.\d+(-ALPHA|-BETA)?$')
> -
> -def main():
> -
> - if len(sys.argv) < 5:
> - print()
> - print('Usage python -u %s BaseURL SvnRevision version tmpDir [
> isSigned(True|False) ] [ -testArgs "-Dwhat=ever [ ... ]" ]'
> - % sys.argv[0])
> - print()
> - print(' example: python3.2 -u dev-tools/scripts/smokeTestRelease.py
> http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-RC1-
> rev1469340 1469340 4.3.0 /path/to/a/tmp/dir')
> - print()
> - sys.exit(1)
> -
> - baseURL = sys.argv[1]
> - svnRevision = sys.argv[2]
> - version = sys.argv[3]
> +version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)')
> +revision_re = re.compile(r'rev(\d+)')
> +def parse_config():
> + epilogue = textwrap.dedent('''
> + Example usage:
> + python3.2 -u dev-tools/scripts/smokeTestRelease.py
> +http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-RC1-
> re
> +v1469340')
> + ''')
> + description = 'Utility to test a release.'
> + parser = argparse.ArgumentParser(description=description,
> epilog=epilogue,
> +
> +formatter_class=argparse.RawDescriptionHelpFormatter)
> + parser.add_argument('--tmp-dir', metavar='PATH',
> + help='Temporary directory to test inside,
> +defaults to /tmp/smoke_lucene_$version_$revision')
> + parser.add_argument('--not-signed', dest='is_signed',
> action='store_false', default=True,
> + help='Indicates the release is not signed')
> + parser.add_argument('--revision',
> + help='SVN revision number that release was built
> +with, defaults to that in URL')
> + parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
> + help='Version of the release, defaults to that in
> +URL')
> + parser.add_argument('url', help='Url pointing to release to test')
> + parser.add_argument('test_args', nargs=argparse.REMAINDER,
> metavar='ARGS',
> + help='Arguments to pass to ant for testing, e.g.
> +-Dwhat=ever')
> + c = parser.parse_args()
> +
> + if c.version is not None:
> + if not version_re.match(c.version):
> + parser.error('version "%s" does not match format
> + X.Y.Z[-ALPHA|-BETA]' % c.version)
> + else:
> + version_match = version_re.search(c.url)
> + if version_match is None:
> + parser.error('Could not find version in URL')
> + c.version = version_match.group(1)
> +
> + if c.revision is None:
> + revision_match = revision_re.search(c.url)
> + if revision_match is None:
> + parser.error('Could not find revision in URL')
> + c.revision = revision_match.group(1)
>
> - if not reAllowedVersion.match(version):
> - raise RuntimeError('version "%s" does not match format X.Y.Z[-ALPHA|-
> BETA]' % version)
> -
> - tmpDirArgNum = 4
> - tmpDir = os.path.abspath(sys.argv[tmpDirArgNum])
> + if c.tmp_dir:
> + c.tmp_dir = os.path.abspath(c.tmp_dir)
> + else:
> + tmp = '/tmp/smoke_lucene_%s_%s' % (c.version, c.revision)
> + c.tmp_dir = tmp
> + i = 1
> + while os.path.exists(c.tmp_dir):
> + c.tmp_dir = tmp + '_%d' % i
> + i += 1
>
> - # TODO: yuck: positional-only args with more than one optional arg totally
> sucks
> - # TODO: consider naming all args
> - isSigned = True
> - testArgs = ''
> - lastArgNum = len(sys.argv) - 1
> - if lastArgNum > tmpDirArgNum:
> - nextArgNum = tmpDirArgNum + 1
> - if sys.argv[nextArgNum] == '-testArgs':
> - if nextArgNum == lastArgNum:
> - raise RuntimeError('missing expected argument to -testArgs')
> - else:
> - # TODO: should there be arg validation here? E.g. starts with '-D'?
> - testArgs = sys.argv[nextArgNum + 1]
> - nextArgNum += 2
> - if nextArgNum <= lastArgNum:
> - isSigned = (sys.argv[nextArgNum].lower() == "true")
> - nextArgNum += 1
> - if nextArgNum <= lastArgNum and testArgs == '':
> - if sys.argv[nextArgNum] == '-testArgs':
> - if nextArgNum == lastArgNum:
> - raise RuntimeError('missing expected argument to -testArgs')
> - else:
> - # TODO: should there be arg validation here? E.g. starts with '-D'?
> - testArgs = sys.argv[nextArgNum + 1]
> + return c
>
> - smokeTest(baseURL, svnRevision, version, tmpDir, isSigned, testArgs)
> +def main():
> + c = parse_config()
> + print('NOTE: output encoding is %s' % sys.stdout.encoding)
> + smokeTest(c.url, c.revision, c.version, c.tmp_dir, c.is_signed, '
> +'.join(c.test_args))
>
> def smokeTest(baseURL, svnRevision, version, tmpDir, isSigned, testArgs):
>
> @@ -1350,10 +1357,8 @@ def smokeTest(baseURL, svnRevision, vers
> print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))
>
> if __name__ == '__main__':
> - print('NOTE: output encoding is %s' % sys.stdout.encoding)
> try:
> main()
> - except:
> - traceback.print_exc()
> - sys.exit(1)
> - sys.exit(0)
> + except KeyboardInterrupt:
> + print('Keyboard interrupt...exiting')
> +
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts: buildAndPushRelease.py smokeTestRelease.py
Posted by Uwe Schindler <uw...@thetaphi.de>.
Latest build succeeded, so the change to argparse is fine now with nightly-smoke:
https://builds.apache.org/job/Lucene-Solr-SmokeRelease-trunk/195/console
Uwe
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: Uwe Schindler [mailto:uwe@thetaphi.de]
> Sent: Thursday, August 28, 2014 11:54 AM
> To: dev@lucene.apache.org
> Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> buildAndPushRelease.py smokeTestRelease.py
>
> Worked on Jenkins, but we have another problem:
>
> [mkdir] Created dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-
> Solr-SmokeRelease-4.x/lucene/build/fakeRelease
> [copy] Copying 431 files to /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-SmokeRelease-
> 4.x/lucene/build/fakeRelease/lucene
> [copy] Copying 239 files to /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-SmokeRelease-
> 4.x/lucene/build/fakeRelease/solr
> [exec] JAVA7_HOME is /home/jenkins/tools/java/latest1.7
> [exec] NOTE: output encoding is US-ASCII
> [exec]
> [exec] Load release URL "file:/usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-SmokeRelease-
> 4.x/lucene/build/fakeRelease/"...
> [exec]
> [exec] Test Lucene...
> [exec] test basics...
> [exec] get KEYS
> [exec] 0.1 MB in 0.01 sec (14.5 MB/sec)
> [exec] check changes HTML...
> [exec] download lucene-4.11.0-src.tgz...
> [exec] 27.7 MB in 0.04 sec (663.2 MB/sec)
> [exec] verify md5/sha1 digests
> [exec] download lucene-4.11.0.tgz...
> [exec] 62.1 MB in 0.15 sec (416.8 MB/sec)
> [exec] verify md5/sha1 digests
> [exec] download lucene-4.11.0.zip...
> [exec] 71.9 MB in 0.12 sec (619.8 MB/sec)
> [exec] verify md5/sha1 digests
> [exec] unpack lucene-4.11.0.tgz...
> [exec] verify JAR metadata/identity/no javax.* or java.* classes...
> [exec] test demo with 1.7...
> [exec] got 5806 hits for query "lucene"
> [exec] checkindex with 1.7...
> [exec] check Lucene's javadoc JAR
> [exec] unpack lucene-4.11.0.zip...
> [exec] verify JAR metadata/identity/no javax.* or java.* classes...
> [exec] test demo with 1.7...
> [exec] got 5806 hits for query "lucene"
> [exec] checkindex with 1.7...
> [exec] check Lucene's javadoc JAR
> [exec] unpack lucene-4.11.0-src.tgz...
> [exec] Traceback (most recent call last):
> [exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 1365, in
> <module>
> [exec] main()
> [exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 1310, in
> main
> [exec] smokeTest(c.url, c.revision, c.version, c.tmp_dir, c.is_signed, '
> '.join(c.test_args))
> [exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 1348, in
> smokeTest
> [exec] unpackAndVerify('lucene', tmpDir, 'lucene-%s-src.tgz' % version,
> svnRevision, version, testArgs, baseURL)
> [exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 639, in
> unpackAndVerify
> [exec] verifyUnpacked(project, artifact, unpackPath, svnRevision,
> version, testArgs, tmpDir, baseURL)
> [exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 710, in
> verifyUnpacked
> [exec] raise RuntimeError('%s: unexpected files/dirs in artifact %s: %s' %
> (project, artifact, l))
> [exec] RuntimeError: lucene: unexpected files/dirs in artifact lucene-
> 4.11.0-src.tgz: ['version.properties']
>
> We should fix the smoke tester to allow this new file. Will look into it!
>
> Uwe
>
> -----
> Uwe Schindler
> H.-H.-Meier-Allee 63, D-28213 Bremen
> http://www.thetaphi.de
> eMail: uwe@thetaphi.de
>
>
> > -----Original Message-----
> > From: Uwe Schindler [mailto:uwe@thetaphi.de]
> > Sent: Thursday, August 28, 2014 11:32 AM
> > To: dev@lucene.apache.org
> > Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> > buildAndPushRelease.py smokeTestRelease.py
> >
> > Hi,
> >
> > I did a quick test, works. I committed that patch with minor modifications:
> >
> > - changed arg line back to arg value for test args. The problem in the
> > python script is the " ".join(testArgs), whoich breaks when test args
> > are escaped. So it is better to pass the whole arg line as one parameter,
> escaped completely.
> > The joining then is a no-op, because there is only one parameter.
> > - removed useless svnversion.
> >
> > I hope this also works on Jenkins. I'll trigger a build.
> > Uwe
> >
> > -----
> > Uwe Schindler
> > H.-H.-Meier-Allee 63, D-28213 Bremen
> > http://www.thetaphi.de
> > eMail: uwe@thetaphi.de
> >
> >
> > > -----Original Message-----
> > > From: Uwe Schindler [mailto:uwe@thetaphi.de]
> > > Sent: Thursday, August 28, 2014 11:01 AM
> > > To: dev@lucene.apache.org
> > > Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-
> tools/scripts:
> > > buildAndPushRelease.py smokeTestRelease.py
> > >
> > > I think the attached patch should be correct:
> > >
> > > Index: build.xml
> > >
> >
> ==========================================================
> > > =========
> > > --- build.xml (revision 1621092)
> > > +++ build.xml (working copy)
> > > @@ -407,13 +407,15 @@
> > > <!-- Tell Python not to write any bytecode cache into the filesystem: --
> >
> > > <arg value="-B"/>
> > > <arg file="dev-tools/scripts/smokeTestRelease.py"/>
> > > - <arg value="${fakeRelease.uri}"/>
> > > + <arg value="--revision"/>
> > > <arg value="skip"/>
> > > + <arg value="--version"/>
> > > <arg value="${fakeReleaseVersion}"/>
> > > + <arg value="--tmp-dir"/>
> > > <arg file="${fakeReleaseTmp}"/>
> > > - <arg value="false"/>
> > > - <arg value="-testArgs"/>
> > > - <arg value="${smokeTestRelease.testArgs}"/>
> > > + <arg value="--not-signed"/>
> > > + <arg value="${fakeRelease.uri}"/>
> > > + <arg line="${smokeTestRelease.testArgs}"/>
> > > <env key="JAVA7_HOME" file="${JAVA7_HOME}"/>
> > > </exec>
> > > <delete dir="${fakeRelease}"/>
> > >
> > > -----
> > > Uwe Schindler
> > > H.-H.-Meier-Allee 63, D-28213 Bremen http://www.thetaphi.de
> > > eMail: uwe@thetaphi.de
> > >
> > >
> > > > -----Original Message-----
> > > > From: Uwe Schindler [mailto:uwe@thetaphi.de]
> > > > Sent: Thursday, August 28, 2014 10:39 AM
> > > > To: dev@lucene.apache.org
> > > > Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-
> > tools/scripts:
> > > > buildAndPushRelease.py smokeTestRelease.py
> > > >
> > > > Hi,
> > > >
> > > > did the parameter names change by that? If yes, you need to fix the
> > > > build.xml in the root folder to pass right params in "nigtly-smoke".
> > > >
> > > > Uwe
> > > >
> > > > -----
> > > > Uwe Schindler
> > > > H.-H.-Meier-Allee 63, D-28213 Bremen http://www.thetaphi.de
> > > > eMail: uwe@thetaphi.de
> > > >
> > > > > -----Original Message-----
> > > > > From: rjernst@apache.org [mailto:rjernst@apache.org]
> > > > > Sent: Thursday, August 28, 2014 10:02 AM
> > > > > To: commits@lucene.apache.org
> > > > > Subject: svn commit: r1621085 - in /lucene/dev/trunk/dev-
> tools/scripts:
> > > > > buildAndPushRelease.py smokeTestRelease.py
> > > > >
> > > > > Author: rjernst
> > > > > Date: Thu Aug 28 08:01:50 2014
> > > > > New Revision: 1621085
> > > > >
> > > > > URL: http://svn.apache.org/r1621085
> > > > > Log:
> > > > > Change buildAndPushRelease and smokeTestRelease scripts to use
> > > > > argparse
> > > > >
> > > > > Modified:
> > > > > lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > > > > lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > > > >
> > > > > Modified:
> > > > > lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > > > > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> > > > >
> > > >
> > >
> >
> tools/scripts/buildAndPushRelease.py?rev=1621085&r1=1621084&r2=162108
> > > > > 5&view=diff
> > > > >
> > > >
> > >
> >
> ==========================================================
> > > > > ====================
> > > > > --- lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > > > > (original)
> > > > > +++ lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> Thu
> > > > > +++ Aug
> > > > > 28
> > > > > +++ 08:01:50 2014
> > > > > @@ -13,6 +13,7 @@
> > > > > # See the License for the specific language governing permissions
> > > > > and # limitations under the License.
> > > > >
> > > > > +import argparse
> > > > > import datetime
> > > > > import re
> > > > > import time
> > > > > @@ -20,16 +21,7 @@ import shutil
> > > > > import os
> > > > > import sys
> > > > > import subprocess
> > > > > -
> > > > > -# Usage: python3.2 -u buildAndPushRelease.py [-sign gpgKey(eg:
> > > > > 6E68DA61)] [-prepare] [-push userName] [-pushLocal dirName]
> > > > > [-smoke tmpDir] /path/to/checkout version(eg: 3.4.0) rcNum(eg: 0) -
> # -
> > # EG:
> > > > > python3.2 -u buildAndPushRelease.py -prepare -push mikemccand
> > > > > -sign
> > > > > 6E68DA61 /path/to/lucene_solr_4_7 4.7.0 0
> > > > > -
> > > > > -# NOTE: if you specify -sign, you have to type in your gpg
> > > > > password at -# some point while this runs; it's VERY confusing
> > > > > because the output -# is directed to /tmp/release.log, so, you
> > > > > have to tail that and when -# GPG wants your password, type it!
> > > > > Also sometimes you have to type -# it twice in a row!
> > > > > +import textwrap
> > > > >
> > > > > LOG = '/tmp/release.log'
> > > > >
> > > > > @@ -226,94 +218,89 @@ def pushLocal(version, root, rev, rcNum,
> > > > >
> > > > > print(' done!')
> > > > > return 'file://%s/%s' % (os.path.abspath(localDir), dir)
> > > > > -
> > > > > -def main():
> > > > > - doPrepare = '-prepare' in sys.argv
> > > > > - if doPrepare:
> > > > > - sys.argv.remove('-prepare')
> > > > >
> > > > > - try:
> > > > > - idx = sys.argv.index('-push')
> > > > > - except ValueError:
> > > > > - doPushRemote = False
> > > > > - else:
> > > > > - doPushRemote = True
> > > > > - username = sys.argv[idx+1]
> > > > > - del sys.argv[idx:idx+2]
> > > > > +def read_version(path):
> > > > > + version_props_file = os.path.join(path, 'lucene',
> > > > > +'version.properties')
> > > > > + return re.search(r'version\.base=(.*)',
> > > > > +open(version_props_file).read()).group(1)
> > > > > +
> > > > > +def parse_config():
> > > > > + epilogue = textwrap.dedent('''
> > > > > + Example usage for a Release Manager:
> > > > > + python3.2 -u buildAndPushRelease.py --push-remote
> mikemccand
> > > > > +--sign
> > > > > +6E68DA61 --rc-num 1 --version 4.7.0 /path/to/lucene_solr_4_7
> > > > > + ''')
> > > > > + description = 'Utility to build, push, and test a release.'
> > > > > + parser = argparse.ArgumentParser(description=description,
> > > > > epilog=epilogue,
> > > > > +
> > > > > +formatter_class=argparse.RawDescriptionHelpFormatter)
> > > > > + parser.add_argument('--no-prepare', dest='prepare',
> > > > > +default=True,
> > > > > action='store_false',
> > > > > + help='Use the already built release in the
> > > > > +provided checkout')
> > > > > + parser.add_argument('--push-remote', metavar='USERNAME',
> > > > > + help='Push the release to people.apache.org
> > > > > +for the given user')
> > > > > + parser.add_argument('--push-local', metavar='PATH',
> > > > > + help='Push the release to the local path')
> > > > > + parser.add_argument('--sign', metavar='KEYID',
> > > > > + help='Sign the release with the given gpg
> > > > > +key')
> > > > > + parser.add_argument('--rc-num', metavar='NUM', type=int,
> > default=1,
> > > > > + help='Release Candidate number, required')
> > > > > + parser.add_argument('--smoke-test', metavar='PATH',
> > > > > + help='Run the smoker tester on the release
> > > > > +in the given directory')
> > > > > + parser.add_argument('root', metavar='checkout_path',
> > > > > + help='Root of SVN checkout for
> > > > > +lucene-solr')
> > > > > + config = parser.parse_args()
> > > > > +
> > > > > + if config.push_remote is not None and config.push_local is not
> None:
> > > > > + parser.error('Cannot specify --push-remote and --push-local
> > > > > + together') if not config.prepare and config.sign:
> > > > > + parser.error('Cannot sign already built release') if
> > > > > + config.push_local is not None and os.path.exists(config.push_local):
> > > > > + parser.error('Cannot push to local path that already exists')
> > > > > + if config.rc_num <= 0:
> > > > > + parser.error('Release Candidate number must be a positive
> > > > > + integer') if not os.path.isdir(config.root):
> > > > > + # TODO: add additional svn check to ensure dir is a real
> > > > > + lucene-solr
> > > > > checkout
> > > > > + parser.error('Root path is not a valid lucene-solr checkout')
> > > > > + if config.smoke_test is not None and
> > > os.path.exists(config.smoke_test):
> > > > > + parser.error('Smoke test path already exists')
> > > > >
> > > > > - try:
> > > > > - idx = sys.argv.index('-smoke')
> > > > > - except ValueError:
> > > > > - smokeTmpDir = None
> > > > > - else:
> > > > > - smokeTmpDir = sys.argv[idx+1]
> > > > > - del sys.argv[idx:idx+2]
> > > > > - if os.path.exists(smokeTmpDir):
> > > > > - print()
> > > > > - print('ERROR: smoke tmpDir "%s" exists; please remove first' %
> > > > > smokeTmpDir)
> > > > > - print()
> > > > > - sys.exit(1)
> > > > > -
> > > > > - try:
> > > > > - idx = sys.argv.index('-pushLocal')
> > > > > - except ValueError:
> > > > > - doPushLocal = False
> > > > > - else:
> > > > > - doPushLocal = True
> > > > > - localStagingDir = sys.argv[idx+1]
> > > > > - del sys.argv[idx:idx+2]
> > > > > - if os.path.exists(localStagingDir):
> > > > > - print()
> > > > > - print('ERROR: pushLocal dir "%s" exists; please remove first' %
> > > > > localStagingDir)
> > > > > - print()
> > > > > - sys.exit(1)
> > > > > -
> > > > > - if doPushRemote and doPushLocal:
> > > > > - print()
> > > > > - print('ERROR: specify at most one of -push or -pushLocal (got
> both)')
> > > > > - print()
> > > > > - sys.exit(1)
> > > > > -
> > > > > - try:
> > > > > - idx = sys.argv.index('-sign')
> > > > > - except ValueError:
> > > > > - gpgKeyID = None
> > > > > - else:
> > > > > - gpgKeyID = sys.argv[idx+1]
> > > > > - del sys.argv[idx:idx+2]
> > > > > + config.version = read_version(config.root) print('Building version:
> > > > > + %s' % config.version)
> > > > >
> > > > > + if config.sign:
> > > > > sys.stdout.flush()
> > > > > import getpass
> > > > > - gpgPassword = getpass.getpass('Enter GPG keystore password: ')
> > > > > + config.key_id = config.sign
> > > > > + config.key_password = getpass.getpass('Enter GPG keystore
> > > password:
> > > > > + ')
> > > > > + else:
> > > > > + config.gpg_password = None
> > > > >
> > > > > - root = os.path.abspath(sys.argv[1])
> > > > > - version = sys.argv[2]
> > > > > - rcNum = int(sys.argv[3])
> > > > > + return config
> > > > > +
> > > > > +def main():
> > > > > + c = parse_config()
> > > > >
> > > > > - if doPrepare:
> > > > > - rev = prepare(root, version, gpgKeyID, gpgPassword,
> smokeTmpDir
> > is
> > > > > None)
> > > > > + if c.prepare:
> > > > > + rev = prepare(c.root, c.version, c.key_id, c.key_password,
> > > > > + key.smoke_test is None)
> > > > > else:
> > > > > os.chdir(root)
> > > > > rev = open('rev.txt', encoding='UTF-8').read()
> > > > >
> > > > > - if doPushRemote:
> > > > > + if c.push_remote:
> > > > > url = push(version, root, rev, rcNum, username)
> > > > > - elif doPushLocal:
> > > > > - url = pushLocal(version, root, rev, rcNum, localStagingDir)
> > > > > + elif c.push_local:
> > > > > + url = pushLocal(version, root, rev, c.rc_num, c.push_local)
> > > > > else:
> > > > > url = None
> > > > >
> > > > > if url is not None:
> > > > > print(' URL: %s' % url)
> > > > >
> > > > > - if smokeTmpDir is not None:
> > > > > + if c.smoke_test is not None:
> > > > > import smokeTestRelease
> > > > > smokeTestRelease.DEBUG = False
> > > > > - smokeTestRelease.smokeTest(url, rev, version, smokeTmpDir,
> > > > gpgKeyID
> > > > > is not None, '')
> > > > > + smokeTestRelease.smokeTest(url, rev, c.version, c.smoke_test,
> > > > > + c.sign is not None, '')
> > > > >
> > > > > if __name__ == '__main__':
> > > > > try:
> > > > > main()
> > > > > - except:
> > > > > - import traceback
> > > > > - traceback.print_exc()
> > > > > + except KeyboardInterrupt:
> > > > > + print('Keyboard interrupt...exiting')
> > > > > +
> > > > >
> > > > > Modified: lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > > > > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> > > > >
> > > >
> > >
> >
> tools/scripts/smokeTestRelease.py?rev=1621085&r1=1621084&r2=1621085&
> > > > > view=diff
> > > > >
> > > >
> > >
> >
> ==========================================================
> > > > > ====================
> > > > > --- lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > > > > (original)
> > > > > +++ lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py Thu
> > Aug
> > > > > +++ 28
> > > > > +++ 08:01:50 2014
> > > > > @@ -13,6 +13,7 @@
> > > > > # See the License for the specific language governing permissions
> > > > > and # limitations under the License.
> > > > >
> > > > > +import argparse
> > > > > import os
> > > > > import zipfile
> > > > > import codecs
> > > > > @@ -40,6 +41,7 @@ import checkJavaDocs import checkJavadocLinks
> > > > > import io import codecs
> > > > > +import textwrap
> > > > >
> > > > > # This tool expects to find /lucene and /solr off the base URL.
> > > > > You # must have a working gpg, tar, unzip in your path. This has
> > > > > been @@ -1248,55
> > > > > +1250,60 @@ def crawl(downloadedFiles, urlString, ta
> > > > > downloadedFiles.append(path)
> > > > > sys.stdout.write('.')
> > > > >
> > > > > -reAllowedVersion = re.compile(r'^\d+\.\d+\.\d+(-ALPHA|-BETA)?$')
> > > > > -
> > > > > -def main():
> > > > > -
> > > > > - if len(sys.argv) < 5:
> > > > > - print()
> > > > > - print('Usage python -u %s BaseURL SvnRevision version tmpDir [
> > > > > isSigned(True|False) ] [ -testArgs "-Dwhat=ever [ ... ]" ]'
> > > > > - % sys.argv[0])
> > > > > - print()
> > > > > - print(' example: python3.2 -u dev-
> > tools/scripts/smokeTestRelease.py
> > > > > http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-
> R
> > > > > C1
> > > > > -
> > > > > rev1469340 1469340 4.3.0 /path/to/a/tmp/dir')
> > > > > - print()
> > > > > - sys.exit(1)
> > > > > -
> > > > > - baseURL = sys.argv[1]
> > > > > - svnRevision = sys.argv[2]
> > > > > - version = sys.argv[3]
> > > > > +version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)')
> > > > > +revision_re = re.compile(r'rev(\d+)') def parse_config():
> > > > > + epilogue = textwrap.dedent('''
> > > > > + Example usage:
> > > > > + python3.2 -u dev-tools/scripts/smokeTestRelease.py
> > > > > +http://people.apache.org/~whoever/staging_area/lucene-solr-
> 4.3.0-
> > > > > +RC
> > > > > +1-
> > > > > re
> > > > > +v1469340')
> > > > > + ''')
> > > > > + description = 'Utility to test a release.'
> > > > > + parser = argparse.ArgumentParser(description=description,
> > > > > epilog=epilogue,
> > > > > +
> > > > > +formatter_class=argparse.RawDescriptionHelpFormatter)
> > > > > + parser.add_argument('--tmp-dir', metavar='PATH',
> > > > > + help='Temporary directory to test inside,
> > > > > +defaults to /tmp/smoke_lucene_$version_$revision')
> > > > > + parser.add_argument('--not-signed', dest='is_signed',
> > > > > action='store_false', default=True,
> > > > > + help='Indicates the release is not signed')
> > > > > + parser.add_argument('--revision',
> > > > > + help='SVN revision number that release was
> > > > > +built with, defaults to that in URL')
> > > > > + parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
> > > > > + help='Version of the release, defaults to
> > > > > +that in
> > > > > +URL')
> > > > > + parser.add_argument('url', help='Url pointing to release to
> > > > > +test')
> > > > > + parser.add_argument('test_args', nargs=argparse.REMAINDER,
> > > > > metavar='ARGS',
> > > > > + help='Arguments to pass to ant for testing, e.g.
> > > > > +-Dwhat=ever')
> > > > > + c = parser.parse_args()
> > > > > +
> > > > > + if c.version is not None:
> > > > > + if not version_re.match(c.version):
> > > > > + parser.error('version "%s" does not match format
> > > > > + X.Y.Z[-ALPHA|-BETA]' % c.version)
> > > > > + else:
> > > > > + version_match = version_re.search(c.url)
> > > > > + if version_match is None:
> > > > > + parser.error('Could not find version in URL')
> > > > > + c.version = version_match.group(1)
> > > > > +
> > > > > + if c.revision is None:
> > > > > + revision_match = revision_re.search(c.url)
> > > > > + if revision_match is None:
> > > > > + parser.error('Could not find revision in URL')
> > > > > + c.revision = revision_match.group(1)
> > > > >
> > > > > - if not reAllowedVersion.match(version):
> > > > > - raise RuntimeError('version "%s" does not match format X.Y.Z[-
> > > ALPHA|-
> > > > > BETA]' % version)
> > > > > -
> > > > > - tmpDirArgNum = 4
> > > > > - tmpDir = os.path.abspath(sys.argv[tmpDirArgNum])
> > > > > + if c.tmp_dir:
> > > > > + c.tmp_dir = os.path.abspath(c.tmp_dir)
> > > > > + else:
> > > > > + tmp = '/tmp/smoke_lucene_%s_%s' % (c.version, c.revision)
> > > > > + c.tmp_dir = tmp
> > > > > + i = 1
> > > > > + while os.path.exists(c.tmp_dir):
> > > > > + c.tmp_dir = tmp + '_%d' % i
> > > > > + i += 1
> > > > >
> > > > > - # TODO: yuck: positional-only args with more than one optional
> > > > > arg totally sucks
> > > > > - # TODO: consider naming all args
> > > > > - isSigned = True
> > > > > - testArgs = ''
> > > > > - lastArgNum = len(sys.argv) - 1
> > > > > - if lastArgNum > tmpDirArgNum:
> > > > > - nextArgNum = tmpDirArgNum + 1
> > > > > - if sys.argv[nextArgNum] == '-testArgs':
> > > > > - if nextArgNum == lastArgNum:
> > > > > - raise RuntimeError('missing expected argument to -testArgs')
> > > > > - else:
> > > > > - # TODO: should there be arg validation here? E.g. starts with '-
> D'?
> > > > > - testArgs = sys.argv[nextArgNum + 1]
> > > > > - nextArgNum += 2
> > > > > - if nextArgNum <= lastArgNum:
> > > > > - isSigned = (sys.argv[nextArgNum].lower() == "true")
> > > > > - nextArgNum += 1
> > > > > - if nextArgNum <= lastArgNum and testArgs == '':
> > > > > - if sys.argv[nextArgNum] == '-testArgs':
> > > > > - if nextArgNum == lastArgNum:
> > > > > - raise RuntimeError('missing expected argument to -testArgs')
> > > > > - else:
> > > > > - # TODO: should there be arg validation here? E.g. starts with '-
> D'?
> > > > > - testArgs = sys.argv[nextArgNum + 1]
> > > > > + return c
> > > > >
> > > > > - smokeTest(baseURL, svnRevision, version, tmpDir, isSigned,
> > > > > testArgs)
> > > > > +def main():
> > > > > + c = parse_config()
> > > > > + print('NOTE: output encoding is %s' % sys.stdout.encoding)
> > > > > + smokeTest(c.url, c.revision, c.version, c.tmp_dir, c.is_signed, '
> > > > > +'.join(c.test_args))
> > > > >
> > > > > def smokeTest(baseURL, svnRevision, version, tmpDir, isSigned,
> > > testArgs):
> > > > >
> > > > > @@ -1350,10 +1357,8 @@ def smokeTest(baseURL, svnRevision, vers
> > > > > print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() -
> > > > > startTime))
> > > > >
> > > > > if __name__ == '__main__':
> > > > > - print('NOTE: output encoding is %s' % sys.stdout.encoding)
> > > > > try:
> > > > > main()
> > > > > - except:
> > > > > - traceback.print_exc()
> > > > > - sys.exit(1)
> > > > > - sys.exit(0)
> > > > > + except KeyboardInterrupt:
> > > > > + print('Keyboard interrupt...exiting')
> > > > > +
> > > >
> > > >
> > > >
> > > > --------------------------------------------------------------------
> > > > - To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For
> > > > additional commands, e-mail: dev-help@lucene.apache.org
> >
> >
> > ---------------------------------------------------------------------
> > To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For additional
> > commands, e-mail: dev-help@lucene.apache.org
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts: buildAndPushRelease.py smokeTestRelease.py
Posted by Uwe Schindler <uw...@thetaphi.de>.
Worked on Jenkins, but we have another problem:
[mkdir] Created dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-4.x/lucene/build/fakeRelease
[copy] Copying 431 files to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-4.x/lucene/build/fakeRelease/lucene
[copy] Copying 239 files to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-4.x/lucene/build/fakeRelease/solr
[exec] JAVA7_HOME is /home/jenkins/tools/java/latest1.7
[exec] NOTE: output encoding is US-ASCII
[exec]
[exec] Load release URL "file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-4.x/lucene/build/fakeRelease/"...
[exec]
[exec] Test Lucene...
[exec] test basics...
[exec] get KEYS
[exec] 0.1 MB in 0.01 sec (14.5 MB/sec)
[exec] check changes HTML...
[exec] download lucene-4.11.0-src.tgz...
[exec] 27.7 MB in 0.04 sec (663.2 MB/sec)
[exec] verify md5/sha1 digests
[exec] download lucene-4.11.0.tgz...
[exec] 62.1 MB in 0.15 sec (416.8 MB/sec)
[exec] verify md5/sha1 digests
[exec] download lucene-4.11.0.zip...
[exec] 71.9 MB in 0.12 sec (619.8 MB/sec)
[exec] verify md5/sha1 digests
[exec] unpack lucene-4.11.0.tgz...
[exec] verify JAR metadata/identity/no javax.* or java.* classes...
[exec] test demo with 1.7...
[exec] got 5806 hits for query "lucene"
[exec] checkindex with 1.7...
[exec] check Lucene's javadoc JAR
[exec] unpack lucene-4.11.0.zip...
[exec] verify JAR metadata/identity/no javax.* or java.* classes...
[exec] test demo with 1.7...
[exec] got 5806 hits for query "lucene"
[exec] checkindex with 1.7...
[exec] check Lucene's javadoc JAR
[exec] unpack lucene-4.11.0-src.tgz...
[exec] Traceback (most recent call last):
[exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 1365, in <module>
[exec] main()
[exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 1310, in main
[exec] smokeTest(c.url, c.revision, c.version, c.tmp_dir, c.is_signed, ' '.join(c.test_args))
[exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 1348, in smokeTest
[exec] unpackAndVerify('lucene', tmpDir, 'lucene-%s-src.tgz' % version, svnRevision, version, testArgs, baseURL)
[exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 639, in unpackAndVerify
[exec] verifyUnpacked(project, artifact, unpackPath, svnRevision, version, testArgs, tmpDir, baseURL)
[exec] File "/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-4.x/dev-tools/scripts/smokeTestRelease.py", line 710, in verifyUnpacked
[exec] raise RuntimeError('%s: unexpected files/dirs in artifact %s: %s' % (project, artifact, l))
[exec] RuntimeError: lucene: unexpected files/dirs in artifact lucene-4.11.0-src.tgz: ['version.properties']
We should fix the smoke tester to allow this new file. Will look into it!
Uwe
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: Uwe Schindler [mailto:uwe@thetaphi.de]
> Sent: Thursday, August 28, 2014 11:32 AM
> To: dev@lucene.apache.org
> Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> buildAndPushRelease.py smokeTestRelease.py
>
> Hi,
>
> I did a quick test, works. I committed that patch with minor modifications:
>
> - changed arg line back to arg value for test args. The problem in the python
> script is the " ".join(testArgs), whoich breaks when test args are escaped. So
> it is better to pass the whole arg line as one parameter, escaped completely.
> The joining then is a no-op, because there is only one parameter.
> - removed useless svnversion.
>
> I hope this also works on Jenkins. I'll trigger a build.
> Uwe
>
> -----
> Uwe Schindler
> H.-H.-Meier-Allee 63, D-28213 Bremen
> http://www.thetaphi.de
> eMail: uwe@thetaphi.de
>
>
> > -----Original Message-----
> > From: Uwe Schindler [mailto:uwe@thetaphi.de]
> > Sent: Thursday, August 28, 2014 11:01 AM
> > To: dev@lucene.apache.org
> > Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> > buildAndPushRelease.py smokeTestRelease.py
> >
> > I think the attached patch should be correct:
> >
> > Index: build.xml
> >
> ==========================================================
> > =========
> > --- build.xml (revision 1621092)
> > +++ build.xml (working copy)
> > @@ -407,13 +407,15 @@
> > <!-- Tell Python not to write any bytecode cache into the filesystem: -->
> > <arg value="-B"/>
> > <arg file="dev-tools/scripts/smokeTestRelease.py"/>
> > - <arg value="${fakeRelease.uri}"/>
> > + <arg value="--revision"/>
> > <arg value="skip"/>
> > + <arg value="--version"/>
> > <arg value="${fakeReleaseVersion}"/>
> > + <arg value="--tmp-dir"/>
> > <arg file="${fakeReleaseTmp}"/>
> > - <arg value="false"/>
> > - <arg value="-testArgs"/>
> > - <arg value="${smokeTestRelease.testArgs}"/>
> > + <arg value="--not-signed"/>
> > + <arg value="${fakeRelease.uri}"/>
> > + <arg line="${smokeTestRelease.testArgs}"/>
> > <env key="JAVA7_HOME" file="${JAVA7_HOME}"/>
> > </exec>
> > <delete dir="${fakeRelease}"/>
> >
> > -----
> > Uwe Schindler
> > H.-H.-Meier-Allee 63, D-28213 Bremen
> > http://www.thetaphi.de
> > eMail: uwe@thetaphi.de
> >
> >
> > > -----Original Message-----
> > > From: Uwe Schindler [mailto:uwe@thetaphi.de]
> > > Sent: Thursday, August 28, 2014 10:39 AM
> > > To: dev@lucene.apache.org
> > > Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-
> tools/scripts:
> > > buildAndPushRelease.py smokeTestRelease.py
> > >
> > > Hi,
> > >
> > > did the parameter names change by that? If yes, you need to fix the
> > > build.xml in the root folder to pass right params in "nigtly-smoke".
> > >
> > > Uwe
> > >
> > > -----
> > > Uwe Schindler
> > > H.-H.-Meier-Allee 63, D-28213 Bremen http://www.thetaphi.de
> > > eMail: uwe@thetaphi.de
> > >
> > > > -----Original Message-----
> > > > From: rjernst@apache.org [mailto:rjernst@apache.org]
> > > > Sent: Thursday, August 28, 2014 10:02 AM
> > > > To: commits@lucene.apache.org
> > > > Subject: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> > > > buildAndPushRelease.py smokeTestRelease.py
> > > >
> > > > Author: rjernst
> > > > Date: Thu Aug 28 08:01:50 2014
> > > > New Revision: 1621085
> > > >
> > > > URL: http://svn.apache.org/r1621085
> > > > Log:
> > > > Change buildAndPushRelease and smokeTestRelease scripts to use
> > > > argparse
> > > >
> > > > Modified:
> > > > lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > > > lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > > >
> > > > Modified:
> > > > lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > > > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> > > >
> > >
> >
> tools/scripts/buildAndPushRelease.py?rev=1621085&r1=1621084&r2=162108
> > > > 5&view=diff
> > > >
> > >
> >
> ==========================================================
> > > > ====================
> > > > --- lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > > > (original)
> > > > +++ lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py Thu
> > > > +++ Aug
> > > > 28
> > > > +++ 08:01:50 2014
> > > > @@ -13,6 +13,7 @@
> > > > # See the License for the specific language governing permissions
> > > > and # limitations under the License.
> > > >
> > > > +import argparse
> > > > import datetime
> > > > import re
> > > > import time
> > > > @@ -20,16 +21,7 @@ import shutil
> > > > import os
> > > > import sys
> > > > import subprocess
> > > > -
> > > > -# Usage: python3.2 -u buildAndPushRelease.py [-sign gpgKey(eg:
> > > > 6E68DA61)] [-prepare] [-push userName] [-pushLocal dirName]
> > > > [-smoke tmpDir] /path/to/checkout version(eg: 3.4.0) rcNum(eg: 0) -# -
> # EG:
> > > > python3.2 -u buildAndPushRelease.py -prepare -push mikemccand
> > > > -sign
> > > > 6E68DA61 /path/to/lucene_solr_4_7 4.7.0 0
> > > > -
> > > > -# NOTE: if you specify -sign, you have to type in your gpg
> > > > password at -# some point while this runs; it's VERY confusing
> > > > because the output -# is directed to /tmp/release.log, so, you
> > > > have to tail that and when -# GPG wants your password, type it!
> > > > Also sometimes you have to type -# it twice in a row!
> > > > +import textwrap
> > > >
> > > > LOG = '/tmp/release.log'
> > > >
> > > > @@ -226,94 +218,89 @@ def pushLocal(version, root, rev, rcNum,
> > > >
> > > > print(' done!')
> > > > return 'file://%s/%s' % (os.path.abspath(localDir), dir)
> > > > -
> > > > -def main():
> > > > - doPrepare = '-prepare' in sys.argv
> > > > - if doPrepare:
> > > > - sys.argv.remove('-prepare')
> > > >
> > > > - try:
> > > > - idx = sys.argv.index('-push')
> > > > - except ValueError:
> > > > - doPushRemote = False
> > > > - else:
> > > > - doPushRemote = True
> > > > - username = sys.argv[idx+1]
> > > > - del sys.argv[idx:idx+2]
> > > > +def read_version(path):
> > > > + version_props_file = os.path.join(path, 'lucene',
> > > > +'version.properties')
> > > > + return re.search(r'version\.base=(.*)',
> > > > +open(version_props_file).read()).group(1)
> > > > +
> > > > +def parse_config():
> > > > + epilogue = textwrap.dedent('''
> > > > + Example usage for a Release Manager:
> > > > + python3.2 -u buildAndPushRelease.py --push-remote mikemccand
> > > > +--sign
> > > > +6E68DA61 --rc-num 1 --version 4.7.0 /path/to/lucene_solr_4_7
> > > > + ''')
> > > > + description = 'Utility to build, push, and test a release.'
> > > > + parser = argparse.ArgumentParser(description=description,
> > > > epilog=epilogue,
> > > > +
> > > > +formatter_class=argparse.RawDescriptionHelpFormatter)
> > > > + parser.add_argument('--no-prepare', dest='prepare',
> > > > +default=True,
> > > > action='store_false',
> > > > + help='Use the already built release in the
> > > > +provided checkout')
> > > > + parser.add_argument('--push-remote', metavar='USERNAME',
> > > > + help='Push the release to people.apache.org
> > > > +for the given user')
> > > > + parser.add_argument('--push-local', metavar='PATH',
> > > > + help='Push the release to the local path')
> > > > + parser.add_argument('--sign', metavar='KEYID',
> > > > + help='Sign the release with the given gpg
> > > > +key')
> > > > + parser.add_argument('--rc-num', metavar='NUM', type=int,
> default=1,
> > > > + help='Release Candidate number, required')
> > > > + parser.add_argument('--smoke-test', metavar='PATH',
> > > > + help='Run the smoker tester on the release
> > > > +in the given directory')
> > > > + parser.add_argument('root', metavar='checkout_path',
> > > > + help='Root of SVN checkout for
> > > > +lucene-solr')
> > > > + config = parser.parse_args()
> > > > +
> > > > + if config.push_remote is not None and config.push_local is not None:
> > > > + parser.error('Cannot specify --push-remote and --push-local
> > > > + together') if not config.prepare and config.sign:
> > > > + parser.error('Cannot sign already built release') if
> > > > + config.push_local is not None and os.path.exists(config.push_local):
> > > > + parser.error('Cannot push to local path that already exists')
> > > > + if config.rc_num <= 0:
> > > > + parser.error('Release Candidate number must be a positive
> > > > + integer') if not os.path.isdir(config.root):
> > > > + # TODO: add additional svn check to ensure dir is a real
> > > > + lucene-solr
> > > > checkout
> > > > + parser.error('Root path is not a valid lucene-solr checkout')
> > > > + if config.smoke_test is not None and
> > os.path.exists(config.smoke_test):
> > > > + parser.error('Smoke test path already exists')
> > > >
> > > > - try:
> > > > - idx = sys.argv.index('-smoke')
> > > > - except ValueError:
> > > > - smokeTmpDir = None
> > > > - else:
> > > > - smokeTmpDir = sys.argv[idx+1]
> > > > - del sys.argv[idx:idx+2]
> > > > - if os.path.exists(smokeTmpDir):
> > > > - print()
> > > > - print('ERROR: smoke tmpDir "%s" exists; please remove first' %
> > > > smokeTmpDir)
> > > > - print()
> > > > - sys.exit(1)
> > > > -
> > > > - try:
> > > > - idx = sys.argv.index('-pushLocal')
> > > > - except ValueError:
> > > > - doPushLocal = False
> > > > - else:
> > > > - doPushLocal = True
> > > > - localStagingDir = sys.argv[idx+1]
> > > > - del sys.argv[idx:idx+2]
> > > > - if os.path.exists(localStagingDir):
> > > > - print()
> > > > - print('ERROR: pushLocal dir "%s" exists; please remove first' %
> > > > localStagingDir)
> > > > - print()
> > > > - sys.exit(1)
> > > > -
> > > > - if doPushRemote and doPushLocal:
> > > > - print()
> > > > - print('ERROR: specify at most one of -push or -pushLocal (got both)')
> > > > - print()
> > > > - sys.exit(1)
> > > > -
> > > > - try:
> > > > - idx = sys.argv.index('-sign')
> > > > - except ValueError:
> > > > - gpgKeyID = None
> > > > - else:
> > > > - gpgKeyID = sys.argv[idx+1]
> > > > - del sys.argv[idx:idx+2]
> > > > + config.version = read_version(config.root) print('Building version:
> > > > + %s' % config.version)
> > > >
> > > > + if config.sign:
> > > > sys.stdout.flush()
> > > > import getpass
> > > > - gpgPassword = getpass.getpass('Enter GPG keystore password: ')
> > > > + config.key_id = config.sign
> > > > + config.key_password = getpass.getpass('Enter GPG keystore
> > password:
> > > > + ')
> > > > + else:
> > > > + config.gpg_password = None
> > > >
> > > > - root = os.path.abspath(sys.argv[1])
> > > > - version = sys.argv[2]
> > > > - rcNum = int(sys.argv[3])
> > > > + return config
> > > > +
> > > > +def main():
> > > > + c = parse_config()
> > > >
> > > > - if doPrepare:
> > > > - rev = prepare(root, version, gpgKeyID, gpgPassword, smokeTmpDir
> is
> > > > None)
> > > > + if c.prepare:
> > > > + rev = prepare(c.root, c.version, c.key_id, c.key_password,
> > > > + key.smoke_test is None)
> > > > else:
> > > > os.chdir(root)
> > > > rev = open('rev.txt', encoding='UTF-8').read()
> > > >
> > > > - if doPushRemote:
> > > > + if c.push_remote:
> > > > url = push(version, root, rev, rcNum, username)
> > > > - elif doPushLocal:
> > > > - url = pushLocal(version, root, rev, rcNum, localStagingDir)
> > > > + elif c.push_local:
> > > > + url = pushLocal(version, root, rev, c.rc_num, c.push_local)
> > > > else:
> > > > url = None
> > > >
> > > > if url is not None:
> > > > print(' URL: %s' % url)
> > > >
> > > > - if smokeTmpDir is not None:
> > > > + if c.smoke_test is not None:
> > > > import smokeTestRelease
> > > > smokeTestRelease.DEBUG = False
> > > > - smokeTestRelease.smokeTest(url, rev, version, smokeTmpDir,
> > > gpgKeyID
> > > > is not None, '')
> > > > + smokeTestRelease.smokeTest(url, rev, c.version, c.smoke_test,
> > > > + c.sign is not None, '')
> > > >
> > > > if __name__ == '__main__':
> > > > try:
> > > > main()
> > > > - except:
> > > > - import traceback
> > > > - traceback.print_exc()
> > > > + except KeyboardInterrupt:
> > > > + print('Keyboard interrupt...exiting')
> > > > +
> > > >
> > > > Modified: lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > > > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> > > >
> > >
> >
> tools/scripts/smokeTestRelease.py?rev=1621085&r1=1621084&r2=1621085&
> > > > view=diff
> > > >
> > >
> >
> ==========================================================
> > > > ====================
> > > > --- lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > > > (original)
> > > > +++ lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py Thu
> Aug
> > > > +++ 28
> > > > +++ 08:01:50 2014
> > > > @@ -13,6 +13,7 @@
> > > > # See the License for the specific language governing permissions
> > > > and # limitations under the License.
> > > >
> > > > +import argparse
> > > > import os
> > > > import zipfile
> > > > import codecs
> > > > @@ -40,6 +41,7 @@ import checkJavaDocs import checkJavadocLinks
> > > > import io import codecs
> > > > +import textwrap
> > > >
> > > > # This tool expects to find /lucene and /solr off the base URL.
> > > > You # must have a working gpg, tar, unzip in your path. This has
> > > > been @@ -1248,55
> > > > +1250,60 @@ def crawl(downloadedFiles, urlString, ta
> > > > downloadedFiles.append(path)
> > > > sys.stdout.write('.')
> > > >
> > > > -reAllowedVersion = re.compile(r'^\d+\.\d+\.\d+(-ALPHA|-BETA)?$')
> > > > -
> > > > -def main():
> > > > -
> > > > - if len(sys.argv) < 5:
> > > > - print()
> > > > - print('Usage python -u %s BaseURL SvnRevision version tmpDir [
> > > > isSigned(True|False) ] [ -testArgs "-Dwhat=ever [ ... ]" ]'
> > > > - % sys.argv[0])
> > > > - print()
> > > > - print(' example: python3.2 -u dev-
> tools/scripts/smokeTestRelease.py
> > > > http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-R
> > > > C1
> > > > -
> > > > rev1469340 1469340 4.3.0 /path/to/a/tmp/dir')
> > > > - print()
> > > > - sys.exit(1)
> > > > -
> > > > - baseURL = sys.argv[1]
> > > > - svnRevision = sys.argv[2]
> > > > - version = sys.argv[3]
> > > > +version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)')
> > > > +revision_re = re.compile(r'rev(\d+)') def parse_config():
> > > > + epilogue = textwrap.dedent('''
> > > > + Example usage:
> > > > + python3.2 -u dev-tools/scripts/smokeTestRelease.py
> > > > +http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-
> > > > +RC
> > > > +1-
> > > > re
> > > > +v1469340')
> > > > + ''')
> > > > + description = 'Utility to test a release.'
> > > > + parser = argparse.ArgumentParser(description=description,
> > > > epilog=epilogue,
> > > > +
> > > > +formatter_class=argparse.RawDescriptionHelpFormatter)
> > > > + parser.add_argument('--tmp-dir', metavar='PATH',
> > > > + help='Temporary directory to test inside,
> > > > +defaults to /tmp/smoke_lucene_$version_$revision')
> > > > + parser.add_argument('--not-signed', dest='is_signed',
> > > > action='store_false', default=True,
> > > > + help='Indicates the release is not signed')
> > > > + parser.add_argument('--revision',
> > > > + help='SVN revision number that release was
> > > > +built with, defaults to that in URL')
> > > > + parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
> > > > + help='Version of the release, defaults to
> > > > +that in
> > > > +URL')
> > > > + parser.add_argument('url', help='Url pointing to release to
> > > > +test')
> > > > + parser.add_argument('test_args', nargs=argparse.REMAINDER,
> > > > metavar='ARGS',
> > > > + help='Arguments to pass to ant for testing, e.g.
> > > > +-Dwhat=ever')
> > > > + c = parser.parse_args()
> > > > +
> > > > + if c.version is not None:
> > > > + if not version_re.match(c.version):
> > > > + parser.error('version "%s" does not match format
> > > > + X.Y.Z[-ALPHA|-BETA]' % c.version)
> > > > + else:
> > > > + version_match = version_re.search(c.url)
> > > > + if version_match is None:
> > > > + parser.error('Could not find version in URL')
> > > > + c.version = version_match.group(1)
> > > > +
> > > > + if c.revision is None:
> > > > + revision_match = revision_re.search(c.url)
> > > > + if revision_match is None:
> > > > + parser.error('Could not find revision in URL')
> > > > + c.revision = revision_match.group(1)
> > > >
> > > > - if not reAllowedVersion.match(version):
> > > > - raise RuntimeError('version "%s" does not match format X.Y.Z[-
> > ALPHA|-
> > > > BETA]' % version)
> > > > -
> > > > - tmpDirArgNum = 4
> > > > - tmpDir = os.path.abspath(sys.argv[tmpDirArgNum])
> > > > + if c.tmp_dir:
> > > > + c.tmp_dir = os.path.abspath(c.tmp_dir)
> > > > + else:
> > > > + tmp = '/tmp/smoke_lucene_%s_%s' % (c.version, c.revision)
> > > > + c.tmp_dir = tmp
> > > > + i = 1
> > > > + while os.path.exists(c.tmp_dir):
> > > > + c.tmp_dir = tmp + '_%d' % i
> > > > + i += 1
> > > >
> > > > - # TODO: yuck: positional-only args with more than one optional
> > > > arg totally sucks
> > > > - # TODO: consider naming all args
> > > > - isSigned = True
> > > > - testArgs = ''
> > > > - lastArgNum = len(sys.argv) - 1
> > > > - if lastArgNum > tmpDirArgNum:
> > > > - nextArgNum = tmpDirArgNum + 1
> > > > - if sys.argv[nextArgNum] == '-testArgs':
> > > > - if nextArgNum == lastArgNum:
> > > > - raise RuntimeError('missing expected argument to -testArgs')
> > > > - else:
> > > > - # TODO: should there be arg validation here? E.g. starts with '-D'?
> > > > - testArgs = sys.argv[nextArgNum + 1]
> > > > - nextArgNum += 2
> > > > - if nextArgNum <= lastArgNum:
> > > > - isSigned = (sys.argv[nextArgNum].lower() == "true")
> > > > - nextArgNum += 1
> > > > - if nextArgNum <= lastArgNum and testArgs == '':
> > > > - if sys.argv[nextArgNum] == '-testArgs':
> > > > - if nextArgNum == lastArgNum:
> > > > - raise RuntimeError('missing expected argument to -testArgs')
> > > > - else:
> > > > - # TODO: should there be arg validation here? E.g. starts with '-D'?
> > > > - testArgs = sys.argv[nextArgNum + 1]
> > > > + return c
> > > >
> > > > - smokeTest(baseURL, svnRevision, version, tmpDir, isSigned,
> > > > testArgs)
> > > > +def main():
> > > > + c = parse_config()
> > > > + print('NOTE: output encoding is %s' % sys.stdout.encoding)
> > > > + smokeTest(c.url, c.revision, c.version, c.tmp_dir, c.is_signed, '
> > > > +'.join(c.test_args))
> > > >
> > > > def smokeTest(baseURL, svnRevision, version, tmpDir, isSigned,
> > testArgs):
> > > >
> > > > @@ -1350,10 +1357,8 @@ def smokeTest(baseURL, svnRevision, vers
> > > > print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() -
> > > > startTime))
> > > >
> > > > if __name__ == '__main__':
> > > > - print('NOTE: output encoding is %s' % sys.stdout.encoding)
> > > > try:
> > > > main()
> > > > - except:
> > > > - traceback.print_exc()
> > > > - sys.exit(1)
> > > > - sys.exit(0)
> > > > + except KeyboardInterrupt:
> > > > + print('Keyboard interrupt...exiting')
> > > > +
> > >
> > >
> > >
> > > --------------------------------------------------------------------
> > > - To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For
> > > additional commands, e-mail: dev-help@lucene.apache.org
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For additional
> commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts: buildAndPushRelease.py smokeTestRelease.py
Posted by Uwe Schindler <uw...@thetaphi.de>.
Hi,
I did a quick test, works. I committed that patch with minor modifications:
- changed arg line back to arg value for test args. The problem in the python script is the " ".join(testArgs), whoich breaks when test args are escaped. So it is better to pass the whole arg line as one parameter, escaped completely. The joining then is a no-op, because there is only one parameter.
- removed useless svnversion.
I hope this also works on Jenkins. I'll trigger a build.
Uwe
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: Uwe Schindler [mailto:uwe@thetaphi.de]
> Sent: Thursday, August 28, 2014 11:01 AM
> To: dev@lucene.apache.org
> Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> buildAndPushRelease.py smokeTestRelease.py
>
> I think the attached patch should be correct:
>
> Index: build.xml
> ==========================================================
> =========
> --- build.xml (revision 1621092)
> +++ build.xml (working copy)
> @@ -407,13 +407,15 @@
> <!-- Tell Python not to write any bytecode cache into the filesystem: -->
> <arg value="-B"/>
> <arg file="dev-tools/scripts/smokeTestRelease.py"/>
> - <arg value="${fakeRelease.uri}"/>
> + <arg value="--revision"/>
> <arg value="skip"/>
> + <arg value="--version"/>
> <arg value="${fakeReleaseVersion}"/>
> + <arg value="--tmp-dir"/>
> <arg file="${fakeReleaseTmp}"/>
> - <arg value="false"/>
> - <arg value="-testArgs"/>
> - <arg value="${smokeTestRelease.testArgs}"/>
> + <arg value="--not-signed"/>
> + <arg value="${fakeRelease.uri}"/>
> + <arg line="${smokeTestRelease.testArgs}"/>
> <env key="JAVA7_HOME" file="${JAVA7_HOME}"/>
> </exec>
> <delete dir="${fakeRelease}"/>
>
> -----
> Uwe Schindler
> H.-H.-Meier-Allee 63, D-28213 Bremen
> http://www.thetaphi.de
> eMail: uwe@thetaphi.de
>
>
> > -----Original Message-----
> > From: Uwe Schindler [mailto:uwe@thetaphi.de]
> > Sent: Thursday, August 28, 2014 10:39 AM
> > To: dev@lucene.apache.org
> > Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> > buildAndPushRelease.py smokeTestRelease.py
> >
> > Hi,
> >
> > did the parameter names change by that? If yes, you need to fix the
> > build.xml in the root folder to pass right params in "nigtly-smoke".
> >
> > Uwe
> >
> > -----
> > Uwe Schindler
> > H.-H.-Meier-Allee 63, D-28213 Bremen
> > http://www.thetaphi.de
> > eMail: uwe@thetaphi.de
> >
> > > -----Original Message-----
> > > From: rjernst@apache.org [mailto:rjernst@apache.org]
> > > Sent: Thursday, August 28, 2014 10:02 AM
> > > To: commits@lucene.apache.org
> > > Subject: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> > > buildAndPushRelease.py smokeTestRelease.py
> > >
> > > Author: rjernst
> > > Date: Thu Aug 28 08:01:50 2014
> > > New Revision: 1621085
> > >
> > > URL: http://svn.apache.org/r1621085
> > > Log:
> > > Change buildAndPushRelease and smokeTestRelease scripts to use
> > > argparse
> > >
> > > Modified:
> > > lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > > lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > >
> > > Modified: lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> > >
> >
> tools/scripts/buildAndPushRelease.py?rev=1621085&r1=1621084&r2=162108
> > > 5&view=diff
> > >
> >
> ==========================================================
> > > ====================
> > > --- lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > > (original)
> > > +++ lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py Thu
> > > +++ Aug
> > > 28
> > > +++ 08:01:50 2014
> > > @@ -13,6 +13,7 @@
> > > # See the License for the specific language governing permissions
> > > and # limitations under the License.
> > >
> > > +import argparse
> > > import datetime
> > > import re
> > > import time
> > > @@ -20,16 +21,7 @@ import shutil
> > > import os
> > > import sys
> > > import subprocess
> > > -
> > > -# Usage: python3.2 -u buildAndPushRelease.py [-sign gpgKey(eg:
> > > 6E68DA61)] [-prepare] [-push userName] [-pushLocal dirName] [-smoke
> > > tmpDir] /path/to/checkout version(eg: 3.4.0) rcNum(eg: 0) -# -# EG:
> > > python3.2 -u buildAndPushRelease.py -prepare -push mikemccand -sign
> > > 6E68DA61 /path/to/lucene_solr_4_7 4.7.0 0
> > > -
> > > -# NOTE: if you specify -sign, you have to type in your gpg password
> > > at -# some point while this runs; it's VERY confusing because the
> > > output -# is directed to /tmp/release.log, so, you have to tail that
> > > and when -# GPG wants your password, type it! Also sometimes you
> > > have to type -# it twice in a row!
> > > +import textwrap
> > >
> > > LOG = '/tmp/release.log'
> > >
> > > @@ -226,94 +218,89 @@ def pushLocal(version, root, rev, rcNum,
> > >
> > > print(' done!')
> > > return 'file://%s/%s' % (os.path.abspath(localDir), dir)
> > > -
> > > -def main():
> > > - doPrepare = '-prepare' in sys.argv
> > > - if doPrepare:
> > > - sys.argv.remove('-prepare')
> > >
> > > - try:
> > > - idx = sys.argv.index('-push')
> > > - except ValueError:
> > > - doPushRemote = False
> > > - else:
> > > - doPushRemote = True
> > > - username = sys.argv[idx+1]
> > > - del sys.argv[idx:idx+2]
> > > +def read_version(path):
> > > + version_props_file = os.path.join(path, 'lucene',
> > > +'version.properties')
> > > + return re.search(r'version\.base=(.*)',
> > > +open(version_props_file).read()).group(1)
> > > +
> > > +def parse_config():
> > > + epilogue = textwrap.dedent('''
> > > + Example usage for a Release Manager:
> > > + python3.2 -u buildAndPushRelease.py --push-remote mikemccand
> > > +--sign
> > > +6E68DA61 --rc-num 1 --version 4.7.0 /path/to/lucene_solr_4_7
> > > + ''')
> > > + description = 'Utility to build, push, and test a release.'
> > > + parser = argparse.ArgumentParser(description=description,
> > > epilog=epilogue,
> > > +
> > > +formatter_class=argparse.RawDescriptionHelpFormatter)
> > > + parser.add_argument('--no-prepare', dest='prepare', default=True,
> > > action='store_false',
> > > + help='Use the already built release in the
> > > +provided checkout')
> > > + parser.add_argument('--push-remote', metavar='USERNAME',
> > > + help='Push the release to people.apache.org
> > > +for the given user')
> > > + parser.add_argument('--push-local', metavar='PATH',
> > > + help='Push the release to the local path')
> > > + parser.add_argument('--sign', metavar='KEYID',
> > > + help='Sign the release with the given gpg
> > > +key')
> > > + parser.add_argument('--rc-num', metavar='NUM', type=int, default=1,
> > > + help='Release Candidate number, required')
> > > + parser.add_argument('--smoke-test', metavar='PATH',
> > > + help='Run the smoker tester on the release in
> > > +the given directory')
> > > + parser.add_argument('root', metavar='checkout_path',
> > > + help='Root of SVN checkout for lucene-solr')
> > > + config = parser.parse_args()
> > > +
> > > + if config.push_remote is not None and config.push_local is not None:
> > > + parser.error('Cannot specify --push-remote and --push-local
> > > + together') if not config.prepare and config.sign:
> > > + parser.error('Cannot sign already built release') if
> > > + config.push_local is not None and os.path.exists(config.push_local):
> > > + parser.error('Cannot push to local path that already exists')
> > > + if config.rc_num <= 0:
> > > + parser.error('Release Candidate number must be a positive
> > > + integer') if not os.path.isdir(config.root):
> > > + # TODO: add additional svn check to ensure dir is a real
> > > + lucene-solr
> > > checkout
> > > + parser.error('Root path is not a valid lucene-solr checkout')
> > > + if config.smoke_test is not None and
> os.path.exists(config.smoke_test):
> > > + parser.error('Smoke test path already exists')
> > >
> > > - try:
> > > - idx = sys.argv.index('-smoke')
> > > - except ValueError:
> > > - smokeTmpDir = None
> > > - else:
> > > - smokeTmpDir = sys.argv[idx+1]
> > > - del sys.argv[idx:idx+2]
> > > - if os.path.exists(smokeTmpDir):
> > > - print()
> > > - print('ERROR: smoke tmpDir "%s" exists; please remove first' %
> > > smokeTmpDir)
> > > - print()
> > > - sys.exit(1)
> > > -
> > > - try:
> > > - idx = sys.argv.index('-pushLocal')
> > > - except ValueError:
> > > - doPushLocal = False
> > > - else:
> > > - doPushLocal = True
> > > - localStagingDir = sys.argv[idx+1]
> > > - del sys.argv[idx:idx+2]
> > > - if os.path.exists(localStagingDir):
> > > - print()
> > > - print('ERROR: pushLocal dir "%s" exists; please remove first' %
> > > localStagingDir)
> > > - print()
> > > - sys.exit(1)
> > > -
> > > - if doPushRemote and doPushLocal:
> > > - print()
> > > - print('ERROR: specify at most one of -push or -pushLocal (got both)')
> > > - print()
> > > - sys.exit(1)
> > > -
> > > - try:
> > > - idx = sys.argv.index('-sign')
> > > - except ValueError:
> > > - gpgKeyID = None
> > > - else:
> > > - gpgKeyID = sys.argv[idx+1]
> > > - del sys.argv[idx:idx+2]
> > > + config.version = read_version(config.root) print('Building version:
> > > + %s' % config.version)
> > >
> > > + if config.sign:
> > > sys.stdout.flush()
> > > import getpass
> > > - gpgPassword = getpass.getpass('Enter GPG keystore password: ')
> > > + config.key_id = config.sign
> > > + config.key_password = getpass.getpass('Enter GPG keystore
> password:
> > > + ')
> > > + else:
> > > + config.gpg_password = None
> > >
> > > - root = os.path.abspath(sys.argv[1])
> > > - version = sys.argv[2]
> > > - rcNum = int(sys.argv[3])
> > > + return config
> > > +
> > > +def main():
> > > + c = parse_config()
> > >
> > > - if doPrepare:
> > > - rev = prepare(root, version, gpgKeyID, gpgPassword, smokeTmpDir is
> > > None)
> > > + if c.prepare:
> > > + rev = prepare(c.root, c.version, c.key_id, c.key_password,
> > > + key.smoke_test is None)
> > > else:
> > > os.chdir(root)
> > > rev = open('rev.txt', encoding='UTF-8').read()
> > >
> > > - if doPushRemote:
> > > + if c.push_remote:
> > > url = push(version, root, rev, rcNum, username)
> > > - elif doPushLocal:
> > > - url = pushLocal(version, root, rev, rcNum, localStagingDir)
> > > + elif c.push_local:
> > > + url = pushLocal(version, root, rev, c.rc_num, c.push_local)
> > > else:
> > > url = None
> > >
> > > if url is not None:
> > > print(' URL: %s' % url)
> > >
> > > - if smokeTmpDir is not None:
> > > + if c.smoke_test is not None:
> > > import smokeTestRelease
> > > smokeTestRelease.DEBUG = False
> > > - smokeTestRelease.smokeTest(url, rev, version, smokeTmpDir,
> > gpgKeyID
> > > is not None, '')
> > > + smokeTestRelease.smokeTest(url, rev, c.version, c.smoke_test,
> > > + c.sign is not None, '')
> > >
> > > if __name__ == '__main__':
> > > try:
> > > main()
> > > - except:
> > > - import traceback
> > > - traceback.print_exc()
> > > + except KeyboardInterrupt:
> > > + print('Keyboard interrupt...exiting')
> > > +
> > >
> > > Modified: lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> > >
> >
> tools/scripts/smokeTestRelease.py?rev=1621085&r1=1621084&r2=1621085&
> > > view=diff
> > >
> >
> ==========================================================
> > > ====================
> > > --- lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > > (original)
> > > +++ lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py Thu Aug
> > > +++ 28
> > > +++ 08:01:50 2014
> > > @@ -13,6 +13,7 @@
> > > # See the License for the specific language governing permissions
> > > and # limitations under the License.
> > >
> > > +import argparse
> > > import os
> > > import zipfile
> > > import codecs
> > > @@ -40,6 +41,7 @@ import checkJavaDocs import checkJavadocLinks
> > > import io import codecs
> > > +import textwrap
> > >
> > > # This tool expects to find /lucene and /solr off the base URL.
> > > You # must have a working gpg, tar, unzip in your path. This has
> > > been @@ -1248,55
> > > +1250,60 @@ def crawl(downloadedFiles, urlString, ta
> > > downloadedFiles.append(path)
> > > sys.stdout.write('.')
> > >
> > > -reAllowedVersion = re.compile(r'^\d+\.\d+\.\d+(-ALPHA|-BETA)?$')
> > > -
> > > -def main():
> > > -
> > > - if len(sys.argv) < 5:
> > > - print()
> > > - print('Usage python -u %s BaseURL SvnRevision version tmpDir [
> > > isSigned(True|False) ] [ -testArgs "-Dwhat=ever [ ... ]" ]'
> > > - % sys.argv[0])
> > > - print()
> > > - print(' example: python3.2 -u dev-tools/scripts/smokeTestRelease.py
> > > http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-RC1
> > > -
> > > rev1469340 1469340 4.3.0 /path/to/a/tmp/dir')
> > > - print()
> > > - sys.exit(1)
> > > -
> > > - baseURL = sys.argv[1]
> > > - svnRevision = sys.argv[2]
> > > - version = sys.argv[3]
> > > +version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)')
> > > +revision_re = re.compile(r'rev(\d+)') def parse_config():
> > > + epilogue = textwrap.dedent('''
> > > + Example usage:
> > > + python3.2 -u dev-tools/scripts/smokeTestRelease.py
> > > +http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-RC
> > > +1-
> > > re
> > > +v1469340')
> > > + ''')
> > > + description = 'Utility to test a release.'
> > > + parser = argparse.ArgumentParser(description=description,
> > > epilog=epilogue,
> > > +
> > > +formatter_class=argparse.RawDescriptionHelpFormatter)
> > > + parser.add_argument('--tmp-dir', metavar='PATH',
> > > + help='Temporary directory to test inside,
> > > +defaults to /tmp/smoke_lucene_$version_$revision')
> > > + parser.add_argument('--not-signed', dest='is_signed',
> > > action='store_false', default=True,
> > > + help='Indicates the release is not signed')
> > > + parser.add_argument('--revision',
> > > + help='SVN revision number that release was
> > > +built with, defaults to that in URL')
> > > + parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
> > > + help='Version of the release, defaults to
> > > +that in
> > > +URL')
> > > + parser.add_argument('url', help='Url pointing to release to
> > > +test')
> > > + parser.add_argument('test_args', nargs=argparse.REMAINDER,
> > > metavar='ARGS',
> > > + help='Arguments to pass to ant for testing, e.g.
> > > +-Dwhat=ever')
> > > + c = parser.parse_args()
> > > +
> > > + if c.version is not None:
> > > + if not version_re.match(c.version):
> > > + parser.error('version "%s" does not match format
> > > + X.Y.Z[-ALPHA|-BETA]' % c.version)
> > > + else:
> > > + version_match = version_re.search(c.url)
> > > + if version_match is None:
> > > + parser.error('Could not find version in URL')
> > > + c.version = version_match.group(1)
> > > +
> > > + if c.revision is None:
> > > + revision_match = revision_re.search(c.url)
> > > + if revision_match is None:
> > > + parser.error('Could not find revision in URL')
> > > + c.revision = revision_match.group(1)
> > >
> > > - if not reAllowedVersion.match(version):
> > > - raise RuntimeError('version "%s" does not match format X.Y.Z[-
> ALPHA|-
> > > BETA]' % version)
> > > -
> > > - tmpDirArgNum = 4
> > > - tmpDir = os.path.abspath(sys.argv[tmpDirArgNum])
> > > + if c.tmp_dir:
> > > + c.tmp_dir = os.path.abspath(c.tmp_dir)
> > > + else:
> > > + tmp = '/tmp/smoke_lucene_%s_%s' % (c.version, c.revision)
> > > + c.tmp_dir = tmp
> > > + i = 1
> > > + while os.path.exists(c.tmp_dir):
> > > + c.tmp_dir = tmp + '_%d' % i
> > > + i += 1
> > >
> > > - # TODO: yuck: positional-only args with more than one optional
> > > arg totally sucks
> > > - # TODO: consider naming all args
> > > - isSigned = True
> > > - testArgs = ''
> > > - lastArgNum = len(sys.argv) - 1
> > > - if lastArgNum > tmpDirArgNum:
> > > - nextArgNum = tmpDirArgNum + 1
> > > - if sys.argv[nextArgNum] == '-testArgs':
> > > - if nextArgNum == lastArgNum:
> > > - raise RuntimeError('missing expected argument to -testArgs')
> > > - else:
> > > - # TODO: should there be arg validation here? E.g. starts with '-D'?
> > > - testArgs = sys.argv[nextArgNum + 1]
> > > - nextArgNum += 2
> > > - if nextArgNum <= lastArgNum:
> > > - isSigned = (sys.argv[nextArgNum].lower() == "true")
> > > - nextArgNum += 1
> > > - if nextArgNum <= lastArgNum and testArgs == '':
> > > - if sys.argv[nextArgNum] == '-testArgs':
> > > - if nextArgNum == lastArgNum:
> > > - raise RuntimeError('missing expected argument to -testArgs')
> > > - else:
> > > - # TODO: should there be arg validation here? E.g. starts with '-D'?
> > > - testArgs = sys.argv[nextArgNum + 1]
> > > + return c
> > >
> > > - smokeTest(baseURL, svnRevision, version, tmpDir, isSigned,
> > > testArgs)
> > > +def main():
> > > + c = parse_config()
> > > + print('NOTE: output encoding is %s' % sys.stdout.encoding)
> > > + smokeTest(c.url, c.revision, c.version, c.tmp_dir, c.is_signed, '
> > > +'.join(c.test_args))
> > >
> > > def smokeTest(baseURL, svnRevision, version, tmpDir, isSigned,
> testArgs):
> > >
> > > @@ -1350,10 +1357,8 @@ def smokeTest(baseURL, svnRevision, vers
> > > print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() -
> > > startTime))
> > >
> > > if __name__ == '__main__':
> > > - print('NOTE: output encoding is %s' % sys.stdout.encoding)
> > > try:
> > > main()
> > > - except:
> > > - traceback.print_exc()
> > > - sys.exit(1)
> > > - sys.exit(0)
> > > + except KeyboardInterrupt:
> > > + print('Keyboard interrupt...exiting')
> > > +
> >
> >
> >
> > ---------------------------------------------------------------------
> > To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For
> > additional commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts: buildAndPushRelease.py smokeTestRelease.py
Posted by Uwe Schindler <uw...@thetaphi.de>.
I think the attached patch should be correct:
Index: build.xml
===================================================================
--- build.xml (revision 1621092)
+++ build.xml (working copy)
@@ -407,13 +407,15 @@
<!-- Tell Python not to write any bytecode cache into the filesystem: -->
<arg value="-B"/>
<arg file="dev-tools/scripts/smokeTestRelease.py"/>
- <arg value="${fakeRelease.uri}"/>
+ <arg value="--revision"/>
<arg value="skip"/>
+ <arg value="--version"/>
<arg value="${fakeReleaseVersion}"/>
+ <arg value="--tmp-dir"/>
<arg file="${fakeReleaseTmp}"/>
- <arg value="false"/>
- <arg value="-testArgs"/>
- <arg value="${smokeTestRelease.testArgs}"/>
+ <arg value="--not-signed"/>
+ <arg value="${fakeRelease.uri}"/>
+ <arg line="${smokeTestRelease.testArgs}"/>
<env key="JAVA7_HOME" file="${JAVA7_HOME}"/>
</exec>
<delete dir="${fakeRelease}"/>
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: Uwe Schindler [mailto:uwe@thetaphi.de]
> Sent: Thursday, August 28, 2014 10:39 AM
> To: dev@lucene.apache.org
> Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> buildAndPushRelease.py smokeTestRelease.py
>
> Hi,
>
> did the parameter names change by that? If yes, you need to fix the
> build.xml in the root folder to pass right params in "nigtly-smoke".
>
> Uwe
>
> -----
> Uwe Schindler
> H.-H.-Meier-Allee 63, D-28213 Bremen
> http://www.thetaphi.de
> eMail: uwe@thetaphi.de
>
> > -----Original Message-----
> > From: rjernst@apache.org [mailto:rjernst@apache.org]
> > Sent: Thursday, August 28, 2014 10:02 AM
> > To: commits@lucene.apache.org
> > Subject: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts:
> > buildAndPushRelease.py smokeTestRelease.py
> >
> > Author: rjernst
> > Date: Thu Aug 28 08:01:50 2014
> > New Revision: 1621085
> >
> > URL: http://svn.apache.org/r1621085
> > Log:
> > Change buildAndPushRelease and smokeTestRelease scripts to use
> > argparse
> >
> > Modified:
> > lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> >
> > Modified: lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> >
> tools/scripts/buildAndPushRelease.py?rev=1621085&r1=1621084&r2=162108
> > 5&view=diff
> >
> ==========================================================
> > ====================
> > --- lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py
> > (original)
> > +++ lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py Thu Aug
> > 28
> > +++ 08:01:50 2014
> > @@ -13,6 +13,7 @@
> > # See the License for the specific language governing permissions and
> > # limitations under the License.
> >
> > +import argparse
> > import datetime
> > import re
> > import time
> > @@ -20,16 +21,7 @@ import shutil
> > import os
> > import sys
> > import subprocess
> > -
> > -# Usage: python3.2 -u buildAndPushRelease.py [-sign gpgKey(eg:
> > 6E68DA61)] [-prepare] [-push userName] [-pushLocal dirName] [-smoke
> > tmpDir] /path/to/checkout version(eg: 3.4.0) rcNum(eg: 0) -# -# EG:
> > python3.2 -u buildAndPushRelease.py -prepare -push mikemccand -sign
> > 6E68DA61 /path/to/lucene_solr_4_7 4.7.0 0
> > -
> > -# NOTE: if you specify -sign, you have to type in your gpg password
> > at -# some point while this runs; it's VERY confusing because the
> > output -# is directed to /tmp/release.log, so, you have to tail that
> > and when -# GPG wants your password, type it! Also sometimes you have
> > to type -# it twice in a row!
> > +import textwrap
> >
> > LOG = '/tmp/release.log'
> >
> > @@ -226,94 +218,89 @@ def pushLocal(version, root, rev, rcNum,
> >
> > print(' done!')
> > return 'file://%s/%s' % (os.path.abspath(localDir), dir)
> > -
> > -def main():
> > - doPrepare = '-prepare' in sys.argv
> > - if doPrepare:
> > - sys.argv.remove('-prepare')
> >
> > - try:
> > - idx = sys.argv.index('-push')
> > - except ValueError:
> > - doPushRemote = False
> > - else:
> > - doPushRemote = True
> > - username = sys.argv[idx+1]
> > - del sys.argv[idx:idx+2]
> > +def read_version(path):
> > + version_props_file = os.path.join(path, 'lucene',
> > +'version.properties')
> > + return re.search(r'version\.base=(.*)',
> > +open(version_props_file).read()).group(1)
> > +
> > +def parse_config():
> > + epilogue = textwrap.dedent('''
> > + Example usage for a Release Manager:
> > + python3.2 -u buildAndPushRelease.py --push-remote mikemccand
> > +--sign
> > +6E68DA61 --rc-num 1 --version 4.7.0 /path/to/lucene_solr_4_7
> > + ''')
> > + description = 'Utility to build, push, and test a release.'
> > + parser = argparse.ArgumentParser(description=description,
> > epilog=epilogue,
> > +
> > +formatter_class=argparse.RawDescriptionHelpFormatter)
> > + parser.add_argument('--no-prepare', dest='prepare', default=True,
> > action='store_false',
> > + help='Use the already built release in the
> > +provided checkout')
> > + parser.add_argument('--push-remote', metavar='USERNAME',
> > + help='Push the release to people.apache.org for
> > +the given user')
> > + parser.add_argument('--push-local', metavar='PATH',
> > + help='Push the release to the local path')
> > + parser.add_argument('--sign', metavar='KEYID',
> > + help='Sign the release with the given gpg key')
> > + parser.add_argument('--rc-num', metavar='NUM', type=int, default=1,
> > + help='Release Candidate number, required')
> > + parser.add_argument('--smoke-test', metavar='PATH',
> > + help='Run the smoker tester on the release in
> > +the given directory')
> > + parser.add_argument('root', metavar='checkout_path',
> > + help='Root of SVN checkout for lucene-solr')
> > + config = parser.parse_args()
> > +
> > + if config.push_remote is not None and config.push_local is not None:
> > + parser.error('Cannot specify --push-remote and --push-local
> > + together') if not config.prepare and config.sign:
> > + parser.error('Cannot sign already built release') if
> > + config.push_local is not None and os.path.exists(config.push_local):
> > + parser.error('Cannot push to local path that already exists') if
> > + config.rc_num <= 0:
> > + parser.error('Release Candidate number must be a positive
> > + integer') if not os.path.isdir(config.root):
> > + # TODO: add additional svn check to ensure dir is a real
> > + lucene-solr
> > checkout
> > + parser.error('Root path is not a valid lucene-solr checkout') if
> > + config.smoke_test is not None and os.path.exists(config.smoke_test):
> > + parser.error('Smoke test path already exists')
> >
> > - try:
> > - idx = sys.argv.index('-smoke')
> > - except ValueError:
> > - smokeTmpDir = None
> > - else:
> > - smokeTmpDir = sys.argv[idx+1]
> > - del sys.argv[idx:idx+2]
> > - if os.path.exists(smokeTmpDir):
> > - print()
> > - print('ERROR: smoke tmpDir "%s" exists; please remove first' %
> > smokeTmpDir)
> > - print()
> > - sys.exit(1)
> > -
> > - try:
> > - idx = sys.argv.index('-pushLocal')
> > - except ValueError:
> > - doPushLocal = False
> > - else:
> > - doPushLocal = True
> > - localStagingDir = sys.argv[idx+1]
> > - del sys.argv[idx:idx+2]
> > - if os.path.exists(localStagingDir):
> > - print()
> > - print('ERROR: pushLocal dir "%s" exists; please remove first' %
> > localStagingDir)
> > - print()
> > - sys.exit(1)
> > -
> > - if doPushRemote and doPushLocal:
> > - print()
> > - print('ERROR: specify at most one of -push or -pushLocal (got both)')
> > - print()
> > - sys.exit(1)
> > -
> > - try:
> > - idx = sys.argv.index('-sign')
> > - except ValueError:
> > - gpgKeyID = None
> > - else:
> > - gpgKeyID = sys.argv[idx+1]
> > - del sys.argv[idx:idx+2]
> > + config.version = read_version(config.root) print('Building version:
> > + %s' % config.version)
> >
> > + if config.sign:
> > sys.stdout.flush()
> > import getpass
> > - gpgPassword = getpass.getpass('Enter GPG keystore password: ')
> > + config.key_id = config.sign
> > + config.key_password = getpass.getpass('Enter GPG keystore password:
> > + ')
> > + else:
> > + config.gpg_password = None
> >
> > - root = os.path.abspath(sys.argv[1])
> > - version = sys.argv[2]
> > - rcNum = int(sys.argv[3])
> > + return config
> > +
> > +def main():
> > + c = parse_config()
> >
> > - if doPrepare:
> > - rev = prepare(root, version, gpgKeyID, gpgPassword, smokeTmpDir is
> > None)
> > + if c.prepare:
> > + rev = prepare(c.root, c.version, c.key_id, c.key_password,
> > + key.smoke_test is None)
> > else:
> > os.chdir(root)
> > rev = open('rev.txt', encoding='UTF-8').read()
> >
> > - if doPushRemote:
> > + if c.push_remote:
> > url = push(version, root, rev, rcNum, username)
> > - elif doPushLocal:
> > - url = pushLocal(version, root, rev, rcNum, localStagingDir)
> > + elif c.push_local:
> > + url = pushLocal(version, root, rev, c.rc_num, c.push_local)
> > else:
> > url = None
> >
> > if url is not None:
> > print(' URL: %s' % url)
> >
> > - if smokeTmpDir is not None:
> > + if c.smoke_test is not None:
> > import smokeTestRelease
> > smokeTestRelease.DEBUG = False
> > - smokeTestRelease.smokeTest(url, rev, version, smokeTmpDir,
> gpgKeyID
> > is not None, '')
> > + smokeTestRelease.smokeTest(url, rev, c.version, c.smoke_test,
> > + c.sign is not None, '')
> >
> > if __name__ == '__main__':
> > try:
> > main()
> > - except:
> > - import traceback
> > - traceback.print_exc()
> > + except KeyboardInterrupt:
> > + print('Keyboard interrupt...exiting')
> > +
> >
> > Modified: lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py
> > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-
> >
> tools/scripts/smokeTestRelease.py?rev=1621085&r1=1621084&r2=1621085&
> > view=diff
> >
> ==========================================================
> > ====================
> > --- lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py (original)
> > +++ lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py Thu Aug 28
> > +++ 08:01:50 2014
> > @@ -13,6 +13,7 @@
> > # See the License for the specific language governing permissions and
> > # limitations under the License.
> >
> > +import argparse
> > import os
> > import zipfile
> > import codecs
> > @@ -40,6 +41,7 @@ import checkJavaDocs import checkJavadocLinks
> > import io import codecs
> > +import textwrap
> >
> > # This tool expects to find /lucene and /solr off the base URL. You # must
> > have a working gpg, tar, unzip in your path. This has been @@ -1248,55
> > +1250,60 @@ def crawl(downloadedFiles, urlString, ta
> > downloadedFiles.append(path)
> > sys.stdout.write('.')
> >
> > -reAllowedVersion = re.compile(r'^\d+\.\d+\.\d+(-ALPHA|-BETA)?$')
> > -
> > -def main():
> > -
> > - if len(sys.argv) < 5:
> > - print()
> > - print('Usage python -u %s BaseURL SvnRevision version tmpDir [
> > isSigned(True|False) ] [ -testArgs "-Dwhat=ever [ ... ]" ]'
> > - % sys.argv[0])
> > - print()
> > - print(' example: python3.2 -u dev-tools/scripts/smokeTestRelease.py
> > http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-RC1-
> > rev1469340 1469340 4.3.0 /path/to/a/tmp/dir')
> > - print()
> > - sys.exit(1)
> > -
> > - baseURL = sys.argv[1]
> > - svnRevision = sys.argv[2]
> > - version = sys.argv[3]
> > +version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)')
> > +revision_re = re.compile(r'rev(\d+)')
> > +def parse_config():
> > + epilogue = textwrap.dedent('''
> > + Example usage:
> > + python3.2 -u dev-tools/scripts/smokeTestRelease.py
> > +http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-RC1-
> > re
> > +v1469340')
> > + ''')
> > + description = 'Utility to test a release.'
> > + parser = argparse.ArgumentParser(description=description,
> > epilog=epilogue,
> > +
> > +formatter_class=argparse.RawDescriptionHelpFormatter)
> > + parser.add_argument('--tmp-dir', metavar='PATH',
> > + help='Temporary directory to test inside,
> > +defaults to /tmp/smoke_lucene_$version_$revision')
> > + parser.add_argument('--not-signed', dest='is_signed',
> > action='store_false', default=True,
> > + help='Indicates the release is not signed')
> > + parser.add_argument('--revision',
> > + help='SVN revision number that release was built
> > +with, defaults to that in URL')
> > + parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
> > + help='Version of the release, defaults to that in
> > +URL')
> > + parser.add_argument('url', help='Url pointing to release to test')
> > + parser.add_argument('test_args', nargs=argparse.REMAINDER,
> > metavar='ARGS',
> > + help='Arguments to pass to ant for testing, e.g.
> > +-Dwhat=ever')
> > + c = parser.parse_args()
> > +
> > + if c.version is not None:
> > + if not version_re.match(c.version):
> > + parser.error('version "%s" does not match format
> > + X.Y.Z[-ALPHA|-BETA]' % c.version)
> > + else:
> > + version_match = version_re.search(c.url)
> > + if version_match is None:
> > + parser.error('Could not find version in URL')
> > + c.version = version_match.group(1)
> > +
> > + if c.revision is None:
> > + revision_match = revision_re.search(c.url)
> > + if revision_match is None:
> > + parser.error('Could not find revision in URL')
> > + c.revision = revision_match.group(1)
> >
> > - if not reAllowedVersion.match(version):
> > - raise RuntimeError('version "%s" does not match format X.Y.Z[-ALPHA|-
> > BETA]' % version)
> > -
> > - tmpDirArgNum = 4
> > - tmpDir = os.path.abspath(sys.argv[tmpDirArgNum])
> > + if c.tmp_dir:
> > + c.tmp_dir = os.path.abspath(c.tmp_dir)
> > + else:
> > + tmp = '/tmp/smoke_lucene_%s_%s' % (c.version, c.revision)
> > + c.tmp_dir = tmp
> > + i = 1
> > + while os.path.exists(c.tmp_dir):
> > + c.tmp_dir = tmp + '_%d' % i
> > + i += 1
> >
> > - # TODO: yuck: positional-only args with more than one optional arg totally
> > sucks
> > - # TODO: consider naming all args
> > - isSigned = True
> > - testArgs = ''
> > - lastArgNum = len(sys.argv) - 1
> > - if lastArgNum > tmpDirArgNum:
> > - nextArgNum = tmpDirArgNum + 1
> > - if sys.argv[nextArgNum] == '-testArgs':
> > - if nextArgNum == lastArgNum:
> > - raise RuntimeError('missing expected argument to -testArgs')
> > - else:
> > - # TODO: should there be arg validation here? E.g. starts with '-D'?
> > - testArgs = sys.argv[nextArgNum + 1]
> > - nextArgNum += 2
> > - if nextArgNum <= lastArgNum:
> > - isSigned = (sys.argv[nextArgNum].lower() == "true")
> > - nextArgNum += 1
> > - if nextArgNum <= lastArgNum and testArgs == '':
> > - if sys.argv[nextArgNum] == '-testArgs':
> > - if nextArgNum == lastArgNum:
> > - raise RuntimeError('missing expected argument to -testArgs')
> > - else:
> > - # TODO: should there be arg validation here? E.g. starts with '-D'?
> > - testArgs = sys.argv[nextArgNum + 1]
> > + return c
> >
> > - smokeTest(baseURL, svnRevision, version, tmpDir, isSigned, testArgs)
> > +def main():
> > + c = parse_config()
> > + print('NOTE: output encoding is %s' % sys.stdout.encoding)
> > + smokeTest(c.url, c.revision, c.version, c.tmp_dir, c.is_signed, '
> > +'.join(c.test_args))
> >
> > def smokeTest(baseURL, svnRevision, version, tmpDir, isSigned, testArgs):
> >
> > @@ -1350,10 +1357,8 @@ def smokeTest(baseURL, svnRevision, vers
> > print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))
> >
> > if __name__ == '__main__':
> > - print('NOTE: output encoding is %s' % sys.stdout.encoding)
> > try:
> > main()
> > - except:
> > - traceback.print_exc()
> > - sys.exit(1)
> > - sys.exit(0)
> > + except KeyboardInterrupt:
> > + print('Keyboard interrupt...exiting')
> > +
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org