You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2018/02/24 00:23:14 UTC
[1/2] lucene-solr:master: LUCENE-8106: collect Ant test sysprops from
Jenkins log and add them to test repro lines
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x bd269fe43 -> 07f7a7b5d
refs/heads/master bd3f30b9a -> 88e00d088
LUCENE-8106: collect Ant test sysprops from Jenkins log and add them to test repro lines
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/88e00d08
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/88e00d08
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/88e00d08
Branch: refs/heads/master
Commit: 88e00d08894cd58f8cf1f51f9de6f36ce3183031
Parents: bd3f30b
Author: Steve Rowe <sa...@apache.org>
Authored: Fri Feb 23 19:20:14 2018 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Fri Feb 23 19:22:49 2018 -0500
----------------------------------------------------------------------
dev-tools/scripts/reproduceJenkinsFailures.py | 27 ++++++++++++++++++++--
1 file changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/88e00d08/dev-tools/scripts/reproduceJenkinsFailures.py
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py
index 775a22a..5e081cb 100644
--- a/dev-tools/scripts/reproduceJenkinsFailures.py
+++ b/dev-tools/scripts/reproduceJenkinsFailures.py
@@ -26,6 +26,17 @@ from textwrap import dedent
# Example: Checking out Revision e441a99009a557f82ea17ee9f9c3e9b89c75cee6 (refs/remotes/origin/master)
reGitRev = re.compile(r'Checking out Revision (\S+)\s+\(refs/remotes/origin/([^)]+)')
+# Policeman Jenkins example: [Lucene-Solr-7.x-Linux] $ /var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/bin/ant "-Dargs=-XX:-UseCompressedOops -XX:+UseConcMarkSweepGC" jenkins-hourly
+# Policeman Jenkins Windows example: [Lucene-Solr-master-Windows] $ cmd.exe /C "C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2\bin\ant.bat '"-Dargs=-client -XX:+UseConcMarkSweepGC"' jenkins-hourly && exit %%ERRORLEVEL%%"
+# ASF Jenkins example: [Lucene-Solr-Tests-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant jenkins-hourly
+# ASF Jenkins nightly example: [checkout] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant -file build.xml -Dtests.multiplier=2 -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt jenkins-nightly
+# ASF Jenkins smoker example: [Lucene-Solr-SmokeRelease-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant nightly-smoke
+reAntInvocation = re.compile(r'\bant(?:\.bat)?\s+.*(?:jenkins-(?:hourly|nightly)|nightly-smoke)')
+reAntSysprops = re.compile(r'"-D[^"]+"|-D[^=]+="[^"]*"|-D\S+')
+
+# sarowe Jenkins example: + export 'ANT_OPTS=-Xmx1150m -XX:+CMSClassUnloadingEnabled -Djava.awt.headless=true -Dargs="-Xmx1g"'
+reAntOptions = re.compile(r"export\s+'?\s*ANT_OPTS=([^'\n\r]+)")
+
# Method example: NOTE: reproduce with: ant test -Dtestcase=ZkSolrClientTest -Dtests.method=testMultipleWatchesAsync -Dtests.seed=6EF5AB70F0032849 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=NST -Dtests.asserts=true -Dtests.file.encoding=UTF-8
# Suite example: NOTE: reproduce with: ant test -Dtestcase=CloudSolrClientTest -Dtests.seed=DB2DF2D8228BAF27 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=America/Argentina/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
reReproLine = re.compile(r'NOTE:\s+reproduce\s+with:(\s+ant\s+test\s+-Dtestcase=(\S+)\s+(?:-Dtests.method=\S+\s+)?(.*))')
@@ -90,7 +101,9 @@ def run(cmd, rememberFailure=True):
def fetchAndParseJenkinsLog(url):
global revisionFromLog
global branchFromLog
+ global antOptions
revisionFromLog = None
+ antOptions = ''
tests = {}
print('[repro] Jenkins log URL: %s\n' % url)
try:
@@ -109,6 +122,16 @@ def fetchAndParseJenkinsLog(url):
testcase = match.group(2)
reproLineWithoutMethod = match.group(3).strip()
tests[testcase] = reproLineWithoutMethod
+ else:
+ match = reAntInvocation.search(line)
+ if match is not None:
+ antOptions = ' '.join(reAntSysprops.findall(line))
+ else:
+ match = reAntOptions.search(line)
+ if match is not None:
+ antOptions = ' '.join(reAntSysprops.findall(line))
+ if len(antOptions) > 0:
+ print('[repro] Ant options: %s' % antOptions)
except urllib.error.URLError as e:
raise RuntimeError('ERROR: fetching %s : %s' % (url, e))
@@ -162,7 +185,7 @@ def groupTestsByModule(tests):
def runTests(testIters, modules, tests):
cwd = os.getcwd()
- testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s'
+ testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s %s'
for module in modules:
moduleTests = list(modules[module])
testList = '|'.join(map(lambda t: '*.%s' % t, moduleTests))
@@ -173,7 +196,7 @@ def runTests(testIters, modules, tests):
try:
if 0 != code:
raise RuntimeError("ERROR: Compile failed in %s/ with code %d. See above." % (module, code))
- run(testCmdline % (testIters, testIters * numTests, testList, params))
+ run(testCmdline % (testIters, testIters * numTests, testList, antOptions, params))
finally:
os.chdir(cwd)
[2/2] lucene-solr:branch_7x: LUCENE-8106: collect Ant test sysprops
from Jenkins log and add them to test repro lines
Posted by sa...@apache.org.
LUCENE-8106: collect Ant test sysprops from Jenkins log and add them to test repro lines
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/07f7a7b5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/07f7a7b5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/07f7a7b5
Branch: refs/heads/branch_7x
Commit: 07f7a7b5ddd08aa86dd65f31cf475546da084204
Parents: bd269fe
Author: Steve Rowe <sa...@apache.org>
Authored: Fri Feb 23 19:20:14 2018 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Fri Feb 23 19:23:06 2018 -0500
----------------------------------------------------------------------
dev-tools/scripts/reproduceJenkinsFailures.py | 27 ++++++++++++++++++++--
1 file changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07f7a7b5/dev-tools/scripts/reproduceJenkinsFailures.py
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py
index 775a22a..5e081cb 100644
--- a/dev-tools/scripts/reproduceJenkinsFailures.py
+++ b/dev-tools/scripts/reproduceJenkinsFailures.py
@@ -26,6 +26,17 @@ from textwrap import dedent
# Example: Checking out Revision e441a99009a557f82ea17ee9f9c3e9b89c75cee6 (refs/remotes/origin/master)
reGitRev = re.compile(r'Checking out Revision (\S+)\s+\(refs/remotes/origin/([^)]+)')
+# Policeman Jenkins example: [Lucene-Solr-7.x-Linux] $ /var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/bin/ant "-Dargs=-XX:-UseCompressedOops -XX:+UseConcMarkSweepGC" jenkins-hourly
+# Policeman Jenkins Windows example: [Lucene-Solr-master-Windows] $ cmd.exe /C "C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2\bin\ant.bat '"-Dargs=-client -XX:+UseConcMarkSweepGC"' jenkins-hourly && exit %%ERRORLEVEL%%"
+# ASF Jenkins example: [Lucene-Solr-Tests-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant jenkins-hourly
+# ASF Jenkins nightly example: [checkout] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant -file build.xml -Dtests.multiplier=2 -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt jenkins-nightly
+# ASF Jenkins smoker example: [Lucene-Solr-SmokeRelease-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant nightly-smoke
+reAntInvocation = re.compile(r'\bant(?:\.bat)?\s+.*(?:jenkins-(?:hourly|nightly)|nightly-smoke)')
+reAntSysprops = re.compile(r'"-D[^"]+"|-D[^=]+="[^"]*"|-D\S+')
+
+# sarowe Jenkins example: + export 'ANT_OPTS=-Xmx1150m -XX:+CMSClassUnloadingEnabled -Djava.awt.headless=true -Dargs="-Xmx1g"'
+reAntOptions = re.compile(r"export\s+'?\s*ANT_OPTS=([^'\n\r]+)")
+
# Method example: NOTE: reproduce with: ant test -Dtestcase=ZkSolrClientTest -Dtests.method=testMultipleWatchesAsync -Dtests.seed=6EF5AB70F0032849 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=NST -Dtests.asserts=true -Dtests.file.encoding=UTF-8
# Suite example: NOTE: reproduce with: ant test -Dtestcase=CloudSolrClientTest -Dtests.seed=DB2DF2D8228BAF27 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=America/Argentina/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
reReproLine = re.compile(r'NOTE:\s+reproduce\s+with:(\s+ant\s+test\s+-Dtestcase=(\S+)\s+(?:-Dtests.method=\S+\s+)?(.*))')
@@ -90,7 +101,9 @@ def run(cmd, rememberFailure=True):
def fetchAndParseJenkinsLog(url):
global revisionFromLog
global branchFromLog
+ global antOptions
revisionFromLog = None
+ antOptions = ''
tests = {}
print('[repro] Jenkins log URL: %s\n' % url)
try:
@@ -109,6 +122,16 @@ def fetchAndParseJenkinsLog(url):
testcase = match.group(2)
reproLineWithoutMethod = match.group(3).strip()
tests[testcase] = reproLineWithoutMethod
+ else:
+ match = reAntInvocation.search(line)
+ if match is not None:
+ antOptions = ' '.join(reAntSysprops.findall(line))
+ else:
+ match = reAntOptions.search(line)
+ if match is not None:
+ antOptions = ' '.join(reAntSysprops.findall(line))
+ if len(antOptions) > 0:
+ print('[repro] Ant options: %s' % antOptions)
except urllib.error.URLError as e:
raise RuntimeError('ERROR: fetching %s : %s' % (url, e))
@@ -162,7 +185,7 @@ def groupTestsByModule(tests):
def runTests(testIters, modules, tests):
cwd = os.getcwd()
- testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s'
+ testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s %s'
for module in modules:
moduleTests = list(modules[module])
testList = '|'.join(map(lambda t: '*.%s' % t, moduleTests))
@@ -173,7 +196,7 @@ def runTests(testIters, modules, tests):
try:
if 0 != code:
raise RuntimeError("ERROR: Compile failed in %s/ with code %d. See above." % (module, code))
- run(testCmdline % (testIters, testIters * numTests, testList, params))
+ run(testCmdline % (testIters, testIters * numTests, testList, antOptions, params))
finally:
os.chdir(cwd)