You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by he...@apache.org on 2019/12/10 23:33:05 UTC
[geode-benchmarks] branch GEODE-7554 updated (2faa477 -> c9549e8)
This is an automated email from the ASF dual-hosted git repository.
heybales pushed a change to branch GEODE-7554
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git.
discard 2faa477 enable all benchmark tests
discard 5abd5a0 write the failed test file up one directory
discard 67a624a only write out failed tests
discard 0028e98 fix noop task
discard 9619a65 delete the extra Noop benchmarks
discard 7fccbcc run 7 NoopBenchmarks and hope that one of them fails
discard b5de57c create the failed file before writing to it
discard 0e5cbf5 write to failed file and exit 1 regardless of test results
discard 9a54d7a Revert "exit in a different place"
discard 7fce0b2 exit in a different place
discard e4534fd exit in a different place
discard 7800b80 fail test without running for testing
discard 429aaeb Revert "only exit 1 from analyze task if not in CI"
discard 224a023 only exit 1 from analyze task if not in CI
discard 9057a37 write each test only once
discard 8310931 close and flush the buffered writer
discard 8ca991a fix append to failed file
discard 256a0a0 include NoopBenchmark
discard fbe3102 write failed tests to file and update --ci readme
add 477d381 Confirm and retry AWS instance allocations. (#119)
new c9549e8 GEODE-7554: Add retry mechanism for failed tests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (2faa477)
\
N -- N -- N refs/heads/GEODE-7554 (c9549e8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../geode/infrastructure/aws/LaunchCluster.java | 67 ++++++++++++++--------
1 file changed, 44 insertions(+), 23 deletions(-)
[geode-benchmarks] 01/01: GEODE-7554: Add retry mechanism for
failed tests
Posted by he...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
heybales pushed a commit to branch GEODE-7554
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git
commit c9549e8b982be8471c6298790e74c7dbe3796858
Author: Helena A. Bales <hb...@pivotal.io>
AuthorDate: Thu Dec 5 09:51:43 2019 -0800
GEODE-7554: Add retry mechanism for failed tests
* write failed tests to a file
* enable all benchmarks
* update help message for --ci option in run_against_baseline.sh
---
geode-benchmarks/build.gradle | 5 -----
.../main/java/org/apache/geode/perftest/analysis/Analyzer.java | 8 ++++++++
infrastructure/scripts/aws/run_against_baseline.sh | 4 ++--
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/geode-benchmarks/build.gradle b/geode-benchmarks/build.gradle
index 0abe0dc..42cafe9 100644
--- a/geode-benchmarks/build.gradle
+++ b/geode-benchmarks/build.gradle
@@ -63,11 +63,6 @@ task benchmark(type: Test) {
useJUnitPlatform()
testLogging { exceptionFormat = 'full' }
- exclude "**/*NonIndexedQueryBenchmark.class"
- exclude "**/PartitionedFunctionExecutionBenchmark.class"
- exclude "**/NoopBenchmark.class"
- exclude "**/*LongBenchmark.class"
-
forkEvery 1
systemProperty 'TEST_HOSTS', project.findProperty('hosts')
diff --git a/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
index 13a90cc..9d69c75 100644
--- a/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
@@ -16,7 +16,9 @@ package org.apache.geode.perftest.analysis;
import static java.lang.Double.isNaN;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
@@ -70,6 +72,9 @@ public class Analyzer {
benchmarkRunResult.writeResult(new PrintWriter(System.out));
/* throw exc if failed? */
+ String errorFilePath = testResultArg + "/../../../failedTests";
+ BufferedWriter writer = new BufferedWriter(new FileWriter(errorFilePath, true));
+
boolean isSignificantlyBetter = false;
boolean isHighWaterCandidate = true;
StringBuilder errorMessage = new StringBuilder();
@@ -79,12 +84,14 @@ public class Analyzer {
if (isNaN(probeResult.baseline) || isNaN(probeResult.test)) {
errorMessage.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
.append(" missing result file.\n");
+ writer.append(benchmarkResult.name + "\n");
} else if (probeResult.description.equals("average latency")) {
if (probeResult.getDifference() > 0) {
isHighWaterCandidate = false;
if (probeResult.getDifference() >= 0.05) {
errorMessage.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
.append(" average latency is 5% worse than baseline.\n");
+ writer.append(benchmarkResult.name + "\n");
}
} else if (probeResult.getDifference() <= -0.5) {
isSignificantlyBetter = true;
@@ -92,6 +99,7 @@ public class Analyzer {
}
}
}
+ writer.close();
if (isCI && isHighWaterCandidate && isSignificantlyBetter) {
System.out.println(
diff --git a/infrastructure/scripts/aws/run_against_baseline.sh b/infrastructure/scripts/aws/run_against_baseline.sh
index f4e54fa..87a202d 100755
--- a/infrastructure/scripts/aws/run_against_baseline.sh
+++ b/infrastructure/scripts/aws/run_against_baseline.sh
@@ -129,7 +129,7 @@ while (( "$#" )); do
echo "-R|--baseline-geode-repo : Geode baseline repo (default: ${DEFAULT_BASELINE_REPO})"
echo "-B|--baseline-geode-branch : Geode baseline branch"
echo "-m|--metadata : Test metadata to output to file, comma-delimited"
- echo "--ci : Set if starting instances for Continuous Integration"
+ echo "--ci : Set if starting instances for Continuous Integration - used to retry failed tests and to name/target AWS instances"
echo "-- : All subsequent arguments are passed to the benchmark task as arguments."
echo "-h|--help|-? : This help message"
exit 1
@@ -208,4 +208,4 @@ if [[ -z "${CI}" ]]; then
./analyze_tests.sh -o ${OUTPUT}
else
./analyze_tests.sh -o ${OUTPUT} --ci
-fi
+fi
\ No newline at end of file