You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by he...@apache.org on 2019/12/10 23:33:06 UTC
[geode-benchmarks] 01/01: GEODE-7554: Add retry mechanism for
failed tests
This is an automated email from the ASF dual-hosted git repository.
heybales pushed a commit to branch GEODE-7554
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git
commit c9549e8b982be8471c6298790e74c7dbe3796858
Author: Helena A. Bales <hb...@pivotal.io>
AuthorDate: Thu Dec 5 09:51:43 2019 -0800
GEODE-7554: Add retry mechanism for failed tests
* write failed tests to a file
* enable all benchmarks
* update help message for --ci option in run_against_baseline.sh
---
geode-benchmarks/build.gradle | 5 -----
.../main/java/org/apache/geode/perftest/analysis/Analyzer.java | 8 ++++++++
infrastructure/scripts/aws/run_against_baseline.sh | 4 ++--
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/geode-benchmarks/build.gradle b/geode-benchmarks/build.gradle
index 0abe0dc..42cafe9 100644
--- a/geode-benchmarks/build.gradle
+++ b/geode-benchmarks/build.gradle
@@ -63,11 +63,6 @@ task benchmark(type: Test) {
useJUnitPlatform()
testLogging { exceptionFormat = 'full' }
- exclude "**/*NonIndexedQueryBenchmark.class"
- exclude "**/PartitionedFunctionExecutionBenchmark.class"
- exclude "**/NoopBenchmark.class"
- exclude "**/*LongBenchmark.class"
-
forkEvery 1
systemProperty 'TEST_HOSTS', project.findProperty('hosts')
diff --git a/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
index 13a90cc..9d69c75 100644
--- a/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
@@ -16,7 +16,9 @@ package org.apache.geode.perftest.analysis;
import static java.lang.Double.isNaN;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
@@ -70,6 +72,9 @@ public class Analyzer {
benchmarkRunResult.writeResult(new PrintWriter(System.out));
/* throw exc if failed? */
+ String errorFilePath = testResultArg + "/../../../failedTests";
+ BufferedWriter writer = new BufferedWriter(new FileWriter(errorFilePath, true));
+
boolean isSignificantlyBetter = false;
boolean isHighWaterCandidate = true;
StringBuilder errorMessage = new StringBuilder();
@@ -79,12 +84,14 @@ public class Analyzer {
if (isNaN(probeResult.baseline) || isNaN(probeResult.test)) {
errorMessage.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
.append(" missing result file.\n");
+ writer.append(benchmarkResult.name + "\n");
} else if (probeResult.description.equals("average latency")) {
if (probeResult.getDifference() > 0) {
isHighWaterCandidate = false;
if (probeResult.getDifference() >= 0.05) {
errorMessage.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
.append(" average latency is 5% worse than baseline.\n");
+ writer.append(benchmarkResult.name + "\n");
}
} else if (probeResult.getDifference() <= -0.5) {
isSignificantlyBetter = true;
@@ -92,6 +99,7 @@ public class Analyzer {
}
}
}
+ writer.close();
if (isCI && isHighWaterCandidate && isSignificantlyBetter) {
System.out.println(
diff --git a/infrastructure/scripts/aws/run_against_baseline.sh b/infrastructure/scripts/aws/run_against_baseline.sh
index f4e54fa..87a202d 100755
--- a/infrastructure/scripts/aws/run_against_baseline.sh
+++ b/infrastructure/scripts/aws/run_against_baseline.sh
@@ -129,7 +129,7 @@ while (( "$#" )); do
echo "-R|--baseline-geode-repo : Geode baseline repo (default: ${DEFAULT_BASELINE_REPO})"
echo "-B|--baseline-geode-branch : Geode baseline branch"
echo "-m|--metadata : Test metadata to output to file, comma-delimited"
- echo "--ci : Set if starting instances for Continuous Integration"
+ echo "--ci : Set if starting instances for Continuous Integration - used to retry failed tests and to name/target AWS instances"
echo "-- : All subsequent arguments are passed to the benchmark task as arguments."
echo "-h|--help|-? : This help message"
exit 1
@@ -208,4 +208,4 @@ if [[ -z "${CI}" ]]; then
./analyze_tests.sh -o ${OUTPUT}
else
./analyze_tests.sh -o ${OUTPUT} --ci
-fi
+fi
\ No newline at end of file