You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by wi...@apache.org on 2018/12/05 17:59:43 UTC
[geode-benchmarks] branch develop updated: GEODE-6145: Allow
multiple results for a single probe analyzer (#14)
This is an automated email from the ASF dual-hosted git repository.
wirebaron pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git
The following commit(s) were added to refs/heads/develop by this push:
new ef2e72f GEODE-6145: Allow multiple results for a single probe analyzer (#14)
ef2e72f is described below
commit ef2e72fec401d6a801744f7a530d9c026c9d77b0
Author: Brian Rowe <br...@pivotal.io>
AuthorDate: Wed Dec 5 09:59:39 2018 -0800
GEODE-6145: Allow multiple results for a single probe analyzer (#14)
This change modifies the ProbeResultParser interface to return a list of
ResultData objects. It takes advantage of this new functionality to return
6 results from the HDRHistogramParser (median latency, 90, 99, 99.9 percentile
latencies, average latency, and latency standard deviation). This is probably
too noisy for the long run, but could prove useful as we evaluate different
platforms.
This change also changes the BenchmarkRunAnalyzer (invoked via gradlew
analyzeRun) to take the baseline directory as its first argument.
---
.../perftest/analysis/BenchmarkRunAnalyzer.java | 24 ++++++++++++++++------
.../geode/perftest/analysis/ProbeResultParser.java | 17 +++++++++++----
.../analysis/YardstickHdrHistogramParser.java | 19 +++++++++--------
.../analysis/YardstickPercentileSensorParser.java | 22 +++++++++-----------
.../analysis/YardstickThroughputSensorParser.java | 10 ++++-----
.../analysis/BenchmarkRunAnalyzerTest.java | 2 +-
6 files changed, 57 insertions(+), 37 deletions(-)
diff --git a/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java b/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
index 764f622..57ae207 100644
--- a/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
@@ -20,6 +20,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
@@ -55,7 +56,7 @@ public class BenchmarkRunAnalyzer {
probes.add(probeResultParser);
}
- public BenchmarkRunResult analyzeTestRun(File testResultDir, File baselineResultDir)
+ public BenchmarkRunResult analyzeTestRun(File baselineResultDir, File testResultDir)
throws IOException {
List<File> benchmarkDirs = Arrays.asList(testResultDir.listFiles());
benchmarkDirs.sort(File::compareTo);
@@ -72,22 +73,33 @@ public class BenchmarkRunAnalyzer {
final BenchmarkRunResult.BenchmarkResult benchmarkResult =
result.addBenchmark(testDir.getName());
for (ProbeResultParser probe : probes) {
- double testResult = getTestResult(testYardstickDirs, probe);
- double baselineResult = getTestResult(baselineYardstickDirs, probe);
+ List<ProbeResultParser.ResultData> testResults = getTestResult(testYardstickDirs, probe);
+ List<ProbeResultParser.ResultData> baselineResults =
+ getTestResult(baselineYardstickDirs, probe);
- benchmarkResult.addProbeResult(probe.getResultDescription(), baselineResult, testResult);
+ Iterator<ProbeResultParser.ResultData> testResultIter = testResults.iterator();
+ Iterator<ProbeResultParser.ResultData> baselineResultIter = baselineResults.iterator();
+
+ while (testResultIter.hasNext()) {
+ ProbeResultParser.ResultData testResult = testResultIter.next();
+ ProbeResultParser.ResultData baselineResult = baselineResultIter.next();
+
+ benchmarkResult.addProbeResult(testResult.description, baselineResult.value,
+ testResult.value);
+ }
}
}
return result;
}
- private double getTestResult(List<File> resultDirs, ProbeResultParser probe) throws IOException {
+ private List<ProbeResultParser.ResultData> getTestResult(List<File> resultDirs,
+ ProbeResultParser probe) throws IOException {
probe.reset();
for (File outputDirectory : resultDirs) {
probe.parseResults(outputDirectory);
}
- return probe.getProbeResult();
+ return probe.getProbeResults();
}
private List<File> getYardstickOutputForBenchmarkDir(File benchmarkDir) throws IOException {
diff --git a/harness/src/main/java/org/apache/geode/perftest/analysis/ProbeResultParser.java b/harness/src/main/java/org/apache/geode/perftest/analysis/ProbeResultParser.java
index 1e94329..dd1935e 100644
--- a/harness/src/main/java/org/apache/geode/perftest/analysis/ProbeResultParser.java
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/ProbeResultParser.java
@@ -16,8 +16,10 @@ package org.apache.geode.perftest.analysis;
import java.io.File;
import java.io.IOException;
+import java.util.List;
public interface ProbeResultParser {
+
// Given a output directory for a benchmark, parse out the data for the desired probe. Note that
// this method may be passed several csv files for a run and is expected to appropriately
// aggregate the result of interest.
@@ -26,9 +28,16 @@ public interface ProbeResultParser {
// Reset the parser to a clean state where parseResults can be called again
void reset();
- // Get a single float value summarizing the data for the probe.
- double getProbeResult();
+ // Get the {description, value} pairs for the probe
+ List<ResultData> getProbeResults();
+
+ class ResultData {
+ public String description;
+ public double value;
- // Get a text description of what the probe result is depicting
- String getResultDescription();
+ public ResultData(String description, double value) {
+ this.description = description;
+ this.value = value;
+ }
+ }
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickHdrHistogramParser.java b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickHdrHistogramParser.java
index 8b2c6da..19d1749 100644
--- a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickHdrHistogramParser.java
+++ b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickHdrHistogramParser.java
@@ -17,6 +17,8 @@ package org.apache.geode.perftest.yardstick.analysis;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.HdrHistogram.Histogram;
import org.HdrHistogram.HistogramLogReader;
@@ -31,7 +33,6 @@ import org.apache.geode.perftest.yardstick.hdrhistogram.HdrHistogramWriter;
*/
public class YardstickHdrHistogramParser implements ProbeResultParser {
public static final String sensorOutputFile = HdrHistogramWriter.FILE_NAME;
- public static final String probeResultDescription = "HDR 99th percentile latency";
public Histogram histogram = null;
@@ -55,13 +56,15 @@ public class YardstickHdrHistogramParser implements ProbeResultParser {
}
@Override
- // Default probe result is the 99th percentile latency for the benchmark
- public double getProbeResult() {
- return histogram.getValueAtPercentile(99);
- }
+ public List<ResultData> getProbeResults() {
+ List<ResultData> results = new ArrayList<>(3);
+ results.add(new ResultData("median latency", histogram.getValueAtPercentile(50)));
+ results.add(new ResultData("90th percentile latency", histogram.getValueAtPercentile(90)));
+ results.add(new ResultData("99th percentile latency", histogram.getValueAtPercentile(99)));
+ results.add(new ResultData("99.9th percentile latency", histogram.getValueAtPercentile(99.9)));
+ results.add(new ResultData("average latency", histogram.getMean()));
+ results.add(new ResultData("latency standard deviation", histogram.getStdDeviation()));
- @Override
- public String getResultDescription() {
- return probeResultDescription;
+ return results;
}
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickPercentileSensorParser.java b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickPercentileSensorParser.java
index fb843ca..d99a538 100644
--- a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickPercentileSensorParser.java
+++ b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickPercentileSensorParser.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
import org.yardstickframework.probes.PercentileProbe;
@@ -32,7 +33,7 @@ import org.apache.geode.perftest.analysis.ProbeResultParser;
*/
public class YardstickPercentileSensorParser implements ProbeResultParser {
public static final String sensorOutputFile = "PercentileProbe.csv";
- public static final String probeResultDescription = "99th percentile latency";
+ public static final String probeResultDescription = "YS 99th percentile latency";
private class SensorBucket {
public int latencyBucket;
@@ -74,17 +75,6 @@ public class YardstickPercentileSensorParser implements ProbeResultParser {
buckets = new ArrayList<>();
}
- @Override
- // Default probe result is the 99th percentile latency for the benchmark
- public double getProbeResult() {
- return getPercentile(99);
- }
-
- @Override
- public String getResultDescription() {
- return probeResultDescription;
- }
-
private void normalizeBuckets() {
float totalPercentage = 0;
for (SensorBucket bucket : buckets) {
@@ -131,4 +121,12 @@ public class YardstickPercentileSensorParser implements ProbeResultParser {
return targetBucket.latencyBucket + bucketSize * percentileLocationInTargetBucket;
}
+
+ @Override
+ public List<ResultData> getProbeResults() {
+ List<ResultData> results = new ArrayList<>(1);
+ results.add(new ResultData(probeResultDescription, getPercentile(99)));
+
+ return results;
+ }
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickThroughputSensorParser.java b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickThroughputSensorParser.java
index 340f3c3..081b263 100644
--- a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickThroughputSensorParser.java
+++ b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickThroughputSensorParser.java
@@ -56,13 +56,11 @@ public class YardstickThroughputSensorParser implements ProbeResultParser {
}
@Override
- public double getProbeResult() {
- return getAverageThroughput();
- }
+ public List<ResultData> getProbeResults() {
+ List<ResultData> results = new ArrayList<>(1);
+ results.add(new ResultData(probeResultDescription, getAverageThroughput()));
- @Override
- public String getResultDescription() {
- return probeResultDescription;
+ return results;
}
public double getAverageThroughput() {
diff --git a/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java b/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
index 12e37d1..f38f355 100644
--- a/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
@@ -81,7 +81,7 @@ public class BenchmarkRunAnalyzerTest {
StringWriter writer = new StringWriter();
- BenchmarkRunResult results = harvester.analyzeTestRun(testFolder, baseFolder);
+ BenchmarkRunResult results = harvester.analyzeTestRun(baseFolder, testFolder);
BenchmarkRunResult expectedBenchmarkResult = new BenchmarkRunResult();
BenchmarkRunResult.BenchmarkResult resultA = expectedBenchmarkResult.addBenchmark("BenchmarkA");