You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by wi...@apache.org on 2018/11/28 18:49:59 UTC
[geode-benchmarks] branch develop updated: GEODE-6086: Adding a
tool to allow command line analysis of test runs (#7)
This is an automated email from the ASF dual-hosted git repository.
wirebaron pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git
The following commit(s) were added to refs/heads/develop by this push:
new 30ae13c GEODE-6086: Adding a tool to allow command line analysis of test runs (#7)
30ae13c is described below
commit 30ae13c036a0eb69a395a83fdc3ad38943828c5b
Author: Brian Rowe <br...@pivotal.io>
AuthorDate: Wed Nov 28 10:49:54 2018 -0800
GEODE-6086: Adding a tool to allow command line analysis of test runs (#7)
This change adds the analyzeRun gradle target, which, when passed a baseline
result directory and test result directory via --args, will analyze the output
and report the variation of the test from the baseline.
This also improves the benchmark analyzer to scan the test result dir and
automatically parse out the benchmarks rather than needing them passed.
---
harness/build.gradle | 5 ++
.../apache/geode/perftest/analysis/Analyzer.java | 62 ++++++++++++++++++++
.../perftest/analysis/BenchmarkRunAnalyzer.java | 67 +++++++---------------
.../analysis/BenchmarkRunAnalyzerTest.java | 15 ++---
4 files changed, 92 insertions(+), 57 deletions(-)
diff --git a/harness/build.gradle b/harness/build.gradle
index bc33562..90d7793 100644
--- a/harness/build.gradle
+++ b/harness/build.gradle
@@ -28,6 +28,11 @@ repositories {
mavenCentral()
}
+task(analyzeRun, dependsOn: 'classes', type: JavaExec) {
+ main = 'org.apache.geode.perftest.analysis.Analyzer'
+ classpath = sourceSets.main.runtimeClasspath
+}
+
dependencies {
testCompile group: 'junit', name: 'junit', version: '4.12'
compile group: 'com.hierynomus', name: 'sshj', version: '0.26.0'
diff --git a/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
new file mode 100644
index 0000000..f0d1988
--- /dev/null
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.perftest.analysis;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import org.apache.geode.perftest.yardstick.analysis.YardstickPercentileSensorParser;
+import org.apache.geode.perftest.yardstick.analysis.YardstickThroughputSensorParser;
+
+public class Analyzer {
+
+ public static void main(String[] args) throws IOException {
+ if (args.length != 2) {
+ System.out.println("Analyzer takes two test output directories as arguments, test results followed by baseline run result.");
+ System.exit(1);
+ return;
+ }
+
+ String testResultArg = args[0];
+ String baselineResultArg = args[1];
+
+ File testResultDir = new File(testResultArg);
+ File baselineResultDir = new File(baselineResultArg);
+
+ boolean valid = true;
+ if (!testResultDir.exists()) {
+ System.out.println("Unable to find test result directory: " + testResultArg);
+ valid = false;
+ }
+ if (!baselineResultDir.exists()) {
+ System.out.println("Unable to find test result directory: " + baselineResultArg);
+ valid = false;
+ }
+ if (!valid) {
+ System.exit(1);
+ return;
+ }
+
+ System.out.println("Running analyzer");
+ System.out.println("Comparing test result at " + testResultArg + " to baseline at " + baselineResultArg);
+
+ BenchmarkRunAnalyzer analyzer = new BenchmarkRunAnalyzer();
+ analyzer.addProbe(new YardstickThroughputSensorParser());
+ analyzer.addProbe(new YardstickPercentileSensorParser());
+
+ analyzer.analyzeTestRun(testResultDir, baselineResultDir, new PrintWriter(System.out));
+ }
+}
diff --git a/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java b/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
index c84346b..19ffb58 100644
--- a/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
@@ -15,13 +15,15 @@
package org.apache.geode.perftest.analysis;
import java.io.File;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.geode.perftest.yardstick.YardstickTask;
@@ -44,18 +46,9 @@ import org.apache.geode.perftest.yardstick.YardstickTask;
* </pre>
*/
public class BenchmarkRunAnalyzer {
- private final List<SensorData> benchmarks = new ArrayList<>();
private final List<ProbeResultParser> probes = new ArrayList<>();
/**
- * Add a benchmark to be analyzed. The benchmark is expected to exist
- * in both result directories passed to {@link #analyzeTestRun(File, File, Writer)}
- */
- public void addBenchmark(String name, String testResultDir, List<String> nodeNames) {
- benchmarks.add(new SensorData(name, testResultDir, nodeNames));
- }
-
- /**
* Add a probe to produce a comparison for. The probe expects to find output files
* in the result directory for each node of each benchmark.
*/
@@ -65,19 +58,26 @@ public class BenchmarkRunAnalyzer {
public void analyzeTestRun(File testResultDir, File baselineResultDir, Writer output)
throws IOException {
+ List<File> benchmarkDirs = Arrays.asList(testResultDir.listFiles());
+
PrintWriter stream = new PrintWriter(output);
- for (SensorData benchmark : benchmarks) {
- stream.println("-- " + benchmark.benchmarkName + " --");
+ for (File testDir : benchmarkDirs) {
+ final List<File> yardstickOutputDirsForTest = getYardstickOutputForBenchmarkDir(testDir);
+ if (yardstickOutputDirsForTest.isEmpty()) {
+ continue;
+ }
+ File baselineDir = new File(baselineResultDir, testDir.getName());
+ stream.println("-- " + testDir.getName() + " --");
for (ProbeResultParser probe : probes) {
stream.println(probe.getResultDescription());
- for (String node : benchmark.nodesToParse) {
- probe.parseResults(getBenchmarkOutputDir(testResultDir, benchmark, node));
+ for (File outputDirectory : yardstickOutputDirsForTest) {
+ probe.parseResults(outputDirectory);
}
double testResult = probe.getProbeResult();
stream.println("Result: " + String.valueOf(testResult));
probe.reset();
- for (String node : benchmark.nodesToParse) {
- probe.parseResults(getBenchmarkOutputDir(baselineResultDir, benchmark, node));
+ for (File outputDirectory : getYardstickOutputForBenchmarkDir(baselineDir)) {
+ probe.parseResults(outputDirectory);
}
double baselineResult = probe.getProbeResult();
stream.println("Baseline: " + String.valueOf(baselineResult));
@@ -89,35 +89,10 @@ public class BenchmarkRunAnalyzer {
stream.flush();
}
- private File getBenchmarkOutputDir(File testResultDir, SensorData benchmark,
- String node) {
- File benchmarkDir = new File(testResultDir, benchmark.benchmarkSubdirectory);
- File nodeDir = new File(benchmarkDir, node);
-
- File[] files = nodeDir.listFiles((dir, name) -> name.contains(YardstickTask.YARDSTICK_OUTPUT));
-
- if(files == null || files.length != 1) {
- throw new IllegalStateException("Expected at least one subdirectory in " + nodeDir
- + " with the name *" +YardstickTask.YARDSTICK_OUTPUT);
- }
-
- return files[0];
+ private List<File> getYardstickOutputForBenchmarkDir(File benchmarkDir) throws IOException {
+ return Files.walk(benchmarkDir.toPath(), 2)
+ .filter(path -> path.toString().endsWith(YardstickTask.YARDSTICK_OUTPUT))
+ .map(Path::toFile)
+ .collect(Collectors.toList());
}
-
- // TODO: depending on how run output is stored, this data may be excessive or insufficient
- // The present assumption is each benchmark contains an arbitrarily named result directory
- // containing subdirectories for each node. Those subdirectories then contain the probe output
- // files for the run, for that node.
- private static class SensorData {
- private final String benchmarkName;
- private final String benchmarkSubdirectory;
- private final List<String> nodesToParse;
-
- public SensorData(String benchmarkName, String benchmarkSubdirectory, List<String> nodesNames) {
- this.benchmarkName = benchmarkName;
- this.benchmarkSubdirectory = benchmarkSubdirectory;
- this.nodesToParse = nodesNames;
- }
- }
-
}
diff --git a/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java b/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
index 387c198..e838733 100644
--- a/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
@@ -23,8 +23,6 @@ import java.io.IOException;
import java.io.PrintStream;
import java.io.StringReader;
import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Scanner;
import org.junit.Assert;
@@ -32,7 +30,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import org.apache.geode.perftest.analysis.BenchmarkRunAnalyzer;
import org.apache.geode.perftest.yardstick.analysis.YardstickPercentileSensorParser;
import org.apache.geode.perftest.yardstick.analysis.YardstickThroughputSensorParser;
@@ -47,6 +44,8 @@ public class BenchmarkRunAnalyzerTest {
final File testBenchmarkA2 = temporaryFolder.newFolder("testFolder","BenchmarkA","client2", "20181121-111516-yardstick-output");
final File testBenchmarkB1 = temporaryFolder.newFolder("testFolder","BenchmarkB","client1", "20181121-111516-yardstick-output");
final File testBenchmarkB2 = temporaryFolder.newFolder("testFolder","BenchmarkB","client2", "20181121-111516-yardstick-output");
+ temporaryFolder.newFolder(testFolder.getName(), "junkfolder");
+ new File(testFolder, "junkfile").createNewFile();
final File baseFolder = temporaryFolder.newFolder("baseFolder");
final File baseBenchmarkA1 = temporaryFolder.newFolder("baseFolder","BenchmarkA","client1", "20181121-111516-yardstick-output");
final File baseBenchmarkA2 = temporaryFolder.newFolder("baseFolder","BenchmarkA","client2", "20181121-111516-yardstick-output");
@@ -71,13 +70,7 @@ public class BenchmarkRunAnalyzerTest {
populateThroughputCSV(baseBenchmarkB2, new double[] {10, 15, 20, 25, 30}); // Avg 20
populatePercentileCSV(baseBenchmarkB2, new double[] {0, 0, 0, 99, 1}); // 300
- List<String> nodes = new ArrayList(2);
- nodes.add("client1");
- nodes.add("client2");
-
BenchmarkRunAnalyzer harvester = new BenchmarkRunAnalyzer();
- harvester.addBenchmark("Alpha", "BenchmarkA", nodes);
- harvester.addBenchmark("Beta", "BenchmarkB", nodes);
harvester.addProbe(new YardstickThroughputSensorParser());
harvester.addProbe(new YardstickPercentileSensorParser());
@@ -88,8 +81,8 @@ public class BenchmarkRunAnalyzerTest {
System.out.println(writer.toString());
BufferedReader resultReader = new BufferedReader(new StringReader(writer.toString()));
- validatedBenchmark(resultReader, "Alpha", 20, 300, 25, 200);
- validatedBenchmark(resultReader, "Beta", 25, 400, 20, 400);
+ validatedBenchmark(resultReader, "BenchmarkA", 20, 300, 25, 200);
+ validatedBenchmark(resultReader, "BenchmarkB", 25, 400, 20, 400);
}
private void populateThroughputCSV(File targetDirectory, double[] perSecondThroughputs)