You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by up...@apache.org on 2021/04/01 22:44:32 UTC
[geode-benchmarks] 03/05: Replacing metadata.json with metadata
properties files
This is an automated email from the ASF dual-hosted git repository.
upthewaterspout pushed a commit to branch feature/redis-performance-testing
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git
commit 002b902f3e79ed59d41728458fa403c56d7f93d5
Author: Dan Smith <da...@vmware.com>
AuthorDate: Thu Apr 1 13:33:05 2021 -0700
Replacing metadata.json with metadata properties files
Getting rid of problematic org.json dependency, and also making
sure we capture *all* system properties that might effect the behavior of the
test.
---
harness/build.gradle | 1 -
.../geode/perftest/runner/DefaultTestRunner.java | 50 +++++-----------------
.../geode/infrastructure/BenchmarkMetadata.java | 2 +-
.../geode/infrastructure/aws/LaunchCluster.java | 20 +++++----
4 files changed, 23 insertions(+), 50 deletions(-)
diff --git a/harness/build.gradle b/harness/build.gradle
index ab85423..a108f00 100644
--- a/harness/build.gradle
+++ b/harness/build.gradle
@@ -55,7 +55,6 @@ dependencies {
compile(group: 'commons-io', name: 'commons-io', version: project.'commons-io.version')
compile(group: 'org.yardstickframework', name: 'yardstick', version: project.'yardstick.version')
compile(group: 'org.hdrhistogram', name: 'HdrHistogram', version: project.'HdrHistogram.version')
- compile(group: 'org.json', name: 'json', version: project.'JSON.version')
compile(group: 'org.apache.geode', name: 'geode-core', version: geodeVersion)
testCompile(group: 'org.mockito', name: 'mockito-all', version: project.'mockito-all.version')
testCompile(group: 'org.awaitility', name: 'awaitility', version: project.'awaitility.version')
diff --git a/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java b/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
index a71b158..ba4d5a8 100644
--- a/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
+++ b/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
@@ -21,15 +21,12 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import org.json.JSONArray;
-import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,42 +74,17 @@ public class DefaultTestRunner implements TestRunner {
}
benchmarkOutput.mkdirs();
- String metadataFilename = outputDir + "/metadata.json";
+ String metadataFilename = outputDir + "/testrunner.properties";
Path metadataOutput = Paths.get(metadataFilename);
- JSONObject JSONmetadata = new JSONObject();
-
- if (metadataOutput.toFile().exists()) {
- JSONmetadata = new JSONObject(new String(Files.readAllBytes(metadataOutput)));
- } else {
- String metadata = System.getProperty("TEST_METADATA");
- if (!(metadata == null) && !metadata.isEmpty()) {
- JSONObject testMetadata = new JSONObject();
- String[] metadataEntries = metadata.split(",");
- for (String data : metadataEntries) {
- String[] kv = data.split(":");
- if (kv.length == 2) {
- testMetadata.put(kv[0], kv[1]);
- }
- }
- addVersionProperties(testMetadata, getVersionProperties());
- JSONmetadata.put("testMetadata", testMetadata);
- }
- }
- try {
- JSONArray testNames = JSONmetadata.getJSONArray("testNames");
- testNames.put(testName);
- JSONmetadata.put("testNames", testNames);
- } catch (org.json.JSONException e) {
- JSONArray testNames = new JSONArray();
- testNames.put(testName);
- JSONmetadata.put("testNames", testNames);
+ if (!metadataOutput.toFile().exists()) {
+ Properties properties = new Properties(System.getProperties());
+ addVersionProperties(properties, getVersionProperties());
+ try (FileWriter writer = new FileWriter(metadataOutput.toFile().getAbsoluteFile())) {
+ properties.store(writer, "Benchmark metadata generated while running tests");
+ }
}
- FileWriter metadataWriter = new FileWriter(metadataOutput.toFile().getAbsoluteFile());
- metadataWriter.write(JSONmetadata.toString());
- metadataWriter.flush();
-
Map<String, Integer> roles = config.getRoles();
Map<String, List<String>> jvmArgs = config.getJvmArgs();
@@ -140,10 +112,10 @@ public class DefaultTestRunner implements TestRunner {
}
- private void addVersionProperties(JSONObject jsonMetadata, Properties versionProperties) {
- jsonMetadata.put("source_version", versionProperties.getProperty("Product-Version"));
- jsonMetadata.put("source_branch", versionProperties.getProperty("Source-Repository"));
- jsonMetadata.put("source_revision", versionProperties.getProperty("Source-Revision"));
+ private void addVersionProperties(Properties jsonMetadata, Properties versionProperties) {
+ jsonMetadata.put("benchmark.source_version", versionProperties.getProperty("Product-Version"));
+ jsonMetadata.put("benchmark.source_branch", versionProperties.getProperty("Source-Repository"));
+ jsonMetadata.put("benchmark.source_revision", versionProperties.getProperty("Source-Revision"));
}
private Properties getVersionProperties() throws IOException {
diff --git a/infrastructure/src/main/java/org/apache/geode/infrastructure/BenchmarkMetadata.java b/infrastructure/src/main/java/org/apache/geode/infrastructure/BenchmarkMetadata.java
index 22c5d48..42a8d3a 100644
--- a/infrastructure/src/main/java/org/apache/geode/infrastructure/BenchmarkMetadata.java
+++ b/infrastructure/src/main/java/org/apache/geode/infrastructure/BenchmarkMetadata.java
@@ -44,6 +44,6 @@ public class BenchmarkMetadata {
}
public static String benchmarkMetadataFileName(String tag) {
- return benchmarkConfigDirectory() + "/" + tag + "-metadata.json";
+ return benchmarkConfigDirectory() + "/" + tag + "-cluster-launch.properties";
}
}
diff --git a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
index 45c126a..5a2e081 100644
--- a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
+++ b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
@@ -19,6 +19,7 @@ package org.apache.geode.infrastructure.aws;
import static java.lang.String.format;
import static java.lang.Thread.sleep;
+import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -32,11 +33,10 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
+import java.util.Properties;
import java.util.UUID;
import java.util.stream.Collectors;
-import org.json.JSONArray;
-import org.json.JSONObject;
import software.amazon.awssdk.services.ec2.Ec2Client;
import software.amazon.awssdk.services.ec2.model.AllocateHostsRequest;
import software.amazon.awssdk.services.ec2.model.AllocateHostsResponse;
@@ -283,19 +283,21 @@ public class LaunchCluster {
private static void createMetadata(String benchmarkTag, List<String> publicIps)
throws IOException {
UUID instanceId = UUID.randomUUID();
- JSONObject metadataJSON = new JSONObject();
-
- metadataJSON.put("instanceId", instanceId.toString());
- metadataJSON.put("publicIps", new JSONArray(publicIps));
+ // TODO - Filter out only benchmark properties from system properties? Maybe not necessary.
+ Properties metadata = new Properties(System.getProperties());
+ metadata.setProperty("benchmark.instanceId", instanceId.toString());
+ metadata.setProperty("benchmark.publicIps", String.join(",", publicIps));
Path configDirectory = Paths.get(BenchmarkMetadata.benchmarkConfigDirectory());
if (!configDirectory.toFile().exists()) {
Files.createDirectories(Paths.get(BenchmarkMetadata.benchmarkConfigDirectory()));
}
- Path metadata = Files.write(Paths.get(AwsBenchmarkMetadata.metadataFileName(benchmarkTag)),
- metadataJSON.toString().getBytes());
- Files.setPosixFilePermissions(metadata, PosixFilePermissions.fromString("rw-------"));
+ Path metadataPath = Paths.get(AwsBenchmarkMetadata.metadataFileName(benchmarkTag));
+ try (FileWriter writer = new FileWriter(metadataPath.toFile())) {
+ metadata.store(writer, "Benchmark metadata generated during cluster launch");
+ }
+ Files.setPosixFilePermissions(metadataPath, PosixFilePermissions.fromString("rw-------"));
}
private static void createLaunchTemplate(String benchmarkTag, Image newestImage) {