You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by nn...@apache.org on 2019/02/25 15:58:19 UTC
[geode-benchmarks] branch develop updated: GEODE-6444: Store GC
logs in files (#56)
This is an automated email from the ASF dual-hosted git repository.
nnag pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git
The following commit(s) were added to refs/heads/develop by this push:
new c477ca0 GEODE-6444: Store GC logs in files (#56)
c477ca0 is described below
commit c477ca02a9a4685c2008d9b9ea98d2e500522592
Author: Nabarun Nag <na...@users.noreply.github.com>
AuthorDate: Mon Feb 25 07:58:13 2019 -0800
GEODE-6444: Store GC logs in files (#56)
* Store the GC logs in a file.
* Print the GC logs in the System.out / System.err in parallel while the benchmark is being executed in parallel.
* Using a thread pool for execution of runAsync and not depend on fork join pools
* Disabling the dstat probe
Co-authored-by: Dan Smith <ds...@pivotal.io>
Co-authored-by: nabarunnag <na...@cs.wisc.edu>
Co-authored-by: Jacob Barrett <jb...@pivotal.io>
---
geode-benchmarks/build.gradle | 2 --
.../infrastructure/ssh/SshInfrastructure.java | 31 ++++++++++++++++++----
.../apache/geode/perftest/jvms/JVMLauncher.java | 3 +++
.../apache/geode/perftest/jvms/rmi/ChildJVM.java | 5 ----
.../apache/geode/perftest/jvms/rmi/Controller.java | 5 +++-
.../geode/perftest/yardstick/YardstickTask.java | 3 +--
.../geode/perftest/jvms/rmi/ChildJVMTest.java | 12 ---------
7 files changed, 34 insertions(+), 27 deletions(-)
diff --git a/geode-benchmarks/build.gradle b/geode-benchmarks/build.gradle
index af8b158..450878a 100644
--- a/geode-benchmarks/build.gradle
+++ b/geode-benchmarks/build.gradle
@@ -61,8 +61,6 @@ task benchmark(type: Test) {
exceptionFormat = 'full'
}
- exclude "**/*Function*Benchmark.class"
- exclude "**/PartitionedIndexQueryBenchmark.class"
forkEvery 1
systemProperty 'TEST_HOSTS', project.findProperty('hosts')
diff --git a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
index 6bea9e2..645e2b1 100644
--- a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
+++ b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
@@ -31,6 +31,8 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -52,6 +54,7 @@ public class SshInfrastructure implements Infrastructure {
private final String user;
private final int port;
public static final Config CONFIG = new DefaultConfig();
+ private ExecutorService streamReaderThreadPool = Executors.newCachedThreadPool();
public SshInfrastructure(Collection<String> hosts, String user) {
this(hosts, user, 22);
@@ -88,18 +91,36 @@ public class SshInfrastructure implements Infrastructure {
try (Session session = client.startSession()) {
logger.info("Executing " + script + " on " + node.getAddress());
final Session.Command cmd = session.exec(script);
- cmd.join();
- copyStream(cmd.getInputStream(), System.out);
- copyStream(cmd.getErrorStream(), System.err);
+ CompletableFuture<Void> copyStdout =
+ copyStreamAsynchronously(cmd.getInputStream(), System.out);
+ CompletableFuture<Void> copyStdErr =
+ copyStreamAsynchronously(cmd.getErrorStream(), System.err);
cmd.join();
+ copyStdout.join();
+ copyStdErr.join();
return cmd.getExitStatus();
}
}
}
- private void copyStream(InputStream inputStream, PrintStream out) throws IOException {
- org.apache.commons.io.IOUtils.copy(inputStream, out);
+ private CompletableFuture<Void> copyStreamAsynchronously(InputStream inputStream,
+ PrintStream out) {
+ return CompletableFuture.runAsync(() -> {
+ try {
+ org.apache.commons.io.IOUtils.copy(inputStream, out);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }, streamReaderThreadPool);
+ }
+
+ private void copyStream(InputStream inputStream, PrintStream out) {
+ try {
+ org.apache.commons.io.IOUtils.copy(inputStream, out);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
}
@Override
diff --git a/harness/src/main/java/org/apache/geode/perftest/jvms/JVMLauncher.java b/harness/src/main/java/org/apache/geode/perftest/jvms/JVMLauncher.java
index 9d3612b..077c2e4 100644
--- a/harness/src/main/java/org/apache/geode/perftest/jvms/JVMLauncher.java
+++ b/harness/src/main/java/org/apache/geode/perftest/jvms/JVMLauncher.java
@@ -56,6 +56,8 @@ class JVMLauncher {
public void run() {
try {
+ infra.onNode(jvmConfig.getNode(), new String[] {"rm", "-rf", jvmConfig.getOutputDir()});
+ infra.onNode(jvmConfig.getNode(), new String[] {"mkdir", "-p", jvmConfig.getOutputDir()});
int result = infra.onNode(jvmConfig.getNode(), shellCommand);
if (result != 0) {
logger.error("ChildJVM exited with error code " + result);
@@ -83,6 +85,7 @@ class JVMLauncher {
command.add("-D" + RemoteJVMFactory.RMI_PORT_PROPERTY + "=" + rmiPort);
command.add("-D" + RemoteJVMFactory.JVM_ID + "=" + jvmConfig.getId());
command.add("-D" + RemoteJVMFactory.OUTPUT_DIR + "=" + jvmConfig.getOutputDir());
+ command.add("-Xloggc:" + jvmConfig.getOutputDir() + "/gc.log");
command.addAll(jvmConfig.getJvmArgs());
command.add(ChildJVM.class.getName());
diff --git a/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/ChildJVM.java b/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/ChildJVM.java
index 5603cb0..ff3b94e 100644
--- a/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/ChildJVM.java
+++ b/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/ChildJVM.java
@@ -21,8 +21,6 @@ import java.io.File;
import java.io.PrintStream;
import java.rmi.RemoteException;
-import org.apache.commons.io.FileUtils;
-
import org.apache.geode.perftest.jdk.RMI;
import org.apache.geode.perftest.jdk.SystemInterface;
import org.apache.geode.perftest.jvms.RemoteJVMFactory;
@@ -62,9 +60,6 @@ public class ChildJVM {
}
File outputDir = new File(OUTPUT_DIR);
- // Clean up the output directory before the test runs
- FileUtils.deleteQuietly(outputDir);
- outputDir.mkdirs();
try (PrintStream out = new PrintStream(new File(outputDir, "system.log"))) {
system.setOut(out);
system.setErr(out);
diff --git a/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/Controller.java b/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/Controller.java
index bd7e0fa..ea4bef2 100644
--- a/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/Controller.java
+++ b/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/Controller.java
@@ -25,6 +25,8 @@ import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.geode.perftest.Task;
@@ -39,6 +41,7 @@ public class Controller extends UnicastRemoteObject implements ControllerRemote
private final Map<Integer, WorkerRemote> workers = new ConcurrentHashMap<>();
private final CountDownLatch workersStarted;
private volatile boolean isClosed;
+ private ExecutorService workerExecutionPool = Executors.newCachedThreadPool();
Controller(int numWorkers, Registry registry, SharedContext context) throws RemoteException {
@@ -85,6 +88,6 @@ public class Controller extends UnicastRemoteObject implements ControllerRemote
} catch (Exception e) {
throw new RuntimeException(e);
}
- });
+ }, workerExecutionPool);
}
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickTask.java b/harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickTask.java
index 5520117..763bddd 100644
--- a/harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickTask.java
+++ b/harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickTask.java
@@ -27,7 +27,6 @@ import org.yardstickframework.BenchmarkProbe;
import org.yardstickframework.impl.BenchmarkLoader;
import org.yardstickframework.impl.BenchmarkProbeSet;
import org.yardstickframework.impl.BenchmarkRunner;
-import org.yardstickframework.probes.DStatProbe;
import org.yardstickframework.probes.PercentileProbe;
import org.yardstickframework.probes.ThroughputLatencyProbe;
import org.yardstickframework.probes.VmStatProbe;
@@ -95,7 +94,7 @@ public class YardstickTask implements Task {
Collection<BenchmarkProbe> probes =
Arrays.asList(new HdrHistogramProbe(new HdrHistogramWriter(context.getOutputDir())),
new ThroughputLatencyProbe(),
- new PercentileProbe(), new DStatProbe(), new VmStatProbe(),
+ new PercentileProbe(), new VmStatProbe(),
testDoneProbe);
BenchmarkLoader loader = new BenchmarkLoader();
loader.initialize(cfg);
diff --git a/harness/src/test/java/org/apache/geode/perftest/jvms/rmi/ChildJVMTest.java b/harness/src/test/java/org/apache/geode/perftest/jvms/rmi/ChildJVMTest.java
index 9282f6f..7fb113b 100644
--- a/harness/src/test/java/org/apache/geode/perftest/jvms/rmi/ChildJVMTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/jvms/rmi/ChildJVMTest.java
@@ -20,7 +20,6 @@ package org.apache.geode.perftest.jvms.rmi;
import static org.apache.geode.perftest.jvms.RemoteJVMFactory.OUTPUT_DIR;
import static org.apache.geode.perftest.jvms.RemoteJVMFactory.RMI_HOST;
import static org.apache.geode.perftest.jvms.RemoteJVMFactory.RMI_PORT_PROPERTY;
-import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
@@ -83,15 +82,4 @@ public class ChildJVMTest {
verify(controller, times(3)).ping();
}
-
- @Test
- public void childCleansOutputDir() throws IOException {
- File expectedFile = new File(folder, "somefile.txt");
- expectedFile.createNewFile();
-
- jvm.run();
-
- assertFalse(expectedFile.exists());
- }
-
}