You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by nn...@apache.org on 2018/12/04 21:33:07 UTC
[geode-benchmarks] 01/01: Changed the way lib directories are
created.
This is an automated email from the ASF dual-hosted git repository.
nnag pushed a commit to branch feature/libPath
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git
commit e146df24a5dc81367db7edb4227c27b8c8e3d1d9
Author: nabarun <na...@cs.wisc.edu>
AuthorDate: Tue Dec 4 13:31:54 2018 -0800
Changed the way lib directories are created.
---
.../apache/geode/benchmark/parameters/JVMParameters.java | 1 +
.../geode/perftest/infrastructure/Infrastructure.java | 6 ++++--
.../perftest/infrastructure/local/LocalInfrastructure.java | 5 ++++-
.../perftest/infrastructure/ssh/SshInfrastructure.java | 10 ++++++----
.../java/org/apache/geode/perftest/jvms/JVMLauncher.java | 12 ++++++------
.../java/org/apache/geode/perftest/jvms/JVMMapping.java | 4 ++++
.../org/apache/geode/perftest/jvms/RemoteJVMFactory.java | 14 ++++++++++----
.../geode/perftest/jvms/classpath/ClassPathCopier.java | 6 ++++--
.../infrastructure/local/LocalInfrastructureTest.java | 2 +-
.../perftest/infrastructure/ssh/SshInfrastructureTest.java | 4 ++--
.../apache/geode/perftest/jvms/RemoteJVMFactoryTest.java | 2 +-
11 files changed, 43 insertions(+), 23 deletions(-)
diff --git a/geode-benchmarks/src/main/java/org/apache/geode/benchmark/parameters/JVMParameters.java b/geode-benchmarks/src/main/java/org/apache/geode/benchmark/parameters/JVMParameters.java
index bf08429..4d39259 100644
--- a/geode-benchmarks/src/main/java/org/apache/geode/benchmark/parameters/JVMParameters.java
+++ b/geode-benchmarks/src/main/java/org/apache/geode/benchmark/parameters/JVMParameters.java
@@ -40,6 +40,7 @@ public class JVMParameters {
"-Dsun.rmi.dgc.server.gcInterval=9223372036854775806",
"-Dgemfire.OSProcess.ENABLE_OUTPUT_REDIRECTION=true",
"-Dgemfire.launcher.registerSignalHandlers=true"
+
};
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/infrastructure/Infrastructure.java b/harness/src/main/java/org/apache/geode/perftest/infrastructure/Infrastructure.java
index 6e93956..38a7b48 100644
--- a/harness/src/main/java/org/apache/geode/perftest/infrastructure/Infrastructure.java
+++ b/harness/src/main/java/org/apache/geode/perftest/infrastructure/Infrastructure.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.Serializable;
import java.net.InetAddress;
import java.util.Set;
+import java.util.function.Function;
/**
* Some deployed infrastructure that the test is running on
@@ -46,11 +47,12 @@ public interface Infrastructure extends AutoCloseable {
* Copy a list of files to a directory on the node.
*
* @param files A list of files on the local system to copy
- * @param destDir The directory on the remote machine to copy to
+ * @param destDirectoryFunction A function that returns the destination directory, given a node
* @param removeExisting If true, remove all existing files in the directory on the remote
* machine
*/
- void copyToNodes(Iterable<File> files, String destDir, boolean removeExisting) throws IOException;
+ void copyToNodes(Iterable<File> files, Function<Node, String> destDirectoryFunction,
+ boolean removeExisting) throws IOException;
void copyFromNode(Node node, String directory, File destDir) throws IOException;
diff --git a/harness/src/main/java/org/apache/geode/perftest/infrastructure/local/LocalInfrastructure.java b/harness/src/main/java/org/apache/geode/perftest/infrastructure/local/LocalInfrastructure.java
index 7f5ff28..904ccc1 100644
--- a/harness/src/main/java/org/apache/geode/perftest/infrastructure/local/LocalInfrastructure.java
+++ b/harness/src/main/java/org/apache/geode/perftest/infrastructure/local/LocalInfrastructure.java
@@ -29,6 +29,7 @@ import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
+import java.util.function.Function;
import org.apache.commons.io.FileUtils;
@@ -85,9 +86,11 @@ public class LocalInfrastructure implements Infrastructure {
}
@Override
- public void copyToNodes(Iterable<File> files, String destDirName, boolean removeExisting)
+ public void copyToNodes(Iterable<File> files, Function<Node, String> destDirFunction,
+ boolean removeExisting)
throws IOException {
for (LocalNode node : nodes) {
+ String destDirName = destDirFunction.apply(node);
Path destDir = new File(node.getWorkingDir(), destDirName).toPath();
destDir.toFile().mkdirs();
diff --git a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
index 4da8ba5..6bea9e2 100644
--- a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
+++ b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
@@ -31,6 +31,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.function.Function;
import java.util.stream.Collectors;
import net.schmizz.sshj.Config;
@@ -107,14 +108,15 @@ public class SshInfrastructure implements Infrastructure {
}
@Override
- public void copyToNodes(Iterable<File> files, String destDir, boolean removeExisting)
+ public void copyToNodes(Iterable<File> files, Function<Node, String> destDirFunction,
+ boolean removeExisting)
throws IOException {
- Set<InetAddress> uniqueNodes =
- getNodes().stream().map(Node::getAddress).collect(Collectors.toSet());
List<CompletableFuture<Void>> futures = new ArrayList<>();
- for (InetAddress address : uniqueNodes) {
+ for (Node node : this.getNodes()) {
futures.add(CompletableFuture.runAsync(() -> {
+ InetAddress address = node.getAddress();
+ String destDir = destDirFunction.apply(node);
try (SSHClient client = getSSHClient(address)) {
client.useCompression();
diff --git a/harness/src/main/java/org/apache/geode/perftest/jvms/JVMLauncher.java b/harness/src/main/java/org/apache/geode/perftest/jvms/JVMLauncher.java
index 106aa30..9d3612b 100644
--- a/harness/src/main/java/org/apache/geode/perftest/jvms/JVMLauncher.java
+++ b/harness/src/main/java/org/apache/geode/perftest/jvms/JVMLauncher.java
@@ -35,21 +35,21 @@ class JVMLauncher {
JVMLauncher() {}
CompletableFuture<Void> launchProcesses(Infrastructure infra, int rmiPort,
- List<JVMMapping> mapping, String libDir)
+ List<JVMMapping> mapping)
throws UnknownHostException {
List<CompletableFuture<Void>> futures = new ArrayList<CompletableFuture<Void>>();
for (JVMMapping entry : mapping) {
- CompletableFuture<Void> future = launchWorker(infra, rmiPort, libDir, entry);
+ CompletableFuture<Void> future = launchWorker(infra, rmiPort, entry);
futures.add(future);
}
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
}
- CompletableFuture<Void> launchWorker(Infrastructure infra, int rmiPort, String libDir,
+ CompletableFuture<Void> launchWorker(Infrastructure infra, int rmiPort,
JVMMapping jvmConfig)
throws UnknownHostException {
String[] shellCommand =
- buildCommand(InetAddress.getLocalHost().getHostAddress(), rmiPort, libDir, jvmConfig);
+ buildCommand(InetAddress.getLocalHost().getHostAddress(), rmiPort, jvmConfig);
CompletableFuture<Void> future = new CompletableFuture<Void>();
Thread thread = new Thread("Worker " + jvmConfig.getNode().getAddress()) {
@@ -73,12 +73,12 @@ class JVMLauncher {
return future;
}
- String[] buildCommand(String rmiHost, int rmiPort, String libDir, JVMMapping jvmConfig) {
+ String[] buildCommand(String rmiHost, int rmiPort, JVMMapping jvmConfig) {
List<String> command = new ArrayList<String>();
command.add("java");
command.add("-classpath");
- command.add(libDir + "/*");
+ command.add(jvmConfig.getLibDir() + "/*");
command.add("-D" + RemoteJVMFactory.RMI_HOST + "=" + rmiHost);
command.add("-D" + RemoteJVMFactory.RMI_PORT_PROPERTY + "=" + rmiPort);
command.add("-D" + RemoteJVMFactory.JVM_ID + "=" + jvmConfig.getId());
diff --git a/harness/src/main/java/org/apache/geode/perftest/jvms/JVMMapping.java b/harness/src/main/java/org/apache/geode/perftest/jvms/JVMMapping.java
index 0b958fc..3f1dc62 100644
--- a/harness/src/main/java/org/apache/geode/perftest/jvms/JVMMapping.java
+++ b/harness/src/main/java/org/apache/geode/perftest/jvms/JVMMapping.java
@@ -55,4 +55,8 @@ public class JVMMapping implements Serializable {
public List<String> getJvmArgs() {
return jvmArgs;
}
+
+ public String getLibDir() {
+ return ".geode-performance/lib/" + role + "-" + id;
+ }
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/jvms/RemoteJVMFactory.java b/harness/src/main/java/org/apache/geode/perftest/jvms/RemoteJVMFactory.java
index 1ca4e81..fe0bc52 100644
--- a/harness/src/main/java/org/apache/geode/perftest/jvms/RemoteJVMFactory.java
+++ b/harness/src/main/java/org/apache/geode/perftest/jvms/RemoteJVMFactory.java
@@ -51,7 +51,6 @@ public class RemoteJVMFactory {
public static final int RMI_PORT = 33333;
public static final String CLASSPATH = System.getProperty("java.class.path");
public static final String JAVA_HOME = System.getProperty("java.home");
- private static final String LIB_DIR = ".geode-performance/lib";
private final JVMLauncher jvmLauncher;
private final ClassPathCopier classPathCopier;
private final ControllerFactory controllerFactory;
@@ -98,10 +97,9 @@ public class RemoteJVMFactory {
Controller controller =
controllerFactory.createController(new SharedContext(mapping), numWorkers);
- classPathCopier.copyToNodes(infra, LIB_DIR);
+ classPathCopier.copyToNodes(infra, node -> getLibDir(mapping, node));
- CompletableFuture<Void> processesExited = jvmLauncher.launchProcesses(infra, RMI_PORT, mapping,
- LIB_DIR);
+ CompletableFuture<Void> processesExited = jvmLauncher.launchProcesses(infra, RMI_PORT, mapping);
if (!controller.waitForWorkers(5, TimeUnit.MINUTES)) {
throw new IllegalStateException("Workers failed to start in 1 minute");
@@ -110,6 +108,14 @@ public class RemoteJVMFactory {
return new RemoteJVMs(infra, mapping, controller, processesExited);
}
+ private String getLibDir(List<JVMMapping> mapping, Infrastructure.Node node) {
+ return mapping.stream()
+ .filter(entry -> entry.getNode().equals(node))
+ .findFirst()
+ .orElseThrow(() -> new IllegalStateException("Could not find lib dir for node " + node))
+ .getLibDir();
+ }
+
public InfrastructureFactory getInfrastructureFactory() {
return infrastructureFactory;
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/jvms/classpath/ClassPathCopier.java b/harness/src/main/java/org/apache/geode/perftest/jvms/classpath/ClassPathCopier.java
index 8f5b6b8..c41cb0f 100644
--- a/harness/src/main/java/org/apache/geode/perftest/jvms/classpath/ClassPathCopier.java
+++ b/harness/src/main/java/org/apache/geode/perftest/jvms/classpath/ClassPathCopier.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Arrays;
+import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.geode.perftest.infrastructure.Infrastructure;
@@ -42,7 +43,8 @@ public class ClassPathCopier {
/**
* Copy the current classpath to a lib directory on all of the nodes in the infrastructure
*/
- public void copyToNodes(Infrastructure infrastructure, String destDir) throws IOException {
+ public void copyToNodes(Infrastructure infrastructure,
+ Function<Infrastructure.Node, String> destDirFunction) throws IOException {
String[] fileArray = classpath.split(File.pathSeparator);
Iterable<File> files = Arrays.asList(fileArray)
@@ -53,7 +55,7 @@ public class ClassPathCopier {
.filter(File::exists)
.collect(Collectors.toSet());
- infrastructure.copyToNodes(files, destDir, true);
+ infrastructure.copyToNodes(files, destDirFunction, true);
}
private File jarDir(File file) {
diff --git a/harness/src/test/java/org/apache/geode/perftest/infrastructure/local/LocalInfrastructureTest.java b/harness/src/test/java/org/apache/geode/perftest/infrastructure/local/LocalInfrastructureTest.java
index af222cf..9dc0427 100644
--- a/harness/src/test/java/org/apache/geode/perftest/infrastructure/local/LocalInfrastructureTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/infrastructure/local/LocalInfrastructureTest.java
@@ -59,7 +59,7 @@ public class LocalInfrastructureTest {
File expectedDir = new File(nodedir, "lib");
assertFalse(expectedDir.exists());
- infra.copyToNodes(Arrays.asList(someFile), "lib", true);
+ infra.copyToNodes(Arrays.asList(someFile), node -> "lib", true);
assertTrue(expectedDir.exists());
assertTrue(new File(expectedDir, someFile.getName()).exists());
diff --git a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
index ca6ad19..b983c5b 100644
--- a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
@@ -75,7 +75,7 @@ public class SshInfrastructureTest {
assertFalse(targetFolder.exists());
- infra.copyToNodes(Arrays.asList(someFile), targetFolder.getPath(), false);
+ infra.copyToNodes(Arrays.asList(someFile), node -> targetFolder.getPath(), false);
assertTrue(targetFolder.exists());
assertTrue(new File(targetFolder, someFile.getName()).exists());
@@ -93,7 +93,7 @@ public class SshInfrastructureTest {
fileToRemove.createNewFile();
assertTrue(fileToRemove.exists());
- infra.copyToNodes(Arrays.asList(someFile), targetFolder.getPath(), true);
+ infra.copyToNodes(Arrays.asList(someFile), node -> targetFolder.getPath(), true);
assertTrue(targetFolder.exists());
assertTrue(new File(targetFolder, someFile.getName()).exists());
diff --git a/harness/src/test/java/org/apache/geode/perftest/jvms/RemoteJVMFactoryTest.java b/harness/src/test/java/org/apache/geode/perftest/jvms/RemoteJVMFactoryTest.java
index 883526b..3539c91 100644
--- a/harness/src/test/java/org/apache/geode/perftest/jvms/RemoteJVMFactoryTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/jvms/RemoteJVMFactoryTest.java
@@ -79,7 +79,7 @@ public class RemoteJVMFactoryTest {
InOrder inOrder = inOrder(controller, controllerFactory, jvmLauncher, classPathCopier, infra);
inOrder.verify(controllerFactory).createController(any(), eq(2));
- inOrder.verify(jvmLauncher).launchProcesses(eq(infra), anyInt(), any(), any());
+ inOrder.verify(jvmLauncher).launchProcesses(eq(infra), anyInt(), any());
inOrder.verify(controller).waitForWorkers(anyInt(), any());