You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@geode.apache.org by GitBox <gi...@apache.org> on 2018/11/26 22:46:50 UTC
[GitHub] upthewaterspout closed pull request #3: Use a Java ssh server
rather than docker for tests
upthewaterspout closed pull request #3: Use a Java ssh server rather than docker for tests
URL: https://github.com/apache/geode-benchmarks/pull/3
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.travis.yml b/.travis.yml
index 620a982..6fe694f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,10 +1,5 @@
-required: sudo
-
language: java
-services:
- - docker
-
jdk:
- openjdk8
@@ -18,13 +13,5 @@ cache:
- $HOME/.gradle/wrapper/
script:
- - ./gradlew buildTestingImage
- - docker run -p 2222:22 -d --name geode-test geode-performance-testing
- - ssh-keygen -N "" -f ./id_rsa
- - docker cp id_rsa geode-test:/root/.ssh/id_rsa
- - docker exec -it geode-test chown root:root /root/.ssh/id_rsa
- - docker cp id_rsa.pub geode-test:/root/.ssh/id_rsa.pub
- - docker exec -it geode-test chown root:root /root/.ssh/id_rsa.pub
- - docker cp id_rsa.pub geode-test:/root/.ssh/authorized_keys
- - docker exec -it geode-test chown root:root /root/.ssh/authorized_keys
- - ssh -o "StrictHostKeyChecking=no" -i id_rsa -p 2222 root@localhost "pushd /geode-performance; ./gradlew geode-benchmarks:test"
+ - ssh-keygen -N "" -f ~/.ssh/id_rsa
+ - ./gradlew check
diff --git a/harness/build.gradle b/harness/build.gradle
index 1658ecf..7b32cd7 100644
--- a/harness/build.gradle
+++ b/harness/build.gradle
@@ -36,4 +36,5 @@ dependencies {
testCompile group: 'org.mockito', name: 'mockito-all', version: '1.10.19'
testCompile group: 'org.awaitility', name: 'awaitility', version: '3.0.0'
testCompile group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.25'
+ testCompile group: 'org.apache.sshd', name: 'sshd-core', version: '2.1.0'
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
index 4ff90e4..bff36b1 100644
--- a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
+++ b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
@@ -49,20 +49,25 @@
private final Set<SshNode> hosts;
private final String user;
+ private final int port;
public static final Config CONFIG = new DefaultConfig();
public SshInfrastructure(Collection<String> hosts, String user) {
+ this(hosts, user, 22);
+ }
+
+ public SshInfrastructure(Collection<String> hosts, String user, int port) {
this.hosts = hosts.stream()
.map(SshNode::new)
.collect(Collectors.toCollection(LinkedHashSet::new));
this.user = user;
+ this.port = port;
}
SSHClient getSSHClient(InetAddress address) throws IOException {
SSHClient client = new SSHClient(CONFIG);
client.addHostKeyVerifier(new PromiscuousVerifier());
- client.loadKnownHosts();
- client.connect(address);
+ client.connect(address, port);
client.authPublickey(user);
return client;
}
@@ -109,19 +114,22 @@ public void copyToNodes(Iterable<File> files, String destDir, boolean removeExis
for(InetAddress address: uniqueNodes) {
futures.add(CompletableFuture.runAsync(() -> {
try (SSHClient client = getSSHClient(address)) {
- try (Session session = client.startSession()) {
client.useCompression();
if(removeExisting) {
- session.exec(String.format("/bin/sh -c \"rm -rf '%s'; mkdir -p '%s'\"", destDir, destDir)).join();
- }else {
+ try (Session session = client.startSession()) {
+ session.exec(String.format("rm -rf '%s'", destDir)).join();
+ }
+ }
+
+ try (Session session = client.startSession()) {
session.exec(String.format("mkdir -p '%s'", destDir)).join();
}
+
for (File file : files) {
logger.info("Copying " + file + " to " + address);
client.newSCPFileTransfer().upload(new FileSystemFile(file), destDir);
}
- }
} catch(IOException e) {
throw new UncheckedIOException(e);
}
@@ -133,14 +141,11 @@ public void copyToNodes(Iterable<File> files, String destDir, boolean removeExis
@Override
public void copyFromNode(Node node, String directory, File destDir) throws IOException {
try (SSHClient client = getSSHClient(node.getAddress())) {
-
- try (Session session = client.startSession()) {
client.useCompression();
destDir.mkdirs();
client.newSCPFileTransfer().download(directory, destDir.getPath());
return;
- }
}
}
diff --git a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
index 2936e80..c1041cf 100644
--- a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
@@ -34,14 +34,19 @@
import org.apache.geode.perftest.infrastructure.Infrastructure;
public class SshInfrastructureTest {
+
+
private static final Set<String> HOSTS = Collections.singleton("localhost");
private static final String USER = System.getProperty("user.name");
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ @Rule
+ public SshServerRule server = new SshServerRule();
+
@Test
public void canFindNodes() throws IOException {
- SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+ SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
assertEquals(1, infra.getNodes().size());
}
@@ -49,10 +54,11 @@ public void canFindNodes() throws IOException {
@Test
public void canExecuteACommandOnNode()
throws IOException {
- SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+ SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
Infrastructure.Node node1 = infra.getNodes().iterator().next();
File folder = temporaryFolder.newFolder();
+ folder.mkdirs();
File expectedFile = new File(folder, "somefile.txt").getAbsoluteFile();
int result = infra.onNode(node1, new String[] {"touch", expectedFile.getPath()}
);
@@ -63,7 +69,7 @@ public void canExecuteACommandOnNode()
@Test
public void copyToNodesPutsFileOnNode() throws IOException, InterruptedException {
- SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+ SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
File someFile = temporaryFolder.newFile();
File targetFolder = new File(temporaryFolder.newFolder(), "dest");
@@ -78,7 +84,7 @@ public void copyToNodesPutsFileOnNode() throws IOException, InterruptedException
@Test
public void copyToNodesCleansDirectory() throws IOException, InterruptedException {
- SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+ SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
File someFile = temporaryFolder.newFile();
File targetFolder = new File(temporaryFolder.newFolder(), "dest");
@@ -99,7 +105,7 @@ public void copyToNodesCleansDirectory() throws IOException, InterruptedExceptio
@Test
public void canCopyFilesFromANode()
throws IOException, ExecutionException, InterruptedException {
- SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+ SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
Infrastructure.Node node1 = infra.getNodes().iterator().next();
infra.onNode(node1, new String[] {"mkdir", "-p", "/tmp/foo"});
diff --git a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshServerRule.java b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshServerRule.java
new file mode 100644
index 0000000..dfc3ca7
--- /dev/null
+++ b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshServerRule.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.geode.perftest.infrastructure.ssh;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Paths;
+
+import org.apache.sshd.server.SshServer;
+import org.apache.sshd.server.command.Command;
+import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
+import org.apache.sshd.server.shell.ProcessShellCommandFactory;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Rule to run an in process ssh server during a test
+ *
+ * This ssh server listens on localhost. It does actually run commands and create
+ * files on the real filesystem. It accepts connections from any user.
+ */
+public class SshServerRule extends TemporaryFolder {
+
+ private SshServer sshd;
+
+ @Override
+ protected void before() throws Throwable {
+ super.before();
+ sshd = SshServer.setUpDefaultServer();
+ sshd.setPort(0);
+ sshd.setHost("localhost");
+ sshd.setPublickeyAuthenticator((username, key, session) -> true);
+ sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(Paths.get(newFolder().getPath(), "hostkey.ser")));
+ sshd.setCommandFactory(new UnescapingCommandFactory());
+ sshd.start();
+ }
+
+ public int getPort() {
+ return sshd.getPort();
+ }
+
+ @Override
+ protected void after() {
+ try {
+ sshd.stop();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private class UnescapingCommandFactory extends ProcessShellCommandFactory {
+ @Override
+ public Command createCommand(String command) {
+ return super.createCommand(command.replace("'", ""));
+ }
+ }
+}
diff --git a/images/geode-performance/Dockerfile b/images/geode-performance/Dockerfile
deleted file mode 100644
index 484c8fd..0000000
--- a/images/geode-performance/Dockerfile
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM ubuntu
-# This should be built from the top level of the repository.
-# docker build -t geode-performance-testing-image -f images/geode-performance/Dockerfile
-RUN apt update -y && \
- apt install -y openjdk-8-jdk-headless openssh-server
-COPY . /geode-performance
-RUN mkdir /var/run/sshd && \
- sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
- sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd && \
- ssh-keygen -N "" -f /root/.ssh/id_rsa && \
- echo "Host localhost" > /root/.ssh/config && \
- echo " StrictHostKeyChecking no" >> /root/.ssh/config && \
- chmod 600 /root/.ssh/config && \
- cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \
- chmod 600 /root/.ssh/authorized_keys && \
- rm -f /geode-performance/Dockerfile
-CMD ["/usr/sbin/sshd", "-D", "-E", "/var/log/sshd.log"]
-EXPOSE 22
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services