You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by iv...@apache.org on 2018/01/19 19:11:30 UTC
[bookkeeper] branch master updated: Utilities for bookies running
in test containers
This is an automated email from the ASF dual-hosted git repository.
ivank pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 6dda0a6 Utilities for bookies running in test containers
6dda0a6 is described below
commit 6dda0a6c68fbaf2ca198cfbb693db4ac93a0feef
Author: Ivan Kelly <iv...@apache.org>
AuthorDate: Fri Jan 19 20:11:08 2018 +0100
Utilities for bookies running in test containers
The changes include:
- Updating configuration in bk_server.conf for one or all bookies
- Run commands on all or one bookie
- Wait for all bookies in the test are up
There are also improvements to DockerUtils#runCommand:
- Better logging
- Catches RunTime exceptions, as Docker-java throws them and it could
fail the test
- Throw an exception is the command exits with anything but 0
Master Issue: #903
Author: Ivan Kelly <iv...@apache.org>
Reviewers: Enrico Olivelli <eo...@gmail.com>, Sijie Guo <si...@apache.org>
This closes #1020 from ivankelly/bookie-util
---
.../bookkeeper/tests/BookKeeperClusterUtils.java | 41 +++++++++++++++++++++-
.../org/apache/bookkeeper/tests/DockerUtils.java | 27 +++++++++++---
2 files changed, 62 insertions(+), 6 deletions(-)
diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/BookKeeperClusterUtils.java
index d70d543..437e5b2 100644
--- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/BookKeeperClusterUtils.java
+++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/BookKeeperClusterUtils.java
@@ -23,6 +23,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import com.github.dockerjava.api.DockerClient;
import java.io.IOException;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -79,6 +80,38 @@ public class BookKeeperClusterUtils {
}
}
+ public static void updateBookieConf(DockerClient docker, String containerId,
+ String version, String key, String value) throws Exception {
+ String confFile = "/opt/bookkeeper/" + version + "/conf/bk_server.conf";
+ String sedProgram = String.format(
+ "/[[:blank:]]*%s[[:blank:]]*=/ { h; s!=.*!=%s!; }; ${x;/^$/ { s//%s=%s/;H; }; x}",
+ key, value, key, value);
+ DockerUtils.runCommand(docker, containerId, "sed", "-i", "-e", sedProgram, confFile);
+ }
+
+ public static void updateAllBookieConf(DockerClient docker, String version, String key, String value)
+ throws Exception {
+ for (String b : DockerUtils.cubeIdsMatching("bookkeeper")) {
+ updateBookieConf(docker, b, version, key, value);
+ }
+ }
+
+ public static boolean runOnAnyBookie(DockerClient docker, String... cmds) throws Exception {
+ Optional<String> bookie = DockerUtils.cubeIdsMatching("bookkeeper").stream().findAny();
+ if (bookie.isPresent()) {
+ DockerUtils.runCommand(docker, bookie.get(), cmds);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public static void runOnAllBookies(DockerClient docker, String... cmds) throws Exception {
+ for (String b : DockerUtils.cubeIdsMatching("bookkeeper")) {
+ DockerUtils.runCommand(docker, b, cmds);
+ }
+ }
+
private static boolean waitBookieState(DockerClient docker, String containerId,
int timeout, TimeUnit timeoutUnit,
boolean upOrDown) {
@@ -142,7 +175,7 @@ public class BookKeeperClusterUtils {
LOG.error("Exception stopping bookie", e);
return false;
}
- return waitBookieDown(docker, containerId, 10, TimeUnit.SECONDS);
+ return waitBookieDown(docker, containerId, 5, TimeUnit.SECONDS);
}
public static boolean stopAllBookies(DockerClient docker) {
@@ -150,4 +183,10 @@ public class BookKeeperClusterUtils {
.map((b) -> stopBookie(docker, b))
.reduce(true, BookKeeperClusterUtils::allTrue);
}
+
+ public static boolean waitAllBookieUp(DockerClient docker) {
+ return DockerUtils.cubeIdsMatching("bookkeeper").stream()
+ .map((b) -> waitBookieUp(docker, b, 10, TimeUnit.SECONDS))
+ .reduce(true, BookKeeperClusterUtils::allTrue);
+ }
}
diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/DockerUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/DockerUtils.java
index 7d183a1..11959af 100644
--- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/DockerUtils.java
+++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/DockerUtils.java
@@ -20,6 +20,7 @@ package org.apache.bookkeeper.tests;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.async.ResultCallback;
+import com.github.dockerjava.api.command.InspectExecResponse;
import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.ContainerNetwork;
@@ -28,10 +29,11 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.IOException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
+import java.util.Arrays;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -88,7 +90,7 @@ public class DockerUtils {
}
});
future.get();
- } catch (ExecutionException|IOException e) {
+ } catch (RuntimeException|ExecutionException|IOException e) {
LOG.error("Error dumping log for {}", containerId, e);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
@@ -116,7 +118,7 @@ public class DockerUtils {
}
entry = stream.getNextTarEntry();
}
- } catch (IOException e) {
+ } catch (RuntimeException|IOException e) {
LOG.error("Error reading bk logs from container {}", containerId, e);
}
}
@@ -132,17 +134,19 @@ public class DockerUtils {
public static void runCommand(DockerClient docker, String containerId, String... cmd) throws Exception {
CompletableFuture<Boolean> future = new CompletableFuture<>();
String execid = docker.execCreateCmd(containerId).withCmd(cmd).exec().getId();
+ String cmdString = Arrays.stream(cmd).collect(Collectors.joining(" "));
docker.execStartCmd(execid).withDetach(false).exec(new ResultCallback<Frame>() {
@Override
public void close() {}
@Override
public void onStart(Closeable closeable) {
+ LOG.info("DOCKER.exec({}:{}): Executing...", containerId, cmdString);
}
@Override
public void onNext(Frame object) {
- LOG.info("DOCKER.exec({}): {}", cmd, object);
+ LOG.info("DOCKER.exec({}:{}): {}", containerId, cmdString, object);
}
@Override
@@ -152,10 +156,23 @@ public class DockerUtils {
@Override
public void onComplete() {
+ LOG.info("DOCKER.exec({}:{}): Done", containerId, cmdString);
future.complete(true);
}
});
future.get();
+
+ InspectExecResponse resp = docker.inspectExecCmd(execid).exec();
+ while (resp.isRunning()) {
+ Thread.sleep(200);
+ resp = docker.inspectExecCmd(execid).exec();
+ }
+ int retCode = resp.getExitCode();
+ if (retCode != 0) {
+ throw new Exception(
+ String.format("cmd(%s) failed on %s with exitcode %d",
+ cmdString, containerId, retCode));
+ }
}
public static Set<String> cubeIdsMatching(String needle) {
--
To stop receiving notification emails like this one, please contact
['"commits@bookkeeper.apache.org" <co...@bookkeeper.apache.org>'].