You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by eo...@apache.org on 2019/07/12 08:02:01 UTC
[bookkeeper] branch master updated: Implement a method to get all
the Bookies
This is an automated email from the ASF dual-hosted git repository.
eolivelli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 762a613 Implement a method to get all the Bookies
762a613 is described below
commit 762a61365408d81a20193ef4ae1d44394746c105
Author: Matteo Minardi <mi...@hotmail.it>
AuthorDate: Fri Jul 12 10:01:55 2019 +0200
Implement a method to get all the Bookies
In this PR I'm adding a `BokKeeperAdmin#getAllBookies` to retrive all the bookies (cookie) retriving them from the `RegistrationClient`
This is still a work in progress, I'm adding some tests in the next commit.
Reviewers: Enrico Olivelli <eo...@gmail.com>, Sijie Guo <si...@apache.org>
This closes #2117 from mino181295/allbookies
---
.../org/apache/bookkeeper/bookie/BookieShell.java | 22 ++++++++++---
.../apache/bookkeeper/client/BookKeeperAdmin.java | 10 ++++++
.../apache/bookkeeper/client/BookieWatcher.java | 1 +
.../bookkeeper/client/BookieWatcherImpl.java | 10 ++++++
.../bookkeeper/discover/RegistrationClient.java | 7 ++++
.../bookkeeper/discover/ZKRegistrationClient.java | 11 +++++--
.../cli/commands/bookies/ListBookiesCommand.java | 15 ++++++++-
.../apache/bookkeeper/bookie/BookieShellTest.java | 17 ++++++++++
.../bookkeeper/client/BookKeeperAdminTest.java | 38 ++++++++++++++++++++++
.../discover/MockRegistrationClient.java | 8 +++++
.../discover/TestZkRegistrationClient.java | 38 ++++++++++++++++++++++
.../metadata/etcd/EtcdRegistrationClient.java | 6 ++++
12 files changed, 176 insertions(+), 7 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 1caa1e1..9849052 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -1104,24 +1104,38 @@ public class BookieShell implements Tool {
super(CMD_LISTBOOKIES);
opts.addOption("rw", "readwrite", false, "Print readwrite bookies");
opts.addOption("ro", "readonly", false, "Print readonly bookies");
+ opts.addOption("a", "all", false, "Print all bookies");
// @deprecated 'rw'/'ro' option print both hostname and ip, so this option is not needed anymore
opts.addOption("h", "hostnames", false, "Also print hostname of the bookie");
}
@Override
public int runCmd(CommandLine cmdLine) throws Exception {
+ int passedCommands = 0;
+
boolean readwrite = cmdLine.hasOption("rw");
+ if (readwrite) {
+ passedCommands++;
+ }
boolean readonly = cmdLine.hasOption("ro");
+ if (readonly) {
+ passedCommands++;
+ }
+ boolean all = cmdLine.hasOption("a");
+ if (all) {
+ passedCommands++;
+ }
- if ((!readwrite && !readonly) || (readwrite && readonly)) {
- LOG.error("One and only one of -readwrite and -readonly must be specified");
+ if (passedCommands != 1) {
+ LOG.error("One and only one of -readwrite, -readonly and -all must be specified");
printUsage();
return 1;
}
ListBookiesCommand.Flags flags = new ListBookiesCommand.Flags()
.readwrite(readwrite)
- .readonly(readonly);
+ .readonly(readonly)
+ .all(all);
ListBookiesCommand command = new ListBookiesCommand(flags);
@@ -1136,7 +1150,7 @@ public class BookieShell implements Tool {
@Override
String getUsage() {
- return "listbookies [-readwrite|-readonly] [-hostnames]";
+ return "listbookies [-readwrite|-readonly|-all] [-hostnames]";
}
@Override
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
index 5a5903d..870db93 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
@@ -215,6 +215,16 @@ public class BookKeeperAdmin implements AutoCloseable {
}
/**
+ * Get a list of all bookies including the not available ones.
+ *
+ * @return a collection of bookie addresses
+ */
+ public Collection<BookieSocketAddress> getAllBookies()
+ throws BKException {
+ return bkc.bookieWatcher.getAllBookies();
+ }
+
+ /**
* Get a list of readonly bookies synchronously.
*
* @return a collection of bookie addresses
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcher.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcher.java
index 0f760a9..b881569 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcher.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcher.java
@@ -26,6 +26,7 @@ import org.apache.bookkeeper.net.BookieSocketAddress;
interface BookieWatcher {
Set<BookieSocketAddress> getBookies() throws BKException;
+ Set<BookieSocketAddress> getAllBookies() throws BKException;
Set<BookieSocketAddress> getReadOnlyBookies() throws BKException;
/**
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java
index f26ee61..e38dcf6 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java
@@ -143,6 +143,16 @@ class BookieWatcherImpl implements BookieWatcher {
}
@Override
+ public Set<BookieSocketAddress> getAllBookies() throws BKException {
+ try {
+ return FutureUtils.result(registrationClient.getAllBookies(), EXCEPTION_FUNC).getValue();
+ } catch (BKInterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw ie;
+ }
+ }
+
+ @Override
public Set<BookieSocketAddress> getReadOnlyBookies()
throws BKException {
try {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/RegistrationClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/RegistrationClient.java
index b53fd24..4056329 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/RegistrationClient.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/RegistrationClient.java
@@ -52,6 +52,13 @@ public interface RegistrationClient extends AutoCloseable {
CompletableFuture<Versioned<Set<BookieSocketAddress>>> getWritableBookies();
/**
+ * Get the list of all bookies identifiers.
+ *
+ * @return a future represents the list of writable bookies.
+ */
+ CompletableFuture<Versioned<Set<BookieSocketAddress>>> getAllBookies();
+
+ /**
* Get the list of readonly bookie identifiers.
*
* @return a future represents the list of readonly bookies.
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/ZKRegistrationClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/ZKRegistrationClient.java
index ce19751..6629f0c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/ZKRegistrationClient.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/ZKRegistrationClient.java
@@ -19,6 +19,7 @@
package org.apache.bookkeeper.discover;
import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE;
+import static org.apache.bookkeeper.util.BookKeeperConstants.COOKIE_NODE;
import static org.apache.bookkeeper.util.BookKeeperConstants.READONLY;
import com.google.common.collect.Sets;
@@ -173,6 +174,7 @@ public class ZKRegistrationClient implements RegistrationClient {
// registration paths
private final String bookieRegistrationPath;
+ private final String bookieAllRegistrationPath;
private final String bookieReadonlyRegistrationPath;
public ZKRegistrationClient(ZooKeeper zk,
@@ -182,6 +184,7 @@ public class ZKRegistrationClient implements RegistrationClient {
this.scheduler = scheduler;
this.bookieRegistrationPath = ledgersRootPath + "/" + AVAILABLE_NODE;
+ this.bookieAllRegistrationPath = ledgersRootPath + "/" + COOKIE_NODE;
this.bookieReadonlyRegistrationPath = this.bookieRegistrationPath + "/" + READONLY;
}
@@ -200,6 +203,11 @@ public class ZKRegistrationClient implements RegistrationClient {
}
@Override
+ public CompletableFuture<Versioned<Set<BookieSocketAddress>>> getAllBookies() {
+ return getChildren(bookieAllRegistrationPath, null);
+ }
+
+ @Override
public CompletableFuture<Versioned<Set<BookieSocketAddress>>> getReadOnlyBookies() {
return getChildren(bookieReadonlyRegistrationPath, null);
}
@@ -209,8 +217,7 @@ public class ZKRegistrationClient implements RegistrationClient {
zk.getChildren(regPath, watcher, (rc, path, ctx, children, stat) -> {
if (Code.OK != rc) {
ZKException zke = new ZKException();
- zke.fillInStackTrace();
- future.completeExceptionally(zke);
+ future.completeExceptionally(zke.fillInStackTrace());
return;
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookies/ListBookiesCommand.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookies/ListBookiesCommand.java
index 4563530..33e1a2d 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookies/ListBookiesCommand.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookies/ListBookiesCommand.java
@@ -64,15 +64,18 @@ public class ListBookiesCommand extends DiscoveryCommand<Flags> {
private boolean readwrite = false;
@Parameter(names = { "-ro", "--readonly" }, description = "Print readonly bookies")
private boolean readonly = false;
+ @Parameter(names = { "-a", "--all" }, description = "Print all bookies")
+ private boolean all = false;
}
@Override
protected void run(RegistrationClient regClient, Flags flags) throws Exception {
- if (!flags.readwrite && !flags.readonly) {
+ if (!flags.readwrite && !flags.readonly && !flags.all) {
// case: no args is provided. list all the bookies by default.
flags.readwrite = true;
flags.readonly = true;
+ flags.all = true;
}
boolean hasBookies = false;
@@ -96,6 +99,16 @@ public class ListBookiesCommand extends DiscoveryCommand<Flags> {
hasBookies = true;
}
}
+ if (flags.all) {
+ Set<BookieSocketAddress> bookies = result(
+ regClient.getAllBookies()
+ ).getValue();
+ if (!bookies.isEmpty()) {
+ System.out.println("All Bookies :");
+ printBookies(bookies);
+ hasBookies = true;
+ }
+ }
if (!hasBookies) {
System.err.println("No bookie exists!");
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieShellTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieShellTest.java
index 9fcb2fb..b0e0a45 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieShellTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieShellTest.java
@@ -395,6 +395,7 @@ public class BookieShellTest {
.apply(same(shell.bkConf), same(mockListBookiesFlags));
verify(mockListBookiesFlags, times(1)).readonly(true);
verify(mockListBookiesFlags, times(1)).readwrite(false);
+ verify(mockListBookiesFlags, times(1)).all(false);
}
@Test
@@ -408,5 +409,21 @@ public class BookieShellTest {
.apply(same(shell.bkConf), same(mockListBookiesFlags));
verify(mockListBookiesFlags, times(1)).readonly(false);
verify(mockListBookiesFlags, times(1)).readwrite(true);
+ verify(mockListBookiesFlags, times(1)).all(false);
}
+
+ @Test
+ public void testListBookiesCmdAll() throws Exception {
+ assertEquals(0, shell.run(new String[] {
+ "listbookies", "-a"
+ }));
+ verifyNew(ListBookiesCommand.class, times(1))
+ .withArguments(same(mockListBookiesFlags));
+ verify(mockListBookiesCommand, times(1))
+ .apply(same(shell.bkConf), same(mockListBookiesFlags));
+ verify(mockListBookiesFlags, times(1)).readonly(false);
+ verify(mockListBookiesFlags, times(1)).readwrite(false);
+ verify(mockListBookiesFlags, times(1)).all(true);
+ }
+
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperAdminTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperAdminTest.java
index 2f419b5..03f182d 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperAdminTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperAdminTest.java
@@ -31,6 +31,7 @@ import static org.junit.Assert.fail;
import com.google.common.net.InetAddresses;
import java.io.File;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
@@ -44,6 +45,7 @@ import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.UnderreplicatedLedger;
import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager;
import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase;
+import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.proto.BookieServer;
import org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
@@ -497,4 +499,40 @@ public class BookKeeperAdminTest extends BookKeeperClusterTestCase {
}
bkc.close();
}
+
+ @Test
+ public void testGetBookies() throws Exception {
+ String ledgersRootPath = "/ledgers";
+ Assert.assertTrue("Cluster rootpath should have been created successfully " + ledgersRootPath,
+ (zkc.exists(ledgersRootPath, false) != null));
+ String bookieCookiePath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf)
+ + "/" + BookKeeperConstants.COOKIE_NODE;
+ Assert.assertTrue("AvailableBookiesPath should have been created successfully " + bookieCookiePath,
+ (zkc.exists(bookieCookiePath, false) != null));
+
+ try (BookKeeperAdmin bkAdmin = new BookKeeperAdmin(zkUtil.getZooKeeperConnectString())) {
+ Collection<BookieSocketAddress> availableBookies = bkAdmin.getAvailableBookies();
+ Assert.assertEquals(availableBookies.size(), bs.size());
+
+ for (int i = 0; i < bs.size(); i++) {
+ availableBookies.contains(bs.get(i).getLocalAddress());
+ }
+
+ BookieServer killedBookie = bs.get(1);
+ killBookieAndWaitForZK(1);
+
+ Collection<BookieSocketAddress> remainingBookies = bkAdmin.getAvailableBookies();
+ Assert.assertFalse(remainingBookies.contains(killedBookie.getLocalAddress()));
+
+ Collection<BookieSocketAddress> allBookies = bkAdmin.getAllBookies();
+ for (int i = 0; i < bs.size(); i++) {
+ remainingBookies.contains(bs.get(i).getLocalAddress());
+ allBookies.contains(bs.get(i).getLocalAddress());
+ }
+
+ Assert.assertEquals(remainingBookies.size(), allBookies.size() - 1);
+ Assert.assertTrue(allBookies.contains(killedBookie.getLocalAddress()));
+ }
+ }
+
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/discover/MockRegistrationClient.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/discover/MockRegistrationClient.java
index 8ce3686..9ba5ae4 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/discover/MockRegistrationClient.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/discover/MockRegistrationClient.java
@@ -38,6 +38,7 @@ public class MockRegistrationClient implements RegistrationClient {
final ExecutorService executor;
private long currentVersion = 0;
private Set<BookieSocketAddress> bookies = new HashSet<BookieSocketAddress>();
+ private Set<BookieSocketAddress> allBookies = new HashSet<BookieSocketAddress>();
private Set<BookieSocketAddress> readOnlyBookies = new HashSet<BookieSocketAddress>();
private Set<RegistrationListener> bookieWatchers = new HashSet<RegistrationListener>();
private Set<RegistrationListener> readOnlyBookieWatchers = new HashSet<RegistrationListener>();
@@ -115,6 +116,13 @@ public class MockRegistrationClient implements RegistrationClient {
}
@Override
+ public CompletableFuture<Versioned<Set<BookieSocketAddress>>> getAllBookies() {
+ CompletableFuture<Versioned<Set<BookieSocketAddress>>> promise = new CompletableFuture<>();
+ executor.submit(() -> promise.complete(versioned(allBookies, currentVersion)));
+ return promise;
+ }
+
+ @Override
public CompletableFuture<Versioned<Set<BookieSocketAddress>>> getReadOnlyBookies() {
CompletableFuture<Versioned<Set<BookieSocketAddress>>> promise = new CompletableFuture<>();
executor.submit(() -> promise.complete(versioned(readOnlyBookies, currentVersion)));
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/discover/TestZkRegistrationClient.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/discover/TestZkRegistrationClient.java
index 1f5e0b6..234a860 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/discover/TestZkRegistrationClient.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/discover/TestZkRegistrationClient.java
@@ -23,6 +23,7 @@ import static org.apache.bookkeeper.common.concurrent.FutureUtils.result;
import static org.apache.bookkeeper.common.testing.MoreAsserts.assertSetEquals;
import static org.apache.bookkeeper.discover.ZKRegistrationClient.ZK_CONNECT_BACKOFF_MS;
import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE;
+import static org.apache.bookkeeper.util.BookKeeperConstants.COOKIE_NODE;
import static org.apache.bookkeeper.util.BookKeeperConstants.READONLY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -88,6 +89,7 @@ public class TestZkRegistrationClient extends MockZooKeeperTestCase {
private String ledgersPath;
private String regPath;
+ private String regAllPath;
private String regReadonlyPath;
private ZKRegistrationClient zkRegistrationClient;
private ScheduledExecutorService mockExecutor;
@@ -100,6 +102,7 @@ public class TestZkRegistrationClient extends MockZooKeeperTestCase {
this.ledgersPath = "/" + runtime.getMethodName();
this.regPath = ledgersPath + "/" + AVAILABLE_NODE;
+ this.regAllPath = ledgersPath + "/" + COOKIE_NODE;
this.regReadonlyPath = regPath + "/" + READONLY;
this.mockExecutor = mock(ScheduledExecutorService.class);
this.controller = new MockExecutorController()
@@ -151,6 +154,27 @@ public class TestZkRegistrationClient extends MockZooKeeperTestCase {
}
@Test
+ public void testGetAllBookies() throws Exception {
+ Set<BookieSocketAddress> addresses = prepareNBookies(10);
+ List<String> children = Lists.newArrayList();
+ for (BookieSocketAddress address : addresses) {
+ children.add(address.toString());
+ }
+ Stat stat = mock(Stat.class);
+ when(stat.getCversion()).thenReturn(1234);
+ mockGetChildren(
+ regAllPath, false,
+ Code.OK.intValue(), children, stat);
+
+ Versioned<Set<BookieSocketAddress>> result =
+ result(zkRegistrationClient.getAllBookies());
+
+ assertEquals(new LongVersion(1234), result.getVersion());
+ assertSetEquals(
+ addresses, result.getValue());
+ }
+
+ @Test
public void testGetReadOnlyBookies() throws Exception {
Set<BookieSocketAddress> addresses = prepareNBookies(10);
List<String> children = Lists.newArrayList();
@@ -186,6 +210,20 @@ public class TestZkRegistrationClient extends MockZooKeeperTestCase {
}
@Test
+ public void testGetAllBookiesFailure() throws Exception {
+ mockGetChildren(
+ regAllPath, false,
+ Code.NONODE.intValue(), null, null);
+
+ try {
+ result(zkRegistrationClient.getAllBookies());
+ fail("Should fail to get all bookies");
+ } catch (ZKException zke) {
+ // expected to throw zookeeper exception
+ }
+ }
+
+ @Test
public void testGetReadOnlyBookiesFailure() throws Exception {
mockGetChildren(
regReadonlyPath, false,
diff --git a/metadata-drivers/etcd/src/main/java/org/apache/bookkeeper/metadata/etcd/EtcdRegistrationClient.java b/metadata-drivers/etcd/src/main/java/org/apache/bookkeeper/metadata/etcd/EtcdRegistrationClient.java
index f1e7715..76d731c 100644
--- a/metadata-drivers/etcd/src/main/java/org/apache/bookkeeper/metadata/etcd/EtcdRegistrationClient.java
+++ b/metadata-drivers/etcd/src/main/java/org/apache/bookkeeper/metadata/etcd/EtcdRegistrationClient.java
@@ -28,6 +28,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.function.Consumer;
import java.util.function.Function;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.common.concurrent.FutureUtils;
import org.apache.bookkeeper.discover.RegistrationClient;
import org.apache.bookkeeper.metadata.etcd.helpers.KeySetReader;
@@ -92,6 +93,11 @@ class EtcdRegistrationClient implements RegistrationClient {
}
@Override
+ public CompletableFuture<Versioned<Set<BookieSocketAddress>>> getAllBookies() {
+ return FutureUtils.exception(new BKException.BKIllegalOpException());
+ }
+
+ @Override
public CompletableFuture<Versioned<Set<BookieSocketAddress>>> getReadOnlyBookies() {
return readonlyBookiesReader.read();
}