You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by iv...@apache.org on 2019/03/01 10:59:09 UTC
[bookkeeper] 01/01: A tools to search and replace bookie ids in
ledger metadata
This is an automated email from the ASF dual-hosted git repository.
ivank pushed a commit to branch bk-replace-bookieid
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 0af5f5a5f517fcc07af55c447fd36590dca4fbbe
Author: Ivan Kelly <iv...@apache.org>
AuthorDate: Fri Mar 1 11:58:02 2019 +0100
A tools to search and replace bookie ids in ledger metadata
To use:
```
bin/bkctl bookieid searchreplace --from <from> --to <to>
```
---
.../bookkeeper/tests/integration/TestCLI.java | 49 ++++++++
.../bookkeeper/tests/integration/TestSmoke.java | 8 +-
.../tools/cli/commands/BookieIdCommandGroup.java | 48 ++++++++
.../bookieid/SearchReplaceBookieIdCommand.java | 128 +++++++++++++++++++++
.../tools/cli/commands/bookieid/package-info.java | 22 ++++
tools/ledger/src/main/resources/commands | 1 +
6 files changed, 252 insertions(+), 4 deletions(-)
diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java
index f31beef..781d28e 100644
--- a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java
+++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java
@@ -19,9 +19,15 @@
package org.apache.bookkeeper.tests.integration;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import com.github.dockerjava.api.DockerClient;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.client.api.WriteHandle;
+import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils;
import org.apache.bookkeeper.tests.integration.utils.DockerUtils;
import org.jboss.arquillian.junit.Arquillian;
@@ -102,4 +108,47 @@ public class TestCLI {
).contains("ReadWrite Bookies :"));
}
+ @Test
+ public void test004_SearchReplaceBookieId() throws Exception {
+ String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker);
+
+ String bookie = BookKeeperClusterUtils.getAnyBookie();
+ int numEntries = 100;
+ try (BookKeeper bk = new BookKeeper(zookeeper)) {
+ long ledgerId;
+ BookieSocketAddress toReplace;
+ BookieSocketAddress replaceWith = new BookieSocketAddress("192.0.2.1:3181");
+ try (WriteHandle writelh = bk.newCreateLedgerOp()
+ .withDigestType(DigestType.CRC32C).withPassword(TestSmoke.PASSWD)
+ .withEnsembleSize(1).withWriteQuorumSize(1).withAckQuorumSize(1).execute().get()) {
+ ledgerId = writelh.getId();
+ toReplace = writelh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+ for (int i = 0; i < numEntries; i++) {
+ writelh.append(("entry-" + i).getBytes());
+ }
+ }
+
+ TestSmoke.readEntries(bk, ledgerId, numEntries);
+
+ DockerUtils.runCommand(docker, bookie,
+ bkctl,
+ "bookieid", "searchreplace",
+ "--from", toReplace.toString(),
+ "--to", replaceWith.toString());
+
+ try {
+ TestSmoke.readEntries(bk, ledgerId, numEntries);
+ fail("Shouldn't be able to read, as bookie id is rubbish");
+ } catch (BKException.BKBookieHandleNotAvailableException e) {
+ // expected
+ }
+
+ DockerUtils.runCommand(docker, bookie,
+ bkctl,
+ "bookieid", "searchreplace",
+ "--from", replaceWith.toString(),
+ "--to", toReplace.toString());
+ TestSmoke.readEntries(bk, ledgerId, numEntries);
+ }
+ }
}
diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java
index 2662b18..1efa63d 100644
--- a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java
+++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java
@@ -57,7 +57,7 @@ import org.junit.runners.MethodSorters;
@RunWith(Arquillian.class)
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class TestSmoke {
- private static final byte[] PASSWD = "foobar".getBytes();
+ static final byte[] PASSWD = "foobar".getBytes();
@ArquillianResource
DockerClient docker;
@@ -112,9 +112,9 @@ public class TestSmoke {
}
}
- private static void readEntries(BookKeeper bk,
- long ledgerId,
- int numExpectedEntries) throws Exception {
+ static void readEntries(BookKeeper bk,
+ long ledgerId,
+ int numExpectedEntries) throws Exception {
try (LedgerHandle readlh = bk.openLedger(ledgerId, BookKeeper.DigestType.CRC32C, PASSWD)) {
long lac = readlh.getLastAddConfirmed();
int i = 0;
diff --git a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieIdCommandGroup.java b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieIdCommandGroup.java
new file mode 100644
index 0000000..f230d09
--- /dev/null
+++ b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieIdCommandGroup.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands;
+
+import static org.apache.bookkeeper.tools.common.BKCommandCategories.CATEGORY_INFRA_SERVICE;
+
+import org.apache.bookkeeper.tools.cli.BKCtl;
+import org.apache.bookkeeper.tools.cli.commands.bookieid.SearchReplaceBookieIdCommand;
+import org.apache.bookkeeper.tools.common.BKFlags;
+import org.apache.bookkeeper.tools.framework.CliCommandGroup;
+import org.apache.bookkeeper.tools.framework.CliSpec;
+
+/**
+ * Commands that operate on bookie IDs.
+ */
+public class BookieIdCommandGroup extends CliCommandGroup<BKFlags> {
+
+ private static final String NAME = "bookieid";
+ private static final String DESC = "Commands operating on bookie ids";
+
+ private static final CliSpec<BKFlags> spec = CliSpec.<BKFlags>newBuilder()
+ .withName(NAME)
+ .withDescription(DESC)
+ .withParent(BKCtl.NAME)
+ .withCategory(CATEGORY_INFRA_SERVICE)
+ .addCommand(new SearchReplaceBookieIdCommand())
+ .build();
+
+ public BookieIdCommandGroup() {
+ super(spec);
+ }
+}
diff --git a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/SearchReplaceBookieIdCommand.java b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/SearchReplaceBookieIdCommand.java
new file mode 100644
index 0000000..2e4f1e7
--- /dev/null
+++ b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/SearchReplaceBookieIdCommand.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookieid;
+
+import com.beust.jcommander.Parameter;
+import com.google.common.util.concurrent.RateLimiter;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.client.LedgerMetadataBuilder;
+import org.apache.bookkeeper.client.api.BookKeeper;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.net.BookieSocketAddress;
+import org.apache.bookkeeper.tools.cli.helpers.ClientCommand;
+import org.apache.bookkeeper.tools.framework.CliFlags;
+import org.apache.bookkeeper.tools.framework.CliSpec;
+import org.apache.bookkeeper.versioning.Versioned;
+
+/**
+ * Search and replace a bookie id in ledger metadata.
+ */
+public class SearchReplaceBookieIdCommand extends ClientCommand<SearchReplaceBookieIdCommand.Flags> {
+
+ private static final String NAME = "searchreplace";
+ private static final String DESC = "Search all ledgers for a bookie ID and replace";
+
+ /**
+ * Flags for replace bookie id.
+ */
+ @Accessors(fluent = true)
+ @Setter
+ public static class Flags extends CliFlags {
+
+ @Parameter(names = { "-f", "--from" }, description = "Bookie ID to search for", required = true)
+ private String from;
+ @Parameter(names = { "-t", "--to" }, description = "Bookie ID to replace with", required = true)
+ private String to;
+ @Parameter(names = { "-m", "--max" }, description = "Maximum number of replacements to make")
+ private long max = Long.MAX_VALUE;
+ @Parameter(names = { "-r", "--rate" }, description = "Rate limit (updates per second)")
+ private int rate = Integer.MAX_VALUE;
+ @Parameter(names = { "--dry-run" }, description = "Don't actually write anything")
+ private boolean dryRun = false;
+ @Parameter(names = { "-v", "--verbose" }, description = "Verbose output")
+ private boolean verbose = false;
+ }
+
+ public SearchReplaceBookieIdCommand() {
+ this(new Flags());
+ }
+
+ public SearchReplaceBookieIdCommand(Flags flags) {
+ super(CliSpec.<Flags>newBuilder()
+ .withName(NAME)
+ .withDescription(DESC)
+ .withFlags(flags)
+ .build());
+ }
+
+ @Override
+ protected void run(BookKeeper bk, Flags flags) throws Exception {
+ BookKeeperAdmin admin = new BookKeeperAdmin((org.apache.bookkeeper.client.BookKeeper) bk);
+ LedgerManager ledgerManager = ((org.apache.bookkeeper.client.BookKeeper) bk).getLedgerManager();
+ long i = 0;
+
+ BookieSocketAddress fromAddr = new BookieSocketAddress(flags.from);
+ BookieSocketAddress toAddr = new BookieSocketAddress(flags.to);
+ System.out.println(String.format("Replacing bookie id %s with %s in metadata", fromAddr, toAddr));
+ RateLimiter limiter = RateLimiter.create(flags.rate);
+ for (Long lid : admin.listLedgers()) {
+ Versioned<LedgerMetadata> md = ledgerManager.readLedgerMetadata(lid).get();
+ if (md.getValue().getAllEnsembles().entrySet().stream().anyMatch(e -> e.getValue().contains(fromAddr))) {
+ limiter.acquire();
+
+ LedgerMetadataBuilder builder = LedgerMetadataBuilder.from(md.getValue());
+ md.getValue().getAllEnsembles().entrySet().stream()
+ .filter(e -> e.getValue().contains(fromAddr))
+ .forEach(e -> {
+ List<BookieSocketAddress> ensemble = new ArrayList<>(e.getValue());
+ ensemble.replaceAll((a) -> {
+ if (a.equals(fromAddr)) {
+ return toAddr;
+ } else {
+ return a;
+ }
+ });
+ builder.replaceEnsembleEntry(e.getKey(), ensemble);
+ });
+ LedgerMetadata newMeta = builder.build();
+ if (flags.verbose) {
+ System.out.println("Replacing ledger " + lid + " metadata ...");
+ System.out.println(md.getValue().toString());
+ System.out.println("with ...");
+ System.out.println(newMeta.toString());
+ }
+ i++;
+ if (!flags.dryRun) {
+ ledgerManager.writeLedgerMetadata(lid, newMeta, md.getVersion()).get();
+ }
+ }
+ if (i >= flags.max) {
+ System.out.println("Max number of ledgers processed, exiting");
+ break;
+ }
+ }
+ System.out.println("Replaced bookie ID in " + i + " ledgers");
+ }
+}
diff --git a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/package-info.java b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/package-info.java
new file mode 100644
index 0000000..131d672
--- /dev/null
+++ b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookieid/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * CLI commands for working with Bookie IDs.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookieid;
diff --git a/tools/ledger/src/main/resources/commands b/tools/ledger/src/main/resources/commands
index 7ea146b..c9af7a1 100644
--- a/tools/ledger/src/main/resources/commands
+++ b/tools/ledger/src/main/resources/commands
@@ -17,6 +17,7 @@
#
org.apache.bookkeeper.tools.cli.commands.BookieCommandGroup
+org.apache.bookkeeper.tools.cli.commands.BookieIdCommandGroup
org.apache.bookkeeper.tools.cli.commands.BookiesCommandGroup
org.apache.bookkeeper.tools.cli.commands.CookieCommandGroup
org.apache.bookkeeper.tools.cli.commands.LedgerCommandGroup