You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by zh...@apache.org on 2019/04/03 13:40:11 UTC
[bookkeeper] branch master updated: Migrate command
`localconsistencycheck`
This is an automated email from the ASF dual-hosted git repository.
zhaijia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 1d4cc71 Migrate command `localconsistencycheck`
1d4cc71 is described below
commit 1d4cc71fd77ab803a3ebec5a8e7f1b8f51de92c9
Author: Yong Zhang <zh...@gmail.com>
AuthorDate: Wed Apr 3 21:40:06 2019 +0800
Migrate command `localconsistencycheck`
Descriptions of the changes in this PR:
#2042
Using bkctl run command localconsistencycheck
```
Validate Ledger Storage internal metadata
Usage: bkctl bookie localconsistencycheck [flags]
Flags:
-h, --help
Display help information
```
Reviewers: Jia Zhai <zh...@apache.org>, Sijie Guo <si...@apache.org>
This closes #2043 from zymap/command-localconsistencycheck and squashes the following commits:
c31f07a00 [Sijie Guo] Merge branch 'master' into command-localconsistencycheck
c391fe58d [Yong Zhang] Migrate command `readlogmetadata`
120d67737 [Yong Zhang] Migrate command `lostbookierecoverydelay`
bf66235e5 [Yong Zhang] Migrate command `deleteledger`
c9bb4a42c [Yong Zhang] Migrate command `localconsistencycheck`
751e55fa4 [Arvin] ISSUE #2020: close db properly to avoid open RocksDB failure at the second time
138a7ae85 [Yong Zhang] Migrate command `metadataformat`
b043d1694 [Yong Zhang] Migrate command `listledgers`
4573285db [Ivan Kelly] Docker autobuild hook
e3d807a32 [Like] Fix IDE complain as there are multi choices for error code
9524a9f4a [Yong Zhang] Migrate command `readjournal`
6c3f33f55 [Yong Zhang] Fix when met unexpect entry id crashed
e35a108c7 [Like] Fix error message for unrecognized number-of-bookies
5902ee27b [Boyang Jerry Peng] fix potential NPE when releasing entry that is null
6aa73ce05 [Ivan Kelly] [RELEASE] Update website to include documentation for 4.8.2
1448d12aa [Yong Zhang] Migrate command `listfilesondisk`
4de598379 [Yong Zhang] Issue #1987: Migrate command `convert-to-interleaved-storage`
468743e7e [Matteo Merli] In DbLedgerStorage use default values when config key is present but empty
f26a4cae0 [Ivan Kelly] Release notes for v4.8.2
ec2636cd2 [Yong Zhang] Issue #1985: Migrate command `convert-to-db-storage`
8cc7239ac [Yong Zhang] Issue #1982: Migrate command `bookiesanity`
fa90f0185 [Yong Zhang] Issue #1980: Migrate command `ledger` from shell to bkctl
---
.../org/apache/bookkeeper/bookie/BookieShell.java | 19 ++----
.../bookie/LocalConsistencyCheckCommand.java | 77 ++++++++++++++++++++++
.../tools/cli/commands/BookieCommandGroup.java | 2 +
.../bookie/LocalConsistencyCheckCommandTest.java | 77 ++++++++++++++++++++++
4 files changed, 160 insertions(+), 15 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 98f276c..9d2bba6 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -85,6 +85,7 @@ import org.apache.bookkeeper.tools.cli.commands.bookie.LastMarkCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.LedgerCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ListFilesOnDiscCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ListLedgersCommand;
+import org.apache.bookkeeper.tools.cli.commands.bookie.LocalConsistencyCheckCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadJournalCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLedgerCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLogCommand;
@@ -961,21 +962,9 @@ public class BookieShell implements Tool {
@Override
public int runCmd(CommandLine cmdLine) throws Exception {
- LOG.info("=== Performing local consistency check ===");
- ServerConfiguration conf = new ServerConfiguration(bkConf);
- LedgerStorage ledgerStorage = Bookie.mountLedgerStorageOffline(conf, null);
- List <LedgerStorage.DetectedInconsistency> errors = ledgerStorage.localConsistencyCheck(
- java.util.Optional.empty());
- if (errors.size() > 0) {
- LOG.info("=== Check returned errors: ===");
- for (LedgerStorage.DetectedInconsistency error : errors) {
- LOG.error("Ledger {}, entry {}: ", error.getLedgerId(), error.getEntryId(), error.getException());
- }
- return 1;
- } else {
- LOG.info("=== Check passed ===");
- return 0;
- }
+ LocalConsistencyCheckCommand cmd = new LocalConsistencyCheckCommand();
+ boolean result = cmd.apply(bkConf, new CliFlags());
+ return (result) ? 0 : 1;
}
@Override
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LocalConsistencyCheckCommand.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LocalConsistencyCheckCommand.java
new file mode 100644
index 0000000..60a94bc
--- /dev/null
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LocalConsistencyCheckCommand.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookie;
+
+import com.google.common.util.concurrent.UncheckedExecutionException;
+import java.io.IOException;
+import java.util.List;
+import org.apache.bookkeeper.bookie.Bookie;
+import org.apache.bookkeeper.bookie.LedgerStorage;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.tools.cli.helpers.BookieCommand;
+import org.apache.bookkeeper.tools.framework.CliFlags;
+import org.apache.bookkeeper.tools.framework.CliSpec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Command to check local storage for inconsistencies.
+ */
+public class LocalConsistencyCheckCommand extends BookieCommand<CliFlags> {
+
+ static final Logger LOG = LoggerFactory.getLogger(LocalConsistencyCheckCommand.class);
+
+ private static final String NAME = "localconsistencycheck";
+ private static final String DESC = "Validate Ledger Storage internal metadata";
+
+ public LocalConsistencyCheckCommand() {
+ super(CliSpec.newBuilder()
+ .withName(NAME)
+ .withDescription(DESC)
+ .withFlags(new CliFlags())
+ .build());
+ }
+
+ @Override
+ public boolean apply(ServerConfiguration conf, CliFlags cmdFlags) {
+ try {
+ return check(conf);
+ } catch (IOException e) {
+ throw new UncheckedExecutionException(e.getMessage(), e);
+ }
+ }
+
+ private boolean check(ServerConfiguration conf) throws IOException {
+ LOG.info("=== Performing local consistency check ===");
+ ServerConfiguration serverConfiguration = new ServerConfiguration(conf);
+ LedgerStorage ledgerStorage = Bookie.mountLedgerStorageOffline(serverConfiguration, null);
+ List<LedgerStorage.DetectedInconsistency> errors = ledgerStorage.localConsistencyCheck(
+ java.util.Optional.empty());
+ if (errors.size() > 0) {
+ LOG.info("=== Check returned errors: ===");
+ for (LedgerStorage.DetectedInconsistency error : errors) {
+ LOG.error("Ledger {}, entry {}: ", error.getLedgerId(), error.getEntryId(), error.getException());
+ }
+ return false;
+ } else {
+ LOG.info("=== Check passed ===");
+ return true;
+ }
+ }
+}
diff --git a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
index 51e579f..95cd459 100644
--- a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
+++ b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java
@@ -29,6 +29,7 @@ import org.apache.bookkeeper.tools.cli.commands.bookie.LastMarkCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.LedgerCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ListFilesOnDiscCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ListLedgersCommand;
+import org.apache.bookkeeper.tools.cli.commands.bookie.LocalConsistencyCheckCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadJournalCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLedgerCommand;
import org.apache.bookkeeper.tools.cli.commands.bookie.ReadLogCommand;
@@ -64,6 +65,7 @@ public class BookieCommandGroup extends CliCommandGroup<BKFlags> {
.addCommand(new ReadLedgerCommand())
.addCommand(new ReadLogCommand())
.addCommand(new ReadLogMetadataCommand())
+ .addCommand(new LocalConsistencyCheckCommand())
.build();
public BookieCommandGroup() {
diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/LocalConsistencyCheckCommandTest.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/LocalConsistencyCheckCommandTest.java
new file mode 100644
index 0000000..3d32f9b
--- /dev/null
+++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/LocalConsistencyCheckCommandTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookie;
+
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.verifyNew;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.bookkeeper.bookie.Bookie;
+import org.apache.bookkeeper.bookie.LedgerStorage;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.tools.cli.helpers.BookieCommandTestBase;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+/**
+ * Unit test for {@link LocalConsistencyCheckCommand}.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ LocalConsistencyCheckCommand.class, Bookie.class })
+public class LocalConsistencyCheckCommandTest extends BookieCommandTestBase {
+
+ @Mock
+ private ServerConfiguration serverConfiguration;
+
+ @Mock
+ private LedgerStorage ledgerStorage;
+
+ public LocalConsistencyCheckCommandTest() {
+ super(3, 0);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ PowerMockito.whenNew(ServerConfiguration.class).withNoArguments().thenReturn(conf);
+ PowerMockito.whenNew(ServerConfiguration.class).withArguments(eq(conf)).thenReturn(serverConfiguration);
+ PowerMockito.mockStatic(Bookie.class);
+ PowerMockito.when(Bookie.mountLedgerStorageOffline(eq(serverConfiguration), eq(null)))
+ .thenReturn(ledgerStorage);
+ List<LedgerStorage.DetectedInconsistency> errors = new ArrayList<>();
+ PowerMockito.when(ledgerStorage.localConsistencyCheck(eq(java.util.Optional.empty()))).thenReturn(errors);
+ }
+
+ @Test
+ public void testCommand() throws Exception {
+ LocalConsistencyCheckCommand cmd = new LocalConsistencyCheckCommand();
+ Assert.assertTrue(cmd.apply(bkFlags, new String[] {}));
+ verifyNew(ServerConfiguration.class, times(1)).withArguments(eq(conf));
+ verify(ledgerStorage, times(1)).localConsistencyCheck(eq(java.util.Optional.empty()));
+ }
+}