You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by yu...@apache.org on 2023/10/07 15:33:03 UTC
[pulsar] branch branch-3.0 updated: [improve] [auto-recovery] Migrate the replication testing from BookKeeper to Pulsar. (#21188)
This is an automated email from the ASF dual-hosted git repository.
yubiao pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new c4de4ee8bfd [improve] [auto-recovery] Migrate the replication testing from BookKeeper to Pulsar. (#21188)
c4de4ee8bfd is described below
commit c4de4ee8bfd98dad577e6776a3c3838a0808b129
Author: Yan Zhao <ho...@apache.org>
AuthorDate: Thu Sep 28 22:14:25 2023 +0800
[improve] [auto-recovery] Migrate the replication testing from BookKeeper to Pulsar. (#21188)
There is no testing for AutoRecovery replication in Pulsar's current test suite, and we need to cover it. So migrate the replication testing from BookKeeper to Pulsar.
(cherry picked from commit 9061d8b4e821e3fd03e067f01d5dfcf05bcdced4)
---
.../metadata/bookkeeper/PulsarLayoutManager.java | 4 +-
.../bookkeeper/PulsarLedgerAuditorManager.java | 4 +-
.../metadata/bookkeeper/PulsarLedgerManager.java | 2 +-
.../replication/AuditorBookieCheckTaskTest.java | 153 +++
.../bookkeeper/replication/AuditorBookieTest.java | 299 +++++
.../AuditorCheckAllLedgersTaskTest.java | 136 +++
.../replication/AuditorLedgerCheckerTest.java | 1058 +++++++++++++++++
.../AuditorPeriodicBookieCheckTest.java | 117 ++
.../replication/AuditorPeriodicCheckTest.java | 852 +++++++++++++
.../AuditorPlacementPolicyCheckTaskTest.java | 124 ++
.../AuditorPlacementPolicyCheckTest.java | 861 ++++++++++++++
.../replication/AuditorReplicasCheckTaskTest.java | 119 ++
.../replication/AuditorReplicasCheckTest.java | 937 +++++++++++++++
.../replication/AuditorRollingRestartTest.java | 112 ++
.../replication/AuthAutoRecoveryTest.java | 119 ++
.../replication/AutoRecoveryMainTest.java | 254 ++++
.../replication/BookieAutoRecoveryTest.java | 652 ++++++++++
.../replication/BookieLedgerIndexTest.java | 248 ++++
.../replication/ReplicationTestUtil.java | 61 +
.../TestAutoRecoveryAlongWithBookieServers.java | 117 ++
.../replication/TestReplicationWorker.java | 1249 ++++++++++++++++++++
21 files changed, 7473 insertions(+), 5 deletions(-)
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java
index a4336b87639..ee06930b3c8 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java
@@ -30,7 +30,7 @@ import org.apache.bookkeeper.util.BookKeeperConstants;
import org.apache.pulsar.metadata.api.MetadataStoreException;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
-class PulsarLayoutManager implements LayoutManager {
+public class PulsarLayoutManager implements LayoutManager {
@Getter(AccessLevel.PACKAGE)
private final MetadataStoreExtended store;
@@ -40,7 +40,7 @@ class PulsarLayoutManager implements LayoutManager {
private final String layoutPath;
- PulsarLayoutManager(MetadataStoreExtended store, String ledgersRootPath) {
+ public PulsarLayoutManager(MetadataStoreExtended store, String ledgersRootPath) {
this.ledgersRootPath = ledgersRootPath;
this.store = store;
this.layoutPath = ledgersRootPath + "/" + BookKeeperConstants.LAYOUT_ZNODE;
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java
index bc35380fec1..957f26ecd2e 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java
@@ -30,9 +30,9 @@ import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl;
@Slf4j
-class PulsarLedgerAuditorManager implements LedgerAuditorManager {
+public class PulsarLedgerAuditorManager implements LedgerAuditorManager {
- private static final String ELECTION_PATH = "leader";
+ public static final String ELECTION_PATH = "leader";
private final CoordinationService coordinationService;
private final LeaderElection<String> leaderElection;
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java
index 59452a3d54d..b003c656353 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java
@@ -377,7 +377,7 @@ public class PulsarLedgerManager implements LedgerManager {
}
}
- private String getLedgerPath(long ledgerId) {
+ public String getLedgerPath(long ledgerId) {
return this.ledgerRootPath + StringUtils.getHybridHierarchicalLedgerPath(ledgerId);
}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java
new file mode 100644
index 00000000000..d30af4de6a8
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE;
+import static org.mockito.ArgumentMatchers.anyCollection;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.testng.AssertJUnit.assertTrue;
+import com.beust.jcommander.internal.Lists;
+import com.beust.jcommander.internal.Sets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.test.TestStatsProvider;
+import org.apache.bookkeeper.versioning.LongVersion;
+import org.apache.bookkeeper.versioning.Versioned;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test {@link AuditorBookieCheckTask}.
+ */
+public class AuditorBookieCheckTaskTest {
+
+ private AuditorStats auditorStats;
+ private BookKeeperAdmin admin;
+ private LedgerManager ledgerManager;
+ private LedgerUnderreplicationManager underreplicationManager;
+ private BookieLedgerIndexer ledgerIndexer;
+ private AuditorBookieCheckTask bookieCheckTask;
+ private final AtomicBoolean shutdownCompleted = new AtomicBoolean(false);
+ private final AuditorTask.ShutdownTaskHandler shutdownTaskHandler = () -> shutdownCompleted.set(true);
+ private long startLedgerId = 0;
+
+ @BeforeMethod
+ public void setup() {
+ ServerConfiguration conf = mock(ServerConfiguration.class);
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ final AuditorStats auditorStats = new AuditorStats(statsLogger);
+ this.auditorStats = spy(auditorStats);
+ admin = mock(BookKeeperAdmin.class);
+ ledgerManager = mock(LedgerManager.class);
+ underreplicationManager = mock(LedgerUnderreplicationManager.class);
+ ledgerIndexer = mock(BookieLedgerIndexer.class);
+ AuditorBookieCheckTask bookieCheckTask1 = new AuditorBookieCheckTask(
+ conf, this.auditorStats, admin, ledgerManager, underreplicationManager,
+ shutdownTaskHandler, ledgerIndexer, null, null);
+ bookieCheckTask = spy(bookieCheckTask1);
+ }
+
+ @Test
+ public void testShutdownAuditBookiesException()
+ throws BKException, ReplicationException.BKAuditException, InterruptedException {
+ doThrow(new ReplicationException.BKAuditException("test failed"))
+ .when(bookieCheckTask)
+ .auditBookies();
+ bookieCheckTask.startAudit(true);
+
+ assertTrue("shutdownTaskHandler should be execute.", shutdownCompleted.get());
+ }
+
+ @Test
+ public void testAuditBookies()
+ throws ReplicationException.UnavailableException, ReplicationException.BKAuditException, BKException {
+ final String bookieId1 = "127.0.0.1:1000";
+ final String bookieId2 = "127.0.0.1:1001";
+ final long bookie1LedgersCount = 10;
+ final long bookie2LedgersCount = 20;
+
+ final Map<String, Set<Long>> bookiesAndLedgers = new HashMap<>();
+ bookiesAndLedgers.put(bookieId1, getLedgers(bookie1LedgersCount));
+ bookiesAndLedgers.put(bookieId2, getLedgers(bookie2LedgersCount));
+ when(ledgerIndexer.getBookieToLedgerIndex()).thenReturn(bookiesAndLedgers);
+ when(underreplicationManager.isLedgerReplicationEnabled()).thenReturn(true);
+
+ CompletableFuture<Versioned<LedgerMetadata>> metaPromise = new CompletableFuture<>();
+ final LongVersion version = mock(LongVersion.class);
+ final LedgerMetadata metadata = mock(LedgerMetadata.class);
+ metaPromise.complete(new Versioned<>(metadata, version));
+ when(ledgerManager.readLedgerMetadata(anyLong())).thenReturn(metaPromise);
+
+ CompletableFuture<Void> markPromise = new CompletableFuture<>();
+ markPromise.complete(null);
+ when(underreplicationManager.markLedgerUnderreplicatedAsync(anyLong(), anyCollection()))
+ .thenReturn(markPromise);
+
+ OpStatsLogger numUnderReplicatedLedgerStats = mock(OpStatsLogger.class);
+ when(auditorStats.getNumUnderReplicatedLedger()).thenReturn(numUnderReplicatedLedgerStats);
+
+ final List<BookieId> availableBookies = Lists.newArrayList();
+ final List<BookieId> readOnlyBookies = Lists.newArrayList();
+ // test bookie1 lost
+ availableBookies.add(BookieId.parse(bookieId2));
+ when(admin.getAvailableBookies()).thenReturn(availableBookies);
+ when(admin.getReadOnlyBookies()).thenReturn(readOnlyBookies);
+ bookieCheckTask.startAudit(true);
+ verify(numUnderReplicatedLedgerStats, times(1))
+ .registerSuccessfulValue(eq(bookie1LedgersCount));
+
+ // test bookie2 lost
+ numUnderReplicatedLedgerStats = mock(OpStatsLogger.class);
+ when(auditorStats.getNumUnderReplicatedLedger()).thenReturn(numUnderReplicatedLedgerStats);
+ availableBookies.clear();
+ availableBookies.add(BookieId.parse(bookieId1));
+ bookieCheckTask.startAudit(true);
+ verify(numUnderReplicatedLedgerStats, times(1))
+ .registerSuccessfulValue(eq(bookie2LedgersCount));
+
+ }
+
+ private Set<Long> getLedgers(long count) {
+ final Set<Long> ledgers = Sets.newHashSet();
+ for (int i = 0; i < count; i++) {
+ ledgers.add(i + startLedgerId++);
+ }
+ return ledgers;
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java
new file mode 100644
index 00000000000..9750fb52d41
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertNotNull;
+import static org.testng.AssertJUnit.assertNotSame;
+import static org.testng.AssertJUnit.assertSame;
+import static org.testng.AssertJUnit.assertTrue;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.proto.BookieServer;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerAuditorManager;
+import org.apache.zookeeper.ZooKeeper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * This test verifies the auditor bookie scenarios which will be monitoring the
+ * bookie failures.
+ */
+public class AuditorBookieTest extends BookKeeperClusterTestCase {
+ // Depending on the taste, select the amount of logging
+ // by decommenting one of the two lines below
+ // private static final Logger LOG = Logger.getRootLogger();
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AuditorBookieTest.class);
+ private String electionPath;
+ private HashMap<String, AuditorElector> auditorElectors = new HashMap<String, AuditorElector>();
+ private List<ZooKeeper> zkClients = new LinkedList<ZooKeeper>();
+
+ public AuditorBookieTest() throws Exception {
+ super(6);
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ electionPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf)
+ + "/underreplication/" + PulsarLedgerAuditorManager.ELECTION_PATH;
+ baseConf.setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ startAuditorElectors();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ stopAuditorElectors();
+ for (ZooKeeper zk : zkClients) {
+ zk.close();
+ }
+ zkClients.clear();
+ super.tearDown();
+ }
+
+ /**
+ * Test should ensure only one should act as Auditor. Starting/shutdown
+ * other than auditor bookie shouldn't initiate re-election and multiple
+ * auditors.
+ */
+ @Test
+ public void testEnsureOnlySingleAuditor() throws Exception {
+ BookieServer auditor = verifyAuditor();
+
+ // shutdown bookie which is not an auditor
+ int indexOf = indexOfServer(auditor);
+ int bkIndexDownBookie;
+ if (indexOf < lastBookieIndex()) {
+ bkIndexDownBookie = indexOf + 1;
+ } else {
+ bkIndexDownBookie = indexOf - 1;
+ }
+ shutdownBookie(serverByIndex(bkIndexDownBookie));
+
+ startNewBookie();
+ startNewBookie();
+ // grace period for the auditor re-election if any
+ BookieServer newAuditor = waitForNewAuditor(auditor);
+ assertSame(
+ "Auditor re-election is not happened for auditor failure!",
+ auditor, newAuditor);
+ }
+
+ /**
+ * Test Auditor crashes should trigger re-election and another bookie should
+ * take over the auditor ship.
+ */
+ @Test
+ public void testSuccessiveAuditorCrashes() throws Exception {
+ BookieServer auditor = verifyAuditor();
+ shutdownBookie(auditor);
+
+ BookieServer newAuditor1 = waitForNewAuditor(auditor);
+ shutdownBookie(newAuditor1);
+ BookieServer newAuditor2 = waitForNewAuditor(newAuditor1);
+ assertNotSame(
+ "Auditor re-election is not happened for auditor failure!",
+ auditor, newAuditor2);
+ }
+
+ /**
+ * Test restarting the entire bookie cluster. It shouldn't create multiple
+ * bookie auditors.
+ */
+ @Test
+ public void testBookieClusterRestart() throws Exception {
+ BookieServer auditor = verifyAuditor();
+ for (AuditorElector auditorElector : auditorElectors.values()) {
+ assertTrue("Auditor elector is not running!", auditorElector
+ .isRunning());
+ }
+ stopBKCluster();
+ stopAuditorElectors();
+
+ startBKCluster(zkUtil.getMetadataServiceUri());
+ //startBKCluster(zkUtil.getMetadataServiceUri()) override the base conf metadataServiceUri
+ baseConf.setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ startAuditorElectors();
+ BookieServer newAuditor = waitForNewAuditor(auditor);
+ assertNotSame(
+ "Auditor re-election is not happened for auditor failure!",
+ auditor, newAuditor);
+ }
+
+ /**
+ * Test the vote is deleting from the ZooKeeper during shutdown.
+ */
+ @Test
+ public void testShutdown() throws Exception {
+ BookieServer auditor = verifyAuditor();
+ shutdownBookie(auditor);
+
+ // waiting for new auditor
+ BookieServer newAuditor = waitForNewAuditor(auditor);
+ assertNotSame(
+ "Auditor re-election is not happened for auditor failure!",
+ auditor, newAuditor);
+
+ List<String> children = zkc.getChildren(electionPath, false);
+ for (String child : children) {
+ byte[] data = zkc.getData(electionPath + '/' + child, false, null);
+ String bookieIP = new String(data);
+ String addr = auditor.getBookieId().toString();
+ assertFalse("AuditorElection cleanup fails", bookieIP
+ .contains(addr));
+ }
+ }
+
+ /**
+ * Test restart of the previous Auditor bookie shouldn't initiate
+ * re-election and should create new vote after restarting.
+ */
+ @Test
+ public void testRestartAuditorBookieAfterCrashing() throws Exception {
+ BookieServer auditor = verifyAuditor();
+
+ String addr = auditor.getBookieId().toString();
+
+ // restarting Bookie with same configurations.
+ ServerConfiguration serverConfiguration = shutdownBookie(auditor);
+
+ auditorElectors.remove(addr);
+ startBookie(serverConfiguration);
+ // starting corresponding auditor elector
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Performing Auditor Election:" + addr);
+ }
+ startAuditorElector(addr);
+
+ // waiting for new auditor to come
+ BookieServer newAuditor = waitForNewAuditor(auditor);
+ assertNotSame(
+ "Auditor re-election is not happened for auditor failure!",
+ auditor, newAuditor);
+ assertFalse("No relection after old auditor rejoins", auditor
+ .getBookieId().equals(newAuditor.getBookieId()));
+ }
+
+ private void startAuditorElector(String addr) throws Exception {
+ AuditorElector auditorElector = new AuditorElector(addr,
+ baseConf);
+ auditorElectors.put(addr, auditorElector);
+ auditorElector.start();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Starting Auditor Elector");
+ }
+ }
+
+ private void startAuditorElectors() throws Exception {
+ for (BookieId addr : bookieAddresses()) {
+ startAuditorElector(addr.toString());
+ }
+ }
+
+ private void stopAuditorElectors() throws Exception {
+ for (AuditorElector auditorElector : auditorElectors.values()) {
+ auditorElector.shutdown();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stopping Auditor Elector!");
+ }
+ }
+ }
+
+ private BookieServer verifyAuditor() throws Exception {
+ List<BookieServer> auditors = getAuditorBookie();
+ assertEquals("Multiple Bookies acting as Auditor!", 1, auditors
+ .size());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Bookie running as Auditor:" + auditors.get(0));
+ }
+ return auditors.get(0);
+ }
+
+ private List<BookieServer> getAuditorBookie() throws Exception {
+ List<BookieServer> auditors = new LinkedList<BookieServer>();
+ byte[] data = zkc.getData(electionPath, false, null);
+ assertNotNull("Auditor election failed", data);
+ for (int i = 0; i < bookieCount(); i++) {
+ BookieServer bks = serverByIndex(i);
+ if (new String(data).contains(bks.getBookieId() + "")) {
+ auditors.add(bks);
+ }
+ }
+ return auditors;
+ }
+
+ private ServerConfiguration shutdownBookie(BookieServer bkServer) throws Exception {
+ int index = indexOfServer(bkServer);
+ String addr = addressByIndex(index).toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Shutting down bookie:" + addr);
+ }
+
+ // shutdown bookie which is an auditor
+ ServerConfiguration conf = killBookie(index);
+
+ // stopping corresponding auditor elector
+ auditorElectors.get(addr).shutdown();
+ return conf;
+ }
+
+ private BookieServer waitForNewAuditor(BookieServer auditor)
+ throws Exception {
+ BookieServer newAuditor = null;
+ int retryCount = 8;
+ while (retryCount > 0) {
+ try {
+ List<BookieServer> auditors = getAuditorBookie();
+ if (auditors.size() > 0) {
+ newAuditor = auditors.get(0);
+ if (auditor != newAuditor) {
+ break;
+ }
+ }
+ } catch (Exception ignore) {
+ }
+
+ Thread.sleep(500);
+ retryCount--;
+ }
+ assertNotNull(
+ "New Auditor is not reelected after auditor crashes",
+ newAuditor);
+ verifyAuditor();
+ return newAuditor;
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java
new file mode 100644
index 00000000000..5c0a3f39325
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE;
+import static org.testng.AssertJUnit.assertEquals;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.meta.LayoutManager;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.test.TestStatsProvider;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test {@link AuditorCheckAllLedgersTask}.
+ */
+public class AuditorCheckAllLedgersTaskTest extends BookKeeperClusterTestCase {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AuditorCheckAllLedgersTaskTest.class);
+
+ private static final int maxNumberOfConcurrentOpenLedgerOperations = 500;
+ private static final int acquireConcurrentOpenLedgerOperationsTimeoutMSec = 120000;
+
+ private BookKeeperAdmin admin;
+ private LedgerManager ledgerManager;
+ private LedgerUnderreplicationManager ledgerUnderreplicationManager;
+
+ public AuditorCheckAllLedgersTaskTest() {
+ super(3);
+ baseConf.setPageLimit(1);
+ baseConf.setAutoRecoveryDaemonEnabled(false);
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ final BookKeeper bookKeeper = new BookKeeper(baseClientConf);
+ admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new ClientConfiguration(baseClientConf));
+
+ String ledgersRoot = "/ledgers";
+ String storeUri = metadataServiceUri.replaceAll("zk://", "").replaceAll("/ledgers", "");
+ MetadataStoreExtended store = MetadataStoreExtended.create(storeUri,
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+ LayoutManager layoutManager = new PulsarLayoutManager(store, ledgersRoot);
+ PulsarLedgerManagerFactory ledgerManagerFactory = new PulsarLedgerManagerFactory();
+
+ ClientConfiguration conf = new ClientConfiguration();
+ conf.setZkLedgersRootPath(ledgersRoot);
+ ledgerManagerFactory.initialize(conf, layoutManager, 1);
+ ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager();
+ ledgerManager = ledgerManagerFactory.newLedgerManager();
+
+ baseConf.setAuditorMaxNumberOfConcurrentOpenLedgerOperations(maxNumberOfConcurrentOpenLedgerOperations);
+ baseConf.setAuditorAcquireConcurrentOpenLedgerOperationsTimeoutMSec(
+ acquireConcurrentOpenLedgerOperationsTimeoutMSec);
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ if (ledgerManager != null) {
+ ledgerManager.close();
+ }
+ if (ledgerUnderreplicationManager != null) {
+ ledgerUnderreplicationManager.close();
+ }
+ if (admin != null) {
+ admin.close();
+ }
+ super.tearDown();
+ }
+
+ @Test
+ public void testCheckAllLedgers() throws Exception {
+ // 1. create ledgers
+ final int numLedgers = 10;
+ List<Long> ids = new LinkedList<Long>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes());
+ ids.add(lh.getId());
+ for (int j = 0; j < 2; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+ // 2. init CheckAllLedgersTask
+ final TestStatsProvider statsProvider = new TestStatsProvider();
+ final TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ final AuditorStats auditorStats = new AuditorStats(statsLogger);
+
+ AuditorCheckAllLedgersTask auditorCheckAllLedgersTask = new AuditorCheckAllLedgersTask(
+ baseConf, auditorStats, admin, ledgerManager,
+ ledgerUnderreplicationManager, null, (flag, throwable) -> flag.set(false));
+
+ // 3. checkAllLedgers
+ auditorCheckAllLedgersTask.runTask();
+
+ // 4. verify
+ assertEquals("CHECK_ALL_LEDGERS_TIME", 1, ((TestStatsProvider.TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.CHECK_ALL_LEDGERS_TIME)).getSuccessCount());
+ assertEquals("NUM_LEDGERS_CHECKED", numLedgers,
+ (long) statsLogger.getCounter(ReplicationStats.NUM_LEDGERS_CHECKED).get());
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
new file mode 100644
index 00000000000..ffd71f9311f
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
@@ -0,0 +1,1058 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertNotNull;
+import static org.testng.AssertJUnit.assertNotSame;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.fail;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import lombok.Cleanup;
+import org.apache.bookkeeper.bookie.BookieImpl;
+import org.apache.bookkeeper.client.AsyncCallback.AddCallback;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.LedgerMetadataBuilder;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
+import org.apache.bookkeeper.common.util.OrderedScheduler;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.meta.LayoutManager;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.meta.MetadataClientDriver;
+import org.apache.bookkeeper.meta.MetadataDrivers;
+import org.apache.bookkeeper.meta.UnderreplicatedLedger;
+import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager;
+import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.net.BookieSocketAddress;
+import org.apache.bookkeeper.proto.BookieServer;
+import org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerAuditorManager;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Tests publishing of under replicated ledgers by the Auditor bookie node when
+ * corresponding bookies identifes as not running.
+ */
+public class AuditorLedgerCheckerTest extends BookKeeperClusterTestCase {
+
+ // Depending on the taste, select the amount of logging
+ // by decommenting one of the two lines below
+ // private static final Logger LOG = Logger.getRootLogger();
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AuditorLedgerCheckerTest.class);
+
+ private static final byte[] ledgerPassword = "aaa".getBytes();
+ private Random rng; // Random Number Generator
+
+ private DigestType digestType;
+
+ private String underreplicatedPath;
+ private Map<String, AuditorElector> auditorElectors = new ConcurrentHashMap<>();
+ private LedgerUnderreplicationManager urLedgerMgr;
+
+ private Set<Long> urLedgerList;
+ private String electionPath;
+
+ private List<Long> ledgerList;
+
+ public AuditorLedgerCheckerTest()
+ throws Exception {
+ this("org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory");
+ }
+
+ AuditorLedgerCheckerTest(String ledgerManagerFactoryClass)
+ throws Exception {
+ super(3);
+ LOG.info("Running test case using ledger manager : "
+ + ledgerManagerFactoryClass);
+ this.digestType = DigestType.CRC32;
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); // set ledger manager name
+ baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactoryClass);
+ baseClientConf
+ .setLedgerManagerFactoryClassName(ledgerManagerFactoryClass);
+ }
+
+ @BeforeMethod
+ public void setUp() throws Exception {
+ super.setUp();
+ underreplicatedPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseClientConf)
+ + "/underreplication/ledgers";
+ electionPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf)
+ + "/underreplication/" + PulsarLedgerAuditorManager.ELECTION_PATH;
+
+ String ledgersRoot = "/ledgers";
+ String storeUri = metadataServiceUri.replaceAll("zk://", "").replaceAll("/ledgers", "");
+ MetadataStoreExtended store = MetadataStoreExtended.create(storeUri,
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+ LayoutManager layoutManager = new PulsarLayoutManager(store, ledgersRoot);
+ PulsarLedgerManagerFactory ledgerManagerFactory = new PulsarLedgerManagerFactory();
+ ClientConfiguration conf = new ClientConfiguration();
+ conf.setZkLedgersRootPath(ledgersRoot);
+ ledgerManagerFactory.initialize(conf, layoutManager, 1);
+ urLedgerMgr = ledgerManagerFactory.newLedgerUnderreplicationManager();
+ urLedgerMgr.setCheckAllLedgersCTime(System.currentTimeMillis());
+
+ baseClientConf.setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ baseConf.setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ startAuditorElectors();
+ rng = new Random(System.currentTimeMillis()); // Initialize the Random
+ urLedgerList = new HashSet<Long>();
+ ledgerList = new ArrayList<Long>(2);
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ stopAuditorElectors();
+ super.tearDown();
+ }
+
+ private void startAuditorElectors() throws Exception {
+ for (String addr : bookieAddresses().stream().map(Object::toString)
+ .collect(Collectors.toList())) {
+ AuditorElector auditorElector = new AuditorElector(addr, baseConf);
+ auditorElectors.put(addr, auditorElector);
+ auditorElector.start();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Starting Auditor Elector");
+ }
+ }
+ }
+
+ private void stopAuditorElectors() throws Exception {
+ for (AuditorElector auditorElector : auditorElectors.values()) {
+ auditorElector.shutdown();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stopping Auditor Elector!");
+ }
+ }
+ }
+
+ /**
+ * Test publishing of under replicated ledgers by the auditor bookie.
+ */
+ @Test
+ public void testSimpleLedger() throws Exception {
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ Long ledgerId = lh1.getId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created ledger : " + ledgerId);
+ }
+ ledgerList.add(ledgerId);
+ lh1.close();
+
+ final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList
+ .size());
+
+ int bkShutdownIndex = lastBookieIndex();
+ String shutdownBookie = shutdownBookie(bkShutdownIndex);
+
+ // grace period for publishing the bk-ledger
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ledgers to be marked as under replicated");
+ }
+ waitForAuditToComplete();
+ underReplicaLatch.await(5, TimeUnit.SECONDS);
+ Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
+ assertEquals("Missed identifying under replicated ledgers", 1,
+ urLedgerList.size());
+
+ /*
+ * Sample data format present in the under replicated ledger path
+ *
+ * {4=replica: "10.18.89.153:5002"}
+ */
+ assertTrue("Ledger is not marked as underreplicated:" + ledgerId,
+ urLedgerList.contains(ledgerId));
+ String data = urLedgerData.get(ledgerId);
+ assertTrue("Bookie " + shutdownBookie
+ + "is not listed in the ledger as missing replica :" + data,
+ data.contains(shutdownBookie));
+ }
+
+ /**
+ * Test once published under replicated ledger should exists even after
+ * restarting respective bookie.
+ */
+ @Test
+ public void testRestartBookie() throws Exception {
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ LedgerHandle lh2 = createAndAddEntriesToLedger();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created following ledgers : {}, {}", lh1, lh2);
+ }
+
+ int bkShutdownIndex = lastBookieIndex();
+ ServerConfiguration bookieConf1 = confByIndex(bkShutdownIndex);
+ String shutdownBookie = shutdownBookie(bkShutdownIndex);
+
+ // restart the failed bookie
+ startAndAddBookie(bookieConf1);
+
+ waitForLedgerMissingReplicas(lh1.getId(), 10, shutdownBookie);
+ waitForLedgerMissingReplicas(lh2.getId(), 10, shutdownBookie);
+ }
+
+ /**
+ * Test publishing of under replicated ledgers when multiple bookie failures
+ * one after another.
+ */
+ @Test
+ public void testMultipleBookieFailures() throws Exception {
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+
+ // failing first bookie
+ shutdownBookie(lastBookieIndex());
+
+ // simulate re-replication
+ doLedgerRereplication(lh1.getId());
+
+ // failing another bookie
+ String shutdownBookie = shutdownBookie(lastBookieIndex());
+
+ // grace period for publishing the bk-ledger
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ledgers to be marked as under replicated");
+ }
+ assertTrue("Ledger should be missing second replica",
+ waitForLedgerMissingReplicas(lh1.getId(), 10, shutdownBookie));
+ }
+
+ @Test
+ public void testToggleLedgerReplication() throws Exception {
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ ledgerList.add(lh1.getId());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created following ledgers : " + ledgerList);
+ }
+
+ // failing another bookie
+ CountDownLatch urReplicaLatch = registerUrLedgerWatcher(ledgerList
+ .size());
+
+ // disabling ledger replication
+ urLedgerMgr.disableLedgerReplication();
+ ArrayList<String> shutdownBookieList = new ArrayList<String>();
+ shutdownBookieList.add(shutdownBookie(lastBookieIndex()));
+ shutdownBookieList.add(shutdownBookie(lastBookieIndex()));
+
+ assertFalse("Ledger replication is not disabled!", urReplicaLatch
+ .await(1, TimeUnit.SECONDS));
+
+ // enabling ledger replication
+ urLedgerMgr.enableLedgerReplication();
+ assertTrue("Ledger replication is not enabled!", urReplicaLatch.await(
+ 5, TimeUnit.SECONDS));
+ }
+
+ @Test
+ public void testDuplicateEnDisableAutoRecovery() throws Exception {
+ urLedgerMgr.disableLedgerReplication();
+ try {
+ urLedgerMgr.disableLedgerReplication();
+ fail("Must throw exception, since AutoRecovery is already disabled");
+ } catch (UnavailableException e) {
+ assertTrue("AutoRecovery is not disabled previously!",
+ e.getCause().getCause() instanceof MetadataStoreException.BadVersionException);
+ }
+ urLedgerMgr.enableLedgerReplication();
+ try {
+ urLedgerMgr.enableLedgerReplication();
+ fail("Must throw exception, since AutoRecovery is already enabled");
+ } catch (UnavailableException e) {
+ assertTrue("AutoRecovery is not enabled previously!",
+ e.getCause().getCause() instanceof MetadataStoreException.NotFoundException);
+ }
+ }
+
+ /**
+ * Test Auditor should consider Readonly bookie as available bookie. Should not publish ur ledgers for
+ * readonly bookies.
+ */
+ @Test
+ public void testReadOnlyBookieExclusionFromURLedgersCheck() throws Exception {
+ LedgerHandle lh = createAndAddEntriesToLedger();
+ ledgerList.add(lh.getId());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created following ledgers : " + ledgerList);
+ }
+
+ int count = ledgerList.size();
+ final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(count);
+
+ final int bkIndex = 2;
+ ServerConfiguration bookieConf = confByIndex(bkIndex);
+ BookieServer bk = serverByIndex(bkIndex);
+ bookieConf.setReadOnlyModeEnabled(true);
+
+ ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode();
+ bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex)))
+ .get(30, TimeUnit.SECONDS);
+
+ // grace period for publishing the bk-ledger
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for Auditor to finish ledger check.");
+ }
+ waitForAuditToComplete();
+ assertFalse("latch should not have completed", underReplicaLatch.await(5, TimeUnit.SECONDS));
+ }
+
+ /**
+ * Test Auditor should consider Readonly bookie fail and publish ur ledgers for readonly bookies.
+ */
+ @Test
+ public void testReadOnlyBookieShutdown() throws Exception {
+ LedgerHandle lh = createAndAddEntriesToLedger();
+ long ledgerId = lh.getId();
+ ledgerList.add(ledgerId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created following ledgers : " + ledgerList);
+ }
+
+ int count = ledgerList.size();
+ final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(count);
+
+ int bkIndex = lastBookieIndex();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Moving bookie {} {} to read only...", bkIndex, serverByIndex(bkIndex));
+ }
+ ServerConfiguration bookieConf = confByIndex(bkIndex);
+ BookieServer bk = serverByIndex(bkIndex);
+ bookieConf.setReadOnlyModeEnabled(true);
+
+ ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode();
+ bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex)))
+ .get(30, TimeUnit.SECONDS);
+
+ // grace period for publishing the bk-ledger
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for Auditor to finish ledger check.");
+ }
+ waitForAuditToComplete();
+ assertFalse("latch should not have completed", underReplicaLatch.await(1, TimeUnit.SECONDS));
+
+ String shutdownBookie = shutdownBookie(bkIndex);
+
+ // grace period for publishing the bk-ledger
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ledgers to be marked as under replicated");
+ }
+ waitForAuditToComplete();
+ underReplicaLatch.await(5, TimeUnit.SECONDS);
+ Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
+ assertEquals("Missed identifying under replicated ledgers", 1, urLedgerList.size());
+
+ /*
+ * Sample data format present in the under replicated ledger path
+ *
+ * {4=replica: "10.18.89.153:5002"}
+ */
+ assertTrue("Ledger is not marked as underreplicated:" + ledgerId, urLedgerList.contains(ledgerId));
+ String data = urLedgerData.get(ledgerId);
+ assertTrue("Bookie " + shutdownBookie + "is not listed in the ledger as missing replica :" + data,
+ data.contains(shutdownBookie));
+ }
+
+ public void testInnerDelayedAuditOfLostBookies() throws Exception {
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ Long ledgerId = lh1.getId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created ledger : " + ledgerId);
+ }
+ ledgerList.add(ledgerId);
+ lh1.close();
+
+ final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList
+ .size());
+
+ // wait for 5 seconds before starting the recovery work when a bookie fails
+ urLedgerMgr.setLostBookieRecoveryDelay(5);
+
+ // shutdown a non auditor bookie; choosing non-auditor to avoid another election
+ String shutdownBookie = shutDownNonAuditorBookie();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ledgers to be marked as under replicated");
+ }
+ assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(4, TimeUnit.SECONDS));
+ assertEquals("under replicated ledgers identified when it was not expected", 0,
+ urLedgerList.size());
+
+ // wait for another 5 seconds for the ledger to get reported as under replicated
+ assertTrue("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS));
+
+ assertTrue("Ledger is not marked as underreplicated:" + ledgerId,
+ urLedgerList.contains(ledgerId));
+ Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
+ String data = urLedgerData.get(ledgerId);
+ assertTrue("Bookie " + shutdownBookie
+ + "is not listed in the ledger as missing replica :" + data,
+ data.contains(shutdownBookie));
+ }
+
+ /**
+ * Test publishing of under replicated ledgers by the auditor
+ * bookie is delayed if LostBookieRecoveryDelay option is set.
+ */
+ @Test
+ public void testDelayedAuditOfLostBookies() throws Exception {
+ // wait for a second so that the initial periodic check finishes
+ Thread.sleep(1000);
+
+ testInnerDelayedAuditOfLostBookies();
+ }
+
+ /**
+ * Test publishing of under replicated ledgers by the auditor
+ * bookie is delayed if LostBookieRecoveryDelay option is set
+ * and it continues to be delayed even when periodic bookie check
+ * is set to run every 2 secs. I.e. periodic bookie check doesn't
+ * override the delay
+ */
+ @Test
+ public void testDelayedAuditWithPeriodicBookieCheck() throws Exception {
+ // enable periodic bookie check on a cadence of every 2 seconds.
+ // this requires us to stop the auditor/auditorElectors, set the
+ // periodic check interval and restart the auditorElectors
+ stopAuditorElectors();
+ baseConf.setAuditorPeriodicBookieCheckInterval(2);
+ startAuditorElectors();
+
+ // wait for a second so that the initial periodic check finishes
+ Thread.sleep(1000);
+
+ // the delaying of audit should just work despite the fact
+ // we have enabled periodic bookie check
+ testInnerDelayedAuditOfLostBookies();
+ }
+
+ @Test
+ public void testRescheduleOfDelayedAuditOfLostBookiesToStartImmediately() throws Exception {
+ // wait for a second so that the initial periodic check finishes
+ Thread.sleep(1000);
+
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ Long ledgerId = lh1.getId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created ledger : " + ledgerId);
+ }
+ ledgerList.add(ledgerId);
+ lh1.close();
+
+ final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList
+ .size());
+
+ // wait for 50 seconds before starting the recovery work when a bookie fails
+ urLedgerMgr.setLostBookieRecoveryDelay(50);
+
+ // shutdown a non auditor bookie; choosing non-auditor to avoid another election
+ String shutdownBookie = shutDownNonAuditorBookie();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ledgers to be marked as under replicated");
+ }
+ assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(4, TimeUnit.SECONDS));
+ assertEquals("under replicated ledgers identified when it was not expected", 0,
+ urLedgerList.size());
+
+ // set lostBookieRecoveryDelay to 0, so that it triggers AuditTask immediately
+ urLedgerMgr.setLostBookieRecoveryDelay(0);
+
+ // wait for 1 second for the ledger to get reported as under replicated
+ assertTrue("audit of lost bookie isn't delayed", underReplicaLatch.await(1, TimeUnit.SECONDS));
+
+ assertTrue("Ledger is not marked as underreplicated:" + ledgerId,
+ urLedgerList.contains(ledgerId));
+ Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
+ String data = urLedgerData.get(ledgerId);
+ assertTrue("Bookie " + shutdownBookie
+ + "is not listed in the ledger as missing replica :" + data,
+ data.contains(shutdownBookie));
+ }
+
+ @Test
+ public void testRescheduleOfDelayedAuditOfLostBookiesToStartLater() throws Exception {
+ // wait for a second so that the initial periodic check finishes
+ Thread.sleep(1000);
+
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ Long ledgerId = lh1.getId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created ledger : " + ledgerId);
+ }
+ ledgerList.add(ledgerId);
+ lh1.close();
+
+ final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList
+ .size());
+
+ // wait for 3 seconds before starting the recovery work when a bookie fails
+ urLedgerMgr.setLostBookieRecoveryDelay(3);
+
+ // shutdown a non auditor bookie; choosing non-auditor to avoid another election
+ String shutdownBookie = shutDownNonAuditorBookie();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ledgers to be marked as under replicated");
+ }
+ assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS));
+ assertEquals("under replicated ledgers identified when it was not expected", 0,
+ urLedgerList.size());
+
+ // set lostBookieRecoveryDelay to 4, so the pending AuditTask is resheduled
+ urLedgerMgr.setLostBookieRecoveryDelay(4);
+
+ // since we changed the BookieRecoveryDelay period to 4, the audittask shouldn't have been executed
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ledgers to be marked as under replicated");
+ }
+ assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS));
+ assertEquals("under replicated ledgers identified when it was not expected", 0,
+ urLedgerList.size());
+
+ // wait for 3 seconds (since we already waited for 2 secs) for the ledger to get reported as under replicated
+ assertTrue("audit of lost bookie isn't delayed", underReplicaLatch.await(3, TimeUnit.SECONDS));
+ assertTrue("Ledger is not marked as underreplicated:" + ledgerId,
+ urLedgerList.contains(ledgerId));
+ Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
+ String data = urLedgerData.get(ledgerId);
+ assertTrue("Bookie " + shutdownBookie
+ + "is not listed in the ledger as missing replica :" + data,
+ data.contains(shutdownBookie));
+ }
+
+ @Test
+ public void testTriggerAuditorWithNoPendingAuditTask() throws Exception {
+ // wait for a second so that the initial periodic check finishes
+ Thread.sleep(1000);
+ int lostBookieRecoveryDelayConfValue = baseConf.getLostBookieRecoveryDelay();
+ Auditor auditorBookiesAuditor = getAuditorBookiesAuditor();
+ Future<?> auditTask = auditorBookiesAuditor.getAuditTask();
+ int lostBookieRecoveryDelayBeforeChange = auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange();
+ assertEquals("auditTask is supposed to be null", null, auditTask);
+ assertEquals(
+ "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay",
+ lostBookieRecoveryDelayConfValue, lostBookieRecoveryDelayBeforeChange);
+
+ @Cleanup("shutdown") OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder()
+ .name("test-scheduler")
+ .numThreads(1)
+ .build();
+ @Cleanup MetadataClientDriver driver =
+ MetadataDrivers.getClientDriver(URI.create(baseClientConf.getMetadataServiceUri()));
+ driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.of(zkc));
+
+ // there is no easy way to validate if the Auditor has executed Audit process (Auditor.startAudit),
+ // without shuttingdown Bookie. To test if by resetting LostBookieRecoveryDelay it does Auditing
+ // even when there is no pending AuditTask, following approach is needed.
+
+ // Here we are creating few ledgers ledgermetadata with non-existing bookies as its ensemble.
+ // When Auditor does audit it recognizes these ledgers as underreplicated and mark them as
+ // under-replicated, since these bookies are not available.
+ int numofledgers = 5;
+ Random rand = new Random();
+ for (int i = 0; i < numofledgers; i++) {
+ ArrayList<BookieId> ensemble = new ArrayList<BookieId>();
+ ensemble.add(new BookieSocketAddress("99.99.99.99:9999").toBookieId());
+ ensemble.add(new BookieSocketAddress("11.11.11.11:1111").toBookieId());
+ ensemble.add(new BookieSocketAddress("88.88.88.88:8888").toBookieId());
+
+ long ledgerId = (Math.abs(rand.nextLong())) % 100000000;
+
+ LedgerMetadata metadata = LedgerMetadataBuilder.create()
+ .withId(ledgerId)
+ .withEnsembleSize(3).withWriteQuorumSize(2).withAckQuorumSize(2)
+ .withPassword("passwd".getBytes())
+ .withDigestType(DigestType.CRC32.toApiDigestType())
+ .newEnsembleEntry(0L, ensemble).build();
+
+ try (LedgerManager lm = driver.getLedgerManagerFactory().newLedgerManager()) {
+ lm.createLedgerMetadata(ledgerId, metadata).get(2000, TimeUnit.MILLISECONDS);
+ }
+ ledgerList.add(ledgerId);
+ }
+
+ final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size());
+ urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelayBeforeChange);
+ assertTrue("Audit should be triggered and created ledgers should be marked as underreplicated",
+ underReplicaLatch.await(2, TimeUnit.SECONDS));
+ assertEquals("All the ledgers should be marked as underreplicated", ledgerList.size(), urLedgerList.size());
+
+ auditTask = auditorBookiesAuditor.getAuditTask();
+ assertEquals("auditTask is supposed to be null", null, auditTask);
+ assertEquals(
+ "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay",
+ lostBookieRecoveryDelayBeforeChange, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange());
+ }
+
+ @Test
+ public void testTriggerAuditorWithPendingAuditTask() throws Exception {
+ // wait for a second so that the initial periodic check finishes
+ Thread.sleep(1000);
+
+ Auditor auditorBookiesAuditor = getAuditorBookiesAuditor();
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ Long ledgerId = lh1.getId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created ledger : " + ledgerId);
+ }
+ ledgerList.add(ledgerId);
+ lh1.close();
+
+ final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList
+ .size());
+
+ int lostBookieRecoveryDelay = 5;
+ // wait for 5 seconds before starting the recovery work when a bookie fails
+ urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay);
+
+ // shutdown a non auditor bookie; choosing non-auditor to avoid another election
+ String shutdownBookie = shutDownNonAuditorBookie();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ledgers to be marked as under replicated");
+ }
+ assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS));
+ assertEquals("under replicated ledgers identified when it was not expected", 0,
+ urLedgerList.size());
+
+ Future<?> auditTask = auditorBookiesAuditor.getAuditTask();
+ assertNotSame("auditTask is not supposed to be null", null, auditTask);
+ assertEquals(
+ "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to what we set",
+ lostBookieRecoveryDelay, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange());
+
+ // set lostBookieRecoveryDelay to 5 (previous value), so that Auditor is triggered immediately
+ urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay);
+ assertTrue("audit of lost bookie shouldn't be delayed", underReplicaLatch.await(2, TimeUnit.SECONDS));
+ assertEquals("all under replicated ledgers should be identified", ledgerList.size(),
+ urLedgerList.size());
+
+ Thread.sleep(100);
+ auditTask = auditorBookiesAuditor.getAuditTask();
+ assertEquals("auditTask is supposed to be null", null, auditTask);
+ assertEquals(
+ "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to previously set value",
+ lostBookieRecoveryDelay, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange());
+ }
+
+ @Test
+ public void testTriggerAuditorBySettingDelayToZeroWithPendingAuditTask() throws Exception {
+ // wait for a second so that the initial periodic check finishes
+ Thread.sleep(1000);
+
+ Auditor auditorBookiesAuditor = getAuditorBookiesAuditor();
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ Long ledgerId = lh1.getId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created ledger : " + ledgerId);
+ }
+ ledgerList.add(ledgerId);
+ lh1.close();
+
+ final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList
+ .size());
+
+ int lostBookieRecoveryDelay = 5;
+ // wait for 5 seconds before starting the recovery work when a bookie fails
+ urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay);
+
+ // shutdown a non auditor bookie; choosing non-auditor to avoid another election
+ String shutdownBookie = shutDownNonAuditorBookie();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ledgers to be marked as under replicated");
+ }
+ assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS));
+ assertEquals("under replicated ledgers identified when it was not expected", 0,
+ urLedgerList.size());
+
+ Future<?> auditTask = auditorBookiesAuditor.getAuditTask();
+ assertNotSame("auditTask is not supposed to be null", null, auditTask);
+ assertEquals(
+ "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to what we set",
+ lostBookieRecoveryDelay, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange());
+
+ // set lostBookieRecoveryDelay to 0, so that Auditor is triggered immediately
+ urLedgerMgr.setLostBookieRecoveryDelay(0);
+ assertTrue("audit of lost bookie shouldn't be delayed", underReplicaLatch.await(1, TimeUnit.SECONDS));
+ assertEquals("all under replicated ledgers should be identified", ledgerList.size(),
+ urLedgerList.size());
+
+ Thread.sleep(100);
+ auditTask = auditorBookiesAuditor.getAuditTask();
+ assertEquals("auditTask is supposed to be null", null, auditTask);
+ assertEquals(
+ "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to previously set value",
+ 0, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange());
+ }
+
+ /**
+ * Test audit of bookies is delayed when one bookie is down. But when
+ * another one goes down, the audit is started immediately.
+ */
+ @Test
+ public void testDelayedAuditWithMultipleBookieFailures() throws Exception {
+ // wait for the periodic bookie check to finish
+ Thread.sleep(1000);
+
+ // create a ledger with a bunch of entries
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ Long ledgerId = lh1.getId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created ledger : " + ledgerId);
+ }
+ ledgerList.add(ledgerId);
+ lh1.close();
+
+ CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size());
+
+ // wait for 10 seconds before starting the recovery work when a bookie fails
+ urLedgerMgr.setLostBookieRecoveryDelay(10);
+
+ // shutdown a non auditor bookie to avoid an election
+ String shutdownBookie1 = shutDownNonAuditorBookie();
+
+ // wait for 3 seconds and there shouldn't be any under replicated ledgers
+ // because we have delayed the start of audit by 10 seconds
+ assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(3, TimeUnit.SECONDS));
+ assertEquals("under replicated ledgers identified when it was not expected", 0,
+ urLedgerList.size());
+
+ // Now shutdown the second non auditor bookie; We want to make sure that
+ // the history about having delayed recovery remains. Hence we make sure
+ // we bring down a non auditor bookie. This should cause the audit to take
+ // place immediately and not wait for the remaining 7 seconds to elapse
+ String shutdownBookie2 = shutDownNonAuditorBookie();
+
+ // 2 second grace period for the ledgers to get reported as under replicated
+ Thread.sleep(2000);
+
+ // If the following checks pass, it means that audit happened
+ // within 2 seconds of second bookie going down and it didn't
+ // wait for 7 more seconds. Hence the second bookie failure doesn't
+ // delay the audit
+ assertTrue("Ledger is not marked as underreplicated:" + ledgerId,
+ urLedgerList.contains(ledgerId));
+ Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
+ String data = urLedgerData.get(ledgerId);
+ assertTrue("Bookie " + shutdownBookie1 + shutdownBookie2
+ + " are not listed in the ledger as missing replicas :" + data,
+ data.contains(shutdownBookie1) && data.contains(shutdownBookie2));
+ }
+
+ /**
+ * Test audit of bookies is delayed during rolling upgrade scenario:
+ * a bookies goes down and comes up, the next bookie go down and up and so on.
+ * At any time only one bookie is down.
+ */
+ @Test
+ public void testDelayedAuditWithRollingUpgrade() throws Exception {
+ // wait for the periodic bookie check to finish
+ Thread.sleep(1000);
+
+ // create a ledger with a bunch of entries
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ Long ledgerId = lh1.getId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created ledger : " + ledgerId);
+ }
+ ledgerList.add(ledgerId);
+ lh1.close();
+
+ CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size());
+
+ // wait for 5 seconds before starting the recovery work when a bookie fails
+ urLedgerMgr.setLostBookieRecoveryDelay(5);
+
+ // shutdown a non auditor bookie to avoid an election
+ int idx1 = getShutDownNonAuditorBookieIdx("");
+ ServerConfiguration conf1 = confByIndex(idx1);
+ String shutdownBookie1 = shutdownBookie(idx1);
+
+ // wait for 2 seconds and there shouldn't be any under replicated ledgers
+ // because we have delayed the start of audit by 5 seconds
+ assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS));
+ assertEquals("under replicated ledgers identified when it was not expected", 0,
+ urLedgerList.size());
+
+ // restart the bookie we shut down above
+ startAndAddBookie(conf1);
+
+ // Now to simulate the rolling upgrade, bring down a bookie different from
+ // the one we brought down/up above.
+ String shutdownBookie2 = shutDownNonAuditorBookie(shutdownBookie1);
+
+ // since the first bookie that was brought down/up has come up, there is only
+ // one bookie down at this time. Hence the lost bookie check shouldn't start
+ // immediately; it will start 5 seconds after the second bookie went down
+ assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS));
+ assertEquals("under replicated ledgers identified when it was not expected", 0,
+ urLedgerList.size());
+
+ // wait for a total of 6 seconds(2+4) for the ledgers to get reported as under replicated
+ Thread.sleep(4000);
+
+ // If the following checks pass, it means that auditing happened
+ // after lostBookieRecoveryDelay during rolling upgrade as expected
+ assertTrue("Ledger is not marked as underreplicated:" + ledgerId,
+ urLedgerList.contains(ledgerId));
+ Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
+ String data = urLedgerData.get(ledgerId);
+ assertTrue("Bookie " + shutdownBookie1 + "wrongly listed as missing the ledger: " + data,
+ !data.contains(shutdownBookie1));
+ assertTrue("Bookie " + shutdownBookie2
+ + " is not listed in the ledger as missing replicas :" + data,
+ data.contains(shutdownBookie2));
+ LOG.info("*****************Test Complete");
+ }
+
+ private void waitForAuditToComplete() throws Exception {
+ long endTime = System.currentTimeMillis() + 5_000;
+ while (System.currentTimeMillis() < endTime) {
+ Auditor auditor = getAuditorBookiesAuditor();
+ if (auditor != null) {
+ Future<?> task = auditor.submitAuditTask();
+ task.get(5, TimeUnit.SECONDS);
+ return;
+ }
+ Thread.sleep(100);
+ }
+ throw new TimeoutException("Could not find an audit within 5 seconds");
+ }
+
+ /**
+ * Wait for ledger to be underreplicated, and to be missing all replicas specified.
+ */
+ private boolean waitForLedgerMissingReplicas(Long ledgerId, long secondsToWait, String... replicas)
+ throws Exception {
+ for (int i = 0; i < secondsToWait; i++) {
+ try {
+ UnderreplicatedLedger data = urLedgerMgr.getLedgerUnreplicationInfo(ledgerId);
+ boolean all = true;
+ for (String r : replicas) {
+ all = all && data.getReplicaList().contains(r);
+ }
+ if (all) {
+ return true;
+ }
+ } catch (Exception e) {
+ // may not find node
+ }
+ Thread.sleep(1000);
+ }
+ return false;
+ }
+
+ private CountDownLatch registerUrLedgerWatcher(int count)
+ throws KeeperException, InterruptedException {
+ final CountDownLatch underReplicaLatch = new CountDownLatch(count);
+ for (Long ledgerId : ledgerList) {
+ Watcher urLedgerWatcher = new ChildWatcher(underReplicaLatch);
+ String znode = ZkLedgerUnderreplicationManager.getUrLedgerZnode(underreplicatedPath,
+ ledgerId);
+ zkc.exists(znode, urLedgerWatcher);
+ }
+ return underReplicaLatch;
+ }
+
+ private void doLedgerRereplication(Long... ledgerIds)
+ throws UnavailableException {
+ for (int i = 0; i < ledgerIds.length; i++) {
+ long lid = urLedgerMgr.getLedgerToRereplicate();
+ assertTrue("Received unexpected ledgerid", Arrays.asList(ledgerIds).contains(lid));
+ urLedgerMgr.markLedgerReplicated(lid);
+ urLedgerMgr.releaseUnderreplicatedLedger(lid);
+ }
+ }
+
+ private String shutdownBookie(int bkShutdownIndex) throws Exception {
+ BookieServer bkServer = serverByIndex(bkShutdownIndex);
+ String bookieAddr = bkServer.getBookieId().toString();
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Shutting down bookie:" + bookieAddr);
+ }
+ killBookie(bkShutdownIndex);
+ auditorElectors.get(bookieAddr).shutdown();
+ auditorElectors.remove(bookieAddr);
+ return bookieAddr;
+ }
+
+ private LedgerHandle createAndAddEntriesToLedger() throws BKException,
+ InterruptedException {
+ int numEntriesToWrite = 100;
+ // Create a ledger
+ LedgerHandle lh = bkc.createLedger(digestType, ledgerPassword);
+ LOG.info("Ledger ID: " + lh.getId());
+ addEntry(numEntriesToWrite, lh);
+ return lh;
+ }
+
+ private void addEntry(int numEntriesToWrite, LedgerHandle lh)
+ throws InterruptedException, BKException {
+ final CountDownLatch completeLatch = new CountDownLatch(numEntriesToWrite);
+ final AtomicInteger rc = new AtomicInteger(BKException.Code.OK);
+
+ for (int i = 0; i < numEntriesToWrite; i++) {
+ ByteBuffer entry = ByteBuffer.allocate(4);
+ entry.putInt(rng.nextInt(Integer.MAX_VALUE));
+ entry.position(0);
+ lh.asyncAddEntry(entry.array(), new AddCallback() {
+ public void addComplete(int rc2, LedgerHandle lh, long entryId, Object ctx) {
+ rc.compareAndSet(BKException.Code.OK, rc2);
+ completeLatch.countDown();
+ }
+ }, null);
+ }
+ completeLatch.await();
+ if (rc.get() != BKException.Code.OK) {
+ throw BKException.create(rc.get());
+ }
+
+ }
+
+ private Map<Long, String> getUrLedgerData(Set<Long> urLedgerList)
+ throws KeeperException, InterruptedException {
+ Map<Long, String> urLedgerData = new HashMap<Long, String>();
+ for (Long ledgerId : urLedgerList) {
+ String znode = ZkLedgerUnderreplicationManager.getUrLedgerZnode(underreplicatedPath,
+ ledgerId);
+ byte[] data = zkc.getData(znode, false, null);
+ urLedgerData.put(ledgerId, new String(data));
+ }
+ return urLedgerData;
+ }
+
+ private class ChildWatcher implements Watcher {
+ private final CountDownLatch underReplicaLatch;
+
+ public ChildWatcher(CountDownLatch underReplicaLatch) {
+ this.underReplicaLatch = underReplicaLatch;
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ LOG.info("Received notification for the ledger path : "
+ + event.getPath());
+ for (Long ledgerId : ledgerList) {
+ if (event.getPath().contains(ledgerId + "")) {
+ urLedgerList.add(ledgerId);
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Count down and waiting for next notification");
+ }
+ // count down and waiting for next notification
+ underReplicaLatch.countDown();
+ }
+ }
+
+ private BookieServer getAuditorBookie() throws Exception {
+ List<BookieServer> auditors = new LinkedList<BookieServer>();
+ byte[] data = zkc.getData(electionPath, false, null);
+ assertNotNull("Auditor election failed", data);
+ for (int i = 0; i < bookieCount(); i++) {
+ BookieId bookieId = addressByIndex(i);
+ if (new String(data).contains(bookieId + "")) {
+ auditors.add(serverByIndex(i));
+ }
+ }
+ assertEquals("Multiple Bookies acting as Auditor!", 1, auditors
+ .size());
+ return auditors.get(0);
+ }
+
+ private Auditor getAuditorBookiesAuditor() throws Exception {
+ BookieServer auditorBookieServer = getAuditorBookie();
+ String bookieAddr = auditorBookieServer.getBookieId().toString();
+ return auditorElectors.get(bookieAddr).auditor;
+ }
+
+ private String shutDownNonAuditorBookie() throws Exception {
+ // shutdown bookie which is not an auditor
+ int indexOf = indexOfServer(getAuditorBookie());
+ int bkIndexDownBookie;
+ if (indexOf < lastBookieIndex()) {
+ bkIndexDownBookie = indexOf + 1;
+ } else {
+ bkIndexDownBookie = indexOf - 1;
+ }
+ return shutdownBookie(bkIndexDownBookie);
+ }
+
+ private int getShutDownNonAuditorBookieIdx(String exclude) throws Exception {
+ // shutdown bookie which is not an auditor
+ int indexOf = indexOfServer(getAuditorBookie());
+ int bkIndexDownBookie = 0;
+ for (int i = 0; i <= lastBookieIndex(); i++) {
+ if (i == indexOf || addressByIndex(i).toString().equals(exclude)) {
+ continue;
+ }
+ bkIndexDownBookie = i;
+ break;
+ }
+ return bkIndexDownBookie;
+ }
+
+ private String shutDownNonAuditorBookie(String exclude) throws Exception {
+ return shutdownBookie(getShutDownNonAuditorBookieIdx(exclude));
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java
new file mode 100644
index 00000000000..4acb207570a
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithLedgerManagerFactory;
+import static org.testng.AssertJUnit.assertEquals;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+import lombok.Cleanup;
+import org.apache.bookkeeper.client.ClientUtil;
+import org.apache.bookkeeper.client.LedgerMetadataBuilder;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.conf.TestBKConfiguration;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.net.BookieSocketAddress;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * This test verifies that the period check on the auditor
+ * will pick up on missing data in the client.
+ */
+public class AuditorPeriodicBookieCheckTest extends BookKeeperClusterTestCase {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AuditorPeriodicBookieCheckTest.class);
+
+ private AuditorElector auditorElector = null;
+
+ private static final int CHECK_INTERVAL = 1; // run every second
+
+ public AuditorPeriodicBookieCheckTest() throws Exception {
+ super(3);
+ baseConf.setPageLimit(1); // to make it easy to push ledger out of cache
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
+ conf.setAuditorPeriodicBookieCheckInterval(CHECK_INTERVAL);
+
+ conf.setMetadataServiceUri(
+ metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ conf.setProperty("clientConnectTimeoutMillis", 500);
+ String addr = addressByIndex(0).toString();
+
+ auditorElector = new AuditorElector(addr, conf);
+ auditorElector.start();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ auditorElector.shutdown();
+ super.tearDown();
+ }
+
+ /**
+ * Test that the periodic bookie checker works.
+ */
+ @Test
+ public void testPeriodicBookieCheckInterval() throws Exception {
+ confByIndex(0).setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ runFunctionWithLedgerManagerFactory(confByIndex(0), mFactory -> {
+ try (LedgerManager ledgerManager = mFactory.newLedgerManager()) {
+ @Cleanup final LedgerUnderreplicationManager underReplicationManager =
+ mFactory.newLedgerUnderreplicationManager();
+ long ledgerId = 12345L;
+ ClientUtil.setupLedger(bkc.getLedgerManager(), ledgerId,
+ LedgerMetadataBuilder.create().withEnsembleSize(3)
+ .withWriteQuorumSize(3).withAckQuorumSize(3)
+ .newEnsembleEntry(0L, Lists.newArrayList(
+ new BookieSocketAddress("192.0.2.1", 1000).toBookieId(),
+ getBookie(0),
+ getBookie(1))));
+ long underReplicatedLedger = -1;
+ for (int i = 0; i < 10; i++) {
+ underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
+ if (underReplicatedLedger != -1) {
+ break;
+ }
+ Thread.sleep(CHECK_INTERVAL * 1000);
+ }
+ assertEquals("Ledger should be under replicated", ledgerId, underReplicatedLedger);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e.getMessage(), e);
+ }
+ return null;
+ });
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java
index c761d46c622..e3b0aa37d7e 100644
--- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java
@@ -18,20 +18,42 @@
*/
package org.apache.bookkeeper.replication;
+import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE;
import static org.testng.AssertJUnit.assertEquals;
import static org.testng.AssertJUnit.assertFalse;
import static org.testng.AssertJUnit.assertNotSame;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.fail;
import io.netty.buffer.ByteBuf;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.net.URI;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.bookkeeper.bookie.Bookie;
+import org.apache.bookkeeper.bookie.BookieAccessor;
import org.apache.bookkeeper.bookie.BookieException;
+import org.apache.bookkeeper.bookie.BookieImpl;
+import org.apache.bookkeeper.bookie.IndexPersistenceMgr;
import org.apache.bookkeeper.bookie.TestBookieImpl;
+import org.apache.bookkeeper.client.AsyncCallback.AddCallback;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.LedgerManagerFactory;
@@ -40,8 +62,15 @@ import org.apache.bookkeeper.meta.MetadataBookieDriver;
import org.apache.bookkeeper.meta.MetadataDrivers;
import org.apache.bookkeeper.net.BookieId;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback;
+import org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
+import org.apache.bookkeeper.stats.Counter;
import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.test.TestStatsProvider;
+import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger;
+import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger;
+import org.awaitility.Awaitility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterTest;
@@ -112,6 +141,829 @@ public class AuditorPeriodicCheckTest extends BookKeeperClusterTestCase {
super.tearDown();
}
+ /**
+ * test that the periodic checking will detect corruptions in
+ * the bookie entry log.
+ */
+ @Test
+ public void testEntryLogCorruption() throws Exception {
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
+ underReplicationManager.disableLedgerReplication();
+
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ long ledgerId = lh.getId();
+ for (int i = 0; i < 100; i++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+
+ BookieAccessor.forceFlush((BookieImpl) serverByIndex(0).getBookie());
+
+
+ File ledgerDir = confByIndex(0).getLedgerDirs()[0];
+ ledgerDir = BookieImpl.getCurrentDirectory(ledgerDir);
+ // corrupt of entryLogs
+ File[] entryLogs = ledgerDir.listFiles(new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".log");
+ }
+ });
+ ByteBuffer junk = ByteBuffer.allocate(1024 * 1024);
+ for (File f : entryLogs) {
+ FileOutputStream out = new FileOutputStream(f);
+ out.getChannel().write(junk);
+ out.close();
+ }
+ restartBookies(); // restart to clear read buffers
+
+ underReplicationManager.enableLedgerReplication();
+ long underReplicatedLedger = -1;
+ for (int i = 0; i < 10; i++) {
+ underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
+ if (underReplicatedLedger != -1) {
+ break;
+ }
+ Thread.sleep(CHECK_INTERVAL * 1000);
+ }
+ assertEquals("Ledger should be under replicated", ledgerId, underReplicatedLedger);
+ underReplicationManager.close();
+ }
+
+ /**
+ * test that the period checker will detect corruptions in
+ * the bookie index files.
+ */
+ @Test
+ public void testIndexCorruption() throws Exception {
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+
+ LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
+
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ long ledgerToCorrupt = lh.getId();
+ for (int i = 0; i < 100; i++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+
+ // push ledgerToCorrupt out of page cache (bookie is configured to only use 1 page)
+ lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ for (int i = 0; i < 100; i++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+
+ BookieAccessor.forceFlush((BookieImpl) serverByIndex(0).getBookie());
+
+ File ledgerDir = confByIndex(0).getLedgerDirs()[0];
+ ledgerDir = BookieImpl.getCurrentDirectory(ledgerDir);
+
+ // corrupt of entryLogs
+ File index = new File(ledgerDir, IndexPersistenceMgr.getLedgerName(ledgerToCorrupt));
+ LOG.info("file to corrupt{}", index);
+ ByteBuffer junk = ByteBuffer.allocate(1024 * 1024);
+ FileOutputStream out = new FileOutputStream(index);
+ out.getChannel().write(junk);
+ out.close();
+
+ long underReplicatedLedger = -1;
+ for (int i = 0; i < 15; i++) {
+ underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
+ if (underReplicatedLedger != -1) {
+ break;
+ }
+ Thread.sleep(CHECK_INTERVAL * 1000);
+ }
+ assertEquals("Ledger should be under replicated", ledgerToCorrupt, underReplicatedLedger);
+ underReplicationManager.close();
+ }
+
+ /**
+ * Test that the period checker will not run when auto replication has been disabled.
+ */
+ @Test
+ public void testPeriodicCheckWhenDisabled() throws Exception {
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
+ final int numLedgers = 10;
+ final int numMsgs = 2;
+ final CountDownLatch completeLatch = new CountDownLatch(numMsgs * numLedgers);
+ final AtomicInteger rc = new AtomicInteger(BKException.Code.OK);
+
+ List<LedgerHandle> lhs = new ArrayList<LedgerHandle>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ lhs.add(lh);
+ for (int j = 0; j < 2; j++) {
+ lh.asyncAddEntry("testdata".getBytes(), new AddCallback() {
+ public void addComplete(int rc2, LedgerHandle lh, long entryId, Object ctx) {
+ if (rc.compareAndSet(BKException.Code.OK, rc2)) {
+ LOG.info("Failed to add entry : {}", BKException.getMessage(rc2));
+ }
+ completeLatch.countDown();
+ }
+ }, null);
+ }
+ }
+ completeLatch.await();
+ if (rc.get() != BKException.Code.OK) {
+ throw BKException.create(rc.get());
+ }
+
+ for (LedgerHandle lh : lhs) {
+ lh.close();
+ }
+
+ underReplicationManager.disableLedgerReplication();
+
+ final AtomicInteger numReads = new AtomicInteger(0);
+ ServerConfiguration conf = killBookie(0);
+
+ Bookie deadBookie = new TestBookieImpl(conf) {
+ @Override
+ public ByteBuf readEntry(long ledgerId, long entryId)
+ throws IOException, NoLedgerException {
+ // we want to disable during checking
+ numReads.incrementAndGet();
+ throw new IOException("Fake I/O exception");
+ }
+ };
+ startAndAddBookie(conf, deadBookie);
+
+ Thread.sleep(CHECK_INTERVAL * 2000);
+ assertEquals("Nothing should have tried to read", 0, numReads.get());
+ underReplicationManager.enableLedgerReplication();
+ Thread.sleep(CHECK_INTERVAL * 2000); // give it time to run
+
+ underReplicationManager.disableLedgerReplication();
+ // give it time to stop, from this point nothing new should be marked
+ Thread.sleep(CHECK_INTERVAL * 2000);
+
+ int numUnderreplicated = 0;
+ long underReplicatedLedger = -1;
+ do {
+ underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
+ if (underReplicatedLedger == -1) {
+ break;
+ }
+ numUnderreplicated++;
+
+ underReplicationManager.markLedgerReplicated(underReplicatedLedger);
+ } while (underReplicatedLedger != -1);
+
+ Thread.sleep(CHECK_INTERVAL * 2000); // give a chance to run again (it shouldn't, it's disabled)
+
+ // ensure that nothing is marked as underreplicated
+ underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
+ assertEquals("There should be no underreplicated ledgers", -1, underReplicatedLedger);
+
+ LOG.info("{} of {} ledgers underreplicated", numUnderreplicated, numUnderreplicated);
+ assertTrue("All should be underreplicated",
+ numUnderreplicated <= numLedgers && numUnderreplicated > 0);
+ }
+
+ /**
+ * Test that the period check will succeed if a ledger is deleted midway.
+ */
+ @Test
+ public void testPeriodicCheckWhenLedgerDeleted() throws Exception {
+ for (AuditorElector e : auditorElectors.values()) {
+ e.shutdown();
+ }
+
+ final int numLedgers = 10;
+ List<Long> ids = new LinkedList<Long>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ ids.add(lh.getId());
+ for (int j = 0; j < 2; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+
+ try (final Auditor auditor = new Auditor(
+ BookieImpl.getBookieId(confByIndex(0)).toString(),
+ confByIndex(0), NullStatsLogger.INSTANCE)) {
+ final AtomicBoolean exceptionCaught = new AtomicBoolean(false);
+ final CountDownLatch latch = new CountDownLatch(1);
+ Thread t = new Thread() {
+ public void run() {
+ try {
+ latch.countDown();
+ for (int i = 0; i < numLedgers; i++) {
+ ((AuditorCheckAllLedgersTask) auditor.auditorCheckAllLedgersTask).checkAllLedgers();
+ }
+ } catch (Exception e) {
+ LOG.error("Caught exception while checking all ledgers", e);
+ exceptionCaught.set(true);
+ }
+ }
+ };
+ t.start();
+ latch.await();
+ for (Long id : ids) {
+ bkc.deleteLedger(id);
+ }
+ t.join();
+ assertFalse("Shouldn't have thrown exception", exceptionCaught.get());
+ }
+ }
+
+ @Test
+ public void testGetLedgerFromZookeeperThrottled() throws Exception {
+ final int numberLedgers = 30;
+
+ // write ledgers into bookkeeper cluster
+ try {
+ for (AuditorElector e : auditorElectors.values()) {
+ e.shutdown();
+ }
+
+ for (int i = 0; i < numberLedgers; ++i) {
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ for (int j = 0; j < 5; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+ } catch (InterruptedException | BKException e) {
+ LOG.error("Failed to shutdown auditor elector or write data to ledgers ", e);
+ fail();
+ }
+
+ // create auditor and call `checkAllLedgers`
+ ServerConfiguration configuration = confByIndex(0);
+ configuration.setAuditorMaxNumberOfConcurrentOpenLedgerOperations(10);
+
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ Counter numLedgersChecked = statsLogger
+ .getCounter(ReplicationStats.NUM_LEDGERS_CHECKED);
+ Auditor auditor = new Auditor(BookieImpl.getBookieId(configuration).toString(),
+ configuration, statsLogger);
+
+ try {
+ ((AuditorCheckAllLedgersTask) auditor.auditorCheckAllLedgersTask).checkAllLedgers();
+ assertEquals("NUM_LEDGERS_CHECKED", numberLedgers, (long) numLedgersChecked.get());
+ } catch (Exception e) {
+ LOG.error("Caught exception while checking all ledgers ", e);
+ fail();
+ }
+ }
+
+ @Test
+ public void testInitialDelayOfCheckAllLedgers() throws Exception {
+ for (AuditorElector e : auditorElectors.values()) {
+ e.shutdown();
+ }
+
+ final int numLedgers = 10;
+ List<Long> ids = new LinkedList<Long>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ ids.add(lh.getId());
+ for (int j = 0; j < 2; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager();
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ validateInitialDelayOfCheckAllLedgers(urm, -1, 1000, servConf, bkc);
+ validateInitialDelayOfCheckAllLedgers(urm, 999, 1000, servConf, bkc);
+ validateInitialDelayOfCheckAllLedgers(urm, 1001, 1000, servConf, bkc);
+ }
+
+ void validateInitialDelayOfCheckAllLedgers(LedgerUnderreplicationManager urm, long timeSinceLastExecutedInSecs,
+ long auditorPeriodicCheckInterval, ServerConfiguration servConf,
+ BookKeeper bkc)
+ throws UnavailableException, UnknownHostException, InterruptedException {
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ TestOpStatsLogger checkAllLedgersStatsLogger = (TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.CHECK_ALL_LEDGERS_TIME);
+ servConf.setAuditorPeriodicCheckInterval(auditorPeriodicCheckInterval);
+ servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0);
+ servConf.setAuditorPeriodicBookieCheckInterval(0);
+
+ final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false,
+ statsLogger, null);
+ CountDownLatch latch = auditor.getLatch();
+ assertEquals("CHECK_ALL_LEDGERS_TIME SuccessCount", 0, checkAllLedgersStatsLogger.getSuccessCount());
+ long curTimeBeforeStart = System.currentTimeMillis();
+ long checkAllLedgersCTime = -1;
+ long initialDelayInMsecs = -1;
+ long nextExpectedCheckAllLedgersExecutionTime = -1;
+ long bufferTimeInMsecs = 12000L;
+ if (timeSinceLastExecutedInSecs == -1) {
+ /*
+ * if we are setting checkAllLedgersCTime to -1, it means that
+ * checkAllLedgers hasn't run before. So initialDelay for
+ * checkAllLedgers should be 0.
+ */
+ checkAllLedgersCTime = -1;
+ initialDelayInMsecs = 0;
+ } else {
+ checkAllLedgersCTime = curTimeBeforeStart - timeSinceLastExecutedInSecs * 1000L;
+ initialDelayInMsecs = timeSinceLastExecutedInSecs > auditorPeriodicCheckInterval ? 0
+ : (auditorPeriodicCheckInterval - timeSinceLastExecutedInSecs) * 1000L;
+ }
+ /*
+ * next checkAllLedgers should happen atleast after
+ * nextExpectedCheckAllLedgersExecutionTime.
+ */
+ nextExpectedCheckAllLedgersExecutionTime = curTimeBeforeStart + initialDelayInMsecs;
+
+ urm.setCheckAllLedgersCTime(checkAllLedgersCTime);
+ auditor.start();
+ /*
+ * since auditorPeriodicCheckInterval are higher values (in the order of
+ * 100s of seconds), its ok bufferTimeInMsecs to be ` 10 secs.
+ */
+ assertTrue("checkAllLedgers should have executed with initialDelay " + initialDelayInMsecs,
+ latch.await(initialDelayInMsecs + bufferTimeInMsecs, TimeUnit.MILLISECONDS));
+ for (int i = 0; i < 10; i++) {
+ Thread.sleep(100);
+ if (checkAllLedgersStatsLogger.getSuccessCount() >= 1) {
+ break;
+ }
+ }
+ assertEquals("CHECK_ALL_LEDGERS_TIME SuccessCount", 1, checkAllLedgersStatsLogger.getSuccessCount());
+ long currentCheckAllLedgersCTime = urm.getCheckAllLedgersCTime();
+ assertTrue(
+ "currentCheckAllLedgersCTime: " + currentCheckAllLedgersCTime
+ + " should be greater than nextExpectedCheckAllLedgersExecutionTime: "
+ + nextExpectedCheckAllLedgersExecutionTime,
+ currentCheckAllLedgersCTime > nextExpectedCheckAllLedgersExecutionTime);
+ assertTrue(
+ "currentCheckAllLedgersCTime: " + currentCheckAllLedgersCTime
+ + " should be lesser than nextExpectedCheckAllLedgersExecutionTime+bufferTimeInMsecs: "
+ + (nextExpectedCheckAllLedgersExecutionTime + bufferTimeInMsecs),
+ currentCheckAllLedgersCTime < (nextExpectedCheckAllLedgersExecutionTime + bufferTimeInMsecs));
+ auditor.close();
+ }
+
+ @Test
+ public void testInitialDelayOfPlacementPolicyCheck() throws Exception {
+ for (AuditorElector e : auditorElectors.values()) {
+ e.shutdown();
+ }
+
+ final int numLedgers = 10;
+ List<Long> ids = new LinkedList<Long>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ ids.add(lh.getId());
+ for (int j = 0; j < 2; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager();
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ validateInitialDelayOfPlacementPolicyCheck(urm, -1, 1000, servConf, bkc);
+ validateInitialDelayOfPlacementPolicyCheck(urm, 999, 1000, servConf, bkc);
+ validateInitialDelayOfPlacementPolicyCheck(urm, 1001, 1000, servConf, bkc);
+ }
+
+ void validateInitialDelayOfPlacementPolicyCheck(LedgerUnderreplicationManager urm, long timeSinceLastExecutedInSecs,
+ long auditorPeriodicPlacementPolicyCheckInterval,
+ ServerConfiguration servConf, BookKeeper bkc)
+ throws UnavailableException, UnknownHostException, InterruptedException {
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ TestOpStatsLogger placementPolicyCheckStatsLogger = (TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME);
+ servConf.setAuditorPeriodicPlacementPolicyCheckInterval(auditorPeriodicPlacementPolicyCheckInterval);
+ servConf.setAuditorPeriodicCheckInterval(0);
+ servConf.setAuditorPeriodicBookieCheckInterval(0);
+
+ final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false,
+ statsLogger, null);
+ CountDownLatch latch = auditor.getLatch();
+ assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, placementPolicyCheckStatsLogger.getSuccessCount());
+ long curTimeBeforeStart = System.currentTimeMillis();
+ long placementPolicyCheckCTime = -1;
+ long initialDelayInMsecs = -1;
+ long nextExpectedPlacementPolicyCheckExecutionTime = -1;
+ long bufferTimeInMsecs = 20000L;
+ if (timeSinceLastExecutedInSecs == -1) {
+ /*
+ * if we are setting placementPolicyCheckCTime to -1, it means that
+ * placementPolicyCheck hasn't run before. So initialDelay for
+ * placementPolicyCheck should be 0.
+ */
+ placementPolicyCheckCTime = -1;
+ initialDelayInMsecs = 0;
+ } else {
+ placementPolicyCheckCTime = curTimeBeforeStart - timeSinceLastExecutedInSecs * 1000L;
+ initialDelayInMsecs = timeSinceLastExecutedInSecs > auditorPeriodicPlacementPolicyCheckInterval ? 0
+ : (auditorPeriodicPlacementPolicyCheckInterval - timeSinceLastExecutedInSecs) * 1000L;
+ }
+ /*
+ * next placementPolicyCheck should happen atleast after
+ * nextExpectedPlacementPolicyCheckExecutionTime.
+ */
+ nextExpectedPlacementPolicyCheckExecutionTime = curTimeBeforeStart + initialDelayInMsecs;
+
+ urm.setPlacementPolicyCheckCTime(placementPolicyCheckCTime);
+ auditor.start();
+ /*
+ * since auditorPeriodicPlacementPolicyCheckInterval are higher values (in the
+ * order of 100s of seconds), its ok bufferTimeInMsecs to be ` 20 secs.
+ */
+ assertTrue("placementPolicyCheck should have executed with initialDelay " + initialDelayInMsecs,
+ latch.await(initialDelayInMsecs + bufferTimeInMsecs, TimeUnit.MILLISECONDS));
+ for (int i = 0; i < 20; i++) {
+ Thread.sleep(100);
+ if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) {
+ break;
+ }
+ }
+ assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, placementPolicyCheckStatsLogger.getSuccessCount());
+ long currentPlacementPolicyCheckCTime = urm.getPlacementPolicyCheckCTime();
+ assertTrue(
+ "currentPlacementPolicyCheckCTime: " + currentPlacementPolicyCheckCTime
+ + " should be greater than nextExpectedPlacementPolicyCheckExecutionTime: "
+ + nextExpectedPlacementPolicyCheckExecutionTime,
+ currentPlacementPolicyCheckCTime > nextExpectedPlacementPolicyCheckExecutionTime);
+ assertTrue(
+ "currentPlacementPolicyCheckCTime: " + currentPlacementPolicyCheckCTime
+ + " should be lesser than nextExpectedPlacementPolicyCheckExecutionTime+bufferTimeInMsecs: "
+ + (nextExpectedPlacementPolicyCheckExecutionTime + bufferTimeInMsecs),
+ currentPlacementPolicyCheckCTime < (nextExpectedPlacementPolicyCheckExecutionTime + bufferTimeInMsecs));
+ auditor.close();
+ }
+
+ @Test
+ public void testInitialDelayOfReplicasCheck() throws Exception {
+ for (AuditorElector e : auditorElectors.values()) {
+ e.shutdown();
+ }
+
+ LedgerHandle lh = bkc.createLedger(3, 2, DigestType.CRC32, "passwd".getBytes());
+ for (int j = 0; j < 5; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+
+ long ledgerId = 100000L;
+ lh = bkc.createLedgerAdv(ledgerId, 3, 2, 2, DigestType.CRC32, "passwd".getBytes(), null);
+ lh.close();
+
+ ledgerId = 100001234L;
+ lh = bkc.createLedgerAdv(ledgerId, 3, 3, 2, DigestType.CRC32, "passwd".getBytes(), null);
+ for (int j = 0; j < 4; j++) {
+ lh.addEntry(j, "testdata".getBytes());
+ }
+ lh.close();
+
+ ledgerId = 991234L;
+ lh = bkc.createLedgerAdv(ledgerId, 3, 2, 2, DigestType.CRC32, "passwd".getBytes(), null);
+ lh.addEntry(0, "testdata".getBytes());
+ lh.close();
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager();
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ validateInitialDelayOfReplicasCheck(urm, -1, 1000, servConf, bkc);
+ validateInitialDelayOfReplicasCheck(urm, 999, 1000, servConf, bkc);
+ validateInitialDelayOfReplicasCheck(urm, 1001, 1000, servConf, bkc);
+ }
+
+ void validateInitialDelayOfReplicasCheck(LedgerUnderreplicationManager urm, long timeSinceLastExecutedInSecs,
+ long auditorPeriodicReplicasCheckInterval, ServerConfiguration servConf,
+ BookKeeper bkc)
+ throws UnavailableException, UnknownHostException, InterruptedException {
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ TestOpStatsLogger replicasCheckStatsLogger = (TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME);
+ servConf.setAuditorPeriodicReplicasCheckInterval(auditorPeriodicReplicasCheckInterval);
+ servConf.setAuditorPeriodicCheckInterval(0);
+ servConf.setAuditorPeriodicBookieCheckInterval(0);
+ final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false,
+ statsLogger, null);
+ CountDownLatch latch = auditor.getLatch();
+ assertEquals("REPLICAS_CHECK_TIME SuccessCount", 0, replicasCheckStatsLogger.getSuccessCount());
+ long curTimeBeforeStart = System.currentTimeMillis();
+ long replicasCheckCTime = -1;
+ long initialDelayInMsecs = -1;
+ long nextExpectedReplicasCheckExecutionTime = -1;
+ long bufferTimeInMsecs = 20000L;
+ if (timeSinceLastExecutedInSecs == -1) {
+ /*
+ * if we are setting replicasCheckCTime to -1, it means that
+ * replicasCheck hasn't run before. So initialDelay for
+ * replicasCheck should be 0.
+ */
+ replicasCheckCTime = -1;
+ initialDelayInMsecs = 0;
+ } else {
+ replicasCheckCTime = curTimeBeforeStart - timeSinceLastExecutedInSecs * 1000L;
+ initialDelayInMsecs = timeSinceLastExecutedInSecs > auditorPeriodicReplicasCheckInterval ? 0
+ : (auditorPeriodicReplicasCheckInterval - timeSinceLastExecutedInSecs) * 1000L;
+ }
+ /*
+ * next replicasCheck should happen atleast after
+ * nextExpectedReplicasCheckExecutionTime.
+ */
+ nextExpectedReplicasCheckExecutionTime = curTimeBeforeStart + initialDelayInMsecs;
+
+ urm.setReplicasCheckCTime(replicasCheckCTime);
+ auditor.start();
+ /*
+ * since auditorPeriodicReplicasCheckInterval are higher values (in the
+ * order of 100s of seconds), its ok bufferTimeInMsecs to be ` 20 secs.
+ */
+ assertTrue("replicasCheck should have executed with initialDelay " + initialDelayInMsecs,
+ latch.await(initialDelayInMsecs + bufferTimeInMsecs, TimeUnit.MILLISECONDS));
+ for (int i = 0; i < 20; i++) {
+ Thread.sleep(100);
+ if (replicasCheckStatsLogger.getSuccessCount() >= 1) {
+ break;
+ }
+ }
+ assertEquals("REPLICAS_CHECK_TIME SuccessCount", 1, replicasCheckStatsLogger.getSuccessCount());
+ long currentReplicasCheckCTime = urm.getReplicasCheckCTime();
+ assertTrue(
+ "currentReplicasCheckCTime: " + currentReplicasCheckCTime
+ + " should be greater than nextExpectedReplicasCheckExecutionTime: "
+ + nextExpectedReplicasCheckExecutionTime,
+ currentReplicasCheckCTime > nextExpectedReplicasCheckExecutionTime);
+ assertTrue(
+ "currentReplicasCheckCTime: " + currentReplicasCheckCTime
+ + " should be lesser than nextExpectedReplicasCheckExecutionTime+bufferTimeInMsecs: "
+ + (nextExpectedReplicasCheckExecutionTime + bufferTimeInMsecs),
+ currentReplicasCheckCTime < (nextExpectedReplicasCheckExecutionTime + bufferTimeInMsecs));
+ auditor.close();
+ }
+
+ @Test
+ public void testDelayBookieAuditOfCheckAllLedgers() throws Exception {
+ for (AuditorElector e : auditorElectors.values()) {
+ e.shutdown();
+ }
+
+ final int numLedgers = 10;
+ List<Long> ids = new LinkedList<Long>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ ids.add(lh.getId());
+ for (int j = 0; j < 2; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager();
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ Counter numBookieAuditsDelayed =
+ statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED);
+ TestOpStatsLogger underReplicatedLedgerTotalSizeStatsLogger = (TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.UNDER_REPLICATED_LEDGERS_TOTAL_SIZE);
+
+ servConf.setAuditorPeriodicCheckInterval(1);
+ servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0);
+ servConf.setAuditorPeriodicBookieCheckInterval(Long.MAX_VALUE);
+
+ urm.setLostBookieRecoveryDelay(Integer.MAX_VALUE);
+
+ AtomicBoolean canRun = new AtomicBoolean(false);
+
+ final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc,
+ false, statsLogger, canRun);
+ final CountDownLatch latch = auditor.getLatch();
+
+ auditor.start();
+
+ killBookie(addressByIndex(0));
+
+ Awaitility.await().untilAsserted(() -> assertEquals(1, (long) numBookieAuditsDelayed.get()));
+ final Future<?> auditTask = auditor.auditTask;
+ assertTrue(auditTask != null && !auditTask.isDone());
+
+ canRun.set(true);
+
+ assertTrue(latch.await(10, TimeUnit.SECONDS));
+ assertTrue(auditor.auditTask.equals(auditTask)
+ && auditor.auditTask != null && !auditor.auditTask.isDone());
+ // wrong num is numLedgers, right num is 0
+ assertEquals("UNDER_REPLICATED_LEDGERS_TOTAL_SIZE",
+ 0,
+ underReplicatedLedgerTotalSizeStatsLogger.getSuccessCount());
+
+ auditor.close();
+ }
+
+ @Test
+ public void testDelayBookieAuditOfPlacementPolicy() throws Exception {
+ for (AuditorElector e : auditorElectors.values()) {
+ e.shutdown();
+ }
+
+ final int numLedgers = 10;
+ List<Long> ids = new LinkedList<Long>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ ids.add(lh.getId());
+ for (int j = 0; j < 2; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager();
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ Counter numBookieAuditsDelayed =
+ statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED);
+ TestOpStatsLogger placementPolicyCheckTime = (TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME);
+
+ servConf.setAuditorPeriodicCheckInterval(0);
+ servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1);
+ servConf.setAuditorPeriodicBookieCheckInterval(Long.MAX_VALUE);
+
+ urm.setLostBookieRecoveryDelay(Integer.MAX_VALUE);
+
+ AtomicBoolean canRun = new AtomicBoolean(false);
+
+ final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc,
+ false, statsLogger, canRun);
+ final CountDownLatch latch = auditor.getLatch();
+
+ auditor.start();
+
+ killBookie(addressByIndex(0));
+
+ Awaitility.await().untilAsserted(() -> assertEquals(1, (long) numBookieAuditsDelayed.get()));
+ final Future<?> auditTask = auditor.auditTask;
+ assertTrue(auditTask != null && !auditTask.isDone());
+ assertEquals("PLACEMENT_POLICY_CHECK_TIME", 0, placementPolicyCheckTime.getSuccessCount());
+
+ canRun.set(true);
+
+ assertTrue(latch.await(10, TimeUnit.SECONDS));
+ assertTrue(auditor.auditTask.equals(auditTask)
+ && auditor.auditTask != null && !auditor.auditTask.isDone());
+ // wrong successCount is > 0, right successCount is = 0
+ assertEquals("PLACEMENT_POLICY_CHECK_TIME", 0, placementPolicyCheckTime.getSuccessCount());
+
+ auditor.close();
+ }
+
+ @Test
+ public void testDelayBookieAuditOfReplicasCheck() throws Exception {
+ for (AuditorElector e : auditorElectors.values()) {
+ e.shutdown();
+ }
+
+ final int numLedgers = 10;
+ List<Long> ids = new LinkedList<Long>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ ids.add(lh.getId());
+ for (int j = 0; j < 2; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager();
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ Counter numBookieAuditsDelayed =
+ statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED);
+ TestOpStatsLogger replicasCheckTime = (TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME);
+
+ servConf.setAuditorPeriodicCheckInterval(0);
+ servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0);
+ servConf.setAuditorPeriodicBookieCheckInterval(Long.MAX_VALUE);
+ servConf.setAuditorPeriodicReplicasCheckInterval(1);
+
+ urm.setLostBookieRecoveryDelay(Integer.MAX_VALUE);
+
+ AtomicBoolean canRun = new AtomicBoolean(false);
+
+ final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc,
+ false, statsLogger, canRun);
+ final CountDownLatch latch = auditor.getLatch();
+
+ auditor.start();
+
+ killBookie(addressByIndex(0));
+
+ Awaitility.await().untilAsserted(() -> assertEquals(1, (long) numBookieAuditsDelayed.get()));
+ final Future<?> auditTask = auditor.auditTask;
+ assertTrue(auditTask != null && !auditTask.isDone());
+ assertEquals("REPLICAS_CHECK_TIME", 0, replicasCheckTime.getSuccessCount());
+
+ canRun.set(true);
+
+ assertTrue(latch.await(10, TimeUnit.SECONDS));
+ assertTrue(auditor.auditTask.equals(auditTask)
+ && auditor.auditTask != null && !auditor.auditTask.isDone());
+ // wrong successCount is > 0, right successCount is = 0
+ assertEquals("REPLICAS_CHECK_TIME", 0, replicasCheckTime.getSuccessCount());
+
+ auditor.close();
+ }
+
+ static class TestAuditor extends Auditor {
+
+ final AtomicReference<CountDownLatch> latchRef = new AtomicReference<CountDownLatch>(new CountDownLatch(1));
+
+ public TestAuditor(String bookieIdentifier, ServerConfiguration conf, BookKeeper bkc, boolean ownBkc,
+ StatsLogger statsLogger, AtomicBoolean exceptedRun) throws UnavailableException {
+ super(bookieIdentifier, conf, bkc, ownBkc, statsLogger);
+ renewAuditorTestWrapperTask(exceptedRun);
+ }
+
+ public TestAuditor(String bookieIdentifier, ServerConfiguration conf, BookKeeper bkc, boolean ownBkc,
+ BookKeeperAdmin bkadmin, boolean ownadmin, StatsLogger statsLogger,
+ AtomicBoolean exceptedRun) throws UnavailableException {
+ super(bookieIdentifier, conf, bkc, ownBkc, bkadmin, ownadmin, statsLogger);
+ renewAuditorTestWrapperTask(exceptedRun);
+ }
+
+ public TestAuditor(final String bookieIdentifier, ServerConfiguration conf, StatsLogger statsLogger,
+ AtomicBoolean exceptedRun)
+ throws UnavailableException {
+ super(bookieIdentifier, conf, statsLogger);
+ renewAuditorTestWrapperTask(exceptedRun);
+ }
+
+ private void renewAuditorTestWrapperTask(AtomicBoolean exceptedRun) {
+ super.auditorCheckAllLedgersTask =
+ new AuditorTestWrapperTask(super.auditorCheckAllLedgersTask, latchRef, exceptedRun);
+ super.auditorPlacementPolicyCheckTask =
+ new AuditorTestWrapperTask(super.auditorPlacementPolicyCheckTask, latchRef, exceptedRun);
+ super.auditorReplicasCheckTask =
+ new AuditorTestWrapperTask(super.auditorReplicasCheckTask, latchRef, exceptedRun);
+ }
+
+ CountDownLatch getLatch() {
+ return latchRef.get();
+ }
+
+ void setLatch(CountDownLatch latch) {
+ latchRef.set(latch);
+ }
+
+ private static class AuditorTestWrapperTask extends AuditorTask {
+ private final AuditorTask innerTask;
+ private final AtomicReference<CountDownLatch> latchRef;
+ private final AtomicBoolean exceptedRun;
+
+ AuditorTestWrapperTask(AuditorTask innerTask,
+ AtomicReference<CountDownLatch> latchRef,
+ AtomicBoolean exceptedRun) {
+ super(null, null, null, null, null,
+ null, null);
+ this.innerTask = innerTask;
+ this.latchRef = latchRef;
+ this.exceptedRun = exceptedRun;
+ }
+
+ @Override
+ protected void runTask() {
+ if (exceptedRun == null || exceptedRun.get()) {
+ innerTask.runTask();
+ latchRef.get().countDown();
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ innerTask.shutdown();
+ }
+ }
+ }
+
private BookieId replaceBookieWithWriteFailingBookie(LedgerHandle lh) throws Exception {
int bookieIdx = -1;
Long entryId = lh.getLedgerMetadata().getAllEnsembles().firstKey();
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java
new file mode 100644
index 00000000000..1bafb8589d9
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE;
+import static org.testng.AssertJUnit.assertEquals;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.test.TestStatsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test {@link AuditorPlacementPolicyCheckTask}.
+ */
+public class AuditorPlacementPolicyCheckTaskTest extends BookKeeperClusterTestCase {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AuditorPlacementPolicyCheckTaskTest.class);
+
+ private BookKeeperAdmin admin;
+ private LedgerManager ledgerManager;
+ private LedgerUnderreplicationManager ledgerUnderreplicationManager;
+
+ public AuditorPlacementPolicyCheckTaskTest() throws Exception {
+ super(3);
+ baseConf.setPageLimit(1);
+ baseConf.setAutoRecoveryDaemonEnabled(false);
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ baseClientConf.setMetadataServiceUri(
+ metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ final BookKeeper bookKeeper = new BookKeeper(baseClientConf);
+ admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new ClientConfiguration(baseClientConf));
+ LedgerManagerFactory ledgerManagerFactory = bookKeeper.getLedgerManagerFactory();
+ ledgerManager = ledgerManagerFactory.newLedgerManager();
+ ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ if (ledgerManager != null) {
+ ledgerManager.close();
+ }
+ if (ledgerUnderreplicationManager != null) {
+ ledgerUnderreplicationManager.close();
+ }
+ super.tearDown();
+ }
+
+ @Test
+ public void testPlacementPolicyCheck() throws BKException, InterruptedException {
+
+ // 1. create ledgers
+ final int numLedgers = 10;
+ List<Long> ids = new LinkedList<Long>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes());
+ ids.add(lh.getId());
+ for (int j = 0; j < 2; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+
+ // 2. init auditorPlacementPolicyCheckTask
+ final TestStatsProvider statsProvider = new TestStatsProvider();
+ final TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ final AuditorStats auditorStats = new AuditorStats(statsLogger);
+
+ AuditorPlacementPolicyCheckTask auditorPlacementPolicyCheckTask = new AuditorPlacementPolicyCheckTask(
+ baseConf, auditorStats, admin, ledgerManager,
+ ledgerUnderreplicationManager, null, (flag, throwable) -> flag.set(false));
+
+ // 3. placementPolicyCheck
+ auditorPlacementPolicyCheckTask.runTask();
+
+ // 4. verify
+ assertEquals("PLACEMENT_POLICY_CHECK_TIME", 1, ((TestStatsProvider.TestOpStatsLogger)
+ statsLogger.getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME)).getSuccessCount());
+ assertEquals("numOfClosedLedgersAuditedInPlacementPolicyCheck",
+ numLedgers,
+ auditorPlacementPolicyCheckTask.getNumOfClosedLedgersAuditedInPlacementPolicyCheck().get());
+ assertEquals("numOfLedgersFoundNotAdheringInPlacementPolicyCheck",
+ numLedgers,
+ auditorPlacementPolicyCheckTask.getNumOfLedgersFoundNotAdheringInPlacementPolicyCheck().get());
+ }
+
+}
\ No newline at end of file
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java
new file mode 100644
index 00000000000..159a4e88a33
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java
@@ -0,0 +1,861 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl.REPP_DNS_RESOLVER_CLASS;
+import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.bookie.BookieImpl;
+import org.apache.bookkeeper.client.LedgerMetadataBuilder;
+import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy;
+import org.apache.bookkeeper.client.ZoneawareEnsemblePlacementPolicy;
+import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.discover.BookieServiceInfo;
+import org.apache.bookkeeper.discover.RegistrationManager;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.meta.MetadataBookieDriver;
+import org.apache.bookkeeper.meta.MetadataDrivers;
+import org.apache.bookkeeper.meta.exceptions.MetadataException;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.net.BookieSocketAddress;
+import org.apache.bookkeeper.replication.AuditorPeriodicCheckTest.TestAuditor;
+import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException;
+import org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
+import org.apache.bookkeeper.stats.Gauge;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.test.TestStatsProvider;
+import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger;
+import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger;
+import org.apache.bookkeeper.util.StaticDNSResolver;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.zookeeper.KeeperException;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Tests the logic of Auditor's PlacementPolicyCheck.
+ */
+public class AuditorPlacementPolicyCheckTest extends BookKeeperClusterTestCase {
+ private MetadataBookieDriver driver;
+
+ public AuditorPlacementPolicyCheckTest() throws Exception {
+ super(1);
+ baseConf.setPageLimit(1); // to make it easy to push ledger out of cache
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ StaticDNSResolver.reset();
+
+ URI uri = URI.create(confByIndex(0).getMetadataServiceUri().replaceAll("zk://", "metadata-store:")
+ .replaceAll("/ledgers", ""));
+ driver = MetadataDrivers.getBookieDriver(uri);
+ ServerConfiguration serverConfiguration = new ServerConfiguration(confByIndex(0));
+ serverConfiguration.setMetadataServiceUri(
+ serverConfiguration.getMetadataServiceUri().replaceAll("zk://", "metadata-store:")
+ .replaceAll("/ledgers", ""));
+ driver.initialize(serverConfiguration, NullStatsLogger.INSTANCE);
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ if (null != driver) {
+ driver.close();
+ }
+ super.tearDown();
+ }
+
+ @Test
+ public void testPlacementPolicyCheckWithBookiesFromDifferentRacks() throws Exception {
+ int numOfBookies = 5;
+ List<BookieId> bookieAddresses = new ArrayList<>();
+ BookieSocketAddress bookieAddress;
+ RegistrationManager regManager = driver.createRegistrationManager();
+ // all the numOfBookies (5) are going to be in different racks
+ for (int i = 0; i < numOfBookies; i++) {
+ bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181);
+ StaticDNSResolver.addNodeToRack(bookieAddress.getHostName(), "/rack" + (i));
+ bookieAddresses.add(bookieAddress.toBookieId());
+ regManager.registerBookie(bookieAddress.toBookieId(), false, BookieServiceInfo.EMPTY);
+ }
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ int ensembleSize = 5;
+ int writeQuorumSize = 4;
+ int ackQuorumSize = 2;
+ int minNumRacksPerWriteQuorumConfValue = 4;
+ Collections.shuffle(bookieAddresses);
+
+ // closed ledger
+ LedgerMetadata initMeta = LedgerMetadataBuilder.create()
+ .withId(1L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses)
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(1L, initMeta).get();
+
+ Collections.shuffle(bookieAddresses);
+ ensembleSize = 4;
+ // closed ledger with multiple segments
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(2L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses.subList(0, 4))
+ .newEnsembleEntry(20L, bookieAddresses.subList(1, 5))
+ .newEnsembleEntry(60L, bookieAddresses.subList(0, 4))
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(2L, initMeta).get();
+
+ Collections.shuffle(bookieAddresses);
+ // non-closed ledger
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(3L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses.subList(0, 4))
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(3L, initMeta).get();
+
+ Collections.shuffle(bookieAddresses);
+ // non-closed ledger with multiple segments
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(4L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses.subList(0, 4))
+ .newEnsembleEntry(20L, bookieAddresses.subList(1, 5))
+ .newEnsembleEntry(60L, bookieAddresses.subList(0, 4))
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(4L, initMeta).get();
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue);
+ setServerConfigPropertiesForRackPlacement(servConf);
+ MutableObject<Auditor> auditorRef = new MutableObject<Auditor>();
+ try {
+ TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef);
+ Gauge<? extends Number> ledgersNotAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY);
+ Gauge<? extends Number> ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY);
+ /*
+ * since all of the bookies are in different racks, there shouldn't be any ledger not adhering
+ * to placement policy.
+ */
+ assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", 0,
+ ledgersNotAdheringToPlacementPolicyGuage.getSample());
+ /*
+ * since all of the bookies are in different racks, there shouldn't be any ledger softly adhering
+ * to placement policy.
+ */
+ assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", 0,
+ ledgersSoftlyAdheringToPlacementPolicyGuage.getSample());
+ } finally {
+ Auditor auditor = auditorRef.getValue();
+ if (auditor != null) {
+ auditor.close();
+ }
+ regManager.close();
+ }
+ }
+
+ @Test
+ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicy() throws Exception {
+ int numOfBookies = 5;
+ int numOfLedgersNotAdheringToPlacementPolicy = 0;
+ List<BookieId> bookieAddresses = new ArrayList<>();
+ RegistrationManager regManager = driver.createRegistrationManager();
+ for (int i = 0; i < numOfBookies; i++) {
+ BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId();
+ bookieAddresses.add(bookieAddress);
+ regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY);
+ }
+
+ // only three racks
+ StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1");
+ StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2");
+ StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3");
+ StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1");
+ StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2");
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ int ensembleSize = 5;
+ int writeQuorumSize = 3;
+ int ackQuorumSize = 2;
+ int minNumRacksPerWriteQuorumConfValue = 3;
+
+ /*
+ * this closed ledger doesn't adhere to placement policy because there are only
+ * 3 racks, and the ensembleSize is 5.
+ */
+ LedgerMetadata initMeta = LedgerMetadataBuilder.create()
+ .withId(1L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses)
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(1L, initMeta).get();
+ numOfLedgersNotAdheringToPlacementPolicy++;
+
+ /*
+ * this is non-closed ledger, so it shouldn't count as ledger not
+ * adhering to placement policy
+ */
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(2L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(2L, initMeta).get();
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue);
+ setServerConfigPropertiesForRackPlacement(servConf);
+ MutableObject<Auditor> auditorRef = new MutableObject<Auditor>();
+ try {
+ TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef);
+ Gauge<? extends Number> ledgersNotAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value",
+ numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample());
+ Gauge<? extends Number> ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value",
+ 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample());
+ } finally {
+ Auditor auditor = auditorRef.getValue();
+ if (auditor != null) {
+ auditor.close();
+ }
+ regManager.close();
+ }
+ }
+
+ @Test
+ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndNotMarkToUnderreplication()
+ throws Exception {
+ int numOfBookies = 5;
+ int numOfLedgersNotAdheringToPlacementPolicy = 0;
+ List<BookieId> bookieAddresses = new ArrayList<>();
+ RegistrationManager regManager = driver.createRegistrationManager();
+ for (int i = 0; i < numOfBookies; i++) {
+ BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId();
+ bookieAddresses.add(bookieAddress);
+ regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY);
+ }
+
+ // only three racks
+ StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1");
+ StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2");
+ StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3");
+ StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1");
+ StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2");
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ int ensembleSize = 5;
+ int writeQuorumSize = 3;
+ int ackQuorumSize = 2;
+ int minNumRacksPerWriteQuorumConfValue = 3;
+
+ /*
+ * this closed ledger doesn't adhere to placement policy because there are only
+ * 3 racks, and the ensembleSize is 5.
+ */
+ LedgerMetadata initMeta = LedgerMetadataBuilder.create()
+ .withId(1L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses)
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(1L, initMeta).get();
+ numOfLedgersNotAdheringToPlacementPolicy++;
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue);
+ setServerConfigPropertiesForRackPlacement(servConf);
+ MutableObject<Auditor> auditorRef = new MutableObject<Auditor>();
+ try {
+ TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef);
+ Gauge<? extends Number> ledgersNotAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value",
+ numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample());
+ Gauge<? extends Number> ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value",
+ 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample());
+ } finally {
+ Auditor auditor = auditorRef.getValue();
+ if (auditor != null) {
+ auditor.close();
+ }
+ regManager.close();
+ }
+ LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager();
+ long unnderReplicateLedgerId = underreplicationManager.pollLedgerToRereplicate();
+ assertEquals(unnderReplicateLedgerId, -1);
+ }
+
+ @Test
+ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndMarkToUnderreplication()
+ throws Exception {
+ int numOfBookies = 5;
+ int numOfLedgersNotAdheringToPlacementPolicy = 0;
+ List<BookieId> bookieAddresses = new ArrayList<>();
+ RegistrationManager regManager = driver.createRegistrationManager();
+ for (int i = 0; i < numOfBookies; i++) {
+ BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId();
+ bookieAddresses.add(bookieAddress);
+ regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY);
+ }
+
+ // only three racks
+ StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1");
+ StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2");
+ StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3");
+ StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1");
+ StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2");
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ int ensembleSize = 5;
+ int writeQuorumSize = 3;
+ int ackQuorumSize = 2;
+ int minNumRacksPerWriteQuorumConfValue = 3;
+
+ /*
+ * this closed ledger doesn't adhere to placement policy because there are only
+ * 3 racks, and the ensembleSize is 5.
+ */
+ LedgerMetadata initMeta = LedgerMetadataBuilder.create()
+ .withId(1L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses)
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(1L, initMeta).get();
+ numOfLedgersNotAdheringToPlacementPolicy++;
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue);
+ servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true);
+ setServerConfigPropertiesForRackPlacement(servConf);
+ MutableObject<Auditor> auditorRef = new MutableObject<Auditor>();
+ try {
+ TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef);
+ Gauge<? extends Number> ledgersNotAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value",
+ numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample());
+ Gauge<? extends Number> ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value",
+ 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample());
+ } finally {
+ Auditor auditor = auditorRef.getValue();
+ if (auditor != null) {
+ auditor.close();
+ }
+ regManager.close();
+ }
+ LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager();
+ long unnderReplicateLedgerId = underreplicationManager.pollLedgerToRereplicate();
+ assertEquals(unnderReplicateLedgerId, 1L);
+ }
+
+ @Test
+ public void testPlacementPolicyCheckForURLedgersElapsedRecoveryGracePeriod() throws Exception {
+ testPlacementPolicyCheckWithURLedgers(true);
+ }
+
+ @Test
+ public void testPlacementPolicyCheckForURLedgersNotElapsedRecoveryGracePeriod() throws Exception {
+ testPlacementPolicyCheckWithURLedgers(false);
+ }
+
+ public void testPlacementPolicyCheckWithURLedgers(boolean timeElapsed) throws Exception {
+ int numOfBookies = 4;
+ /*
+ * in timeElapsed=true scenario, set some low value, otherwise set some
+ * highValue.
+ */
+ int underreplicatedLedgerRecoveryGracePeriod = timeElapsed ? 1 : 1000;
+ int numOfURLedgersElapsedRecoveryGracePeriod = 0;
+ List<BookieId> bookieAddresses = new ArrayList<BookieId>();
+ RegistrationManager regManager = driver.createRegistrationManager();
+ for (int i = 0; i < numOfBookies; i++) {
+ BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId();
+ bookieAddresses.add(bookieAddress);
+ regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY);
+ }
+
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager();
+ int ensembleSize = 4;
+ int writeQuorumSize = 3;
+ int ackQuorumSize = 2;
+
+ long ledgerId1 = 1L;
+ LedgerMetadata initMeta = LedgerMetadataBuilder.create()
+ .withId(ledgerId1)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses)
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(ledgerId1, initMeta).get();
+ underreplicationManager.markLedgerUnderreplicated(ledgerId1, bookieAddresses.get(0).toString());
+ if (timeElapsed) {
+ numOfURLedgersElapsedRecoveryGracePeriod++;
+ }
+
+ /*
+ * this is non-closed ledger, it should also be reported as
+ * URLedgersElapsedRecoveryGracePeriod
+ */
+ ensembleSize = 3;
+ long ledgerId2 = 21234561L;
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(ledgerId2)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L,
+ Arrays.asList(bookieAddresses.get(0), bookieAddresses.get(1), bookieAddresses.get(2)))
+ .newEnsembleEntry(100L,
+ Arrays.asList(bookieAddresses.get(3), bookieAddresses.get(1), bookieAddresses.get(2)))
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(ledgerId2, initMeta).get();
+ underreplicationManager.markLedgerUnderreplicated(ledgerId2, bookieAddresses.get(0).toString());
+ if (timeElapsed) {
+ numOfURLedgersElapsedRecoveryGracePeriod++;
+ }
+
+ /*
+ * this ledger is not marked underreplicated.
+ */
+ long ledgerId3 = 31234561L;
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(ledgerId3)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L,
+ Arrays.asList(bookieAddresses.get(1), bookieAddresses.get(2), bookieAddresses.get(3)))
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(ledgerId3, initMeta).get();
+
+ if (timeElapsed) {
+ /*
+ * in timeelapsed scenario, by waiting for
+ * underreplicatedLedgerRecoveryGracePeriod, recovery time must be
+ * elapsed.
+ */
+ Thread.sleep((underreplicatedLedgerRecoveryGracePeriod + 1) * 1000);
+ } else {
+ /*
+ * in timeElapsed=false scenario, since
+ * underreplicatedLedgerRecoveryGracePeriod is set to some high
+ * value, there is no value in waiting. So just wait for some time
+ * and make sure urledgers are not reported as recoverytime elapsed
+ * urledgers.
+ */
+ Thread.sleep(5000);
+ }
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ servConf.setUnderreplicatedLedgerRecoveryGracePeriod(underreplicatedLedgerRecoveryGracePeriod);
+ setServerConfigPropertiesForRackPlacement(servConf);
+ MutableObject<Auditor> auditorRef = new MutableObject<Auditor>();
+ try {
+ TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef);
+ Gauge<? extends Number> underreplicatedLedgersElapsedRecoveryGracePeriodGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_UNDERREPLICATED_LEDGERS_ELAPSED_RECOVERY_GRACE_PERIOD);
+ assertEquals("NUM_UNDERREPLICATED_LEDGERS_ELAPSED_RECOVERY_GRACE_PERIOD guage value",
+ numOfURLedgersElapsedRecoveryGracePeriod,
+ underreplicatedLedgersElapsedRecoveryGracePeriodGuage.getSample());
+ } finally {
+ Auditor auditor = auditorRef.getValue();
+ if (auditor != null) {
+ auditor.close();
+ }
+ regManager.close();
+ }
+ }
+
+ @Test
+ public void testPlacementPolicyCheckWithLedgersNotAdheringToPolicyWithMultipleSegments() throws Exception {
+ int numOfBookies = 7;
+ int numOfLedgersNotAdheringToPlacementPolicy = 0;
+ List<BookieId> bookieAddresses = new ArrayList<>();
+ RegistrationManager regManager = driver.createRegistrationManager();
+ for (int i = 0; i < numOfBookies; i++) {
+ BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId();
+ bookieAddresses.add(bookieAddress);
+ regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY);
+ }
+
+ // only three racks
+ StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1");
+ StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2");
+ StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3");
+ StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack4");
+ StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack1");
+ StaticDNSResolver.addNodeToRack("98.98.98.5", "/rack2");
+ StaticDNSResolver.addNodeToRack("98.98.98.6", "/rack3");
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ int ensembleSize = 5;
+ int writeQuorumSize = 5;
+ int ackQuorumSize = 2;
+ int minNumRacksPerWriteQuorumConfValue = 4;
+
+ /*
+ * this closed ledger in each writeQuorumSize (5), there would be
+ * atleast minNumRacksPerWriteQuorumConfValue (4) racks. So it wont be
+ * counted as ledgers not adhering to placement policy.
+ */
+ LedgerMetadata initMeta = LedgerMetadataBuilder.create()
+ .withId(1L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses.subList(0, 5))
+ .newEnsembleEntry(20L, bookieAddresses.subList(1, 6))
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(1L, initMeta).get();
+
+ /*
+ * for the second segment bookies are from /rack1, /rack2 and /rack3,
+ * which is < minNumRacksPerWriteQuorumConfValue (4). So it is not
+ * adhering to placement policy.
+ *
+ * also for the third segment are from /rack1, /rack2 and /rack3, which
+ * is < minNumRacksPerWriteQuorumConfValue (4). So it is not adhering to
+ * placement policy.
+ *
+ * Though there are multiple segments are not adhering to placement
+ * policy, it should be counted as single ledger.
+ */
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(2L)
+ .withEnsembleSize(ensembleSize)
+ .withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize)
+ .newEnsembleEntry(0L, bookieAddresses.subList(0, 5))
+ .newEnsembleEntry(20L,
+ Arrays.asList(bookieAddresses.get(0), bookieAddresses.get(1), bookieAddresses.get(2),
+ bookieAddresses.get(4), bookieAddresses.get(5)))
+ .newEnsembleEntry(40L,
+ Arrays.asList(bookieAddresses.get(0), bookieAddresses.get(1), bookieAddresses.get(2),
+ bookieAddresses.get(4), bookieAddresses.get(6)))
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(2L, initMeta).get();
+ numOfLedgersNotAdheringToPlacementPolicy++;
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue);
+ setServerConfigPropertiesForRackPlacement(servConf);
+ MutableObject<Auditor> auditorRef = new MutableObject<Auditor>();
+ try {
+ TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef);
+ Gauge<? extends Number> ledgersNotAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY gauge value",
+ numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample());
+ Gauge<? extends Number> ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY gauge value",
+ 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample());
+ } finally {
+ Auditor auditor = auditorRef.getValue();
+ if (auditor != null) {
+ auditor.close();
+ }
+ regManager.close();
+ }
+ }
+
+ @Test
+ public void testZoneawarePlacementPolicyCheck() throws Exception {
+ int numOfBookies = 6;
+ int numOfLedgersNotAdheringToPlacementPolicy = 0;
+ int numOfLedgersSoftlyAdheringToPlacementPolicy = 0;
+ List<BookieId> bookieAddresses = new ArrayList<BookieId>();
+ RegistrationManager regManager = driver.createRegistrationManager();
+ /*
+ * 6 bookies - 3 zones and 2 uds
+ */
+ for (int i = 0; i < numOfBookies; i++) {
+ BookieSocketAddress bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181);
+ bookieAddresses.add(bookieAddress.toBookieId());
+ regManager.registerBookie(bookieAddress.toBookieId(), false, BookieServiceInfo.EMPTY);
+ String zone = "/zone" + (i % 3);
+ String upgradeDomain = "/ud" + (i % 2);
+ String networkLocation = zone + upgradeDomain;
+ StaticDNSResolver.addNodeToRack(bookieAddress.getHostName(), networkLocation);
+ }
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ servConf.setDesiredNumZonesPerWriteQuorum(3);
+ servConf.setMinNumZonesPerWriteQuorum(2);
+ setServerConfigPropertiesForZonePlacement(servConf);
+
+ /*
+ * this closed ledger adheres to ZoneAwarePlacementPolicy, since
+ * ensemble is spread across 3 zones and 2 UDs
+ */
+ LedgerMetadata initMeta = LedgerMetadataBuilder.create()
+ .withId(1L)
+ .withEnsembleSize(6)
+ .withWriteQuorumSize(6)
+ .withAckQuorumSize(2)
+ .newEnsembleEntry(0L, bookieAddresses)
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(1L, initMeta).get();
+
+ /*
+ * this is non-closed ledger, so though ensemble is not adhering to
+ * placement policy (since ensemble is not multiple of writeQuorum),
+ * this shouldn't be reported
+ */
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(2L)
+ .withEnsembleSize(6)
+ .withWriteQuorumSize(5)
+ .withAckQuorumSize(2)
+ .newEnsembleEntry(0L, bookieAddresses)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(2L, initMeta).get();
+
+ /*
+ * this is closed ledger, since ensemble is not multiple of writeQuorum,
+ * this ledger is not adhering to placement policy.
+ */
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(3L)
+ .withEnsembleSize(6)
+ .withWriteQuorumSize(5)
+ .withAckQuorumSize(2)
+ .newEnsembleEntry(0L, bookieAddresses)
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(3L, initMeta).get();
+ numOfLedgersNotAdheringToPlacementPolicy++;
+
+ /*
+ * this closed ledger adheres softly to ZoneAwarePlacementPolicy, since
+ * ensemble/writeQuorum of size 4 has spread across just
+ * minNumZonesPerWriteQuorum (2).
+ */
+ List<BookieId> newEnsemble = new ArrayList<BookieId>();
+ newEnsemble.add(bookieAddresses.get(0));
+ newEnsemble.add(bookieAddresses.get(1));
+ newEnsemble.add(bookieAddresses.get(3));
+ newEnsemble.add(bookieAddresses.get(4));
+ initMeta = LedgerMetadataBuilder.create()
+ .withId(4L)
+ .withEnsembleSize(4)
+ .withWriteQuorumSize(4)
+ .withAckQuorumSize(2)
+ .newEnsembleEntry(0L, newEnsemble)
+ .withClosedState()
+ .withLastEntryId(100)
+ .withLength(10000)
+ .withDigestType(DigestType.DUMMY)
+ .withPassword(new byte[0])
+ .build();
+ lm.createLedgerMetadata(4L, initMeta).get();
+ numOfLedgersSoftlyAdheringToPlacementPolicy++;
+
+ MutableObject<Auditor> auditorRef = new MutableObject<Auditor>();
+ try {
+ TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef);
+ Gauge<? extends Number> ledgersNotAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value",
+ numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample());
+ Gauge<? extends Number> ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value",
+ numOfLedgersSoftlyAdheringToPlacementPolicy,
+ ledgersSoftlyAdheringToPlacementPolicyGuage.getSample());
+ } finally {
+ Auditor auditor = auditorRef.getValue();
+ if (auditor != null) {
+ auditor.close();
+ }
+ regManager.close();
+ }
+ }
+
+ private void setServerConfigPropertiesForRackPlacement(ServerConfiguration servConf) {
+ setServerConfigProperties(servConf, RackawareEnsemblePlacementPolicy.class.getName());
+ }
+
+ private void setServerConfigPropertiesForZonePlacement(ServerConfiguration servConf) {
+ setServerConfigProperties(servConf, ZoneawareEnsemblePlacementPolicy.class.getName());
+ }
+
+ private void setServerConfigProperties(ServerConfiguration servConf, String ensemblePlacementPolicyClass) {
+ servConf.setProperty(REPP_DNS_RESOLVER_CLASS, StaticDNSResolver.class.getName());
+ servConf.setProperty(ClientConfiguration.ENSEMBLE_PLACEMENT_POLICY, ensemblePlacementPolicyClass);
+ servConf.setAuditorPeriodicCheckInterval(0);
+ servConf.setAuditorPeriodicBookieCheckInterval(0);
+ servConf.setAuditorPeriodicReplicasCheckInterval(0);
+ servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000);
+ }
+
+ private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf,
+ MutableObject<Auditor> auditorRef) throws MetadataException, CompatibilityException, KeeperException,
+ InterruptedException, UnavailableException, UnknownHostException {
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager();
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ TestOpStatsLogger placementPolicyCheckStatsLogger = (TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME);
+
+ final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf,
+ statsLogger, null);
+ auditorRef.setValue(auditor);
+ CountDownLatch latch = auditor.getLatch();
+ assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, placementPolicyCheckStatsLogger.getSuccessCount());
+ urm.setPlacementPolicyCheckCTime(-1);
+ auditor.start();
+ /*
+ * since placementPolicyCheckCTime is set to -1, placementPolicyCheck should be
+ * scheduled to run with no initialdelay
+ */
+ assertTrue("placementPolicyCheck should have executed", latch.await(20, TimeUnit.SECONDS));
+ for (int i = 0; i < 20; i++) {
+ Thread.sleep(100);
+ if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) {
+ break;
+ }
+ }
+ assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, placementPolicyCheckStatsLogger.getSuccessCount());
+ return statsLogger;
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java
new file mode 100644
index 00000000000..21dd2807b75
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE;
+import static org.testng.AssertJUnit.assertEquals;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.test.TestStatsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test {@link AuditorReplicasCheckTask}.
+ */
+public class AuditorReplicasCheckTaskTest extends BookKeeperClusterTestCase {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AuditorReplicasCheckTaskTest.class);
+
+ private BookKeeperAdmin admin;
+ private LedgerManager ledgerManager;
+ private LedgerUnderreplicationManager ledgerUnderreplicationManager;
+
+ public AuditorReplicasCheckTaskTest() throws Exception {
+ super(3);
+ baseConf.setPageLimit(1);
+ baseConf.setAutoRecoveryDaemonEnabled(false);
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ baseClientConf.setMetadataServiceUri(
+ metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ final BookKeeper bookKeeper = new BookKeeper(baseClientConf);
+ admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new ClientConfiguration(baseClientConf));
+ LedgerManagerFactory ledgerManagerFactory = bookKeeper.getLedgerManagerFactory();
+ ledgerManager = ledgerManagerFactory.newLedgerManager();
+ ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ if (ledgerManager != null) {
+ ledgerManager.close();
+ }
+ if (ledgerUnderreplicationManager != null) {
+ ledgerUnderreplicationManager.close();
+ }
+ if (admin != null) {
+ admin.close();
+ }
+ super.tearDown();
+ }
+
+ @Test
+ public void testReplicasCheck() throws BKException, InterruptedException {
+
+ // 1. create ledgers
+ final int numLedgers = 10;
+ List<Long> ids = new LinkedList<Long>();
+ for (int i = 0; i < numLedgers; i++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes());
+ ids.add(lh.getId());
+ for (int j = 0; j < 2; j++) {
+ lh.addEntry("testdata".getBytes());
+ }
+ lh.close();
+ }
+
+ // 2. init auditorReplicasCheckTask
+ final TestStatsProvider statsProvider = new TestStatsProvider();
+ final TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ final AuditorStats auditorStats = new AuditorStats(statsLogger);
+ AuditorReplicasCheckTask auditorReplicasCheckTask = new AuditorReplicasCheckTask(
+ baseConf, auditorStats, admin, ledgerManager,
+ ledgerUnderreplicationManager, null, (flag, throwable) -> flag.set(false));
+
+ // 3. replicasCheck
+ auditorReplicasCheckTask.runTask();
+
+ // 4. verify
+ assertEquals("REPLICAS_CHECK_TIME", 1, ((TestStatsProvider.TestOpStatsLogger)
+ statsLogger.getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME)).getSuccessCount());
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java
new file mode 100644
index 00000000000..a4d6d86dece
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java
@@ -0,0 +1,937 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.bookie.BookieException;
+import org.apache.bookkeeper.bookie.BookieImpl;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.client.LedgerMetadataBuilder;
+import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.discover.BookieServiceInfo;
+import org.apache.bookkeeper.discover.RegistrationManager;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.meta.MetadataBookieDriver;
+import org.apache.bookkeeper.meta.MetadataDrivers;
+import org.apache.bookkeeper.meta.exceptions.MetadataException;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.net.BookieSocketAddress;
+import org.apache.bookkeeper.replication.AuditorPeriodicCheckTest.TestAuditor;
+import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException;
+import org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
+import org.apache.bookkeeper.stats.Gauge;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.test.TestStatsProvider;
+import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger;
+import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger;
+import org.apache.bookkeeper.util.AvailabilityOfEntriesOfLedger;
+import org.apache.bookkeeper.util.StaticDNSResolver;
+import org.apache.commons.collections4.map.MultiKeyMap;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.zookeeper.KeeperException;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Tests the logic of Auditor's ReplicasCheck.
+ */
+public class AuditorReplicasCheckTest extends BookKeeperClusterTestCase {
+ private MetadataBookieDriver driver;
+ private RegistrationManager regManager;
+
+ public AuditorReplicasCheckTest() throws Exception {
+ super(1);
+ baseConf.setPageLimit(1); // to make it easy to push ledger out of cache
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ StaticDNSResolver.reset();
+
+ URI uri = URI.create(confByIndex(0).getMetadataServiceUri().replaceAll("zk://", "metadata-store:")
+ .replaceAll("/ledgers", ""));
+ driver = MetadataDrivers.getBookieDriver(uri);
+ ServerConfiguration serverConfiguration = new ServerConfiguration(confByIndex(0));
+ serverConfiguration.setMetadataServiceUri(
+ serverConfiguration.getMetadataServiceUri().replaceAll("zk://", "metadata-store:")
+ .replaceAll("/ledgers", ""));
+ driver.initialize(serverConfiguration, NullStatsLogger.INSTANCE);
+ regManager = driver.createRegistrationManager();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ if (null != regManager) {
+ regManager.close();
+ }
+ if (null != driver) {
+ driver.close();
+ }
+ super.tearDown();
+ }
+
+ private class TestBookKeeperAdmin extends BookKeeperAdmin {
+
+ private final MultiKeyMap<String, AvailabilityOfEntriesOfLedger> returnAvailabilityOfEntriesOfLedger;
+ private final MultiKeyMap<String, Integer> errorReturnValueForGetAvailabilityOfEntriesOfLedger;
+
+ public TestBookKeeperAdmin(BookKeeper bkc, StatsLogger statsLogger,
+ MultiKeyMap<String, AvailabilityOfEntriesOfLedger> returnAvailabilityOfEntriesOfLedger,
+ MultiKeyMap<String, Integer> errorReturnValueForGetAvailabilityOfEntriesOfLedger) {
+ super(bkc, statsLogger, baseClientConf);
+ this.returnAvailabilityOfEntriesOfLedger = returnAvailabilityOfEntriesOfLedger;
+ this.errorReturnValueForGetAvailabilityOfEntriesOfLedger =
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger;
+ }
+
+ @Override
+ public CompletableFuture<AvailabilityOfEntriesOfLedger> asyncGetListOfEntriesOfLedger(
+ BookieId address, long ledgerId) {
+ CompletableFuture<AvailabilityOfEntriesOfLedger> futureResult =
+ new CompletableFuture<AvailabilityOfEntriesOfLedger>();
+ Integer errorReturnValue = errorReturnValueForGetAvailabilityOfEntriesOfLedger.get(address.toString(),
+ Long.toString(ledgerId));
+ if (errorReturnValue != null) {
+ futureResult.completeExceptionally(BKException.create(errorReturnValue).fillInStackTrace());
+ } else {
+ AvailabilityOfEntriesOfLedger availabilityOfEntriesOfLedger = returnAvailabilityOfEntriesOfLedger
+ .get(address.toString(), Long.toString(ledgerId));
+ futureResult.complete(availabilityOfEntriesOfLedger);
+ }
+ return futureResult;
+ }
+ }
+
+ private TestStatsLogger startAuditorAndWaitForReplicasCheck(ServerConfiguration servConf,
+ MutableObject<Auditor> auditorRef,
+ MultiKeyMap<String, AvailabilityOfEntriesOfLedger> expectedReturnAvailabilityOfEntriesOfLedger,
+ MultiKeyMap<String, Integer> errorReturnValueForGetAvailabilityOfEntriesOfLedger)
+ throws MetadataException, CompatibilityException, KeeperException, InterruptedException,
+ UnavailableException, UnknownHostException {
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager();
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ TestOpStatsLogger replicasCheckStatsLogger = (TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME);
+
+ final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, true,
+ new TestBookKeeperAdmin(bkc, statsLogger, expectedReturnAvailabilityOfEntriesOfLedger,
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger),
+ true, statsLogger, null);
+ auditorRef.setValue(auditor);
+ CountDownLatch latch = auditor.getLatch();
+ assertEquals("REPLICAS_CHECK_TIME SuccessCount", 0, replicasCheckStatsLogger.getSuccessCount());
+ urm.setReplicasCheckCTime(-1);
+ auditor.start();
+ /*
+ * since replicasCheckCTime is set to -1, replicasCheck should be
+ * scheduled to run with no initialdelay
+ */
+ assertTrue("replicasCheck should have executed", latch.await(20, TimeUnit.SECONDS));
+ for (int i = 0; i < 200; i++) {
+ Thread.sleep(100);
+ if (replicasCheckStatsLogger.getSuccessCount() >= 1) {
+ break;
+ }
+ }
+ assertEquals("REPLICAS_CHECK_TIME SuccessCount", 1, replicasCheckStatsLogger.getSuccessCount());
+ return statsLogger;
+ }
+
+ private void setServerConfigProperties(ServerConfiguration servConf) {
+ servConf.setAuditorPeriodicCheckInterval(0);
+ servConf.setAuditorPeriodicBookieCheckInterval(0);
+ servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0);
+ servConf.setAuditorPeriodicReplicasCheckInterval(1000);
+ }
+
+ List<BookieId> addAndRegisterBookies(int numOfBookies)
+ throws BookieException {
+ BookieId bookieAddress;
+ List<BookieId> bookieAddresses = new ArrayList<BookieId>();
+ for (int i = 0; i < numOfBookies; i++) {
+ bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId();
+ bookieAddresses.add(bookieAddress);
+ regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY);
+ }
+ return bookieAddresses;
+ }
+
+ private void createClosedLedgerMetadata(LedgerManager lm, long ledgerId, int ensembleSize, int writeQuorumSize,
+ int ackQuorumSize, Map<Long, List<BookieId>> segmentEnsembles, long lastEntryId, int length,
+ DigestType digestType, byte[] password) throws InterruptedException, ExecutionException {
+ LedgerMetadataBuilder ledgerMetadataBuilder = LedgerMetadataBuilder.create();
+ ledgerMetadataBuilder.withId(ledgerId).withEnsembleSize(ensembleSize).withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize).withClosedState().withLastEntryId(lastEntryId).withLength(length)
+ .withDigestType(digestType).withPassword(password);
+ for (Map.Entry<Long, List<BookieId>> mapEntry : segmentEnsembles.entrySet()) {
+ ledgerMetadataBuilder.newEnsembleEntry(mapEntry.getKey(), mapEntry.getValue());
+ }
+ LedgerMetadata initMeta = ledgerMetadataBuilder.build();
+ lm.createLedgerMetadata(ledgerId, initMeta).get();
+ }
+
+ private void createNonClosedLedgerMetadata(LedgerManager lm, long ledgerId, int ensembleSize, int writeQuorumSize,
+ int ackQuorumSize, Map<Long, List<BookieId>> segmentEnsembles, DigestType digestType,
+ byte[] password) throws InterruptedException, ExecutionException {
+ LedgerMetadataBuilder ledgerMetadataBuilder = LedgerMetadataBuilder.create();
+ ledgerMetadataBuilder.withId(ledgerId).withEnsembleSize(ensembleSize).withWriteQuorumSize(writeQuorumSize)
+ .withAckQuorumSize(ackQuorumSize).withDigestType(digestType).withPassword(password);
+ for (Map.Entry<Long, List<BookieId>> mapEntry : segmentEnsembles.entrySet()) {
+ ledgerMetadataBuilder.newEnsembleEntry(mapEntry.getKey(), mapEntry.getValue());
+ }
+ LedgerMetadata initMeta = ledgerMetadataBuilder.build();
+ lm.createLedgerMetadata(ledgerId, initMeta).get();
+ }
+
+ private void runTestScenario(MultiKeyMap<String, AvailabilityOfEntriesOfLedger> returnAvailabilityOfEntriesOfLedger,
+ MultiKeyMap<String, Integer> errorReturnValueForGetAvailabilityOfEntriesOfLedger,
+ int expectedNumLedgersFoundHavingNoReplicaOfAnEntry,
+ int expectedNumLedgersHavingLessThanAQReplicasOfAnEntry,
+ int expectedNumLedgersHavingLessThanWQReplicasOfAnEntry) throws Exception {
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ setServerConfigProperties(servConf);
+ MutableObject<Auditor> auditorRef = new MutableObject<Auditor>();
+ try {
+ TestStatsLogger statsLogger = startAuditorAndWaitForReplicasCheck(servConf, auditorRef,
+ returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger);
+ checkReplicasCheckStats(statsLogger, expectedNumLedgersFoundHavingNoReplicaOfAnEntry,
+ expectedNumLedgersHavingLessThanAQReplicasOfAnEntry,
+ expectedNumLedgersHavingLessThanWQReplicasOfAnEntry);
+ } finally {
+ Auditor auditor = auditorRef.getValue();
+ if (auditor != null) {
+ auditor.close();
+ }
+ }
+ }
+
+ private void checkReplicasCheckStats(TestStatsLogger statsLogger,
+ int expectedNumLedgersFoundHavingNoReplicaOfAnEntry,
+ int expectedNumLedgersHavingLessThanAQReplicasOfAnEntry,
+ int expectedNumLedgersHavingLessThanWQReplicasOfAnEntry) {
+ Gauge<? extends Number> numLedgersFoundHavingNoReplicaOfAnEntryGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_HAVING_NO_REPLICA_OF_AN_ENTRY);
+ Gauge<? extends Number> numLedgersHavingLessThanAQReplicasOfAnEntryGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_HAVING_LESS_THAN_AQ_REPLICAS_OF_AN_ENTRY);
+ Gauge<? extends Number> numLedgersHavingLessThanWQReplicasOfAnEntryGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_HAVING_LESS_THAN_WQ_REPLICAS_OF_AN_ENTRY);
+
+ assertEquals("NUM_LEDGERS_HAVING_NO_REPLICA_OF_AN_ENTRY guage value",
+ expectedNumLedgersFoundHavingNoReplicaOfAnEntry,
+ numLedgersFoundHavingNoReplicaOfAnEntryGuage.getSample());
+ assertEquals("NUM_LEDGERS_HAVING_LESS_THAN_AQ_REPLICAS_OF_AN_ENTRY guage value",
+ expectedNumLedgersHavingLessThanAQReplicasOfAnEntry,
+ numLedgersHavingLessThanAQReplicasOfAnEntryGuage.getSample());
+ assertEquals("NUM_LEDGERS_HAVING_LESS_THAN_WQ_REPLICAS_OF_AN_ENTRY guage value",
+ expectedNumLedgersHavingLessThanWQReplicasOfAnEntry,
+ numLedgersHavingLessThanWQReplicasOfAnEntryGuage.getSample());
+ }
+
+ /*
+ * For all the ledgers and for all the bookies,
+ * asyncGetListOfEntriesOfLedger would return
+ * BookieHandleNotAvailableException, so these ledgers wouldn't be counted
+ * against expectedNumLedgersFoundHavingNoReplicaOfAnEntry /
+ * LessThanAQReplicasOfAnEntry / LessThanWQReplicasOfAnEntry.
+ */
+ @Test
+ public void testReplicasCheckForBookieHandleNotAvailable() throws Exception {
+ int numOfBookies = 5;
+ MultiKeyMap<String, AvailabilityOfEntriesOfLedger> returnAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, AvailabilityOfEntriesOfLedger>();
+ MultiKeyMap<String, Integer> errorReturnValueForGetAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, Integer>();
+ List<BookieId> bookieAddresses = addAndRegisterBookies(numOfBookies);
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ int ensembleSize = 5;
+ int writeQuorumSize = 4;
+ int ackQuorumSize = 2;
+ long lastEntryId = 100;
+ int length = 10000;
+ DigestType digestType = DigestType.DUMMY;
+ byte[] password = new byte[0];
+ Collections.shuffle(bookieAddresses);
+
+ /*
+ * closed ledger
+ *
+ * for this ledger, for all the bookies we are setting
+ * errorReturnValueForGetAvailabilityOfEntriesOfLedger to
+ * BookieHandleNotAvailableException so asyncGetListOfEntriesOfLedger will
+ * return BookieHandleNotAvailableException.
+ */
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
+ segmentEnsembles.put(0L, bookieAddresses);
+ long ledgerId = 1L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ for (BookieId bookieSocketAddress : bookieAddresses) {
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(),
+ Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException);
+ }
+
+ ensembleSize = 4;
+ /*
+ * closed ledger with multiple segments
+ *
+ * for this ledger, for all the bookies we are setting
+ * errorReturnValueForGetAvailabilityOfEntriesOfLedger to
+ * BookieHandleNotAvailableException so asyncGetListOfEntriesOfLedger will
+ * return BookieHandleNotAvailableException.
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(20L, bookieAddresses.subList(1, 5));
+ segmentEnsembles.put(60L, bookieAddresses.subList(0, 4));
+ ledgerId = 2L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ for (BookieId bookieSocketAddress : bookieAddresses) {
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(),
+ Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException);
+ }
+
+ /*
+ * non-closed ledger
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ ledgerId = 3L;
+ createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ digestType, password);
+ for (BookieId bookieSocketAddress : bookieAddresses) {
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(),
+ Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException);
+ }
+
+ /*
+ * non-closed ledger with multiple segments
+ *
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(20L, bookieAddresses.subList(1, 5));
+ segmentEnsembles.put(60L, bookieAddresses.subList(0, 4));
+ ledgerId = 4L;
+ createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ digestType, password);
+ for (BookieId bookieSocketAddress : bookieAddresses) {
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(),
+ Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException);
+ }
+
+ runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, 0, 0,
+ 0);
+ }
+
+ /*
+ * In this testscenario all the ledgers have a missing entry. So all closed
+ * ledgers should be counted towards
+ * numLedgersFoundHavingNoReplicaOfAnEntry.
+ */
+ @Test
+ public void testReplicasCheckForLedgersFoundHavingNoReplica() throws Exception {
+ int numOfBookies = 5;
+ MultiKeyMap<String, AvailabilityOfEntriesOfLedger> returnAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, AvailabilityOfEntriesOfLedger>();
+ MultiKeyMap<String, Integer> errorReturnValueForGetAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, Integer>();
+ List<BookieId> bookieAddresses = addAndRegisterBookies(numOfBookies);
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ int ensembleSize = 5;
+ int writeQuorumSize = 4;
+ int ackQuorumSize = 2;
+ long lastEntryId = 100;
+ int length = 10000;
+ DigestType digestType = DigestType.DUMMY;
+ byte[] password = new byte[0];
+ Collections.shuffle(bookieAddresses);
+
+ int numLedgersFoundHavingNoReplicaOfAnEntry = 0;
+
+ /*
+ * closed ledger
+ *
+ * for this ledger we are setting returnAvailabilityOfEntriesOfLedger to
+ * Empty one for all of the bookies, so this ledger would be counted in
+ * ledgersFoundHavingNoReplicaOfAnEntry .
+ */
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
+ segmentEnsembles.put(0L, bookieAddresses);
+ long ledgerId = 1L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ for (BookieId bookieSocketAddress : bookieAddresses) {
+ returnAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), Long.toString(ledgerId),
+ AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER);
+ }
+ numLedgersFoundHavingNoReplicaOfAnEntry++;
+
+ ensembleSize = 4;
+ /*
+ * closed ledger with multiple segments
+ *
+ * for this ledger we are setting
+ * errorReturnValueForGetAvailabilityOfEntriesOfLedger to
+ * NoSuchLedgerExistsException. This is equivalent to
+ * EMPTY_AVAILABILITYOFENTRIESOFLEDGER. So this ledger would be counted
+ * in ledgersFoundHavingNoReplicaOfAnEntry
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(20L, bookieAddresses.subList(1, 5));
+ segmentEnsembles.put(60L, bookieAddresses.subList(0, 4));
+ ledgerId = 2L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ for (BookieId bookieSocketAddress : bookieAddresses) {
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(),
+ Long.toString(ledgerId), BKException.Code.NoSuchLedgerExistsException);
+ }
+ numLedgersFoundHavingNoReplicaOfAnEntry++;
+
+ /*
+ * non-closed ledger
+ *
+ * since this is non-closed ledger, it should not be counted in
+ * ledgersFoundHavingNoReplicaOfAnEntry
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ ledgerId = 3L;
+ createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ digestType, password);
+ for (BookieId bookieSocketAddress : bookieAddresses) {
+ returnAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), Long.toString(ledgerId),
+ AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER);
+ }
+
+ ensembleSize = 3;
+ writeQuorumSize = 3;
+ ackQuorumSize = 2;
+ lastEntryId = 1;
+ length = 1000;
+ /*
+ * closed ledger
+ *
+ * for this ledger we are setting returnAvailabilityOfEntriesOfLedger to
+ * just {0l} for all of the bookies and entry 1l is missing for all of
+ * the bookies, so this ledger would be counted in
+ * ledgersFoundHavingNoReplicaOfAnEntry
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 3));
+ ledgerId = 4L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ for (BookieId bookieSocketAddress : bookieAddresses) {
+ returnAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0L }));
+ }
+ numLedgersFoundHavingNoReplicaOfAnEntry++;
+
+ /*
+ * For this closed ledger, entry 1 is missing. So it should be counted
+ * towards numLedgersFoundHavingNoReplicaOfAnEntry.
+ */
+ ensembleSize = 4;
+ writeQuorumSize = 3;
+ ackQuorumSize = 2;
+ lastEntryId = 3;
+ length = 10000;
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ ledgerId = 5L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 2, 3 }));
+ numLedgersFoundHavingNoReplicaOfAnEntry++;
+
+ runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger,
+ numLedgersFoundHavingNoReplicaOfAnEntry, 0, 0);
+ }
+
+ /*
+ * In this testscenario all the ledgers have an entry with less than AQ
+ * number of copies. So all closed ledgers should be counted towards
+ * numLedgersFoundHavingLessThanAQReplicasOfAnEntry.
+ */
+ @Test
+ public void testReplicasCheckForLedgersFoundHavingLessThanAQReplicasOfAnEntry() throws Exception {
+ int numOfBookies = 5;
+ MultiKeyMap<String, AvailabilityOfEntriesOfLedger> returnAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, AvailabilityOfEntriesOfLedger>();
+ MultiKeyMap<String, Integer> errorReturnValueForGetAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, Integer>();
+ List<BookieId> bookieAddresses = addAndRegisterBookies(numOfBookies);
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ DigestType digestType = DigestType.DUMMY;
+ byte[] password = new byte[0];
+ Collections.shuffle(bookieAddresses);
+
+ int numLedgersFoundHavingLessThanAQReplicasOfAnEntry = 0;
+
+ /*
+ * closed ledger
+ *
+ * for this ledger there is only one copy of entry 2, so this ledger
+ * would be counted towards
+ * ledgersFoundHavingLessThanAQReplicasOfAnEntry.
+ */
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
+ int ensembleSize = 4;
+ int writeQuorumSize = 3;
+ int ackQuorumSize = 2;
+ long lastEntryId = 3;
+ int length = 10000;
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ long ledgerId = 1L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1, 2, 3 }));
+ numLedgersFoundHavingLessThanAQReplicasOfAnEntry++;
+
+ /*
+ * closed ledger with multiple segments.
+ *
+ * for this ledger there is only one copy of entry 2, so this ledger
+ * would be counted towards
+ * ledgersFoundHavingLessThanAQReplicasOfAnEntry.
+ *
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(2L, bookieAddresses.subList(1, 5));
+ ledgerId = 2L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] {}));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 3 }));
+ numLedgersFoundHavingLessThanAQReplicasOfAnEntry++;
+
+ /*
+ * closed ledger with multiple segments
+ *
+ * for this ledger entry 2 is overrreplicated, but it has only one copy
+ * in the set of bookies it is supposed to be. So it should be counted
+ * towards ledgersFoundHavingLessThanAQReplicasOfAnEntry.
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(2L, bookieAddresses.subList(1, 5));
+ ledgerId = 3L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 2 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 3 }));
+ numLedgersFoundHavingLessThanAQReplicasOfAnEntry++;
+
+ /*
+ * non-closed ledger
+ *
+ * since this is non-closed ledger, it should not be counted towards
+ * ledgersFoundHavingLessThanAQReplicasOfAnEntry
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(2L, bookieAddresses.subList(1, 5));
+ ledgerId = 4L;
+ createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] {}));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 3 }));
+
+ /*
+ * this is closed ledger.
+ *
+ * For third bookie, asyncGetListOfEntriesOfLedger will fail with
+ * BookieHandleNotAvailableException, so this should not be counted
+ * against missing copies of an entry. Other than that, for both entries
+ * 0 and 1, two copies are missing. Hence this should be counted towards
+ * numLedgersFoundHavingLessThanAQReplicasOfAnEntry.
+ */
+ ensembleSize = 3;
+ writeQuorumSize = 3;
+ ackQuorumSize = 2;
+ lastEntryId = 1;
+ length = 1000;
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 3));
+ ledgerId = 5L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER);
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(),
+ Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException);
+ numLedgersFoundHavingLessThanAQReplicasOfAnEntry++;
+
+ runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, 0,
+ numLedgersFoundHavingLessThanAQReplicasOfAnEntry, 0);
+ }
+
+ /*
+ * In this testscenario all the ledgers have an entry with less than WQ
+ * number of copies but greater than AQ. So all closed ledgers should be
+ * counted towards numLedgersFoundHavingLessThanWQReplicasOfAnEntry.
+ */
+ @Test
+ public void testReplicasCheckForLedgersFoundHavingLessThanWQReplicasOfAnEntry() throws Exception {
+ int numOfBookies = 5;
+ MultiKeyMap<String, AvailabilityOfEntriesOfLedger> returnAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, AvailabilityOfEntriesOfLedger>();
+ MultiKeyMap<String, Integer> errorReturnValueForGetAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, Integer>();
+ List<BookieId> bookieAddresses = addAndRegisterBookies(numOfBookies);
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ DigestType digestType = DigestType.DUMMY;
+ byte[] password = new byte[0];
+ Collections.shuffle(bookieAddresses);
+
+ int numLedgersFoundHavingLessThanWQReplicasOfAnEntry = 0;
+
+ /*
+ * closed ledger
+ *
+ * for this ledger a copy of entry 3, so this ledger would be counted
+ * towards ledgersFoundHavingLessThanWQReplicasOfAnEntry.
+ */
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
+ int ensembleSize = 4;
+ int writeQuorumSize = 3;
+ int ackQuorumSize = 2;
+ long lastEntryId = 3;
+ int length = 10000;
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ long ledgerId = 1L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1, 2, 3 }));
+ numLedgersFoundHavingLessThanWQReplicasOfAnEntry++;
+
+ /*
+ * closed ledger with multiple segments
+ *
+ * for this ledger a copy of entry 0 and entry 2 are missing, so this
+ * ledger would be counted towards
+ * ledgersFoundHavingLessThanWQReplicasOfAnEntry.
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(2L, bookieAddresses.subList(1, 5));
+ ledgerId = 2L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] {}));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 2, 3 }));
+ numLedgersFoundHavingLessThanWQReplicasOfAnEntry++;
+
+ /*
+ * non-closed ledger with multiple segments
+ *
+ * since this is non-closed ledger, it should not be counted towards
+ * ledgersFoundHavingLessThanWQReplicasOfAnEntry
+ */
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(2L, bookieAddresses.subList(1, 5));
+ ledgerId = 3L;
+ createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ digestType, password);
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(),
+ Long.toString(ledgerId), BKException.Code.NoSuchLedgerExistsException);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 2, 3 }));
+
+ /*
+ * closed ledger.
+ *
+ * for this ledger entry 0 is overrreplicated, but a copy is missing in
+ * the set of bookies it is supposed to be. So it should be counted
+ * towards ledgersFoundHavingLessThanWQReplicasOfAnEntry.
+ */
+ ensembleSize = 4;
+ writeQuorumSize = 3;
+ ackQuorumSize = 2;
+ lastEntryId = 1;
+ length = 1000;
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ ledgerId = 4L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1, 3 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0 }));
+ numLedgersFoundHavingLessThanWQReplicasOfAnEntry++;
+
+ /*
+ * this is closed ledger.
+ *
+ * For third bookie, asyncGetListOfEntriesOfLedger will fail with
+ * BookieHandleNotAvailableException, so this should not be counted
+ * against missing copies of an entry. Other than that, for both entries
+ * 0 and 1, a copy is missing. Hence this should be counted towards
+ * numLedgersFoundHavingLessThanWQReplicasOfAnEntry.
+ */
+ ensembleSize = 3;
+ writeQuorumSize = 3;
+ ackQuorumSize = 2;
+ lastEntryId = 1;
+ length = 1000;
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 3));
+ ledgerId = 5L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 }));
+ errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(),
+ Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException);
+ numLedgersFoundHavingLessThanWQReplicasOfAnEntry++;
+
+ runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, 0, 0,
+ numLedgersFoundHavingLessThanWQReplicasOfAnEntry);
+ }
+
+ /*
+ * In this testscenario all the ledgers have empty segments.
+ */
+ @Test
+ public void testReplicasCheckForLedgersWithEmptySegments() throws Exception {
+ int numOfBookies = 5;
+ MultiKeyMap<String, AvailabilityOfEntriesOfLedger> returnAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, AvailabilityOfEntriesOfLedger>();
+ MultiKeyMap<String, Integer> errorReturnValueForGetAvailabilityOfEntriesOfLedger =
+ new MultiKeyMap<String, Integer>();
+ List<BookieId> bookieAddresses = addAndRegisterBookies(numOfBookies);
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerManager lm = mFactory.newLedgerManager();
+ DigestType digestType = DigestType.DUMMY;
+ byte[] password = new byte[0];
+ Collections.shuffle(bookieAddresses);
+
+ int numLedgersFoundHavingNoReplicaOfAnEntry = 0;
+ int numLedgersFoundHavingLessThanAQReplicasOfAnEntry = 0;
+ int numLedgersFoundHavingLessThanWQReplicasOfAnEntry = 0;
+
+ /*
+ * closed ledger.
+ *
+ * This closed Ledger has no entry. So it should not be counted towards
+ * numLedgersFoundHavingNoReplicaOfAnEntry/LessThanAQReplicasOfAnEntry
+ * /WQReplicasOfAnEntry.
+ */
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
+ int ensembleSize = 4;
+ int writeQuorumSize = 3;
+ int ackQuorumSize = 2;
+ long lastEntryId = -1L;
+ int length = 0;
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ long ledgerId = 1L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+
+ /*
+ * closed ledger with multiple segments.
+ *
+ * This ledger has empty last segment, but all the entries have
+ * writeQuorumSize number of copies, So it should not be counted towards
+ * numLedgersFoundHavingNoReplicaOfAnEntry/LessThanAQReplicasOfAnEntry/
+ * WQReplicasOfAnEntry.
+ */
+ lastEntryId = 2;
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(1, 5));
+ ledgerId = 2L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1, 2 }));
+
+ /*
+ * Closed ledger with multiple segments.
+ *
+ * Segment0, Segment1, Segment3, Segment5 and Segment6 are empty.
+ * Entries from entryid 3 are missing. So it should be counted towards
+ * numLedgersFoundHavingNoReplicaOfAnEntry.
+ */
+ lastEntryId = 5;
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(1, 5));
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(4L, bookieAddresses.subList(1, 5));
+ segmentEnsembles.put(4L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(1, 5));
+ segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(0, 4));
+ ledgerId = 3L;
+ createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ lastEntryId, length, digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1, 2 }));
+ numLedgersFoundHavingNoReplicaOfAnEntry++;
+
+ /*
+ * non-closed ledger with multiple segments
+ *
+ * since this is non-closed ledger, it should not be counted towards
+ * ledgersFoundHavingLessThanWQReplicasOfAnEntry
+ */
+ lastEntryId = 2;
+ segmentEnsembles.clear();
+ segmentEnsembles.put(0L, bookieAddresses.subList(0, 4));
+ segmentEnsembles.put(0L, bookieAddresses.subList(1, 5));
+ segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(1, 5));
+ ledgerId = 4L;
+ createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
+ digestType, password);
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2 }));
+ returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId),
+ new AvailabilityOfEntriesOfLedger(new long[] { 1, 2 }));
+
+ runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger,
+ numLedgersFoundHavingNoReplicaOfAnEntry, numLedgersFoundHavingLessThanAQReplicasOfAnEntry,
+ numLedgersFoundHavingLessThanWQReplicasOfAnEntry);
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java
new file mode 100644
index 00000000000..2c458d635f5
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithLedgerManagerFactory;
+import static org.testng.AssertJUnit.assertEquals;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+import lombok.Cleanup;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.meta.LedgerAuditorManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.test.TestCallbacks;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Test auditor behaviours during a rolling restart.
+ */
+public class AuditorRollingRestartTest extends BookKeeperClusterTestCase {
+
+ public AuditorRollingRestartTest() throws Exception {
+ super(3, 600);
+ // run the daemon within the bookie
+ setAutoRecoveryEnabled(true);
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ @Override
+ protected void startBKCluster(String metadataServiceUri) throws Exception {
+ super.startBKCluster(metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ }
+
+ /**
+ * Test no auditing during restart if disabled.
+ */
+ @Test
+ public void testAuditingDuringRollingRestart() throws Exception {
+ confByIndex(0).setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ runFunctionWithLedgerManagerFactory(
+ confByIndex(0),
+ mFactory -> {
+ try {
+ testAuditingDuringRollingRestart(mFactory);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e.getMessage(), e);
+ }
+ return null;
+ }
+ );
+ }
+
+ private void testAuditingDuringRollingRestart(LedgerManagerFactory mFactory) throws Exception {
+ final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
+
+ LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+ for (int i = 0; i < 10; i++) {
+ lh.asyncAddEntry("foobar".getBytes(), new TestCallbacks.AddCallbackFuture(i), null);
+ }
+ lh.addEntry("foobar".getBytes());
+ lh.close();
+
+ assertEquals("shouldn't be anything under replicated",
+ underReplicationManager.pollLedgerToRereplicate(), -1);
+ underReplicationManager.disableLedgerReplication();
+
+ @Cleanup
+ LedgerAuditorManager lam = mFactory.newLedgerAuditorManager();
+ BookieId auditor = lam.getCurrentAuditor();
+ ServerConfiguration conf = killBookie(auditor);
+ Thread.sleep(2000);
+ startBookie(conf);
+ Thread.sleep(2000); // give it time to run
+ assertEquals("shouldn't be anything under replicated", -1,
+ underReplicationManager.pollLedgerToRereplicate());
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java
new file mode 100644
index 00000000000..db338d1bb4b
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+import org.apache.bookkeeper.auth.AuthCallbacks;
+import org.apache.bookkeeper.auth.AuthToken;
+import org.apache.bookkeeper.auth.ClientAuthProvider;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.proto.ClientConnectionPeer;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * This test verifies the auditor bookie scenarios from the auth point-of-view.
+ */
+public class AuthAutoRecoveryTest extends BookKeeperClusterTestCase {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AuthAutoRecoveryTest.class);
+
+ public static final String TEST_AUTH_PROVIDER_PLUGIN_NAME = "TestAuthProviderPlugin";
+
+ private static String clientSideRole;
+
+ private static class AuditorClientAuthInterceptorFactory
+ implements ClientAuthProvider.Factory {
+
+ @Override
+ public String getPluginName() {
+ return TEST_AUTH_PROVIDER_PLUGIN_NAME;
+ }
+
+ @Override
+ public void init(ClientConfiguration conf) {
+ clientSideRole = conf.getClientRole();
+ }
+
+ @Override
+ public ClientAuthProvider newProvider(ClientConnectionPeer addr,
+ final AuthCallbacks.GenericCallback<Void> completeCb) {
+ return new ClientAuthProvider() {
+ public void init(AuthCallbacks.GenericCallback<AuthToken> cb) {
+ completeCb.operationComplete(BKException.Code.OK, null);
+ }
+
+ public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
+ }
+ };
+ }
+ }
+
+ protected ServerConfiguration newServerConfiguration() throws Exception {
+ ServerConfiguration conf = super.newServerConfiguration();
+ conf.setClientAuthProviderFactoryClass(AuditorClientAuthInterceptorFactory.class.getName());
+ return conf;
+ }
+
+ public AuthAutoRecoveryTest() {
+ super(6);
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ /*
+ * test the client role of the auditor
+ */
+ @Test
+ public void testAuthClientRole() throws Exception {
+ ServerConfiguration config = confByIndex(0);
+ assertEquals(AuditorClientAuthInterceptorFactory.class.getName(), config.getClientAuthProviderFactoryClass());
+ AutoRecoveryMain main = new AutoRecoveryMain(config);
+ try {
+ main.start();
+ Thread.sleep(500);
+ assertTrue("AuditorElector should be running",
+ main.auditorElector.isRunning());
+ assertTrue("Replication worker should be running",
+ main.replicationWorker.isRunning());
+ } finally {
+ main.shutdown();
+ }
+ assertEquals(ClientConfiguration.CLIENT_ROLE_SYSTEM, clientSideRole);
+ }
+
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java
new file mode 100644
index 00000000000..62416968142
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertNotNull;
+import static org.testng.AssertJUnit.assertTrue;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.concurrent.TimeUnit;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.bookie.BookieImpl;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.util.TestUtils;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory;
+import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver;
+import org.apache.pulsar.metadata.impl.ZKMetadataStore;
+import org.apache.zookeeper.ZooKeeper;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Test the AuditorPeer.
+ */
+@Slf4j
+public class AutoRecoveryMainTest extends BookKeeperClusterTestCase {
+
+ public AutoRecoveryMainTest() throws Exception {
+ super(3);
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ /**
+ * Test the startup of the auditorElector and RW.
+ */
+ @Test
+ public void testStartup() throws Exception {
+ log.info("testStartup()");
+ confByIndex(0).setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ AutoRecoveryMain main = new AutoRecoveryMain(confByIndex(0));
+ try {
+ main.start();
+ Thread.sleep(500);
+ assertTrue("AuditorElector should be running",
+ main.auditorElector.isRunning());
+ assertTrue("Replication worker should be running",
+ main.replicationWorker.isRunning());
+ } finally {
+ main.shutdown();
+ }
+ }
+
+ /*
+ * Test the shutdown of all daemons
+ */
+ @Test
+ public void testShutdown() throws Exception {
+ log.info("testShutdown()");
+ confByIndex(0).setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ AutoRecoveryMain main = new AutoRecoveryMain(confByIndex(0));
+ main.start();
+ Thread.sleep(500);
+ assertTrue("AuditorElector should be running",
+ main.auditorElector.isRunning());
+ assertTrue("Replication worker should be running",
+ main.replicationWorker.isRunning());
+
+ main.shutdown();
+ assertFalse("AuditorElector should not be running",
+ main.auditorElector.isRunning());
+ assertFalse("Replication worker should not be running",
+ main.replicationWorker.isRunning());
+ }
+
+ /**
+ * Test that, if an autorecovery looses its ZK connection/session it will
+ * shutdown.
+ */
+ @Test
+ public void testAutoRecoverySessionLoss() throws Exception {
+ log.info("testAutoRecoverySessionLoss()");
+ confByIndex(0).setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ confByIndex(1).setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ confByIndex(2).setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ /*
+ * initialize three AutoRecovery instances.
+ */
+ AutoRecoveryMain main1 = new AutoRecoveryMain(confByIndex(0));
+ AutoRecoveryMain main2 = new AutoRecoveryMain(confByIndex(1));
+ AutoRecoveryMain main3 = new AutoRecoveryMain(confByIndex(2));
+
+ /*
+ * start main1, make sure all the components are started and main1 is
+ * the current Auditor
+ */
+ PulsarMetadataClientDriver pulsarMetadataClientDriver1 = startAutoRecoveryMain(main1);
+ ZooKeeper zk1 = getZk(pulsarMetadataClientDriver1);
+
+ // Wait until auditor gets elected
+ for (int i = 0; i < 10; i++) {
+ try {
+ if (main1.auditorElector.getCurrentAuditor() != null) {
+ break;
+ } else {
+ Thread.sleep(1000);
+ }
+ } catch (IOException e) {
+ Thread.sleep(1000);
+ }
+ }
+ BookieId currentAuditor = main1.auditorElector.getCurrentAuditor();
+ assertNotNull(currentAuditor);
+ Auditor auditor1 = main1.auditorElector.getAuditor();
+ assertEquals("Current Auditor should be AR1", currentAuditor, BookieImpl.getBookieId(confByIndex(0)));
+ Awaitility.waitAtMost(30, TimeUnit.SECONDS).untilAsserted(() -> {
+ assertNotNull(auditor1);
+ assertTrue("Auditor of AR1 should be running", auditor1.isRunning());
+ });
+
+
+ /*
+ * start main2 and main3
+ */
+ PulsarMetadataClientDriver pulsarMetadataClientDriver2 = startAutoRecoveryMain(main2);
+ ZooKeeper zk2 = getZk(pulsarMetadataClientDriver2);
+
+ PulsarMetadataClientDriver pulsarMetadataClientDriver3 = startAutoRecoveryMain(main3);
+ ZooKeeper zk3 = getZk(pulsarMetadataClientDriver3);
+
+
+ /*
+ * make sure AR1 is still the current Auditor and AR2's and AR3's
+ * auditors are not running.
+ */
+ assertEquals("Current Auditor should still be AR1", currentAuditor, BookieImpl.getBookieId(confByIndex(0)));
+ Awaitility.await().untilAsserted(() -> {
+ assertTrue("AR2's Auditor should not be running", (main2.auditorElector.getAuditor() == null
+ || !main2.auditorElector.getAuditor().isRunning()));
+ assertTrue("AR3's Auditor should not be running", (main3.auditorElector.getAuditor() == null
+ || !main3.auditorElector.getAuditor().isRunning()));
+ });
+
+
+ /*
+ * expire zk2 and zk1 sessions.
+ */
+ zkUtil.expireSession(zk2);
+ zkUtil.expireSession(zk1);
+
+ /*
+ * wait for some time for all the components of AR1 and AR2 are
+ * shutdown.
+ */
+ for (int i = 0; i < 10; i++) {
+ if (!main1.auditorElector.isRunning() && !main1.replicationWorker.isRunning()
+ && !main1.isAutoRecoveryRunning() && !main2.auditorElector.isRunning()
+ && !main2.replicationWorker.isRunning() && !main2.isAutoRecoveryRunning()) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+
+ /*
+ * the AR3 should be current auditor.
+ */
+ currentAuditor = main3.auditorElector.getCurrentAuditor();
+ assertEquals("Current Auditor should be AR3", currentAuditor, BookieImpl.getBookieId(confByIndex(2)));
+ Awaitility.await().untilAsserted(() -> {
+ assertNotNull(main3.auditorElector.getAuditor());
+ assertTrue("Auditor of AR3 should be running", main3.auditorElector.getAuditor().isRunning());
+ });
+
+ Awaitility.waitAtMost(100, TimeUnit.SECONDS).untilAsserted(() -> {
+ /*
+ * since AR3 is current auditor, AR1's auditor should not be running
+ * anymore.
+ */
+ assertFalse("AR1's auditor should not be running", auditor1.isRunning());
+
+ /*
+ * components of AR2 and AR3 should not be running since zk1 and zk2
+ * sessions are expired.
+ */
+ assertFalse("Elector1 should have shutdown", main1.auditorElector.isRunning());
+ assertFalse("RW1 should have shutdown", main1.replicationWorker.isRunning());
+ assertFalse("AR1 should have shutdown", main1.isAutoRecoveryRunning());
+ assertFalse("Elector2 should have shutdown", main2.auditorElector.isRunning());
+ assertFalse("RW2 should have shutdown", main2.replicationWorker.isRunning());
+ assertFalse("AR2 should have shutdown", main2.isAutoRecoveryRunning());
+ });
+
+ }
+
+ /*
+ * start autoRecoveryMain and make sure all its components are running and
+ * myVote node is existing
+ */
+ PulsarMetadataClientDriver startAutoRecoveryMain(AutoRecoveryMain autoRecoveryMain) throws Exception {
+ autoRecoveryMain.start();
+ PulsarMetadataClientDriver pulsarMetadataClientDriver = (PulsarMetadataClientDriver) autoRecoveryMain.bkc
+ .getMetadataClientDriver();
+ TestUtils.assertEventuallyTrue("autoRecoveryMain components should be running",
+ () -> autoRecoveryMain.auditorElector.isRunning()
+ && autoRecoveryMain.replicationWorker.isRunning() && autoRecoveryMain.isAutoRecoveryRunning());
+ return pulsarMetadataClientDriver;
+ }
+
+ private ZooKeeper getZk(PulsarMetadataClientDriver pulsarMetadataClientDriver) throws Exception {
+ PulsarLedgerManagerFactory pulsarLedgerManagerFactory =
+ (PulsarLedgerManagerFactory) pulsarMetadataClientDriver.getLedgerManagerFactory();
+ Field field = pulsarLedgerManagerFactory.getClass().getDeclaredField("store");
+ field.setAccessible(true);
+ ZKMetadataStore zkMetadataStore = (ZKMetadataStore) field.get(pulsarLedgerManagerFactory);
+ return zkMetadataStore.getZkClient();
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java
new file mode 100644
index 00000000000..c8c76302b89
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java
@@ -0,0 +1,652 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertNotNull;
+import static org.testng.AssertJUnit.assertNull;
+import static org.testng.AssertJUnit.assertTrue;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.SortedMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.client.BookKeeperTestClient;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.common.util.OrderedScheduler;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.meta.MetadataClientDriver;
+import org.apache.bookkeeper.meta.MetadataDrivers;
+import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.proto.BookieServer;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Integration tests verifies the complete functionality of the
+ * Auditor-rereplication process: Auditor will publish the bookie failures,
+ * consequently ReplicationWorker will get the notifications and act on it.
+ */
+public class BookieAutoRecoveryTest extends BookKeeperClusterTestCase {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(BookieAutoRecoveryTest.class);
+ private static final byte[] PASSWD = "admin".getBytes();
+ private static final byte[] data = "TESTDATA".getBytes();
+ private static final String openLedgerRereplicationGracePeriod = "3000"; // milliseconds
+
+ private DigestType digestType;
+ private MetadataClientDriver metadataClientDriver;
+ private LedgerManagerFactory mFactory;
+ private LedgerUnderreplicationManager underReplicationManager;
+ private LedgerManager ledgerManager;
+ private OrderedScheduler scheduler;
+
+ private final String underreplicatedPath = "/ledgers/underreplication/ledgers";
+
+ public BookieAutoRecoveryTest() throws Exception {
+ super(3);
+
+ baseConf.setLedgerManagerFactoryClassName(
+ "org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory");
+ baseConf.setOpenLedgerRereplicationGracePeriod(openLedgerRereplicationGracePeriod);
+ baseConf.setRwRereplicateBackoffMs(500);
+ baseClientConf.setLedgerManagerFactoryClassName(
+ "org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory");
+ this.digestType = DigestType.MAC;
+ setAutoRecoveryEnabled(true);
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ baseConf.setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ baseClientConf.setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+
+ scheduler = OrderedScheduler.newSchedulerBuilder()
+ .name("test-scheduler")
+ .numThreads(1)
+ .build();
+
+ metadataClientDriver = MetadataDrivers.getClientDriver(
+ URI.create(baseClientConf.getMetadataServiceUri()));
+ metadataClientDriver.initialize(
+ baseClientConf,
+ scheduler,
+ NullStatsLogger.INSTANCE,
+ Optional.empty());
+
+ // initialize urReplicationManager
+ mFactory = metadataClientDriver.getLedgerManagerFactory();
+ underReplicationManager = mFactory.newLedgerUnderreplicationManager();
+ ledgerManager = mFactory.newLedgerManager();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+
+ if (null != underReplicationManager) {
+ underReplicationManager.close();
+ underReplicationManager = null;
+ }
+ if (null != ledgerManager) {
+ ledgerManager.close();
+ ledgerManager = null;
+ }
+ if (null != metadataClientDriver) {
+ metadataClientDriver.close();
+ metadataClientDriver = null;
+ }
+ if (null != scheduler) {
+ scheduler.shutdown();
+ }
+ }
+
+ /**
+ * Test verifies publish urLedger by Auditor and replication worker is
+ * picking up the entries and finishing the rereplication of open ledger.
+ */
+ @Test
+ public void testOpenLedgers() throws Exception {
+ List<LedgerHandle> listOfLedgerHandle = createLedgersAndAddEntries(1, 5);
+ LedgerHandle lh = listOfLedgerHandle.get(0);
+ int ledgerReplicaIndex = 0;
+ BookieId replicaToKillAddr = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+
+ final String urLedgerZNode = getUrLedgerZNode(lh);
+ ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr);
+
+ CountDownLatch latch = new CountDownLatch(1);
+ assertNull("UrLedger already exists!",
+ watchUrLedgerNode(urLedgerZNode, latch));
+
+ LOG.info("Killing Bookie :" + replicaToKillAddr);
+ killBookie(replicaToKillAddr);
+
+ // waiting to publish urLedger znode by Auditor
+ latch.await();
+ latch = new CountDownLatch(1);
+ LOG.info("Watching on urLedgerPath:" + urLedgerZNode
+ + " to know the status of rereplication process");
+ assertNotNull("UrLedger doesn't exists!",
+ watchUrLedgerNode(urLedgerZNode, latch));
+
+ // starting the replication service, so that he will be able to act as
+ // target bookie
+ startNewBookie();
+ int newBookieIndex = lastBookieIndex();
+ BookieServer newBookieServer = serverByIndex(newBookieIndex);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting to finish the replication of failed bookie : "
+ + replicaToKillAddr);
+ }
+ latch.await();
+
+ // grace period to update the urledger metadata in zookeeper
+ LOG.info("Waiting to update the urledger metadata in zookeeper");
+
+ verifyLedgerEnsembleMetadataAfterReplication(newBookieServer,
+ listOfLedgerHandle.get(0), ledgerReplicaIndex);
+ }
+
+ /**
+ * Test verifies publish urLedger by Auditor and replication worker is
+ * picking up the entries and finishing the rereplication of closed ledgers.
+ */
+ @Test
+ public void testClosedLedgers() throws Exception {
+ List<Integer> listOfReplicaIndex = new ArrayList<Integer>();
+ List<LedgerHandle> listOfLedgerHandle = createLedgersAndAddEntries(1, 5);
+ closeLedgers(listOfLedgerHandle);
+ LedgerHandle lhandle = listOfLedgerHandle.get(0);
+ int ledgerReplicaIndex = 0;
+ BookieId replicaToKillAddr = lhandle.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+
+ CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size());
+ for (LedgerHandle lh : listOfLedgerHandle) {
+ ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr);
+ listOfReplicaIndex.add(ledgerReplicaIndex);
+ assertNull("UrLedger already exists!",
+ watchUrLedgerNode(getUrLedgerZNode(lh), latch));
+ }
+
+ LOG.info("Killing Bookie :" + replicaToKillAddr);
+ killBookie(replicaToKillAddr);
+
+ // waiting to publish urLedger znode by Auditor
+ latch.await();
+
+ // Again watching the urLedger znode to know the replication status
+ latch = new CountDownLatch(listOfLedgerHandle.size());
+ for (LedgerHandle lh : listOfLedgerHandle) {
+ String urLedgerZNode = getUrLedgerZNode(lh);
+ LOG.info("Watching on urLedgerPath:" + urLedgerZNode
+ + " to know the status of rereplication process");
+ assertNotNull("UrLedger doesn't exists!",
+ watchUrLedgerNode(urLedgerZNode, latch));
+ }
+
+ // starting the replication service, so that he will be able to act as
+ // target bookie
+ startNewBookie();
+ int newBookieIndex = lastBookieIndex();
+ BookieServer newBookieServer = serverByIndex(newBookieIndex);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting to finish the replication of failed bookie : "
+ + replicaToKillAddr);
+ }
+
+ // waiting to finish replication
+ latch.await();
+
+ // grace period to update the urledger metadata in zookeeper
+ LOG.info("Waiting to update the urledger metadata in zookeeper");
+
+ for (int index = 0; index < listOfLedgerHandle.size(); index++) {
+ verifyLedgerEnsembleMetadataAfterReplication(newBookieServer,
+ listOfLedgerHandle.get(index),
+ listOfReplicaIndex.get(index));
+ }
+ }
+
+ /**
+ * Test stopping replica service while replication in progress. Considering
+ * when there is an exception will shutdown Auditor and RW processes. After
+ * restarting should be able to finish the re-replication activities
+ */
+ @Test
+ public void testStopWhileReplicationInProgress() throws Exception {
+ int numberOfLedgers = 2;
+ List<Integer> listOfReplicaIndex = new ArrayList<Integer>();
+ List<LedgerHandle> listOfLedgerHandle = createLedgersAndAddEntries(
+ numberOfLedgers, 5);
+ closeLedgers(listOfLedgerHandle);
+ LedgerHandle handle = listOfLedgerHandle.get(0);
+ BookieId replicaToKillAddr = handle.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+ LOG.info("Killing Bookie:" + replicaToKillAddr);
+
+ // Each ledger, there will be two events : create urLedger and after
+ // rereplication delete urLedger
+ CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size());
+ for (int i = 0; i < listOfLedgerHandle.size(); i++) {
+ final String urLedgerZNode = getUrLedgerZNode(listOfLedgerHandle
+ .get(i));
+ assertNull("UrLedger already exists!",
+ watchUrLedgerNode(urLedgerZNode, latch));
+ int replicaIndexInLedger = getReplicaIndexInLedger(
+ listOfLedgerHandle.get(i), replicaToKillAddr);
+ listOfReplicaIndex.add(replicaIndexInLedger);
+ }
+
+ LOG.info("Killing Bookie :" + replicaToKillAddr);
+ killBookie(replicaToKillAddr);
+
+ // waiting to publish urLedger znode by Auditor
+ latch.await();
+
+ // Again watching the urLedger znode to know the replication status
+ latch = new CountDownLatch(listOfLedgerHandle.size());
+ for (LedgerHandle lh : listOfLedgerHandle) {
+ String urLedgerZNode = getUrLedgerZNode(lh);
+ LOG.info("Watching on urLedgerPath:" + urLedgerZNode
+ + " to know the status of rereplication process");
+ assertNotNull("UrLedger doesn't exists!",
+ watchUrLedgerNode(urLedgerZNode, latch));
+ }
+
+ // starting the replication service, so that he will be able to act as
+ // target bookie
+ startNewBookie();
+ int newBookieIndex = lastBookieIndex();
+ BookieServer newBookieServer = serverByIndex(newBookieIndex);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting to finish the replication of failed bookie : "
+ + replicaToKillAddr);
+ }
+ while (true) {
+ if (latch.getCount() < numberOfLedgers || latch.getCount() <= 0) {
+ stopReplicationService();
+ LOG.info("Latch Count is:" + latch.getCount());
+ break;
+ }
+ // grace period to take breath
+ Thread.sleep(1000);
+ }
+
+ startReplicationService();
+
+ LOG.info("Waiting to finish rereplication processes");
+ latch.await();
+
+ // grace period to update the urledger metadata in zookeeper
+ LOG.info("Waiting to update the urledger metadata in zookeeper");
+
+ for (int index = 0; index < listOfLedgerHandle.size(); index++) {
+ verifyLedgerEnsembleMetadataAfterReplication(newBookieServer,
+ listOfLedgerHandle.get(index),
+ listOfReplicaIndex.get(index));
+ }
+ }
+
+ /**
+ * Verify the published urledgers of deleted ledgers(those ledgers where
+ * deleted after publishing as urledgers by Auditor) should be cleared off
+ * by the newly selected replica bookie.
+ */
+ @Test
+ public void testNoSuchLedgerExists() throws Exception {
+ List<LedgerHandle> listOfLedgerHandle = createLedgersAndAddEntries(2, 5);
+ CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size());
+ for (LedgerHandle lh : listOfLedgerHandle) {
+ assertNull("UrLedger already exists!",
+ watchUrLedgerNode(getUrLedgerZNode(lh), latch));
+ }
+ BookieId replicaToKillAddr = listOfLedgerHandle.get(0)
+ .getLedgerMetadata().getAllEnsembles()
+ .get(0L).get(0);
+ killBookie(replicaToKillAddr);
+ replicaToKillAddr = listOfLedgerHandle.get(0)
+ .getLedgerMetadata().getAllEnsembles()
+ .get(0L).get(0);
+ killBookie(replicaToKillAddr);
+ // waiting to publish urLedger znode by Auditor
+ latch.await();
+
+ latch = new CountDownLatch(listOfLedgerHandle.size());
+ for (LedgerHandle lh : listOfLedgerHandle) {
+ assertNotNull("UrLedger doesn't exists!",
+ watchUrLedgerNode(getUrLedgerZNode(lh), latch));
+ }
+
+ // delete ledgers
+ for (LedgerHandle lh : listOfLedgerHandle) {
+ bkc.deleteLedger(lh.getId());
+ }
+ startNewBookie();
+
+ // waiting to delete published urledgers, since it doesn't exists
+ latch.await();
+
+ for (LedgerHandle lh : listOfLedgerHandle) {
+ assertNull("UrLedger still exists after rereplication",
+ watchUrLedgerNode(getUrLedgerZNode(lh), latch));
+ }
+ }
+
+ /**
+ * Test that if a empty ledger loses the bookie not in the quorum for entry 0, it will
+ * still be openable when it loses enough bookies to lose a whole quorum.
+ */
+ @Test
+ public void testEmptyLedgerLosesQuorumEventually() throws Exception {
+ LedgerHandle lh = bkc.createLedger(3, 2, 2, DigestType.CRC32, PASSWD);
+ CountDownLatch latch = new CountDownLatch(1);
+ String urZNode = getUrLedgerZNode(lh);
+ watchUrLedgerNode(urZNode, latch);
+
+ BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(2);
+ LOG.info("Killing last bookie, {}, in ensemble {}", replicaToKill,
+ lh.getLedgerMetadata().getAllEnsembles().get(0L));
+ killBookie(replicaToKill);
+ startNewBookie();
+
+ getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs
+
+ assertTrue("Should be marked as underreplicated", latch.await(5, TimeUnit.SECONDS));
+ latch = new CountDownLatch(1);
+ Stat s = watchUrLedgerNode(urZNode, latch); // should be marked as replicated
+ if (s != null) {
+ assertTrue("Should be marked as replicated", latch.await(15, TimeUnit.SECONDS));
+ }
+
+ replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(1);
+ LOG.info("Killing second bookie, {}, in ensemble {}", replicaToKill,
+ lh.getLedgerMetadata().getAllEnsembles().get(0L));
+ killBookie(replicaToKill);
+
+ getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs
+
+ assertTrue("Should be marked as underreplicated", latch.await(5, TimeUnit.SECONDS));
+ latch = new CountDownLatch(1);
+ s = watchUrLedgerNode(urZNode, latch); // should be marked as replicated
+
+ startNewBookie();
+ getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs
+
+ if (s != null) {
+ assertTrue("Should be marked as replicated", latch.await(20, TimeUnit.SECONDS));
+ }
+
+ // should be able to open ledger without issue
+ bkc.openLedger(lh.getId(), DigestType.CRC32, PASSWD);
+ }
+
+ /**
+ * Test verifies bookie recovery, the host (recorded via ipaddress in
+ * ledgermetadata).
+ */
+ @Test
+ public void testLedgerMetadataContainsIpAddressAsBookieID()
+ throws Exception {
+ stopBKCluster();
+ bkc = new BookKeeperTestClient(baseClientConf);
+ // start bookie with useHostNameAsBookieID=false, as old bookie
+ ServerConfiguration serverConf1 = newServerConfiguration();
+ // start 2 more bookies with useHostNameAsBookieID=true
+ ServerConfiguration serverConf2 = newServerConfiguration();
+ serverConf2.setUseHostNameAsBookieID(true);
+ ServerConfiguration serverConf3 = newServerConfiguration();
+ serverConf3.setUseHostNameAsBookieID(true);
+ startAndAddBookie(serverConf1);
+ startAndAddBookie(serverConf2);
+ startAndAddBookie(serverConf3);
+
+ List<LedgerHandle> listOfLedgerHandle = createLedgersAndAddEntries(1, 5);
+ LedgerHandle lh = listOfLedgerHandle.get(0);
+ int ledgerReplicaIndex = 0;
+ final SortedMap<Long, ? extends List<BookieId>> ensembles = lh.getLedgerMetadata().getAllEnsembles();
+ final List<BookieId> bkAddresses = ensembles.get(0L);
+ BookieId replicaToKillAddr = bkAddresses.get(0);
+ for (BookieId bookieSocketAddress : bkAddresses) {
+ if (!isCreatedFromIp(bookieSocketAddress)) {
+ replicaToKillAddr = bookieSocketAddress;
+ LOG.info("Kill bookie which has registered using hostname");
+ break;
+ }
+ }
+
+ final String urLedgerZNode = getUrLedgerZNode(lh);
+ ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr);
+
+ CountDownLatch latch = new CountDownLatch(1);
+ assertNull("UrLedger already exists!",
+ watchUrLedgerNode(urLedgerZNode, latch));
+
+ LOG.info("Killing Bookie :" + replicaToKillAddr);
+ killBookie(replicaToKillAddr);
+
+ // waiting to publish urLedger znode by Auditor
+ latch.await();
+ latch = new CountDownLatch(1);
+ LOG.info("Watching on urLedgerPath:" + urLedgerZNode
+ + " to know the status of rereplication process");
+ assertNotNull("UrLedger doesn't exists!",
+ watchUrLedgerNode(urLedgerZNode, latch));
+
+ // starting the replication service, so that he will be able to act as
+ // target bookie
+ ServerConfiguration serverConf = newServerConfiguration();
+ serverConf.setUseHostNameAsBookieID(false);
+ startAndAddBookie(serverConf);
+
+ int newBookieIndex = lastBookieIndex();
+ BookieServer newBookieServer = serverByIndex(newBookieIndex);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting to finish the replication of failed bookie : "
+ + replicaToKillAddr);
+ }
+ latch.await();
+
+ // grace period to update the urledger metadata in zookeeper
+ LOG.info("Waiting to update the urledger metadata in zookeeper");
+
+ verifyLedgerEnsembleMetadataAfterReplication(newBookieServer,
+ listOfLedgerHandle.get(0), ledgerReplicaIndex);
+
+ }
+
+ /**
+ * Test verifies bookie recovery, the host (recorded via useHostName in
+ * ledgermetadata).
+ */
+ @Test
+ public void testLedgerMetadataContainsHostNameAsBookieID()
+ throws Exception {
+ stopBKCluster();
+
+ bkc = new BookKeeperTestClient(baseClientConf);
+ // start bookie with useHostNameAsBookieID=false, as old bookie
+ ServerConfiguration serverConf1 = newServerConfiguration();
+ // start 2 more bookies with useHostNameAsBookieID=true
+ ServerConfiguration serverConf2 = newServerConfiguration();
+ serverConf2.setUseHostNameAsBookieID(true);
+ ServerConfiguration serverConf3 = newServerConfiguration();
+ serverConf3.setUseHostNameAsBookieID(true);
+ startAndAddBookie(serverConf1);
+ startAndAddBookie(serverConf2);
+ startAndAddBookie(serverConf3);
+
+ List<LedgerHandle> listOfLedgerHandle = createLedgersAndAddEntries(1, 5);
+ LedgerHandle lh = listOfLedgerHandle.get(0);
+ int ledgerReplicaIndex = 0;
+ final SortedMap<Long, ? extends List<BookieId>> ensembles = lh.getLedgerMetadata().getAllEnsembles();
+ final List<BookieId> bkAddresses = ensembles.get(0L);
+ BookieId replicaToKillAddr = bkAddresses.get(0);
+ for (BookieId bookieSocketAddress : bkAddresses) {
+ if (isCreatedFromIp(bookieSocketAddress)) {
+ replicaToKillAddr = bookieSocketAddress;
+ LOG.info("Kill bookie which has registered using ipaddress");
+ break;
+ }
+ }
+
+ final String urLedgerZNode = getUrLedgerZNode(lh);
+ ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr);
+
+ CountDownLatch latch = new CountDownLatch(1);
+ assertNull("UrLedger already exists!",
+ watchUrLedgerNode(urLedgerZNode, latch));
+
+ LOG.info("Killing Bookie :" + replicaToKillAddr);
+ killBookie(replicaToKillAddr);
+
+ // waiting to publish urLedger znode by Auditor
+ latch.await();
+ latch = new CountDownLatch(1);
+ LOG.info("Watching on urLedgerPath:" + urLedgerZNode
+ + " to know the status of rereplication process");
+ assertNotNull("UrLedger doesn't exists!",
+ watchUrLedgerNode(urLedgerZNode, latch));
+
+ // creates new bkclient
+ bkc = new BookKeeperTestClient(baseClientConf);
+ // starting the replication service, so that he will be able to act as
+ // target bookie
+ ServerConfiguration serverConf = newServerConfiguration();
+ serverConf.setUseHostNameAsBookieID(true);
+ startAndAddBookie(serverConf);
+
+ int newBookieIndex = lastBookieIndex();
+ BookieServer newBookieServer = serverByIndex(newBookieIndex);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting to finish the replication of failed bookie : "
+ + replicaToKillAddr);
+ }
+ latch.await();
+
+ // grace period to update the urledger metadata in zookeeper
+ LOG.info("Waiting to update the urledger metadata in zookeeper");
+
+ verifyLedgerEnsembleMetadataAfterReplication(newBookieServer,
+ listOfLedgerHandle.get(0), ledgerReplicaIndex);
+
+ }
+
+ private int getReplicaIndexInLedger(LedgerHandle lh, BookieId replicaToKill) {
+ SortedMap<Long, ? extends List<BookieId>> ensembles = lh.getLedgerMetadata().getAllEnsembles();
+ int ledgerReplicaIndex = -1;
+ for (BookieId addr : ensembles.get(0L)) {
+ ++ledgerReplicaIndex;
+ if (addr.equals(replicaToKill)) {
+ break;
+ }
+ }
+ return ledgerReplicaIndex;
+ }
+
+ private void verifyLedgerEnsembleMetadataAfterReplication(
+ BookieServer newBookieServer, LedgerHandle lh,
+ int ledgerReplicaIndex) throws Exception {
+ LedgerHandle openLedger = bkc
+ .openLedger(lh.getId(), digestType, PASSWD);
+
+ BookieId inetSocketAddress = openLedger.getLedgerMetadata().getAllEnsembles().get(0L)
+ .get(ledgerReplicaIndex);
+ assertEquals("Rereplication has been failed and ledgerReplicaIndex :"
+ + ledgerReplicaIndex, newBookieServer.getBookieId(),
+ inetSocketAddress);
+ openLedger.close();
+ }
+
+ private void closeLedgers(List<LedgerHandle> listOfLedgerHandle)
+ throws InterruptedException, BKException {
+ for (LedgerHandle lh : listOfLedgerHandle) {
+ lh.close();
+ }
+ }
+
+ private List<LedgerHandle> createLedgersAndAddEntries(int numberOfLedgers,
+ int numberOfEntries)
+ throws InterruptedException, BKException {
+ List<LedgerHandle> listOfLedgerHandle = new ArrayList<LedgerHandle>(
+ numberOfLedgers);
+ for (int index = 0; index < numberOfLedgers; index++) {
+ LedgerHandle lh = bkc.createLedger(3, 3, digestType, PASSWD);
+ listOfLedgerHandle.add(lh);
+ for (int i = 0; i < numberOfEntries; i++) {
+ lh.addEntry(data);
+ }
+ }
+ return listOfLedgerHandle;
+ }
+
+ private String getUrLedgerZNode(LedgerHandle lh) {
+ return ZkLedgerUnderreplicationManager.getUrLedgerZnode(
+ underreplicatedPath, lh.getId());
+ }
+
+ private Stat watchUrLedgerNode(final String znode,
+ final CountDownLatch latch) throws KeeperException,
+ InterruptedException {
+ return zkc.exists(znode, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getType() == EventType.NodeDeleted) {
+ LOG.info("Received Ledger rereplication completion event :"
+ + event.getType());
+ latch.countDown();
+ }
+ if (event.getType() == EventType.NodeCreated) {
+ LOG.info("Received urLedger publishing event :"
+ + event.getType());
+ latch.countDown();
+ }
+ }
+ });
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java
new file mode 100644
index 00000000000..eb9f95ffdf7
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.fail;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.meta.LayoutManager;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Tests verifies bookie vs ledger mapping generating by the BookieLedgerIndexer.
+ */
+public class BookieLedgerIndexTest extends BookKeeperClusterTestCase {
+
+ // Depending on the taste, select the amount of logging
+ // by decommenting one of the two lines below
+ // private final static Logger LOG = Logger.getRootLogger();
+ private static final Logger LOG = LoggerFactory
+ .getLogger(BookieLedgerIndexTest.class);
+
+ private Random rng; // Random Number Generator
+ private ArrayList<byte[]> entries; // generated entries
+ private final DigestType digestType = DigestType.CRC32;
+ private int numberOfLedgers = 3;
+ private List<Long> ledgerList;
+ private LedgerManagerFactory newLedgerManagerFactory;
+ private LedgerManager ledgerManager;
+
+ public BookieLedgerIndexTest() throws Exception {
+ this("org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory");
+ }
+
+ BookieLedgerIndexTest(String ledgerManagerFactory) throws Exception {
+ super(3);
+ LOG.info("Running test case using ledger manager : "
+ + ledgerManagerFactory);
+ // set ledger manager name
+ baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
+ baseClientConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ public void setUp() throws Exception {
+ super.setUp();
+ baseConf.setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ rng = new Random(System.currentTimeMillis()); // Initialize the Random
+ // Number Generator
+ entries = new ArrayList<byte[]>(); // initialize the entries list
+ ledgerList = new ArrayList<Long>(3);
+
+ String ledgersRoot = "/ledgers";
+ String storeUri = metadataServiceUri.replaceAll("zk://", "").replaceAll("/ledgers", "");
+ MetadataStoreExtended store = MetadataStoreExtended.create(storeUri,
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+ LayoutManager layoutManager = new PulsarLayoutManager(store, ledgersRoot);
+ newLedgerManagerFactory = new PulsarLedgerManagerFactory();
+
+ ClientConfiguration conf = new ClientConfiguration();
+ conf.setZkLedgersRootPath(ledgersRoot);
+ newLedgerManagerFactory.initialize(conf, layoutManager, 1);
+ ledgerManager = newLedgerManagerFactory.newLedgerManager();
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ super.tearDown();
+ if (null != newLedgerManagerFactory) {
+ newLedgerManagerFactory.close();
+ newLedgerManagerFactory = null;
+ }
+ if (null != ledgerManager) {
+ ledgerManager.close();
+ ledgerManager = null;
+ }
+ }
+
+ /**
+ * Verify the bookie-ledger mapping with minimum number of bookies and few
+ * ledgers.
+ */
+ @Test
+ public void testSimpleBookieLedgerMapping() throws Exception {
+
+ for (int i = 0; i < numberOfLedgers; i++) {
+ createAndAddEntriesToLedger().close();
+ }
+
+ BookieLedgerIndexer bookieLedgerIndex = new BookieLedgerIndexer(
+ ledgerManager);
+
+ Map<String, Set<Long>> bookieToLedgerIndex = bookieLedgerIndex
+ .getBookieToLedgerIndex();
+
+ assertEquals("Missed few bookies in the bookie-ledger mapping!", 3,
+ bookieToLedgerIndex.size());
+ Collection<Set<Long>> bk2ledgerEntry = bookieToLedgerIndex.values();
+ for (Set<Long> ledgers : bk2ledgerEntry) {
+ assertEquals("Missed few ledgers in the bookie-ledger mapping!", 3,
+ ledgers.size());
+ for (Long ledgerId : ledgers) {
+ assertTrue("Unknown ledger-bookie mapping", ledgerList
+ .contains(ledgerId));
+ }
+ }
+ }
+
+ /**
+ * Verify ledger index with failed bookies and throws exception.
+ */
+ @SuppressWarnings("deprecation")
+// @Test
+// public void testWithoutZookeeper() throws Exception {
+// // This test case is for ledger metadata that stored in ZooKeeper. As
+// // far as MSLedgerManagerFactory, ledger metadata are stored in other
+// // storage. So this test is not suitable for MSLedgerManagerFactory.
+// if (newLedgerManagerFactory instanceof org.apache.bookkeeper.meta.MSLedgerManagerFactory) {
+// return;
+// }
+//
+// for (int i = 0; i < numberOfLedgers; i++) {
+// createAndAddEntriesToLedger().close();
+// }
+//
+// BookieLedgerIndexer bookieLedgerIndex = new BookieLedgerIndexer(
+// ledgerManager);
+// stopZKCluster();
+// try {
+// bookieLedgerIndex.getBookieToLedgerIndex();
+// fail("Must throw exception as zookeeper are not running!");
+// } catch (BKAuditException bkAuditException) {
+// // expected behaviour
+// }
+// }
+
+ /**
+ * Verify indexing with multiple ensemble reformation.
+ */
+ @Test
+ public void testEnsembleReformation() throws Exception {
+ try {
+ LedgerHandle lh1 = createAndAddEntriesToLedger();
+ LedgerHandle lh2 = createAndAddEntriesToLedger();
+
+ startNewBookie();
+ shutdownBookie(lastBookieIndex() - 1);
+
+ // add few more entries after ensemble reformation
+ for (int i = 0; i < 10; i++) {
+ ByteBuffer entry = ByteBuffer.allocate(4);
+ entry.putInt(rng.nextInt(Integer.MAX_VALUE));
+ entry.position(0);
+
+ entries.add(entry.array());
+ lh1.addEntry(entry.array());
+ lh2.addEntry(entry.array());
+ }
+
+ BookieLedgerIndexer bookieLedgerIndex = new BookieLedgerIndexer(
+ ledgerManager);
+
+ Map<String, Set<Long>> bookieToLedgerIndex = bookieLedgerIndex
+ .getBookieToLedgerIndex();
+ assertEquals("Missed few bookies in the bookie-ledger mapping!", 4,
+ bookieToLedgerIndex.size());
+ Collection<Set<Long>> bk2ledgerEntry = bookieToLedgerIndex.values();
+ for (Set<Long> ledgers : bk2ledgerEntry) {
+ assertEquals(
+ "Missed few ledgers in the bookie-ledger mapping!", 2,
+ ledgers.size());
+ for (Long ledgerNode : ledgers) {
+ assertTrue("Unknown ledger-bookie mapping", ledgerList
+ .contains(ledgerNode));
+ }
+ }
+ } catch (BKException e) {
+ LOG.error("Test failed", e);
+ fail("Test failed due to BookKeeper exception");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.error("Test failed", e);
+ fail("Test failed due to interruption");
+ }
+ }
+
+ private void shutdownBookie(int bkShutdownIndex) throws Exception {
+ killBookie(bkShutdownIndex);
+ }
+
+ private LedgerHandle createAndAddEntriesToLedger() throws BKException,
+ InterruptedException {
+ int numEntriesToWrite = 20;
+ // Create a ledger
+ LedgerHandle lh = bkc.createLedger(digestType, "admin".getBytes());
+ LOG.info("Ledger ID: " + lh.getId());
+ for (int i = 0; i < numEntriesToWrite; i++) {
+ ByteBuffer entry = ByteBuffer.allocate(4);
+ entry.putInt(rng.nextInt(Integer.MAX_VALUE));
+ entry.position(0);
+
+ entries.add(entry.array());
+ lh.addEntry(entry.array());
+ }
+ ledgerList.add(lh.getId());
+ return lh;
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java
new file mode 100644
index 00000000000..4360bf32546
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import java.util.List;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * Utility class for replication tests.
+ */
+public class ReplicationTestUtil {
+
+ /**
+ * Checks whether ledger is in under-replication.
+ */
+ public static boolean isLedgerInUnderReplication(ZooKeeper zkc, long id,
+ String basePath) throws KeeperException, InterruptedException {
+ List<String> children;
+ try {
+ children = zkc.getChildren(basePath, true);
+ } catch (KeeperException.NoNodeException nne) {
+ return false;
+ }
+
+ boolean isMatched = false;
+ for (String child : children) {
+ if (child.startsWith("urL") && child.contains(String.valueOf(id))) {
+ isMatched = true;
+ break;
+ } else {
+ String path = basePath + '/' + child;
+ try {
+ if (zkc.getChildren(path, false).size() > 0) {
+ isMatched = isLedgerInUnderReplication(zkc, id, path);
+ }
+ } catch (KeeperException.NoNodeException nne) {
+ return false;
+ }
+ }
+
+ }
+ return isMatched;
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java
new file mode 100644
index 00000000000..8a2e7f2747a
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Map.Entry;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.util.BookKeeperConstants;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Test auto recovery.
+ */
+public class TestAutoRecoveryAlongWithBookieServers extends
+ BookKeeperClusterTestCase {
+
+ private String basePath = "";
+
+ public TestAutoRecoveryAlongWithBookieServers() throws Exception {
+ super(3);
+ setAutoRecoveryEnabled(true);
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ basePath = BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH + '/'
+ + BookKeeperConstants.UNDER_REPLICATION_NODE
+ + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH;
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ @Override
+ protected void startBKCluster(String metadataServiceUri) throws Exception {
+ super.startBKCluster(metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ }
+
+ /**
+ * Tests that the auto recovery service along with Bookie servers itself.
+ */
+ @Test
+ public void testAutoRecoveryAlongWithBookieServers() throws Exception {
+ LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32,
+ "testpasswd".getBytes());
+ byte[] testData = "testBuiltAutoRecovery".getBytes();
+
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(testData);
+ }
+ lh.close();
+ BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+
+ killBookie(replicaToKill);
+
+ BookieId newBkAddr = startNewBookieAndReturnBookieId();
+
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(),
+ basePath)) {
+ Thread.sleep(100);
+ }
+
+ // Killing all bookies except newly replicated bookie
+ for (Entry<Long, ? extends List<BookieId>> entry :
+ lh.getLedgerMetadata().getAllEnsembles().entrySet()) {
+ List<BookieId> bookies = entry.getValue();
+ for (BookieId bookie : bookies) {
+ if (bookie.equals(newBkAddr)) {
+ continue;
+ }
+ killBookie(bookie);
+ }
+ }
+
+ // Should be able to read the entries from 0-9
+ LedgerHandle lhs = bkc.openLedgerNoRecovery(lh.getId(),
+ BookKeeper.DigestType.CRC32, "testpasswd".getBytes());
+ Enumeration<LedgerEntry> entries = lhs.readEntries(0, 9);
+ assertTrue("Should have the elements", entries.hasMoreElements());
+ while (entries.hasMoreElements()) {
+ LedgerEntry entry = entries.nextElement();
+ assertEquals("testBuiltAutoRecovery", new String(entry.getEntry()));
+ }
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java
new file mode 100644
index 00000000000..ca02f91d1de
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java
@@ -0,0 +1,1249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.replication;
+
+import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE;
+import static org.apache.bookkeeper.replication.ReplicationStats.NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION;
+import static org.apache.bookkeeper.replication.ReplicationStats.REPLICATION_SCOPE;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertNotNull;
+import static org.testng.AssertJUnit.assertNull;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.fail;
+import io.netty.util.HashedWheelTimer;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.TimerTask;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import lombok.Cleanup;
+import org.apache.bookkeeper.bookie.BookieImpl;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.BookKeeperTestClient;
+import org.apache.bookkeeper.client.ClientUtil;
+import org.apache.bookkeeper.client.EnsemblePlacementPolicy;
+import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy;
+import org.apache.bookkeeper.client.ZoneawareEnsemblePlacementPolicy;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
+import org.apache.bookkeeper.common.util.OrderedScheduler;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.feature.FeatureProvider;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+import org.apache.bookkeeper.meta.MetadataBookieDriver;
+import org.apache.bookkeeper.meta.MetadataClientDriver;
+import org.apache.bookkeeper.meta.MetadataDrivers;
+import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager;
+import org.apache.bookkeeper.meta.exceptions.MetadataException;
+import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.net.DNSToSwitchMapping;
+import org.apache.bookkeeper.proto.BookieAddressResolver;
+import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException;
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.Gauge;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.test.TestStatsProvider;
+import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger;
+import org.apache.bookkeeper.util.BookKeeperConstants;
+import org.apache.bookkeeper.util.StaticDNSResolver;
+import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory;
+import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver;
+import org.apache.pulsar.metadata.impl.ZKMetadataStore;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+import org.awaitility.Awaitility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Test the ReplicationWroker, where it has to replicate the fragments from
+ * failed Bookies to given target Bookie.
+ */
+public class TestReplicationWorker extends BookKeeperClusterTestCase {
+
+ private static final byte[] TESTPASSWD = "testpasswd".getBytes();
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestReplicationWorker.class);
+ private String basePath = "";
+ private String baseLockPath = "";
+ private MetadataBookieDriver driver;
+ private LedgerManagerFactory mFactory;
+ private LedgerUnderreplicationManager underReplicationManager;
+ private LedgerManager ledgerManager;
+ private static byte[] data = "TestReplicationWorker".getBytes();
+ private OrderedScheduler scheduler;
+ private String zkLedgersRootPath;
+
+ public TestReplicationWorker() throws Exception {
+ this("org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory");
+ }
+
+ TestReplicationWorker(String ledgerManagerFactory) throws Exception {
+ super(3, 300);
+ LOG.info("Running test case using ledger manager : "
+ + ledgerManagerFactory);
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver");
+ Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver");
+ // set ledger manager name
+ baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
+ baseClientConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
+ baseConf.setRereplicationEntryBatchSize(3);
+ baseConf.setZkTimeout(7000);
+ baseConf.setZkRetryBackoffMaxMs(500);
+ baseConf.setZkRetryBackoffStartMs(10);
+ }
+
+ @BeforeMethod
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ zkLedgersRootPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseClientConf);
+ basePath = zkLedgersRootPath + '/'
+ + BookKeeperConstants.UNDER_REPLICATION_NODE
+ + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH;
+ baseLockPath = zkLedgersRootPath + '/'
+ + BookKeeperConstants.UNDER_REPLICATION_NODE
+ + "/locks";
+ baseClientConf.setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ baseConf.setMetadataServiceUri(
+ zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", ""));
+ this.scheduler = OrderedScheduler.newSchedulerBuilder()
+ .name("test-scheduler")
+ .numThreads(1)
+ .build();
+
+ this.driver = MetadataDrivers.getBookieDriver(
+ URI.create(baseConf.getMetadataServiceUri()));
+ this.driver.initialize(
+ baseConf,
+ NullStatsLogger.INSTANCE);
+ // initialize urReplicationManager
+ mFactory = driver.getLedgerManagerFactory();
+ ledgerManager = mFactory.newLedgerManager();
+ underReplicationManager = mFactory.newLedgerUnderreplicationManager();
+ }
+
+ @AfterMethod
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ if (null != ledgerManager) {
+ ledgerManager.close();
+ ledgerManager = null;
+ }
+ if (null != underReplicationManager) {
+ underReplicationManager.close();
+ underReplicationManager = null;
+ }
+ if (null != driver) {
+ driver.close();
+ }
+ if (null != scheduler) {
+ scheduler.shutdown();
+ scheduler = null;
+ }
+ if (null != mFactory) {
+ mFactory.close();
+ }
+ }
+
+ /**
+ * Tests that replication worker should replicate the failed bookie
+ * fragments to target bookie given to the worker.
+ */
+ @Test
+ public void testRWShouldReplicateFragmentsToTargetBookie() throws Exception {
+ LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32,
+ TESTPASSWD);
+
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(data);
+ }
+ BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+
+ LOG.info("Killing Bookie : {}", replicaToKill);
+ killBookie(replicaToKill);
+
+ BookieId newBkAddr = startNewBookieAndReturnBookieId();
+ LOG.info("New Bookie addr : {}", newBkAddr);
+
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(data);
+ }
+
+ ReplicationWorker rw = new ReplicationWorker(baseConf);
+
+ rw.start();
+ try {
+
+ underReplicationManager.markLedgerUnderreplicated(lh.getId(),
+ replicaToKill.toString());
+
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
+ .getId(), basePath)) {
+ Thread.sleep(100);
+ }
+
+ killAllBookies(lh, newBkAddr);
+
+ // Should be able to read the entries from 0-9
+ verifyRecoveredLedgers(lh, 0, 9);
+ } finally {
+ rw.shutdown();
+ }
+ }
+
+ /**
+ * Tests that replication worker should retry for replication until enough
+ * bookies available for replication.
+ */
+ @Test
+ public void testRWShouldRetryUntilThereAreEnoughBksAvailableForReplication()
+ throws Exception {
+ LedgerHandle lh = bkc.createLedger(1, 1, BookKeeper.DigestType.CRC32,
+ TESTPASSWD);
+
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(data);
+ }
+ lh.close();
+ BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+ LOG.info("Killing Bookie : {}", replicaToKill);
+ ServerConfiguration killedBookieConfig = killBookie(replicaToKill);
+
+ BookieId newBkAddr = startNewBookieAndReturnBookieId();
+ LOG.info("New Bookie addr :" + newBkAddr);
+
+ killAllBookies(lh, newBkAddr);
+ ReplicationWorker rw = new ReplicationWorker(baseConf);
+
+ rw.start();
+ try {
+ underReplicationManager.markLedgerUnderreplicated(lh.getId(),
+ replicaToKill.toString());
+ int counter = 30;
+ while (counter-- > 0) {
+ assertTrue("Expecting that replication should not complete",
+ ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
+ .getId(), basePath));
+ Thread.sleep(100);
+ }
+ // restart killed bookie
+ startAndAddBookie(killedBookieConfig);
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
+ .getId(), basePath)) {
+ Thread.sleep(100);
+ }
+ // Should be able to read the entries from 0-9
+ verifyRecoveredLedgers(lh, 0, 9);
+ } finally {
+ rw.shutdown();
+ }
+ }
+
+ /**
+ * Tests that replication worker1 should take one fragment replication and
+ * other replication worker also should compete for the replication.
+ */
+ @Test
+ public void test2RWsShouldCompeteForReplicationOf2FragmentsAndCompleteReplication()
+ throws Exception {
+ LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32,
+ TESTPASSWD);
+
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(data);
+ }
+ lh.close();
+ BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+ LOG.info("Killing Bookie : {}", replicaToKill);
+ ServerConfiguration killedBookieConfig = killBookie(replicaToKill);
+
+ killAllBookies(lh, null);
+ // Starte RW1
+ BookieId newBkAddr1 = startNewBookieAndReturnBookieId();
+ LOG.info("New Bookie addr : {}", newBkAddr1);
+ ReplicationWorker rw1 = new ReplicationWorker(baseConf);
+
+ // Starte RW2
+ BookieId newBkAddr2 = startNewBookieAndReturnBookieId();
+ LOG.info("New Bookie addr : {}", newBkAddr2);
+ ReplicationWorker rw2 = new ReplicationWorker(baseConf);
+ rw1.start();
+ rw2.start();
+
+ try {
+ underReplicationManager.markLedgerUnderreplicated(lh.getId(),
+ replicaToKill.toString());
+ int counter = 10;
+ while (counter-- > 0) {
+ assertTrue("Expecting that replication should not complete",
+ ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
+ .getId(), basePath));
+ Thread.sleep(100);
+ }
+ // restart killed bookie
+ startAndAddBookie(killedBookieConfig);
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
+ .getId(), basePath)) {
+ Thread.sleep(100);
+ }
+ // Should be able to read the entries from 0-9
+ verifyRecoveredLedgers(lh, 0, 9);
+ } finally {
+ rw1.shutdown();
+ rw2.shutdown();
+ }
+ }
+
+ /**
+ * Tests that Replication worker should clean the leadger under replication
+ * node of the ledger already deleted.
+ */
+ @Test
+ public void testRWShouldCleanTheLedgerFromUnderReplicationIfLedgerAlreadyDeleted()
+ throws Exception {
+ LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32,
+ TESTPASSWD);
+
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(data);
+ }
+ lh.close();
+ BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+ LOG.info("Killing Bookie : {}", replicaToKill);
+ killBookie(replicaToKill);
+
+ BookieId newBkAddr = startNewBookieAndReturnBookieId();
+ LOG.info("New Bookie addr : {}", newBkAddr);
+ ReplicationWorker rw = new ReplicationWorker(baseConf);
+ rw.start();
+
+ try {
+ bkc.deleteLedger(lh.getId()); // Deleting the ledger
+ // Also mark ledger as in UnderReplication
+ underReplicationManager.markLedgerUnderreplicated(lh.getId(),
+ replicaToKill.toString());
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
+ .getId(), basePath)) {
+ Thread.sleep(100);
+ }
+ } finally {
+ rw.shutdown();
+ }
+
+ }
+
+ @Test
+ public void testMultipleLedgerReplicationWithReplicationWorker()
+ throws Exception {
+ // Ledger1
+ LedgerHandle lh1 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32,
+ TESTPASSWD);
+
+ for (int i = 0; i < 10; i++) {
+ lh1.addEntry(data);
+ }
+ BookieId replicaToKillFromFirstLedger = lh1.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+
+ LOG.info("Killing Bookie : {}", replicaToKillFromFirstLedger);
+
+ // Ledger2
+ LedgerHandle lh2 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32,
+ TESTPASSWD);
+
+ for (int i = 0; i < 10; i++) {
+ lh2.addEntry(data);
+ }
+ BookieId replicaToKillFromSecondLedger = lh2.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+
+ LOG.info("Killing Bookie : {}", replicaToKillFromSecondLedger);
+
+ // Kill ledger1
+ killBookie(replicaToKillFromFirstLedger);
+ lh1.close();
+ // Kill ledger2
+ killBookie(replicaToKillFromFirstLedger);
+ lh2.close();
+
+ BookieId newBkAddr = startNewBookieAndReturnBookieId();
+ LOG.info("New Bookie addr : {}", newBkAddr);
+
+ ReplicationWorker rw = new ReplicationWorker(baseConf);
+
+ rw.start();
+ try {
+
+ // Mark ledger1 and 2 as underreplicated
+ underReplicationManager.markLedgerUnderreplicated(lh1.getId(),
+ replicaToKillFromFirstLedger.toString());
+ underReplicationManager.markLedgerUnderreplicated(lh2.getId(),
+ replicaToKillFromSecondLedger.toString());
+
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh1
+ .getId(), basePath)) {
+ Thread.sleep(100);
+ }
+
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh2
+ .getId(), basePath)) {
+ Thread.sleep(100);
+ }
+
+ killAllBookies(lh1, newBkAddr);
+
+ // Should be able to read the entries from 0-9
+ verifyRecoveredLedgers(lh1, 0, 9);
+ verifyRecoveredLedgers(lh2, 0, 9);
+ } finally {
+ rw.shutdown();
+ }
+
+ }
+
+ /**
+ * Tests that ReplicationWorker should fence the ledger and release ledger
+ * lock after timeout. Then replication should happen normally.
+ */
+ @Test
+ public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsUR()
+ throws Exception {
+ LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32,
+ TESTPASSWD);
+
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(data);
+ }
+ BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+
+ LOG.info("Killing Bookie : {}", replicaToKill);
+ killBookie(replicaToKill);
+
+ BookieId newBkAddr = startNewBookieAndReturnBookieId();
+ LOG.info("New Bookie addr : {}", newBkAddr);
+
+ // set to 3s instead of default 30s
+ baseConf.setOpenLedgerRereplicationGracePeriod("3000");
+ ReplicationWorker rw = new ReplicationWorker(baseConf);
+
+ @Cleanup MetadataClientDriver clientDriver = MetadataDrivers.getClientDriver(
+ URI.create(baseClientConf.getMetadataServiceUri()));
+ clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty());
+
+ LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory();
+
+ LedgerUnderreplicationManager underReplicationManager = mFactory
+ .newLedgerUnderreplicationManager();
+ rw.start();
+ try {
+
+ underReplicationManager.markLedgerUnderreplicated(lh.getId(),
+ replicaToKill.toString());
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
+ .getId(), basePath)) {
+ Thread.sleep(100);
+ }
+ killAllBookies(lh, newBkAddr);
+ // Should be able to read the entries from 0-9
+ verifyRecoveredLedgers(lh, 0, 9);
+ lh = bkc.openLedgerNoRecovery(lh.getId(),
+ BookKeeper.DigestType.CRC32, TESTPASSWD);
+ assertFalse("Ledger must have been closed by RW", ClientUtil
+ .isLedgerOpen(lh));
+ } finally {
+ rw.shutdown();
+ underReplicationManager.close();
+ }
+
+ }
+
+ @Test
+ public void testBookiesNotAvailableScenarioForReplicationWorker() throws Exception {
+ int ensembleSize = 3;
+ LedgerHandle lh = bkc.createLedger(ensembleSize, ensembleSize, BookKeeper.DigestType.CRC32, TESTPASSWD);
+
+ int numOfEntries = 7;
+ for (int i = 0; i < numOfEntries; i++) {
+ lh.addEntry(data);
+ }
+ lh.close();
+
+ BookieId[] bookiesKilled = new BookieId[ensembleSize];
+ ServerConfiguration[] killedBookiesConfig = new ServerConfiguration[ensembleSize];
+
+ // kill all bookies
+ for (int i = 0; i < ensembleSize; i++) {
+ bookiesKilled[i] = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(i);
+ killedBookiesConfig[i] = getBkConf(bookiesKilled[i]);
+ LOG.info("Killing Bookie : {}", bookiesKilled[i]);
+ killBookie(bookiesKilled[i]);
+ }
+
+ // start new bookiesToKill number of bookies
+ for (int i = 0; i < ensembleSize; i++) {
+ BookieId newBkAddr = startNewBookieAndReturnBookieId();
+ }
+
+ // create couple of replicationworkers
+ ServerConfiguration newRWConf = new ServerConfiguration(baseConf);
+ newRWConf.setLockReleaseOfFailedLedgerGracePeriod("64");
+ ReplicationWorker rw1 = new ReplicationWorker(newRWConf);
+ ReplicationWorker rw2 = new ReplicationWorker(newRWConf);
+
+ @Cleanup
+ MetadataClientDriver clientDriver = MetadataDrivers
+ .getClientDriver(URI.create(baseClientConf.getMetadataServiceUri()));
+ clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty());
+
+ LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory();
+
+ LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
+ try {
+ //mark ledger underreplicated
+ for (int i = 0; i < bookiesKilled.length; i++) {
+ underReplicationManager.markLedgerUnderreplicated(lh.getId(), bookiesKilled[i].toString());
+ }
+ while (!ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) {
+ Thread.sleep(100);
+ }
+ rw1.start();
+ rw2.start();
+
+ AtomicBoolean isBookieRestarted = new AtomicBoolean(false);
+
+ (new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(3000);
+ isBookieRestarted.set(true);
+ /*
+ * after sleeping for 3000 msecs, restart one of the
+ * bookie, so that replication can succeed.
+ */
+ startBookie(killedBookiesConfig[0]);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ })).start();
+
+ int rw1PrevFailedAttemptsCount = 0;
+ int rw2PrevFailedAttemptsCount = 0;
+ while (!isBookieRestarted.get()) {
+ /*
+ * since all the bookies containing the ledger entries are down
+ * replication wouldnt have succeeded.
+ */
+ assertTrue("Ledger: " + lh.getId() + " should be underreplicated",
+ ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath));
+
+ // the number of failed attempts should have increased.
+ int rw1CurFailedAttemptsCount = rw1.replicationFailedLedgers.get(lh.getId()).get();
+ assertTrue(
+ "The current number of failed attempts: " + rw1CurFailedAttemptsCount
+ + " should be greater than or equal to previous value: " + rw1PrevFailedAttemptsCount,
+ rw1CurFailedAttemptsCount >= rw1PrevFailedAttemptsCount);
+ rw1PrevFailedAttemptsCount = rw1CurFailedAttemptsCount;
+
+ int rw2CurFailedAttemptsCount = rw2.replicationFailedLedgers.get(lh.getId()).get();
+ assertTrue(
+ "The current number of failed attempts: " + rw2CurFailedAttemptsCount
+ + " should be greater than or equal to previous value: " + rw2PrevFailedAttemptsCount,
+ rw2CurFailedAttemptsCount >= rw2PrevFailedAttemptsCount);
+ rw2PrevFailedAttemptsCount = rw2CurFailedAttemptsCount;
+
+ Thread.sleep(50);
+ }
+
+ /**
+ * since one of the killed bookie is restarted, replicationworker
+ * should succeed in replicating this under replicated ledger and it
+ * shouldn't be under replicated anymore.
+ */
+ int timeToWaitForReplicationToComplete = 20000;
+ int timeWaited = 0;
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) {
+ Thread.sleep(100);
+ timeWaited += 100;
+ if (timeWaited == timeToWaitForReplicationToComplete) {
+ fail("Ledger should be replicated by now");
+ }
+ }
+
+ rw1PrevFailedAttemptsCount = rw1.replicationFailedLedgers.get(lh.getId()).get();
+ rw2PrevFailedAttemptsCount = rw2.replicationFailedLedgers.get(lh.getId()).get();
+ Thread.sleep(2000);
+ // now since the ledger is replicated, number of failed attempts
+ // counter shouldn't be increased even after sleeping for sometime.
+ assertEquals("rw1 failedattempts", rw1PrevFailedAttemptsCount,
+ rw1.replicationFailedLedgers.get(lh.getId()).get());
+ assertEquals("rw2 failed attempts ", rw2PrevFailedAttemptsCount,
+ rw2.replicationFailedLedgers.get(lh.getId()).get());
+
+ /*
+ * Since these entries are eventually available, and replication has
+ * eventually succeeded, in one of the RW
+ * unableToReadEntriesForReplication should be 0.
+ */
+ int rw1UnableToReadEntriesForReplication = rw1.unableToReadEntriesForReplication.get(lh.getId()).size();
+ int rw2UnableToReadEntriesForReplication = rw2.unableToReadEntriesForReplication.get(lh.getId()).size();
+ assertTrue(
+ "unableToReadEntriesForReplication in RW1: " + rw1UnableToReadEntriesForReplication
+ + " in RW2: "
+ + rw2UnableToReadEntriesForReplication,
+ (rw1UnableToReadEntriesForReplication == 0)
+ || (rw2UnableToReadEntriesForReplication == 0));
+ } finally {
+ rw1.shutdown();
+ rw2.shutdown();
+ underReplicationManager.close();
+ }
+ }
+
+ class InjectedReplicationWorker extends ReplicationWorker {
+ CopyOnWriteArrayList<Long> delayReplicationPeriods;
+
+ public InjectedReplicationWorker(ServerConfiguration conf, StatsLogger statsLogger,
+ CopyOnWriteArrayList<Long> delayReplicationPeriods)
+ throws CompatibilityException, ReplicationException.UnavailableException,
+ InterruptedException, IOException {
+ super(conf, statsLogger);
+ this.delayReplicationPeriods = delayReplicationPeriods;
+ }
+
+ @Override
+ protected void scheduleTaskWithDelay(TimerTask timerTask, long delayPeriod) {
+ delayReplicationPeriods.add(delayPeriod);
+ super.scheduleTaskWithDelay(timerTask, delayPeriod);
+ }
+ }
+
+ @Test
+ public void testDeferLedgerLockReleaseForReplicationWorker() throws Exception {
+ int ensembleSize = 3;
+ LedgerHandle lh = bkc.createLedger(ensembleSize, ensembleSize, BookKeeper.DigestType.CRC32, TESTPASSWD);
+ int numOfEntries = 7;
+ for (int i = 0; i < numOfEntries; i++) {
+ lh.addEntry(data);
+ }
+ lh.close();
+
+ BookieId[] bookiesKilled = new BookieId[ensembleSize];
+ ServerConfiguration[] killedBookiesConfig = new ServerConfiguration[ensembleSize];
+
+ // kill all bookies
+ for (int i = 0; i < ensembleSize; i++) {
+ bookiesKilled[i] = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(i);
+ killedBookiesConfig[i] = getBkConf(bookiesKilled[i]);
+ LOG.info("Killing Bookie : {}", bookiesKilled[i]);
+ killBookie(bookiesKilled[i]);
+ }
+
+ // start new bookiesToKill number of bookies
+ for (int i = 0; i < ensembleSize; i++) {
+ startNewBookieAndReturnBookieId();
+ }
+
+ // create couple of replicationworkers
+ long lockReleaseOfFailedLedgerGracePeriod = 64L;
+ long baseBackoffForLockReleaseOfFailedLedger = lockReleaseOfFailedLedgerGracePeriod
+ / (int) Math.pow(2, ReplicationWorker.NUM_OF_EXPONENTIAL_BACKOFF_RETRIALS);
+ ServerConfiguration newRWConf = new ServerConfiguration(baseConf);
+ newRWConf.setLockReleaseOfFailedLedgerGracePeriod(Long.toString(lockReleaseOfFailedLedgerGracePeriod));
+ newRWConf.setRereplicationEntryBatchSize(1000);
+ CopyOnWriteArrayList<Long> rw1DelayReplicationPeriods = new CopyOnWriteArrayList<Long>();
+ CopyOnWriteArrayList<Long> rw2DelayReplicationPeriods = new CopyOnWriteArrayList<Long>();
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger1 = statsProvider.getStatsLogger("rw1");
+ TestStatsLogger statsLogger2 = statsProvider.getStatsLogger("rw2");
+ ReplicationWorker rw1 = new InjectedReplicationWorker(newRWConf, statsLogger1, rw1DelayReplicationPeriods);
+ ReplicationWorker rw2 = new InjectedReplicationWorker(newRWConf, statsLogger2, rw2DelayReplicationPeriods);
+
+ Counter numEntriesUnableToReadForReplication1 = statsLogger1
+ .getCounter(NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION);
+ Counter numEntriesUnableToReadForReplication2 = statsLogger2
+ .getCounter(NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION);
+ @Cleanup
+ MetadataClientDriver clientDriver = MetadataDrivers
+ .getClientDriver(URI.create(baseClientConf.getMetadataServiceUri()));
+ clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty());
+
+ LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory();
+
+ LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
+ try {
+ // mark ledger underreplicated
+ for (int i = 0; i < bookiesKilled.length; i++) {
+ underReplicationManager.markLedgerUnderreplicated(lh.getId(), bookiesKilled[i].toString());
+ }
+ while (!ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) {
+ Thread.sleep(100);
+ }
+ rw1.start();
+ rw2.start();
+
+ // wait for RWs to complete 'numOfAttemptsToWaitFor' failed attempts
+ int numOfAttemptsToWaitFor = 10;
+ while ((rw1.replicationFailedLedgers.get(lh.getId()).get() < numOfAttemptsToWaitFor)
+ || rw2.replicationFailedLedgers.get(lh.getId()).get() < numOfAttemptsToWaitFor) {
+ Thread.sleep(500);
+ }
+
+ /*
+ * since all the bookies containing the ledger entries are down
+ * replication wouldn't have succeeded.
+ */
+ assertTrue("Ledger: " + lh.getId() + " should be underreplicated",
+ ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath));
+
+ /*
+ * since RW failed 'numOfAttemptsToWaitFor' number of times, we
+ * should have atleast (numOfAttemptsToWaitFor - 1)
+ * delayReplicationPeriods and their value should be
+ * (lockReleaseOfFailedLedgerGracePeriod/16) , 2 * previous value,..
+ * with max : lockReleaseOfFailedLedgerGracePeriod
+ */
+ for (int i = 0; i < ((numOfAttemptsToWaitFor - 1)); i++) {
+ long expectedDelayValue = Math.min(lockReleaseOfFailedLedgerGracePeriod,
+ baseBackoffForLockReleaseOfFailedLedger * (1 << i));
+ assertEquals("RW1 delayperiod", (Long) expectedDelayValue, rw1DelayReplicationPeriods.get(i));
+ assertEquals("RW2 delayperiod", (Long) expectedDelayValue, rw2DelayReplicationPeriods.get(i));
+ }
+
+ /*
+ * RW wont try to replicate until and unless RW succeed in reading
+ * those failed entries before proceeding with replication of under
+ * replicated fragment, so the numEntriesUnableToReadForReplication
+ * should be just 'numOfEntries', though RW failed to replicate
+ * multiple times.
+ */
+ assertEquals("numEntriesUnableToReadForReplication for RW1", Long.valueOf((long) numOfEntries),
+ numEntriesUnableToReadForReplication1.get());
+ assertEquals("numEntriesUnableToReadForReplication for RW2", Long.valueOf((long) numOfEntries),
+ numEntriesUnableToReadForReplication2.get());
+
+ /*
+ * Since these entries are unavailable,
+ * unableToReadEntriesForReplication should be of size numOfEntries.
+ */
+ assertEquals("RW1 unabletoreadentries", numOfEntries,
+ rw1.unableToReadEntriesForReplication.get(lh.getId()).size());
+ assertEquals("RW2 unabletoreadentries", numOfEntries,
+ rw2.unableToReadEntriesForReplication.get(lh.getId()).size());
+ } finally {
+ rw1.shutdown();
+ rw2.shutdown();
+ underReplicationManager.close();
+ }
+ }
+
+ /**
+ * Tests that ReplicationWorker should not have identified for postponing
+ * the replication if ledger is in open state and lastFragment is not in
+ * underReplication state. Note that RW should not fence such ledgers.
+ */
+ @Test
+ public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR()
+ throws Exception {
+ LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32,
+ TESTPASSWD);
+
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(data);
+ }
+ BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0);
+
+ LOG.info("Killing Bookie : {}", replicaToKill);
+ killBookie(replicaToKill);
+
+ BookieId newBkAddr = startNewBookieAndReturnBookieId();
+ LOG.info("New Bookie addr : {}", newBkAddr);
+
+ // Reform ensemble...Making sure that last fragment is not in
+ // under-replication
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(data);
+ }
+
+ ReplicationWorker rw = new ReplicationWorker(baseConf);
+
+ baseClientConf.setMetadataServiceUri(zkUtil.getMetadataServiceUri());
+
+ @Cleanup MetadataClientDriver driver = MetadataDrivers.getClientDriver(
+ URI.create(baseClientConf.getMetadataServiceUri()));
+ driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty());
+
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+
+ LedgerUnderreplicationManager underReplicationManager = mFactory
+ .newLedgerUnderreplicationManager();
+
+ rw.start();
+ try {
+
+ underReplicationManager.markLedgerUnderreplicated(lh.getId(),
+ replicaToKill.toString());
+ while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh
+ .getId(), basePath)) {
+ Thread.sleep(100);
+ }
+
+ killAllBookies(lh, newBkAddr);
+
+ // Should be able to read the entries from 0-9
+ verifyRecoveredLedgers(lh, 0, 9);
+ lh = bkc.openLedgerNoRecovery(lh.getId(),
+ BookKeeper.DigestType.CRC32, TESTPASSWD);
+
+ // Ledger should be still in open state
+ assertTrue("Ledger must have been closed by RW", ClientUtil
+ .isLedgerOpen(lh));
+ } finally {
+ rw.shutdown();
+ underReplicationManager.close();
+ }
+
+ }
+
+ /**
+ * Test that the replication worker will not shutdown on a simple ZK disconnection.
+ */
+ @Test
+ public void testRWZKConnectionLost() throws Exception {
+ try (ZooKeeperClient zk = ZooKeeperClient.newBuilder()
+ .connectString(zkUtil.getZooKeeperConnectString())
+ .sessionTimeoutMs(10000)
+ .build()) {
+
+ ReplicationWorker rw = new ReplicationWorker(baseConf);
+ rw.start();
+ for (int i = 0; i < 10; i++) {
+ if (rw.isRunning()) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ assertTrue("Replication worker should be running", rw.isRunning());
+
+ stopZKCluster();
+ // ZK is down for shorter period than reconnect timeout
+ Thread.sleep(1000);
+ startZKCluster();
+
+ assertTrue("Replication worker should not shutdown", rw.isRunning());
+ }
+ }
+
+ /**
+ * Test that the replication worker shuts down on non-recoverable ZK connection loss.
+ */
+ @Test
+ public void testRWZKConnectionLostOnNonRecoverableZkError() throws Exception {
+ for (int j = 0; j < 3; j++) {
+ LedgerHandle lh = bkc.createLedger(1, 1, 1,
+ BookKeeper.DigestType.CRC32, TESTPASSWD,
+ null);
+ final long createdLedgerId = lh.getId();
+ for (int i = 0; i < 10; i++) {
+ lh.addEntry(data);
+ }
+ lh.close();
+ }
+
+ killBookie(2);
+ killBookie(1);
+ startNewBookie();
+ startNewBookie();
+
+ servers.get(0).getConfiguration().setRwRereplicateBackoffMs(100);
+ servers.get(0).startAutoRecovery();
+
+ Auditor auditor = getAuditor(10, TimeUnit.SECONDS);
+ ReplicationWorker rw = servers.get(0).getReplicationWorker();
+
+ ZkLedgerUnderreplicationManager ledgerUnderreplicationManager =
+ (ZkLedgerUnderreplicationManager) FieldUtils.readField(auditor,
+ "ledgerUnderreplicationManager", true);
+
+ ZooKeeper zkc = (ZooKeeper) FieldUtils.readField(ledgerUnderreplicationManager, "zkc", true);
+ auditor.submitAuditTask().get();
+
+ assertTrue(zkc.getState().isConnected());
+ zkc.close();
+ assertFalse(zkc.getState().isConnected());
+
+ auditor.submitAuditTask();
+ rw.run();
+
+ for (int i = 0; i < 10; i++) {
+ if (!rw.isRunning() && !auditor.isRunning()) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ assertFalse("Replication worker should NOT be running", rw.isRunning());
+ assertFalse("Auditor should NOT be running", auditor.isRunning());
+ }
+
+ private void killAllBookies(LedgerHandle lh, BookieId excludeBK)
+ throws Exception {
+ // Killing all bookies except newly replicated bookie
+ for (Entry<Long, ? extends List<BookieId>> entry :
+ lh.getLedgerMetadata().getAllEnsembles().entrySet()) {
+ List<BookieId> bookies = entry.getValue();
+ for (BookieId bookie : bookies) {
+ if (bookie.equals(excludeBK)) {
+ continue;
+ }
+ killBookie(bookie);
+ }
+ }
+ }
+
+ private void verifyRecoveredLedgers(LedgerHandle lh, long startEntryId,
+ long endEntryId) throws BKException, InterruptedException {
+ LedgerHandle lhs = bkc.openLedgerNoRecovery(lh.getId(),
+ BookKeeper.DigestType.CRC32, TESTPASSWD);
+ Enumeration<LedgerEntry> entries = lhs.readEntries(startEntryId,
+ endEntryId);
+ assertTrue("Should have the elements", entries.hasMoreElements());
+ while (entries.hasMoreElements()) {
+ LedgerEntry entry = entries.nextElement();
+ assertEquals("TestReplicationWorker", new String(entry.getEntry()));
+ }
+ }
+
+ @Test
+ public void testReplicateEmptyOpenStateLedger() throws Exception {
+ LedgerHandle lh = bkc.createLedger(3, 3, 2, BookKeeper.DigestType.CRC32, TESTPASSWD);
+ assertFalse(lh.getLedgerMetadata().isClosed());
+
+ List<BookieId> firstEnsemble = lh.getLedgerMetadata().getAllEnsembles().firstEntry().getValue();
+ List<BookieId> ensemble = lh.getLedgerMetadata().getAllEnsembles().entrySet().iterator().next().getValue();
+ killBookie(ensemble.get(1));
+
+ startNewBookie();
+ baseConf.setOpenLedgerRereplicationGracePeriod(String.valueOf(30));
+ ReplicationWorker replicationWorker = new ReplicationWorker(baseConf);
+ replicationWorker.start();
+
+ try {
+ underReplicationManager.markLedgerUnderreplicated(lh.getId(), ensemble.get(1).toString());
+ Awaitility.waitAtMost(60, TimeUnit.SECONDS).untilAsserted(() ->
+ assertFalse(ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath))
+ );
+
+ LedgerHandle lh1 = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD);
+ assertTrue(lh1.getLedgerMetadata().isClosed());
+ } finally {
+ replicationWorker.shutdown();
+ }
+ }
+
+ @Test
+ public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsOnRack() throws Exception {
+ testRepairedNotAdheringPlacementPolicyLedgerFragments(RackawareEnsemblePlacementPolicy.class, null);
+ }
+
+ @Test
+ public void testReplicationStats() throws Exception {
+ BiConsumer<Boolean, ReplicationWorker> checkReplicationStats = (first, rw) -> {
+ try {
+ final Method rereplicate = rw.getClass().getDeclaredMethod("rereplicate");
+ rereplicate.setAccessible(true);
+ final Object result = rereplicate.invoke(rw);
+ final Field statsLoggerField = rw.getClass().getDeclaredField("statsLogger");
+ statsLoggerField.setAccessible(true);
+ final TestStatsLogger statsLogger = (TestStatsLogger) statsLoggerField.get(rw);
+
+ final Counter numDeferLedgerLockReleaseOfFailedLedgerCounter =
+ statsLogger.getCounter(ReplicationStats.NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER);
+ final Counter numLedgersReplicatedCounter =
+ statsLogger.getCounter(ReplicationStats.NUM_FULL_OR_PARTIAL_LEDGERS_REPLICATED);
+ final Counter numNotAdheringPlacementLedgersCounter = statsLogger
+ .getCounter(ReplicationStats.NUM_NOT_ADHERING_PLACEMENT_LEDGERS_REPLICATED);
+
+ assertEquals("NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER",
+ 1, numDeferLedgerLockReleaseOfFailedLedgerCounter.get().longValue());
+
+ if (first) {
+ assertFalse((boolean) result);
+ assertEquals("NUM_FULL_OR_PARTIAL_LEDGERS_REPLICATED",
+ 0, numLedgersReplicatedCounter.get().longValue());
+ assertEquals("NUM_NOT_ADHERING_PLACEMENT_LEDGERS_REPLICATED",
+ 0, numNotAdheringPlacementLedgersCounter.get().longValue());
+
+ } else {
+ assertTrue((boolean) result);
+ assertEquals("NUM_FULL_OR_PARTIAL_LEDGERS_REPLICATED",
+ 1, numLedgersReplicatedCounter.get().longValue());
+ assertEquals("NUM_NOT_ADHERING_PLACEMENT_LEDGERS_REPLICATED",
+ 1, numNotAdheringPlacementLedgersCounter.get().longValue());
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
+ testRepairedNotAdheringPlacementPolicyLedgerFragments(
+ RackawareEnsemblePlacementPolicy.class, checkReplicationStats);
+ }
+
+ private void testRepairedNotAdheringPlacementPolicyLedgerFragments(
+ Class<? extends EnsemblePlacementPolicy> placementPolicyClass,
+ BiConsumer<Boolean, ReplicationWorker> checkReplicationStats) throws Exception {
+ List<BookieId> firstThreeBookies = servers.stream().map(ele -> {
+ try {
+ return ele.getServer().getBookieId();
+ } catch (UnknownHostException e) {
+ return null;
+ }
+ }).filter(Objects::nonNull).collect(Collectors.toList());
+
+ baseClientConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName());
+ baseClientConf.setProperty("enforceStrictZoneawarePlacement", false);
+ bkc.close();
+ bkc = new BookKeeperTestClient(baseClientConf) {
+ @Override
+ protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf,
+ DNSToSwitchMapping dnsResolver,
+ HashedWheelTimer timer,
+ FeatureProvider featureProvider,
+ StatsLogger statsLogger,
+ BookieAddressResolver bookieAddressResolver)
+ throws IOException {
+ EnsemblePlacementPolicy ensemblePlacementPolicy = null;
+ if (ZoneawareEnsemblePlacementPolicy.class == placementPolicyClass) {
+ ensemblePlacementPolicy = buildZoneAwareEnsemblePlacementPolicy(firstThreeBookies);
+ } else if (RackawareEnsemblePlacementPolicy.class == placementPolicyClass) {
+ ensemblePlacementPolicy = buildRackAwareEnsemblePlacementPolicy(firstThreeBookies);
+ }
+ ensemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer,
+ featureProvider, statsLogger, bookieAddressResolver);
+ return ensemblePlacementPolicy;
+ }
+ };
+
+ //This ledger not adhering placement policy, the combine(0,1,2) rack is 1.
+ LedgerHandle lh = bkc.createLedger(3, 3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
+
+ int entrySize = 10;
+ for (int i = 0; i < entrySize; i++) {
+ lh.addEntry(data);
+ }
+ lh.close();
+
+ int minNumRacksPerWriteQuorumConfValue = 2;
+
+ ServerConfiguration servConf = new ServerConfiguration(confByIndex(0));
+ servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue);
+ servConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName());
+ servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000);
+ servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true);
+
+ MutableObject<Auditor> auditorRef = new MutableObject<Auditor>();
+ try {
+ TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef);
+ Gauge<? extends Number> ledgersNotAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value",
+ 1, ledgersNotAdheringToPlacementPolicyGuage.getSample());
+ Gauge<? extends Number> ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger
+ .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY);
+ assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value",
+ 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample());
+ } finally {
+ Auditor auditor = auditorRef.getValue();
+ if (auditor != null) {
+ auditor.close();
+ }
+ }
+
+ ZooKeeper zk = getZk((PulsarMetadataClientDriver) bkc.getMetadataClientDriver());
+
+
+ Stat stat = zk.exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false);
+ assertNotNull(stat);
+
+ baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true);
+ BookKeeper bookKeeper = new BookKeeperTestClient(baseClientConf) {
+ @Override
+ protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf,
+ DNSToSwitchMapping dnsResolver,
+ HashedWheelTimer timer,
+ FeatureProvider featureProvider,
+ StatsLogger statsLogger,
+ BookieAddressResolver bookieAddressResolver)
+ throws IOException {
+ EnsemblePlacementPolicy ensemblePlacementPolicy = null;
+ if (ZoneawareEnsemblePlacementPolicy.class == placementPolicyClass) {
+ ensemblePlacementPolicy = buildZoneAwareEnsemblePlacementPolicy(firstThreeBookies);
+ } else if (RackawareEnsemblePlacementPolicy.class == placementPolicyClass) {
+ ensemblePlacementPolicy = buildRackAwareEnsemblePlacementPolicy(firstThreeBookies);
+ }
+ ensemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer,
+ featureProvider, statsLogger, bookieAddressResolver);
+ return ensemblePlacementPolicy;
+ }
+ };
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(REPLICATION_SCOPE);
+ ReplicationWorker rw = new ReplicationWorker(baseConf, bookKeeper, false, statsLogger);
+
+ if (checkReplicationStats != null) {
+ checkReplicationStats.accept(true, rw);
+ } else {
+ rw.start();
+ }
+
+ //start new bookie, the rack is /rack2
+ BookieId newBookieId = startNewBookieAndReturnBookieId();
+
+ if (checkReplicationStats != null) {
+ checkReplicationStats.accept(false, rw);
+ }
+
+ Awaitility.await().untilAsserted(() -> {
+ LedgerMetadata metadata = bkc.getLedgerManager().readLedgerMetadata(lh.getId()).get().getValue();
+ List<BookieId> newBookies = metadata.getAllEnsembles().get(0L);
+ assertTrue(newBookies.contains(newBookieId));
+ });
+
+ Awaitility.await().untilAsserted(() -> {
+ Stat stat1 = zk.exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false);
+ assertNull(stat1);
+ });
+
+ for (BookieId rack1Book : firstThreeBookies) {
+ killBookie(rack1Book);
+ }
+
+ verifyRecoveredLedgers(lh, 0, entrySize - 1);
+
+ if (checkReplicationStats == null) {
+ rw.shutdown();
+ }
+ baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(false);
+ bookKeeper.close();
+ }
+
+ private EnsemblePlacementPolicy buildRackAwareEnsemblePlacementPolicy(List<BookieId> bookieIds) {
+ return new RackawareEnsemblePlacementPolicy() {
+ @Override
+ public String resolveNetworkLocation(BookieId addr) {
+ if (bookieIds.contains(addr)) {
+ return "/rack1";
+ }
+ //The other bookie is /rack2
+ return "/rack2";
+ }
+ };
+ }
+
+ private EnsemblePlacementPolicy buildZoneAwareEnsemblePlacementPolicy(List<BookieId> firstThreeBookies) {
+ return new ZoneawareEnsemblePlacementPolicy() {
+ @Override
+ protected String resolveNetworkLocation(BookieId addr) {
+ //The first three bookie 1 is /zone1/ud1
+ //The first three bookie 2,3 is /zone1/ud2
+ if (firstThreeBookies.get(0).equals(addr)) {
+ return "/zone1/ud1";
+ } else if (firstThreeBookies.contains(addr)) {
+ return "/zone1/ud2";
+ }
+ //The other bookie is /zone2/ud1
+ return "/zone2/ud1";
+ }
+ };
+ }
+
+ private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf,
+ MutableObject<Auditor> auditorRef)
+ throws MetadataException, CompatibilityException, KeeperException,
+ InterruptedException, ReplicationException.UnavailableException, UnknownHostException {
+ LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
+ LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager();
+ TestStatsProvider statsProvider = new TestStatsProvider();
+ TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE);
+ TestStatsProvider.TestOpStatsLogger placementPolicyCheckStatsLogger =
+ (TestStatsProvider.TestOpStatsLogger) statsLogger
+ .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME);
+
+ final AuditorPeriodicCheckTest.TestAuditor auditor = new AuditorPeriodicCheckTest.TestAuditor(
+ BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, statsLogger, null);
+ auditorRef.setValue(auditor);
+ CountDownLatch latch = auditor.getLatch();
+ assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0,
+ placementPolicyCheckStatsLogger.getSuccessCount());
+ urm.setPlacementPolicyCheckCTime(-1);
+ auditor.start();
+ /*
+ * since placementPolicyCheckCTime is set to -1, placementPolicyCheck should be
+ * scheduled to run with no initialdelay
+ */
+ assertTrue("placementPolicyCheck should have executed", latch.await(20, TimeUnit.SECONDS));
+ for (int i = 0; i < 20; i++) {
+ Thread.sleep(100);
+ if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) {
+ break;
+ }
+ }
+ assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1,
+ placementPolicyCheckStatsLogger.getSuccessCount());
+ return statsLogger;
+ }
+
+ private ZooKeeper getZk(PulsarMetadataClientDriver pulsarMetadataClientDriver) throws Exception {
+ PulsarLedgerManagerFactory pulsarLedgerManagerFactory =
+ (PulsarLedgerManagerFactory) pulsarMetadataClientDriver.getLedgerManagerFactory();
+ Field field = pulsarLedgerManagerFactory.getClass().getDeclaredField("store");
+ field.setAccessible(true);
+ ZKMetadataStore zkMetadataStore = (ZKMetadataStore) field.get(pulsarLedgerManagerFactory);
+ return zkMetadataStore.getZkClient();
+ }
+}