You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by te...@apache.org on 2023/12/03 11:16:33 UTC
(pulsar) branch branch-2.10 updated: [fix][broker] Fix the deadlock when using BookieRackAffinityMapping with rackaware policy (#21481)
This is an automated email from the ASF dual-hosted git repository.
technoboy pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 269b336b25a [fix][broker] Fix the deadlock when using BookieRackAffinityMapping with rackaware policy (#21481)
269b336b25a is described below
commit 269b336b25ab717b13a917b2c5f9fa14d3f5c431
Author: erobot <er...@users.noreply.github.com>
AuthorDate: Fri Nov 10 18:54:01 2023 +0800
[fix][broker] Fix the deadlock when using BookieRackAffinityMapping with rackaware policy (#21481)
---
.../rackawareness/BookieRackAffinityMapping.java | 9 +--
.../BookieRackAffinityMappingTest.java | 68 ++++++++++++++++++++++
2 files changed, 73 insertions(+), 4 deletions(-)
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
index 3654b7b54c3..098e2d24515 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
@@ -244,6 +244,7 @@ public class BookieRackAffinityMapping extends AbstractDNSToSwitchMapping
bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH)
.thenAccept(optVal -> {
+ Set<BookieId> bookieIdSet = new HashSet<>();
synchronized (this) {
LOG.info("Bookie rack info updated to {}. Notifying rackaware policy.", optVal);
this.updateRacksWithHost(optVal.orElseGet(BookiesRackConfiguration::new));
@@ -258,12 +259,12 @@ public class BookieRackAffinityMapping extends AbstractDNSToSwitchMapping
LOG.debug("Bookies with rack update from {} to {}", bookieAddressListLastTime,
bookieAddressList);
}
- Set<BookieId> bookieIdSet = new HashSet<>(bookieAddressList);
+ bookieIdSet.addAll(bookieAddressList);
bookieIdSet.addAll(bookieAddressListLastTime);
bookieAddressListLastTime = bookieAddressList;
- if (rackawarePolicy != null) {
- rackawarePolicy.onBookieRackChange(new ArrayList<>(bookieIdSet));
- }
+ }
+ if (rackawarePolicy != null) {
+ rackawarePolicy.onBookieRackChange(new ArrayList<>(bookieIdSet));
}
});
}
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java
index cf6e0c7cdef..f5fe88b7d4d 100644
--- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.bookie.rackawareness;
import static org.apache.bookkeeper.feature.SettableFeatureProvider.DISABLE_ALL;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -28,6 +29,7 @@ import io.netty.util.HashedWheelTimer;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -35,8 +37,12 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import lombok.Cleanup;
import org.apache.bookkeeper.client.DefaultBookieAddressResolver;
import org.apache.bookkeeper.client.EnsemblePlacementPolicy;
import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy;
@@ -47,6 +53,7 @@ import org.apache.bookkeeper.discover.RegistrationClient;
import org.apache.bookkeeper.net.BookieId;
import org.apache.bookkeeper.net.BookieNode;
import org.apache.bookkeeper.net.BookieSocketAddress;
+import org.apache.bookkeeper.net.NetworkTopology;
import org.apache.bookkeeper.proto.BookieAddressResolver;
import org.apache.bookkeeper.stats.NullStatsLogger;
import org.apache.bookkeeper.stats.StatsLogger;
@@ -56,6 +63,8 @@ import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.MetadataStoreConfig;
import org.apache.pulsar.metadata.api.MetadataStoreFactory;
+import org.apache.pulsar.metadata.api.Notification;
+import org.apache.pulsar.metadata.api.NotificationType;
import org.apache.pulsar.metadata.bookkeeper.BookieServiceInfoSerde;
import org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient;
import org.awaitility.Awaitility;
@@ -332,4 +341,63 @@ public class BookieRackAffinityMappingTest {
timer.stop();
}
+
+ @Test
+ public void testNoDeadlockWithRackawarePolicy() throws Exception {
+ ClientConfiguration bkClientConf = new ClientConfiguration();
+ bkClientConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store);
+
+ BookieRackAffinityMapping mapping = new BookieRackAffinityMapping();
+ mapping.setBookieAddressResolver(BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER);
+ mapping.setConf(bkClientConf);
+
+ @Cleanup("stop")
+ HashedWheelTimer timer = new HashedWheelTimer(new ThreadFactoryBuilder().setNameFormat("TestTimer-%d").build(),
+ bkClientConf.getTimeoutTimerTickDurationMs(), TimeUnit.MILLISECONDS,
+ bkClientConf.getTimeoutTimerNumTicks());
+
+ RackawareEnsemblePlacementPolicy repp = new RackawareEnsemblePlacementPolicy();
+ repp.initialize(bkClientConf, Optional.of(mapping), timer,
+ DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER);
+ repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK);
+
+ mapping.registerRackChangeListener(repp);
+
+ @Cleanup("shutdownNow")
+ ExecutorService executor1 = Executors.newSingleThreadExecutor();
+ @Cleanup("shutdownNow")
+ ExecutorService executor2 = Executors.newSingleThreadExecutor();
+
+ CountDownLatch count = new CountDownLatch(2);
+
+ executor1.submit(() -> {
+ try {
+ Method handleUpdates =
+ BookieRackAffinityMapping.class.getDeclaredMethod("handleUpdates", Notification.class);
+ handleUpdates.setAccessible(true);
+ Notification n =
+ new Notification(NotificationType.Modified, BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH);
+ long start = System.currentTimeMillis();
+ while (System.currentTimeMillis() - start < 2_000) {
+ handleUpdates.invoke(mapping, n);
+ }
+ count.countDown();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ executor2.submit(() -> {
+ Set<BookieId> writableBookies = new HashSet<>();
+ writableBookies.add(BOOKIE1.toBookieId());
+ long start = System.currentTimeMillis();
+ while (System.currentTimeMillis() - start < 2_000) {
+ repp.onClusterChanged(writableBookies, Collections.emptySet());
+ repp.onClusterChanged(Collections.emptySet(), Collections.emptySet());
+ }
+ count.countDown();
+ });
+
+ assertTrue(count.await(3, TimeUnit.SECONDS));
+ }
}