You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/10 19:31:39 UTC
(pulsar) branch master updated: [fix][broker] Fix assignment and ownership cleanup races in the extensible load manager (#26520)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 816eddb45976 [fix][broker] Fix assignment and ownership cleanup races in the extensible load manager (#26520)
816eddb45976 is described below
commit 816eddb459764f02a7bb5cfadefa17795bbf4bc2
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Thu Sep 10 22:31:32 2026 +0300
[fix][broker] Fix assignment and ownership cleanup races in the extensible load manager (#26520)
---
.../channel/ServiceUnitStateChannelImpl.java | 100 ++++++++++++----
.../extensions/ExtensibleLoadManagerCloseTest.java | 41 +++++--
.../channel/ServiceUnitStateChannelTest.java | 129 ++++++++++++++++++++-
3 files changed, 232 insertions(+), 38 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index e38a74c4a938..da290e9d890a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -146,6 +146,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
private long totalInactiveBrokerCleanupIgnoredCnt = 0;
private long totalInactiveBrokerCleanupCancelledCnt = 0;
private volatile ChannelState channelState;
+ private final Set<CompletableFuture<Void>> pendingAssignPublishes = new HashSet<>();
private volatile long lastOwnEventHandledAt = 0;
private long lastOwnedServiceUnitCountAt = 0;
private int totalOwnedServiceUnitCnt = 0;
@@ -243,6 +244,16 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
return cleanupJobs;
}
+ @VisibleForTesting
+ ServiceUnitStateTableView getTableView() {
+ return tableview;
+ }
+
+ @VisibleForTesting
+ void setTableView(ServiceUnitStateTableView tableview) {
+ this.tableview = tableview;
+ }
+
@Override
public void scheduleOwnershipMonitor() {
if (monitorTask == null) {
@@ -272,6 +283,20 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
@Override
public void cleanOwnerships() {
disable();
+ List<CompletableFuture<Void>> pending;
+ synchronized (pendingAssignPublishes) {
+ pending = List.copyOf(pendingAssignPublishes);
+ }
+ try {
+ // A lookup may have selected a broker before disable(). Finish accepted assignment writes
+ // before scanning ownership, while rejecting any new assignments after disable().
+ FutureUtil.waitForAll(pending).get(config.getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ log.warn().exception(e).log("Interrupted while waiting for in-flight assignment writes");
+ } catch (ExecutionException | TimeoutException e) {
+ log.warn().exception(e).log("Failed to finish in-flight assignment writes before ownership cleanup");
+ }
doCleanup(brokerId, true);
}
@@ -658,25 +683,48 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
@Override
public CompletableFuture<String> publishAssignEventAsync(String serviceUnit, String brokerId) {
- if (!validateChannelState(Started, true)) {
- return CompletableFuture.failedFuture(
- new IllegalStateException("Invalid channel state:" + channelState.name()));
+ CompletableFuture<Void> published = new CompletableFuture<>();
+ synchronized (pendingAssignPublishes) {
+ if (channelState != Started) {
+ return CompletableFuture.failedFuture(
+ new IllegalStateException("Invalid channel state:" + channelState.name()));
+ }
+ pendingAssignPublishes.add(published);
}
+ published.whenComplete((__, ex) -> {
+ synchronized (pendingAssignPublishes) {
+ pendingAssignPublishes.remove(published);
+ }
+ });
EventType eventType = Assign;
eventCounters.get(eventType).getTotal().incrementAndGet();
- CompletableFuture<String> getOwnerRequest = dedupeGetOwnerRequest(serviceUnit);
+ CompletableFuture<String> getOwnerRequest;
+ try {
+ getOwnerRequest = dedupeGetOwnerRequest(serviceUnit);
+ } catch (Throwable e) {
+ published.complete(null);
+ eventCounters.get(eventType).getFailure().incrementAndGet();
+ return CompletableFuture.failedFuture(e);
+ }
- pubAsync(serviceUnit,
- new ServiceUnitStateData(Assigning, brokerId, getNextVersionId(serviceUnit)))
- .whenComplete((__, ex) -> {
- if (ex != null) {
- getOwnerRequests.remove(serviceUnit, getOwnerRequest);
- if (!getOwnerRequest.isCompletedExceptionally()) {
- getOwnerRequest.completeExceptionally(ex);
- }
- eventCounters.get(eventType).getFailure().incrementAndGet();
- }
- });
+ CompletableFuture<Void> publishFuture;
+ try {
+ publishFuture = pubAsync(serviceUnit,
+ new ServiceUnitStateData(Assigning, brokerId, getNextVersionId(serviceUnit)));
+ } catch (Throwable e) {
+ publishFuture = CompletableFuture.failedFuture(e);
+ }
+ publishFuture.whenComplete((__, ex) -> {
+ // Complete outside the lock: completion may run the waiting cleanup's callbacks.
+ published.complete(null);
+ if (ex != null) {
+ getOwnerRequests.remove(serviceUnit, getOwnerRequest);
+ if (!getOwnerRequest.isCompletedExceptionally()) {
+ getOwnerRequest.completeExceptionally(ex);
+ }
+ eventCounters.get(eventType).getFailure().incrementAndGet();
+ }
+ });
return getOwnerRequest;
}
@@ -1536,31 +1584,35 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
});
}
- private void waitForCleanups(String broker, boolean excludeSystemTopics, int maxWaitTimeInMillis) {
+ private void waitForCleanups(String broker, boolean gracefully, int maxWaitTimeInMillis) {
long started = System.currentTimeMillis();
while (System.currentTimeMillis() - started < maxWaitTimeInMillis) {
boolean cleaned = true;
+ List<CompletableFuture<Void>> overrideFutures = new ArrayList<>();
for (var etr : tableview.entrySet()) {
var serviceUnit = etr.getKey();
var data = etr.getValue();
- if (excludeSystemTopics && serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
+ if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
continue;
}
if (data.state() == Owned && broker.equals(data.dstBroker())) {
cleaned = false;
- log.info().attr("broker", broker).attr("bundle", serviceUnit).attr("data", data)
- .log("Bundle is still owned by this broker");
- break;
+ // An in-flight assignment can win against the initial cleanup override at the same
+ // version. A successful publish does not mean the override was accepted. Retry using
+ // the current state, keeping version conflict checks so a new owner's state is safe.
+ overrideFutures.add(overrideOwnership(serviceUnit, data, broker, gracefully));
+ tryWaitForOverrides(overrideFutures, false);
}
}
if (cleaned) {
break;
} else {
+ tryWaitForOverrides(overrideFutures, true);
try {
tableview.flush(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS / 2);
- Thread.sleep(OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS / 2);
+ Thread.sleep(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS / 2);
} catch (InterruptedException e) {
log.warn().attr("broker", brokerId)
.log("Interrupted while delaying the next service unit clean-up. Cleaning broker");
@@ -1704,7 +1756,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
// can cause the cluster to be temporarily unstable.
// Hence, we clean the non-system bundles first and gracefully wait for them.
// After that, we clean the system bundles, if any.
- waitForCleanups(broker, true, OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS);
+ waitForCleanups(broker, gracefully, OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS);
this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt;
this.totalInactiveBrokerCleanupCnt++;
}
@@ -2089,7 +2141,9 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
@VisibleForTesting
protected void disable() {
- channelState = Disabled;
+ synchronized (pendingAssignPublishes) {
+ channelState = Disabled;
+ }
}
@VisibleForTesting
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java
index f0734fbe8bbe..1d7390c42dde 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import lombok.CustomLog;
import org.apache.pulsar.broker.PulsarService;
@@ -34,7 +35,9 @@ import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@CustomLog
@@ -51,12 +54,11 @@ public class ExtensibleLoadManagerCloseTest {
bk.start();
}
- private void setupBrokers(int numBrokers) throws Exception {
- brokers.clear();
+ private void setupBrokers(int numBrokers, boolean topicPoliciesEnabled) throws Exception {
for (int i = 0; i < numBrokers; i++) {
- final var broker = new PulsarService(brokerConfig());
- broker.start();
+ final var broker = new PulsarService(brokerConfig(topicPoliciesEnabled));
brokers.add(broker);
+ broker.start();
}
final var admin = brokers.get(0).getAdminClient();
if (!admin.clusters().getClusters().contains(clusterName)) {
@@ -68,12 +70,21 @@ public class ExtensibleLoadManagerCloseTest {
}
+ @AfterMethod(alwaysRun = true, timeOut = 30000)
+ public void cleanupBrokers() throws Exception {
+ try {
+ FutureUtil.waitForAll(brokers.stream().map(PulsarService::closeAsync).toList()).get();
+ } finally {
+ brokers.clear();
+ }
+ }
+
@AfterClass(alwaysRun = true, timeOut = 30000)
public void cleanup() throws Exception {
bk.stop();
}
- private ServiceConfiguration brokerConfig() {
+ private ServiceConfiguration brokerConfig(boolean topicPoliciesEnabled) {
final var config = new ServiceConfiguration();
config.setClusterName(clusterName);
config.setAdvertisedAddress("localhost");
@@ -85,6 +96,7 @@ public class ExtensibleLoadManagerCloseTest {
config.setManagedLedgerDefaultEnsembleSize(1);
config.setDefaultNumberOfNamespaceBundles(16);
config.setLoadBalancerAutoBundleSplitEnabled(false);
+ config.setTopicLevelPoliciesEnabled(topicPoliciesEnabled);
config.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
config.setLoadBalancerDebugModeEnabled(true);
config.setBrokerShutdownTimeoutMs(100);
@@ -96,9 +108,14 @@ public class ExtensibleLoadManagerCloseTest {
}
- @Test(invocationCount = 10)
- public void testCloseAfterLoadingBundles() throws Exception {
- setupBrokers(3);
+ @DataProvider
+ public Object[][] topicPoliciesEnabled() {
+ return new Object[][]{{false}, {true}};
+ }
+
+ @Test(invocationCount = 10, dataProvider = "topicPoliciesEnabled")
+ public void testCloseAfterLoadingBundles(boolean topicPoliciesEnabled) throws Exception {
+ setupBrokers(3, topicPoliciesEnabled);
final var topic = "test-" + System.currentTimeMillis();
final var admin = brokers.get(0).getAdminClient();
admin.topics().createPartitionedTopic(topic, 20);
@@ -120,10 +137,10 @@ public class ExtensibleLoadManagerCloseTest {
}
}
- @Test
- public void testLookup() throws Exception {
- setupBrokers(1);
- final var topic = "test-lookup";
+ @Test(dataProvider = "topicPoliciesEnabled")
+ public void testLookup(boolean topicPoliciesEnabled) throws Exception {
+ setupBrokers(1, topicPoliciesEnabled);
+ final var topic = "test-lookup-" + UUID.randomUUID();
final var numPartitions = 16;
final var admin = brokers.get(0).getAdminClient();
admin.topics().createPartitionedTopic(topic, numPartitions);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index 373068c38d74..e96d1570613e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -42,6 +42,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
@@ -2226,6 +2227,129 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest {
}
}
+ @Test
+ public void testCleanupDrainsAssignmentsAndRejectsNewOnes() throws Exception {
+ var channel = (ServiceUnitStateChannelImpl) channel1;
+ var tableView = channel.getTableView();
+ var delayedTableView = spy(tableView);
+ String serviceUnit = namespaceName + "/0x00000000_0xffffffff";
+ String lateServiceUnit = namespaceName2 + "/0x00000000_0xffffffff";
+ var pendingAssignment = new CompletableFuture<ServiceUnitStateData>();
+ var published = new CompletableFuture<Void>();
+ var disabled = new CompletableFuture<Void>();
+ doReturn(CompletableFuture.completedFuture(Optional.empty()))
+ .when(loadManager).selectAsync(any(), any(), any());
+ doAnswer(invocation -> {
+ ServiceUnitStateData data = invocation.getArgument(1);
+ if (data.state() == Assigning && !data.force()) {
+ pendingAssignment.complete(data);
+ return published;
+ }
+ return tableView.put(serviceUnit, data);
+ }).when(delayedTableView).put(eq(serviceUnit), any(ServiceUnitStateData.class));
+ doAnswer(invocation -> {
+ invocation.callRealMethod();
+ disabled.complete(null);
+ return null;
+ }).when(channel).disable();
+ channel.setTableView(delayedTableView);
+ @Cleanup("shutdownNow")
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ CompletableFuture<Void> cleanup = null;
+ try {
+ channel.publishAssignEventAsync(serviceUnit, brokerId1);
+ var assignment = pendingAssignment.get(10, TimeUnit.SECONDS);
+ cleanup = CompletableFuture.runAsync(channel::cleanOwnerships, executor);
+ disabled.get(10, TimeUnit.SECONDS);
+ assertFalse(cleanup.isDone(), "Cleanup must wait for the accepted assignment write");
+ var rejected = expectThrows(ExecutionException.class,
+ () -> channel.publishAssignEventAsync(lateServiceUnit, brokerId1).get(10, TimeUnit.SECONDS));
+ assertTrue(rejected.getCause() instanceof IllegalStateException);
+ assertNull(tableView.get(lateServiceUnit));
+ tableView.put(serviceUnit, assignment).get(10, TimeUnit.SECONDS);
+ published.complete(null);
+ cleanup.get(10, TimeUnit.SECONDS);
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+ var remaining = tableView.get(serviceUnit);
+ assertTrue(remaining == null || remaining.state() == Free,
+ "An accepted assignment must be included in cleanup: " + remaining);
+ });
+ } finally {
+ published.complete(null);
+ if (cleanup != null) {
+ cleanup.get(10, TimeUnit.SECONDS);
+ }
+ doCallRealMethod().when(channel).disable();
+ channel.setTableView(tableView);
+ channel.enable();
+ tableView.delete(serviceUnit).get(10, TimeUnit.SECONDS);
+ tableView.delete(lateServiceUnit).get(10, TimeUnit.SECONDS);
+ }
+ }
+
+ @DataProvider
+ public Object[][] cleanupDestination() {
+ return new Object[][]{{false}, {true}};
+ }
+
+ @Test(dataProvider = "cleanupDestination")
+ public void testCleanupRetriesConcurrentAssignment(boolean hasDestinationBroker) throws Exception {
+ var channel = (ServiceUnitStateChannelImpl) channel1;
+ var tableView = channel.getTableView();
+ var delayedTableView = spy(tableView);
+ String serviceUnit = namespaceName + "/0x00000000_0xffffffff";
+ var pendingOwned = new CompletableFuture<ServiceUnitStateData>();
+ var ownedPublished = new CompletableFuture<Void>();
+ var conflictingOverride = new CompletableFuture<ServiceUnitStateData>();
+ doReturn(CompletableFuture.completedFuture(
+ hasDestinationBroker ? Optional.of(brokerId2) : Optional.empty()))
+ .when(loadManager).selectAsync(any(), any(), any());
+ doAnswer(invocation -> {
+ ServiceUnitStateData data = invocation.getArgument(1);
+ if (data.state() == Owned && !data.force()) {
+ pendingOwned.complete(data);
+ return ownedPublished;
+ }
+ var owned = pendingOwned.getNow(null);
+ if (data.force() && owned != null && data.versionId() == owned.versionId()) {
+ // Publish the delayed Owned update before the cleanup's same-version override. Both writes
+ // succeed, but the real conflict resolver discards the stale cleanup update.
+ return tableView.put(serviceUnit, owned).thenCompose(__ -> {
+ ownedPublished.complete(null);
+ conflictingOverride.complete(data);
+ return tableView.put(serviceUnit, data);
+ });
+ }
+ return tableView.put(serviceUnit, data);
+ }).when(delayedTableView).put(eq(serviceUnit), any(ServiceUnitStateData.class));
+ channel.setTableView(delayedTableView);
+ try {
+ var assignment = channel.publishAssignEventAsync(serviceUnit, brokerId1);
+ var owned = pendingOwned.get(10, TimeUnit.SECONDS);
+ assertEquals(2L, owned.versionId());
+ channel.cleanOwnerships();
+ assertEquals(owned.versionId(), conflictingOverride.get(10, TimeUnit.SECONDS).versionId());
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+ var remaining = tableView.get(serviceUnit);
+ if (hasDestinationBroker) {
+ assertNotNull(remaining);
+ assertEquals(Owned, remaining.state());
+ assertEquals(brokerId2, remaining.dstBroker());
+ } else {
+ assertTrue(remaining == null || remaining.state() == Free,
+ "Cleanup must not leave the concurrently assigned bundle owned by the stopped broker: "
+ + remaining);
+ }
+ });
+ assertTrue(assignment.isDone());
+ } finally {
+ ownedPublished.complete(null);
+ channel.setTableView(tableView);
+ channel.enable();
+ tableView.delete(serviceUnit).get(10, TimeUnit.SECONDS);
+ }
+ }
+
private static ConcurrentHashMap<String, CompletableFuture<String>> getOwnerRequests(
ServiceUnitStateChannel channel) throws IllegalAccessException {
return (ConcurrentHashMap<String, CompletableFuture<String>>)
@@ -2301,14 +2425,13 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest {
private static ServiceUnitStateTableView getTableView(ServiceUnitStateChannel channel)
throws IllegalAccessException {
- return (ServiceUnitStateTableView)
- FieldUtils.readField(channel, "tableview", true);
+ return ((ServiceUnitStateChannelImpl) channel).getTableView();
}
private static void setTableView(ServiceUnitStateChannel channel,
ServiceUnitStateTableView tableView)
throws IllegalAccessException {
- FieldUtils.writeField(channel, "tableview", tableView, true);
+ ((ServiceUnitStateChannelImpl) channel).setTableView(tableView);
}
private static void waitUntilState(ServiceUnitStateChannel channel, String key)