You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/10 19:31:39 UTC

(pulsar) branch master updated: [fix][broker] Fix assignment and ownership cleanup races in the extensible load manager (#26520)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 816eddb45976 [fix][broker] Fix assignment and ownership cleanup races in the extensible load manager (#26520)
816eddb45976 is described below

commit 816eddb459764f02a7bb5cfadefa17795bbf4bc2
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Thu Sep 10 22:31:32 2026 +0300

    [fix][broker] Fix assignment and ownership cleanup races in the extensible load manager (#26520)
---
 .../channel/ServiceUnitStateChannelImpl.java       | 100 ++++++++++++----
 .../extensions/ExtensibleLoadManagerCloseTest.java |  41 +++++--
 .../channel/ServiceUnitStateChannelTest.java       | 129 ++++++++++++++++++++-
 3 files changed, 232 insertions(+), 38 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index e38a74c4a938..da290e9d890a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -146,6 +146,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
     private long totalInactiveBrokerCleanupIgnoredCnt = 0;
     private long totalInactiveBrokerCleanupCancelledCnt = 0;
     private volatile ChannelState channelState;
+    private final Set<CompletableFuture<Void>> pendingAssignPublishes = new HashSet<>();
     private volatile long lastOwnEventHandledAt = 0;
     private long lastOwnedServiceUnitCountAt = 0;
     private int totalOwnedServiceUnitCnt = 0;
@@ -243,6 +244,16 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
         return cleanupJobs;
     }
 
+    @VisibleForTesting
+    ServiceUnitStateTableView getTableView() {
+        return tableview;
+    }
+
+    @VisibleForTesting
+    void setTableView(ServiceUnitStateTableView tableview) {
+        this.tableview = tableview;
+    }
+
     @Override
     public void scheduleOwnershipMonitor() {
         if (monitorTask == null) {
@@ -272,6 +283,20 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
     @Override
     public void cleanOwnerships() {
         disable();
+        List<CompletableFuture<Void>> pending;
+        synchronized (pendingAssignPublishes) {
+            pending = List.copyOf(pendingAssignPublishes);
+        }
+        try {
+            // A lookup may have selected a broker before disable(). Finish accepted assignment writes
+            // before scanning ownership, while rejecting any new assignments after disable().
+            FutureUtil.waitForAll(pending).get(config.getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            log.warn().exception(e).log("Interrupted while waiting for in-flight assignment writes");
+        } catch (ExecutionException | TimeoutException e) {
+            log.warn().exception(e).log("Failed to finish in-flight assignment writes before ownership cleanup");
+        }
         doCleanup(brokerId, true);
     }
 
@@ -658,25 +683,48 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
 
     @Override
     public CompletableFuture<String> publishAssignEventAsync(String serviceUnit, String brokerId) {
-        if (!validateChannelState(Started, true)) {
-            return CompletableFuture.failedFuture(
-                    new IllegalStateException("Invalid channel state:" + channelState.name()));
+        CompletableFuture<Void> published = new CompletableFuture<>();
+        synchronized (pendingAssignPublishes) {
+            if (channelState != Started) {
+                return CompletableFuture.failedFuture(
+                        new IllegalStateException("Invalid channel state:" + channelState.name()));
+            }
+            pendingAssignPublishes.add(published);
         }
+        published.whenComplete((__, ex) -> {
+            synchronized (pendingAssignPublishes) {
+                pendingAssignPublishes.remove(published);
+            }
+        });
         EventType eventType = Assign;
         eventCounters.get(eventType).getTotal().incrementAndGet();
-        CompletableFuture<String> getOwnerRequest = dedupeGetOwnerRequest(serviceUnit);
+        CompletableFuture<String> getOwnerRequest;
+        try {
+            getOwnerRequest = dedupeGetOwnerRequest(serviceUnit);
+        } catch (Throwable e) {
+            published.complete(null);
+            eventCounters.get(eventType).getFailure().incrementAndGet();
+            return CompletableFuture.failedFuture(e);
+        }
 
-        pubAsync(serviceUnit,
-                new ServiceUnitStateData(Assigning, brokerId, getNextVersionId(serviceUnit)))
-                .whenComplete((__, ex) -> {
-                    if (ex != null) {
-                        getOwnerRequests.remove(serviceUnit, getOwnerRequest);
-                        if (!getOwnerRequest.isCompletedExceptionally()) {
-                            getOwnerRequest.completeExceptionally(ex);
-                        }
-                        eventCounters.get(eventType).getFailure().incrementAndGet();
-                    }
-                });
+        CompletableFuture<Void> publishFuture;
+        try {
+            publishFuture = pubAsync(serviceUnit,
+                    new ServiceUnitStateData(Assigning, brokerId, getNextVersionId(serviceUnit)));
+        } catch (Throwable e) {
+            publishFuture = CompletableFuture.failedFuture(e);
+        }
+        publishFuture.whenComplete((__, ex) -> {
+            // Complete outside the lock: completion may run the waiting cleanup's callbacks.
+            published.complete(null);
+            if (ex != null) {
+                getOwnerRequests.remove(serviceUnit, getOwnerRequest);
+                if (!getOwnerRequest.isCompletedExceptionally()) {
+                    getOwnerRequest.completeExceptionally(ex);
+                }
+                eventCounters.get(eventType).getFailure().incrementAndGet();
+            }
+        });
 
         return getOwnerRequest;
     }
@@ -1536,31 +1584,35 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
                 });
     }
 
-    private void waitForCleanups(String broker, boolean excludeSystemTopics, int maxWaitTimeInMillis) {
+    private void waitForCleanups(String broker, boolean gracefully, int maxWaitTimeInMillis) {
         long started = System.currentTimeMillis();
         while (System.currentTimeMillis() - started < maxWaitTimeInMillis) {
             boolean cleaned = true;
+            List<CompletableFuture<Void>> overrideFutures = new ArrayList<>();
             for (var etr : tableview.entrySet()) {
                 var serviceUnit = etr.getKey();
                 var data = etr.getValue();
 
-                if (excludeSystemTopics && serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
+                if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
                     continue;
                 }
 
                 if (data.state() == Owned && broker.equals(data.dstBroker())) {
                     cleaned = false;
-                    log.info().attr("broker", broker).attr("bundle", serviceUnit).attr("data", data)
-                            .log("Bundle is still owned by this broker");
-                    break;
+                    // An in-flight assignment can win against the initial cleanup override at the same
+                    // version. A successful publish does not mean the override was accepted. Retry using
+                    // the current state, keeping version conflict checks so a new owner's state is safe.
+                    overrideFutures.add(overrideOwnership(serviceUnit, data, broker, gracefully));
+                    tryWaitForOverrides(overrideFutures, false);
                 }
             }
             if (cleaned) {
                 break;
             } else {
+                tryWaitForOverrides(overrideFutures, true);
                 try {
                     tableview.flush(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS / 2);
-                    Thread.sleep(OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS / 2);
+                    Thread.sleep(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS / 2);
                 } catch (InterruptedException e) {
                     log.warn().attr("broker", brokerId)
                             .log("Interrupted while delaying the next service unit clean-up. Cleaning broker");
@@ -1704,7 +1756,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
             // can cause the cluster to be temporarily unstable.
             // Hence, we clean the non-system bundles first and gracefully wait for them.
             // After that, we clean the system bundles, if any.
-            waitForCleanups(broker, true, OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS);
+            waitForCleanups(broker, gracefully, OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS);
             this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt;
             this.totalInactiveBrokerCleanupCnt++;
         }
@@ -2089,7 +2141,9 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
 
     @VisibleForTesting
     protected void disable() {
-        channelState = Disabled;
+        synchronized (pendingAssignPublishes) {
+            channelState = Disabled;
+        }
     }
 
     @VisibleForTesting
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java
index f0734fbe8bbe..1d7390c42dde 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import lombok.CustomLog;
 import org.apache.pulsar.broker.PulsarService;
@@ -34,7 +35,9 @@ import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 @CustomLog
@@ -51,12 +54,11 @@ public class ExtensibleLoadManagerCloseTest {
         bk.start();
     }
 
-    private void setupBrokers(int numBrokers) throws Exception {
-        brokers.clear();
+    private void setupBrokers(int numBrokers, boolean topicPoliciesEnabled) throws Exception {
         for (int i = 0; i < numBrokers; i++) {
-            final var broker = new PulsarService(brokerConfig());
-            broker.start();
+            final var broker = new PulsarService(brokerConfig(topicPoliciesEnabled));
             brokers.add(broker);
+            broker.start();
         }
         final var admin = brokers.get(0).getAdminClient();
         if (!admin.clusters().getClusters().contains(clusterName)) {
@@ -68,12 +70,21 @@ public class ExtensibleLoadManagerCloseTest {
     }
 
 
+    @AfterMethod(alwaysRun = true, timeOut = 30000)
+    public void cleanupBrokers() throws Exception {
+        try {
+            FutureUtil.waitForAll(brokers.stream().map(PulsarService::closeAsync).toList()).get();
+        } finally {
+            brokers.clear();
+        }
+    }
+
     @AfterClass(alwaysRun = true, timeOut = 30000)
     public void cleanup() throws Exception {
         bk.stop();
     }
 
-    private ServiceConfiguration brokerConfig() {
+    private ServiceConfiguration brokerConfig(boolean topicPoliciesEnabled) {
         final var config = new ServiceConfiguration();
         config.setClusterName(clusterName);
         config.setAdvertisedAddress("localhost");
@@ -85,6 +96,7 @@ public class ExtensibleLoadManagerCloseTest {
         config.setManagedLedgerDefaultEnsembleSize(1);
         config.setDefaultNumberOfNamespaceBundles(16);
         config.setLoadBalancerAutoBundleSplitEnabled(false);
+        config.setTopicLevelPoliciesEnabled(topicPoliciesEnabled);
         config.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         config.setLoadBalancerDebugModeEnabled(true);
         config.setBrokerShutdownTimeoutMs(100);
@@ -96,9 +108,14 @@ public class ExtensibleLoadManagerCloseTest {
     }
 
 
-    @Test(invocationCount = 10)
-    public void testCloseAfterLoadingBundles() throws Exception {
-        setupBrokers(3);
+    @DataProvider
+    public Object[][] topicPoliciesEnabled() {
+        return new Object[][]{{false}, {true}};
+    }
+
+    @Test(invocationCount = 10, dataProvider = "topicPoliciesEnabled")
+    public void testCloseAfterLoadingBundles(boolean topicPoliciesEnabled) throws Exception {
+        setupBrokers(3, topicPoliciesEnabled);
         final var topic = "test-" + System.currentTimeMillis();
         final var admin = brokers.get(0).getAdminClient();
         admin.topics().createPartitionedTopic(topic, 20);
@@ -120,10 +137,10 @@ public class ExtensibleLoadManagerCloseTest {
         }
     }
 
-    @Test
-    public void testLookup() throws Exception {
-        setupBrokers(1);
-        final var topic = "test-lookup";
+    @Test(dataProvider = "topicPoliciesEnabled")
+    public void testLookup(boolean topicPoliciesEnabled) throws Exception {
+        setupBrokers(1, topicPoliciesEnabled);
+        final var topic = "test-lookup-" + UUID.randomUUID();
         final var numPartitions = 16;
         final var admin = brokers.get(0).getAdminClient();
         admin.topics().createPartitionedTopic(topic, numPartitions);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index 373068c38d74..e96d1570613e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -42,6 +42,7 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doCallRealMethod;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.reset;
@@ -2226,6 +2227,129 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest {
         }
     }
 
+    @Test
+    public void testCleanupDrainsAssignmentsAndRejectsNewOnes() throws Exception {
+        var channel = (ServiceUnitStateChannelImpl) channel1;
+        var tableView = channel.getTableView();
+        var delayedTableView = spy(tableView);
+        String serviceUnit = namespaceName + "/0x00000000_0xffffffff";
+        String lateServiceUnit = namespaceName2 + "/0x00000000_0xffffffff";
+        var pendingAssignment = new CompletableFuture<ServiceUnitStateData>();
+        var published = new CompletableFuture<Void>();
+        var disabled = new CompletableFuture<Void>();
+        doReturn(CompletableFuture.completedFuture(Optional.empty()))
+                .when(loadManager).selectAsync(any(), any(), any());
+        doAnswer(invocation -> {
+            ServiceUnitStateData data = invocation.getArgument(1);
+            if (data.state() == Assigning && !data.force()) {
+                pendingAssignment.complete(data);
+                return published;
+            }
+            return tableView.put(serviceUnit, data);
+        }).when(delayedTableView).put(eq(serviceUnit), any(ServiceUnitStateData.class));
+        doAnswer(invocation -> {
+            invocation.callRealMethod();
+            disabled.complete(null);
+            return null;
+        }).when(channel).disable();
+        channel.setTableView(delayedTableView);
+        @Cleanup("shutdownNow")
+        ExecutorService executor = Executors.newSingleThreadExecutor();
+        CompletableFuture<Void> cleanup = null;
+        try {
+            channel.publishAssignEventAsync(serviceUnit, brokerId1);
+            var assignment = pendingAssignment.get(10, TimeUnit.SECONDS);
+            cleanup = CompletableFuture.runAsync(channel::cleanOwnerships, executor);
+            disabled.get(10, TimeUnit.SECONDS);
+            assertFalse(cleanup.isDone(), "Cleanup must wait for the accepted assignment write");
+            var rejected = expectThrows(ExecutionException.class,
+                    () -> channel.publishAssignEventAsync(lateServiceUnit, brokerId1).get(10, TimeUnit.SECONDS));
+            assertTrue(rejected.getCause() instanceof IllegalStateException);
+            assertNull(tableView.get(lateServiceUnit));
+            tableView.put(serviceUnit, assignment).get(10, TimeUnit.SECONDS);
+            published.complete(null);
+            cleanup.get(10, TimeUnit.SECONDS);
+            Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+                var remaining = tableView.get(serviceUnit);
+                assertTrue(remaining == null || remaining.state() == Free,
+                        "An accepted assignment must be included in cleanup: " + remaining);
+            });
+        } finally {
+            published.complete(null);
+            if (cleanup != null) {
+                cleanup.get(10, TimeUnit.SECONDS);
+            }
+            doCallRealMethod().when(channel).disable();
+            channel.setTableView(tableView);
+            channel.enable();
+            tableView.delete(serviceUnit).get(10, TimeUnit.SECONDS);
+            tableView.delete(lateServiceUnit).get(10, TimeUnit.SECONDS);
+        }
+    }
+
+    @DataProvider
+    public Object[][] cleanupDestination() {
+        return new Object[][]{{false}, {true}};
+    }
+
+    @Test(dataProvider = "cleanupDestination")
+    public void testCleanupRetriesConcurrentAssignment(boolean hasDestinationBroker) throws Exception {
+        var channel = (ServiceUnitStateChannelImpl) channel1;
+        var tableView = channel.getTableView();
+        var delayedTableView = spy(tableView);
+        String serviceUnit = namespaceName + "/0x00000000_0xffffffff";
+        var pendingOwned = new CompletableFuture<ServiceUnitStateData>();
+        var ownedPublished = new CompletableFuture<Void>();
+        var conflictingOverride = new CompletableFuture<ServiceUnitStateData>();
+        doReturn(CompletableFuture.completedFuture(
+                hasDestinationBroker ? Optional.of(brokerId2) : Optional.empty()))
+                .when(loadManager).selectAsync(any(), any(), any());
+        doAnswer(invocation -> {
+            ServiceUnitStateData data = invocation.getArgument(1);
+            if (data.state() == Owned && !data.force()) {
+                pendingOwned.complete(data);
+                return ownedPublished;
+            }
+            var owned = pendingOwned.getNow(null);
+            if (data.force() && owned != null && data.versionId() == owned.versionId()) {
+                // Publish the delayed Owned update before the cleanup's same-version override. Both writes
+                // succeed, but the real conflict resolver discards the stale cleanup update.
+                return tableView.put(serviceUnit, owned).thenCompose(__ -> {
+                    ownedPublished.complete(null);
+                    conflictingOverride.complete(data);
+                    return tableView.put(serviceUnit, data);
+                });
+            }
+            return tableView.put(serviceUnit, data);
+        }).when(delayedTableView).put(eq(serviceUnit), any(ServiceUnitStateData.class));
+        channel.setTableView(delayedTableView);
+        try {
+            var assignment = channel.publishAssignEventAsync(serviceUnit, brokerId1);
+            var owned = pendingOwned.get(10, TimeUnit.SECONDS);
+            assertEquals(2L, owned.versionId());
+            channel.cleanOwnerships();
+            assertEquals(owned.versionId(), conflictingOverride.get(10, TimeUnit.SECONDS).versionId());
+            Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+                var remaining = tableView.get(serviceUnit);
+                if (hasDestinationBroker) {
+                    assertNotNull(remaining);
+                    assertEquals(Owned, remaining.state());
+                    assertEquals(brokerId2, remaining.dstBroker());
+                } else {
+                    assertTrue(remaining == null || remaining.state() == Free,
+                            "Cleanup must not leave the concurrently assigned bundle owned by the stopped broker: "
+                                    + remaining);
+                }
+            });
+            assertTrue(assignment.isDone());
+        } finally {
+            ownedPublished.complete(null);
+            channel.setTableView(tableView);
+            channel.enable();
+            tableView.delete(serviceUnit).get(10, TimeUnit.SECONDS);
+        }
+    }
+
     private static ConcurrentHashMap<String, CompletableFuture<String>> getOwnerRequests(
             ServiceUnitStateChannel channel) throws IllegalAccessException {
         return (ConcurrentHashMap<String, CompletableFuture<String>>)
@@ -2301,14 +2425,13 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest {
 
     private static ServiceUnitStateTableView getTableView(ServiceUnitStateChannel channel)
             throws IllegalAccessException {
-        return (ServiceUnitStateTableView)
-                FieldUtils.readField(channel, "tableview", true);
+        return ((ServiceUnitStateChannelImpl) channel).getTableView();
     }
 
     private static void setTableView(ServiceUnitStateChannel channel,
                                      ServiceUnitStateTableView tableView)
             throws IllegalAccessException {
-        FieldUtils.writeField(channel, "tableview", tableView, true);
+        ((ServiceUnitStateChannelImpl) channel).setTableView(tableView);
     }
 
     private static void waitUntilState(ServiceUnitStateChannel channel, String key)