You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/08/17 14:19:20 UTC

(pulsar) branch branch-4.0 updated: [fix][test] Fix flaky AdminApiTest caused by leaked brokerShutdownTimeoutMs (#26249)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new bd3110777743 [fix][test] Fix flaky AdminApiTest caused by leaked brokerShutdownTimeoutMs (#26249)
bd3110777743 is described below

commit bd31107777435edf9be84ab26c4abe08a9fa60c7
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Mon Jul 27 11:30:41 2026 +0300

    [fix][test] Fix flaky AdminApiTest caused by leaked brokerShutdownTimeoutMs (#26249)
    
    (cherry picked from commit 6a109ac2e25c1b9d9d54d9121945ea00f57c79e2)
    
    Backport note: branch-4.0 still carries V1AdminApiTest, the copy of AdminApiTest that
    PIP-457 (#25304) removed from master, and it cannot inherit this fix. It has the same
    defect: testGetDynamicLocalConfiguration leaves the brokerShutdownTimeoutMs dynamic
    configuration at 10ms, which poisons the stopBroker() in
    testInvalidDynamicConfigContentInZK and turns the @AfterMethod (reset) into a class-wide
    skip. The same fix is therefore applied to V1AdminApiTest as well.
    
    In V1AdminApiTest the added wait is placed before the pre-existing in-memory
    setBrokerShutdownTimeoutMs(defaultValue) statement instead of at the end of the method:
    that statement restores only the in-memory value and never removes the dynamic
    configuration, so appending the wait after it would make the wait unsatisfiable.
---
 .../apache/pulsar/broker/admin/AdminApiTest.java   | 31 ++++++++++++++++++++
 .../pulsar/broker/admin/v1/V1AdminApiTest.java     | 33 ++++++++++++++++++++++
 2 files changed, 64 insertions(+)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
index 9a5c72ff2404..27f5be1137b9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
@@ -158,6 +158,8 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest {
 
     private static final Logger LOG = LoggerFactory.getLogger(AdminApiTest.class);
 
+    private static final String BROKER_SHUTDOWN_TIMEOUT_MS = "brokerShutdownTimeoutMs";
+
     private MockedPulsarService mockPulsarSetup;
 
     private PulsarService otherPulsar;
@@ -220,6 +222,14 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest {
 
     @AfterMethod(alwaysRun = true)
     public void resetClusters() throws Exception {
+        if (pulsar == null || !pulsar.isRunning()) {
+            // A test method left the broker stopped, for example because stopBroker() failed. Throwing from
+            // here is a TestNG configuration failure, which makes TestNG skip every remaining test method of
+            // this class. TestRetrySupport recreates the shared test context before the next test method.
+            log.warn("Broker isn't running, skipping the cleanup of the previous test method");
+            return;
+        }
+        restoreBrokerShutdownTimeout();
         pulsar.getConfiguration().setForceDeleteTenantAllowed(true);
         pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true);
         for (String tenant : admin.tenants().getTenants()) {
@@ -241,6 +251,25 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest {
         setupClusters();
     }
 
+    /**
+     * Several test methods of this class lower the {@code brokerShutdownTimeoutMs} dynamic configuration to 10ms.
+     * The broker applies dynamic configuration changes asynchronously, so when the value is left behind it can
+     * become effective while a later test method is shutting the broker down in stopBroker() or restartBroker().
+     * PulsarService#close then fails with "Timeout in close" and leaves the test holding a broker that is no
+     * longer listening, which fails this @AfterMethod and skips the remaining test methods of the class.
+     */
+    private void restoreBrokerShutdownTimeout() throws Exception {
+        String overriddenValue = admin.brokers().getAllDynamicConfigurations().get(BROKER_SHUTDOWN_TIMEOUT_MS);
+        if (overriddenValue == null) {
+            return;
+        }
+        long overriddenTimeoutMs = Long.parseLong(overriddenValue);
+        // removing the dynamic configuration makes the broker restore the value it was started with
+        admin.brokers().deleteDynamicConfiguration(BROKER_SHUTDOWN_TIMEOUT_MS);
+        Awaitility.await().untilAsserted(
+                () -> assertNotEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), overriddenTimeoutMs));
+    }
+
     private void setupClusters() throws PulsarAdminException {
         admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build());
         TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test"));
@@ -758,6 +787,8 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest {
         admin.brokers().updateDynamicConfiguration(configName, Long.toString(shutdownTime));
         // Now, znode is created: updateConfigurationAndRegisterListeners and check if configuration updated
         assertEquals(Long.parseLong(admin.brokers().getAllDynamicConfigurations().get(configName)), shutdownTime);
+        // wait until the broker has applied the value, so that the @AfterMethod always has to restore it
+        Awaitility.await().until(() -> pulsar.getConfiguration().getBrokerShutdownTimeoutMs() == shutdownTime);
     }
 
     @Test
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1AdminApiTest.java
index 2d1a0acab843..48e4dcbbb017 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1AdminApiTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1AdminApiTest.java
@@ -113,6 +113,7 @@ import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.compaction.Compactor;
 import org.apache.pulsar.compaction.PulsarCompactionServiceFactory;
 import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl;
+import org.awaitility.Awaitility;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
@@ -128,6 +129,8 @@ public class V1AdminApiTest extends MockedPulsarServiceBaseTest {
 
     private static final Logger LOG = LoggerFactory.getLogger(V1AdminApiTest.class);
 
+    private static final String BROKER_SHUTDOWN_TIMEOUT_MS = "brokerShutdownTimeoutMs";
+
     private MockedPulsarService mockPulsarSetup;
 
     private PulsarService otherPulsar;
@@ -189,6 +192,14 @@ public class V1AdminApiTest extends MockedPulsarServiceBaseTest {
 
     @AfterMethod(alwaysRun = true)
     public void reset() throws Exception {
+        if (pulsar == null || !pulsar.isRunning()) {
+            // A test method left the broker stopped, for example because stopBroker() failed. Throwing from
+            // here is a TestNG configuration failure, which makes TestNG skip every remaining test method of
+            // this class. TestRetrySupport recreates the shared test context before the next test method.
+            log.warn("Broker isn't running, skipping the cleanup of the previous test method");
+            return;
+        }
+        restoreBrokerShutdownTimeout();
         pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true);
         for (String tenant : admin.tenants().getTenants()) {
             for (String namespace : admin.namespaces().getNamespaces(tenant)) {
@@ -211,6 +222,25 @@ public class V1AdminApiTest extends MockedPulsarServiceBaseTest {
         admin.namespaces().createNamespace("prop-xyz/use/ns1");
     }
 
+    /**
+     * Several test methods of this class lower the {@code brokerShutdownTimeoutMs} dynamic configuration to 10ms.
+     * The broker applies dynamic configuration changes asynchronously, so when the value is left behind it can
+     * become effective while a later test method is shutting the broker down in stopBroker() or restartBroker().
+     * PulsarService#close then fails with "Timeout in close" and leaves the test holding a broker that is no
+     * longer listening, which fails this @AfterMethod and skips the remaining test methods of the class.
+     */
+    private void restoreBrokerShutdownTimeout() throws Exception {
+        String overriddenValue = admin.brokers().getAllDynamicConfigurations().get(BROKER_SHUTDOWN_TIMEOUT_MS);
+        if (overriddenValue == null) {
+            return;
+        }
+        long overriddenTimeoutMs = Long.parseLong(overriddenValue);
+        // removing the dynamic configuration makes the broker restore the value it was started with
+        admin.brokers().deleteDynamicConfiguration(BROKER_SHUTDOWN_TIMEOUT_MS);
+        Awaitility.await().untilAsserted(
+                () -> assertNotEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), overriddenTimeoutMs));
+    }
+
     @DataProvider(name = "numBundles")
     public static Object[][] numBundles() {
         return new Object[][] { { 1 }, { 4 } };
@@ -622,6 +652,9 @@ public class V1AdminApiTest extends MockedPulsarServiceBaseTest {
         admin.brokers().updateDynamicConfiguration(configName, Long.toString(shutdownTime));
         // Now, znode is created: updateConfigurationAndregisterListeners and check if configuration updated
         assertEquals(Long.parseLong(admin.brokers().getAllDynamicConfigurations().get(configName)), shutdownTime);
+        // wait until the broker has applied the value before restoring it below, so that the asynchronous
+        // update cannot set it back to shutdownTime once this test method has completed
+        Awaitility.await().until(() -> pulsar.getConfiguration().getBrokerShutdownTimeoutMs() == shutdownTime);
 
         pulsar.getConfiguration().setBrokerShutdownTimeoutMs(defaultValue);
     }