You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2025/04/09 12:40:05 UTC

(pulsar) branch branch-3.0 updated (1035accffd1 -> 59a2218dea3)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git


    from 1035accffd1 Bump version to next snapshot version
     new ff6c5785f79 [fix][test] Fix flaky BrokerServiceChaosTest (#24162)
     new 59a2218dea3 [fix][test] Fix flaky BrokerServiceChaosTest.testFetchPartitionedTopicMetadataWithCacheRefresh (#24161)

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../broker/service/BkEnsemblesChaosTest.java       |  4 +-
 .../broker/service/BrokerServiceChaosTest.java     | 14 +++---
 .../CanReconnectZKClientPulsarServiceBaseTest.java | 56 +++++++++++++++-------
 3 files changed, 47 insertions(+), 27 deletions(-)


(pulsar) 02/02: [fix][test] Fix flaky BrokerServiceChaosTest.testFetchPartitionedTopicMetadataWithCacheRefresh (#24161)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 59a2218dea34135d2afa3526bd4e9aba37193f51
Author: Zixuan Liu <no...@gmail.com>
AuthorDate: Wed Apr 9 12:10:38 2025 +0800

    [fix][test] Fix flaky BrokerServiceChaosTest.testFetchPartitionedTopicMetadataWithCacheRefresh (#24161)
    
    Signed-off-by: Zixuan Liu <no...@gmail.com>
    Co-authored-by: Lari Hotari <lh...@users.noreply.github.com>
    (cherry picked from commit 9bcad7cc010348df252b065c59fae42915aca80b)
---
 .../broker/service/CanReconnectZKClientPulsarServiceBaseTest.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
index 683fa702b4e..5a8f946fe2c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
@@ -34,12 +34,14 @@ import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.apache.pulsar.common.policies.data.TopicType;
+import org.apache.pulsar.metadata.api.extended.SessionEvent;
 import org.apache.pulsar.metadata.impl.ZKMetadataStore;
 import org.apache.pulsar.tests.TestRetrySupport;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.apache.pulsar.zookeeper.ZookeeperServerTest;
 import org.apache.zookeeper.ClientCnxn;
 import org.apache.zookeeper.ZooKeeper;
+import org.awaitility.Awaitility;
 import org.awaitility.reflect.WhiteboxImpl;
 
 @Slf4j
@@ -58,6 +60,7 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetr
     protected PulsarAdmin admin;
     protected PulsarClient client;
     protected ZooKeeper localZkOfBroker;
+    protected volatile SessionEvent sessionEvent;
     protected Object localMetaDataStoreClientCnx;
     protected final AtomicBoolean connectionTerminationThreadKeepRunning = new AtomicBoolean();
     private volatile Thread connectionTerminationThread;
@@ -80,6 +83,10 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetr
         broker = pulsar.getBrokerService();
         ZKMetadataStore zkMetadataStore = (ZKMetadataStore) pulsar.getLocalMetadataStore();
         localZkOfBroker = zkMetadataStore.getZkClient();
+        zkMetadataStore.registerSessionListener(n -> {
+            log.info("Received session event: {}", n);
+            sessionEvent = n;
+        });
         ClientCnxn cnxn = WhiteboxImpl.getInternalState(localZkOfBroker, "cnxn");
         Object sendThread = WhiteboxImpl.getInternalState(cnxn, "sendThread");
         localMetaDataStoreClientCnx = WhiteboxImpl.getInternalState(sendThread, "clientCnxnSocket");
@@ -150,6 +157,7 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetr
             connectionTerminationThread.join();
             connectionTerminationThread = null;
         }
+        Awaitility.await().until(() -> SessionEvent.Reconnected.equals(sessionEvent));
     }
 
     protected void createDefaultTenantsAndClustersAndNamespace() throws Exception {


(pulsar) 01/02: [fix][test] Fix flaky BrokerServiceChaosTest (#24162)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit ff6c5785f7907ac76d340ff5d1a5d33b9892f3fd
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Apr 8 20:38:35 2025 +0300

    [fix][test] Fix flaky BrokerServiceChaosTest (#24162)
    
    (cherry picked from commit d59664ca9ca7300a8a7bc3faa1869f623eaac8ae)
---
 .../broker/service/BkEnsemblesChaosTest.java       |  4 +-
 .../broker/service/BrokerServiceChaosTest.java     | 14 +++----
 .../CanReconnectZKClientPulsarServiceBaseTest.java | 48 ++++++++++++++--------
 3 files changed, 39 insertions(+), 27 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java
index d49489d8a84..a1f0f8fa256 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java
@@ -55,13 +55,13 @@ public class BkEnsemblesChaosTest extends CanReconnectZKClientPulsarServiceBaseT
         for (int i = 0; i < numberOfBookies - 1; i++){
             bkEnsemble.stopBK(i);
         }
-        makeLocalMetadataStoreKeepReconnect();
+        startLocalMetadataStoreConnectionTermination();
         for (int i = 0; i < numberOfBookies - 1; i++){
             bkEnsemble.startBK(i);
         }
         // Sleep 100ms to lose the notifications of ZK node create.
         Thread.sleep(100);
-        stopLocalMetadataStoreAlwaysReconnect();
+        stopLocalMetadataStoreConnectionTermination();
 
         // Ensure broker still works.
         admin.topics().unload(topicName);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java
index 4187364e46f..5650fe6e72f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java
@@ -20,8 +20,9 @@ package org.apache.pulsar.broker.service;
 
 import static org.testng.Assert.assertEquals;
 import java.nio.charset.StandardCharsets;
-import java.util.UUID;
+import lombok.Cleanup;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
 import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride;
@@ -55,9 +56,10 @@ public class BrokerServiceChaosTest extends CanReconnectZKClientPulsarServiceBas
     public void testFetchPartitionedTopicMetadataWithCacheRefresh() throws Exception {
         final String configMetadataStoreConnectString =
                 WhiteboxImpl.getInternalState(pulsar.getConfigurationMetadataStore(), "zkConnectString");
+        @Cleanup
         final ZooKeeper anotherZKCli = new ZooKeeper(configMetadataStoreConnectString, 5000, null);
         // Set policy of auto create topic to PARTITIONED.
-        final String ns = defaultTenant + "/ns_" + UUID.randomUUID().toString().replaceAll("-", "");
+        final String ns = BrokerTestUtil.newUniqueName(defaultTenant + "/ns");
         final TopicName topicName1 = TopicName.get("persistent://" + ns + "/tp1");
         final TopicName topicName2 = TopicName.get("persistent://" + ns + "/tp2");
         admin.namespaces().createNamespace(ns);
@@ -79,11 +81,11 @@ public class BrokerServiceChaosTest extends CanReconnectZKClientPulsarServiceBas
 
         // Create the partitioned metadata by another zk client.
         // Make a error to make the cache could not update.
-        makeLocalMetadataStoreKeepReconnect();
+        startLocalMetadataStoreConnectionTermination();
         anotherZKCli.create("/admin/partitioned-topics/" + ns + "/persistent/" + topicName2.getLocalName(),
                 "{\"partitions\":3}".getBytes(StandardCharsets.UTF_8),
                 ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-        stopLocalMetadataStoreAlwaysReconnect();
+        stopLocalMetadataStoreConnectionTermination();
 
         // Get the partitioned metadata from cache, there is 90% chance that partitions count of metadata is 0.
         PartitionedTopicMetadata partitionedTopicMetadata2 =
@@ -95,9 +97,5 @@ public class BrokerServiceChaosTest extends CanReconnectZKClientPulsarServiceBas
         PartitionedTopicMetadata partitionedTopicMetadata3 =
                 pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2, true).get();
         assertEquals(partitionedTopicMetadata3.partitions, 3);
-
-        // cleanup.
-        admin.topics().deletePartitionedTopic(topicName2.toString());
-        anotherZKCli.close();
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
index a1cb4abc4c3..683fa702b4e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
@@ -24,6 +24,7 @@ import java.net.URL;
 import java.nio.channels.SelectionKey;
 import java.util.Collections;
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.PulsarService;
@@ -58,7 +59,8 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetr
     protected PulsarClient client;
     protected ZooKeeper localZkOfBroker;
     protected Object localMetaDataStoreClientCnx;
-    protected final AtomicBoolean LocalMetadataStoreInReconnectFinishSignal = new AtomicBoolean();
+    protected final AtomicBoolean connectionTerminationThreadKeepRunning = new AtomicBoolean();
+    private volatile Thread connectionTerminationThread;
 
     protected void startZKAndBK() throws Exception {
         // Start ZK.
@@ -88,25 +90,29 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetr
         client = PulsarClient.builder().serviceUrl(url.toString()).build();
     }
 
-    protected void makeLocalMetadataStoreKeepReconnect() throws Exception {
-        if (!LocalMetadataStoreInReconnectFinishSignal.compareAndSet(false, true)) {
-            throw new RuntimeException("Local metadata store is already keeping reconnect");
+    protected void startLocalMetadataStoreConnectionTermination() throws Exception {
+        if (!connectionTerminationThreadKeepRunning.compareAndSet(false, true)) {
+            throw new RuntimeException("Local metadata store connection is already being terminated");
         }
+        CompletableFuture<Void> future = new CompletableFuture<>();
         if (localMetaDataStoreClientCnx.getClass().getSimpleName().equals("ClientCnxnSocketNIO")) {
-            makeLocalMetadataStoreKeepReconnectNIO();
+            startNIOImplTermination(future);
         } else {
             // ClientCnxnSocketNetty.
-            makeLocalMetadataStoreKeepReconnectNetty();
+            startNettyImplTermination(future);
         }
+        // wait until connection is closed at least once
+        future.get();
     }
 
-    protected void makeLocalMetadataStoreKeepReconnectNIO() {
-        new Thread(() -> {
-            while (LocalMetadataStoreInReconnectFinishSignal.get()) {
+    private void startNIOImplTermination(CompletableFuture<Void> future) {
+        connectionTerminationThread = new Thread(() -> {
+            while (connectionTerminationThreadKeepRunning.get()) {
                 try {
                     SelectionKey sockKey = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "sockKey");
                     if (sockKey != null) {
                         sockKey.channel().close();
+                        future.complete(null);
                     }
                     // Prevents high cpu usage.
                     Thread.sleep(5);
@@ -114,16 +120,18 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetr
                     log.error("Try close the ZK connection of local metadata store failed: {}", e.toString());
                 }
             }
-        }).start();
+        });
+        connectionTerminationThread.start();
     }
 
-    protected void makeLocalMetadataStoreKeepReconnectNetty() {
-        new Thread(() -> {
-            while (LocalMetadataStoreInReconnectFinishSignal.get()) {
+    private void startNettyImplTermination(CompletableFuture<Void> future) {
+        connectionTerminationThread = new Thread(() -> {
+            while (connectionTerminationThreadKeepRunning.get()) {
                 try {
                     Channel channel = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "channel");
                     if (channel != null) {
                         channel.close();
+                        future.complete(null);
                     }
                     // Prevents high cpu usage.
                     Thread.sleep(5);
@@ -131,11 +139,17 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetr
                     log.error("Try close the ZK connection of local metadata store failed: {}", e.toString());
                 }
             }
-        }).start();
+        });
+        connectionTerminationThread.start();
     }
 
-    protected void stopLocalMetadataStoreAlwaysReconnect() {
-        LocalMetadataStoreInReconnectFinishSignal.set(false);
+    protected void stopLocalMetadataStoreConnectionTermination() throws InterruptedException {
+        connectionTerminationThreadKeepRunning.set(false);
+        if (connectionTerminationThread != null) {
+            // Wait for the reconnect thread to finish.
+            connectionTerminationThread.join();
+            connectionTerminationThread = null;
+        }
     }
 
     protected void createDefaultTenantsAndClustersAndNamespace() throws Exception {
@@ -195,7 +209,7 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetr
         markCurrentSetupNumberCleaned();
         log.info("--- Shutting down ---");
 
-        stopLocalMetadataStoreAlwaysReconnect();
+        stopLocalMetadataStoreConnectionTermination();
 
         // Stop brokers.
         if (client != null) {