You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/20 12:54:28 UTC

(pulsar) branch master updated: [fix][broker] Fix topic listing memory limit enforcement (#26945)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new e4c0e7f8a5c9 [fix][broker] Fix topic listing memory limit enforcement (#26945)
e4c0e7f8a5c9 is described below

commit e4c0e7f8a5c9f76295b7a61f4fa048faaf2f4cb7
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Oct 20 15:54:21 2026 +0300

    [fix][broker] Fix topic listing memory limit enforcement (#26945)
---
 .../topiclistlimit/TopicListSizeResultCache.java   | 287 ++++++++++++-
 .../pulsar/broker/topiclistlimit/TopicListing.java |  65 +++
 .../TopicListSizeResultCacheTest.java              | 186 +++++++-
 .../apache/pulsar/broker/admin/AdminResource.java  |  15 +
 .../pulsar/broker/admin/impl/NamespacesBase.java   |  95 ++---
 .../broker/admin/impl/PersistentTopicsBase.java    |  55 ++-
 .../admin/impl/TopicListResponseMemoryLimiter.java | 283 +++++++++++++
 .../apache/pulsar/broker/admin/v2/Namespaces.java  |  10 +-
 .../broker/admin/v2/NonPersistentTopics.java       |  90 ++--
 .../pulsar/broker/admin/v2/PersistentTopics.java   |   8 +-
 .../pulsar/broker/admin/v2/ScalableTopics.java     |  19 +-
 .../pulsar/broker/service/BrokerService.java       |   9 +-
 .../pulsar/broker/service/PulsarCommandSender.java |  21 +-
 .../broker/service/PulsarCommandSenderImpl.java    |  15 +-
 .../apache/pulsar/broker/service/ServerCnx.java    |  76 +++-
 .../pulsar/broker/service/TopicListService.java    | 450 +++++++++++++++-----
 .../broker/admin/TopicListRestMemoryLimitTest.java | 209 +++++++++
 .../impl/TopicListResponseMemoryLimiterTest.java   | 365 ++++++++++++++++
 .../pulsar/broker/service/ServerCnxTest.java       | 128 ++++++
 .../broker/service/TopicListServiceTest.java       | 469 ++++++++++++++++++++-
 .../common/semaphore/AsyncDualMemoryLimiter.java   |  18 +-
 .../semaphore/AsyncDualMemoryLimiterUtil.java      |  93 +++-
 .../pulsar/common/semaphore/AsyncSemaphore.java    |  16 +
 .../semaphore/AsyncDualMemoryLimiterUtilTest.java  | 160 +++++++
 .../pulsar/proxy/server/LookupProxyHandler.java    |  99 +++--
 .../apache/pulsar/proxy/server/ProxyService.java   |   9 +-
 26 files changed, 2928 insertions(+), 322 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCache.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCache.java
index 90e348447906..3a3896ef0f85 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCache.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCache.java
@@ -20,8 +20,18 @@ package org.apache.pulsar.broker.topiclistlimit;
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Comparator;
+import java.util.Map;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
 import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
 
 /**
@@ -32,18 +42,151 @@ import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
  * - Maintain and update size estimates based on actual topic list sizes
  * - Prevent thundering herd problems when multiple concurrent requests for a namespace are made without
  *   a previous size estimate
- * The cache uses namespace name and topic list mode (PERSISTENT/NON_PERSISTENT/ALL) as keys
+ * The cache uses the topic listing ({@link TopicListing}: namespace name, topic list mode
+ * (PERSISTENT/NON_PERSISTENT/ALL) and listing type) and the optional topic listing properties (PIP-452) as keys
  * and maintains size estimates that are refined with actual usage.
  */
 public class TopicListSizeResultCache {
     // 10kB initial estimate for topic list heap size
     private static final long INITIAL_TOPIC_LIST_HEAP_SIZE = 10 * 1024;
+    // how soon after measuring an oversized topic list the listing may measure it again while the limiter is idle
+    private static final Duration DEFAULT_OVERSIZED_REMEASURE_INTERVAL = Duration.ofMinutes(1);
+    // how soon after measuring an oversized topic list the listing may measure it again even if the limiter is busy
+    private static final Duration DEFAULT_OVERSIZED_FORCED_REMEASURE_INTERVAL = Duration.ofMinutes(10);
+    // how long a re-measuring request blocks other re-measuring requests unless it completes before
+    private static final Duration DEFAULT_REMEASURE_GATE_TIMEOUT = Duration.ofMinutes(1);
 
-    private Cache<CacheKey, ResultHolder> topicListSizeCache = Caffeine.newBuilder()
+    private final Cache<CacheKey, ResultHolder> topicListSizeCache = Caffeine.newBuilder()
             .maximumSize(10000)
             .build();
+    private final OversizedRemeasurePolicy remeasurePolicy;
 
-    record CacheKey(String namespaceName, CommandGetTopicsOfNamespace.Mode mode) {
+    /**
+     * Creates a cache without a size limit.
+     */
+    public TopicListSizeResultCache() {
+        this(0, () -> true);
+    }
+
+    /**
+     * Creates a cache for listings that are limited by the given topic list heap memory limit.
+     *
+     * <p>A topic list larger than the limit can't be served. Requests for a listing whose estimate is larger than
+     * the limit get the estimate as-is, so that they fail without waiting for permits. To let the listing recover
+     * after its topic list has become smaller, a request may get the limit as its estimate so that it lists the
+     * topics and measures the list again. That request needs all permits, and permits are granted in order, so
+     * while it waits for the permits of other listings, all later listings wait behind it. Therefore:
+     * <ul>
+     *     <li>a listing is measured again at the earliest one interval after its oversized size was measured,</li>
+     *     <li>only one listing at a time is measured again,</li>
+     *     <li>and only while no other topic list is in flight or waiting for permits. When the limiter hasn't been
+     *     idle for a longer interval, the listing is measured again anyway, so that it can recover on a broker that
+     *     is never idle.</li>
+     * </ul>
+     *
+     * @param sizeLimit the topic list heap memory limit, use {@code <= 0} for no limit
+     * @param limiterIdle supplier that returns true when no topic list permits are acquired or requested
+     */
+    public TopicListSizeResultCache(long sizeLimit, BooleanSupplier limiterIdle) {
+        this(sizeLimit, DEFAULT_OVERSIZED_REMEASURE_INTERVAL, DEFAULT_OVERSIZED_FORCED_REMEASURE_INTERVAL,
+                limiterIdle);
+    }
+
+    @VisibleForTesting
+    TopicListSizeResultCache(long sizeLimit, Duration oversizedRemeasureInterval,
+                             Duration oversizedForcedRemeasureInterval, BooleanSupplier limiterIdle) {
+        this.remeasurePolicy = new OversizedRemeasurePolicy(sizeLimit > 0 ? sizeLimit : Long.MAX_VALUE,
+                oversizedRemeasureInterval.toNanos(), oversizedForcedRemeasureInterval.toNanos(),
+                DEFAULT_REMEASURE_GATE_TIMEOUT.toNanos(), limiterIdle, new AtomicReference<>(),
+                new AtomicLong(System.nanoTime()));
+    }
+
+    /**
+     * Returns the topic list heap memory limit, {@link Long#MAX_VALUE} when there's no limit. An estimate larger
+     * than the limit can't be granted.
+     */
+    public long getSizeLimit() {
+        return remeasurePolicy.sizeLimit();
+    }
+
+    /**
+     * The listing that is being measured again, and the time when it stops blocking other listings from being
+     * measured again unless it records its size before that.
+     */
+    private record RemeasureGate(ResultHolder owner, long expiresAtNanos) {
+    }
+
+    /**
+     * The size limit and the rules for measuring an oversized topic list again.
+     *
+     * @param gate the listing that is being measured again. Only one listing is measured again at a time.
+     * @param nextForcedRemeasureNanos the earliest time when any listing may be measured again while the limiter
+     *                                 is busy. The listings have separate estimates for each set of properties,
+     *                                 which come from the clients, so the number of forced re-measures is limited
+     *                                 for the whole broker.
+     */
+    private record OversizedRemeasurePolicy(long sizeLimit, long intervalNanos, long forcedIntervalNanos,
+                                            long gateTimeoutNanos, BooleanSupplier limiterIdle,
+                                            AtomicReference<RemeasureGate> gate,
+                                            AtomicLong nextForcedRemeasureNanos) {
+        static final OversizedRemeasurePolicy UNLIMITED = new OversizedRemeasurePolicy(Long.MAX_VALUE, 0, 0, 0,
+                () -> true, new AtomicReference<>(), new AtomicLong());
+
+        boolean tryForcedRemeasure(long now) {
+            long next = nextForcedRemeasureNanos.get();
+            return now - next >= 0 && nextForcedRemeasureNanos.compareAndSet(next, now + forcedIntervalNanos);
+        }
+
+        boolean tryEnterGate(ResultHolder holder, long now) {
+            RemeasureGate current = gate.get();
+            return (current == null || now - current.expiresAtNanos() >= 0)
+                    && gate.compareAndSet(current, new RemeasureGate(holder, now + gateTimeoutNanos));
+        }
+
+        void exitGate(ResultHolder holder) {
+            RemeasureGate current = gate.get();
+            if (current != null && current.owner() == holder) {
+                gate.compareAndSet(current, null);
+            }
+        }
+    }
+
+    /**
+     * Cache key. The topic listing properties are stored as a hash so that the client provided property values
+     * aren't retained in the cache. A hash collision only results in sharing a size estimate.
+     */
+    record CacheKey(TopicListing listing, HashCode propertiesHash) {
+        CacheKey(String namespaceName, CommandGetTopicsOfNamespace.Mode mode) {
+            this(TopicListing.namespaceTopics(namespaceName, mode), (HashCode) null);
+        }
+
+        CacheKey(TopicListing listing, Map<String, String> properties) {
+            this(listing, hashProperties(properties));
+        }
+
+        private static HashCode hashProperties(Map<String, String> properties) {
+            if (properties == null || properties.isEmpty()) {
+                return null;
+            }
+            Hasher hasher = Hashing.murmur3_128().newHasher();
+            properties.entrySet().stream()
+                    .sorted(Map.Entry.comparingByKey(Comparator.nullsFirst(Comparator.naturalOrder())))
+                    .forEach(entry -> {
+                        putString(hasher, entry.getKey());
+                        putString(hasher, entry.getValue());
+                    });
+            return hasher.hash();
+        }
+
+        private static void putString(Hasher hasher, String value) {
+            if (value == null) {
+                hasher.putInt(-1);
+            } else {
+                // length prefix keeps the encoding of the entries unambiguous
+                hasher.putInt(value.length());
+                hasher.putString(value, StandardCharsets.UTF_8);
+            }
+        }
     }
 
     /**
@@ -54,6 +197,29 @@ public class TopicListSizeResultCache {
     public static class ResultHolder {
         private final AtomicReference<CompletableFuture<Long>> topicListSizeFuture =
                 new AtomicReference<>(null);
+        private final OversizedRemeasurePolicy policy;
+        private final long sizeLimit;
+        private final long initialEstimate;
+        // the earliest time when a request for an oversized listing may measure the topic list again
+        private volatile long nextRemeasureNanos;
+        // the earliest time when the topic list is measured again even if the limiter is busy
+        private volatile long nextForcedRemeasureNanos;
+        // whether the topic list of this listing has been measured
+        private volatile boolean measured;
+
+        public ResultHolder() {
+            this(OversizedRemeasurePolicy.UNLIMITED, 0);
+        }
+
+        private ResultHolder(OversizedRemeasurePolicy policy, long knownSizeEstimate) {
+            this.policy = policy;
+            this.sizeLimit = policy.sizeLimit();
+            this.initialEstimate = Math.max(Math.min(INITIAL_TOPIC_LIST_HEAP_SIZE, sizeLimit), knownSizeEstimate);
+            long now = System.nanoTime();
+            // a listing that starts from an oversized known estimate hasn't been measured yet
+            this.nextRemeasureNanos = now;
+            this.nextForcedRemeasureNanos = now + policy.forcedIntervalNanos();
+        }
 
         /**
          * Get the topic list size estimate. The first request will return the initial estimate
@@ -61,16 +227,46 @@ public class TopicListSizeResultCache {
          * will wait for the first request to complete and use the estimate of the first request.
          * Subsequent requests will use estimate which gets updated based on the returned size of the topic list
          * of each request.
+         * When the estimate is larger than the size limit, it's returned as-is so that the request fails without
+         * waiting for permits, except for a request that gets the size limit so that it can measure the topic
+         * list again (see {@link TopicListSizeResultCache#TopicListSizeResultCache(long, BooleanSupplier)}).
          * @return a future that will return the topic list size estimate
          */
         public CompletableFuture<Long> getSizeAsync() {
+            CompletableFuture<Long> estimate;
             if (topicListSizeFuture.compareAndSet(null, new CompletableFuture<>())) {
                 // let the first request proceed with the initial estimate
-                return CompletableFuture.completedFuture(INITIAL_TOPIC_LIST_HEAP_SIZE);
+                estimate = CompletableFuture.completedFuture(initialEstimate);
             } else {
                 // all other requests wait for the first one to complete
-                return topicListSizeFuture.get();
+                estimate = topicListSizeFuture.get();
+            }
+            return sizeLimit == Long.MAX_VALUE ? estimate : estimate.thenApply(this::limitOversizedEstimate);
+        }
+
+        private long limitOversizedEstimate(long estimate) {
+            if (estimate <= sizeLimit) {
+                return estimate;
+            }
+            long now = System.nanoTime();
+            // Requesting all permits while other topic lists are in flight would make the request wait for them,
+            // and all later requests would wait behind it, since the permits are granted in order. Measure the
+            // list again only when no other topic list is in flight, or when the limiter hasn't been idle for a
+            // longer time, and only one listing at a time.
+            boolean limiterIdle = policy.limiterIdle().getAsBoolean();
+            if (now - nextRemeasureNanos >= 0
+                    && (limiterIdle || now - nextForcedRemeasureNanos >= 0)
+                    && policy.tryEnterGate(this, now)) {
+                if (limiterIdle || policy.tryForcedRemeasure(now)) {
+                    nextRemeasureNanos = now + policy.intervalNanos();
+                    nextForcedRemeasureNanos = now + policy.forcedIntervalNanos();
+                    // a request that can be granted, so that the topic list gets measured again
+                    return sizeLimit;
+                }
+                policy.exitGate(this);
             }
+            // acquiring permits for an estimate larger than the limit fails right away
+            return estimate;
         }
 
         /**
@@ -79,6 +275,15 @@ public class TopicListSizeResultCache {
          * @param actualSize the actual size of the topic list
          */
         public void updateSize(long actualSize) {
+            measured = true;
+            if (actualSize > sizeLimit) {
+                // the topic list can be measured again after the interval
+                long now = System.nanoTime();
+                nextRemeasureNanos = now + policy.intervalNanos();
+                nextForcedRemeasureNanos = now + policy.forcedIntervalNanos();
+            }
+            // the topic list has been measured, another listing may be measured again
+            policy.exitGate(this);
             CompletableFuture<Long> currentFuture = topicListSizeFuture.get();
             if (currentFuture != null && !currentFuture.isDone()) {
                 // complete the future if it's not done yet
@@ -103,9 +308,31 @@ public class TopicListSizeResultCache {
             });
             // let all current requests proceed with the initial estimate if this were to happen
             if (currentFuture != null && !currentFuture.isDone()) {
-                currentFuture.complete(INITIAL_TOPIC_LIST_HEAP_SIZE);
+                currentFuture.complete(initialEstimate);
             }
         }
+
+        /**
+         * Returns true when the topic list of this listing has been measured. A listing with a new set of properties
+         * starts from the estimate of the same listing without properties, which can be larger than the limit
+         * although its own topic list hasn't been measured yet.
+         *
+         * @return true if the topic list has been measured
+         */
+        public boolean isMeasured() {
+            return measured;
+        }
+
+        /**
+         * Returns the current size estimate, or 0 when there's no measured estimate yet.
+         */
+        long getKnownSizeEstimate() {
+            CompletableFuture<Long> currentFuture = topicListSizeFuture.get();
+            if (currentFuture == null || !currentFuture.isDone() || currentFuture.isCompletedExceptionally()) {
+                return 0;
+            }
+            return currentFuture.getNow(0L);
+        }
     }
 
     /**
@@ -116,7 +343,51 @@ public class TopicListSizeResultCache {
      */
     public ResultHolder getTopicListSize(String namespaceName,
                                          CommandGetTopicsOfNamespace.Mode mode) {
-        return topicListSizeCache.asMap()
-                .computeIfAbsent(new CacheKey(namespaceName, mode), __ -> new ResultHolder());
+        return getTopicListSize(namespaceName, mode, null);
+    }
+
+    /**
+     * Get the topic list size result holder for the given namespace, mode and topic listing properties.
+     * The properties are passed to a customized topic listing implementation (PIP-452) which can return
+     * a different topic list for different properties. Therefore, the estimate is kept separately for each
+     * distinct set of properties.
+     * @param namespaceName the namespace name in the format of "tenant/namespace"
+     * @param mode the mode of the topic list request (PERSISTENT, NON_PERSISTENT, ALL)
+     * @param properties the topic listing properties, null or empty when not provided
+     * @return the topic list size result holder
+     */
+    public ResultHolder getTopicListSize(String namespaceName,
+                                         CommandGetTopicsOfNamespace.Mode mode,
+                                         Map<String, String> properties) {
+        return getTopicListSize(TopicListing.namespaceTopics(namespaceName, mode), properties);
+    }
+
+    /**
+     * Get the topic list size result holder for the given topic listing and topic listing properties.
+     * @param listing the topic listing
+     * @param properties the topic listing properties (PIP-452), null or empty when not provided
+     * @return the topic list size result holder
+     */
+    public ResultHolder getTopicListSize(TopicListing listing, Map<String, String> properties) {
+        CacheKey key = new CacheKey(listing, properties);
+        ResultHolder holder = topicListSizeCache.getIfPresent(key);
+        if (holder != null) {
+            return holder;
+        }
+        // The properties come from the client. A listing with a new set of properties starts from the estimate of
+        // the same listing without properties, so that a request can't start below the known size of the listing
+        // by sending new property values. With the default topic listing, which ignores the properties, the lists
+        // are the same.
+        long knownSizeEstimate = 0;
+        if (key.propertiesHash() != null) {
+            ResultHolder holderWithoutProperties =
+                    topicListSizeCache.getIfPresent(new CacheKey(listing, (HashCode) null));
+            if (holderWithoutProperties != null) {
+                knownSizeEstimate = holderWithoutProperties.getKnownSizeEstimate();
+            }
+        }
+        long initialSizeEstimate = knownSizeEstimate;
+        return topicListSizeCache.asMap().computeIfAbsent(key,
+                __ -> new ResultHolder(remeasurePolicy, initialSizeEstimate));
     }
 }
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListing.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListing.java
new file mode 100644
index 000000000000..e741fae5c8d7
--- /dev/null
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListing.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.topiclistlimit;
+
+import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode;
+
+/**
+ * Identifies a topic listing for the topic list size estimates of {@link TopicListSizeResultCache}.
+ *
+ * <p>Listings that return different topic lists need separate size estimates, so that their estimates don't
+ * replace each other.
+ *
+ * @param namespaceName the namespace name in the format of "tenant/namespace"
+ * @param mode the mode of the listing (PERSISTENT, NON_PERSISTENT, ALL)
+ * @param type the type of the listing
+ * @param bundleRange the bundle range for listings of a single bundle, null otherwise
+ */
+public record TopicListing(String namespaceName, Mode mode, Type type, String bundleRange) {
+
+    /**
+     * The type of a topic listing.
+     */
+    public enum Type {
+        /** The topics of a namespace. */
+        NAMESPACE_TOPICS,
+        /** The topics that the owner of a bundle has loaded in that bundle. */
+        OWNED_BUNDLE_TOPICS,
+        /** The partitioned topics of a namespace. */
+        PARTITIONED_TOPICS,
+        /** The scalable topics of a namespace. */
+        SCALABLE_TOPICS
+    }
+
+    public static TopicListing namespaceTopics(String namespaceName, Mode mode) {
+        return new TopicListing(namespaceName, mode, Type.NAMESPACE_TOPICS, null);
+    }
+
+    public static TopicListing ownedBundleTopics(String namespaceName, Mode mode, String bundleRange) {
+        return new TopicListing(namespaceName, mode, Type.OWNED_BUNDLE_TOPICS, bundleRange);
+    }
+
+    public static TopicListing partitionedTopics(String namespaceName, Mode mode) {
+        return new TopicListing(namespaceName, mode, Type.PARTITIONED_TOPICS, null);
+    }
+
+    public static TopicListing scalableTopics(String namespaceName) {
+        return new TopicListing(namespaceName, Mode.ALL, Type.SCALABLE_TOPICS, null);
+    }
+}
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCacheTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCacheTest.java
index 397582d7022e..68480ddc7494 100644
--- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCacheTest.java
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCacheTest.java
@@ -25,15 +25,20 @@ import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNotSame;
 import static org.testng.Assert.assertSame;
 import static org.testng.Assert.assertTrue;
+import java.time.Duration;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
+import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -67,6 +72,53 @@ public class TopicListSizeResultCacheTest {
         assertSame(holder1, holder2, "Should return the same ResultHolder instance for the same key");
     }
 
+    @Test
+    public void testGetTopicListSize_keysByProperties() {
+        String namespace = "tenant/namespace";
+        CommandGetTopicsOfNamespace.Mode mode = CommandGetTopicsOfNamespace.Mode.PERSISTENT;
+
+        TopicListSizeResultCache.ResultHolder noProperties = cache.getTopicListSize(namespace, mode);
+        assertSame(cache.getTopicListSize(namespace, mode, null), noProperties);
+        assertSame(cache.getTopicListSize(namespace, mode, Map.of()), noProperties);
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put("k1", "v1");
+        properties.put("k2", "v2");
+        TopicListSizeResultCache.ResultHolder withProperties = cache.getTopicListSize(namespace, mode, properties);
+        assertNotSame(withProperties, noProperties,
+                "A customized topic listing with properties should have a separate size estimate");
+        assertSame(cache.getTopicListSize(namespace, mode, Map.of("k2", "v2", "k1", "v1")), withProperties);
+        // the key must not be affected by later modifications of the passed map
+        properties.put("k3", "v3");
+        assertSame(cache.getTopicListSize(namespace, mode, Map.of("k2", "v2", "k1", "v1")), withProperties);
+        assertNotSame(cache.getTopicListSize(namespace, mode, Map.of("k1", "other")), withProperties);
+    }
+
+    @Test
+    public void testGetTopicListSize_keysByListing() {
+        String namespace = "tenant/namespace";
+        CommandGetTopicsOfNamespace.Mode mode = CommandGetTopicsOfNamespace.Mode.NON_PERSISTENT;
+
+        TopicListSizeResultCache.ResultHolder namespaceTopics = cache.getTopicListSize(namespace, mode);
+        assertSame(cache.getTopicListSize(TopicListing.namespaceTopics(namespace, mode), null), namespaceTopics);
+
+        List<TopicListing> otherListings = List.of(
+                TopicListing.ownedBundleTopics(namespace, mode, "0x00000000_0x80000000"),
+                TopicListing.ownedBundleTopics(namespace, mode, "0x80000000_0xffffffff"),
+                TopicListing.partitionedTopics(namespace, mode),
+                TopicListing.scalableTopics(namespace));
+        List<TopicListSizeResultCache.ResultHolder> holders = new ArrayList<>();
+        holders.add(namespaceTopics);
+        for (TopicListing listing : otherListings) {
+            TopicListSizeResultCache.ResultHolder holder = cache.getTopicListSize(listing, null);
+            assertSame(cache.getTopicListSize(listing, Map.of()), holder);
+            for (TopicListSizeResultCache.ResultHolder other : holders) {
+                assertNotSame(holder, other, "Each topic listing should have a separate size estimate: " + listing);
+            }
+            holders.add(holder);
+        }
+    }
+
     @Test
     public void testGetTopicListSize_returnsDifferentInstancesForDifferentKeys() {
         String namespace1 = "tenant/namespace1";
@@ -329,4 +381,136 @@ public class TopicListSizeResultCacheTest {
             assertNotNull(future.get(5, TimeUnit.SECONDS), "All futures should complete");
         }
     }
-}
\ No newline at end of file
+
+    @Test
+    public void testOversizedEstimateFailsFastUntilIntervalHasPassed() {
+        TopicListSizeResultCache limitedCache =
+                new TopicListSizeResultCache(100_000, Duration.ofHours(1), Duration.ofHours(1), () -> true);
+        TopicListSizeResultCache.ResultHolder holder =
+                limitedCache.getTopicListSize("tenant/namespace", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+        holder.getSizeAsync();
+        // a topic list larger than the limit
+        holder.updateSize(500_000);
+        // the requests get the estimate, which is larger than the limit, so that they fail right away instead of
+        // listing the topics again right after the oversized list was measured
+        assertEquals(holder.getSizeAsync().join().longValue(), 500_000L);
+        assertEquals(holder.getSizeAsync().join().longValue(), 500_000L);
+        // the estimate follows the list when it becomes smaller than the limit
+        holder.updateSize(50_000);
+        assertEquals(holder.getSizeAsync().join().longValue(), 50_000L);
+    }
+
+    @Test
+    public void testOversizedEstimateIsRemeasuredOnlyWhenLimiterIsIdle() {
+        AtomicBoolean limiterIdle = new AtomicBoolean(false);
+        TopicListSizeResultCache limitedCache =
+                new TopicListSizeResultCache(100_000, Duration.ZERO, Duration.ofHours(1), limiterIdle::get);
+        TopicListSizeResultCache.ResultHolder holder =
+                limitedCache.getTopicListSize("tenant/namespace", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+        holder.getSizeAsync();
+        holder.updateSize(500_000);
+        // other topic lists are in flight: requesting all permits would make the later requests wait behind it
+        assertEquals(holder.getSizeAsync().join().longValue(), 500_000L);
+        // no other topic list is in flight: the request gets the limit so that it can measure the list again
+        limiterIdle.set(true);
+        assertEquals(holder.getSizeAsync().join().longValue(), 100_000L);
+    }
+
+    @Test
+    public void testNewPropertiesStartFromEstimateWithoutProperties() {
+        String namespace = "tenant/namespace";
+        CommandGetTopicsOfNamespace.Mode mode = CommandGetTopicsOfNamespace.Mode.PERSISTENT;
+        // without a known estimate, a listing with properties starts from the initial estimate
+        TopicListSizeResultCache.ResultHolder first = cache.getTopicListSize(namespace, mode, Map.of("k", "v0"));
+        assertEquals(first.getSizeAsync().join().longValue(), 10 * 1024L);
+
+        TopicListSizeResultCache.ResultHolder withoutProperties = cache.getTopicListSize(namespace, mode);
+        withoutProperties.getSizeAsync();
+        withoutProperties.updateSize(500_000);
+        // a listing with new properties can't start below the known size of the listing without properties
+        TopicListSizeResultCache.ResultHolder withNewProperties =
+                cache.getTopicListSize(namespace, mode, Map.of("k", "v1"));
+        assertEquals(withNewProperties.getSizeAsync().join().longValue(), 500_000L);
+        // and then follows its own topic list
+        withNewProperties.updateSize(1000);
+        assertEquals(withNewProperties.getSizeAsync().join().longValue(), 1000L);
+        assertEquals(withoutProperties.getSizeAsync().join().longValue(), 500_000L);
+    }
+
+    @Test
+    public void testNewPropertiesWithOversizedEstimateFailFastWhileLimiterIsBusy() {
+        AtomicBoolean limiterIdle = new AtomicBoolean(false);
+        TopicListSizeResultCache limitedCache =
+                new TopicListSizeResultCache(100_000, Duration.ofHours(1), Duration.ofHours(1), limiterIdle::get);
+        String namespace = "tenant/namespace";
+        CommandGetTopicsOfNamespace.Mode mode = CommandGetTopicsOfNamespace.Mode.PERSISTENT;
+        TopicListSizeResultCache.ResultHolder withoutProperties = limitedCache.getTopicListSize(namespace, mode);
+        withoutProperties.getSizeAsync();
+        withoutProperties.updateSize(500_000);
+
+        TopicListSizeResultCache.ResultHolder withNewProperties =
+                limitedCache.getTopicListSize(namespace, mode, Map.of("k", "v1"));
+        assertEquals(withNewProperties.getSizeAsync().join().longValue(), 500_000L);
+        // the request is rejected, which resets the estimate that it was initializing
+        withNewProperties.resetIfInitializing();
+        // a listing with new properties hasn't been measured, so it may be measured when the limiter is idle
+        limiterIdle.set(true);
+        assertEquals(withNewProperties.getSizeAsync().join().longValue(), 100_000L);
+    }
+
+    @Test
+    public void testOnlyOneOversizedListingIsRemeasuredAtATime() {
+        TopicListSizeResultCache limitedCache =
+                new TopicListSizeResultCache(100_000, Duration.ZERO, Duration.ofHours(1), () -> true);
+        TopicListSizeResultCache.ResultHolder first =
+                limitedCache.getTopicListSize("tenant/ns1", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+        TopicListSizeResultCache.ResultHolder second =
+                limitedCache.getTopicListSize("tenant/ns2", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+        for (TopicListSizeResultCache.ResultHolder holder : List.of(first, second)) {
+            holder.getSizeAsync();
+            holder.updateSize(500_000);
+        }
+        // the first listing is measured again, the second one fails fast while that's in progress
+        assertEquals(first.getSizeAsync().join().longValue(), 100_000L);
+        assertEquals(second.getSizeAsync().join().longValue(), 500_000L);
+        // the first listing measured its size, so the second listing may be measured again
+        first.updateSize(500_000);
+        assertEquals(second.getSizeAsync().join().longValue(), 100_000L);
+    }
+
+    @Test
+    public void testOversizedEstimateIsRemeasuredWhenLimiterIsNeverIdle() {
+        TopicListSizeResultCache limitedCache =
+                new TopicListSizeResultCache(100_000, Duration.ZERO, Duration.ZERO, () -> false);
+        TopicListSizeResultCache.ResultHolder holder =
+                limitedCache.getTopicListSize("tenant/namespace", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+        holder.getSizeAsync();
+        holder.updateSize(500_000);
+        // after the longer interval, the listing is measured again although other topic lists are in flight
+        assertEquals(holder.getSizeAsync().join().longValue(), 100_000L);
+    }
+
+    @Test
+    public void testForcedRemeasuresAreLimitedForTheWholeCache() throws Exception {
+        Duration forcedInterval = Duration.ofMillis(500);
+        TopicListSizeResultCache limitedCache =
+                new TopicListSizeResultCache(100_000, Duration.ZERO, forcedInterval, () -> false);
+        // listings with different properties have separate estimates
+        TopicListSizeResultCache.ResultHolder first = limitedCache.getTopicListSize("tenant/namespace",
+                CommandGetTopicsOfNamespace.Mode.PERSISTENT, Map.of("k", "v1"));
+        TopicListSizeResultCache.ResultHolder second = limitedCache.getTopicListSize("tenant/namespace",
+                CommandGetTopicsOfNamespace.Mode.PERSISTENT, Map.of("k", "v2"));
+        for (TopicListSizeResultCache.ResultHolder holder : List.of(first, second)) {
+            holder.getSizeAsync();
+            holder.updateSize(500_000);
+        }
+        Thread.sleep(forcedInterval.toMillis() + 100);
+        // the limiter is busy: the first listing is measured again after the longer interval
+        assertEquals(first.getSizeAsync().join().longValue(), 100_000L);
+        first.updateSize(500_000);
+        // the second listing's interval has also passed, but forced re-measures are limited for the whole cache
+        assertEquals(second.getSizeAsync().join().longValue(), 500_000L);
+        Awaitility.await().untilAsserted(
+                () -> assertEquals(second.getSizeAsync().join().longValue(), 100_000L));
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
index b94dcfde51c2..84a548ddc5ce 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
@@ -45,6 +45,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.admin.impl.TopicListResponseMemoryLimiter;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
 import org.apache.pulsar.broker.authorization.AuthorizationService;
 import org.apache.pulsar.broker.namespace.TopicExistsInfo;
@@ -1020,6 +1021,20 @@ public abstract class AdminResource extends PulsarWebResource {
                 == Status.NOT_FOUND.getStatusCode();
     }
 
+    /**
+     * Checks whether a topic listing failed because the topic list memory limit rejected it (429) or because the
+     * request was cancelled (503). These are expected responses that don't need to be logged as errors.
+     */
+    protected static boolean isTopicListLimitRejection(Throwable ex) {
+        Throwable realCause = FutureUtil.unwrapCompletionException(ex);
+        if (!(realCause instanceof WebApplicationException webApplicationException)) {
+            return false;
+        }
+        // other 503 responses, such as a bundle owner that isn't ready, are logged as errors
+        return webApplicationException.getResponse().getStatus() == Status.TOO_MANY_REQUESTS.getStatusCode()
+                || TopicListResponseMemoryLimiter.isRequestCancelled(realCause);
+    }
+
     protected static boolean is4xxRestException(Throwable ex) {
         Throwable realCause = FutureUtil.unwrapCompletionException(ex);
         return realCause instanceof WebApplicationException
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index 9cc0473dd5fc..b0f9ffaae813 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -22,7 +22,6 @@ import static org.apache.pulsar.common.policies.data.PoliciesUtil.getBundles;
 import com.google.common.collect.Sets;
 import jakarta.ws.rs.WebApplicationException;
 import jakarta.ws.rs.container.AsyncResponse;
-import jakarta.ws.rs.container.CompletionCallback;
 import jakarta.ws.rs.core.Response;
 import jakarta.ws.rs.core.Response.Status;
 import jakarta.ws.rs.core.UriBuilder;
@@ -44,10 +43,8 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiFunction;
-import java.util.function.BooleanSupplier;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -70,8 +67,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentReplicator;
 import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.service.scalable.AutoScaleConfig;
-import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
-import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
 import org.apache.pulsar.broker.web.RestException;
 import org.apache.pulsar.client.admin.GrantTopicPermissionOptions;
 import org.apache.pulsar.client.admin.Namespaces;
@@ -122,8 +118,6 @@ import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.common.policies.data.ValidateResult;
 import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl;
 import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl;
-import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
-import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
 import org.apache.pulsar.common.util.Codec;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
@@ -186,76 +180,39 @@ public abstract class NamespacesBase extends AdminResource {
                                                                       CommandGetTopicsOfNamespace.Mode mode) {
         // Use maxTopicListInFlightLimiter to limit inflight get topic listing responses
         // to avoid OOME caused by a lot of clients using HTTP service lookups to list topics
-        AsyncDualMemoryLimiterImpl maxTopicListInFlightLimiter =
-                pulsar().getBrokerService().getMaxTopicListInFlightLimiter();
-        TopicListSizeResultCache.ResultHolder listSizeHolder = pulsar().getBrokerService().getTopicListSizeResultCache()
-                .getTopicListSize(namespaceName.toString(), mode);
-        // setup the permit cancellation function
-        AtomicBoolean permitRequestCancelled = new AtomicBoolean(false);
-        BooleanSupplier isPermitRequestCancelled = permitRequestCancelled::get;
-        // add callback that releases permits when the response completes
-        AtomicReference<AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit> initialPermitsRef =
-                new AtomicReference<>();
-        AtomicReference<AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit> permitsRef = new AtomicReference<>();
-        response.register(new CompletionCallback() {
-            @Override
-            public void onComplete(Throwable throwable) {
-                if (throwable != null) {
-                    // for failed request
-                    // handle resetting the TopicListSizeResultCache.ResultHolder
-                    listSizeHolder.resetIfInitializing();
-                    // cancel any pending permit request
-                    permitRequestCancelled.set(true);
-                }
-                AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit initialPermit = initialPermitsRef.get();
-                if (initialPermit != null) {
-                    maxTopicListInFlightLimiter.release(initialPermit);
-                }
-                AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit permits = permitsRef.get();
-                if (permits != null) {
-                    maxTopicListInFlightLimiter.release(permits);
-                }
-            }
-        });
-        return listSizeHolder.getSizeAsync().thenCompose(initialSize -> maxTopicListInFlightLimiter.acquire(initialSize,
-                AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY, isPermitRequestCancelled).exceptionally(t -> {
-            throw new CompletionException(
-                    new RestException(Status.TOO_MANY_REQUESTS, "Failed due to heap memory limit exceeded"));
-        }).thenCompose(initialPermits -> {
-            initialPermitsRef.set(initialPermits);
-            // perform the actual get list of topics operation
-            return doInternalGetListOfTopics(policies, mode).thenCompose(topicList -> {
-                long actualSize = TopicListMemoryLimiter.estimateTopicListSize(topicList);
-                listSizeHolder.updateSize(actualSize);
-                return maxTopicListInFlightLimiter.update(initialPermits, actualSize, isPermitRequestCancelled)
-                        .exceptionally(t -> {
-                            throw new CompletionException(new RestException(Status.TOO_MANY_REQUESTS,
-                                    "Failed due to heap memory limit exceeded"));
-                        }).thenApply(permits -> {
-                            permitsRef.set(permits);
-                            initialPermitsRef.set(null);
-                            return topicList;
-                        });
-            });
-        }));
-    }
-
-    private CompletableFuture<List<String>> doInternalGetListOfTopics(Policies policies,
-                                                                     CommandGetTopicsOfNamespace.Mode mode) {
         switch (mode) {
             case ALL:
-                return pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName)
-                        .thenCombine(internalGetNonPersistentTopics(policies),
-                                (persistentTopics, nonPersistentTopics) ->
-                                        ListUtils.union(persistentTopics, nonPersistentTopics));
+                // The non-persistent topics are collected first from the bundle listings, which acquire permits
+                // from the same limiter, possibly on this broker, so no permits are held while waiting for them.
+                // The listing then acquires permits once, for the whole topic list, with the estimate of the ALL
+                // listing, so that it doesn't hold permits for one part while waiting for permits for the other.
+                return TopicListResponseMemoryLimiter.listTopicsAfterCollecting(pulsar().getBrokerService(),
+                        response, TopicListing.namespaceTopics(namespaceName.toString(), mode), null,
+                        () -> internalGetNonPersistentTopics(policies),
+                        nonPersistentTopics -> pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName)
+                                .thenApply(persistentTopics ->
+                                        ListUtils.union(persistentTopics, nonPersistentTopics)));
             case NON_PERSISTENT:
-                return internalGetNonPersistentTopics(policies);
+                return listNonPersistentTopicsWithMemoryLimit(response, policies);
             case PERSISTENT:
             default:
-                return pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName);
+                return listPersistentTopicsWithMemoryLimit(response);
         }
     }
 
+    private CompletableFuture<List<String>> listPersistentTopicsWithMemoryLimit(AsyncResponse response) {
+        return TopicListResponseMemoryLimiter.listTopics(pulsar().getBrokerService(), response,
+                TopicListing.namespaceTopics(namespaceName.toString(), CommandGetTopicsOfNamespace.Mode.PERSISTENT),
+                null, () -> pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName));
+    }
+
+    private CompletableFuture<List<String>> listNonPersistentTopicsWithMemoryLimit(AsyncResponse response,
+                                                                                   Policies policies) {
+        // the non-persistent topics are collected from the bundle topic listings of the bundle owners
+        return TopicListResponseMemoryLimiter.listTopicsFromOtherListings(pulsar().getBrokerService(), response,
+                () -> internalGetNonPersistentTopics(policies));
+    }
+
     protected CompletableFuture<List<String>> internalGetNonPersistentTopics(Policies policies) {
         final List<CompletableFuture<List<String>>> futures = new ArrayList<>();
         final List<String> boundaries = policies.bundles.getBoundaries();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 139c106a4f46..0eaa171770e8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -56,6 +56,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
@@ -91,6 +92,7 @@ import org.apache.pulsar.broker.service.TopicPoliciesService;
 import org.apache.pulsar.broker.service.persistent.PersistentReplicator;
 import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
 import org.apache.pulsar.broker.web.RestException;
 import org.apache.pulsar.client.admin.LongRunningProcessStatus;
 import org.apache.pulsar.client.admin.OffloadProcessStatus;
@@ -170,6 +172,21 @@ public class PersistentTopicsBase extends AdminResource {
 
     protected CompletableFuture<List<String>> internalGetListAsync(Optional<String> bundle,
                                                                    @Nullable Map<String, String> properties) {
+        return internalGetListAsync(null, bundle, properties);
+    }
+
+    /**
+     * Get the list of persistent topics in the namespace.
+     *
+     * @param asyncResponse the async response of the REST request. When provided, the topic list heap memory limit
+     *                      is applied and the permits are held until the response completes.
+     * @param bundle the optional bundle to filter the topics
+     * @param properties the topic listing properties (PIP-452)
+     * @return the list of topics
+     */
+    protected CompletableFuture<List<String>> internalGetListAsync(@Nullable AsyncResponse asyncResponse,
+                                                                   Optional<String> bundle,
+                                                                   @Nullable Map<String, String> properties) {
         return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPICS)
             .thenCompose(__ -> namespaceResources().namespaceExistsAsync(namespaceName))
             .thenAccept(exists -> {
@@ -177,8 +194,18 @@ public class PersistentTopicsBase extends AdminResource {
                     throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
                 }
             })
-            .thenCompose(__ -> pulsar().getNamespaceService().getListOfTopicsByProperties(namespaceName,
-               CommandGetTopicsOfNamespace.Mode.PERSISTENT, properties))
+            .thenCompose(__ -> {
+                Supplier<CompletableFuture<List<String>>> topicsSupplier = () -> pulsar().getNamespaceService()
+                        .getListOfTopicsByProperties(namespaceName, CommandGetTopicsOfNamespace.Mode.PERSISTENT,
+                                properties);
+                if (asyncResponse == null) {
+                    return topicsSupplier.get();
+                }
+                return TopicListResponseMemoryLimiter.listTopics(pulsar().getBrokerService(), asyncResponse,
+                        TopicListing.namespaceTopics(namespaceName.toString(),
+                                CommandGetTopicsOfNamespace.Mode.PERSISTENT),
+                        properties, topicsSupplier);
+            })
             .thenApply(topics ->
                 topics.stream()
                     .filter(topic -> {
@@ -197,6 +224,18 @@ public class PersistentTopicsBase extends AdminResource {
     }
 
     protected CompletableFuture<List<String>> internalGetPartitionedTopicListAsync() {
+        return internalGetPartitionedTopicListAsync(null);
+    }
+
+    /**
+     * Get the list of partitioned topics in the namespace.
+     *
+     * @param asyncResponse the async response of the REST request. When provided, the topic list heap memory limit
+     *                      is applied and the permits are held until the response completes.
+     * @return the list of partitioned topics
+     */
+    protected CompletableFuture<List<String>> internalGetPartitionedTopicListAsync(
+            @Nullable AsyncResponse asyncResponse) {
         return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPICS)
                 .thenCompose(__ -> namespaceResources().namespaceExistsAsync(namespaceName))
                 .thenCompose(namespaceExists -> {
@@ -206,9 +245,17 @@ public class PersistentTopicsBase extends AdminResource {
                                 .attr("namespace", namespaceName)
                                 .log("Failed to get partitioned topic list : Namespace does not exist");
                         throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
-                    } else {
-                        return getPartitionedTopicListAsync(TopicDomain.getEnum(domain()));
                     }
+                    TopicDomain topicDomain = TopicDomain.getEnum(domain());
+                    if (asyncResponse == null) {
+                        return getPartitionedTopicListAsync(topicDomain);
+                    }
+                    CommandGetTopicsOfNamespace.Mode mode = topicDomain == TopicDomain.persistent
+                            ? CommandGetTopicsOfNamespace.Mode.PERSISTENT
+                            : CommandGetTopicsOfNamespace.Mode.NON_PERSISTENT;
+                    return TopicListResponseMemoryLimiter.listTopics(pulsar().getBrokerService(), asyncResponse,
+                            TopicListing.partitionedTopics(namespaceName.toString(), mode), null,
+                            () -> getPartitionedTopicListAsync(topicDomain));
                 });
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiter.java
new file mode 100644
index 000000000000..7f11764c0b7a
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiter.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.impl;
+
+import jakarta.ws.rs.container.AsyncResponse;
+import jakarta.ws.rs.container.CompletionCallback;
+import jakarta.ws.rs.core.Response.Status;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
+import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
+import org.apache.pulsar.broker.web.RestException;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil;
+import org.apache.pulsar.common.semaphore.AsyncSemaphore;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.jspecify.annotations.Nullable;
+
+/**
+ * Applies the topic list heap memory limit (PIP-442) to REST topic listing requests.
+ *
+ * <p>The heap memory permits are held until the {@link AsyncResponse} has completed, since the topic list is
+ * referenced until the response has been written. The request is treated as cancelled once the response is done
+ * (for example, after the async response has timed out), so that a request that is no longer waited for doesn't
+ * list the topics, and permits that are granted after the response has completed are released immediately.
+ * A response can combine several listings, each of them holds its own permits until the response completes.
+ */
+public final class TopicListResponseMemoryLimiter {
+
+    /**
+     * The 503 response of a topic listing that was cancelled while it was waiting for permits, for example since
+     * the response had already completed.
+     */
+    static final class RequestCancelledException extends RestException {
+        private static final long serialVersionUID = 1L;
+
+        RequestCancelledException() {
+            super(Status.SERVICE_UNAVAILABLE, "Request was cancelled");
+        }
+    }
+
+    /**
+     * Checks whether a topic listing failed since it was cancelled while it was waiting for permits.
+     *
+     * @param ex the failure, possibly wrapped in a {@link CompletionException}
+     * @return true if the request was cancelled
+     */
+    public static boolean isRequestCancelled(Throwable ex) {
+        return FutureUtil.unwrapCompletionException(ex) instanceof RequestCancelledException;
+    }
+    private TopicListResponseMemoryLimiter() {
+    }
+
+    /**
+     * Lists topics while holding heap memory permits for the estimated size of the topic list.
+     *
+     * @param brokerService the broker service
+     * @param response the async response of the REST request, the permits are released when it completes
+     * @param listing the topic listing, which identifies the topic list size estimate
+     * @param properties the topic listing properties (PIP-452), null when not provided
+     * @param topicsSupplier supplier that performs the topic listing
+     * @return a future that completes with the topic list
+     */
+    public static CompletableFuture<List<String>> listTopics(BrokerService brokerService, AsyncResponse response,
+                                                             TopicListing listing,
+                                                             @Nullable Map<String, String> properties,
+                                                             Supplier<CompletableFuture<List<String>>>
+                                                                     topicsSupplier) {
+        return listTopicsAfterCollecting(brokerService, response, listing, properties,
+                () -> CompletableFuture.completedFuture(null), __ -> topicsSupplier.get());
+    }
+
+    /**
+     * Lists topics while holding heap memory permits for the estimated size of the topic list, after collecting
+     * part of the topics from other topic listings without holding permits.
+     *
+     * <p>Use this for a listing that combines topics from topic listings that apply the topic list memory limit
+     * themselves, such as the bundle topic listings of the bundle owners, which can be this broker, with topics
+     * that it lists itself. The size estimate is checked before collecting, so that a listing whose estimate is
+     * larger than the limit fails without collecting. The permits are acquired once for the whole topic list
+     * after collecting, so that the listing doesn't hold permits while waiting for the other listings.
+     *
+     * @param brokerService the broker service
+     * @param response the async response of the REST request, the permits are released when it completes
+     * @param listing the topic listing, which identifies the topic list size estimate
+     * @param properties the topic listing properties (PIP-452), null when not provided
+     * @param collector supplier that collects the topics from the other listings
+     * @param topicsSupplier function that performs the topic listing, given the collected topics
+     * @return a future that completes with the topic list
+     */
+    public static <CollectedT> CompletableFuture<List<String>> listTopicsAfterCollecting(
+            BrokerService brokerService, AsyncResponse response, TopicListing listing,
+            @Nullable Map<String, String> properties, Supplier<CompletableFuture<CollectedT>> collector,
+            Function<CollectedT, CompletableFuture<List<String>>> topicsSupplier) {
+        TopicListSizeResultCache sizeCache = brokerService.getTopicListSizeResultCache();
+        TopicListSizeResultCache.ResultHolder listSizeHolder = sizeCache.getTopicListSize(listing, properties);
+        ResponsePermits permits =
+                new ResponsePermits(brokerService.getMaxTopicListInFlightLimiter(), response, listSizeHolder);
+        response.register(permits);
+        return listSizeHolder.getSizeAsync()
+                .thenCompose(initialSize -> {
+                    if (initialSize > sizeCache.getSizeLimit()) {
+                        // the permits can't be granted, skip collecting
+                        return permits.acquire(initialSize).thenCompose(__ -> topicsSupplier.apply(null));
+                    }
+                    return collector.get().thenCompose(collected -> permits.acquire(initialSize)
+                            .thenCompose(__ -> topicsSupplier.apply(collected)));
+                })
+                .thenCompose(topicList -> {
+                    long actualSize = TopicListMemoryLimiter.estimateTopicListSize(topicList);
+                    listSizeHolder.updateSize(actualSize);
+                    return permits.update(actualSize).thenApply(___ -> topicList);
+                })
+                .whenComplete((__, t) -> {
+                    if (t != null) {
+                        // let concurrent requests waiting for the size estimate proceed
+                        listSizeHolder.resetIfInitializing();
+                    }
+                });
+    }
+
+    /**
+     * Lists topics by collecting them from other topic listings, and then holds heap memory permits for the
+     * collected topic list until the response completes.
+     *
+     * <p>Use this for a listing that gets its topics from topic listings that apply the topic list memory limit
+     * themselves, such as the bundle topic listings of the bundle owners, which can be this broker. Unlike
+     * {@link #listTopics}, no permits are held while the topics are collected: the collected topic lists are
+     * covered by the permits of the other listings until they have responded, and holding permits for an
+     * estimate here could leave too few permits for a listing on this broker that this request waits for.
+     *
+     * @param brokerService the broker service
+     * @param response the async response of the REST request, the permits are released when it completes
+     * @param topicsSupplier supplier that collects the topics from the other listings
+     * @return a future that completes with the topic list
+     */
+    public static CompletableFuture<List<String>> listTopicsFromOtherListings(
+            BrokerService brokerService, AsyncResponse response,
+            Supplier<CompletableFuture<List<String>>> topicsSupplier) {
+        ResponsePermits permits = new ResponsePermits(brokerService.getMaxTopicListInFlightLimiter(), response, null);
+        response.register(permits);
+        if (permits.isCancelled()) {
+            return CompletableFuture.failedFuture(ResponsePermits.cancelled());
+        }
+        return topicsSupplier.get().thenCompose(topicList ->
+                permits.acquire(TopicListMemoryLimiter.estimateTopicListSize(topicList)).thenApply(__ -> topicList));
+    }
+
+    /**
+     * Holds the heap memory permits of a single REST request until its {@link AsyncResponse} completes.
+     */
+    private static final class ResponsePermits implements CompletionCallback {
+        private final AsyncDualMemoryLimiter limiter;
+        private final AsyncResponse response;
+        private final TopicListSizeResultCache.ResultHolder listSizeHolder;
+        // guarded by this
+        private boolean completed;
+        // guarded by this
+        private AsyncDualMemoryLimiterPermit permit;
+
+        ResponsePermits(AsyncDualMemoryLimiter limiter, AsyncResponse response,
+                        TopicListSizeResultCache.ResultHolder listSizeHolder) {
+            this.limiter = limiter;
+            this.response = response;
+            this.listSizeHolder = listSizeHolder;
+        }
+
+        boolean isCancelled() {
+            synchronized (this) {
+                if (completed) {
+                    return true;
+                }
+            }
+            // the response is resumed only after the topic list has been retrieved, so a response that is
+            // already done has been cancelled or has timed out
+            return response.isDone();
+        }
+
+        CompletableFuture<Void> acquire(long size) {
+            return holdOrFail(() -> limiter.acquire(size, AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY,
+                    this::isCancelled));
+        }
+
+        CompletableFuture<Void> update(long size) {
+            AsyncDualMemoryLimiterPermit currentPermit;
+            synchronized (this) {
+                currentPermit = permit;
+            }
+            if (currentPermit == null) {
+                // the response has already completed and the permits have been released
+                return CompletableFuture.failedFuture(cancelled());
+            }
+            return holdOrFail(() -> limiter.update(currentPermit, size, this::isCancelled));
+        }
+
+        private CompletableFuture<Void> holdOrFail(
+                Supplier<CompletableFuture<AsyncDualMemoryLimiterPermit>> permitFutureSupplier) {
+            return AsyncDualMemoryLimiterUtil.requestPermits(permitFutureSupplier).handle((acquiredPermit, t) -> {
+                if (t != null) {
+                    String message = FutureUtil.unwrapCompletionException(t)
+                            instanceof AsyncSemaphore.PermitAcquireLimitExceededException
+                            ? "Failed since the topic list is larger than the topic list heap memory limit"
+                            : "Failed due to heap memory limit exceeded";
+                    throw new CompletionException(new RestException(Status.TOO_MANY_REQUESTS, message));
+                }
+                hold(acquiredPermit);
+                // checked after storing the permit: if the response completes concurrently, either onComplete or
+                // this check releases the permit
+                if (isCancelled()) {
+                    // don't continue processing a request that has been cancelled while waiting for the permits
+                    releaseHeldPermit();
+                    throw cancelled();
+                }
+                return null;
+            });
+        }
+
+        private static CompletionException cancelled() {
+            return new CompletionException(new RequestCancelledException());
+        }
+
+        /**
+         * Stores the permit so that it gets released when the response completes.
+         */
+        private void hold(AsyncDualMemoryLimiterPermit newPermit) {
+            AsyncDualMemoryLimiterPermit previousPermit;
+            synchronized (this) {
+                previousPermit = permit;
+                permit = newPermit;
+            }
+            if (previousPermit != null && previousPermit != newPermit) {
+                // after a permit update the previous permit has no remaining permits and releasing it is a no-op,
+                // it's released in case of an unexpected state to avoid leaking permits
+                limiter.release(previousPermit);
+            }
+        }
+
+        private void releaseHeldPermit() {
+            AsyncDualMemoryLimiterPermit permitToRelease;
+            synchronized (this) {
+                permitToRelease = permit;
+                permit = null;
+            }
+            if (permitToRelease != null) {
+                limiter.release(permitToRelease);
+            }
+        }
+
+        @Override
+        public void onComplete(Throwable throwable) {
+            if (throwable != null && listSizeHolder != null) {
+                listSizeHolder.resetIfInitializing();
+            }
+            synchronized (this) {
+                completed = true;
+            }
+            releaseHeldPermit();
+        }
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
index 183e2b1718d3..f68bb3efe0ec 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
@@ -147,10 +147,12 @@ public class Namespaces extends NamespacesBase {
                 .thenApply(topics -> filterSystemTopic(topics, includeSystemTopic))
                 .thenAccept(response::resume)
                 .exceptionally(ex -> {
-                    log.error()
-                            .attr("namespace", namespaceName)
-                            .exception(ex)
-                            .log("Failed to get topics list for namespace");
+                    if (!isTopicListLimitRejection(ex)) {
+                        log.error()
+                                .attr("namespace", namespaceName)
+                                .exception(ex)
+                                .log("Failed to get topics list for namespace");
+                    }
                     resumeAsyncResponseExceptionally(response, ex);
                     return null;
                 });
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java
index bc3a5d5d7c6a..d42be7bed089 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java
@@ -54,8 +54,12 @@ import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.admin.impl.TopicListResponseMemoryLimiter;
 import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
 import org.apache.pulsar.broker.web.RestException;
+import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
+import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
 import org.apache.pulsar.common.policies.data.EntryFilters;
@@ -451,8 +455,21 @@ public class NonPersistentTopics extends PersistentTopics {
             return;
         }
 
-        final List<CompletableFuture<List<String>>> futures = new ArrayList<>();
         final List<String> boundaries = policies.bundles.getBoundaries();
+        // the topics are collected from the bundle topic listings (getListFromBundle) of the bundle owners
+        TopicListResponseMemoryLimiter.listTopicsFromOtherListings(pulsar().getBrokerService(), asyncResponse,
+                () -> getNonPersistentTopicsFromBundles(boundaries, nsBundle))
+                .thenAccept(nonPersistentTopics ->
+                        asyncResponse.resume(filterSystemTopic(nonPersistentTopics, includeSystemTopic)))
+                .exceptionally(ex -> {
+                    resumeAsyncResponseExceptionally(asyncResponse, ex);
+                    return null;
+                });
+    }
+
+    private CompletableFuture<List<String>> getNonPersistentTopicsFromBundles(List<String> boundaries,
+                                                                              String nsBundle) {
+        final List<CompletableFuture<List<String>>> futures = new ArrayList<>();
         for (int i = 0; i < boundaries.size() - 1; i++) {
             final String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1));
             if (StringUtils.isNotBlank(nsBundle) && !nsBundle.equals(bundle)) {
@@ -466,28 +483,20 @@ public class NonPersistentTopics extends PersistentTopics {
                         .attr("bundle", bundle)
                         .exception(e)
                         .log("Failed to get list of topics under namespace");
-                asyncResponse.resume(new RestException(e));
-                return;
+                return CompletableFuture.failedFuture(new RestException(e));
             }
         }
-
-        FutureUtil.waitForAll(futures).whenComplete((result, ex) -> {
-            if (ex != null) {
-                resumeAsyncResponseExceptionally(asyncResponse, ex);
-            } else {
-                final List<String> topics = new ArrayList<>();
-                for (int i = 0; i < futures.size(); i++) {
-                    List<String> topicList = futures.get(i).join();
-                    if (topicList != null) {
-                        topics.addAll(topicList);
-                    }
+        return FutureUtil.waitForAll(futures).thenApply(__ -> {
+            final List<String> topics = new ArrayList<>();
+            for (int i = 0; i < futures.size(); i++) {
+                List<String> topicList = futures.get(i).join();
+                if (topicList != null) {
+                    topics.addAll(topicList);
                 }
-                final List<String> nonPersistentTopics =
-                        topics.stream()
-                                .filter(name -> !TopicName.get(name).isPersistent())
-                                .collect(Collectors.toList());
-                asyncResponse.resume(filterSystemTopic(nonPersistentTopics, includeSystemTopic));
             }
+            return topics.stream()
+                    .filter(name -> !TopicName.get(name).isPersistent())
+                    .collect(Collectors.toList());
         });
     }
 
@@ -534,24 +543,15 @@ public class NonPersistentTopics extends PersistentTopics {
                 asyncResponse.resume(Response.noContent().build());
             } else {
                 validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, true, true)
-                        .thenAccept(nsBundle -> {
-                            final var bundleTopics = pulsar().getBrokerService().getMultiLayerTopicsMap()
-                                    .get(namespaceName.toString());
-                            if (bundleTopics == null || bundleTopics.isEmpty()) {
-                                asyncResponse.resume(Collections.emptyList());
-                                return;
-                            }
-                            final List<String> topicList = new ArrayList<>();
-                            String bundleKey = namespaceName.toString() + "/" + nsBundle.getBundleRange();
-                            final var topicMap = bundleTopics.get(bundleKey);
-                            if (topicMap != null) {
-                                topicList.addAll(topicMap.keySet().stream()
-                                        .filter(name -> !TopicName.get(name).isPersistent())
-                                        .collect(Collectors.toList()));
-                            }
-                            asyncResponse.resume(topicList);
-                        }).exceptionally(ex -> {
-                            if (isNot307And404Exception(ex)) {
+                        .thenCompose(nsBundle -> TopicListResponseMemoryLimiter.listTopics(
+                                pulsar().getBrokerService(), asyncResponse,
+                                TopicListing.ownedBundleTopics(namespaceName.toString(),
+                                        CommandGetTopicsOfNamespace.Mode.NON_PERSISTENT, nsBundle.getBundleRange()),
+                                null,
+                                () -> CompletableFuture.completedFuture(getNonPersistentTopicsOfBundle(nsBundle))))
+                        .thenAccept(asyncResponse::resume)
+                        .exceptionally(ex -> {
+                            if (isNot307And404Exception(ex) && !isTopicListLimitRejection(ex)) {
                                 log.error()
                                         .attr("namespace", namespaceName)
                                         .attr("bundleRange", bundleRange)
@@ -701,6 +701,22 @@ public class NonPersistentTopics extends PersistentTopics {
                 });
     }
 
+    private List<String> getNonPersistentTopicsOfBundle(NamespaceBundle nsBundle) {
+        final var bundleTopics = pulsar().getBrokerService().getMultiLayerTopicsMap()
+                .get(namespaceName.toString());
+        if (bundleTopics == null || bundleTopics.isEmpty()) {
+            return Collections.emptyList();
+        }
+        String bundleKey = namespaceName.toString() + "/" + nsBundle.getBundleRange();
+        final var topicMap = bundleTopics.get(bundleKey);
+        if (topicMap == null) {
+            return Collections.emptyList();
+        }
+        return topicMap.keySet().stream()
+                .filter(name -> !TopicName.get(name).isPersistent())
+                .collect(Collectors.toList());
+    }
+
     private Topic getTopicReference(TopicName topicName) {
         try {
             return pulsar().getBrokerService().getTopicIfExists(topicName.toString())
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index e4b60620c56f..bdd63c81d46d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -135,10 +135,10 @@ public class PersistentTopics extends PersistentTopicsBase {
             @Parameter(description = "properties for customized topic listing plugin, format: k1=v1,k2=v2")
             @QueryParam("properties") String propertiesStr) {
         validateNamespaceName(tenant, namespace);
-        internalGetListAsync(Optional.ofNullable(bundle), parseProperties(propertiesStr))
+        internalGetListAsync(asyncResponse, Optional.ofNullable(bundle), parseProperties(propertiesStr))
             .thenAccept(topicList -> asyncResponse.resume(filterSystemTopic(topicList, includeSystemTopic)))
             .exceptionally(ex -> {
-                if (isNot307And404Exception(ex)) {
+                if (isNot307And404Exception(ex) && !isTopicListLimitRejection(ex)) {
                     log.error()
                             .attr("namespace", namespaceName)
                             .exception(ex)
@@ -172,11 +172,11 @@ public class PersistentTopics extends PersistentTopicsBase {
             @Parameter(description = "Include system topic")
             @QueryParam("includeSystemTopic") boolean includeSystemTopic) {
         validateNamespaceName(tenant, namespace);
-        internalGetPartitionedTopicListAsync()
+        internalGetPartitionedTopicListAsync(asyncResponse)
                 .thenAccept(partitionedTopicList -> asyncResponse.resume(
                         filterSystemTopic(partitionedTopicList, includeSystemTopic)))
                 .exceptionally(ex -> {
-                    if (isNot307And404Exception(ex)) {
+                    if (isNot307And404Exception(ex) && !isTopicListLimitRejection(ex)) {
                         log.error()
                                 .attr("namespace", namespaceName)
                                 .exception(ex)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
index c412bd93efd1..f1a606f3b364 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
@@ -54,6 +54,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.function.Function;
 import lombok.CustomLog;
 import org.apache.pulsar.broker.admin.AdminResource;
+import org.apache.pulsar.broker.admin.impl.TopicListResponseMemoryLimiter;
 import org.apache.pulsar.broker.resources.ScalableTopicMetadata;
 import org.apache.pulsar.broker.resources.ScalableTopicResources;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
@@ -61,6 +62,7 @@ import org.apache.pulsar.broker.service.scalable.AutoScaleConfig;
 import org.apache.pulsar.broker.service.scalable.ScalableTopicController;
 import org.apache.pulsar.broker.service.scalable.ScalableTopicService;
 import org.apache.pulsar.broker.service.scalable.SegmentTopicStatsBuilder;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
 import org.apache.pulsar.broker.web.RestException;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.common.naming.TopicDomain;
@@ -122,14 +124,19 @@ public class ScalableTopics extends AdminResource {
         validateNamespaceName(tenant, namespace);
         Map<String, String> propertyFilters = parseKeyValuePairs(properties);
         validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPICS)
-                .thenCompose(__ -> propertyFilters.isEmpty()
-                        ? resources().listScalableTopicsAsync(namespaceName)
-                        : resources().findScalableTopicsByPropertiesAsync(
-                                namespaceName, propertyFilters))
+                .thenCompose(__ -> TopicListResponseMemoryLimiter.listTopics(pulsar().getBrokerService(),
+                        asyncResponse,
+                        TopicListing.scalableTopics(namespaceName.toString()), propertyFilters,
+                        () -> propertyFilters.isEmpty()
+                                ? resources().listScalableTopicsAsync(namespaceName)
+                                : resources().findScalableTopicsByPropertiesAsync(
+                                        namespaceName, propertyFilters)))
                 .thenAccept(asyncResponse::resume)
                 .exceptionally(ex -> {
-                    log.error().attr("clientAppId", clientAppId()).attr("namespace", namespaceName)
-                            .exception(ex).log("Failed to list scalable topics");
+                    if (!isTopicListLimitRejection(ex)) {
+                        log.error().attr("clientAppId", clientAppId()).attr("namespace", namespaceName)
+                                .exception(ex).log("Failed to list scalable topics");
+                    }
                     resumeAsyncResponseExceptionally(asyncResponse, ex);
                     return null;
                 });
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index f9c2bc45d3f8..512696a2a2f8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -198,7 +198,9 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl;
 import org.apache.pulsar.common.protocol.schema.SchemaVersion;
 import org.apache.pulsar.common.scalable.SegmentLoadStats;
 import org.apache.pulsar.common.scalable.SegmentTopicName;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.apache.pulsar.common.semaphore.AsyncSemaphore;
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.FieldParser;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -279,7 +281,7 @@ public class BrokerService implements Closeable {
     @Getter
     private final AsyncDualMemoryLimiterImpl maxTopicListInFlightLimiter;
     @Getter
-    private final TopicListSizeResultCache topicListSizeResultCache = new TopicListSizeResultCache();
+    private final TopicListSizeResultCache topicListSizeResultCache;
     protected final AtomicReference<Semaphore> topicLoadRequestSemaphore;
 
     public static final String TOPIC_LOOKUP_USAGE_METRIC_NAME = "pulsar.broker.request.topic.lookup.concurrent.usage";
@@ -446,6 +448,11 @@ public class BrokerService implements Closeable {
                 pulsar.getConfiguration().getMaxTopicListInFlightDirectMemSizeMB() * 1024L * 1024L,
                 pulsar.getConfiguration().getMaxTopicListInFlightDirectMemSizePermitsAcquireQueueSize(),
                 pulsar.getConfiguration().getMaxTopicListInFlightDirectMemSizePermitsAcquireTimeoutMillis());
+        AsyncSemaphore topicListHeapLimiter =
+                maxTopicListInFlightLimiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY);
+        this.topicListSizeResultCache = new TopicListSizeResultCache(
+                pulsar.getConfiguration().getMaxTopicListInFlightHeapMemSizeMB() * 1024L * 1024L,
+                () -> topicListHeapLimiter.getQueueSize() == 0 && topicListHeapLimiter.getAcquiredPermits() == 0);
         this.topicLoadRequestSemaphore = new AtomicReference<>(
                 new Semaphore(pulsar.getConfiguration().getMaxConcurrentTopicLoadRequest(), false));
         if (pulsar.getConfiguration().getMaxUnackedMessagesPerBroker() > 0
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSender.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSender.java
index d98fc3585832..519cd444b471 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSender.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSender.java
@@ -24,6 +24,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.function.BooleanSupplier;
 import java.util.function.Function;
 import org.apache.bookkeeper.mledger.Entry;
 import org.apache.pulsar.client.api.transaction.TxnID;
@@ -100,15 +101,31 @@ public interface PulsarCommandSender {
 
     void sendEndTxnErrorResponse(long requestId, TxnID txnID, ServerError error, String message);
 
+    /**
+     * Sends the topic list watcher success response after acquiring direct memory permits for it.
+     *
+     * @param isCancelled supplier that returns true when the response is no longer needed, for example when the
+     *                    watcher has been closed. The response isn't sent when the connection is inactive or the
+     *                    request has been cancelled.
+     */
     CompletableFuture<Void> sendWatchTopicListSuccess(long requestId, long watcherId, String topicsHash,
                                                       Collection<String> topics,
                                                       Function<Throwable, CompletableFuture<Void>>
-                                                              permitAcquireErrorHandler);
+                                                              permitAcquireErrorHandler,
+                                                      BooleanSupplier isCancelled);
 
+    /**
+     * Sends a topic list watcher update after acquiring direct memory permits for it.
+     *
+     * @param isCancelled supplier that returns true when the update is no longer needed, for example when the
+     *                    watcher has been closed. The update isn't sent when the connection is inactive or the
+     *                    request has been cancelled.
+     */
     CompletableFuture<Void> sendWatchTopicListUpdate(long watcherId,
                                   List<String> newTopics, List<String> deletedTopics, String topicsHash,
                                   Function<Throwable, CompletableFuture<Void>>
-                                                             permitAcquireErrorHandler);
+                                                             permitAcquireErrorHandler,
+                                  BooleanSupplier isCancelled);
 
     /**
      * Send the response to a scalable-topic subscribe request on success. The caller is
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java
index 7faa0b58bb94..dcbe1e91b077 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java
@@ -28,6 +28,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.function.BooleanSupplier;
 import java.util.function.Function;
 import lombok.CustomLog;
 import org.apache.bookkeeper.mledger.Entry;
@@ -466,11 +467,12 @@ public class PulsarCommandSenderImpl implements PulsarCommandSender {
     public CompletableFuture<Void> sendWatchTopicListSuccess(long requestId, long watcherId, String topicsHash,
                                                              Collection<String> topics,
                                                              Function<Throwable, CompletableFuture<Void>>
-                                                                         permitAcquireErrorHandler) {
+                                                                         permitAcquireErrorHandler,
+                                                             BooleanSupplier isCancelled) {
         BaseCommand command = Commands.newWatchTopicListSuccess(requestId, watcherId, topicsHash, topics);
         safeIntercept(command, cnx);
-        return acquireDirectMemoryPermitsAndWriteAndFlush(cnx.ctx(), maxTopicListInFlightLimiter, () -> !cnx.isActive(),
-                command, permitAcquireErrorHandler);
+        return acquireDirectMemoryPermitsAndWriteAndFlush(cnx.ctx(), maxTopicListInFlightLimiter,
+                () -> !cnx.isActive() || isCancelled.getAsBoolean(), command, permitAcquireErrorHandler);
     }
 
     /***
@@ -481,11 +483,12 @@ public class PulsarCommandSenderImpl implements PulsarCommandSender {
     public CompletableFuture<Void> sendWatchTopicListUpdate(long watcherId, List<String> newTopics,
                                                             List<String> deletedTopics, String topicsHash,
                                                             Function<Throwable, CompletableFuture<Void>>
-                                                                        permitAcquireErrorHandler) {
+                                                                        permitAcquireErrorHandler,
+                                                            BooleanSupplier isCancelled) {
         BaseCommand command = Commands.newWatchTopicUpdate(watcherId, newTopics, deletedTopics, topicsHash);
         safeIntercept(command, cnx);
-        return acquireDirectMemoryPermitsAndWriteAndFlush(cnx.ctx(), maxTopicListInFlightLimiter, () -> !cnx.isActive(),
-                command, permitAcquireErrorHandler);
+        return acquireDirectMemoryPermitsAndWriteAndFlush(cnx.ctx(), maxTopicListInFlightLimiter,
+                () -> !cnx.isActive() || isCancelled.getAsBoolean(), command, permitAcquireErrorHandler);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 23f766feab7a..1e437b59d8f2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -201,6 +201,7 @@ import org.apache.pulsar.common.protocol.schema.SchemaVersion;
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil;
 import org.apache.pulsar.common.topics.TopicList;
 import org.apache.pulsar.common.topics.TopicsPattern;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -3642,8 +3643,8 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
                                                     Map<String, String> properties,
                                                     Semaphore lookupSemaphore) {
         BooleanSupplier isPermitRequestCancelled = () -> !ctx().channel().isActive();
-        TopicListSizeResultCache.ResultHolder
-                listSizeHolder = service.getTopicListSizeResultCache().getTopicListSize(namespaceName.toString(), mode);
+        TopicListSizeResultCache.ResultHolder listSizeHolder = service.getTopicListSizeResultCache()
+                .getTopicListSize(namespaceName.toString(), mode, properties);
         listSizeHolder.getSizeAsync().thenAccept(initialSize -> {
             maxTopicListInFlightLimiter.withAcquiredPermits(initialSize,
                     AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY, isPermitRequestCancelled, initialPermits -> {
@@ -3688,22 +3689,22 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
                                                 return commandSender.sendGetTopicsOfNamespaceResponse(filteredTopics,
                                                         hash,
                                                         filterTopics, !hashUnchanged, requestId, ex -> {
-                                                            log.warn()
-                                                                    .exceptionMessage(ex)
-                                                                    .log("Failed to acquire direct memory permits for "
-                                                                            + "GetTopicsOfNamespace");
-                                                            commandSender.sendErrorResponse(requestId,
-                                                                    ServerError.TooManyRequests,
-                                                                    "Cannot acquire permits for direct memory");
+                                                            if (!isTopicListRequestCancelled(ex,
+                                                                    "Failed to acquire direct memory permits for "
+                                                                            + "GetTopicsOfNamespace")) {
+                                                                sendTopicListPermitsErrorResponse(requestId, ex,
+                                                                        true, "direct",
+                                                                        "Cannot acquire permits for direct memory");
+                                                            }
                                                             return CompletableFuture.completedFuture(null);
                                                         });
                                             }, t -> {
-                                                log.warn()
-                                                        .exceptionMessage(t)
-                                                        .log("Failed to acquire heap memory permits for "
-                                                                + "GetTopicsOfNamespace");
-                                                writeAndFlush(Commands.newError(requestId, ServerError.TooManyRequests,
-                                                        "Failed due to heap memory limit exceeded"));
+                                                if (!isTopicListRequestCancelled(t,
+                                                        "Failed to acquire heap memory permits for "
+                                                                + "GetTopicsOfNamespace")) {
+                                                    sendTopicListPermitsErrorResponse(requestId, t, true,
+                                                            "heap", "Failed due to heap memory limit exceeded");
+                                                }
                                                 return CompletableFuture.completedFuture(null);
                                             });
                                 }).whenComplete((__, ___) -> {
@@ -3720,18 +3721,53 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
                                     return null;
                                 });
                     }, t -> {
-                        log.warn()
-                                .exceptionMessage(t)
-                                .log("Failed to acquire initial heap memory permits for GetTopicsOfNamespace");
                         listSizeHolder.resetIfInitializing();
-                        writeAndFlush(Commands.newError(requestId, ServerError.TooManyRequests,
-                                "Failed due to heap memory limit exceeded"));
+                        if (!isTopicListRequestCancelled(t,
+                                "Failed to acquire initial heap memory permits for GetTopicsOfNamespace")) {
+                            sendTopicListPermitsErrorResponse(requestId, t, listSizeHolder.isMeasured(), "heap",
+                                    "Failed due to heap memory limit exceeded");
+                        }
                         lookupSemaphore.release();
                         return CompletableFuture.completedFuture(null);
                     });
         });
     }
 
+    /**
+     * Responds to a GetTopicsOfNamespace request that couldn't acquire topic list memory permits. A topic list that
+     * has been measured to be larger than the limit gets a non-retriable error, since a retry fails the same way
+     * until the list is measured again. A permit timeout or a full queue, or an estimate larger than the limit for
+     * a topic list that hasn't been measured, gets {@link ServerError#TooManyRequests}, which the client retries.
+     */
+    private void sendTopicListPermitsErrorResponse(long requestId, Throwable t, boolean topicListMeasured,
+                                                   String memoryType, String tooManyRequestsMessage) {
+        if (topicListMeasured && AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(t)) {
+            commandSender.sendErrorResponse(requestId, ServerError.NotAllowedError,
+                    "Failed since the topic list is larger than the topic list " + memoryType + " memory limit");
+        } else {
+            commandSender.sendErrorResponse(requestId, ServerError.TooManyRequests, tooManyRequestsMessage);
+        }
+    }
+
+    /**
+     * Logs a failure to acquire topic list memory permits. A request that was cancelled, because the connection
+     * was closed, is logged at debug level, since there is no client to respond to.
+     *
+     * @return true if the request was cancelled and no error response should be sent
+     */
+    private boolean isTopicListRequestCancelled(Throwable t, String failureMessage) {
+        if (AsyncDualMemoryLimiterUtil.isPermitAcquireCancelled(t)) {
+            log.debug()
+                    .attr("remoteAddress", remoteAddress)
+                    .log("GetTopicsOfNamespace was cancelled since the connection was closed");
+            return true;
+        }
+        log.warn()
+                .exceptionMessage(t)
+                .log(failureMessage);
+        return false;
+    }
+
 
     @Override
     protected void handleGetSchema(CommandGetSchema commandGetSchema) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
index 124122c8ee58..299168b382c9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
@@ -37,6 +37,7 @@ import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -56,6 +57,7 @@ import org.apache.pulsar.common.api.proto.ServerError;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil;
 import org.apache.pulsar.common.semaphore.AsyncSemaphore;
 import org.apache.pulsar.common.topics.TopicList;
 import org.apache.pulsar.common.topics.TopicsPattern;
@@ -81,12 +83,16 @@ public class TopicListService {
         private final Executor executor;
         private volatile boolean closed = false;
         private boolean sendingInProgress;
+        // the number of watch request success responses being sent, topic changes are sent after them
+        private int pendingSuccessResponses;
         private final BlockingDeque<Runnable> sendTopicListUpdateTasks;
         @Getter(onMethod_ = @VisibleForTesting)
         private boolean updatingTopics;
         private List<String> matchingTopicsBeforeDisconnected;
         private boolean disconnected;
-        private List<Runnable> updateCallbacks = new LinkedList<>();
+        // callbacks of requests waiting for the ongoing topic list update, called with a snapshot of the matching
+        // topics when the update completes, or with the failure
+        private List<BiConsumer<Collection<String>, Throwable>> updateCallbacks = new LinkedList<>();
         private boolean updatingWhileDisconnected;
 
         public TopicListWatcher(TopicListService topicListService, long id,
@@ -168,6 +174,15 @@ public class TopicListService {
                 sendTopicListUpdateTasks.clear();
                 return;
             }
+            if (updatingTopics) {
+                // The update calls this again when it completes. Until then, topic changes are queued, since they
+                // would be sent with the hash of the partially rebuilt matching topics.
+                return;
+            }
+            if (pendingSuccessResponses > 0) {
+                // the last success response to complete calls this again, topic changes are sent after it
+                return;
+            }
             // Execute the next task
             Runnable task = sendTopicListUpdateTasks.poll();
             if (task != null) {
@@ -177,20 +192,104 @@ public class TopicListService {
             }
         }
 
-        public synchronized void close() {
-            closed = true;
-            sendTopicListUpdateTasks.clear();
-            updateCallbacks.clear();
+        public void close() {
+            List<BiConsumer<Collection<String>, Throwable>> pendingCallbacks;
+            synchronized (this) {
+                closed = true;
+                sendTopicListUpdateTasks.clear();
+                pendingCallbacks = new ArrayList<>(updateCallbacks);
+                updateCallbacks.clear();
+            }
+            // complete the requests waiting for the topic list update so that they don't wait forever
+            if (!pendingCallbacks.isEmpty()) {
+                IllegalStateException closedException = newClosedException();
+                for (BiConsumer<Collection<String>, Throwable> callback : pendingCallbacks) {
+                    runUpdateCallback(callback, null, closedException);
+                }
+            }
+        }
+
+        /**
+         * Completes the requests waiting for the ongoing update with the given failure. The update continues.
+         */
+        /**
+         * Called before sending the success response of a watch request. Topic changes are queued until
+         * {@link #successResponseSendingCompleted()} has been called for each success response, so that they are
+         * sent after it.
+         */
+        synchronized void successResponseSendingStarted() {
+            pendingSuccessResponses++;
+            sendingInProgress = true;
+        }
+
+        synchronized void successResponseSendingCompleted() {
+            if (pendingSuccessResponses > 0) {
+                pendingSuccessResponses--;
+            }
+            sendingCompleted();
+        }
+
+        void failUpdateCallbacks(Throwable failure) {
+            List<BiConsumer<Collection<String>, Throwable>> pendingCallbacks;
+            synchronized (this) {
+                pendingCallbacks = new ArrayList<>(updateCallbacks);
+                updateCallbacks.clear();
+            }
+            for (BiConsumer<Collection<String>, Throwable> callback : pendingCallbacks) {
+                runUpdateCallback(callback, null, failure);
+            }
+        }
+
+        /**
+         * Calls the callback with a snapshot of the matching topics. While a topic list update is in progress, the
+         * matching topics aren't complete, so the callback is called when the update completes.
+         */
+        void whenMatchingTopicsAvailable(BiConsumer<Collection<String>, Throwable> callback) {
+            Collection<String> snapshot;
+            synchronized (this) {
+                if (!closed && updatingTopics) {
+                    updateCallbacks.add(callback);
+                    return;
+                }
+                snapshot = closed ? null : new ArrayList<>(matchingTopics);
+            }
+            runUpdateCallback(callback, snapshot, snapshot == null ? newClosedException() : null);
+        }
+
+        private IllegalStateException newClosedException() {
+            return new IllegalStateException("Topic list watcher " + id + " was closed");
+        }
+
+        private static void runUpdateCallback(BiConsumer<Collection<String>, Throwable> callback,
+                                              Collection<String> matchingTopics, Throwable failure) {
+            try {
+                callback.accept(matchingTopics, failure);
+            } catch (Exception e) {
+                log.warn()
+                        .attr("callback", callback)
+                        .exception(e)
+                        .log("Error executing topic list update callback");
+            }
         }
 
         /**
          * Returns true if the topic list update is prepared for execution. It is expected that the caller initiates
          * the update. The callback is registered to be executed after the update, either existing or upcoming is
          * completed.
-         * @param afterUpdateCompletionCallback callback to be executed after the update is completed.
+         * @param afterUpdateCompletionCallback callback to be executed after the update is completed. It's called
+         *                                      with a snapshot of the matching topics when the update completes,
+         *                                      or with the failure when the update fails or the watcher is closed
+         *                                      before the update completes.
          * @return true if an existing update wasn't ongoing and a new update is prepared for execution.
          */
-        synchronized boolean prepareUpdateTopics(Runnable afterUpdateCompletionCallback) {
+        synchronized boolean prepareUpdateTopics(
+                BiConsumer<Collection<String>, Throwable> afterUpdateCompletionCallback) {
+            if (closed) {
+                if (afterUpdateCompletionCallback != null) {
+                    runUpdateCallback(afterUpdateCompletionCallback, null, newClosedException());
+                }
+                return false;
+            }
             if (!updatingTopics) {
                 updatingTopics = true;
                 sendingInProgress = true;
@@ -208,29 +307,32 @@ public class TopicListService {
             }
         }
 
-        synchronized void updateTopics(List<String> topics) {
-            if (closed) {
-                return;
-            }
-            matchingTopics.clear();
-            TopicList.filterTopicsToStream(topics, topicsPattern).forEach(matchingTopics::add);
-            updatingTopics = false;
-            if (disconnected) {
-                handleNewAndDeletedTopicsWhileDisconnected();
-                matchingTopicsBeforeDisconnected = null;
-                disconnected = false;
-            }
-            for (Runnable callback : updateCallbacks) {
-                try {
-                    callback.run();
-                } catch (Exception e) {
-                    log.warn()
-                            .attr("callback", callback)
-                            .exception(e)
-                            .log("Error executing topic list update callback");
+        void updateTopics(List<String> topics) {
+            List<BiConsumer<Collection<String>, Throwable>> callbacks;
+            Collection<String> snapshot;
+            synchronized (this) {
+                if (closed) {
+                    return;
+                }
+                matchingTopics.clear();
+                TopicList.filterTopicsToStream(topics, topicsPattern).forEach(matchingTopics::add);
+                updatingTopics = false;
+                if (disconnected) {
+                    handleNewAndDeletedTopicsWhileDisconnected();
+                    matchingTopicsBeforeDisconnected = null;
+                    disconnected = false;
                 }
+                callbacks = new ArrayList<>(updateCallbacks);
+                updateCallbacks.clear();
+                // the waiting requests respond with the result of this update, also if the matching topics change
+                // after the lock has been released, for example when another update starts
+                snapshot = callbacks.isEmpty() ? null : new ArrayList<>(matchingTopics);
+            }
+            // the callbacks run without holding the lock, since a callback can close the watcher, for example when
+            // sending the response of a waiting request fails
+            for (BiConsumer<Collection<String>, Throwable> callback : callbacks) {
+                runUpdateCallback(callback, snapshot, null);
             }
-            updateCallbacks.clear();
             sendingCompleted();
         }
 
@@ -379,23 +481,24 @@ public class TopicListService {
         CompletableFuture<TopicListWatcher> watcherFuture = new CompletableFuture<>();
         CompletableFuture<TopicListWatcher> existingWatcherFuture = watchers.putIfAbsent(watcherId, watcherFuture);
 
+        // the watcher and the snapshot of its matching topics to respond with
+        CompletableFuture<WatcherTopics> watcherTopicsFuture;
         if (existingWatcherFuture != null) {
             log.debug()
                     .attr("connection", connection)
                     .attr("watcherId", watcherId)
                     .log("Watcher with the same watcherId is already created. Refreshing.");
             // use the existing watcher if it's already created
-            watcherFuture = existingWatcherFuture.thenCompose(watcher -> {
-                CompletableFuture<TopicListWatcher> future = new CompletableFuture<>();
-                Runnable callback = () -> future.complete(watcher);
+            watcherTopicsFuture = existingWatcherFuture.thenCompose(watcher -> {
+                CompletableFuture<WatcherTopics> future = new CompletableFuture<>();
+                BiConsumer<Collection<String>, Throwable> callback = watcherTopicsCallback(watcher, future);
                 // trigger a new update unless an update is already ongoing. Register the callback to complete
-                // when the update completes.
+                // when the update completes, or when the watcher is closed before that.
                 if (watcher.prepareUpdateTopics(callback)) {
                     updateTopicListWatcher(watcher)
-                            // run the callback also in failure cases
-                            // prepareUpdateTopics handles it for success cases
+                            // complete the request also when the update fails
                             .exceptionally(ex -> {
-                                callback.run();
+                                callback.accept(null, ex);
                                 return null;
                             });
                 }
@@ -403,11 +506,20 @@ public class TopicListService {
             });
         } else {
             initializeTopicsListWatcher(watcherFuture, namespaceName, watcherId, topicsPattern);
+            watcherTopicsFuture = watcherFuture.thenCompose(watcher -> {
+                CompletableFuture<WatcherTopics> future = new CompletableFuture<>();
+                // waits for an update that started after the watcher was created, for example after a metadata
+                // store session event, since the matching topics are incomplete until the update completes
+                watcher.whenMatchingTopicsAvailable(watcherTopicsCallback(watcher, future));
+                return future;
+            });
         }
 
-        CompletableFuture<TopicListWatcher> finalWatcherFuture = watcherFuture;
-        finalWatcherFuture.thenAccept(watcher -> {
-                    Collection<String> topicList = watcher.getMatchingTopics();
+        // a failed request removes the watcher that it created, not an existing watcher
+        CompletableFuture<TopicListWatcher> createdWatcherFuture = existingWatcherFuture == null ? watcherFuture : null;
+        watcherTopicsFuture.thenAccept(watcherTopics -> {
+                    TopicListWatcher watcher = watcherTopics.watcher();
+                    Collection<String> topicList = watcherTopics.matchingTopics();
                     String hash = TopicList.calculateHash(topicList);
                     if (hash.equals(topicsHash)) {
                         topicList = Collections.emptyList();
@@ -417,48 +529,105 @@ public class TopicListService {
                             .attr("namespace", namespaceName)
                             .attr("requestId", requestId)
                             .log("Received WatchTopicList");
+                    watcher.successResponseSendingStarted();
                     sendTopicListSuccessWithPermitAcquiringRetries(watcherId, requestId, topicList, hash,
-                            watcher::sendingCompleted, watcher::close);
+                            watcher::successResponseSendingCompleted, failure -> {
+                                if (!isCancellation(failure)) {
+                                    // the watcher can't be used without the success response
+                                    deleteTopicListWatcher(watcherId);
+                                }
+                                watcher.successResponseSendingCompleted();
+                                // the request is waiting for a response unless the connection was closed
+                                if (connection.isActive()) {
+                                    connection.getCommandSender().sendErrorResponse(requestId,
+                                            getWatchTopicListErrorCode(failure), failure.getMessage());
+                                }
+                            });
                     lookupSemaphore.release();
                 })
                 .exceptionally(ex -> {
-                    log.warn()
-                            .attr("connection", connection)
-                            .attr("namespace", namespaceName)
-                            .attr("requestId", requestId)
-                            .exceptionMessage(ex)
-                            .log("Error WatchTopicList");
-                    connection.getCommandSender().sendErrorResponse(requestId,
-                            BrokerServiceException.getClientErrorCode(
-                                    new BrokerServiceException.ServerMetadataException(ex)), ex.getMessage());
-                    watchers.remove(watcherId, finalWatcherFuture);
+                    if (connection.isActive()) {
+                        log.warn()
+                                .attr("connection", connection)
+                                .attr("namespace", namespaceName)
+                                .attr("requestId", requestId)
+                                .exceptionMessage(ex)
+                                .log("Error WatchTopicList");
+                        connection.getCommandSender().sendErrorResponse(requestId,
+                                getWatchTopicListErrorCode(ex), ex.getMessage());
+                    } else {
+                        // the connection was closed, there's no client to respond to
+                        log.debug()
+                                .attr("connection", connection)
+                                .attr("namespace", namespaceName)
+                                .attr("requestId", requestId)
+                                .exceptionMessage(ex)
+                                .log("WatchTopicList was cancelled since the connection was closed");
+                    }
+                    if (createdWatcherFuture != null && watchers.remove(watcherId, createdWatcherFuture)) {
+                        // the request can fail after it created the watcher and registered it as a topic listener,
+                        // for example when an update that started after the watcher was created fails
+                        createdWatcherFuture.thenAccept(createdWatcher -> {
+                            topicResources.deregisterPersistentTopicListener(createdWatcher);
+                            createdWatcher.close();
+                        });
+                    }
                     lookupSemaphore.release();
                     return null;
                 });
     }
 
+    /**
+     * A watcher and a snapshot of its matching topics.
+     */
+    private record WatcherTopics(TopicListWatcher watcher, Collection<String> matchingTopics) {
+    }
+
+    private static BiConsumer<Collection<String>, Throwable> watcherTopicsCallback(
+            TopicListWatcher watcher, CompletableFuture<WatcherTopics> future) {
+        return (matchingTopics, failure) -> {
+            if (failure == null) {
+                future.complete(new WatcherTopics(watcher, matchingTopics));
+            } else {
+                future.completeExceptionally(failure);
+            }
+        };
+    }
+
     private void sendTopicListSuccessWithPermitAcquiringRetries(long watcherId, long requestId,
                                                                 Collection<String> topicList,
                                                                 String hash,
                                                                 Runnable successfulCompletionCallback,
-                                                                Runnable failedCompletionCallback) {
-        performOperationWithPermitAcquiringRetries(watcherId, "topic list success", permitAcquireErrorHandler ->
+                                                                Consumer<Throwable> failedCompletionCallback) {
+        // the success response is the response to the watch request, so a cancelled success response is reported
+        // as a failure, so that the request can be completed with an error response
+        performOperationWithPermitAcquiringRetries(watcherId, "topic list success", true, permitAcquireErrorHandler ->
                 () -> connection.getCommandSender()
-                        .sendWatchTopicListSuccess(requestId, watcherId, hash, topicList, permitAcquireErrorHandler)
-                        .whenComplete((__, t) -> {
-                            if (t != null) {
-                                // this is an unexpected case
-                                log.warn()
-                                        .attr("connection", connection)
-                                        .attr("watcherId", watcherId)
-                                        .exception(t)
-                                        .log("Failed to send topic list success. Watcher is not active.");
-                                failedCompletionCallback.run();
-                            } else {
-                                // completed successfully, run the callback
-                                successfulCompletionCallback.run();
-                            }
-                        }));
+                        .sendWatchTopicListSuccess(requestId, watcherId, hash, topicList, permitAcquireErrorHandler,
+                                () -> !watchers.containsKey(watcherId)))
+                // handled on the future of the whole operation, which also completes when the watcher was removed
+                // before the first attempt and after retries
+                .whenComplete((__, t) -> {
+                    if (t != null) {
+                        Throwable failure = FutureUtil.unwrapCompletionException(t);
+                        if (isCancellation(failure)) {
+                            log.debug()
+                                    .attr("connection", connection)
+                                    .attr("watcherId", watcherId)
+                                    .log("Topic list success wasn't sent since the watcher was closed");
+                        } else {
+                            log.warn()
+                                    .attr("connection", connection)
+                                    .attr("watcherId", watcherId)
+                                    .exception(t)
+                                    .log("Failed to send topic list success. Watcher is not active.");
+                        }
+                        failedCompletionCallback.accept(failure);
+                    } else {
+                        // completed successfully, run the callback
+                        successfulCompletionCallback.run();
+                    }
+                });
     }
 
     /***
@@ -498,12 +667,20 @@ public class TopicListService {
                             () -> initializeTopicsListWatcher(watcherFuture, namespace, watcherId, topicsPattern),
                             retryAfterMillis, TimeUnit.MILLISECONDS);
                 } else {
-                    log.warn()
-                            .attr("connection", connection)
-                            .attr("watcherId", watcherId)
-                            .attr("namespace", namespace)
-                            .exception(unwrappedException)
-                            .log("Failed to initialize topic list watcher");
+                    if (isCancellation(unwrappedException)) {
+                        log.debug()
+                                .attr("connection", connection)
+                                .attr("watcherId", watcherId)
+                                .attr("namespace", namespace)
+                                .log("Topic list watcher initialization was cancelled");
+                    } else {
+                        log.warn()
+                                .attr("connection", connection)
+                                .attr("watcherId", watcherId)
+                                .attr("namespace", namespace)
+                                .exception(unwrappedException)
+                                .log("Failed to initialize topic list watcher");
+                    }
                     watcherFuture.completeExceptionally(unwrappedException);
                 }
             } else {
@@ -569,6 +746,12 @@ public class TopicListService {
                                     }
                                 }, connection.ctx().executor());
                             }, CompletableFuture::failedFuture)
+                    .whenComplete((__, t) -> {
+                        if (t != null) {
+                            // let concurrent requests waiting for the size estimate proceed
+                            listSizeHolder.resetIfInitializing();
+                        }
+                    })
                     .thenApplyAsync(Function.identity(), connection.ctx().executor());
         });
     }
@@ -589,10 +772,35 @@ public class TopicListService {
         getTopics(namespace, watcherId).whenComplete((topics, exception) -> {
             if (exception != null) {
                 Throwable unwrappedException = FutureUtil.unwrapCompletionException(exception);
-                if (connection.isActive() && !watcher.closed
-                        && (unwrappedException instanceof AsyncSemaphore.PermitAcquireTimeoutException
-                        || unwrappedException instanceof AsyncSemaphore.PermitAcquireQueueFullException)) {
-                    // retry with backoff if permit acquisition fails due to timeout or queue full
+                if (!connection.isActive() || watcher.closed || isCancellation(unwrappedException)) {
+                    // the connection or the watcher was closed, closing the watcher completes the requests
+                    // waiting for the update
+                    log.debug()
+                            .attr("connection", connection)
+                            .attr("watcherId", watcherId)
+                            .attr("namespace", namespace)
+                            .log("Topic list watcher update was cancelled");
+                    future.completeExceptionally(unwrappedException);
+                } else if (unwrappedException instanceof AsyncSemaphore.PermitAcquireLimitExceededException) {
+                    // the topic list can't be listed within the topic list memory limit, retrying wouldn't help.
+                    // The watcher can't send updates without a successful update, so close it.
+                    log.warn()
+                            .attr("connection", connection)
+                            .attr("watcherId", watcherId)
+                            .attr("namespace", namespace)
+                            .exceptionMessage(unwrappedException)
+                            .log("Closing topic list watcher since the topic list is larger than the topic list "
+                                    + "memory limit");
+                    // fail the requests waiting for the update with this failure before closing the watcher fails
+                    // them with a watcher closed error, so that they get a non-retriable error response
+                    watcher.failUpdateCallbacks(unwrappedException);
+                    deleteTopicListWatcher(watcherId);
+                    future.completeExceptionally(unwrappedException);
+                } else {
+                    // Retry with backoff. Until the update completes, the watcher queues the topic changes and
+                    // doesn't send updates, so giving up would leave the watcher without updates.
+                    // Requests waiting for the update get an error response instead of waiting for the retries.
+                    watcher.failUpdateCallbacks(unwrappedException);
                     long retryAfterMillis = this.retryBackoff.next().toMillis();
                     log.info()
                             .attr("connection", connection)
@@ -600,18 +808,10 @@ public class TopicListService {
                             .attr("watcherId", watcherId)
                             .attr("namespace", namespace)
                             .attr("retryAfterMillis", retryAfterMillis)
-                            .log("when updating topic list watcher for namespace. Retrying");
+                            .log("Failed to update topic list watcher for namespace. Retrying");
                     connection.ctx().executor()
                             .schedule(() -> internalUpdateTopicListWatcher(watcher, future),
                                     retryAfterMillis, TimeUnit.MILLISECONDS);
-                } else {
-                    log.warn()
-                            .attr("connection", connection)
-                            .attr("watcherId", watcherId)
-                            .attr("namespace", namespace)
-                            .exception(unwrappedException)
-                            .log("Failed to update topic list watcher");
-                    future.completeExceptionally(unwrappedException);
                 }
             } else {
                 watcher.updateTopics(topics);
@@ -673,27 +873,32 @@ public class TopicListService {
      */
     public void sendTopicListUpdate(long watcherId, String topicsHash, List<String> deletedTopics,
                                     List<String> newTopics, Runnable completionCallback) {
-        performOperationWithPermitAcquiringRetries(watcherId, "topic list update", permitAcquireErrorHandler ->
+        performOperationWithPermitAcquiringRetries(watcherId, "topic list update", false,
+                permitAcquireErrorHandler ->
                 () -> connection.getCommandSender()
                         .sendWatchTopicListUpdate(watcherId, newTopics, deletedTopics, topicsHash,
-                                permitAcquireErrorHandler)
-                        .whenComplete((__, t) -> {
-                            if (t != null) {
-                                // this is an unexpected case
-                                log.warn()
-                                        .attr("connection", connection)
-                                        .attr("watcherId", watcherId)
-                                        .exception(t)
-                                        .log("Failed to send topic list update for watcherId=. Watcher will be in "
-                                                + "inconsistent state.");
-                            }
-                            completionCallback.run();
-                        }));
+                                permitAcquireErrorHandler, () -> !watchers.containsKey(watcherId)))
+                // handled on the future of the whole operation, so that the callback runs once also after retries
+                .whenComplete((__, t) -> {
+                    if (t != null) {
+                        log.warn()
+                                .attr("connection", connection)
+                                .attr("watcherId", watcherId)
+                                .exception(t)
+                                .log("Failed to send topic list update for watcherId=. Watcher will be in "
+                                        + "inconsistent state.");
+                    }
+                    completionCallback.run();
+                });
     }
 
     // performs an operation with infinite permit acquiring retries.
-    // If acquiring permits fails, it will retry after a backoff period
-    private void performOperationWithPermitAcquiringRetries(long watcherId, String operationName,
+    // If acquiring permits fails, it will retry after a backoff period.
+    // When failOnCancellation is true, an operation that is skipped because the connection was closed or the
+    // watcher was removed completes exceptionally with PermitAcquireCancelledException instead of successfully.
+    // The returned future completes when the operation has completed, including the retries.
+    private CompletableFuture<Void> performOperationWithPermitAcquiringRetries(long watcherId, String operationName,
+                                                            boolean failOnCancellation,
                                                             Function<Function<Throwable, CompletableFuture<Void>>,
                                                                     Supplier<CompletableFuture<Void>>>
                                                                     asyncOperationFactory) {
@@ -702,36 +907,43 @@ public class TopicListService {
         AtomicReference<Supplier<CompletableFuture<Void>>> operationRef = new AtomicReference<>();
         // create the error handler for the operation
         Function<Throwable, CompletableFuture<Void>> permitAcquireErrorHandler =
-                createPermitAcquireErrorHandler(watcherId, operationName, () -> operationRef.get().get());
+                createPermitAcquireErrorHandler(watcherId, operationName, failOnCancellation,
+                        () -> operationRef.get().get());
         // create the async operation using the factory function. Pass the error handler to the factory function.
         Supplier<CompletableFuture<Void>> asyncOperation = asyncOperationFactory.apply(permitAcquireErrorHandler);
         // set the operation to run into the operation reference
         operationRef.set(() -> {
             if (!connection.isActive() || !watchers.containsKey(watcherId)) {
                 // do nothing if the connection has already been closed or the watcher has been removed
-                return CompletableFuture.completedFuture(null);
+                return failOnCancellation ? CompletableFuture.failedFuture(newWatcherClosedException(watcherId))
+                        : CompletableFuture.completedFuture(null);
             }
             return asyncOperation.get().thenRun(() -> retryBackoff.reset());
         });
         // run the operation
-        operationRef.get().get();
+        return operationRef.get().get();
     }
 
     // retries acquiring permits until the connection is closed or the watcher is removed
     private Function<Throwable, CompletableFuture<Void>> createPermitAcquireErrorHandler(long watcherId,
                                                                                          String operationName,
+                                                                                         boolean failOnCancellation,
                                                                                          Supplier<CompletableFuture
                                                                                                  <Void>> operationRef) {
         ScheduledExecutorService scheduledExecutor = connection.ctx().channel().eventLoop();
         AtomicInteger retryCount = new AtomicInteger(0);
         return t -> {
             Throwable unwrappedException = FutureUtil.unwrapCompletionException(t);
-            if (unwrappedException instanceof AsyncSemaphore.PermitAcquireCancelledException
-                    || unwrappedException instanceof AsyncSemaphore.PermitAcquireAlreadyClosedException
+            if (isCancellation(unwrappedException)
                     || !connection.isActive()
                     || !watchers.containsKey(watcherId)) {
-                // stop retrying and complete successfully
-                return CompletableFuture.completedFuture(null);
+                // stop retrying
+                return failOnCancellation ? CompletableFuture.failedFuture(newWatcherClosedException(watcherId))
+                        : CompletableFuture.completedFuture(null);
+            }
+            if (unwrappedException instanceof AsyncSemaphore.PermitAcquireLimitExceededException) {
+                // the command is larger than the direct memory limit, retrying wouldn't help
+                return CompletableFuture.failedFuture(unwrappedException);
             }
             long retryDelay = retryBackoff.next().toMillis();
             retryCount.incrementAndGet();
@@ -749,6 +961,28 @@ public class TopicListService {
         };
     }
 
+    /**
+     * Maps a failure of a watch request to the error response. A topic list or a response that is larger than the
+     * topic list memory limit gets a non-retriable error, since a retry fails the same way until the topic list is
+     * measured again.
+     */
+    private static ServerError getWatchTopicListErrorCode(Throwable failure) {
+        if (AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(failure)) {
+            return ServerError.NotAllowedError;
+        }
+        return BrokerServiceException.getClientErrorCode(new BrokerServiceException.ServerMetadataException(failure));
+    }
+
+    private static AsyncSemaphore.PermitAcquireCancelledException newWatcherClosedException(long watcherId) {
+        return new AsyncSemaphore.PermitAcquireCancelledException("Topic list watcher " + watcherId
+                + " was closed");
+    }
+
+    private static boolean isCancellation(Throwable unwrappedException) {
+        return unwrappedException instanceof AsyncSemaphore.PermitAcquireCancelledException
+                || unwrappedException instanceof AsyncSemaphore.PermitAcquireAlreadyClosedException;
+    }
+
     @VisibleForTesting
     CompletableFuture<TopicListWatcher> getWatcherFuture(long watcherId) {
         return watchers.get(watcherId);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicListRestMemoryLimitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicListRestMemoryLimitTest.java
new file mode 100644
index 000000000000..3618dbdbb2b8
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicListRestMemoryLimitTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.doReturn;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
+import org.apache.pulsar.broker.namespace.NamespaceService;
+import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
+import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
+import org.apache.pulsar.client.admin.ListNamespaceTopicsOptions;
+import org.apache.pulsar.client.admin.ListTopicsOptions;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicDomain;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.LimitType;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Verifies that the REST API topic listing endpoints apply the topic list heap memory limit (PIP-442).
+ */
+@Test(groups = "broker-admin")
+public class TopicListRestMemoryLimitTest extends MockedPulsarServiceBaseTest {
+    private static final String NAMESPACE = "public/default";
+
+    @Override
+    protected void doInitConf() throws Exception {
+        super.doInitConf();
+        conf.setMaxTopicListInFlightHeapMemSizeMB(1);
+        conf.setMaxTopicListInFlightHeapMemSizePermitsAcquireTimeoutMillis(200);
+    }
+
+    @BeforeClass
+    @Override
+    protected void setup() throws Exception {
+        super.internalSetup();
+        setupDefaultTenantAndNamespace();
+    }
+
+    @AfterClass(alwaysRun = true)
+    @Override
+    protected void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    @Test
+    public void testTopicListingEndpointsApplyHeapMemoryLimit() throws Exception {
+        String persistentTopic = "persistent://" + NAMESPACE + "/limited-persistent";
+        String partitionedTopic = "persistent://" + NAMESPACE + "/limited-partitioned";
+        String nonPersistentTopic = "non-persistent://" + NAMESPACE + "/limited-non-persistent";
+        String scalableTopic = "topic://" + NAMESPACE + "/limited-scalable";
+        admin.topics().createNonPartitionedTopic(persistentTopic);
+        admin.topics().createPartitionedTopic(partitionedTopic, 2);
+        admin.scalableTopics().createScalableTopic(scalableTopic, 1, Map.of("team", "a"));
+        // a connected producer keeps the non-persistent topic and its bundle loaded
+        @Cleanup
+        Producer<byte[]> producer = pulsarClient.newProducer().topic(nonPersistentTopic).create();
+        String bundleRange = admin.lookups().getBundleRange(nonPersistentTopic);
+
+        Map<String, ThrowingCallable> listings = Map.of(
+                "namespaces topics", () -> admin.namespaces().getTopics(NAMESPACE),
+                "persistent topics", () -> admin.topics().getList(NAMESPACE),
+                "persistent partitioned topics", () -> admin.topics().getPartitionedTopicList(NAMESPACE),
+                "non-persistent topics", () -> admin.topics().getList(NAMESPACE, TopicDomain.non_persistent),
+                "non-persistent bundle topics", () -> admin.topics().getListInBundle(NAMESPACE, bundleRange),
+                "scalable topics", () -> admin.scalableTopics().listScalableTopics(NAMESPACE),
+                "scalable topics by properties",
+                () -> admin.scalableTopics().listScalableTopicsByProperties(NAMESPACE, Map.of("team", "a")));
+
+        AsyncDualMemoryLimiterImpl limiter = pulsar.getBrokerService().getMaxTopicListInFlightLimiter();
+        long maxHeapPermits = limiter.getLimiter(LimitType.HEAP_MEMORY).getAvailablePermits();
+        // occupy all heap memory permits so that topic listings can't acquire permits
+        AsyncDualMemoryLimiterPermit permit = limiter.acquire(maxHeapPermits, LimitType.HEAP_MEMORY, () -> false)
+                .get(5, TimeUnit.SECONDS);
+        try {
+            listings.forEach((name, listing) -> assertThatThrownBy(listing)
+                    .as(name)
+                    .isInstanceOfSatisfying(PulsarAdminException.class,
+                            e -> assertThat(e.getStatusCode()).as(name).isEqualTo(429)));
+        } finally {
+            limiter.release(permit);
+        }
+
+        // the listings succeed when permits are available
+        assertThat(admin.namespaces().getTopics(NAMESPACE)).contains(persistentTopic);
+        assertThat(admin.topics().getList(NAMESPACE)).contains(persistentTopic);
+        assertThat(admin.topics().getPartitionedTopicList(NAMESPACE)).contains(partitionedTopic);
+        assertThat(admin.topics().getList(NAMESPACE, TopicDomain.non_persistent)).contains(nonPersistentTopic);
+        assertThat(admin.topics().getListInBundle(NAMESPACE, bundleRange)).contains(nonPersistentTopic);
+        assertThat(admin.scalableTopics().listScalableTopics(NAMESPACE)).contains(scalableTopic);
+        assertThat(admin.scalableTopics().listScalableTopicsByProperties(NAMESPACE, Map.of("team", "a")))
+                .isEqualTo(List.of(scalableTopic));
+        // the permits are released after the responses have been written
+        assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isZero();
+    }
+
+    /**
+     * The non-persistent topic listings collect the topics from the bundle topic listings of the bundle owners,
+     * which are on this broker. They must not hold permits while waiting for those listings, otherwise the size
+     * estimates of the two listings could exceed the limit together and the listings would time out on each other.
+     */
+    @Test
+    public void testNonPersistentListingsDontWaitForTheirOwnPermits() throws Exception {
+        String nonPersistentTopic = "non-persistent://" + NAMESPACE + "/nested-listing";
+        @Cleanup
+        Producer<byte[]> producer = pulsarClient.newProducer().topic(nonPersistentTopic).create();
+        String bundleRange = admin.lookups().getBundleRange(nonPersistentTopic);
+
+        AsyncDualMemoryLimiterImpl limiter = pulsar.getBrokerService().getMaxTopicListInFlightLimiter();
+        long maxHeapPermits = limiter.getLimiter(LimitType.HEAP_MEMORY).getAvailablePermits();
+        long estimate = maxHeapPermits * 6 / 10;
+        TopicListSizeResultCache sizeCache = pulsar.getBrokerService().getTopicListSizeResultCache();
+        for (int i = 0; i < 3; i++) {
+            // estimates from earlier, larger topic lists: each listing alone fits within the limit, both don't
+            sizeCache.getTopicListSize(TopicListing.ownedBundleTopics(NAMESPACE, Mode.NON_PERSISTENT, bundleRange),
+                    null).updateSize(estimate);
+            sizeCache.getTopicListSize(TopicListing.namespaceTopics(NAMESPACE, Mode.NON_PERSISTENT), null)
+                    .updateSize(estimate);
+            sizeCache.getTopicListSize(TopicListing.namespaceTopics(NAMESPACE, Mode.ALL), null)
+                    .updateSize(estimate);
+
+            assertThat(admin.topics().getList(NAMESPACE, TopicDomain.non_persistent)).contains(nonPersistentTopic);
+            assertThat(admin.topics().getList(NAMESPACE, TopicDomain.non_persistent,
+                    ListTopicsOptions.builder().bundle(bundleRange).build())).contains(nonPersistentTopic);
+            for (org.apache.pulsar.client.admin.Mode mode : List.of(org.apache.pulsar.client.admin.Mode.NON_PERSISTENT,
+                    org.apache.pulsar.client.admin.Mode.ALL)) {
+                assertThat(admin.namespaces().getTopics(NAMESPACE,
+                        ListNamespaceTopicsOptions.builder().mode(mode).build()))
+                        .as(mode.name()).contains(nonPersistentTopic);
+            }
+        }
+        assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isZero();
+    }
+
+    /**
+     * The ALL mode collects the non-persistent topics from the bundle listings, which acquire permits from the
+     * same limiter on this broker, before it acquires permits for the whole topic list. Holding the permits of a
+     * large persistent topic list while waiting for the bundle listings could leave too few permits for them.
+     */
+    @Test
+    public void testAllModeDoesNotHoldPermitsWhileWaitingForBundleListings() throws Exception {
+        String nonPersistentTopic = "non-persistent://" + NAMESPACE + "/all-mode-listing";
+        @Cleanup
+        Producer<byte[]> producer = pulsarClient.newProducer().topic(nonPersistentTopic).create();
+        String bundleRange = admin.lookups().getBundleRange(nonPersistentTopic);
+
+        AsyncDualMemoryLimiterImpl limiter = pulsar.getBrokerService().getMaxTopicListInFlightLimiter();
+        long maxHeapPermits = limiter.getLimiter(LimitType.HEAP_MEMORY).getAvailablePermits();
+        // a persistent topic list of about 60% of the limit
+        List<String> persistentTopics = new ArrayList<>();
+        String longName = "t".repeat(10_000);
+        while (TopicListMemoryLimiter.estimateTopicListSize(persistentTopics) < maxHeapPermits * 6 / 10) {
+            persistentTopics.add("persistent://" + NAMESPACE + "/" + longName + persistentTopics.size());
+        }
+        NamespaceService namespaceService = pulsar.getNamespaceService();
+        doReturn(CompletableFuture.completedFuture(persistentTopics))
+                .when(namespaceService).getListOfPersistentTopics(NamespaceName.get(NAMESPACE));
+        try {
+            TopicListSizeResultCache sizeCache = pulsar.getBrokerService().getTopicListSizeResultCache();
+            sizeCache.getTopicListSize(TopicListing.namespaceTopics(NAMESPACE, Mode.ALL), null)
+                    .updateSize(TopicListMemoryLimiter.estimateTopicListSize(persistentTopics));
+            // the bundle listing's estimate fits within the limit, but not together with the persistent topic list
+            sizeCache.getTopicListSize(TopicListing.ownedBundleTopics(NAMESPACE, Mode.NON_PERSISTENT, bundleRange),
+                    null).updateSize(maxHeapPermits * 6 / 10);
+
+            for (int i = 0; i < 3; i++) {
+                assertThat(admin.namespaces().getTopics(NAMESPACE, ListNamespaceTopicsOptions.builder()
+                        .mode(org.apache.pulsar.client.admin.Mode.ALL).build()))
+                        .contains(nonPersistentTopic, persistentTopics.get(0));
+            }
+        } finally {
+            doCallRealMethod().when(namespaceService).getListOfPersistentTopics(any());
+        }
+        assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isZero();
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiterTest.java
new file mode 100644
index 000000000000..0fa068c70c55
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiterTest.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import jakarta.ws.rs.container.AsyncResponse;
+import jakarta.ws.rs.container.CompletionCallback;
+import jakarta.ws.rs.core.Response;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
+import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
+import org.apache.pulsar.broker.web.RestException;
+import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.LimitType;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class TopicListResponseMemoryLimiterTest {
+    private static final String NAMESPACE = "tenant/ns";
+    private static final long MAX_HEAP_MEMORY = 100_000;
+    private static final List<String> TOPICS =
+            List.of("persistent://tenant/ns/topic-1", "persistent://tenant/ns/topic-2");
+
+    private AsyncDualMemoryLimiterImpl limiter;
+    private TopicListSizeResultCache sizeCache;
+    private BrokerService brokerService;
+    private AsyncResponse response;
+    private AtomicBoolean responseDone;
+    private AtomicReference<CompletionCallback> completionCallback;
+
+    @BeforeMethod
+    public void setup() {
+        limiter = new AsyncDualMemoryLimiterImpl(MAX_HEAP_MEMORY, 10, 5000, MAX_HEAP_MEMORY, 10, 5000);
+        sizeCache = new TopicListSizeResultCache();
+        brokerService = mock(BrokerService.class);
+        when(brokerService.getMaxTopicListInFlightLimiter()).thenAnswer(__ -> limiter);
+        when(brokerService.getTopicListSizeResultCache()).thenReturn(sizeCache);
+        responseDone = new AtomicBoolean();
+        completionCallback = new AtomicReference<>();
+        response = mock(AsyncResponse.class);
+        when(response.isDone()).thenAnswer(__ -> responseDone.get());
+        doAnswer(invocation -> {
+            completionCallback.set(invocation.getArgument(0));
+            return List.of();
+        }).when(response).register(any(Object.class));
+    }
+
+    @AfterMethod(alwaysRun = true)
+    public void cleanup() {
+        if (limiter != null) {
+            limiter.close();
+            limiter = null;
+        }
+    }
+
+    private long acquiredHeapPermits() {
+        return limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits();
+    }
+
+    private void completeResponse(Throwable throwable) {
+        responseDone.set(true);
+        completionCallback.get().onComplete(throwable);
+    }
+
+    private CompletableFuture<List<String>> listTopics(Supplier<CompletableFuture<List<String>>> topicsSupplier) {
+        return TopicListResponseMemoryLimiter.listTopics(brokerService, response,
+                TopicListing.namespaceTopics(NAMESPACE, Mode.PERSISTENT), null, topicsSupplier);
+    }
+
+    @Test
+    public void testPermitsAreHeldUntilResponseCompletes() {
+        CompletableFuture<List<String>> result = listTopics(() -> CompletableFuture.completedFuture(TOPICS));
+
+        assertThat(result).succeedsWithin(Duration.ofSeconds(5)).isEqualTo(TOPICS);
+        // the topic list is referenced until the response has been written
+        assertThat(acquiredHeapPermits()).isEqualTo(TopicListMemoryLimiter.estimateTopicListSize(TOPICS));
+
+        completeResponse(null);
+        assertThat(acquiredHeapPermits()).isZero();
+    }
+
+    @Test
+    public void testDoesNotListTopicsWhenResponseIsAlreadyDone() {
+        responseDone.set(true);
+        AtomicInteger listingCalls = new AtomicInteger();
+
+        CompletableFuture<List<String>> result = listTopics(() -> {
+            listingCalls.incrementAndGet();
+            return CompletableFuture.completedFuture(TOPICS);
+        });
+
+        assertThat(result).failsWithin(Duration.ofSeconds(5)).withThrowableThat()
+                .havingRootCause().isInstanceOf(RestException.class);
+        assertThat(listingCalls).hasValue(0);
+        assertThat(acquiredHeapPermits()).isZero();
+        // concurrent requests waiting for the size estimate must not be left waiting
+        assertThat(sizeCache.getTopicListSize(NAMESPACE, Mode.PERSISTENT).getSizeAsync()).isDone();
+    }
+
+    @Test
+    public void testDoesNotListTopicsWhenResponseTimesOutWhileWaitingForPermits() throws Exception {
+        // occupy all permits so that the request waits in the queue
+        AsyncDualMemoryLimiterPermit blockingPermit =
+                limiter.acquire(MAX_HEAP_MEMORY, LimitType.HEAP_MEMORY, () -> false).get(5, TimeUnit.SECONDS);
+        AtomicInteger listingCalls = new AtomicInteger();
+        CompletableFuture<List<String>> result = listTopics(() -> {
+            listingCalls.incrementAndGet();
+            return CompletableFuture.completedFuture(TOPICS);
+        });
+        Awaitility.await().until(() -> limiter.getLimiter(LimitType.HEAP_MEMORY).getQueueSize() == 1);
+
+        // an async response timeout completes the response with a mapped response, without a throwable
+        completeResponse(null);
+        limiter.release(blockingPermit);
+
+        assertThat(result).failsWithin(Duration.ofSeconds(5));
+        assertThat(listingCalls).hasValue(0);
+        assertThat(acquiredHeapPermits()).isZero();
+    }
+
+    @Test
+    public void testPermitsAreReleasedWhenResponseCompletesDuringListing() {
+        CompletableFuture<List<String>> result = listTopics(() -> {
+            assertThat(acquiredHeapPermits()).isPositive();
+            completeResponse(null);
+            return CompletableFuture.completedFuture(TOPICS);
+        });
+
+        assertThat(result).failsWithin(Duration.ofSeconds(5));
+        assertThat(acquiredHeapPermits()).isZero();
+    }
+
+    @Test
+    public void testPermitGrantedAfterResponseCompletionIsReleased() {
+        // simulate a permit grant that races with the completion of the response
+        AsyncDualMemoryLimiterImpl mockedLimiter = mock(AsyncDualMemoryLimiterImpl.class);
+        AsyncDualMemoryLimiterPermit permit = mock(AsyncDualMemoryLimiterPermit.class);
+        when(mockedLimiter.acquire(anyLong(), any(), any())).thenAnswer(__ -> {
+            completionCallback.get().onComplete(null);
+            return CompletableFuture.completedFuture(permit);
+        });
+        when(brokerService.getMaxTopicListInFlightLimiter()).thenReturn(mockedLimiter);
+        AtomicInteger listingCalls = new AtomicInteger();
+
+        CompletableFuture<List<String>> result = listTopics(() -> {
+            listingCalls.incrementAndGet();
+            return CompletableFuture.completedFuture(TOPICS);
+        });
+
+        assertThat(result).failsWithin(Duration.ofSeconds(5));
+        assertThat(listingCalls).hasValue(0);
+        verify(mockedLimiter, times(1)).release(permit);
+    }
+
+    @Test
+    public void testPermitGrantedAfterResponseIsDoneIsReleased() {
+        // simulate a permit grant that races with the cancellation of the response, before the completion
+        // callback has been called
+        AsyncDualMemoryLimiterImpl mockedLimiter = mock(AsyncDualMemoryLimiterImpl.class);
+        AsyncDualMemoryLimiterPermit permit = mock(AsyncDualMemoryLimiterPermit.class);
+        when(mockedLimiter.acquire(anyLong(), any(), any())).thenAnswer(__ -> {
+            responseDone.set(true);
+            return CompletableFuture.completedFuture(permit);
+        });
+        when(brokerService.getMaxTopicListInFlightLimiter()).thenReturn(mockedLimiter);
+        AtomicInteger listingCalls = new AtomicInteger();
+
+        CompletableFuture<List<String>> result = listTopics(() -> {
+            listingCalls.incrementAndGet();
+            return CompletableFuture.completedFuture(TOPICS);
+        });
+
+        assertThat(result).failsWithin(Duration.ofSeconds(5));
+        assertThat(listingCalls).hasValue(0);
+        verify(mockedLimiter, times(1)).release(permit);
+        // the cancellation is recognized as the limiter's own 503, unlike other 503 responses
+        Throwable failure = result.handle((__, t) -> t).join();
+        assertThat(TopicListResponseMemoryLimiter.isRequestCancelled(failure)).isTrue();
+        assertThat(TopicListResponseMemoryLimiter.isRequestCancelled(
+                new RestException(Response.Status.SERVICE_UNAVAILABLE, "not ready"))).isFalse();
+    }
+
+    @Test
+    public void testTopicListLargerThanLimitFailsWithTooManyRequests() {
+        List<String> hugeTopicList = List.of("persistent://tenant/ns/" + "t".repeat((int) MAX_HEAP_MEMORY));
+
+        // the limiter rejects the update to the actual size synchronously
+        CompletableFuture<List<String>> result =
+                listTopics(() -> CompletableFuture.completedFuture(hugeTopicList));
+        assertThat(result).failsWithin(Duration.ofSeconds(5)).withThrowableThat()
+                .havingRootCause().isInstanceOfSatisfying(RestException.class,
+                        e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+        completeResponse(new RuntimeException("failed"));
+        assertThat(acquiredHeapPermits()).isZero();
+
+        // the cached estimate is now larger than the limit and the initial acquisition is rejected synchronously
+        responseDone.set(false);
+        CompletableFuture<List<String>> result2 = listTopics(() -> CompletableFuture.completedFuture(TOPICS));
+        assertThat(result2).failsWithin(Duration.ofSeconds(5)).withThrowableThat()
+                .havingRootCause().isInstanceOfSatisfying(RestException.class,
+                        e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+    }
+
+    @Test
+    public void testPermitAcquireFailureFailsWithTooManyRequests() throws Exception {
+        limiter.close();
+        // a queue size of 1 with a queued request causes the next request to fail
+        limiter = new AsyncDualMemoryLimiterImpl(MAX_HEAP_MEMORY, 1, 5000, MAX_HEAP_MEMORY, 1, 5000);
+        AsyncDualMemoryLimiterPermit blockingPermit =
+                limiter.acquire(MAX_HEAP_MEMORY, LimitType.HEAP_MEMORY, () -> false).get(5, TimeUnit.SECONDS);
+        CompletableFuture<AsyncDualMemoryLimiterPermit> queued =
+                limiter.acquire(1, LimitType.HEAP_MEMORY, () -> false);
+
+        CompletableFuture<List<String>> result = listTopics(() -> CompletableFuture.completedFuture(TOPICS));
+
+        assertThat(result).failsWithin(Duration.ofSeconds(5)).withThrowableThat()
+                .havingRootCause().isInstanceOfSatisfying(RestException.class,
+                        e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+        limiter.release(blockingPermit);
+        limiter.release(queued.get(5, TimeUnit.SECONDS));
+        assertThat(acquiredHeapPermits()).isZero();
+    }
+
+    @Test
+    public void testListTopicsFromOtherListingsHoldsPermitsOnlyAfterListing() {
+        AtomicLong permitsDuringListing = new AtomicLong(-1);
+        CompletableFuture<List<String>> result = TopicListResponseMemoryLimiter.listTopicsFromOtherListings(
+                brokerService, response, () -> {
+                    permitsDuringListing.set(acquiredHeapPermits());
+                    return CompletableFuture.completedFuture(TOPICS);
+                });
+
+        assertThat(result).succeedsWithin(Duration.ofSeconds(5)).isEqualTo(TOPICS);
+        // no permits are held while the topics are collected from the other listings
+        assertThat(permitsDuringListing).hasValue(0);
+        // permits for the collected list are held until the response has been written
+        assertThat(acquiredHeapPermits()).isEqualTo(TopicListMemoryLimiter.estimateTopicListSize(TOPICS));
+        completeResponse(null);
+        assertThat(acquiredHeapPermits()).isZero();
+    }
+
+    @Test
+    public void testListTopicsFromOtherListingsSkipsListingWhenResponseIsAlreadyDone() {
+        responseDone.set(true);
+        AtomicInteger listingCalls = new AtomicInteger();
+
+        CompletableFuture<List<String>> result = TopicListResponseMemoryLimiter.listTopicsFromOtherListings(
+                brokerService, response, () -> {
+                    listingCalls.incrementAndGet();
+                    return CompletableFuture.completedFuture(TOPICS);
+                });
+
+        assertThat(result).failsWithin(Duration.ofSeconds(5));
+        assertThat(listingCalls).hasValue(0);
+        assertThat(acquiredHeapPermits()).isZero();
+    }
+
+    @Test
+    public void testOversizedListingFailsWithoutWaitingForPermits() throws Exception {
+        sizeCache = new TopicListSizeResultCache(MAX_HEAP_MEMORY, () -> true);
+        when(brokerService.getTopicListSizeResultCache()).thenReturn(sizeCache);
+        TopicListSizeResultCache.ResultHolder holder =
+                sizeCache.getTopicListSize(NAMESPACE, Mode.PERSISTENT);
+        // a previous listing measured a topic list larger than the limit
+        holder.updateSize(MAX_HEAP_MEMORY * 2);
+        // another topic listing is in flight
+        AsyncDualMemoryLimiterPermit otherListingPermit =
+                limiter.acquire(1, LimitType.HEAP_MEMORY, () -> false).get(5, TimeUnit.SECONDS);
+        AtomicInteger listingCalls = new AtomicInteger();
+
+        long startNanos = System.nanoTime();
+        CompletableFuture<List<String>> result = listTopics(() -> {
+            listingCalls.incrementAndGet();
+            return CompletableFuture.completedFuture(TOPICS);
+        });
+
+        // fails right away with 429 instead of waiting in the queue until no other topic list is in flight
+        assertThat(result).failsWithin(Duration.ofSeconds(2)).withThrowableThat()
+                .havingRootCause().isInstanceOfSatisfying(RestException.class,
+                        e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+        assertThat(Duration.ofNanos(System.nanoTime() - startNanos)).isLessThan(Duration.ofSeconds(2));
+        assertThat(listingCalls).hasValue(0);
+        assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getQueueSize()).isZero();
+        limiter.release(otherListingPermit);
+    }
+
+    @Test
+    public void testListTopicsAfterCollectingSkipsCollectingWhenEstimateIsLargerThanLimit() {
+        sizeCache = new TopicListSizeResultCache(MAX_HEAP_MEMORY, () -> true);
+        when(brokerService.getTopicListSizeResultCache()).thenReturn(sizeCache);
+        TopicListing listing = TopicListing.namespaceTopics(NAMESPACE, Mode.ALL);
+        // a previous listing measured a topic list larger than the limit
+        sizeCache.getTopicListSize(listing, null).updateSize(MAX_HEAP_MEMORY * 2);
+        AtomicInteger collectorCalls = new AtomicInteger();
+        AtomicInteger listingCalls = new AtomicInteger();
+
+        CompletableFuture<List<String>> result = TopicListResponseMemoryLimiter.listTopicsAfterCollecting(
+                brokerService, response, listing, null, () -> {
+                    collectorCalls.incrementAndGet();
+                    return CompletableFuture.completedFuture(TOPICS);
+                }, collected -> {
+                    listingCalls.incrementAndGet();
+                    return CompletableFuture.completedFuture(collected);
+                });
+
+        assertThat(result).failsWithin(Duration.ofSeconds(2)).withThrowableThat()
+                .havingRootCause().isInstanceOfSatisfying(RestException.class,
+                        e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+        assertThat(collectorCalls).hasValue(0);
+        assertThat(listingCalls).hasValue(0);
+    }
+
+    @Test
+    public void testListTopicsAfterCollectingAcquiresPermitsAfterCollecting() {
+        AtomicLong permitsWhileCollecting = new AtomicLong(-1);
+        CompletableFuture<List<String>> result = TopicListResponseMemoryLimiter.listTopicsAfterCollecting(
+                brokerService, response, TopicListing.namespaceTopics(NAMESPACE, Mode.ALL), null, () -> {
+                    permitsWhileCollecting.set(acquiredHeapPermits());
+                    return CompletableFuture.completedFuture(TOPICS);
+                }, CompletableFuture::completedFuture);
+
+        assertThat(result).succeedsWithin(Duration.ofSeconds(5)).isEqualTo(TOPICS);
+        assertThat(permitsWhileCollecting).hasValue(0);
+        assertThat(acquiredHeapPermits()).isEqualTo(TopicListMemoryLimiter.estimateTopicListSize(TOPICS));
+        completeResponse(null);
+        assertThat(acquiredHeapPermits()).isZero();
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
index 6722440e0ca7..5e871e500780 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
@@ -30,6 +30,7 @@ import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.clearInvocations;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.matches;
@@ -128,6 +129,8 @@ import org.apache.pulsar.broker.service.scalable.ScalableTopicService;
 import org.apache.pulsar.broker.service.schema.SchemaRegistryService;
 import org.apache.pulsar.broker.service.utils.ClientChannelHelper;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
+import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
+import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
 import org.apache.pulsar.broker.web.RestException;
 import org.apache.pulsar.client.api.ProducerAccessMode;
 import org.apache.pulsar.client.api.PulsarClientException;
@@ -187,6 +190,8 @@ import org.apache.pulsar.common.protocol.schema.EmptyVersion;
 import org.apache.pulsar.common.protocol.schema.SchemaData;
 import org.apache.pulsar.common.protocol.schema.SchemaVersion;
 import org.apache.pulsar.common.scalable.HashRange;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
 import org.apache.pulsar.common.topics.TopicList;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap;
@@ -4979,4 +4984,127 @@ public class ServerCnxTest {
         channel.finish();
     }
 
+
+    @Test(timeOut = 30000)
+    public void testGetTopicsOfNamespaceLargerThanHeapMemoryLimit() throws Exception {
+        long topicListSize = TopicListMemoryLimiter.estimateTopicListSize(topics);
+        // both the initial size estimate and the topic list are larger than the heap memory limit
+        AsyncDualMemoryLimiterImpl limiter =
+                new AsyncDualMemoryLimiterImpl(topicListSize - 1, 10, 1000, 1_000_000, 10, 1000);
+        try {
+            doReturn(limiter).when(brokerService).getMaxTopicListInFlightLimiter();
+            doReturn(new TopicListSizeResultCache()).when(brokerService).getTopicListSizeResultCache();
+            resetChannel();
+            setChannelConnected();
+            var lookupSemaphore = brokerService.getLookupRequestSemaphore();
+            int initialPermits = lookupSemaphore.availablePermits();
+            for (long requestId = 1; requestId <= 3; requestId++) {
+                channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+                        "use/ns-abc", requestId, CommandGetTopicsOfNamespace.Mode.ALL, null, null, null));
+                // the request is rejected instead of being left without a response. The topic list hasn't been
+                // measured, so the error is retriable.
+                CommandError error = (CommandError) getResponse();
+                assertEquals(error.getRequestId(), requestId);
+                assertEquals(error.getError(), ServerError.TooManyRequests);
+                // the lookup permit is released
+                assertEquals(lookupSemaphore.availablePermits(), initialPermits);
+            }
+        } finally {
+            channel.finish();
+            limiter.close();
+        }
+    }
+
+    @Test(timeOut = 30000)
+    public void testGetTopicsOfNamespaceWithNewPropertiesAndOversizedEstimateIsRetriable() throws Exception {
+        long topicListSize = TopicListMemoryLimiter.estimateTopicListSize(topics);
+        AsyncDualMemoryLimiterImpl limiter =
+                new AsyncDualMemoryLimiterImpl(topicListSize - 1, 10, 1000, 1_000_000, 10, 1000);
+        try {
+            doReturn(limiter).when(brokerService).getMaxTopicListInFlightLimiter();
+            // the limiter is never idle, so an oversized estimate isn't measured again within the test
+            doReturn(new TopicListSizeResultCache(topicListSize - 1, () -> false))
+                    .when(brokerService).getTopicListSizeResultCache();
+            resetChannel();
+            setChannelConnected();
+            clearInvocations(namespaceService);
+            // the listing without properties is measured to be larger than the limit
+            channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+                    "use/ns-abc", 1, CommandGetTopicsOfNamespace.Mode.ALL, null, null, null));
+            CommandError error = (CommandError) getResponse();
+            assertEquals(error.getRequestId(), 1);
+            assertEquals(error.getError(), ServerError.NotAllowedError);
+
+            // a listing with new properties starts from that estimate, but its own topic list hasn't been measured,
+            // so it gets a retriable error
+            channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+                    "use/ns-abc", 2, CommandGetTopicsOfNamespace.Mode.ALL, null, null, Map.of("label", "x")));
+            error = (CommandError) getResponse();
+            assertEquals(error.getRequestId(), 2);
+            assertEquals(error.getError(), ServerError.TooManyRequests);
+            verify(namespaceService, times(1)).getListOfUserTopicsByProperties(any(), any(), any());
+        } finally {
+            channel.finish();
+            limiter.close();
+        }
+    }
+
+    @Test(timeOut = 30000)
+    public void testGetTopicsOfNamespacePermitTimeoutIsRetriable() throws Exception {
+        long heapMemoryLimit = 1_000_000;
+        // the heap memory limit fits the topic list, but all of it is held by another request
+        AsyncDualMemoryLimiterImpl limiter =
+                new AsyncDualMemoryLimiterImpl(heapMemoryLimit, 10, 100, 1_000_000, 10, 1000);
+        AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit heldPermit = limiter.acquire(heapMemoryLimit,
+                AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY, () -> false).get(10, TimeUnit.SECONDS);
+        try {
+            doReturn(limiter).when(brokerService).getMaxTopicListInFlightLimiter();
+            doReturn(new TopicListSizeResultCache()).when(brokerService).getTopicListSizeResultCache();
+            resetChannel();
+            setChannelConnected();
+            channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+                    "use/ns-abc", 1, CommandGetTopicsOfNamespace.Mode.ALL, null, null, null));
+            // a permit timeout gets an error that the client retries, unlike a topic list larger than the limit
+            CommandError error = (CommandError) getResponse();
+            assertEquals(error.getRequestId(), 1);
+            assertEquals(error.getError(), ServerError.TooManyRequests);
+        } finally {
+            limiter.release(heldPermit);
+            channel.finish();
+            limiter.close();
+        }
+    }
+
+    @Test(timeOut = 30000)
+    public void testGetTopicsOfNamespaceLargerThanHeapMemoryLimitFailsFastAfterMeasuring() throws Exception {
+        long topicListSize = TopicListMemoryLimiter.estimateTopicListSize(topics);
+        AsyncDualMemoryLimiterImpl limiter =
+                new AsyncDualMemoryLimiterImpl(topicListSize - 1, 10, 1000, 1_000_000, 10, 1000);
+        try {
+            doReturn(limiter).when(brokerService).getMaxTopicListInFlightLimiter();
+            // like the broker's cache: the initial estimate fits within the limit, so the first request lists the
+            // topics and is rejected when the permits are updated to the size of the topic list
+            doReturn(new TopicListSizeResultCache(topicListSize - 1, () -> true))
+                    .when(brokerService).getTopicListSizeResultCache();
+            resetChannel();
+            setChannelConnected();
+            clearInvocations(namespaceService);
+            var lookupSemaphore = brokerService.getLookupRequestSemaphore();
+            int initialPermits = lookupSemaphore.availablePermits();
+            for (long requestId = 1; requestId <= 3; requestId++) {
+                channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+                        "use/ns-abc", requestId, CommandGetTopicsOfNamespace.Mode.ALL, null, null, null));
+                CommandError error = (CommandError) getResponse();
+                assertEquals(error.getRequestId(), requestId);
+                assertEquals(error.getError(), ServerError.NotAllowedError);
+                assertEquals(lookupSemaphore.availablePermits(), initialPermits);
+            }
+            // only the first request listed the topics, the later requests failed without listing them
+            verify(namespaceService, times(1)).getListOfUserTopicsByProperties(any(), any(), any());
+            assertEquals(limiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY).getAcquiredPermits(), 0);
+        } finally {
+            channel.finish();
+            limiter.close();
+        }
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java
index e39ab45a0a84..8c2ade341fb9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java
@@ -29,8 +29,10 @@ import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
@@ -52,8 +54,11 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.IntStream;
@@ -63,6 +68,7 @@ import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.resources.PulsarResources;
+import org.apache.pulsar.broker.resources.TopicListener;
 import org.apache.pulsar.broker.resources.TopicResources;
 import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
 import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
@@ -75,6 +81,7 @@ import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
 import org.apache.pulsar.common.semaphore.AsyncSemaphore;
 import org.apache.pulsar.common.topics.TopicList;
 import org.apache.pulsar.common.topics.TopicsPattern;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.Notification;
 import org.apache.pulsar.metadata.api.NotificationType;
 import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
@@ -104,6 +111,8 @@ public class TopicListServiceTest {
     private ScheduledExecutorService scheduledExecutorService;
     private PulsarService pulsar;
     private NamespaceService namespaceService;
+    private TopicListSizeResultCache.ResultHolder resultHolder;
+    private AtomicBoolean connectionActive;
 
     @BeforeMethod(alwaysRun = true)
     public void setup() throws Exception {
@@ -132,7 +141,7 @@ public class TopicListServiceTest {
         when(pulsar.getBrokerService()).thenReturn(brokerService);
         TopicListSizeResultCache topicListSizeResultCache = mock(TopicListSizeResultCache.class);
         when(brokerService.getTopicListSizeResultCache()).thenReturn(topicListSizeResultCache);
-        TopicListSizeResultCache.ResultHolder resultHolder = mock(TopicListSizeResultCache.ResultHolder.class);
+        resultHolder = mock(TopicListSizeResultCache.ResultHolder.class);
         doReturn(resultHolder).when(topicListSizeResultCache).getTopicListSize(anyString(), any());
         doReturn(CompletableFuture.completedFuture(1L)).when(resultHolder).getSizeAsync();
 
@@ -143,10 +152,11 @@ public class TopicListServiceTest {
         when(connection.getRemoteAddress()).thenReturn(new InetSocketAddress(10000));
         pulsarCommandSender = mock(PulsarCommandSender.class);
         when(connection.getCommandSender()).thenReturn(pulsarCommandSender);
-        when(connection.isActive()).thenReturn(true);
-        when(pulsarCommandSender.sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any()))
+        connectionActive = new AtomicBoolean(true);
+        when(connection.isActive()).thenAnswer(__ -> connectionActive.get());
+        when(pulsarCommandSender.sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any()))
                 .thenReturn(CompletableFuture.completedFuture(null));
-        when(pulsarCommandSender.sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any()))
+        when(pulsarCommandSender.sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any()))
                 .thenReturn(CompletableFuture.completedFuture(null));
 
         scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
@@ -210,7 +220,7 @@ public class TopicListServiceTest {
         verify(topicResources).registerPersistentTopicListener(any(TopicListService.TopicListWatcher.class));
         Collection<String> expectedTopics = new ArrayList<>(topics);
         verify(connection.getCommandSender()).sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics),
-                any());
+                any(), any());
     }
 
     @Test
@@ -238,7 +248,31 @@ public class TopicListServiceTest {
         verify(topicResources).registerPersistentTopicListener(any(TopicListService.TopicListWatcher.class));
         Collection<String> expectedTopics = new ArrayList<>(topics);
         verify(connection.getCommandSender()).sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics),
-                any());
+                any(), any());
+    }
+
+    @Test
+    public void testCancelledWatchResetsTopicListSizeEstimate() throws Exception {
+        // acquire all permits so that the watcher's permit request waits in the queue
+        AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit permit =
+                memoryLimiter.acquire(1_000_000, AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY,
+                                Boolean.FALSE::booleanValue)
+                        .get();
+        topicListService.handleWatchTopicList(
+                NamespaceName.get("tenant/ns"),
+                13,
+                7,
+                "persistent://tenant/ns/topic\\d",
+                topicsPatternImplementation, null,
+                lookupSemaphore);
+        Awaitility.await().untilAsserted(() -> Assert.assertEquals(
+                memoryLimiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY).getQueueSize(), 1));
+        // the connection closes while the request waits for permits
+        connectionActive.set(false);
+        memoryLimiter.release(permit);
+        // concurrent requests waiting for the size estimate of the cancelled request must be able to proceed
+        Awaitility.await().untilAsserted(() -> verify(resultHolder).resetIfInitializing());
+        verify(namespaceService, never()).getListOfUserTopics(any(), any());
     }
 
     @Test
@@ -299,12 +333,12 @@ public class TopicListServiceTest {
             } else {
                 return CompletableFuture.completedFuture(null);
             }
-        }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any());
+        }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any());
         topicListFuture.complete(topics);
         assertThat(topicListService.getWatcherFuture(13)).succeedsWithin(Duration.ofSeconds(2));
         Collection<String> expectedTopics = new ArrayList<>(topics);
         verify(connection.getCommandSender(), timeout(2000L).times(3))
-                .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any());
+                .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any(), any());
     }
 
     @Test
@@ -325,13 +359,13 @@ public class TopicListServiceTest {
         notificationConsumer.accept(
                 new Notification(NotificationType.Created, "/managed-ledgers/tenant/ns/persistent/topic2"));
         verify(connection.getCommandSender(), timeout(2000L))
-                .sendWatchTopicListUpdate(eq(13L), eq(newTopics), any(), eq(hash), any());
+                .sendWatchTopicListUpdate(eq(13L), eq(newTopics), any(), eq(hash), any(), any());
 
         hash = TopicList.calculateHash(newTopics);
         notificationConsumer.accept(
                 new Notification(NotificationType.Deleted, "/managed-ledgers/tenant/ns/persistent/topic1"));
         verify(connection.getCommandSender(), timeout(2000L))
-                .sendWatchTopicListUpdate(eq(13L), eq(List.of()), eq(topics), eq(hash), any());
+                .sendWatchTopicListUpdate(eq(13L), eq(List.of()), eq(topics), eq(hash), any(), any());
     }
 
     @Test
@@ -348,7 +382,7 @@ public class TopicListServiceTest {
             } else {
                 return CompletableFuture.completedFuture(null);
             }
-        }).when(pulsarCommandSender).sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any());
+        }).when(pulsarCommandSender).sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
 
         topicListService.handleWatchTopicList(
                 NamespaceName.get("tenant/ns"),
@@ -369,9 +403,9 @@ public class TopicListServiceTest {
                 new Notification(NotificationType.Deleted, "/managed-ledgers/tenant/ns/persistent/topic2"));
         InOrder inOrder = inOrder(connection.getCommandSender());
         inOrder.verify(connection.getCommandSender(), timeout(2000L).times(3))
-                .sendWatchTopicListUpdate(eq(13L), eq(newTopics), eq(List.of()), eq(hash), any());
+                .sendWatchTopicListUpdate(eq(13L), eq(newTopics), eq(List.of()), eq(hash), any(), any());
         inOrder.verify(connection.getCommandSender(), timeout(2000L).times(1))
-                .sendWatchTopicListUpdate(eq(13L), eq(List.of()), eq(newTopics), any(), any());
+                .sendWatchTopicListUpdate(eq(13L), eq(List.of()), eq(newTopics), any(), any(), any());
     }
 
     @Test
@@ -420,12 +454,12 @@ public class TopicListServiceTest {
                 topicsPatternImplementation, hash,
                 lookupSemaphore);
         doReturn(CompletableFuture.completedFuture(null)).when(pulsarCommandSender)
-                .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any());
+                .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any());
         topicListFuture.complete(topics);
         assertThat(topicListService.getWatcherFuture(13)).succeedsWithin(Duration.ofSeconds(2));
         Collection<String> expectedTopics = List.of();
         verify(connection.getCommandSender(), timeout(2000L).times(1))
-                .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any());
+                .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any(), any());
     }
 
     @Test
@@ -440,12 +474,12 @@ public class TopicListServiceTest {
                 topicsPatternImplementation, "INVALID_HASH",
                 lookupSemaphore);
         doReturn(CompletableFuture.completedFuture(null)).when(pulsarCommandSender)
-                .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any());
+                .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any());
         topicListFuture.complete(topics);
         assertThat(topicListService.getWatcherFuture(13)).succeedsWithin(Duration.ofSeconds(2));
         Collection<String> expectedTopics = topics;
         verify(connection.getCommandSender(), timeout(2000L).times(1))
-                .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any());
+                .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any(), any());
     }
 
     @Test
@@ -503,7 +537,7 @@ public class TopicListServiceTest {
                 .sendWatchTopicListUpdate(eq(13L),
                         argThat(newTopics -> new HashSet<>(newTopics).equals(new HashSet<>(expectedNew))),
                         argThat(deletedTopics -> new HashSet<>(deletedTopics).equals(new HashSet<>(expectedDeleted))),
-                        eq(expectedHash), any());
+                        eq(expectedHash), any(), any());
 
         // Verify the watcher's matching topics are updated
         assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(topicsAfterReconnect);
@@ -560,7 +594,7 @@ public class TopicListServiceTest {
                 .sendWatchTopicListUpdate(eq(13L),
                         argThat(newTopics -> new HashSet<>(newTopics).equals(new HashSet<>(expectedNew))),
                         argThat(deletedTopics -> new HashSet<>(deletedTopics).equals(new HashSet<>(expectedDeleted))),
-                        eq(expectedHash), any());
+                        eq(expectedHash), any(), any());
 
         assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(topicsAfterReconnect);
     }
@@ -604,7 +638,7 @@ public class TopicListServiceTest {
 
         // Verify that sendWatchTopicListUpdate was NOT called (no changes to report)
         verify(connection.getCommandSender(), timeout(500L).times(0))
-                .sendWatchTopicListUpdate(eq(13L), any(), any(), any(), any());
+                .sendWatchTopicListUpdate(eq(13L), any(), any(), any(), any(), any());
     }
 
     @Test
@@ -734,4 +768,399 @@ public class TopicListServiceTest {
         Awaitility.await().untilAsserted(() ->
                 assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(initialTopics));
     }
+
+    private TopicListService.TopicListWatcher createWatcher(List<String> topics) throws Exception {
+        topicListService.handleWatchTopicList(
+                NamespaceName.get("tenant/ns"),
+                13,
+                7,
+                "persistent://tenant/ns/topic\\d",
+                topicsPatternImplementation, null,
+                lookupSemaphore);
+        topicListFuture.complete(topics);
+        TopicListService.TopicListWatcher watcher =
+                topicListService.getWatcherFuture(13).get(2, TimeUnit.SECONDS);
+        Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+        return watcher;
+    }
+
+    @Test
+    public void testDuplicateWatchRequestCompletesWhenWatcherIsClosedDuringUpdate() throws Exception {
+        TopicListService.TopicListWatcher watcher =
+                createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+        // start an update that doesn't complete
+        topicListFuture = new CompletableFuture<>();
+        assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+        topicListService.updateTopicListWatcher(watcher);
+
+        // a duplicate watch request waits for the ongoing update
+        lookupSemaphore.acquire();
+        topicListService.handleWatchTopicList(
+                NamespaceName.get("tenant/ns"),
+                13,
+                8,
+                "persistent://tenant/ns/topic\\d",
+                topicsPatternImplementation, null,
+                lookupSemaphore);
+        assertThat(lookupSemaphore.availablePermits()).isZero();
+
+        // closing the watcher completes the waiting request with an error and releases its lookup permit
+        topicListService.deleteTopicListWatcher(13L);
+        Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+        verify(pulsarCommandSender).sendErrorResponse(eq(8L), any(ServerError.class), anyString());
+        verify(pulsarCommandSender, never())
+                .sendWatchTopicListSuccess(eq(8L), anyLong(), anyString(), any(), any(), any());
+    }
+
+    @Test
+    public void testDuplicateWatchRequestGetsNonRetriableErrorWhenTopicListIsLargerThanMemoryLimit()
+            throws Exception {
+        TopicListService.TopicListWatcher watcher =
+                createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+        // start an update that doesn't complete yet
+        topicListFuture = new CompletableFuture<>();
+        assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+        topicListService.updateTopicListWatcher(watcher);
+
+        // a duplicate watch request waits for the ongoing update
+        lookupSemaphore.acquire();
+        topicListService.handleWatchTopicList(
+                NamespaceName.get("tenant/ns"),
+                13,
+                8,
+                "persistent://tenant/ns/topic\\d",
+                topicsPatternImplementation, null,
+                lookupSemaphore);
+
+        // the estimated size of the topic list is larger than the 1_000_000 bytes heap memory limit
+        topicListFuture.complete(List.of("persistent://tenant/ns/topic1" + "x".repeat(1_000_000)));
+
+        // the waiting request gets the non-retriable error of the update, not the error of closing the watcher
+        verify(pulsarCommandSender, timeout(2000L))
+                .sendErrorResponse(eq(8L), eq(ServerError.NotAllowedError), anyString());
+        Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+        assertThat(topicListService.getWatcherFuture(13)).isNull();
+    }
+
+    @Test
+    public void testFailedWatcherUpdateIsRetried() throws Exception {
+        TopicListService.TopicListWatcher watcher =
+                createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+        List<String> refreshedTopics = List.of("persistent://tenant/ns/topic1", "persistent://tenant/ns/topic2");
+        AtomicInteger listingCalls = new AtomicInteger();
+        doAnswer(invocationOnMock -> listingCalls.incrementAndGet() == 1
+                ? CompletableFuture.failedFuture(new PulsarServerException("Metadata error"))
+                : CompletableFuture.completedFuture(refreshedTopics))
+                .when(namespaceService).getListOfUserTopics(any(), eq(CommandGetTopicsOfNamespace.Mode.PERSISTENT));
+
+        assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+        assertThat(topicListService.updateTopicListWatcher(watcher)).succeedsWithin(Duration.ofSeconds(10));
+
+        assertThat(listingCalls).hasValue(2);
+        assertThat(watcher.isUpdatingTopics()).isFalse();
+        assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(refreshedTopics);
+    }
+
+    @Test
+    public void testWatcherIsClosedWhenTopicListIsLargerThanMemoryLimit() throws Exception {
+        TopicListService.TopicListWatcher watcher =
+                createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+        // the estimated size of the topic list is larger than the 1_000_000 bytes heap memory limit
+        List<String> hugeTopicList = List.of("persistent://tenant/ns/topic1" + "x".repeat(1_000_000));
+        doReturn(CompletableFuture.completedFuture(hugeTopicList))
+                .when(namespaceService).getListOfUserTopics(any(), eq(CommandGetTopicsOfNamespace.Mode.PERSISTENT));
+
+        assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+        Throwable failure = topicListService.updateTopicListWatcher(watcher)
+                .handle((__, t) -> t).get(10, TimeUnit.SECONDS);
+        assertThat(FutureUtil.unwrapCompletionException(failure))
+                .isInstanceOf(AsyncSemaphore.PermitAcquireLimitExceededException.class);
+
+        assertThat(topicListService.getWatcherFuture(13)).isNull();
+        verify(topicResources).deregisterPersistentTopicListener(watcher);
+        assertThat(memoryLimiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY).getAcquiredPermits())
+                .isZero();
+    }
+
+    @Test
+    public void testWatchSendsAreCancelledWhenWatcherIsClosed() throws Exception {
+        AtomicReference<BooleanSupplier> successCancelledRef = new AtomicReference<>();
+        doAnswer(invocationOnMock -> {
+            successCancelledRef.set(invocationOnMock.getArgument(5));
+            return CompletableFuture.completedFuture(null);
+        }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+                any());
+        AtomicReference<BooleanSupplier> updateCancelledRef = new AtomicReference<>();
+        doAnswer(invocationOnMock -> {
+            updateCancelledRef.set(invocationOnMock.getArgument(5));
+            return CompletableFuture.completedFuture(null);
+        }).when(pulsarCommandSender).sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
+        createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+        notificationConsumer.accept(
+                new Notification(NotificationType.Created, "/managed-ledgers/tenant/ns/persistent/topic2"));
+        Awaitility.await().untilAsserted(() -> assertThat(updateCancelledRef.get()).isNotNull());
+        assertThat(successCancelledRef.get().getAsBoolean()).isFalse();
+        assertThat(updateCancelledRef.get().getAsBoolean()).isFalse();
+
+        topicListService.deleteTopicListWatcher(13L);
+
+        // sends that are still waiting for direct memory permits are cancelled
+        assertThat(successCancelledRef.get().getAsBoolean()).isTrue();
+        assertThat(updateCancelledRef.get().getAsBoolean()).isTrue();
+    }
+
+    @Test
+    public void testWatcherIsDeletedWhenSendingSuccessResponseFails() throws Exception {
+        doReturn(CompletableFuture.failedFuture(new RuntimeException("Sending failed")))
+                .when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+                        any());
+        topicListService.handleWatchTopicList(
+                NamespaceName.get("tenant/ns"),
+                13,
+                7,
+                "persistent://tenant/ns/topic\\d",
+                topicsPatternImplementation, null,
+                lookupSemaphore);
+        topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+        Awaitility.await().untilAsserted(() -> assertThat(topicListService.getWatcherFuture(13)).isNull());
+        verify(topicResources).deregisterPersistentTopicListener(any(TopicListService.TopicListWatcher.class));
+        // the request gets an error response instead of waiting for a response that never comes
+        verify(pulsarCommandSender).sendErrorResponse(eq(7L), any(ServerError.class), eq("Sending failed"));
+        assertThat(lookupSemaphore.availablePermits()).isEqualTo(1);
+    }
+
+    @Test
+    public void testWatchRequestGetsErrorWhenSuccessResponseIsLargerThanDirectMemoryLimit() throws Exception {
+        doAnswer(invocationOnMock -> {
+            Function<Throwable, CompletableFuture<Void>> permitAcquireErrorHandler = invocationOnMock.getArgument(4);
+            return permitAcquireErrorHandler.apply(new AsyncSemaphore.PermitAcquireLimitExceededException(
+                    "Requested permits are larger than the limit", new IllegalArgumentException()));
+        }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+                any());
+        topicListService.handleWatchTopicList(
+                NamespaceName.get("tenant/ns"),
+                13,
+                7,
+                "persistent://tenant/ns/topic\\d",
+                topicsPatternImplementation, null,
+                lookupSemaphore);
+        topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+        Awaitility.await().untilAsserted(() -> assertThat(topicListService.getWatcherFuture(13)).isNull());
+        verify(pulsarCommandSender, timeout(2000L))
+                .sendErrorResponse(eq(7L), eq(ServerError.NotAllowedError), anyString());
+        // the success response isn't retried
+        verify(pulsarCommandSender, times(1))
+                .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any());
+        assertThat(lookupSemaphore.availablePermits()).isEqualTo(1);
+    }
+
+    @Test
+    public void testWatchRequestGetsErrorWhenWatcherIsClosedBeforeSuccessResponseIsSent() throws Exception {
+        doAnswer(invocationOnMock -> {
+            // the watcher is closed while the success response waits for direct memory permits, the permit
+            // request is then cancelled
+            topicListService.deleteTopicListWatcher(13L);
+            BooleanSupplier isCancelled = invocationOnMock.getArgument(5);
+            assertThat(isCancelled.getAsBoolean()).isTrue();
+            Function<Throwable, CompletableFuture<Void>> permitAcquireErrorHandler = invocationOnMock.getArgument(4);
+            return permitAcquireErrorHandler.apply(
+                    new AsyncSemaphore.PermitAcquireCancelledException("Permit acquisition was cancelled"));
+        }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+                any());
+        topicListService.handleWatchTopicList(
+                NamespaceName.get("tenant/ns"),
+                13,
+                7,
+                "persistent://tenant/ns/topic\\d",
+                topicsPatternImplementation, null,
+                lookupSemaphore);
+        topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+        // the connection is still active, so the request gets an error response instead of no response
+        verify(pulsarCommandSender, timeout(2000L)).sendErrorResponse(eq(7L), any(ServerError.class), anyString());
+        Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+    }
+
+    @Test
+    public void testDuplicateWatchRequestGetsErrorWhenUpdateFails() throws Exception {
+        TopicListService.TopicListWatcher watcher =
+                createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+        // the update keeps failing until the listing is allowed to succeed
+        AtomicBoolean listingFails = new AtomicBoolean(true);
+        List<String> refreshedTopics = List.of("persistent://tenant/ns/topic1", "persistent://tenant/ns/topic2");
+        doAnswer(invocationOnMock -> listingFails.get()
+                ? CompletableFuture.failedFuture(new PulsarServerException("Metadata error"))
+                : CompletableFuture.completedFuture(refreshedTopics))
+                .when(namespaceService).getListOfUserTopics(any(), eq(CommandGetTopicsOfNamespace.Mode.PERSISTENT));
+
+        // a duplicate watch request triggers an update and waits for it
+        lookupSemaphore.acquire();
+        topicListService.handleWatchTopicList(
+                NamespaceName.get("tenant/ns"),
+                13,
+                8,
+                "persistent://tenant/ns/topic\\d",
+                topicsPatternImplementation, null,
+                lookupSemaphore);
+
+        // the request gets an error response after the first failure instead of waiting for the retries
+        verify(pulsarCommandSender, timeout(5000L)).sendErrorResponse(eq(8L), any(ServerError.class), anyString());
+        Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+        assertThat(watcher.isUpdatingTopics()).isTrue();
+
+        // the update keeps being retried and completes once the listing succeeds
+        listingFails.set(false);
+        Awaitility.await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> {
+            assertThat(watcher.isUpdatingTopics()).isFalse();
+            assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(refreshedTopics);
+        });
+    }
+
+    @Test
+    public void testWatchRequestGetsErrorWhenWatcherIsRemovedBeforeFirstSendAttempt() throws Exception {
+        createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+        // the update triggered by the duplicate watch requests waits for the listing
+        topicListFuture = new CompletableFuture<>();
+        doAnswer(invocationOnMock -> {
+            long requestId = invocationOnMock.getArgument(0);
+            if (requestId == 8L) {
+                // the watcher is closed while the next waiting request is about to send its success response
+                topicListService.deleteTopicListWatcher(13L);
+            }
+            return CompletableFuture.completedFuture(null);
+        }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+                any());
+        lookupSemaphore.release(2);
+        for (long requestId = 8; requestId <= 9; requestId++) {
+            lookupSemaphore.acquire();
+            topicListService.handleWatchTopicList(
+                    NamespaceName.get("tenant/ns"),
+                    13,
+                    requestId,
+                    "persistent://tenant/ns/topic\\d",
+                    topicsPatternImplementation, null,
+                    lookupSemaphore);
+        }
+
+        topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+        // request 9 finds the watcher removed before its first send attempt and gets an error response
+        verify(pulsarCommandSender, timeout(2000L)).sendErrorResponse(eq(9L), any(ServerError.class), anyString());
+        verify(pulsarCommandSender, never())
+                .sendWatchTopicListSuccess(eq(9L), anyLong(), anyString(), any(), any(), any());
+        Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(3));
+    }
+
+    @Test
+    public void testWaitingWatchRequestRespondsWithSnapshotOfItsUpdate() throws Exception {
+        List<String> topics = Collections.singletonList("persistent://tenant/ns/topic1");
+        TopicListService.TopicListWatcher watcher = createWatcher(topics);
+        // the update triggered by the duplicate watch requests waits for the listing
+        topicListFuture = new CompletableFuture<>();
+        doAnswer(invocationOnMock -> {
+            long requestId = invocationOnMock.getArgument(0);
+            if (requestId == 8L) {
+                // the metadata store connection is lost after the update has completed, which clears the matching
+                // topics until the next update, while the next waiting request is about to respond
+                watcher.onSessionEvent(SessionEvent.ConnectionLost);
+            }
+            return CompletableFuture.completedFuture(null);
+        }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+                any());
+        lookupSemaphore.release(2);
+        for (long requestId = 8; requestId <= 9; requestId++) {
+            lookupSemaphore.acquire();
+            topicListService.handleWatchTopicList(
+                    NamespaceName.get("tenant/ns"),
+                    13,
+                    requestId,
+                    "persistent://tenant/ns/topic\\d",
+                    topicsPatternImplementation, null,
+                    lookupSemaphore);
+        }
+
+        topicListFuture.complete(topics);
+
+        // request 9 responds with the matching topics of the completed update, not with the cleared list
+        verify(pulsarCommandSender, timeout(2000L)).sendWatchTopicListSuccess(eq(9L), eq(13L),
+                eq(TopicList.calculateHash(topics)), eq(new ArrayList<>(topics)), any(), any());
+    }
+
+    @Test
+    public void testFailedWatchRequestClosesTheWatcherItCreated() throws Exception {
+        AtomicReference<TopicListService.TopicListWatcher> createdWatcher = new AtomicReference<>();
+        doAnswer(invocationOnMock -> {
+            invocationOnMock.callRealMethod();
+            TopicListService.TopicListWatcher watcher = invocationOnMock.getArgument(0);
+            createdWatcher.set(watcher);
+            // the metadata store connection is lost while the watcher is being created, which starts an update
+            watcher.onSessionEvent(SessionEvent.ConnectionLost);
+            return null;
+        }).when(topicResources).registerPersistentTopicListener(any(TopicListener.class));
+        topicListService.handleWatchTopicList(
+                NamespaceName.get("tenant/ns"),
+                13,
+                7,
+                "persistent://tenant/ns/topic\\d",
+                topicsPatternImplementation, null,
+                lookupSemaphore);
+        topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+        // the permits are granted and the watcher is created asynchronously
+        Awaitility.await().untilAsserted(() -> assertThat(createdWatcher.get()).isNotNull());
+        // the watch request waits for the update since the matching topics are incomplete
+        Awaitility.await().untilAsserted(() -> assertThat(topicListService.getWatcherFuture(13)).isCompleted());
+        assertThat(lookupSemaphore.availablePermits()).isZero();
+
+        // the update fails, which fails the waiting watch request
+        doReturn(CompletableFuture.failedFuture(new PulsarServerException("Metadata error")))
+                .when(namespaceService).getListOfUserTopics(any(), eq(CommandGetTopicsOfNamespace.Mode.PERSISTENT));
+        topicListService.updateTopicListWatcher(createdWatcher.get());
+
+        // the request gets an error response, and the watcher that it created is removed, deregistered and closed
+        verify(pulsarCommandSender, timeout(2000L)).sendErrorResponse(eq(7L), any(ServerError.class), anyString());
+        Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+        assertThat(topicListService.getWatcherFuture(13)).isNull();
+        verify(topicResources, timeout(2000L)).deregisterPersistentTopicListener(createdWatcher.get());
+    }
+
+    @Test
+    public void testTopicChangesAreSentAfterPendingSuccessResponse() throws Exception {
+        TopicListService.TopicListWatcher watcher =
+                createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+        // the success response of a watch request that waited for an update is being sent
+        watcher.successResponseSendingStarted();
+        assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+        // completing the update doesn't send the queued topic changes before the success response
+        watcher.updateTopics(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+        notificationConsumer.accept(
+                new Notification(NotificationType.Created, "/managed-ledgers/tenant/ns/persistent/topic2"));
+        Thread.sleep(500);
+        verify(pulsarCommandSender, never())
+                .sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
+
+        // the topic change is sent after the success response has been sent
+        watcher.successResponseSendingCompleted();
+        verify(pulsarCommandSender, timeout(2000L))
+                .sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
+    }
+
+    @Test
+    public void testTopicChangesAreQueuedWhileUpdateIsInProgress() throws Exception {
+        TopicListService.TopicListWatcher watcher =
+                createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+        // an update starts, for example after a metadata store session event
+        assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+        // a send that was in progress before the update completes
+        watcher.sendingCompleted();
+
+        // a topic change during the update isn't sent, since the matching topics are being rebuilt
+        notificationConsumer.accept(
+                new Notification(NotificationType.Created, "/managed-ledgers/tenant/ns/persistent/topic2"));
+        Thread.sleep(500);
+        verify(pulsarCommandSender, never())
+                .sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiter.java b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiter.java
index 59be8db89239..444ddb199454 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiter.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiter.java
@@ -70,8 +70,13 @@ public interface AsyncDualMemoryLimiter {
     void release(AsyncDualMemoryLimiterPermit permit);
     /**
      * Execute the specified function with acquired permits and release the permits after the returned future completes.
+     * When the permits can't be acquired, including when the memory size is larger than the limit
+     * ({@link AsyncSemaphore.PermitAcquireLimitExceededException}), permitAcquireErrorHandler is called instead.
      * @param memorySize memory size to acquire permits for
      * @param limitType memory limit type to acquire permits for
+     * @param isCancelled supplier that returns true if the request has been cancelled. When it returns true after
+     *                    the permits have been acquired, the function isn't executed, the permits are released and
+     *                    permitAcquireErrorHandler is called with AsyncSemaphore.PermitAcquireCancelledException
      * @param function function to execute with acquired permits
      * @return result of the function
      * @param <T> type of the CompletableFuture returned by the function
@@ -82,14 +87,18 @@ public interface AsyncDualMemoryLimiter {
                                                                  CompletableFuture<T>> function,
                                                          Function<Throwable, CompletableFuture<T>>
                                                                  permitAcquireErrorHandler) {
-        return AsyncDualMemoryLimiterUtil.withPermitsFuture(acquire(memorySize, limitType, isCancelled), function,
-                permitAcquireErrorHandler, this::release);
+        return AsyncDualMemoryLimiterUtil.withPermitsFuture(
+                AsyncDualMemoryLimiterUtil.requestPermits(() -> acquire(memorySize, limitType, isCancelled)),
+                isCancelled, function, permitAcquireErrorHandler, this::release);
     }
 
     /**
      * Executed the specified function with updated permits and release the permits after the returned future completes.
      * @param initialPermit initial permit to update
      * @param newMemorySize new memory size to update to
+     * @param isCancelled supplier that returns true if the request has been cancelled. When it returns true after
+     *                    the permits have been updated, the function isn't executed, the permits are released and
+     *                    permitAcquireErrorHandler is called with AsyncSemaphore.PermitAcquireCancelledException
      * @param function function to execute with updated permits
      * @return result of the function
      * @param <T> type of the CompletableFuture returned by the function
@@ -100,8 +109,9 @@ public interface AsyncDualMemoryLimiter {
                                                                 CompletableFuture<T>> function,
                                                         Function<Throwable, CompletableFuture<T>>
                                                                 permitAcquireErrorHandler) {
-        return AsyncDualMemoryLimiterUtil.withPermitsFuture(update(initialPermit, newMemorySize, isCancelled), function,
-                permitAcquireErrorHandler, this::release);
+        return AsyncDualMemoryLimiterUtil.withPermitsFuture(
+                AsyncDualMemoryLimiterUtil.requestPermits(() -> update(initialPermit, newMemorySize, isCancelled)),
+                isCancelled, function, permitAcquireErrorHandler, this::release);
     }
 
     /**
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtil.java
index 9f3880a2c09b..e7233de492de 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtil.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtil.java
@@ -24,11 +24,13 @@ import java.util.concurrent.CompletableFuture;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.Supplier;
 import lombok.experimental.UtilityClass;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.common.api.proto.BaseCommand;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
+import org.apache.pulsar.common.util.FutureUtil;
 
 @UtilityClass
 public class AsyncDualMemoryLimiterUtil {
@@ -52,6 +54,33 @@ public class AsyncDualMemoryLimiterUtil {
             Function<Throwable, CompletableFuture<T>>
                     permitAcquireErrorHandler,
             Consumer<AsyncDualMemoryLimiterPermit> releaser) {
+        return withPermitsFuture(permitsFuture, () -> false, function, permitAcquireErrorHandler, releaser);
+    }
+
+    /**
+     * Execute a function with acquired permits and ensure permits are released after completion.
+     * This method handles the lifecycle of permits - acquisition, usage, and release, including error cases.
+     * If the request has been cancelled by the time the permits are acquired, the permits are released
+     * without executing the function, and the permit acquire error handler is called with
+     * {@link AsyncSemaphore.PermitAcquireCancelledException}.
+     *
+     * @param permitsFuture             Future that will complete with the required permits
+     * @param isCancelled               supplier that returns true if the request has been cancelled
+     * @param function                  Function to execute once permits are acquired that returns a CompletableFuture
+     * @param permitAcquireErrorHandler Handler for permit acquisition errors that returns a CompletableFuture
+     * @param releaser                  Consumer that handles releasing the permits
+     * @param <T>                       The type of result returned by the function
+     * @return CompletableFuture that completes with the result of the function execution
+     */
+    public static <T> CompletableFuture<T> withPermitsFuture(
+            CompletableFuture<AsyncDualMemoryLimiterPermit>
+                    permitsFuture,
+            BooleanSupplier isCancelled,
+            Function<AsyncDualMemoryLimiterPermit,
+                    CompletableFuture<T>> function,
+            Function<Throwable, CompletableFuture<T>>
+                    permitAcquireErrorHandler,
+            Consumer<AsyncDualMemoryLimiterPermit> releaser) {
         return permitsFuture
                 // combine the permits and error into a single pair so that it can be used in thenCompose
                 .handle((permits, permitAcquireError) ->
@@ -63,6 +92,11 @@ public class AsyncDualMemoryLimiterUtil {
                     } else {
                         // permits were acquired
                         AsyncDualMemoryLimiterPermit permits = permitsAndError.getLeft();
+                        if (isCancelled.getAsBoolean()) {
+                            // the request was cancelled while waiting for the permits, skip the operation
+                            releaser.accept(permits);
+                            return permitAcquireErrorHandler.apply(newCancelledException());
+                        }
                         try {
                             return function.apply(permits)
                                     .whenComplete((__, ___) ->
@@ -83,7 +117,10 @@ public class AsyncDualMemoryLimiterUtil {
      *
      * @param ctx the channel handler context used for writing the response
      * @param dualMemoryLimiter the memory limiter used to acquire and release memory permits
-     * @param isCancelled supplier that indicates if the permit acquisition should be cancelled
+     * @param isCancelled supplier that indicates if the permit acquisition should be cancelled. When it returns true
+     *                    after the permits have been acquired, the command isn't serialized or written, the permits
+     *                    are released and permitAcquireErrorHandler is called with
+     *                    AsyncSemaphore.PermitAcquireCancelledException
      * @param command the base command to serialize and write to the channel
      * @param permitAcquireErrorHandler handler for errors that occur during permit acquisition
      * @return a future that completes when the command has been written to the channel's outbound buffer
@@ -100,11 +137,17 @@ public class AsyncDualMemoryLimiterUtil {
         // Calculate serialized size before acquiring permits
         int serializedSize = command.getSerializedSize();
         // Acquire permits
-        return dualMemoryLimiter.acquire(serializedSize, AsyncDualMemoryLimiter.LimitType.DIRECT_MEMORY, isCancelled)
+        return requestPermits(() -> dualMemoryLimiter.acquire(serializedSize,
+                        AsyncDualMemoryLimiter.LimitType.DIRECT_MEMORY, isCancelled))
                 .handle((permits, t) -> {
                     if (t != null) {
                         return permitAcquireErrorHandler.apply(t);
                     }
+                    if (isCancelled.getAsBoolean()) {
+                        // the request was cancelled while waiting for the permits, skip serializing and writing
+                        dualMemoryLimiter.release(permits);
+                        return permitAcquireErrorHandler.apply(newCancelledException());
+                    }
                     try {
                         // Serialize the response
                         ByteBuf outBuf = Commands.serializeWithPrecalculatedSerializedSize(command, serializedSize);
@@ -121,4 +164,50 @@ public class AsyncDualMemoryLimiterUtil {
                     return CompletableFuture.<Void>completedFuture(null);
                 }).thenCompose(Function.identity());
     }
+
+    /**
+     * Requests permits so that all failures are reported through the returned future. The limiter rejects a request
+     * that is larger than the limit by throwing {@link IllegalArgumentException}, which is converted to a future
+     * that completes exceptionally with {@link AsyncSemaphore.PermitAcquireLimitExceededException}. Callers that
+     * handle failures only through the future would otherwise never complete the request.
+     *
+     * @param permitRequest supplier that requests the permits from the limiter
+     * @return future that completes with the permits or exceptionally when the permits can't be acquired
+     */
+    public static CompletableFuture<AsyncDualMemoryLimiterPermit> requestPermits(
+            Supplier<CompletableFuture<AsyncDualMemoryLimiterPermit>> permitRequest) {
+        try {
+            return permitRequest.get();
+        } catch (IllegalArgumentException e) {
+            return CompletableFuture.failedFuture(
+                    new AsyncSemaphore.PermitAcquireLimitExceededException(e.getMessage(), e));
+        }
+    }
+
+    /**
+     * Checks whether the permit request failed because the request was cancelled.
+     *
+     * @param throwable the failure, possibly wrapped in a {@link java.util.concurrent.CompletionException}
+     * @return true if the request was cancelled
+     */
+    public static boolean isPermitAcquireCancelled(Throwable throwable) {
+        return FutureUtil.unwrapCompletionException(throwable)
+                instanceof AsyncSemaphore.PermitAcquireCancelledException;
+    }
+
+    /**
+     * Checks whether the permit request failed because more permits were requested than the limit. Unlike a permit
+     * timeout or a full queue, retrying such a request fails the same way.
+     *
+     * @param throwable the failure, possibly wrapped in a {@link java.util.concurrent.CompletionException}
+     * @return true if the request was larger than the limit
+     */
+    public static boolean isPermitAcquireLimitExceeded(Throwable throwable) {
+        return FutureUtil.unwrapCompletionException(throwable)
+                instanceof AsyncSemaphore.PermitAcquireLimitExceededException;
+    }
+
+    private static AsyncSemaphore.PermitAcquireCancelledException newCancelledException() {
+        return new AsyncSemaphore.PermitAcquireCancelledException("Request was cancelled before permits were used");
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncSemaphore.java b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncSemaphore.java
index 99479d9b3e54..dd15be67cdb3 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncSemaphore.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncSemaphore.java
@@ -84,6 +84,10 @@ public interface AsyncSemaphore {
         public PermitAcquireException(String message) {
             super(message);
         }
+
+        public PermitAcquireException(String message, Throwable cause) {
+            super(message, cause);
+        }
     }
 
     /**
@@ -119,6 +123,18 @@ public interface AsyncSemaphore {
         }
     }
 
+    /**
+     * Exception used when the requested permits are larger than the maximum number of permits, so the request
+     * can never be granted.
+     */
+    class PermitAcquireLimitExceededException extends PermitAcquireException {
+        private static final long serialVersionUID = 1L;
+
+        public PermitAcquireLimitExceededException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
     /**
      * Exception thrown when permit acquisition is cancelled.
      */
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtilTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtilTest.java
index cf202397cb77..a5f78e3eec8c 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtilTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtilTest.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.common.semaphore;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
@@ -36,6 +37,7 @@ import io.netty.channel.ChannelHandlerContext;
 import io.netty.util.concurrent.GenericFutureListener;
 import java.time.Duration;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -45,6 +47,7 @@ import org.apache.pulsar.common.api.proto.BaseCommand;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.LimitType;
 import org.apache.pulsar.common.semaphore.AsyncSemaphore.PermitAcquireCancelledException;
+import org.apache.pulsar.common.semaphore.AsyncSemaphore.PermitAcquireLimitExceededException;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
@@ -600,6 +603,163 @@ public class AsyncDualMemoryLimiterUtilTest {
         assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAvailablePermits()).isEqualTo(10000);
     }
 
+    @Test
+    public void testWithAcquiredPermitsSkipsFunctionWhenCancelledAfterPermitsAreGranted() throws Exception {
+        // the request gets cancelled after the permits have been granted, but before the function is executed
+        AtomicBoolean cancelled = new AtomicBoolean(false);
+        CompletableFuture<AsyncDualMemoryLimiterPermit> permitFuture =
+                limiter.acquire(100, LimitType.HEAP_MEMORY, cancelled::get);
+        assertThat(permitFuture).succeedsWithin(Duration.ofSeconds(1));
+        cancelled.set(true);
+
+        AtomicBoolean functionExecuted = new AtomicBoolean(false);
+        AtomicReference<Throwable> capturedError = new AtomicReference<>();
+        CompletableFuture<String> result = AsyncDualMemoryLimiterUtil.withPermitsFuture(
+                permitFuture,
+                cancelled::get,
+                permit -> {
+                    functionExecuted.set(true);
+                    return CompletableFuture.completedFuture("success");
+                },
+                throwable -> {
+                    capturedError.set(throwable);
+                    return CompletableFuture.completedFuture("cancelled");
+                },
+                limiter::release
+        );
+
+        assertThat(result).succeedsWithin(Duration.ofSeconds(1)).isEqualTo("cancelled");
+        assertThat(functionExecuted.get()).isFalse();
+        assertThat(capturedError.get()).isInstanceOf(PermitAcquireCancelledException.class);
+        assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isEqualTo(0);
+    }
+
+    @Test
+    public void testWithUpdatedPermitsSkipsFunctionWhenCancelledAfterPermitsAreGranted() throws Exception {
+        AtomicBoolean cancelled = new AtomicBoolean(false);
+        AtomicBoolean innerFunctionExecuted = new AtomicBoolean(false);
+        AtomicReference<Throwable> capturedError = new AtomicReference<>();
+        CompletableFuture<String> result = limiter.withAcquiredPermits(100, LimitType.HEAP_MEMORY, cancelled::get,
+                initialPermit -> {
+                    // the request is cancelled while the listing is in progress
+                    cancelled.set(true);
+                    return limiter.withUpdatedPermits(initialPermit, 50, cancelled::get, updatedPermit -> {
+                        innerFunctionExecuted.set(true);
+                        return CompletableFuture.completedFuture("success");
+                    }, throwable -> {
+                        capturedError.set(throwable);
+                        return CompletableFuture.completedFuture("cancelled");
+                    });
+                }, CompletableFuture::failedFuture);
+
+        assertThat(result).succeedsWithin(Duration.ofSeconds(1)).isEqualTo("cancelled");
+        assertThat(innerFunctionExecuted.get()).isFalse();
+        assertThat(capturedError.get()).isInstanceOf(PermitAcquireCancelledException.class);
+        assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isEqualTo(0);
+    }
+
+    @Test
+    public void testAcquireDirectMemoryPermitsAndWriteAndFlushSkipsWriteWhenCancelledAfterPermitsAreGranted()
+            throws Exception {
+        // simulate a permit grant that races with the cancellation of the request
+        AsyncDualMemoryLimiter dualMemoryLimiter = mock(AsyncDualMemoryLimiter.class);
+        AsyncDualMemoryLimiterPermit permit = mock(AsyncDualMemoryLimiterPermit.class);
+        when(dualMemoryLimiter.acquire(anyLong(), any(), any())).thenReturn(CompletableFuture.completedFuture(permit));
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        AtomicReference<Throwable> capturedError = new AtomicReference<>();
+
+        CompletableFuture<Void> result = AsyncDualMemoryLimiterUtil.acquireDirectMemoryPermitsAndWriteAndFlush(
+                ctx,
+                dualMemoryLimiter,
+                () -> true,
+                createTestCommand(),
+                throwable -> {
+                    capturedError.set(throwable);
+                    return CompletableFuture.completedFuture(null);
+                }
+        );
+
+        assertThat(result).succeedsWithin(Duration.ofSeconds(1));
+        verify(ctx, never()).writeAndFlush(any());
+        verify(dualMemoryLimiter, times(1)).release(permit);
+        assertThat(capturedError.get()).isInstanceOf(PermitAcquireCancelledException.class);
+    }
+
+    @Test
+    public void testWithAcquiredPermitsCallsErrorHandlerWhenRequestIsLargerThanLimit() {
+        AtomicBoolean functionExecuted = new AtomicBoolean(false);
+        AtomicReference<Throwable> capturedError = new AtomicReference<>();
+
+        // the limiter rejects the request synchronously, the failure is passed to the error handler
+        CompletableFuture<String> result = limiter.withAcquiredPermits(10001, LimitType.HEAP_MEMORY, () -> false,
+                permit -> {
+                    functionExecuted.set(true);
+                    return CompletableFuture.completedFuture("success");
+                }, throwable -> {
+                    capturedError.set(throwable);
+                    return CompletableFuture.completedFuture("too large");
+                });
+
+        assertThat(result).succeedsWithin(Duration.ofSeconds(1)).isEqualTo("too large");
+        assertThat(functionExecuted.get()).isFalse();
+        assertThat(capturedError.get()).isInstanceOf(PermitAcquireLimitExceededException.class);
+        assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isEqualTo(0);
+    }
+
+    @Test
+    public void testWithUpdatedPermitsCallsErrorHandlerWhenRequestIsLargerThanLimit() {
+        AtomicBoolean innerFunctionExecuted = new AtomicBoolean(false);
+        AtomicReference<Throwable> capturedError = new AtomicReference<>();
+
+        CompletableFuture<String> result = limiter.withAcquiredPermits(100, LimitType.HEAP_MEMORY, () -> false,
+                initialPermit -> limiter.withUpdatedPermits(initialPermit, 10001, () -> false, updatedPermit -> {
+                    innerFunctionExecuted.set(true);
+                    return CompletableFuture.completedFuture("success");
+                }, throwable -> {
+                    capturedError.set(throwable);
+                    return CompletableFuture.completedFuture("too large");
+                }), CompletableFuture::failedFuture);
+
+        assertThat(result).succeedsWithin(Duration.ofSeconds(1)).isEqualTo("too large");
+        assertThat(innerFunctionExecuted.get()).isFalse();
+        assertThat(capturedError.get()).isInstanceOf(PermitAcquireLimitExceededException.class);
+        // the initial permits are released after the outer function completes
+        assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isEqualTo(0);
+    }
+
+    @Test
+    public void testAcquireDirectMemoryPermitsAndWriteAndFlushCallsErrorHandlerWhenCommandIsLargerThanLimit() {
+        limiter.close();
+        // a direct memory limit smaller than the serialized command
+        limiter = new AsyncDualMemoryLimiterImpl(10000, 100, 5000, 1, 100, 5000);
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        AtomicReference<Throwable> capturedError = new AtomicReference<>();
+
+        CompletableFuture<Void> result = AsyncDualMemoryLimiterUtil.acquireDirectMemoryPermitsAndWriteAndFlush(
+                ctx, limiter, () -> false, createTestCommand(), throwable -> {
+                    capturedError.set(throwable);
+                    return CompletableFuture.completedFuture(null);
+                });
+
+        assertThat(result).succeedsWithin(Duration.ofSeconds(1));
+        verify(ctx, never()).writeAndFlush(any());
+        assertThat(capturedError.get()).isInstanceOf(PermitAcquireLimitExceededException.class);
+    }
+
+    @Test
+    public void testIsPermitAcquireLimitExceeded() {
+        PermitAcquireLimitExceededException limitExceeded =
+                new PermitAcquireLimitExceededException("larger than the limit", new IllegalArgumentException());
+        assertThat(AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(limitExceeded)).isTrue();
+        assertThat(AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(new CompletionException(limitExceeded)))
+                .isTrue();
+        // a permit timeout or a full queue can succeed when retried
+        assertThat(AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(
+                new AsyncSemaphore.PermitAcquireTimeoutException("timeout"))).isFalse();
+        assertThat(AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(
+                new AsyncSemaphore.PermitAcquireQueueFullException("queue full"))).isFalse();
+    }
+
     private BaseCommand createTestCommand() {
         BaseCommand command = new BaseCommand().setType(BaseCommand.Type.PING);
         command.setPing();
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
index 9d07c5fd203b..75dd59e86210 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.proxy.server;
 import static org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil.acquireDirectMemoryPermitsAndWriteAndFlush;
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.ChannelHandlerContext;
+import io.netty.util.ReferenceCountUtil;
 import io.prometheus.client.Counter;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
@@ -55,6 +56,8 @@ import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion;
 import org.apache.pulsar.common.protocol.schema.SchemaVersion;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.netty.NettyChannelUtil;
 import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData;
 
@@ -374,7 +377,8 @@ public class LookupProxyHandler {
             command = Commands.newGetTopicsOfNamespaceRequest(namespaceName, requestId, mode,
                     topicsPattern, topicsHash, propertiesMap);
 
-            internalPerformGetTopicsOfNamespace(clientRequestId, namespaceName, mode, clientCnx, command, requestId);
+            internalPerformGetTopicsOfNamespace(clientRequestId, namespaceName, mode, propertiesMap, clientCnx,
+                    command, requestId);
             proxyConnection.getConnectionPool().releaseConnection(clientCnx);
         }).exceptionally(ex -> {
             // Failed to connect to backend broker
@@ -385,11 +389,12 @@ public class LookupProxyHandler {
     }
 
     private void internalPerformGetTopicsOfNamespace(long clientRequestId, String namespaceName,
-                                                     CommandGetTopicsOfNamespace.Mode mode, ClientCnx clientCnx,
+                                                     CommandGetTopicsOfNamespace.Mode mode,
+                                                     Map<String, String> properties, ClientCnx clientCnx,
                                                      ByteBuf command, long requestId) {
         BooleanSupplier isPermitRequestCancelled = () -> !proxyConnection.ctx().channel().isActive();
         TopicListSizeResultCache.ResultHolder
-                listSizeHolder = topicListSizeResultCache.getTopicListSize(namespaceName, mode);
+                listSizeHolder = topicListSizeResultCache.getTopicListSize(namespaceName, mode, properties);
         listSizeHolder.getSizeAsync().thenAccept(initialSize -> {
             maxTopicListInFlightLimiter.withAcquiredPermits(initialSize,
                     AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY, isPermitRequestCancelled, initialPermits -> {
@@ -401,7 +406,8 @@ public class LookupProxyHandler {
                                         .exceptionMessage(t)
                                         .log("Failed to get TopicsOfNamespace");
                                 listSizeHolder.resetIfInitializing();
-                                writeAndFlush(Commands.newError(clientRequestId, getServerError(t), t.getMessage()));
+                                writeAndFlush(Commands.newError(clientRequestId, getTopicListServerError(t),
+                                        t.getMessage()));
                                 return CompletableFuture.completedFuture(null);
                             } else {
                                 long actualSize = TopicListMemoryLimiter.estimateTopicListSize(
@@ -412,27 +418,23 @@ public class LookupProxyHandler {
                                             return handleWritingGetTopicsResponse(clientRequestId, r,
                                                     isPermitRequestCancelled);
                                         }, t2 -> {
-                                            log.warn()
-                                                    .attr("clientAddress", clientAddress)
-                                                    .attr("GetTopicsOfNamespace", t2.getMessage())
-                                                    .log("Failed to acquire actual heap memory permits for " + "");
-                                            writeAndFlush(
-                                                    Commands.newError(clientRequestId, ServerError.TooManyRequests,
-                                                            "Failed due to heap memory limit exceeded"));
-
+                                            if (!isTopicListRequestCancelled(t2,
+                                                    "Failed to acquire actual heap memory permits for "
+                                                            + "GetTopicsOfNamespace")) {
+                                                writeTopicListPermitsError(clientRequestId, t2, true, "heap");
+                                            }
                                             return CompletableFuture.completedFuture(null);
                                         });
                             }
                         });
                     }, t -> {
-                        log.warn()
-                                .attr("clientAddress", clientAddress)
-                                .attr("GetTopicsOfNamespace", t.getMessage())
-                                .log("Failed to acquire initial heap memory permits for");
+                        // the command wasn't sent to the broker, release it
+                        ReferenceCountUtil.safeRelease(command);
                         listSizeHolder.resetIfInitializing();
-                        writeAndFlush(Commands.newError(clientRequestId, ServerError.TooManyRequests,
-                                "Failed due to heap memory limit exceeded"));
-
+                        if (!isTopicListRequestCancelled(t,
+                                "Failed to acquire initial heap memory permits for GetTopicsOfNamespace")) {
+                            writeTopicListPermitsError(clientRequestId, t, listSizeHolder.isMeasured(), "heap");
+                        }
                         return CompletableFuture.completedFuture(null);
                     }).exceptionally(ex -> {
                 listSizeHolder.resetIfInitializing();
@@ -449,16 +451,51 @@ public class LookupProxyHandler {
                 r.isChanged(), clientRequestId);
         return acquireDirectMemoryPermitsAndWriteAndFlush(proxyConnection.ctx(), maxTopicListInFlightLimiter,
                 isCancelled, responseCommand, t -> {
-                    log.warn()
-                            .attr("clientAddress", clientAddress)
-                            .attr("GetTopicsOfNamespace", t.getMessage())
-                            .log("Failed to acquire actual direct memory permits for");
-                    writeAndFlush(Commands.newError(clientRequestId, ServerError.TooManyRequests,
-                            "Failed due to direct memory limit exceeded"));
+                    if (!isTopicListRequestCancelled(t,
+                            "Failed to acquire actual direct memory permits for GetTopicsOfNamespace")) {
+                        writeTopicListPermitsError(clientRequestId, t, true, "direct");
+                    }
                     return CompletableFuture.completedFuture(null);
                 });
     }
 
+    /**
+     * Responds to a GetTopicsOfNamespace request that couldn't acquire topic list memory permits. A topic list that
+     * has been measured to be larger than the limit gets a non-retriable error, since a retry fails the same way
+     * until the list is measured again. A permit timeout or a full queue, or an estimate larger than the limit for
+     * a topic list that hasn't been measured, gets {@link ServerError#TooManyRequests}, which the client retries.
+     */
+    private void writeTopicListPermitsError(long clientRequestId, Throwable t, boolean topicListMeasured,
+                                            String memoryType) {
+        if (topicListMeasured && AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(t)) {
+            writeAndFlush(Commands.newError(clientRequestId, ServerError.NotAllowedError,
+                    "Failed since the topic list is larger than the topic list " + memoryType + " memory limit"));
+        } else {
+            writeAndFlush(Commands.newError(clientRequestId, ServerError.TooManyRequests,
+                    "Failed due to " + memoryType + " memory limit exceeded"));
+        }
+    }
+
+    /**
+     * Logs a failure to acquire topic list memory permits. A request that was cancelled, because the client
+     * connection was closed, is logged at debug level, since there is no client to respond to.
+     *
+     * @return true if the request was cancelled and no error response should be sent
+     */
+    private boolean isTopicListRequestCancelled(Throwable t, String failureMessage) {
+        if (AsyncDualMemoryLimiterUtil.isPermitAcquireCancelled(t)) {
+            log.debug()
+                    .attr("clientAddress", clientAddress)
+                    .log("GetTopicsOfNamespace was cancelled since the client connection was closed");
+            return true;
+        }
+        log.warn()
+                .attr("clientAddress", clientAddress)
+                .exceptionMessage(t)
+                .log(failureMessage);
+        return false;
+    }
+
     public void handleGetSchema(CommandGetSchema commandGetSchema) {
         GET_SCHEMA_REQUESTS.inc();
         log.debug()
@@ -568,6 +605,18 @@ public class LookupProxyHandler {
         return responseError;
     }
 
+    /**
+     * Maps a broker's GetTopicsOfNamespace failure to the error relayed to the client. The broker rejects a topic
+     * list that is larger than its topic list memory limit with {@link ServerError#NotAllowedError}, which is passed
+     * through so that the client doesn't retry it.
+     */
+    private ServerError getTopicListServerError(Throwable error) {
+        if (FutureUtil.unwrapCompletionException(error) instanceof PulsarClientException.NotAllowedException) {
+            return ServerError.NotAllowedError;
+        }
+        return getServerError(error);
+    }
+
     private void writeAndFlush(ByteBuf cmd) {
         final ChannelHandlerContext ctx = proxyConnection.ctx();
         NettyChannelUtil.writeAndFlushWithVoidPromise(ctx, cmd);
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
index 84bb62ac79b2..a15f15614880 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
@@ -77,7 +77,9 @@ import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
 import org.apache.pulsar.client.impl.tls.ClientTlsFactorySupport;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
 import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
 import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.apache.pulsar.common.semaphore.AsyncSemaphore;
 import org.apache.pulsar.common.tls.impl.TlsContextAcquisition;
 import org.apache.pulsar.common.tls.impl.TlsSynthesisSpec;
 import org.apache.pulsar.common.util.netty.DnsResolverUtil;
@@ -190,7 +192,7 @@ public class ProxyService implements Closeable {
     private final AsyncDualMemoryLimiterImpl maxTopicListInFlightLimiter;
 
     @Getter
-    private final TopicListSizeResultCache topicListSizeResultCache = new TopicListSizeResultCache();
+    private final TopicListSizeResultCache topicListSizeResultCache;
 
     private boolean gracefulShutdown = true;
 
@@ -272,6 +274,11 @@ public class ProxyService implements Closeable {
                 proxyConfig.getMaxTopicListInFlightDirectMemSizeMB() * 1024L * 1024L,
                 proxyConfig.getMaxTopicListInFlightDirectMemSizePermitsAcquireQueueSize(),
                 proxyConfig.getMaxTopicListInFlightDirectMemSizePermitsAcquireTimeoutMillis());
+        AsyncSemaphore topicListHeapLimiter =
+                maxTopicListInFlightLimiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY);
+        this.topicListSizeResultCache = new TopicListSizeResultCache(
+                proxyConfig.getMaxTopicListInFlightHeapMemSizeMB() * 1024L * 1024L,
+                () -> topicListHeapLimiter.getQueueSize() == 0 && topicListHeapLimiter.getAcquiredPermits() == 0);
     }
 
     public void start() throws Exception {