You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/20 12:54:28 UTC
(pulsar) branch master updated: [fix][broker] Fix topic listing memory limit enforcement (#26945)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new e4c0e7f8a5c9 [fix][broker] Fix topic listing memory limit enforcement (#26945)
e4c0e7f8a5c9 is described below
commit e4c0e7f8a5c9f76295b7a61f4fa048faaf2f4cb7
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Oct 20 15:54:21 2026 +0300
[fix][broker] Fix topic listing memory limit enforcement (#26945)
---
.../topiclistlimit/TopicListSizeResultCache.java | 287 ++++++++++++-
.../pulsar/broker/topiclistlimit/TopicListing.java | 65 +++
.../TopicListSizeResultCacheTest.java | 186 +++++++-
.../apache/pulsar/broker/admin/AdminResource.java | 15 +
.../pulsar/broker/admin/impl/NamespacesBase.java | 95 ++---
.../broker/admin/impl/PersistentTopicsBase.java | 55 ++-
.../admin/impl/TopicListResponseMemoryLimiter.java | 283 +++++++++++++
.../apache/pulsar/broker/admin/v2/Namespaces.java | 10 +-
.../broker/admin/v2/NonPersistentTopics.java | 90 ++--
.../pulsar/broker/admin/v2/PersistentTopics.java | 8 +-
.../pulsar/broker/admin/v2/ScalableTopics.java | 19 +-
.../pulsar/broker/service/BrokerService.java | 9 +-
.../pulsar/broker/service/PulsarCommandSender.java | 21 +-
.../broker/service/PulsarCommandSenderImpl.java | 15 +-
.../apache/pulsar/broker/service/ServerCnx.java | 76 +++-
.../pulsar/broker/service/TopicListService.java | 450 +++++++++++++++-----
.../broker/admin/TopicListRestMemoryLimitTest.java | 209 +++++++++
.../impl/TopicListResponseMemoryLimiterTest.java | 365 ++++++++++++++++
.../pulsar/broker/service/ServerCnxTest.java | 128 ++++++
.../broker/service/TopicListServiceTest.java | 469 ++++++++++++++++++++-
.../common/semaphore/AsyncDualMemoryLimiter.java | 18 +-
.../semaphore/AsyncDualMemoryLimiterUtil.java | 93 +++-
.../pulsar/common/semaphore/AsyncSemaphore.java | 16 +
.../semaphore/AsyncDualMemoryLimiterUtilTest.java | 160 +++++++
.../pulsar/proxy/server/LookupProxyHandler.java | 99 +++--
.../apache/pulsar/proxy/server/ProxyService.java | 9 +-
26 files changed, 2928 insertions(+), 322 deletions(-)
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCache.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCache.java
index 90e348447906..3a3896ef0f85 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCache.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCache.java
@@ -20,8 +20,18 @@ package org.apache.pulsar.broker.topiclistlimit;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Comparator;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
/**
@@ -32,18 +42,151 @@ import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
* - Maintain and update size estimates based on actual topic list sizes
* - Prevent thundering herd problems when multiple concurrent requests for a namespace are made without
* a previous size estimate
- * The cache uses namespace name and topic list mode (PERSISTENT/NON_PERSISTENT/ALL) as keys
+ * The cache uses the topic listing ({@link TopicListing}: namespace name, topic list mode
+ * (PERSISTENT/NON_PERSISTENT/ALL) and listing type) and the optional topic listing properties (PIP-452) as keys
* and maintains size estimates that are refined with actual usage.
*/
public class TopicListSizeResultCache {
// 10kB initial estimate for topic list heap size
private static final long INITIAL_TOPIC_LIST_HEAP_SIZE = 10 * 1024;
+ // how soon after measuring an oversized topic list the listing may measure it again while the limiter is idle
+ private static final Duration DEFAULT_OVERSIZED_REMEASURE_INTERVAL = Duration.ofMinutes(1);
+ // how soon after measuring an oversized topic list the listing may measure it again even if the limiter is busy
+ private static final Duration DEFAULT_OVERSIZED_FORCED_REMEASURE_INTERVAL = Duration.ofMinutes(10);
+ // how long a re-measuring request blocks other re-measuring requests unless it completes before
+ private static final Duration DEFAULT_REMEASURE_GATE_TIMEOUT = Duration.ofMinutes(1);
- private Cache<CacheKey, ResultHolder> topicListSizeCache = Caffeine.newBuilder()
+ private final Cache<CacheKey, ResultHolder> topicListSizeCache = Caffeine.newBuilder()
.maximumSize(10000)
.build();
+ private final OversizedRemeasurePolicy remeasurePolicy;
- record CacheKey(String namespaceName, CommandGetTopicsOfNamespace.Mode mode) {
+ /**
+ * Creates a cache without a size limit.
+ */
+ public TopicListSizeResultCache() {
+ this(0, () -> true);
+ }
+
+ /**
+ * Creates a cache for listings that are limited by the given topic list heap memory limit.
+ *
+ * <p>A topic list larger than the limit can't be served. Requests for a listing whose estimate is larger than
+ * the limit get the estimate as-is, so that they fail without waiting for permits. To let the listing recover
+ * after its topic list has become smaller, a request may get the limit as its estimate so that it lists the
+ * topics and measures the list again. That request needs all permits, and permits are granted in order, so
+ * while it waits for the permits of other listings, all later listings wait behind it. Therefore:
+ * <ul>
+ * <li>a listing is measured again at the earliest one interval after its oversized size was measured,</li>
+ * <li>only one listing at a time is measured again,</li>
+ * <li>and only while no other topic list is in flight or waiting for permits. When the limiter hasn't been
+ * idle for a longer interval, the listing is measured again anyway, so that it can recover on a broker that
+ * is never idle.</li>
+ * </ul>
+ *
+ * @param sizeLimit the topic list heap memory limit, use {@code <= 0} for no limit
+ * @param limiterIdle supplier that returns true when no topic list permits are acquired or requested
+ */
+ public TopicListSizeResultCache(long sizeLimit, BooleanSupplier limiterIdle) {
+ this(sizeLimit, DEFAULT_OVERSIZED_REMEASURE_INTERVAL, DEFAULT_OVERSIZED_FORCED_REMEASURE_INTERVAL,
+ limiterIdle);
+ }
+
+ @VisibleForTesting
+ TopicListSizeResultCache(long sizeLimit, Duration oversizedRemeasureInterval,
+ Duration oversizedForcedRemeasureInterval, BooleanSupplier limiterIdle) {
+ this.remeasurePolicy = new OversizedRemeasurePolicy(sizeLimit > 0 ? sizeLimit : Long.MAX_VALUE,
+ oversizedRemeasureInterval.toNanos(), oversizedForcedRemeasureInterval.toNanos(),
+ DEFAULT_REMEASURE_GATE_TIMEOUT.toNanos(), limiterIdle, new AtomicReference<>(),
+ new AtomicLong(System.nanoTime()));
+ }
+
+ /**
+ * Returns the topic list heap memory limit, {@link Long#MAX_VALUE} when there's no limit. An estimate larger
+ * than the limit can't be granted.
+ */
+ public long getSizeLimit() {
+ return remeasurePolicy.sizeLimit();
+ }
+
+ /**
+ * The listing that is being measured again, and the time when it stops blocking other listings from being
+ * measured again unless it records its size before that.
+ */
+ private record RemeasureGate(ResultHolder owner, long expiresAtNanos) {
+ }
+
+ /**
+ * The size limit and the rules for measuring an oversized topic list again.
+ *
+ * @param gate the listing that is being measured again. Only one listing is measured again at a time.
+ * @param nextForcedRemeasureNanos the earliest time when any listing may be measured again while the limiter
+ * is busy. The listings have separate estimates for each set of properties,
+ * which come from the clients, so the number of forced re-measures is limited
+ * for the whole broker.
+ */
+ private record OversizedRemeasurePolicy(long sizeLimit, long intervalNanos, long forcedIntervalNanos,
+ long gateTimeoutNanos, BooleanSupplier limiterIdle,
+ AtomicReference<RemeasureGate> gate,
+ AtomicLong nextForcedRemeasureNanos) {
+ static final OversizedRemeasurePolicy UNLIMITED = new OversizedRemeasurePolicy(Long.MAX_VALUE, 0, 0, 0,
+ () -> true, new AtomicReference<>(), new AtomicLong());
+
+ boolean tryForcedRemeasure(long now) {
+ long next = nextForcedRemeasureNanos.get();
+ return now - next >= 0 && nextForcedRemeasureNanos.compareAndSet(next, now + forcedIntervalNanos);
+ }
+
+ boolean tryEnterGate(ResultHolder holder, long now) {
+ RemeasureGate current = gate.get();
+ return (current == null || now - current.expiresAtNanos() >= 0)
+ && gate.compareAndSet(current, new RemeasureGate(holder, now + gateTimeoutNanos));
+ }
+
+ void exitGate(ResultHolder holder) {
+ RemeasureGate current = gate.get();
+ if (current != null && current.owner() == holder) {
+ gate.compareAndSet(current, null);
+ }
+ }
+ }
+
+ /**
+ * Cache key. The topic listing properties are stored as a hash so that the client provided property values
+ * aren't retained in the cache. A hash collision only results in sharing a size estimate.
+ */
+ record CacheKey(TopicListing listing, HashCode propertiesHash) {
+ CacheKey(String namespaceName, CommandGetTopicsOfNamespace.Mode mode) {
+ this(TopicListing.namespaceTopics(namespaceName, mode), (HashCode) null);
+ }
+
+ CacheKey(TopicListing listing, Map<String, String> properties) {
+ this(listing, hashProperties(properties));
+ }
+
+ private static HashCode hashProperties(Map<String, String> properties) {
+ if (properties == null || properties.isEmpty()) {
+ return null;
+ }
+ Hasher hasher = Hashing.murmur3_128().newHasher();
+ properties.entrySet().stream()
+ .sorted(Map.Entry.comparingByKey(Comparator.nullsFirst(Comparator.naturalOrder())))
+ .forEach(entry -> {
+ putString(hasher, entry.getKey());
+ putString(hasher, entry.getValue());
+ });
+ return hasher.hash();
+ }
+
+ private static void putString(Hasher hasher, String value) {
+ if (value == null) {
+ hasher.putInt(-1);
+ } else {
+ // length prefix keeps the encoding of the entries unambiguous
+ hasher.putInt(value.length());
+ hasher.putString(value, StandardCharsets.UTF_8);
+ }
+ }
}
/**
@@ -54,6 +197,29 @@ public class TopicListSizeResultCache {
public static class ResultHolder {
private final AtomicReference<CompletableFuture<Long>> topicListSizeFuture =
new AtomicReference<>(null);
+ private final OversizedRemeasurePolicy policy;
+ private final long sizeLimit;
+ private final long initialEstimate;
+ // the earliest time when a request for an oversized listing may measure the topic list again
+ private volatile long nextRemeasureNanos;
+ // the earliest time when the topic list is measured again even if the limiter is busy
+ private volatile long nextForcedRemeasureNanos;
+ // whether the topic list of this listing has been measured
+ private volatile boolean measured;
+
+ public ResultHolder() {
+ this(OversizedRemeasurePolicy.UNLIMITED, 0);
+ }
+
+ private ResultHolder(OversizedRemeasurePolicy policy, long knownSizeEstimate) {
+ this.policy = policy;
+ this.sizeLimit = policy.sizeLimit();
+ this.initialEstimate = Math.max(Math.min(INITIAL_TOPIC_LIST_HEAP_SIZE, sizeLimit), knownSizeEstimate);
+ long now = System.nanoTime();
+ // a listing that starts from an oversized known estimate hasn't been measured yet
+ this.nextRemeasureNanos = now;
+ this.nextForcedRemeasureNanos = now + policy.forcedIntervalNanos();
+ }
/**
* Get the topic list size estimate. The first request will return the initial estimate
@@ -61,16 +227,46 @@ public class TopicListSizeResultCache {
* will wait for the first request to complete and use the estimate of the first request.
* Subsequent requests will use estimate which gets updated based on the returned size of the topic list
* of each request.
+ * When the estimate is larger than the size limit, it's returned as-is so that the request fails without
+ * waiting for permits, except for a request that gets the size limit so that it can measure the topic
+ * list again (see {@link TopicListSizeResultCache#TopicListSizeResultCache(long, BooleanSupplier)}).
* @return a future that will return the topic list size estimate
*/
public CompletableFuture<Long> getSizeAsync() {
+ CompletableFuture<Long> estimate;
if (topicListSizeFuture.compareAndSet(null, new CompletableFuture<>())) {
// let the first request proceed with the initial estimate
- return CompletableFuture.completedFuture(INITIAL_TOPIC_LIST_HEAP_SIZE);
+ estimate = CompletableFuture.completedFuture(initialEstimate);
} else {
// all other requests wait for the first one to complete
- return topicListSizeFuture.get();
+ estimate = topicListSizeFuture.get();
+ }
+ return sizeLimit == Long.MAX_VALUE ? estimate : estimate.thenApply(this::limitOversizedEstimate);
+ }
+
+ private long limitOversizedEstimate(long estimate) {
+ if (estimate <= sizeLimit) {
+ return estimate;
+ }
+ long now = System.nanoTime();
+ // Requesting all permits while other topic lists are in flight would make the request wait for them,
+ // and all later requests would wait behind it, since the permits are granted in order. Measure the
+ // list again only when no other topic list is in flight, or when the limiter hasn't been idle for a
+ // longer time, and only one listing at a time.
+ boolean limiterIdle = policy.limiterIdle().getAsBoolean();
+ if (now - nextRemeasureNanos >= 0
+ && (limiterIdle || now - nextForcedRemeasureNanos >= 0)
+ && policy.tryEnterGate(this, now)) {
+ if (limiterIdle || policy.tryForcedRemeasure(now)) {
+ nextRemeasureNanos = now + policy.intervalNanos();
+ nextForcedRemeasureNanos = now + policy.forcedIntervalNanos();
+ // a request that can be granted, so that the topic list gets measured again
+ return sizeLimit;
+ }
+ policy.exitGate(this);
}
+ // acquiring permits for an estimate larger than the limit fails right away
+ return estimate;
}
/**
@@ -79,6 +275,15 @@ public class TopicListSizeResultCache {
* @param actualSize the actual size of the topic list
*/
public void updateSize(long actualSize) {
+ measured = true;
+ if (actualSize > sizeLimit) {
+ // the topic list can be measured again after the interval
+ long now = System.nanoTime();
+ nextRemeasureNanos = now + policy.intervalNanos();
+ nextForcedRemeasureNanos = now + policy.forcedIntervalNanos();
+ }
+ // the topic list has been measured, another listing may be measured again
+ policy.exitGate(this);
CompletableFuture<Long> currentFuture = topicListSizeFuture.get();
if (currentFuture != null && !currentFuture.isDone()) {
// complete the future if it's not done yet
@@ -103,9 +308,31 @@ public class TopicListSizeResultCache {
});
// let all current requests proceed with the initial estimate if this were to happen
if (currentFuture != null && !currentFuture.isDone()) {
- currentFuture.complete(INITIAL_TOPIC_LIST_HEAP_SIZE);
+ currentFuture.complete(initialEstimate);
}
}
+
+ /**
+ * Returns true when the topic list of this listing has been measured. A listing with a new set of properties
+ * starts from the estimate of the same listing without properties, which can be larger than the limit
+ * although its own topic list hasn't been measured yet.
+ *
+ * @return true if the topic list has been measured
+ */
+ public boolean isMeasured() {
+ return measured;
+ }
+
+ /**
+ * Returns the current size estimate, or 0 when there's no measured estimate yet.
+ */
+ long getKnownSizeEstimate() {
+ CompletableFuture<Long> currentFuture = topicListSizeFuture.get();
+ if (currentFuture == null || !currentFuture.isDone() || currentFuture.isCompletedExceptionally()) {
+ return 0;
+ }
+ return currentFuture.getNow(0L);
+ }
}
/**
@@ -116,7 +343,51 @@ public class TopicListSizeResultCache {
*/
public ResultHolder getTopicListSize(String namespaceName,
CommandGetTopicsOfNamespace.Mode mode) {
- return topicListSizeCache.asMap()
- .computeIfAbsent(new CacheKey(namespaceName, mode), __ -> new ResultHolder());
+ return getTopicListSize(namespaceName, mode, null);
+ }
+
+ /**
+ * Get the topic list size result holder for the given namespace, mode and topic listing properties.
+ * The properties are passed to a customized topic listing implementation (PIP-452) which can return
+ * a different topic list for different properties. Therefore, the estimate is kept separately for each
+ * distinct set of properties.
+ * @param namespaceName the namespace name in the format of "tenant/namespace"
+ * @param mode the mode of the topic list request (PERSISTENT, NON_PERSISTENT, ALL)
+ * @param properties the topic listing properties, null or empty when not provided
+ * @return the topic list size result holder
+ */
+ public ResultHolder getTopicListSize(String namespaceName,
+ CommandGetTopicsOfNamespace.Mode mode,
+ Map<String, String> properties) {
+ return getTopicListSize(TopicListing.namespaceTopics(namespaceName, mode), properties);
+ }
+
+ /**
+ * Get the topic list size result holder for the given topic listing and topic listing properties.
+ * @param listing the topic listing
+ * @param properties the topic listing properties (PIP-452), null or empty when not provided
+ * @return the topic list size result holder
+ */
+ public ResultHolder getTopicListSize(TopicListing listing, Map<String, String> properties) {
+ CacheKey key = new CacheKey(listing, properties);
+ ResultHolder holder = topicListSizeCache.getIfPresent(key);
+ if (holder != null) {
+ return holder;
+ }
+ // The properties come from the client. A listing with a new set of properties starts from the estimate of
+ // the same listing without properties, so that a request can't start below the known size of the listing
+ // by sending new property values. With the default topic listing, which ignores the properties, the lists
+ // are the same.
+ long knownSizeEstimate = 0;
+ if (key.propertiesHash() != null) {
+ ResultHolder holderWithoutProperties =
+ topicListSizeCache.getIfPresent(new CacheKey(listing, (HashCode) null));
+ if (holderWithoutProperties != null) {
+ knownSizeEstimate = holderWithoutProperties.getKnownSizeEstimate();
+ }
+ }
+ long initialSizeEstimate = knownSizeEstimate;
+ return topicListSizeCache.asMap().computeIfAbsent(key,
+ __ -> new ResultHolder(remeasurePolicy, initialSizeEstimate));
}
}
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListing.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListing.java
new file mode 100644
index 000000000000..e741fae5c8d7
--- /dev/null
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/topiclistlimit/TopicListing.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.topiclistlimit;
+
+import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode;
+
+/**
+ * Identifies a topic listing for the topic list size estimates of {@link TopicListSizeResultCache}.
+ *
+ * <p>Listings that return different topic lists need separate size estimates, so that their estimates don't
+ * replace each other.
+ *
+ * @param namespaceName the namespace name in the format of "tenant/namespace"
+ * @param mode the mode of the listing (PERSISTENT, NON_PERSISTENT, ALL)
+ * @param type the type of the listing
+ * @param bundleRange the bundle range for listings of a single bundle, null otherwise
+ */
+public record TopicListing(String namespaceName, Mode mode, Type type, String bundleRange) {
+
+ /**
+ * The type of a topic listing.
+ */
+ public enum Type {
+ /** The topics of a namespace. */
+ NAMESPACE_TOPICS,
+ /** The topics that the owner of a bundle has loaded in that bundle. */
+ OWNED_BUNDLE_TOPICS,
+ /** The partitioned topics of a namespace. */
+ PARTITIONED_TOPICS,
+ /** The scalable topics of a namespace. */
+ SCALABLE_TOPICS
+ }
+
+ public static TopicListing namespaceTopics(String namespaceName, Mode mode) {
+ return new TopicListing(namespaceName, mode, Type.NAMESPACE_TOPICS, null);
+ }
+
+ public static TopicListing ownedBundleTopics(String namespaceName, Mode mode, String bundleRange) {
+ return new TopicListing(namespaceName, mode, Type.OWNED_BUNDLE_TOPICS, bundleRange);
+ }
+
+ public static TopicListing partitionedTopics(String namespaceName, Mode mode) {
+ return new TopicListing(namespaceName, mode, Type.PARTITIONED_TOPICS, null);
+ }
+
+ public static TopicListing scalableTopics(String namespaceName) {
+ return new TopicListing(namespaceName, Mode.ALL, Type.SCALABLE_TOPICS, null);
+ }
+}
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCacheTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCacheTest.java
index 397582d7022e..68480ddc7494 100644
--- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCacheTest.java
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/topiclistlimit/TopicListSizeResultCacheTest.java
@@ -25,15 +25,20 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNotSame;
import static org.testng.Assert.assertSame;
import static org.testng.Assert.assertTrue;
+import java.time.Duration;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
+import org.awaitility.Awaitility;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -67,6 +72,53 @@ public class TopicListSizeResultCacheTest {
assertSame(holder1, holder2, "Should return the same ResultHolder instance for the same key");
}
+ @Test
+ public void testGetTopicListSize_keysByProperties() {
+ String namespace = "tenant/namespace";
+ CommandGetTopicsOfNamespace.Mode mode = CommandGetTopicsOfNamespace.Mode.PERSISTENT;
+
+ TopicListSizeResultCache.ResultHolder noProperties = cache.getTopicListSize(namespace, mode);
+ assertSame(cache.getTopicListSize(namespace, mode, null), noProperties);
+ assertSame(cache.getTopicListSize(namespace, mode, Map.of()), noProperties);
+
+ Map<String, String> properties = new HashMap<>();
+ properties.put("k1", "v1");
+ properties.put("k2", "v2");
+ TopicListSizeResultCache.ResultHolder withProperties = cache.getTopicListSize(namespace, mode, properties);
+ assertNotSame(withProperties, noProperties,
+ "A customized topic listing with properties should have a separate size estimate");
+ assertSame(cache.getTopicListSize(namespace, mode, Map.of("k2", "v2", "k1", "v1")), withProperties);
+ // the key must not be affected by later modifications of the passed map
+ properties.put("k3", "v3");
+ assertSame(cache.getTopicListSize(namespace, mode, Map.of("k2", "v2", "k1", "v1")), withProperties);
+ assertNotSame(cache.getTopicListSize(namespace, mode, Map.of("k1", "other")), withProperties);
+ }
+
+ @Test
+ public void testGetTopicListSize_keysByListing() {
+ String namespace = "tenant/namespace";
+ CommandGetTopicsOfNamespace.Mode mode = CommandGetTopicsOfNamespace.Mode.NON_PERSISTENT;
+
+ TopicListSizeResultCache.ResultHolder namespaceTopics = cache.getTopicListSize(namespace, mode);
+ assertSame(cache.getTopicListSize(TopicListing.namespaceTopics(namespace, mode), null), namespaceTopics);
+
+ List<TopicListing> otherListings = List.of(
+ TopicListing.ownedBundleTopics(namespace, mode, "0x00000000_0x80000000"),
+ TopicListing.ownedBundleTopics(namespace, mode, "0x80000000_0xffffffff"),
+ TopicListing.partitionedTopics(namespace, mode),
+ TopicListing.scalableTopics(namespace));
+ List<TopicListSizeResultCache.ResultHolder> holders = new ArrayList<>();
+ holders.add(namespaceTopics);
+ for (TopicListing listing : otherListings) {
+ TopicListSizeResultCache.ResultHolder holder = cache.getTopicListSize(listing, null);
+ assertSame(cache.getTopicListSize(listing, Map.of()), holder);
+ for (TopicListSizeResultCache.ResultHolder other : holders) {
+ assertNotSame(holder, other, "Each topic listing should have a separate size estimate: " + listing);
+ }
+ holders.add(holder);
+ }
+ }
+
@Test
public void testGetTopicListSize_returnsDifferentInstancesForDifferentKeys() {
String namespace1 = "tenant/namespace1";
@@ -329,4 +381,136 @@ public class TopicListSizeResultCacheTest {
assertNotNull(future.get(5, TimeUnit.SECONDS), "All futures should complete");
}
}
-}
\ No newline at end of file
+
+ @Test
+ public void testOversizedEstimateFailsFastUntilIntervalHasPassed() {
+ TopicListSizeResultCache limitedCache =
+ new TopicListSizeResultCache(100_000, Duration.ofHours(1), Duration.ofHours(1), () -> true);
+ TopicListSizeResultCache.ResultHolder holder =
+ limitedCache.getTopicListSize("tenant/namespace", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+ holder.getSizeAsync();
+ // a topic list larger than the limit
+ holder.updateSize(500_000);
+ // the requests get the estimate, which is larger than the limit, so that they fail right away instead of
+ // listing the topics again right after the oversized list was measured
+ assertEquals(holder.getSizeAsync().join().longValue(), 500_000L);
+ assertEquals(holder.getSizeAsync().join().longValue(), 500_000L);
+ // the estimate follows the list when it becomes smaller than the limit
+ holder.updateSize(50_000);
+ assertEquals(holder.getSizeAsync().join().longValue(), 50_000L);
+ }
+
+ @Test
+ public void testOversizedEstimateIsRemeasuredOnlyWhenLimiterIsIdle() {
+ AtomicBoolean limiterIdle = new AtomicBoolean(false);
+ TopicListSizeResultCache limitedCache =
+ new TopicListSizeResultCache(100_000, Duration.ZERO, Duration.ofHours(1), limiterIdle::get);
+ TopicListSizeResultCache.ResultHolder holder =
+ limitedCache.getTopicListSize("tenant/namespace", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+ holder.getSizeAsync();
+ holder.updateSize(500_000);
+ // other topic lists are in flight: requesting all permits would make the later requests wait behind it
+ assertEquals(holder.getSizeAsync().join().longValue(), 500_000L);
+ // no other topic list is in flight: the request gets the limit so that it can measure the list again
+ limiterIdle.set(true);
+ assertEquals(holder.getSizeAsync().join().longValue(), 100_000L);
+ }
+
+ @Test
+ public void testNewPropertiesStartFromEstimateWithoutProperties() {
+ String namespace = "tenant/namespace";
+ CommandGetTopicsOfNamespace.Mode mode = CommandGetTopicsOfNamespace.Mode.PERSISTENT;
+ // without a known estimate, a listing with properties starts from the initial estimate
+ TopicListSizeResultCache.ResultHolder first = cache.getTopicListSize(namespace, mode, Map.of("k", "v0"));
+ assertEquals(first.getSizeAsync().join().longValue(), 10 * 1024L);
+
+ TopicListSizeResultCache.ResultHolder withoutProperties = cache.getTopicListSize(namespace, mode);
+ withoutProperties.getSizeAsync();
+ withoutProperties.updateSize(500_000);
+ // a listing with new properties can't start below the known size of the listing without properties
+ TopicListSizeResultCache.ResultHolder withNewProperties =
+ cache.getTopicListSize(namespace, mode, Map.of("k", "v1"));
+ assertEquals(withNewProperties.getSizeAsync().join().longValue(), 500_000L);
+ // and then follows its own topic list
+ withNewProperties.updateSize(1000);
+ assertEquals(withNewProperties.getSizeAsync().join().longValue(), 1000L);
+ assertEquals(withoutProperties.getSizeAsync().join().longValue(), 500_000L);
+ }
+
+ @Test
+ public void testNewPropertiesWithOversizedEstimateFailFastWhileLimiterIsBusy() {
+ AtomicBoolean limiterIdle = new AtomicBoolean(false);
+ TopicListSizeResultCache limitedCache =
+ new TopicListSizeResultCache(100_000, Duration.ofHours(1), Duration.ofHours(1), limiterIdle::get);
+ String namespace = "tenant/namespace";
+ CommandGetTopicsOfNamespace.Mode mode = CommandGetTopicsOfNamespace.Mode.PERSISTENT;
+ TopicListSizeResultCache.ResultHolder withoutProperties = limitedCache.getTopicListSize(namespace, mode);
+ withoutProperties.getSizeAsync();
+ withoutProperties.updateSize(500_000);
+
+ TopicListSizeResultCache.ResultHolder withNewProperties =
+ limitedCache.getTopicListSize(namespace, mode, Map.of("k", "v1"));
+ assertEquals(withNewProperties.getSizeAsync().join().longValue(), 500_000L);
+ // the request is rejected, which resets the estimate that it was initializing
+ withNewProperties.resetIfInitializing();
+ // a listing with new properties hasn't been measured, so it may be measured when the limiter is idle
+ limiterIdle.set(true);
+ assertEquals(withNewProperties.getSizeAsync().join().longValue(), 100_000L);
+ }
+
+ @Test
+ public void testOnlyOneOversizedListingIsRemeasuredAtATime() {
+ TopicListSizeResultCache limitedCache =
+ new TopicListSizeResultCache(100_000, Duration.ZERO, Duration.ofHours(1), () -> true);
+ TopicListSizeResultCache.ResultHolder first =
+ limitedCache.getTopicListSize("tenant/ns1", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+ TopicListSizeResultCache.ResultHolder second =
+ limitedCache.getTopicListSize("tenant/ns2", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+ for (TopicListSizeResultCache.ResultHolder holder : List.of(first, second)) {
+ holder.getSizeAsync();
+ holder.updateSize(500_000);
+ }
+ // the first listing is measured again, the second one fails fast while that's in progress
+ assertEquals(first.getSizeAsync().join().longValue(), 100_000L);
+ assertEquals(second.getSizeAsync().join().longValue(), 500_000L);
+ // the first listing measured its size, so the second listing may be measured again
+ first.updateSize(500_000);
+ assertEquals(second.getSizeAsync().join().longValue(), 100_000L);
+ }
+
+ @Test
+ public void testOversizedEstimateIsRemeasuredWhenLimiterIsNeverIdle() {
+ TopicListSizeResultCache limitedCache =
+ new TopicListSizeResultCache(100_000, Duration.ZERO, Duration.ZERO, () -> false);
+ TopicListSizeResultCache.ResultHolder holder =
+ limitedCache.getTopicListSize("tenant/namespace", CommandGetTopicsOfNamespace.Mode.PERSISTENT);
+ holder.getSizeAsync();
+ holder.updateSize(500_000);
+ // after the longer interval, the listing is measured again although other topic lists are in flight
+ assertEquals(holder.getSizeAsync().join().longValue(), 100_000L);
+ }
+
+ @Test
+ public void testForcedRemeasuresAreLimitedForTheWholeCache() throws Exception {
+ Duration forcedInterval = Duration.ofMillis(500);
+ TopicListSizeResultCache limitedCache =
+ new TopicListSizeResultCache(100_000, Duration.ZERO, forcedInterval, () -> false);
+ // listings with different properties have separate estimates
+ TopicListSizeResultCache.ResultHolder first = limitedCache.getTopicListSize("tenant/namespace",
+ CommandGetTopicsOfNamespace.Mode.PERSISTENT, Map.of("k", "v1"));
+ TopicListSizeResultCache.ResultHolder second = limitedCache.getTopicListSize("tenant/namespace",
+ CommandGetTopicsOfNamespace.Mode.PERSISTENT, Map.of("k", "v2"));
+ for (TopicListSizeResultCache.ResultHolder holder : List.of(first, second)) {
+ holder.getSizeAsync();
+ holder.updateSize(500_000);
+ }
+ Thread.sleep(forcedInterval.toMillis() + 100);
+ // the limiter is busy: the first listing is measured again after the longer interval
+ assertEquals(first.getSizeAsync().join().longValue(), 100_000L);
+ first.updateSize(500_000);
+ // the second listing's interval has also passed, but forced re-measures are limited for the whole cache
+ assertEquals(second.getSizeAsync().join().longValue(), 500_000L);
+ Awaitility.await().untilAsserted(
+ () -> assertEquals(second.getSizeAsync().join().longValue(), 100_000L));
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
index b94dcfde51c2..84a548ddc5ce 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
@@ -45,6 +45,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.admin.impl.TopicListResponseMemoryLimiter;
import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
import org.apache.pulsar.broker.authorization.AuthorizationService;
import org.apache.pulsar.broker.namespace.TopicExistsInfo;
@@ -1020,6 +1021,20 @@ public abstract class AdminResource extends PulsarWebResource {
== Status.NOT_FOUND.getStatusCode();
}
+ /**
+ * Checks whether a topic listing failed because the topic list memory limit rejected it (429) or because the
+ * request was cancelled (503). These are expected responses that don't need to be logged as errors.
+ */
+ protected static boolean isTopicListLimitRejection(Throwable ex) {
+ Throwable realCause = FutureUtil.unwrapCompletionException(ex);
+ if (!(realCause instanceof WebApplicationException webApplicationException)) {
+ return false;
+ }
+ // other 503 responses, such as a bundle owner that isn't ready, are logged as errors
+ return webApplicationException.getResponse().getStatus() == Status.TOO_MANY_REQUESTS.getStatusCode()
+ || TopicListResponseMemoryLimiter.isRequestCancelled(realCause);
+ }
+
protected static boolean is4xxRestException(Throwable ex) {
Throwable realCause = FutureUtil.unwrapCompletionException(ex);
return realCause instanceof WebApplicationException
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index 9cc0473dd5fc..b0f9ffaae813 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -22,7 +22,6 @@ import static org.apache.pulsar.common.policies.data.PoliciesUtil.getBundles;
import com.google.common.collect.Sets;
import jakarta.ws.rs.WebApplicationException;
import jakarta.ws.rs.container.AsyncResponse;
-import jakarta.ws.rs.container.CompletionCallback;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.Response.Status;
import jakarta.ws.rs.core.UriBuilder;
@@ -44,10 +43,8 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
-import java.util.function.BooleanSupplier;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -70,8 +67,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentReplicator;
import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.broker.service.scalable.AutoScaleConfig;
-import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
-import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
import org.apache.pulsar.broker.web.RestException;
import org.apache.pulsar.client.admin.GrantTopicPermissionOptions;
import org.apache.pulsar.client.admin.Namespaces;
@@ -122,8 +118,6 @@ import org.apache.pulsar.common.policies.data.TopicType;
import org.apache.pulsar.common.policies.data.ValidateResult;
import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl;
import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl;
-import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
-import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
import org.apache.pulsar.common.util.Codec;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.MetadataStoreException;
@@ -186,76 +180,39 @@ public abstract class NamespacesBase extends AdminResource {
CommandGetTopicsOfNamespace.Mode mode) {
// Use maxTopicListInFlightLimiter to limit inflight get topic listing responses
// to avoid OOME caused by a lot of clients using HTTP service lookups to list topics
- AsyncDualMemoryLimiterImpl maxTopicListInFlightLimiter =
- pulsar().getBrokerService().getMaxTopicListInFlightLimiter();
- TopicListSizeResultCache.ResultHolder listSizeHolder = pulsar().getBrokerService().getTopicListSizeResultCache()
- .getTopicListSize(namespaceName.toString(), mode);
- // setup the permit cancellation function
- AtomicBoolean permitRequestCancelled = new AtomicBoolean(false);
- BooleanSupplier isPermitRequestCancelled = permitRequestCancelled::get;
- // add callback that releases permits when the response completes
- AtomicReference<AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit> initialPermitsRef =
- new AtomicReference<>();
- AtomicReference<AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit> permitsRef = new AtomicReference<>();
- response.register(new CompletionCallback() {
- @Override
- public void onComplete(Throwable throwable) {
- if (throwable != null) {
- // for failed request
- // handle resetting the TopicListSizeResultCache.ResultHolder
- listSizeHolder.resetIfInitializing();
- // cancel any pending permit request
- permitRequestCancelled.set(true);
- }
- AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit initialPermit = initialPermitsRef.get();
- if (initialPermit != null) {
- maxTopicListInFlightLimiter.release(initialPermit);
- }
- AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit permits = permitsRef.get();
- if (permits != null) {
- maxTopicListInFlightLimiter.release(permits);
- }
- }
- });
- return listSizeHolder.getSizeAsync().thenCompose(initialSize -> maxTopicListInFlightLimiter.acquire(initialSize,
- AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY, isPermitRequestCancelled).exceptionally(t -> {
- throw new CompletionException(
- new RestException(Status.TOO_MANY_REQUESTS, "Failed due to heap memory limit exceeded"));
- }).thenCompose(initialPermits -> {
- initialPermitsRef.set(initialPermits);
- // perform the actual get list of topics operation
- return doInternalGetListOfTopics(policies, mode).thenCompose(topicList -> {
- long actualSize = TopicListMemoryLimiter.estimateTopicListSize(topicList);
- listSizeHolder.updateSize(actualSize);
- return maxTopicListInFlightLimiter.update(initialPermits, actualSize, isPermitRequestCancelled)
- .exceptionally(t -> {
- throw new CompletionException(new RestException(Status.TOO_MANY_REQUESTS,
- "Failed due to heap memory limit exceeded"));
- }).thenApply(permits -> {
- permitsRef.set(permits);
- initialPermitsRef.set(null);
- return topicList;
- });
- });
- }));
- }
-
- private CompletableFuture<List<String>> doInternalGetListOfTopics(Policies policies,
- CommandGetTopicsOfNamespace.Mode mode) {
switch (mode) {
case ALL:
- return pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName)
- .thenCombine(internalGetNonPersistentTopics(policies),
- (persistentTopics, nonPersistentTopics) ->
- ListUtils.union(persistentTopics, nonPersistentTopics));
+ // The non-persistent topics are collected first from the bundle listings, which acquire permits
+ // from the same limiter, possibly on this broker, so no permits are held while waiting for them.
+ // The listing then acquires permits once, for the whole topic list, with the estimate of the ALL
+ // listing, so that it doesn't hold permits for one part while waiting for permits for the other.
+ return TopicListResponseMemoryLimiter.listTopicsAfterCollecting(pulsar().getBrokerService(),
+ response, TopicListing.namespaceTopics(namespaceName.toString(), mode), null,
+ () -> internalGetNonPersistentTopics(policies),
+ nonPersistentTopics -> pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName)
+ .thenApply(persistentTopics ->
+ ListUtils.union(persistentTopics, nonPersistentTopics)));
case NON_PERSISTENT:
- return internalGetNonPersistentTopics(policies);
+ return listNonPersistentTopicsWithMemoryLimit(response, policies);
case PERSISTENT:
default:
- return pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName);
+ return listPersistentTopicsWithMemoryLimit(response);
}
}
+ private CompletableFuture<List<String>> listPersistentTopicsWithMemoryLimit(AsyncResponse response) {
+ return TopicListResponseMemoryLimiter.listTopics(pulsar().getBrokerService(), response,
+ TopicListing.namespaceTopics(namespaceName.toString(), CommandGetTopicsOfNamespace.Mode.PERSISTENT),
+ null, () -> pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName));
+ }
+
+ private CompletableFuture<List<String>> listNonPersistentTopicsWithMemoryLimit(AsyncResponse response,
+ Policies policies) {
+ // the non-persistent topics are collected from the bundle topic listings of the bundle owners
+ return TopicListResponseMemoryLimiter.listTopicsFromOtherListings(pulsar().getBrokerService(), response,
+ () -> internalGetNonPersistentTopics(policies));
+ }
+
protected CompletableFuture<List<String>> internalGetNonPersistentTopics(Policies policies) {
final List<CompletableFuture<List<String>>> futures = new ArrayList<>();
final List<String> boundaries = policies.bundles.getBoundaries();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 139c106a4f46..0eaa171770e8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -56,6 +56,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
@@ -91,6 +92,7 @@ import org.apache.pulsar.broker.service.TopicPoliciesService;
import org.apache.pulsar.broker.service.persistent.PersistentReplicator;
import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
import org.apache.pulsar.broker.web.RestException;
import org.apache.pulsar.client.admin.LongRunningProcessStatus;
import org.apache.pulsar.client.admin.OffloadProcessStatus;
@@ -170,6 +172,21 @@ public class PersistentTopicsBase extends AdminResource {
protected CompletableFuture<List<String>> internalGetListAsync(Optional<String> bundle,
@Nullable Map<String, String> properties) {
+ return internalGetListAsync(null, bundle, properties);
+ }
+
+ /**
+ * Get the list of persistent topics in the namespace.
+ *
+ * @param asyncResponse the async response of the REST request. When provided, the topic list heap memory limit
+ * is applied and the permits are held until the response completes.
+ * @param bundle the optional bundle to filter the topics
+ * @param properties the topic listing properties (PIP-452)
+ * @return the list of topics
+ */
+ protected CompletableFuture<List<String>> internalGetListAsync(@Nullable AsyncResponse asyncResponse,
+ Optional<String> bundle,
+ @Nullable Map<String, String> properties) {
return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPICS)
.thenCompose(__ -> namespaceResources().namespaceExistsAsync(namespaceName))
.thenAccept(exists -> {
@@ -177,8 +194,18 @@ public class PersistentTopicsBase extends AdminResource {
throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
}
})
- .thenCompose(__ -> pulsar().getNamespaceService().getListOfTopicsByProperties(namespaceName,
- CommandGetTopicsOfNamespace.Mode.PERSISTENT, properties))
+ .thenCompose(__ -> {
+ Supplier<CompletableFuture<List<String>>> topicsSupplier = () -> pulsar().getNamespaceService()
+ .getListOfTopicsByProperties(namespaceName, CommandGetTopicsOfNamespace.Mode.PERSISTENT,
+ properties);
+ if (asyncResponse == null) {
+ return topicsSupplier.get();
+ }
+ return TopicListResponseMemoryLimiter.listTopics(pulsar().getBrokerService(), asyncResponse,
+ TopicListing.namespaceTopics(namespaceName.toString(),
+ CommandGetTopicsOfNamespace.Mode.PERSISTENT),
+ properties, topicsSupplier);
+ })
.thenApply(topics ->
topics.stream()
.filter(topic -> {
@@ -197,6 +224,18 @@ public class PersistentTopicsBase extends AdminResource {
}
protected CompletableFuture<List<String>> internalGetPartitionedTopicListAsync() {
+ return internalGetPartitionedTopicListAsync(null);
+ }
+
+ /**
+ * Get the list of partitioned topics in the namespace.
+ *
+ * @param asyncResponse the async response of the REST request. When provided, the topic list heap memory limit
+ * is applied and the permits are held until the response completes.
+ * @return the list of partitioned topics
+ */
+ protected CompletableFuture<List<String>> internalGetPartitionedTopicListAsync(
+ @Nullable AsyncResponse asyncResponse) {
return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPICS)
.thenCompose(__ -> namespaceResources().namespaceExistsAsync(namespaceName))
.thenCompose(namespaceExists -> {
@@ -206,9 +245,17 @@ public class PersistentTopicsBase extends AdminResource {
.attr("namespace", namespaceName)
.log("Failed to get partitioned topic list : Namespace does not exist");
throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
- } else {
- return getPartitionedTopicListAsync(TopicDomain.getEnum(domain()));
}
+ TopicDomain topicDomain = TopicDomain.getEnum(domain());
+ if (asyncResponse == null) {
+ return getPartitionedTopicListAsync(topicDomain);
+ }
+ CommandGetTopicsOfNamespace.Mode mode = topicDomain == TopicDomain.persistent
+ ? CommandGetTopicsOfNamespace.Mode.PERSISTENT
+ : CommandGetTopicsOfNamespace.Mode.NON_PERSISTENT;
+ return TopicListResponseMemoryLimiter.listTopics(pulsar().getBrokerService(), asyncResponse,
+ TopicListing.partitionedTopics(namespaceName.toString(), mode), null,
+ () -> getPartitionedTopicListAsync(topicDomain));
});
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiter.java
new file mode 100644
index 000000000000..7f11764c0b7a
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiter.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.impl;
+
+import jakarta.ws.rs.container.AsyncResponse;
+import jakarta.ws.rs.container.CompletionCallback;
+import jakarta.ws.rs.core.Response.Status;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
+import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
+import org.apache.pulsar.broker.web.RestException;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil;
+import org.apache.pulsar.common.semaphore.AsyncSemaphore;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.jspecify.annotations.Nullable;
+
+/**
+ * Applies the topic list heap memory limit (PIP-442) to REST topic listing requests.
+ *
+ * <p>The heap memory permits are held until the {@link AsyncResponse} has completed, since the topic list is
+ * referenced until the response has been written. The request is treated as cancelled once the response is done
+ * (for example, after the async response has timed out), so that a request that is no longer waited for doesn't
+ * list the topics, and permits that are granted after the response has completed are released immediately.
+ * A response can combine several listings, each of them holds its own permits until the response completes.
+ */
+public final class TopicListResponseMemoryLimiter {
+
+ /**
+ * The 503 response of a topic listing that was cancelled while it was waiting for permits, for example since
+ * the response had already completed.
+ */
+ static final class RequestCancelledException extends RestException {
+ private static final long serialVersionUID = 1L;
+
+ RequestCancelledException() {
+ super(Status.SERVICE_UNAVAILABLE, "Request was cancelled");
+ }
+ }
+
+ /**
+ * Checks whether a topic listing failed since it was cancelled while it was waiting for permits.
+ *
+ * @param ex the failure, possibly wrapped in a {@link CompletionException}
+ * @return true if the request was cancelled
+ */
+ public static boolean isRequestCancelled(Throwable ex) {
+ return FutureUtil.unwrapCompletionException(ex) instanceof RequestCancelledException;
+ }
+ private TopicListResponseMemoryLimiter() {
+ }
+
+ /**
+ * Lists topics while holding heap memory permits for the estimated size of the topic list.
+ *
+ * @param brokerService the broker service
+ * @param response the async response of the REST request, the permits are released when it completes
+ * @param listing the topic listing, which identifies the topic list size estimate
+ * @param properties the topic listing properties (PIP-452), null when not provided
+ * @param topicsSupplier supplier that performs the topic listing
+ * @return a future that completes with the topic list
+ */
+ public static CompletableFuture<List<String>> listTopics(BrokerService brokerService, AsyncResponse response,
+ TopicListing listing,
+ @Nullable Map<String, String> properties,
+ Supplier<CompletableFuture<List<String>>>
+ topicsSupplier) {
+ return listTopicsAfterCollecting(brokerService, response, listing, properties,
+ () -> CompletableFuture.completedFuture(null), __ -> topicsSupplier.get());
+ }
+
+ /**
+ * Lists topics while holding heap memory permits for the estimated size of the topic list, after collecting
+ * part of the topics from other topic listings without holding permits.
+ *
+ * <p>Use this for a listing that combines topics from topic listings that apply the topic list memory limit
+ * themselves, such as the bundle topic listings of the bundle owners, which can be this broker, with topics
+ * that it lists itself. The size estimate is checked before collecting, so that a listing whose estimate is
+ * larger than the limit fails without collecting. The permits are acquired once for the whole topic list
+ * after collecting, so that the listing doesn't hold permits while waiting for the other listings.
+ *
+ * @param brokerService the broker service
+ * @param response the async response of the REST request, the permits are released when it completes
+ * @param listing the topic listing, which identifies the topic list size estimate
+ * @param properties the topic listing properties (PIP-452), null when not provided
+ * @param collector supplier that collects the topics from the other listings
+ * @param topicsSupplier function that performs the topic listing, given the collected topics
+ * @return a future that completes with the topic list
+ */
+ public static <CollectedT> CompletableFuture<List<String>> listTopicsAfterCollecting(
+ BrokerService brokerService, AsyncResponse response, TopicListing listing,
+ @Nullable Map<String, String> properties, Supplier<CompletableFuture<CollectedT>> collector,
+ Function<CollectedT, CompletableFuture<List<String>>> topicsSupplier) {
+ TopicListSizeResultCache sizeCache = brokerService.getTopicListSizeResultCache();
+ TopicListSizeResultCache.ResultHolder listSizeHolder = sizeCache.getTopicListSize(listing, properties);
+ ResponsePermits permits =
+ new ResponsePermits(brokerService.getMaxTopicListInFlightLimiter(), response, listSizeHolder);
+ response.register(permits);
+ return listSizeHolder.getSizeAsync()
+ .thenCompose(initialSize -> {
+ if (initialSize > sizeCache.getSizeLimit()) {
+ // the permits can't be granted, skip collecting
+ return permits.acquire(initialSize).thenCompose(__ -> topicsSupplier.apply(null));
+ }
+ return collector.get().thenCompose(collected -> permits.acquire(initialSize)
+ .thenCompose(__ -> topicsSupplier.apply(collected)));
+ })
+ .thenCompose(topicList -> {
+ long actualSize = TopicListMemoryLimiter.estimateTopicListSize(topicList);
+ listSizeHolder.updateSize(actualSize);
+ return permits.update(actualSize).thenApply(___ -> topicList);
+ })
+ .whenComplete((__, t) -> {
+ if (t != null) {
+ // let concurrent requests waiting for the size estimate proceed
+ listSizeHolder.resetIfInitializing();
+ }
+ });
+ }
+
+ /**
+ * Lists topics by collecting them from other topic listings, and then holds heap memory permits for the
+ * collected topic list until the response completes.
+ *
+ * <p>Use this for a listing that gets its topics from topic listings that apply the topic list memory limit
+ * themselves, such as the bundle topic listings of the bundle owners, which can be this broker. Unlike
+ * {@link #listTopics}, no permits are held while the topics are collected: the collected topic lists are
+ * covered by the permits of the other listings until they have responded, and holding permits for an
+ * estimate here could leave too few permits for a listing on this broker that this request waits for.
+ *
+ * @param brokerService the broker service
+ * @param response the async response of the REST request, the permits are released when it completes
+ * @param topicsSupplier supplier that collects the topics from the other listings
+ * @return a future that completes with the topic list
+ */
+ public static CompletableFuture<List<String>> listTopicsFromOtherListings(
+ BrokerService brokerService, AsyncResponse response,
+ Supplier<CompletableFuture<List<String>>> topicsSupplier) {
+ ResponsePermits permits = new ResponsePermits(brokerService.getMaxTopicListInFlightLimiter(), response, null);
+ response.register(permits);
+ if (permits.isCancelled()) {
+ return CompletableFuture.failedFuture(ResponsePermits.cancelled());
+ }
+ return topicsSupplier.get().thenCompose(topicList ->
+ permits.acquire(TopicListMemoryLimiter.estimateTopicListSize(topicList)).thenApply(__ -> topicList));
+ }
+
+ /**
+ * Holds the heap memory permits of a single REST request until its {@link AsyncResponse} completes.
+ */
+ private static final class ResponsePermits implements CompletionCallback {
+ private final AsyncDualMemoryLimiter limiter;
+ private final AsyncResponse response;
+ private final TopicListSizeResultCache.ResultHolder listSizeHolder;
+ // guarded by this
+ private boolean completed;
+ // guarded by this
+ private AsyncDualMemoryLimiterPermit permit;
+
+ ResponsePermits(AsyncDualMemoryLimiter limiter, AsyncResponse response,
+ TopicListSizeResultCache.ResultHolder listSizeHolder) {
+ this.limiter = limiter;
+ this.response = response;
+ this.listSizeHolder = listSizeHolder;
+ }
+
+ boolean isCancelled() {
+ synchronized (this) {
+ if (completed) {
+ return true;
+ }
+ }
+ // the response is resumed only after the topic list has been retrieved, so a response that is
+ // already done has been cancelled or has timed out
+ return response.isDone();
+ }
+
+ CompletableFuture<Void> acquire(long size) {
+ return holdOrFail(() -> limiter.acquire(size, AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY,
+ this::isCancelled));
+ }
+
+ CompletableFuture<Void> update(long size) {
+ AsyncDualMemoryLimiterPermit currentPermit;
+ synchronized (this) {
+ currentPermit = permit;
+ }
+ if (currentPermit == null) {
+ // the response has already completed and the permits have been released
+ return CompletableFuture.failedFuture(cancelled());
+ }
+ return holdOrFail(() -> limiter.update(currentPermit, size, this::isCancelled));
+ }
+
+ private CompletableFuture<Void> holdOrFail(
+ Supplier<CompletableFuture<AsyncDualMemoryLimiterPermit>> permitFutureSupplier) {
+ return AsyncDualMemoryLimiterUtil.requestPermits(permitFutureSupplier).handle((acquiredPermit, t) -> {
+ if (t != null) {
+ String message = FutureUtil.unwrapCompletionException(t)
+ instanceof AsyncSemaphore.PermitAcquireLimitExceededException
+ ? "Failed since the topic list is larger than the topic list heap memory limit"
+ : "Failed due to heap memory limit exceeded";
+ throw new CompletionException(new RestException(Status.TOO_MANY_REQUESTS, message));
+ }
+ hold(acquiredPermit);
+ // checked after storing the permit: if the response completes concurrently, either onComplete or
+ // this check releases the permit
+ if (isCancelled()) {
+ // don't continue processing a request that has been cancelled while waiting for the permits
+ releaseHeldPermit();
+ throw cancelled();
+ }
+ return null;
+ });
+ }
+
+ private static CompletionException cancelled() {
+ return new CompletionException(new RequestCancelledException());
+ }
+
+ /**
+ * Stores the permit so that it gets released when the response completes.
+ */
+ private void hold(AsyncDualMemoryLimiterPermit newPermit) {
+ AsyncDualMemoryLimiterPermit previousPermit;
+ synchronized (this) {
+ previousPermit = permit;
+ permit = newPermit;
+ }
+ if (previousPermit != null && previousPermit != newPermit) {
+ // after a permit update the previous permit has no remaining permits and releasing it is a no-op,
+ // it's released in case of an unexpected state to avoid leaking permits
+ limiter.release(previousPermit);
+ }
+ }
+
+ private void releaseHeldPermit() {
+ AsyncDualMemoryLimiterPermit permitToRelease;
+ synchronized (this) {
+ permitToRelease = permit;
+ permit = null;
+ }
+ if (permitToRelease != null) {
+ limiter.release(permitToRelease);
+ }
+ }
+
+ @Override
+ public void onComplete(Throwable throwable) {
+ if (throwable != null && listSizeHolder != null) {
+ listSizeHolder.resetIfInitializing();
+ }
+ synchronized (this) {
+ completed = true;
+ }
+ releaseHeldPermit();
+ }
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
index 183e2b1718d3..f68bb3efe0ec 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
@@ -147,10 +147,12 @@ public class Namespaces extends NamespacesBase {
.thenApply(topics -> filterSystemTopic(topics, includeSystemTopic))
.thenAccept(response::resume)
.exceptionally(ex -> {
- log.error()
- .attr("namespace", namespaceName)
- .exception(ex)
- .log("Failed to get topics list for namespace");
+ if (!isTopicListLimitRejection(ex)) {
+ log.error()
+ .attr("namespace", namespaceName)
+ .exception(ex)
+ .log("Failed to get topics list for namespace");
+ }
resumeAsyncResponseExceptionally(response, ex);
return null;
});
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java
index bc3a5d5d7c6a..d42be7bed089 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java
@@ -54,8 +54,12 @@ import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.admin.impl.TopicListResponseMemoryLimiter;
import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
import org.apache.pulsar.broker.web.RestException;
+import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
+import org.apache.pulsar.common.naming.NamespaceBundle;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
import org.apache.pulsar.common.policies.data.EntryFilters;
@@ -451,8 +455,21 @@ public class NonPersistentTopics extends PersistentTopics {
return;
}
- final List<CompletableFuture<List<String>>> futures = new ArrayList<>();
final List<String> boundaries = policies.bundles.getBoundaries();
+ // the topics are collected from the bundle topic listings (getListFromBundle) of the bundle owners
+ TopicListResponseMemoryLimiter.listTopicsFromOtherListings(pulsar().getBrokerService(), asyncResponse,
+ () -> getNonPersistentTopicsFromBundles(boundaries, nsBundle))
+ .thenAccept(nonPersistentTopics ->
+ asyncResponse.resume(filterSystemTopic(nonPersistentTopics, includeSystemTopic)))
+ .exceptionally(ex -> {
+ resumeAsyncResponseExceptionally(asyncResponse, ex);
+ return null;
+ });
+ }
+
+ private CompletableFuture<List<String>> getNonPersistentTopicsFromBundles(List<String> boundaries,
+ String nsBundle) {
+ final List<CompletableFuture<List<String>>> futures = new ArrayList<>();
for (int i = 0; i < boundaries.size() - 1; i++) {
final String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1));
if (StringUtils.isNotBlank(nsBundle) && !nsBundle.equals(bundle)) {
@@ -466,28 +483,20 @@ public class NonPersistentTopics extends PersistentTopics {
.attr("bundle", bundle)
.exception(e)
.log("Failed to get list of topics under namespace");
- asyncResponse.resume(new RestException(e));
- return;
+ return CompletableFuture.failedFuture(new RestException(e));
}
}
-
- FutureUtil.waitForAll(futures).whenComplete((result, ex) -> {
- if (ex != null) {
- resumeAsyncResponseExceptionally(asyncResponse, ex);
- } else {
- final List<String> topics = new ArrayList<>();
- for (int i = 0; i < futures.size(); i++) {
- List<String> topicList = futures.get(i).join();
- if (topicList != null) {
- topics.addAll(topicList);
- }
+ return FutureUtil.waitForAll(futures).thenApply(__ -> {
+ final List<String> topics = new ArrayList<>();
+ for (int i = 0; i < futures.size(); i++) {
+ List<String> topicList = futures.get(i).join();
+ if (topicList != null) {
+ topics.addAll(topicList);
}
- final List<String> nonPersistentTopics =
- topics.stream()
- .filter(name -> !TopicName.get(name).isPersistent())
- .collect(Collectors.toList());
- asyncResponse.resume(filterSystemTopic(nonPersistentTopics, includeSystemTopic));
}
+ return topics.stream()
+ .filter(name -> !TopicName.get(name).isPersistent())
+ .collect(Collectors.toList());
});
}
@@ -534,24 +543,15 @@ public class NonPersistentTopics extends PersistentTopics {
asyncResponse.resume(Response.noContent().build());
} else {
validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, true, true)
- .thenAccept(nsBundle -> {
- final var bundleTopics = pulsar().getBrokerService().getMultiLayerTopicsMap()
- .get(namespaceName.toString());
- if (bundleTopics == null || bundleTopics.isEmpty()) {
- asyncResponse.resume(Collections.emptyList());
- return;
- }
- final List<String> topicList = new ArrayList<>();
- String bundleKey = namespaceName.toString() + "/" + nsBundle.getBundleRange();
- final var topicMap = bundleTopics.get(bundleKey);
- if (topicMap != null) {
- topicList.addAll(topicMap.keySet().stream()
- .filter(name -> !TopicName.get(name).isPersistent())
- .collect(Collectors.toList()));
- }
- asyncResponse.resume(topicList);
- }).exceptionally(ex -> {
- if (isNot307And404Exception(ex)) {
+ .thenCompose(nsBundle -> TopicListResponseMemoryLimiter.listTopics(
+ pulsar().getBrokerService(), asyncResponse,
+ TopicListing.ownedBundleTopics(namespaceName.toString(),
+ CommandGetTopicsOfNamespace.Mode.NON_PERSISTENT, nsBundle.getBundleRange()),
+ null,
+ () -> CompletableFuture.completedFuture(getNonPersistentTopicsOfBundle(nsBundle))))
+ .thenAccept(asyncResponse::resume)
+ .exceptionally(ex -> {
+ if (isNot307And404Exception(ex) && !isTopicListLimitRejection(ex)) {
log.error()
.attr("namespace", namespaceName)
.attr("bundleRange", bundleRange)
@@ -701,6 +701,22 @@ public class NonPersistentTopics extends PersistentTopics {
});
}
+ private List<String> getNonPersistentTopicsOfBundle(NamespaceBundle nsBundle) {
+ final var bundleTopics = pulsar().getBrokerService().getMultiLayerTopicsMap()
+ .get(namespaceName.toString());
+ if (bundleTopics == null || bundleTopics.isEmpty()) {
+ return Collections.emptyList();
+ }
+ String bundleKey = namespaceName.toString() + "/" + nsBundle.getBundleRange();
+ final var topicMap = bundleTopics.get(bundleKey);
+ if (topicMap == null) {
+ return Collections.emptyList();
+ }
+ return topicMap.keySet().stream()
+ .filter(name -> !TopicName.get(name).isPersistent())
+ .collect(Collectors.toList());
+ }
+
private Topic getTopicReference(TopicName topicName) {
try {
return pulsar().getBrokerService().getTopicIfExists(topicName.toString())
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index e4b60620c56f..bdd63c81d46d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -135,10 +135,10 @@ public class PersistentTopics extends PersistentTopicsBase {
@Parameter(description = "properties for customized topic listing plugin, format: k1=v1,k2=v2")
@QueryParam("properties") String propertiesStr) {
validateNamespaceName(tenant, namespace);
- internalGetListAsync(Optional.ofNullable(bundle), parseProperties(propertiesStr))
+ internalGetListAsync(asyncResponse, Optional.ofNullable(bundle), parseProperties(propertiesStr))
.thenAccept(topicList -> asyncResponse.resume(filterSystemTopic(topicList, includeSystemTopic)))
.exceptionally(ex -> {
- if (isNot307And404Exception(ex)) {
+ if (isNot307And404Exception(ex) && !isTopicListLimitRejection(ex)) {
log.error()
.attr("namespace", namespaceName)
.exception(ex)
@@ -172,11 +172,11 @@ public class PersistentTopics extends PersistentTopicsBase {
@Parameter(description = "Include system topic")
@QueryParam("includeSystemTopic") boolean includeSystemTopic) {
validateNamespaceName(tenant, namespace);
- internalGetPartitionedTopicListAsync()
+ internalGetPartitionedTopicListAsync(asyncResponse)
.thenAccept(partitionedTopicList -> asyncResponse.resume(
filterSystemTopic(partitionedTopicList, includeSystemTopic)))
.exceptionally(ex -> {
- if (isNot307And404Exception(ex)) {
+ if (isNot307And404Exception(ex) && !isTopicListLimitRejection(ex)) {
log.error()
.attr("namespace", namespaceName)
.exception(ex)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
index c412bd93efd1..f1a606f3b364 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
@@ -54,6 +54,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
import lombok.CustomLog;
import org.apache.pulsar.broker.admin.AdminResource;
+import org.apache.pulsar.broker.admin.impl.TopicListResponseMemoryLimiter;
import org.apache.pulsar.broker.resources.ScalableTopicMetadata;
import org.apache.pulsar.broker.resources.ScalableTopicResources;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
@@ -61,6 +62,7 @@ import org.apache.pulsar.broker.service.scalable.AutoScaleConfig;
import org.apache.pulsar.broker.service.scalable.ScalableTopicController;
import org.apache.pulsar.broker.service.scalable.ScalableTopicService;
import org.apache.pulsar.broker.service.scalable.SegmentTopicStatsBuilder;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
import org.apache.pulsar.broker.web.RestException;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.common.naming.TopicDomain;
@@ -122,14 +124,19 @@ public class ScalableTopics extends AdminResource {
validateNamespaceName(tenant, namespace);
Map<String, String> propertyFilters = parseKeyValuePairs(properties);
validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPICS)
- .thenCompose(__ -> propertyFilters.isEmpty()
- ? resources().listScalableTopicsAsync(namespaceName)
- : resources().findScalableTopicsByPropertiesAsync(
- namespaceName, propertyFilters))
+ .thenCompose(__ -> TopicListResponseMemoryLimiter.listTopics(pulsar().getBrokerService(),
+ asyncResponse,
+ TopicListing.scalableTopics(namespaceName.toString()), propertyFilters,
+ () -> propertyFilters.isEmpty()
+ ? resources().listScalableTopicsAsync(namespaceName)
+ : resources().findScalableTopicsByPropertiesAsync(
+ namespaceName, propertyFilters)))
.thenAccept(asyncResponse::resume)
.exceptionally(ex -> {
- log.error().attr("clientAppId", clientAppId()).attr("namespace", namespaceName)
- .exception(ex).log("Failed to list scalable topics");
+ if (!isTopicListLimitRejection(ex)) {
+ log.error().attr("clientAppId", clientAppId()).attr("namespace", namespaceName)
+ .exception(ex).log("Failed to list scalable topics");
+ }
resumeAsyncResponseExceptionally(asyncResponse, ex);
return null;
});
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index f9c2bc45d3f8..512696a2a2f8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -198,7 +198,9 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl;
import org.apache.pulsar.common.protocol.schema.SchemaVersion;
import org.apache.pulsar.common.scalable.SegmentLoadStats;
import org.apache.pulsar.common.scalable.SegmentTopicName;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.apache.pulsar.common.semaphore.AsyncSemaphore;
import org.apache.pulsar.common.stats.Metrics;
import org.apache.pulsar.common.util.FieldParser;
import org.apache.pulsar.common.util.FutureUtil;
@@ -279,7 +281,7 @@ public class BrokerService implements Closeable {
@Getter
private final AsyncDualMemoryLimiterImpl maxTopicListInFlightLimiter;
@Getter
- private final TopicListSizeResultCache topicListSizeResultCache = new TopicListSizeResultCache();
+ private final TopicListSizeResultCache topicListSizeResultCache;
protected final AtomicReference<Semaphore> topicLoadRequestSemaphore;
public static final String TOPIC_LOOKUP_USAGE_METRIC_NAME = "pulsar.broker.request.topic.lookup.concurrent.usage";
@@ -446,6 +448,11 @@ public class BrokerService implements Closeable {
pulsar.getConfiguration().getMaxTopicListInFlightDirectMemSizeMB() * 1024L * 1024L,
pulsar.getConfiguration().getMaxTopicListInFlightDirectMemSizePermitsAcquireQueueSize(),
pulsar.getConfiguration().getMaxTopicListInFlightDirectMemSizePermitsAcquireTimeoutMillis());
+ AsyncSemaphore topicListHeapLimiter =
+ maxTopicListInFlightLimiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY);
+ this.topicListSizeResultCache = new TopicListSizeResultCache(
+ pulsar.getConfiguration().getMaxTopicListInFlightHeapMemSizeMB() * 1024L * 1024L,
+ () -> topicListHeapLimiter.getQueueSize() == 0 && topicListHeapLimiter.getAcquiredPermits() == 0);
this.topicLoadRequestSemaphore = new AtomicReference<>(
new Semaphore(pulsar.getConfiguration().getMaxConcurrentTopicLoadRequest(), false));
if (pulsar.getConfiguration().getMaxUnackedMessagesPerBroker() > 0
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSender.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSender.java
index d98fc3585832..519cd444b471 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSender.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSender.java
@@ -24,6 +24,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
+import java.util.function.BooleanSupplier;
import java.util.function.Function;
import org.apache.bookkeeper.mledger.Entry;
import org.apache.pulsar.client.api.transaction.TxnID;
@@ -100,15 +101,31 @@ public interface PulsarCommandSender {
void sendEndTxnErrorResponse(long requestId, TxnID txnID, ServerError error, String message);
+ /**
+ * Sends the topic list watcher success response after acquiring direct memory permits for it.
+ *
+ * @param isCancelled supplier that returns true when the response is no longer needed, for example when the
+ * watcher has been closed. The response isn't sent when the connection is inactive or the
+ * request has been cancelled.
+ */
CompletableFuture<Void> sendWatchTopicListSuccess(long requestId, long watcherId, String topicsHash,
Collection<String> topics,
Function<Throwable, CompletableFuture<Void>>
- permitAcquireErrorHandler);
+ permitAcquireErrorHandler,
+ BooleanSupplier isCancelled);
+ /**
+ * Sends a topic list watcher update after acquiring direct memory permits for it.
+ *
+ * @param isCancelled supplier that returns true when the update is no longer needed, for example when the
+ * watcher has been closed. The update isn't sent when the connection is inactive or the
+ * request has been cancelled.
+ */
CompletableFuture<Void> sendWatchTopicListUpdate(long watcherId,
List<String> newTopics, List<String> deletedTopics, String topicsHash,
Function<Throwable, CompletableFuture<Void>>
- permitAcquireErrorHandler);
+ permitAcquireErrorHandler,
+ BooleanSupplier isCancelled);
/**
* Send the response to a scalable-topic subscribe request on success. The caller is
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java
index 7faa0b58bb94..dcbe1e91b077 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java
@@ -28,6 +28,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
+import java.util.function.BooleanSupplier;
import java.util.function.Function;
import lombok.CustomLog;
import org.apache.bookkeeper.mledger.Entry;
@@ -466,11 +467,12 @@ public class PulsarCommandSenderImpl implements PulsarCommandSender {
public CompletableFuture<Void> sendWatchTopicListSuccess(long requestId, long watcherId, String topicsHash,
Collection<String> topics,
Function<Throwable, CompletableFuture<Void>>
- permitAcquireErrorHandler) {
+ permitAcquireErrorHandler,
+ BooleanSupplier isCancelled) {
BaseCommand command = Commands.newWatchTopicListSuccess(requestId, watcherId, topicsHash, topics);
safeIntercept(command, cnx);
- return acquireDirectMemoryPermitsAndWriteAndFlush(cnx.ctx(), maxTopicListInFlightLimiter, () -> !cnx.isActive(),
- command, permitAcquireErrorHandler);
+ return acquireDirectMemoryPermitsAndWriteAndFlush(cnx.ctx(), maxTopicListInFlightLimiter,
+ () -> !cnx.isActive() || isCancelled.getAsBoolean(), command, permitAcquireErrorHandler);
}
/***
@@ -481,11 +483,12 @@ public class PulsarCommandSenderImpl implements PulsarCommandSender {
public CompletableFuture<Void> sendWatchTopicListUpdate(long watcherId, List<String> newTopics,
List<String> deletedTopics, String topicsHash,
Function<Throwable, CompletableFuture<Void>>
- permitAcquireErrorHandler) {
+ permitAcquireErrorHandler,
+ BooleanSupplier isCancelled) {
BaseCommand command = Commands.newWatchTopicUpdate(watcherId, newTopics, deletedTopics, topicsHash);
safeIntercept(command, cnx);
- return acquireDirectMemoryPermitsAndWriteAndFlush(cnx.ctx(), maxTopicListInFlightLimiter, () -> !cnx.isActive(),
- command, permitAcquireErrorHandler);
+ return acquireDirectMemoryPermitsAndWriteAndFlush(cnx.ctx(), maxTopicListInFlightLimiter,
+ () -> !cnx.isActive() || isCancelled.getAsBoolean(), command, permitAcquireErrorHandler);
}
@Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 23f766feab7a..1e437b59d8f2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -201,6 +201,7 @@ import org.apache.pulsar.common.protocol.schema.SchemaVersion;
import org.apache.pulsar.common.schema.SchemaType;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil;
import org.apache.pulsar.common.topics.TopicList;
import org.apache.pulsar.common.topics.TopicsPattern;
import org.apache.pulsar.common.util.FutureUtil;
@@ -3642,8 +3643,8 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
Map<String, String> properties,
Semaphore lookupSemaphore) {
BooleanSupplier isPermitRequestCancelled = () -> !ctx().channel().isActive();
- TopicListSizeResultCache.ResultHolder
- listSizeHolder = service.getTopicListSizeResultCache().getTopicListSize(namespaceName.toString(), mode);
+ TopicListSizeResultCache.ResultHolder listSizeHolder = service.getTopicListSizeResultCache()
+ .getTopicListSize(namespaceName.toString(), mode, properties);
listSizeHolder.getSizeAsync().thenAccept(initialSize -> {
maxTopicListInFlightLimiter.withAcquiredPermits(initialSize,
AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY, isPermitRequestCancelled, initialPermits -> {
@@ -3688,22 +3689,22 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
return commandSender.sendGetTopicsOfNamespaceResponse(filteredTopics,
hash,
filterTopics, !hashUnchanged, requestId, ex -> {
- log.warn()
- .exceptionMessage(ex)
- .log("Failed to acquire direct memory permits for "
- + "GetTopicsOfNamespace");
- commandSender.sendErrorResponse(requestId,
- ServerError.TooManyRequests,
- "Cannot acquire permits for direct memory");
+ if (!isTopicListRequestCancelled(ex,
+ "Failed to acquire direct memory permits for "
+ + "GetTopicsOfNamespace")) {
+ sendTopicListPermitsErrorResponse(requestId, ex,
+ true, "direct",
+ "Cannot acquire permits for direct memory");
+ }
return CompletableFuture.completedFuture(null);
});
}, t -> {
- log.warn()
- .exceptionMessage(t)
- .log("Failed to acquire heap memory permits for "
- + "GetTopicsOfNamespace");
- writeAndFlush(Commands.newError(requestId, ServerError.TooManyRequests,
- "Failed due to heap memory limit exceeded"));
+ if (!isTopicListRequestCancelled(t,
+ "Failed to acquire heap memory permits for "
+ + "GetTopicsOfNamespace")) {
+ sendTopicListPermitsErrorResponse(requestId, t, true,
+ "heap", "Failed due to heap memory limit exceeded");
+ }
return CompletableFuture.completedFuture(null);
});
}).whenComplete((__, ___) -> {
@@ -3720,18 +3721,53 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
return null;
});
}, t -> {
- log.warn()
- .exceptionMessage(t)
- .log("Failed to acquire initial heap memory permits for GetTopicsOfNamespace");
listSizeHolder.resetIfInitializing();
- writeAndFlush(Commands.newError(requestId, ServerError.TooManyRequests,
- "Failed due to heap memory limit exceeded"));
+ if (!isTopicListRequestCancelled(t,
+ "Failed to acquire initial heap memory permits for GetTopicsOfNamespace")) {
+ sendTopicListPermitsErrorResponse(requestId, t, listSizeHolder.isMeasured(), "heap",
+ "Failed due to heap memory limit exceeded");
+ }
lookupSemaphore.release();
return CompletableFuture.completedFuture(null);
});
});
}
+ /**
+ * Responds to a GetTopicsOfNamespace request that couldn't acquire topic list memory permits. A topic list that
+ * has been measured to be larger than the limit gets a non-retriable error, since a retry fails the same way
+ * until the list is measured again. A permit timeout or a full queue, or an estimate larger than the limit for
+ * a topic list that hasn't been measured, gets {@link ServerError#TooManyRequests}, which the client retries.
+ */
+ private void sendTopicListPermitsErrorResponse(long requestId, Throwable t, boolean topicListMeasured,
+ String memoryType, String tooManyRequestsMessage) {
+ if (topicListMeasured && AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(t)) {
+ commandSender.sendErrorResponse(requestId, ServerError.NotAllowedError,
+ "Failed since the topic list is larger than the topic list " + memoryType + " memory limit");
+ } else {
+ commandSender.sendErrorResponse(requestId, ServerError.TooManyRequests, tooManyRequestsMessage);
+ }
+ }
+
+ /**
+ * Logs a failure to acquire topic list memory permits. A request that was cancelled, because the connection
+ * was closed, is logged at debug level, since there is no client to respond to.
+ *
+ * @return true if the request was cancelled and no error response should be sent
+ */
+ private boolean isTopicListRequestCancelled(Throwable t, String failureMessage) {
+ if (AsyncDualMemoryLimiterUtil.isPermitAcquireCancelled(t)) {
+ log.debug()
+ .attr("remoteAddress", remoteAddress)
+ .log("GetTopicsOfNamespace was cancelled since the connection was closed");
+ return true;
+ }
+ log.warn()
+ .exceptionMessage(t)
+ .log(failureMessage);
+ return false;
+ }
+
@Override
protected void handleGetSchema(CommandGetSchema commandGetSchema) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
index 124122c8ee58..299168b382c9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
@@ -37,6 +37,7 @@ import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -56,6 +57,7 @@ import org.apache.pulsar.common.api.proto.ServerError;
import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil;
import org.apache.pulsar.common.semaphore.AsyncSemaphore;
import org.apache.pulsar.common.topics.TopicList;
import org.apache.pulsar.common.topics.TopicsPattern;
@@ -81,12 +83,16 @@ public class TopicListService {
private final Executor executor;
private volatile boolean closed = false;
private boolean sendingInProgress;
+ // the number of watch request success responses being sent, topic changes are sent after them
+ private int pendingSuccessResponses;
private final BlockingDeque<Runnable> sendTopicListUpdateTasks;
@Getter(onMethod_ = @VisibleForTesting)
private boolean updatingTopics;
private List<String> matchingTopicsBeforeDisconnected;
private boolean disconnected;
- private List<Runnable> updateCallbacks = new LinkedList<>();
+ // callbacks of requests waiting for the ongoing topic list update, called with a snapshot of the matching
+ // topics when the update completes, or with the failure
+ private List<BiConsumer<Collection<String>, Throwable>> updateCallbacks = new LinkedList<>();
private boolean updatingWhileDisconnected;
public TopicListWatcher(TopicListService topicListService, long id,
@@ -168,6 +174,15 @@ public class TopicListService {
sendTopicListUpdateTasks.clear();
return;
}
+ if (updatingTopics) {
+ // The update calls this again when it completes. Until then, topic changes are queued, since they
+ // would be sent with the hash of the partially rebuilt matching topics.
+ return;
+ }
+ if (pendingSuccessResponses > 0) {
+ // the last success response to complete calls this again, topic changes are sent after it
+ return;
+ }
// Execute the next task
Runnable task = sendTopicListUpdateTasks.poll();
if (task != null) {
@@ -177,20 +192,104 @@ public class TopicListService {
}
}
- public synchronized void close() {
- closed = true;
- sendTopicListUpdateTasks.clear();
- updateCallbacks.clear();
+ public void close() {
+ List<BiConsumer<Collection<String>, Throwable>> pendingCallbacks;
+ synchronized (this) {
+ closed = true;
+ sendTopicListUpdateTasks.clear();
+ pendingCallbacks = new ArrayList<>(updateCallbacks);
+ updateCallbacks.clear();
+ }
+ // complete the requests waiting for the topic list update so that they don't wait forever
+ if (!pendingCallbacks.isEmpty()) {
+ IllegalStateException closedException = newClosedException();
+ for (BiConsumer<Collection<String>, Throwable> callback : pendingCallbacks) {
+ runUpdateCallback(callback, null, closedException);
+ }
+ }
+ }
+
+ /**
+ * Completes the requests waiting for the ongoing update with the given failure. The update continues.
+ */
+ /**
+ * Called before sending the success response of a watch request. Topic changes are queued until
+ * {@link #successResponseSendingCompleted()} has been called for each success response, so that they are
+ * sent after it.
+ */
+ synchronized void successResponseSendingStarted() {
+ pendingSuccessResponses++;
+ sendingInProgress = true;
+ }
+
+ synchronized void successResponseSendingCompleted() {
+ if (pendingSuccessResponses > 0) {
+ pendingSuccessResponses--;
+ }
+ sendingCompleted();
+ }
+
+ void failUpdateCallbacks(Throwable failure) {
+ List<BiConsumer<Collection<String>, Throwable>> pendingCallbacks;
+ synchronized (this) {
+ pendingCallbacks = new ArrayList<>(updateCallbacks);
+ updateCallbacks.clear();
+ }
+ for (BiConsumer<Collection<String>, Throwable> callback : pendingCallbacks) {
+ runUpdateCallback(callback, null, failure);
+ }
+ }
+
+ /**
+ * Calls the callback with a snapshot of the matching topics. While a topic list update is in progress, the
+ * matching topics aren't complete, so the callback is called when the update completes.
+ */
+ void whenMatchingTopicsAvailable(BiConsumer<Collection<String>, Throwable> callback) {
+ Collection<String> snapshot;
+ synchronized (this) {
+ if (!closed && updatingTopics) {
+ updateCallbacks.add(callback);
+ return;
+ }
+ snapshot = closed ? null : new ArrayList<>(matchingTopics);
+ }
+ runUpdateCallback(callback, snapshot, snapshot == null ? newClosedException() : null);
+ }
+
+ private IllegalStateException newClosedException() {
+ return new IllegalStateException("Topic list watcher " + id + " was closed");
+ }
+
+ private static void runUpdateCallback(BiConsumer<Collection<String>, Throwable> callback,
+ Collection<String> matchingTopics, Throwable failure) {
+ try {
+ callback.accept(matchingTopics, failure);
+ } catch (Exception e) {
+ log.warn()
+ .attr("callback", callback)
+ .exception(e)
+ .log("Error executing topic list update callback");
+ }
}
/**
* Returns true if the topic list update is prepared for execution. It is expected that the caller initiates
* the update. The callback is registered to be executed after the update, either existing or upcoming is
* completed.
- * @param afterUpdateCompletionCallback callback to be executed after the update is completed.
+ * @param afterUpdateCompletionCallback callback to be executed after the update is completed. It's called
+ * with a snapshot of the matching topics when the update completes,
+ * or with the failure when the update fails or the watcher is closed
+ * before the update completes.
* @return true if an existing update wasn't ongoing and a new update is prepared for execution.
*/
- synchronized boolean prepareUpdateTopics(Runnable afterUpdateCompletionCallback) {
+ synchronized boolean prepareUpdateTopics(
+ BiConsumer<Collection<String>, Throwable> afterUpdateCompletionCallback) {
+ if (closed) {
+ if (afterUpdateCompletionCallback != null) {
+ runUpdateCallback(afterUpdateCompletionCallback, null, newClosedException());
+ }
+ return false;
+ }
if (!updatingTopics) {
updatingTopics = true;
sendingInProgress = true;
@@ -208,29 +307,32 @@ public class TopicListService {
}
}
- synchronized void updateTopics(List<String> topics) {
- if (closed) {
- return;
- }
- matchingTopics.clear();
- TopicList.filterTopicsToStream(topics, topicsPattern).forEach(matchingTopics::add);
- updatingTopics = false;
- if (disconnected) {
- handleNewAndDeletedTopicsWhileDisconnected();
- matchingTopicsBeforeDisconnected = null;
- disconnected = false;
- }
- for (Runnable callback : updateCallbacks) {
- try {
- callback.run();
- } catch (Exception e) {
- log.warn()
- .attr("callback", callback)
- .exception(e)
- .log("Error executing topic list update callback");
+ void updateTopics(List<String> topics) {
+ List<BiConsumer<Collection<String>, Throwable>> callbacks;
+ Collection<String> snapshot;
+ synchronized (this) {
+ if (closed) {
+ return;
+ }
+ matchingTopics.clear();
+ TopicList.filterTopicsToStream(topics, topicsPattern).forEach(matchingTopics::add);
+ updatingTopics = false;
+ if (disconnected) {
+ handleNewAndDeletedTopicsWhileDisconnected();
+ matchingTopicsBeforeDisconnected = null;
+ disconnected = false;
}
+ callbacks = new ArrayList<>(updateCallbacks);
+ updateCallbacks.clear();
+ // the waiting requests respond with the result of this update, also if the matching topics change
+ // after the lock has been released, for example when another update starts
+ snapshot = callbacks.isEmpty() ? null : new ArrayList<>(matchingTopics);
+ }
+ // the callbacks run without holding the lock, since a callback can close the watcher, for example when
+ // sending the response of a waiting request fails
+ for (BiConsumer<Collection<String>, Throwable> callback : callbacks) {
+ runUpdateCallback(callback, snapshot, null);
}
- updateCallbacks.clear();
sendingCompleted();
}
@@ -379,23 +481,24 @@ public class TopicListService {
CompletableFuture<TopicListWatcher> watcherFuture = new CompletableFuture<>();
CompletableFuture<TopicListWatcher> existingWatcherFuture = watchers.putIfAbsent(watcherId, watcherFuture);
+ // the watcher and the snapshot of its matching topics to respond with
+ CompletableFuture<WatcherTopics> watcherTopicsFuture;
if (existingWatcherFuture != null) {
log.debug()
.attr("connection", connection)
.attr("watcherId", watcherId)
.log("Watcher with the same watcherId is already created. Refreshing.");
// use the existing watcher if it's already created
- watcherFuture = existingWatcherFuture.thenCompose(watcher -> {
- CompletableFuture<TopicListWatcher> future = new CompletableFuture<>();
- Runnable callback = () -> future.complete(watcher);
+ watcherTopicsFuture = existingWatcherFuture.thenCompose(watcher -> {
+ CompletableFuture<WatcherTopics> future = new CompletableFuture<>();
+ BiConsumer<Collection<String>, Throwable> callback = watcherTopicsCallback(watcher, future);
// trigger a new update unless an update is already ongoing. Register the callback to complete
- // when the update completes.
+ // when the update completes, or when the watcher is closed before that.
if (watcher.prepareUpdateTopics(callback)) {
updateTopicListWatcher(watcher)
- // run the callback also in failure cases
- // prepareUpdateTopics handles it for success cases
+ // complete the request also when the update fails
.exceptionally(ex -> {
- callback.run();
+ callback.accept(null, ex);
return null;
});
}
@@ -403,11 +506,20 @@ public class TopicListService {
});
} else {
initializeTopicsListWatcher(watcherFuture, namespaceName, watcherId, topicsPattern);
+ watcherTopicsFuture = watcherFuture.thenCompose(watcher -> {
+ CompletableFuture<WatcherTopics> future = new CompletableFuture<>();
+ // waits for an update that started after the watcher was created, for example after a metadata
+ // store session event, since the matching topics are incomplete until the update completes
+ watcher.whenMatchingTopicsAvailable(watcherTopicsCallback(watcher, future));
+ return future;
+ });
}
- CompletableFuture<TopicListWatcher> finalWatcherFuture = watcherFuture;
- finalWatcherFuture.thenAccept(watcher -> {
- Collection<String> topicList = watcher.getMatchingTopics();
+ // a failed request removes the watcher that it created, not an existing watcher
+ CompletableFuture<TopicListWatcher> createdWatcherFuture = existingWatcherFuture == null ? watcherFuture : null;
+ watcherTopicsFuture.thenAccept(watcherTopics -> {
+ TopicListWatcher watcher = watcherTopics.watcher();
+ Collection<String> topicList = watcherTopics.matchingTopics();
String hash = TopicList.calculateHash(topicList);
if (hash.equals(topicsHash)) {
topicList = Collections.emptyList();
@@ -417,48 +529,105 @@ public class TopicListService {
.attr("namespace", namespaceName)
.attr("requestId", requestId)
.log("Received WatchTopicList");
+ watcher.successResponseSendingStarted();
sendTopicListSuccessWithPermitAcquiringRetries(watcherId, requestId, topicList, hash,
- watcher::sendingCompleted, watcher::close);
+ watcher::successResponseSendingCompleted, failure -> {
+ if (!isCancellation(failure)) {
+ // the watcher can't be used without the success response
+ deleteTopicListWatcher(watcherId);
+ }
+ watcher.successResponseSendingCompleted();
+ // the request is waiting for a response unless the connection was closed
+ if (connection.isActive()) {
+ connection.getCommandSender().sendErrorResponse(requestId,
+ getWatchTopicListErrorCode(failure), failure.getMessage());
+ }
+ });
lookupSemaphore.release();
})
.exceptionally(ex -> {
- log.warn()
- .attr("connection", connection)
- .attr("namespace", namespaceName)
- .attr("requestId", requestId)
- .exceptionMessage(ex)
- .log("Error WatchTopicList");
- connection.getCommandSender().sendErrorResponse(requestId,
- BrokerServiceException.getClientErrorCode(
- new BrokerServiceException.ServerMetadataException(ex)), ex.getMessage());
- watchers.remove(watcherId, finalWatcherFuture);
+ if (connection.isActive()) {
+ log.warn()
+ .attr("connection", connection)
+ .attr("namespace", namespaceName)
+ .attr("requestId", requestId)
+ .exceptionMessage(ex)
+ .log("Error WatchTopicList");
+ connection.getCommandSender().sendErrorResponse(requestId,
+ getWatchTopicListErrorCode(ex), ex.getMessage());
+ } else {
+ // the connection was closed, there's no client to respond to
+ log.debug()
+ .attr("connection", connection)
+ .attr("namespace", namespaceName)
+ .attr("requestId", requestId)
+ .exceptionMessage(ex)
+ .log("WatchTopicList was cancelled since the connection was closed");
+ }
+ if (createdWatcherFuture != null && watchers.remove(watcherId, createdWatcherFuture)) {
+ // the request can fail after it created the watcher and registered it as a topic listener,
+ // for example when an update that started after the watcher was created fails
+ createdWatcherFuture.thenAccept(createdWatcher -> {
+ topicResources.deregisterPersistentTopicListener(createdWatcher);
+ createdWatcher.close();
+ });
+ }
lookupSemaphore.release();
return null;
});
}
+ /**
+ * A watcher and a snapshot of its matching topics.
+ */
+ private record WatcherTopics(TopicListWatcher watcher, Collection<String> matchingTopics) {
+ }
+
+ private static BiConsumer<Collection<String>, Throwable> watcherTopicsCallback(
+ TopicListWatcher watcher, CompletableFuture<WatcherTopics> future) {
+ return (matchingTopics, failure) -> {
+ if (failure == null) {
+ future.complete(new WatcherTopics(watcher, matchingTopics));
+ } else {
+ future.completeExceptionally(failure);
+ }
+ };
+ }
+
private void sendTopicListSuccessWithPermitAcquiringRetries(long watcherId, long requestId,
Collection<String> topicList,
String hash,
Runnable successfulCompletionCallback,
- Runnable failedCompletionCallback) {
- performOperationWithPermitAcquiringRetries(watcherId, "topic list success", permitAcquireErrorHandler ->
+ Consumer<Throwable> failedCompletionCallback) {
+ // the success response is the response to the watch request, so a cancelled success response is reported
+ // as a failure, so that the request can be completed with an error response
+ performOperationWithPermitAcquiringRetries(watcherId, "topic list success", true, permitAcquireErrorHandler ->
() -> connection.getCommandSender()
- .sendWatchTopicListSuccess(requestId, watcherId, hash, topicList, permitAcquireErrorHandler)
- .whenComplete((__, t) -> {
- if (t != null) {
- // this is an unexpected case
- log.warn()
- .attr("connection", connection)
- .attr("watcherId", watcherId)
- .exception(t)
- .log("Failed to send topic list success. Watcher is not active.");
- failedCompletionCallback.run();
- } else {
- // completed successfully, run the callback
- successfulCompletionCallback.run();
- }
- }));
+ .sendWatchTopicListSuccess(requestId, watcherId, hash, topicList, permitAcquireErrorHandler,
+ () -> !watchers.containsKey(watcherId)))
+ // handled on the future of the whole operation, which also completes when the watcher was removed
+ // before the first attempt and after retries
+ .whenComplete((__, t) -> {
+ if (t != null) {
+ Throwable failure = FutureUtil.unwrapCompletionException(t);
+ if (isCancellation(failure)) {
+ log.debug()
+ .attr("connection", connection)
+ .attr("watcherId", watcherId)
+ .log("Topic list success wasn't sent since the watcher was closed");
+ } else {
+ log.warn()
+ .attr("connection", connection)
+ .attr("watcherId", watcherId)
+ .exception(t)
+ .log("Failed to send topic list success. Watcher is not active.");
+ }
+ failedCompletionCallback.accept(failure);
+ } else {
+ // completed successfully, run the callback
+ successfulCompletionCallback.run();
+ }
+ });
}
/***
@@ -498,12 +667,20 @@ public class TopicListService {
() -> initializeTopicsListWatcher(watcherFuture, namespace, watcherId, topicsPattern),
retryAfterMillis, TimeUnit.MILLISECONDS);
} else {
- log.warn()
- .attr("connection", connection)
- .attr("watcherId", watcherId)
- .attr("namespace", namespace)
- .exception(unwrappedException)
- .log("Failed to initialize topic list watcher");
+ if (isCancellation(unwrappedException)) {
+ log.debug()
+ .attr("connection", connection)
+ .attr("watcherId", watcherId)
+ .attr("namespace", namespace)
+ .log("Topic list watcher initialization was cancelled");
+ } else {
+ log.warn()
+ .attr("connection", connection)
+ .attr("watcherId", watcherId)
+ .attr("namespace", namespace)
+ .exception(unwrappedException)
+ .log("Failed to initialize topic list watcher");
+ }
watcherFuture.completeExceptionally(unwrappedException);
}
} else {
@@ -569,6 +746,12 @@ public class TopicListService {
}
}, connection.ctx().executor());
}, CompletableFuture::failedFuture)
+ .whenComplete((__, t) -> {
+ if (t != null) {
+ // let concurrent requests waiting for the size estimate proceed
+ listSizeHolder.resetIfInitializing();
+ }
+ })
.thenApplyAsync(Function.identity(), connection.ctx().executor());
});
}
@@ -589,10 +772,35 @@ public class TopicListService {
getTopics(namespace, watcherId).whenComplete((topics, exception) -> {
if (exception != null) {
Throwable unwrappedException = FutureUtil.unwrapCompletionException(exception);
- if (connection.isActive() && !watcher.closed
- && (unwrappedException instanceof AsyncSemaphore.PermitAcquireTimeoutException
- || unwrappedException instanceof AsyncSemaphore.PermitAcquireQueueFullException)) {
- // retry with backoff if permit acquisition fails due to timeout or queue full
+ if (!connection.isActive() || watcher.closed || isCancellation(unwrappedException)) {
+ // the connection or the watcher was closed, closing the watcher completes the requests
+ // waiting for the update
+ log.debug()
+ .attr("connection", connection)
+ .attr("watcherId", watcherId)
+ .attr("namespace", namespace)
+ .log("Topic list watcher update was cancelled");
+ future.completeExceptionally(unwrappedException);
+ } else if (unwrappedException instanceof AsyncSemaphore.PermitAcquireLimitExceededException) {
+ // the topic list can't be listed within the topic list memory limit, retrying wouldn't help.
+ // The watcher can't send updates without a successful update, so close it.
+ log.warn()
+ .attr("connection", connection)
+ .attr("watcherId", watcherId)
+ .attr("namespace", namespace)
+ .exceptionMessage(unwrappedException)
+ .log("Closing topic list watcher since the topic list is larger than the topic list "
+ + "memory limit");
+ // fail the requests waiting for the update with this failure before closing the watcher fails
+ // them with a watcher closed error, so that they get a non-retriable error response
+ watcher.failUpdateCallbacks(unwrappedException);
+ deleteTopicListWatcher(watcherId);
+ future.completeExceptionally(unwrappedException);
+ } else {
+ // Retry with backoff. Until the update completes, the watcher queues the topic changes and
+ // doesn't send updates, so giving up would leave the watcher without updates.
+ // Requests waiting for the update get an error response instead of waiting for the retries.
+ watcher.failUpdateCallbacks(unwrappedException);
long retryAfterMillis = this.retryBackoff.next().toMillis();
log.info()
.attr("connection", connection)
@@ -600,18 +808,10 @@ public class TopicListService {
.attr("watcherId", watcherId)
.attr("namespace", namespace)
.attr("retryAfterMillis", retryAfterMillis)
- .log("when updating topic list watcher for namespace. Retrying");
+ .log("Failed to update topic list watcher for namespace. Retrying");
connection.ctx().executor()
.schedule(() -> internalUpdateTopicListWatcher(watcher, future),
retryAfterMillis, TimeUnit.MILLISECONDS);
- } else {
- log.warn()
- .attr("connection", connection)
- .attr("watcherId", watcherId)
- .attr("namespace", namespace)
- .exception(unwrappedException)
- .log("Failed to update topic list watcher");
- future.completeExceptionally(unwrappedException);
}
} else {
watcher.updateTopics(topics);
@@ -673,27 +873,32 @@ public class TopicListService {
*/
public void sendTopicListUpdate(long watcherId, String topicsHash, List<String> deletedTopics,
List<String> newTopics, Runnable completionCallback) {
- performOperationWithPermitAcquiringRetries(watcherId, "topic list update", permitAcquireErrorHandler ->
+ performOperationWithPermitAcquiringRetries(watcherId, "topic list update", false,
+ permitAcquireErrorHandler ->
() -> connection.getCommandSender()
.sendWatchTopicListUpdate(watcherId, newTopics, deletedTopics, topicsHash,
- permitAcquireErrorHandler)
- .whenComplete((__, t) -> {
- if (t != null) {
- // this is an unexpected case
- log.warn()
- .attr("connection", connection)
- .attr("watcherId", watcherId)
- .exception(t)
- .log("Failed to send topic list update for watcherId=. Watcher will be in "
- + "inconsistent state.");
- }
- completionCallback.run();
- }));
+ permitAcquireErrorHandler, () -> !watchers.containsKey(watcherId)))
+ // handled on the future of the whole operation, so that the callback runs once also after retries
+ .whenComplete((__, t) -> {
+ if (t != null) {
+ log.warn()
+ .attr("connection", connection)
+ .attr("watcherId", watcherId)
+ .exception(t)
+ .log("Failed to send topic list update for watcherId=. Watcher will be in "
+ + "inconsistent state.");
+ }
+ completionCallback.run();
+ });
}
// performs an operation with infinite permit acquiring retries.
- // If acquiring permits fails, it will retry after a backoff period
- private void performOperationWithPermitAcquiringRetries(long watcherId, String operationName,
+ // If acquiring permits fails, it will retry after a backoff period.
+ // When failOnCancellation is true, an operation that is skipped because the connection was closed or the
+ // watcher was removed completes exceptionally with PermitAcquireCancelledException instead of successfully.
+ // The returned future completes when the operation has completed, including the retries.
+ private CompletableFuture<Void> performOperationWithPermitAcquiringRetries(long watcherId, String operationName,
+ boolean failOnCancellation,
Function<Function<Throwable, CompletableFuture<Void>>,
Supplier<CompletableFuture<Void>>>
asyncOperationFactory) {
@@ -702,36 +907,43 @@ public class TopicListService {
AtomicReference<Supplier<CompletableFuture<Void>>> operationRef = new AtomicReference<>();
// create the error handler for the operation
Function<Throwable, CompletableFuture<Void>> permitAcquireErrorHandler =
- createPermitAcquireErrorHandler(watcherId, operationName, () -> operationRef.get().get());
+ createPermitAcquireErrorHandler(watcherId, operationName, failOnCancellation,
+ () -> operationRef.get().get());
// create the async operation using the factory function. Pass the error handler to the factory function.
Supplier<CompletableFuture<Void>> asyncOperation = asyncOperationFactory.apply(permitAcquireErrorHandler);
// set the operation to run into the operation reference
operationRef.set(() -> {
if (!connection.isActive() || !watchers.containsKey(watcherId)) {
// do nothing if the connection has already been closed or the watcher has been removed
- return CompletableFuture.completedFuture(null);
+ return failOnCancellation ? CompletableFuture.failedFuture(newWatcherClosedException(watcherId))
+ : CompletableFuture.completedFuture(null);
}
return asyncOperation.get().thenRun(() -> retryBackoff.reset());
});
// run the operation
- operationRef.get().get();
+ return operationRef.get().get();
}
// retries acquiring permits until the connection is closed or the watcher is removed
private Function<Throwable, CompletableFuture<Void>> createPermitAcquireErrorHandler(long watcherId,
String operationName,
+ boolean failOnCancellation,
Supplier<CompletableFuture
<Void>> operationRef) {
ScheduledExecutorService scheduledExecutor = connection.ctx().channel().eventLoop();
AtomicInteger retryCount = new AtomicInteger(0);
return t -> {
Throwable unwrappedException = FutureUtil.unwrapCompletionException(t);
- if (unwrappedException instanceof AsyncSemaphore.PermitAcquireCancelledException
- || unwrappedException instanceof AsyncSemaphore.PermitAcquireAlreadyClosedException
+ if (isCancellation(unwrappedException)
|| !connection.isActive()
|| !watchers.containsKey(watcherId)) {
- // stop retrying and complete successfully
- return CompletableFuture.completedFuture(null);
+ // stop retrying
+ return failOnCancellation ? CompletableFuture.failedFuture(newWatcherClosedException(watcherId))
+ : CompletableFuture.completedFuture(null);
+ }
+ if (unwrappedException instanceof AsyncSemaphore.PermitAcquireLimitExceededException) {
+ // the command is larger than the direct memory limit, retrying wouldn't help
+ return CompletableFuture.failedFuture(unwrappedException);
}
long retryDelay = retryBackoff.next().toMillis();
retryCount.incrementAndGet();
@@ -749,6 +961,28 @@ public class TopicListService {
};
}
+ /**
+ * Maps a failure of a watch request to the error response. A topic list or a response that is larger than the
+ * topic list memory limit gets a non-retriable error, since a retry fails the same way until the topic list is
+ * measured again.
+ */
+ private static ServerError getWatchTopicListErrorCode(Throwable failure) {
+ if (AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(failure)) {
+ return ServerError.NotAllowedError;
+ }
+ return BrokerServiceException.getClientErrorCode(new BrokerServiceException.ServerMetadataException(failure));
+ }
+
+ private static AsyncSemaphore.PermitAcquireCancelledException newWatcherClosedException(long watcherId) {
+ return new AsyncSemaphore.PermitAcquireCancelledException("Topic list watcher " + watcherId
+ + " was closed");
+ }
+
+ private static boolean isCancellation(Throwable unwrappedException) {
+ return unwrappedException instanceof AsyncSemaphore.PermitAcquireCancelledException
+ || unwrappedException instanceof AsyncSemaphore.PermitAcquireAlreadyClosedException;
+ }
+
@VisibleForTesting
CompletableFuture<TopicListWatcher> getWatcherFuture(long watcherId) {
return watchers.get(watcherId);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicListRestMemoryLimitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicListRestMemoryLimitTest.java
new file mode 100644
index 000000000000..3618dbdbb2b8
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicListRestMemoryLimitTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.doReturn;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
+import org.apache.pulsar.broker.namespace.NamespaceService;
+import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
+import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
+import org.apache.pulsar.client.admin.ListNamespaceTopicsOptions;
+import org.apache.pulsar.client.admin.ListTopicsOptions;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicDomain;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.LimitType;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Verifies that the REST API topic listing endpoints apply the topic list heap memory limit (PIP-442).
+ */
+@Test(groups = "broker-admin")
+public class TopicListRestMemoryLimitTest extends MockedPulsarServiceBaseTest {
+ private static final String NAMESPACE = "public/default";
+
+ @Override
+ protected void doInitConf() throws Exception {
+ super.doInitConf();
+ conf.setMaxTopicListInFlightHeapMemSizeMB(1);
+ conf.setMaxTopicListInFlightHeapMemSizePermitsAcquireTimeoutMillis(200);
+ }
+
+ @BeforeClass
+ @Override
+ protected void setup() throws Exception {
+ super.internalSetup();
+ setupDefaultTenantAndNamespace();
+ }
+
+ @AfterClass(alwaysRun = true)
+ @Override
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ @Test
+ public void testTopicListingEndpointsApplyHeapMemoryLimit() throws Exception {
+ String persistentTopic = "persistent://" + NAMESPACE + "/limited-persistent";
+ String partitionedTopic = "persistent://" + NAMESPACE + "/limited-partitioned";
+ String nonPersistentTopic = "non-persistent://" + NAMESPACE + "/limited-non-persistent";
+ String scalableTopic = "topic://" + NAMESPACE + "/limited-scalable";
+ admin.topics().createNonPartitionedTopic(persistentTopic);
+ admin.topics().createPartitionedTopic(partitionedTopic, 2);
+ admin.scalableTopics().createScalableTopic(scalableTopic, 1, Map.of("team", "a"));
+ // a connected producer keeps the non-persistent topic and its bundle loaded
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(nonPersistentTopic).create();
+ String bundleRange = admin.lookups().getBundleRange(nonPersistentTopic);
+
+ Map<String, ThrowingCallable> listings = Map.of(
+ "namespaces topics", () -> admin.namespaces().getTopics(NAMESPACE),
+ "persistent topics", () -> admin.topics().getList(NAMESPACE),
+ "persistent partitioned topics", () -> admin.topics().getPartitionedTopicList(NAMESPACE),
+ "non-persistent topics", () -> admin.topics().getList(NAMESPACE, TopicDomain.non_persistent),
+ "non-persistent bundle topics", () -> admin.topics().getListInBundle(NAMESPACE, bundleRange),
+ "scalable topics", () -> admin.scalableTopics().listScalableTopics(NAMESPACE),
+ "scalable topics by properties",
+ () -> admin.scalableTopics().listScalableTopicsByProperties(NAMESPACE, Map.of("team", "a")));
+
+ AsyncDualMemoryLimiterImpl limiter = pulsar.getBrokerService().getMaxTopicListInFlightLimiter();
+ long maxHeapPermits = limiter.getLimiter(LimitType.HEAP_MEMORY).getAvailablePermits();
+ // occupy all heap memory permits so that topic listings can't acquire permits
+ AsyncDualMemoryLimiterPermit permit = limiter.acquire(maxHeapPermits, LimitType.HEAP_MEMORY, () -> false)
+ .get(5, TimeUnit.SECONDS);
+ try {
+ listings.forEach((name, listing) -> assertThatThrownBy(listing)
+ .as(name)
+ .isInstanceOfSatisfying(PulsarAdminException.class,
+ e -> assertThat(e.getStatusCode()).as(name).isEqualTo(429)));
+ } finally {
+ limiter.release(permit);
+ }
+
+ // the listings succeed when permits are available
+ assertThat(admin.namespaces().getTopics(NAMESPACE)).contains(persistentTopic);
+ assertThat(admin.topics().getList(NAMESPACE)).contains(persistentTopic);
+ assertThat(admin.topics().getPartitionedTopicList(NAMESPACE)).contains(partitionedTopic);
+ assertThat(admin.topics().getList(NAMESPACE, TopicDomain.non_persistent)).contains(nonPersistentTopic);
+ assertThat(admin.topics().getListInBundle(NAMESPACE, bundleRange)).contains(nonPersistentTopic);
+ assertThat(admin.scalableTopics().listScalableTopics(NAMESPACE)).contains(scalableTopic);
+ assertThat(admin.scalableTopics().listScalableTopicsByProperties(NAMESPACE, Map.of("team", "a")))
+ .isEqualTo(List.of(scalableTopic));
+ // the permits are released after the responses have been written
+ assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isZero();
+ }
+
+ /**
+ * The non-persistent topic listings collect the topics from the bundle topic listings of the bundle owners,
+ * which are on this broker. They must not hold permits while waiting for those listings, otherwise the size
+ * estimates of the two listings could exceed the limit together and the listings would time out on each other.
+ */
+ @Test
+ public void testNonPersistentListingsDontWaitForTheirOwnPermits() throws Exception {
+ String nonPersistentTopic = "non-persistent://" + NAMESPACE + "/nested-listing";
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(nonPersistentTopic).create();
+ String bundleRange = admin.lookups().getBundleRange(nonPersistentTopic);
+
+ AsyncDualMemoryLimiterImpl limiter = pulsar.getBrokerService().getMaxTopicListInFlightLimiter();
+ long maxHeapPermits = limiter.getLimiter(LimitType.HEAP_MEMORY).getAvailablePermits();
+ long estimate = maxHeapPermits * 6 / 10;
+ TopicListSizeResultCache sizeCache = pulsar.getBrokerService().getTopicListSizeResultCache();
+ for (int i = 0; i < 3; i++) {
+ // estimates from earlier, larger topic lists: each listing alone fits within the limit, both don't
+ sizeCache.getTopicListSize(TopicListing.ownedBundleTopics(NAMESPACE, Mode.NON_PERSISTENT, bundleRange),
+ null).updateSize(estimate);
+ sizeCache.getTopicListSize(TopicListing.namespaceTopics(NAMESPACE, Mode.NON_PERSISTENT), null)
+ .updateSize(estimate);
+ sizeCache.getTopicListSize(TopicListing.namespaceTopics(NAMESPACE, Mode.ALL), null)
+ .updateSize(estimate);
+
+ assertThat(admin.topics().getList(NAMESPACE, TopicDomain.non_persistent)).contains(nonPersistentTopic);
+ assertThat(admin.topics().getList(NAMESPACE, TopicDomain.non_persistent,
+ ListTopicsOptions.builder().bundle(bundleRange).build())).contains(nonPersistentTopic);
+ for (org.apache.pulsar.client.admin.Mode mode : List.of(org.apache.pulsar.client.admin.Mode.NON_PERSISTENT,
+ org.apache.pulsar.client.admin.Mode.ALL)) {
+ assertThat(admin.namespaces().getTopics(NAMESPACE,
+ ListNamespaceTopicsOptions.builder().mode(mode).build()))
+ .as(mode.name()).contains(nonPersistentTopic);
+ }
+ }
+ assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isZero();
+ }
+
+ /**
+ * The ALL mode collects the non-persistent topics from the bundle listings, which acquire permits from the
+ * same limiter on this broker, before it acquires permits for the whole topic list. Holding the permits of a
+ * large persistent topic list while waiting for the bundle listings could leave too few permits for them.
+ */
+ @Test
+ public void testAllModeDoesNotHoldPermitsWhileWaitingForBundleListings() throws Exception {
+ String nonPersistentTopic = "non-persistent://" + NAMESPACE + "/all-mode-listing";
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(nonPersistentTopic).create();
+ String bundleRange = admin.lookups().getBundleRange(nonPersistentTopic);
+
+ AsyncDualMemoryLimiterImpl limiter = pulsar.getBrokerService().getMaxTopicListInFlightLimiter();
+ long maxHeapPermits = limiter.getLimiter(LimitType.HEAP_MEMORY).getAvailablePermits();
+ // a persistent topic list of about 60% of the limit
+ List<String> persistentTopics = new ArrayList<>();
+ String longName = "t".repeat(10_000);
+ while (TopicListMemoryLimiter.estimateTopicListSize(persistentTopics) < maxHeapPermits * 6 / 10) {
+ persistentTopics.add("persistent://" + NAMESPACE + "/" + longName + persistentTopics.size());
+ }
+ NamespaceService namespaceService = pulsar.getNamespaceService();
+ doReturn(CompletableFuture.completedFuture(persistentTopics))
+ .when(namespaceService).getListOfPersistentTopics(NamespaceName.get(NAMESPACE));
+ try {
+ TopicListSizeResultCache sizeCache = pulsar.getBrokerService().getTopicListSizeResultCache();
+ sizeCache.getTopicListSize(TopicListing.namespaceTopics(NAMESPACE, Mode.ALL), null)
+ .updateSize(TopicListMemoryLimiter.estimateTopicListSize(persistentTopics));
+ // the bundle listing's estimate fits within the limit, but not together with the persistent topic list
+ sizeCache.getTopicListSize(TopicListing.ownedBundleTopics(NAMESPACE, Mode.NON_PERSISTENT, bundleRange),
+ null).updateSize(maxHeapPermits * 6 / 10);
+
+ for (int i = 0; i < 3; i++) {
+ assertThat(admin.namespaces().getTopics(NAMESPACE, ListNamespaceTopicsOptions.builder()
+ .mode(org.apache.pulsar.client.admin.Mode.ALL).build()))
+ .contains(nonPersistentTopic, persistentTopics.get(0));
+ }
+ } finally {
+ doCallRealMethod().when(namespaceService).getListOfPersistentTopics(any());
+ }
+ assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isZero();
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiterTest.java
new file mode 100644
index 000000000000..0fa068c70c55
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/impl/TopicListResponseMemoryLimiterTest.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import jakarta.ws.rs.container.AsyncResponse;
+import jakarta.ws.rs.container.CompletionCallback;
+import jakarta.ws.rs.core.Response;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
+import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
+import org.apache.pulsar.broker.topiclistlimit.TopicListing;
+import org.apache.pulsar.broker.web.RestException;
+import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.LimitType;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class TopicListResponseMemoryLimiterTest {
+ private static final String NAMESPACE = "tenant/ns";
+ private static final long MAX_HEAP_MEMORY = 100_000;
+ private static final List<String> TOPICS =
+ List.of("persistent://tenant/ns/topic-1", "persistent://tenant/ns/topic-2");
+
+ private AsyncDualMemoryLimiterImpl limiter;
+ private TopicListSizeResultCache sizeCache;
+ private BrokerService brokerService;
+ private AsyncResponse response;
+ private AtomicBoolean responseDone;
+ private AtomicReference<CompletionCallback> completionCallback;
+
+ @BeforeMethod
+ public void setup() {
+ limiter = new AsyncDualMemoryLimiterImpl(MAX_HEAP_MEMORY, 10, 5000, MAX_HEAP_MEMORY, 10, 5000);
+ sizeCache = new TopicListSizeResultCache();
+ brokerService = mock(BrokerService.class);
+ when(brokerService.getMaxTopicListInFlightLimiter()).thenAnswer(__ -> limiter);
+ when(brokerService.getTopicListSizeResultCache()).thenReturn(sizeCache);
+ responseDone = new AtomicBoolean();
+ completionCallback = new AtomicReference<>();
+ response = mock(AsyncResponse.class);
+ when(response.isDone()).thenAnswer(__ -> responseDone.get());
+ doAnswer(invocation -> {
+ completionCallback.set(invocation.getArgument(0));
+ return List.of();
+ }).when(response).register(any(Object.class));
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void cleanup() {
+ if (limiter != null) {
+ limiter.close();
+ limiter = null;
+ }
+ }
+
+ private long acquiredHeapPermits() {
+ return limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits();
+ }
+
+ private void completeResponse(Throwable throwable) {
+ responseDone.set(true);
+ completionCallback.get().onComplete(throwable);
+ }
+
+ private CompletableFuture<List<String>> listTopics(Supplier<CompletableFuture<List<String>>> topicsSupplier) {
+ return TopicListResponseMemoryLimiter.listTopics(brokerService, response,
+ TopicListing.namespaceTopics(NAMESPACE, Mode.PERSISTENT), null, topicsSupplier);
+ }
+
+ @Test
+ public void testPermitsAreHeldUntilResponseCompletes() {
+ CompletableFuture<List<String>> result = listTopics(() -> CompletableFuture.completedFuture(TOPICS));
+
+ assertThat(result).succeedsWithin(Duration.ofSeconds(5)).isEqualTo(TOPICS);
+ // the topic list is referenced until the response has been written
+ assertThat(acquiredHeapPermits()).isEqualTo(TopicListMemoryLimiter.estimateTopicListSize(TOPICS));
+
+ completeResponse(null);
+ assertThat(acquiredHeapPermits()).isZero();
+ }
+
+ @Test
+ public void testDoesNotListTopicsWhenResponseIsAlreadyDone() {
+ responseDone.set(true);
+ AtomicInteger listingCalls = new AtomicInteger();
+
+ CompletableFuture<List<String>> result = listTopics(() -> {
+ listingCalls.incrementAndGet();
+ return CompletableFuture.completedFuture(TOPICS);
+ });
+
+ assertThat(result).failsWithin(Duration.ofSeconds(5)).withThrowableThat()
+ .havingRootCause().isInstanceOf(RestException.class);
+ assertThat(listingCalls).hasValue(0);
+ assertThat(acquiredHeapPermits()).isZero();
+ // concurrent requests waiting for the size estimate must not be left waiting
+ assertThat(sizeCache.getTopicListSize(NAMESPACE, Mode.PERSISTENT).getSizeAsync()).isDone();
+ }
+
+ @Test
+ public void testDoesNotListTopicsWhenResponseTimesOutWhileWaitingForPermits() throws Exception {
+ // occupy all permits so that the request waits in the queue
+ AsyncDualMemoryLimiterPermit blockingPermit =
+ limiter.acquire(MAX_HEAP_MEMORY, LimitType.HEAP_MEMORY, () -> false).get(5, TimeUnit.SECONDS);
+ AtomicInteger listingCalls = new AtomicInteger();
+ CompletableFuture<List<String>> result = listTopics(() -> {
+ listingCalls.incrementAndGet();
+ return CompletableFuture.completedFuture(TOPICS);
+ });
+ Awaitility.await().until(() -> limiter.getLimiter(LimitType.HEAP_MEMORY).getQueueSize() == 1);
+
+ // an async response timeout completes the response with a mapped response, without a throwable
+ completeResponse(null);
+ limiter.release(blockingPermit);
+
+ assertThat(result).failsWithin(Duration.ofSeconds(5));
+ assertThat(listingCalls).hasValue(0);
+ assertThat(acquiredHeapPermits()).isZero();
+ }
+
+ @Test
+ public void testPermitsAreReleasedWhenResponseCompletesDuringListing() {
+ CompletableFuture<List<String>> result = listTopics(() -> {
+ assertThat(acquiredHeapPermits()).isPositive();
+ completeResponse(null);
+ return CompletableFuture.completedFuture(TOPICS);
+ });
+
+ assertThat(result).failsWithin(Duration.ofSeconds(5));
+ assertThat(acquiredHeapPermits()).isZero();
+ }
+
+ @Test
+ public void testPermitGrantedAfterResponseCompletionIsReleased() {
+ // simulate a permit grant that races with the completion of the response
+ AsyncDualMemoryLimiterImpl mockedLimiter = mock(AsyncDualMemoryLimiterImpl.class);
+ AsyncDualMemoryLimiterPermit permit = mock(AsyncDualMemoryLimiterPermit.class);
+ when(mockedLimiter.acquire(anyLong(), any(), any())).thenAnswer(__ -> {
+ completionCallback.get().onComplete(null);
+ return CompletableFuture.completedFuture(permit);
+ });
+ when(brokerService.getMaxTopicListInFlightLimiter()).thenReturn(mockedLimiter);
+ AtomicInteger listingCalls = new AtomicInteger();
+
+ CompletableFuture<List<String>> result = listTopics(() -> {
+ listingCalls.incrementAndGet();
+ return CompletableFuture.completedFuture(TOPICS);
+ });
+
+ assertThat(result).failsWithin(Duration.ofSeconds(5));
+ assertThat(listingCalls).hasValue(0);
+ verify(mockedLimiter, times(1)).release(permit);
+ }
+
+ @Test
+ public void testPermitGrantedAfterResponseIsDoneIsReleased() {
+ // simulate a permit grant that races with the cancellation of the response, before the completion
+ // callback has been called
+ AsyncDualMemoryLimiterImpl mockedLimiter = mock(AsyncDualMemoryLimiterImpl.class);
+ AsyncDualMemoryLimiterPermit permit = mock(AsyncDualMemoryLimiterPermit.class);
+ when(mockedLimiter.acquire(anyLong(), any(), any())).thenAnswer(__ -> {
+ responseDone.set(true);
+ return CompletableFuture.completedFuture(permit);
+ });
+ when(brokerService.getMaxTopicListInFlightLimiter()).thenReturn(mockedLimiter);
+ AtomicInteger listingCalls = new AtomicInteger();
+
+ CompletableFuture<List<String>> result = listTopics(() -> {
+ listingCalls.incrementAndGet();
+ return CompletableFuture.completedFuture(TOPICS);
+ });
+
+ assertThat(result).failsWithin(Duration.ofSeconds(5));
+ assertThat(listingCalls).hasValue(0);
+ verify(mockedLimiter, times(1)).release(permit);
+ // the cancellation is recognized as the limiter's own 503, unlike other 503 responses
+ Throwable failure = result.handle((__, t) -> t).join();
+ assertThat(TopicListResponseMemoryLimiter.isRequestCancelled(failure)).isTrue();
+ assertThat(TopicListResponseMemoryLimiter.isRequestCancelled(
+ new RestException(Response.Status.SERVICE_UNAVAILABLE, "not ready"))).isFalse();
+ }
+
+ @Test
+ public void testTopicListLargerThanLimitFailsWithTooManyRequests() {
+ List<String> hugeTopicList = List.of("persistent://tenant/ns/" + "t".repeat((int) MAX_HEAP_MEMORY));
+
+ // the limiter rejects the update to the actual size synchronously
+ CompletableFuture<List<String>> result =
+ listTopics(() -> CompletableFuture.completedFuture(hugeTopicList));
+ assertThat(result).failsWithin(Duration.ofSeconds(5)).withThrowableThat()
+ .havingRootCause().isInstanceOfSatisfying(RestException.class,
+ e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+ completeResponse(new RuntimeException("failed"));
+ assertThat(acquiredHeapPermits()).isZero();
+
+ // the cached estimate is now larger than the limit and the initial acquisition is rejected synchronously
+ responseDone.set(false);
+ CompletableFuture<List<String>> result2 = listTopics(() -> CompletableFuture.completedFuture(TOPICS));
+ assertThat(result2).failsWithin(Duration.ofSeconds(5)).withThrowableThat()
+ .havingRootCause().isInstanceOfSatisfying(RestException.class,
+ e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+ }
+
+ @Test
+ public void testPermitAcquireFailureFailsWithTooManyRequests() throws Exception {
+ limiter.close();
+ // a queue size of 1 with a queued request causes the next request to fail
+ limiter = new AsyncDualMemoryLimiterImpl(MAX_HEAP_MEMORY, 1, 5000, MAX_HEAP_MEMORY, 1, 5000);
+ AsyncDualMemoryLimiterPermit blockingPermit =
+ limiter.acquire(MAX_HEAP_MEMORY, LimitType.HEAP_MEMORY, () -> false).get(5, TimeUnit.SECONDS);
+ CompletableFuture<AsyncDualMemoryLimiterPermit> queued =
+ limiter.acquire(1, LimitType.HEAP_MEMORY, () -> false);
+
+ CompletableFuture<List<String>> result = listTopics(() -> CompletableFuture.completedFuture(TOPICS));
+
+ assertThat(result).failsWithin(Duration.ofSeconds(5)).withThrowableThat()
+ .havingRootCause().isInstanceOfSatisfying(RestException.class,
+ e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+ limiter.release(blockingPermit);
+ limiter.release(queued.get(5, TimeUnit.SECONDS));
+ assertThat(acquiredHeapPermits()).isZero();
+ }
+
+ @Test
+ public void testListTopicsFromOtherListingsHoldsPermitsOnlyAfterListing() {
+ AtomicLong permitsDuringListing = new AtomicLong(-1);
+ CompletableFuture<List<String>> result = TopicListResponseMemoryLimiter.listTopicsFromOtherListings(
+ brokerService, response, () -> {
+ permitsDuringListing.set(acquiredHeapPermits());
+ return CompletableFuture.completedFuture(TOPICS);
+ });
+
+ assertThat(result).succeedsWithin(Duration.ofSeconds(5)).isEqualTo(TOPICS);
+ // no permits are held while the topics are collected from the other listings
+ assertThat(permitsDuringListing).hasValue(0);
+ // permits for the collected list are held until the response has been written
+ assertThat(acquiredHeapPermits()).isEqualTo(TopicListMemoryLimiter.estimateTopicListSize(TOPICS));
+ completeResponse(null);
+ assertThat(acquiredHeapPermits()).isZero();
+ }
+
+ @Test
+ public void testListTopicsFromOtherListingsSkipsListingWhenResponseIsAlreadyDone() {
+ responseDone.set(true);
+ AtomicInteger listingCalls = new AtomicInteger();
+
+ CompletableFuture<List<String>> result = TopicListResponseMemoryLimiter.listTopicsFromOtherListings(
+ brokerService, response, () -> {
+ listingCalls.incrementAndGet();
+ return CompletableFuture.completedFuture(TOPICS);
+ });
+
+ assertThat(result).failsWithin(Duration.ofSeconds(5));
+ assertThat(listingCalls).hasValue(0);
+ assertThat(acquiredHeapPermits()).isZero();
+ }
+
+ @Test
+ public void testOversizedListingFailsWithoutWaitingForPermits() throws Exception {
+ sizeCache = new TopicListSizeResultCache(MAX_HEAP_MEMORY, () -> true);
+ when(brokerService.getTopicListSizeResultCache()).thenReturn(sizeCache);
+ TopicListSizeResultCache.ResultHolder holder =
+ sizeCache.getTopicListSize(NAMESPACE, Mode.PERSISTENT);
+ // a previous listing measured a topic list larger than the limit
+ holder.updateSize(MAX_HEAP_MEMORY * 2);
+ // another topic listing is in flight
+ AsyncDualMemoryLimiterPermit otherListingPermit =
+ limiter.acquire(1, LimitType.HEAP_MEMORY, () -> false).get(5, TimeUnit.SECONDS);
+ AtomicInteger listingCalls = new AtomicInteger();
+
+ long startNanos = System.nanoTime();
+ CompletableFuture<List<String>> result = listTopics(() -> {
+ listingCalls.incrementAndGet();
+ return CompletableFuture.completedFuture(TOPICS);
+ });
+
+ // fails right away with 429 instead of waiting in the queue until no other topic list is in flight
+ assertThat(result).failsWithin(Duration.ofSeconds(2)).withThrowableThat()
+ .havingRootCause().isInstanceOfSatisfying(RestException.class,
+ e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+ assertThat(Duration.ofNanos(System.nanoTime() - startNanos)).isLessThan(Duration.ofSeconds(2));
+ assertThat(listingCalls).hasValue(0);
+ assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getQueueSize()).isZero();
+ limiter.release(otherListingPermit);
+ }
+
+ @Test
+ public void testListTopicsAfterCollectingSkipsCollectingWhenEstimateIsLargerThanLimit() {
+ sizeCache = new TopicListSizeResultCache(MAX_HEAP_MEMORY, () -> true);
+ when(brokerService.getTopicListSizeResultCache()).thenReturn(sizeCache);
+ TopicListing listing = TopicListing.namespaceTopics(NAMESPACE, Mode.ALL);
+ // a previous listing measured a topic list larger than the limit
+ sizeCache.getTopicListSize(listing, null).updateSize(MAX_HEAP_MEMORY * 2);
+ AtomicInteger collectorCalls = new AtomicInteger();
+ AtomicInteger listingCalls = new AtomicInteger();
+
+ CompletableFuture<List<String>> result = TopicListResponseMemoryLimiter.listTopicsAfterCollecting(
+ brokerService, response, listing, null, () -> {
+ collectorCalls.incrementAndGet();
+ return CompletableFuture.completedFuture(TOPICS);
+ }, collected -> {
+ listingCalls.incrementAndGet();
+ return CompletableFuture.completedFuture(collected);
+ });
+
+ assertThat(result).failsWithin(Duration.ofSeconds(2)).withThrowableThat()
+ .havingRootCause().isInstanceOfSatisfying(RestException.class,
+ e -> assertThat(e.getResponse().getStatus()).isEqualTo(429));
+ assertThat(collectorCalls).hasValue(0);
+ assertThat(listingCalls).hasValue(0);
+ }
+
+ @Test
+ public void testListTopicsAfterCollectingAcquiresPermitsAfterCollecting() {
+ AtomicLong permitsWhileCollecting = new AtomicLong(-1);
+ CompletableFuture<List<String>> result = TopicListResponseMemoryLimiter.listTopicsAfterCollecting(
+ brokerService, response, TopicListing.namespaceTopics(NAMESPACE, Mode.ALL), null, () -> {
+ permitsWhileCollecting.set(acquiredHeapPermits());
+ return CompletableFuture.completedFuture(TOPICS);
+ }, CompletableFuture::completedFuture);
+
+ assertThat(result).succeedsWithin(Duration.ofSeconds(5)).isEqualTo(TOPICS);
+ assertThat(permitsWhileCollecting).hasValue(0);
+ assertThat(acquiredHeapPermits()).isEqualTo(TopicListMemoryLimiter.estimateTopicListSize(TOPICS));
+ completeResponse(null);
+ assertThat(acquiredHeapPermits()).isZero();
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
index 6722440e0ca7..5e871e500780 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
@@ -30,6 +30,7 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.clearInvocations;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.matches;
@@ -128,6 +129,8 @@ import org.apache.pulsar.broker.service.scalable.ScalableTopicService;
import org.apache.pulsar.broker.service.schema.SchemaRegistryService;
import org.apache.pulsar.broker.service.utils.ClientChannelHelper;
import org.apache.pulsar.broker.testcontext.PulsarTestContext;
+import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
+import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
import org.apache.pulsar.broker.web.RestException;
import org.apache.pulsar.client.api.ProducerAccessMode;
import org.apache.pulsar.client.api.PulsarClientException;
@@ -187,6 +190,8 @@ import org.apache.pulsar.common.protocol.schema.EmptyVersion;
import org.apache.pulsar.common.protocol.schema.SchemaData;
import org.apache.pulsar.common.protocol.schema.SchemaVersion;
import org.apache.pulsar.common.scalable.HashRange;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
import org.apache.pulsar.common.topics.TopicList;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap;
@@ -4979,4 +4984,127 @@ public class ServerCnxTest {
channel.finish();
}
+
+ @Test(timeOut = 30000)
+ public void testGetTopicsOfNamespaceLargerThanHeapMemoryLimit() throws Exception {
+ long topicListSize = TopicListMemoryLimiter.estimateTopicListSize(topics);
+ // both the initial size estimate and the topic list are larger than the heap memory limit
+ AsyncDualMemoryLimiterImpl limiter =
+ new AsyncDualMemoryLimiterImpl(topicListSize - 1, 10, 1000, 1_000_000, 10, 1000);
+ try {
+ doReturn(limiter).when(brokerService).getMaxTopicListInFlightLimiter();
+ doReturn(new TopicListSizeResultCache()).when(brokerService).getTopicListSizeResultCache();
+ resetChannel();
+ setChannelConnected();
+ var lookupSemaphore = brokerService.getLookupRequestSemaphore();
+ int initialPermits = lookupSemaphore.availablePermits();
+ for (long requestId = 1; requestId <= 3; requestId++) {
+ channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+ "use/ns-abc", requestId, CommandGetTopicsOfNamespace.Mode.ALL, null, null, null));
+ // the request is rejected instead of being left without a response. The topic list hasn't been
+ // measured, so the error is retriable.
+ CommandError error = (CommandError) getResponse();
+ assertEquals(error.getRequestId(), requestId);
+ assertEquals(error.getError(), ServerError.TooManyRequests);
+ // the lookup permit is released
+ assertEquals(lookupSemaphore.availablePermits(), initialPermits);
+ }
+ } finally {
+ channel.finish();
+ limiter.close();
+ }
+ }
+
+ @Test(timeOut = 30000)
+ public void testGetTopicsOfNamespaceWithNewPropertiesAndOversizedEstimateIsRetriable() throws Exception {
+ long topicListSize = TopicListMemoryLimiter.estimateTopicListSize(topics);
+ AsyncDualMemoryLimiterImpl limiter =
+ new AsyncDualMemoryLimiterImpl(topicListSize - 1, 10, 1000, 1_000_000, 10, 1000);
+ try {
+ doReturn(limiter).when(brokerService).getMaxTopicListInFlightLimiter();
+ // the limiter is never idle, so an oversized estimate isn't measured again within the test
+ doReturn(new TopicListSizeResultCache(topicListSize - 1, () -> false))
+ .when(brokerService).getTopicListSizeResultCache();
+ resetChannel();
+ setChannelConnected();
+ clearInvocations(namespaceService);
+ // the listing without properties is measured to be larger than the limit
+ channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+ "use/ns-abc", 1, CommandGetTopicsOfNamespace.Mode.ALL, null, null, null));
+ CommandError error = (CommandError) getResponse();
+ assertEquals(error.getRequestId(), 1);
+ assertEquals(error.getError(), ServerError.NotAllowedError);
+
+ // a listing with new properties starts from that estimate, but its own topic list hasn't been measured,
+ // so it gets a retriable error
+ channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+ "use/ns-abc", 2, CommandGetTopicsOfNamespace.Mode.ALL, null, null, Map.of("label", "x")));
+ error = (CommandError) getResponse();
+ assertEquals(error.getRequestId(), 2);
+ assertEquals(error.getError(), ServerError.TooManyRequests);
+ verify(namespaceService, times(1)).getListOfUserTopicsByProperties(any(), any(), any());
+ } finally {
+ channel.finish();
+ limiter.close();
+ }
+ }
+
+ @Test(timeOut = 30000)
+ public void testGetTopicsOfNamespacePermitTimeoutIsRetriable() throws Exception {
+ long heapMemoryLimit = 1_000_000;
+ // the heap memory limit fits the topic list, but all of it is held by another request
+ AsyncDualMemoryLimiterImpl limiter =
+ new AsyncDualMemoryLimiterImpl(heapMemoryLimit, 10, 100, 1_000_000, 10, 1000);
+ AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit heldPermit = limiter.acquire(heapMemoryLimit,
+ AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY, () -> false).get(10, TimeUnit.SECONDS);
+ try {
+ doReturn(limiter).when(brokerService).getMaxTopicListInFlightLimiter();
+ doReturn(new TopicListSizeResultCache()).when(brokerService).getTopicListSizeResultCache();
+ resetChannel();
+ setChannelConnected();
+ channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+ "use/ns-abc", 1, CommandGetTopicsOfNamespace.Mode.ALL, null, null, null));
+ // a permit timeout gets an error that the client retries, unlike a topic list larger than the limit
+ CommandError error = (CommandError) getResponse();
+ assertEquals(error.getRequestId(), 1);
+ assertEquals(error.getError(), ServerError.TooManyRequests);
+ } finally {
+ limiter.release(heldPermit);
+ channel.finish();
+ limiter.close();
+ }
+ }
+
+ @Test(timeOut = 30000)
+ public void testGetTopicsOfNamespaceLargerThanHeapMemoryLimitFailsFastAfterMeasuring() throws Exception {
+ long topicListSize = TopicListMemoryLimiter.estimateTopicListSize(topics);
+ AsyncDualMemoryLimiterImpl limiter =
+ new AsyncDualMemoryLimiterImpl(topicListSize - 1, 10, 1000, 1_000_000, 10, 1000);
+ try {
+ doReturn(limiter).when(brokerService).getMaxTopicListInFlightLimiter();
+ // like the broker's cache: the initial estimate fits within the limit, so the first request lists the
+ // topics and is rejected when the permits are updated to the size of the topic list
+ doReturn(new TopicListSizeResultCache(topicListSize - 1, () -> true))
+ .when(brokerService).getTopicListSizeResultCache();
+ resetChannel();
+ setChannelConnected();
+ clearInvocations(namespaceService);
+ var lookupSemaphore = brokerService.getLookupRequestSemaphore();
+ int initialPermits = lookupSemaphore.availablePermits();
+ for (long requestId = 1; requestId <= 3; requestId++) {
+ channel.writeInbound(Commands.newGetTopicsOfNamespaceRequest(
+ "use/ns-abc", requestId, CommandGetTopicsOfNamespace.Mode.ALL, null, null, null));
+ CommandError error = (CommandError) getResponse();
+ assertEquals(error.getRequestId(), requestId);
+ assertEquals(error.getError(), ServerError.NotAllowedError);
+ assertEquals(lookupSemaphore.availablePermits(), initialPermits);
+ }
+ // only the first request listed the topics, the later requests failed without listing them
+ verify(namespaceService, times(1)).getListOfUserTopicsByProperties(any(), any(), any());
+ assertEquals(limiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY).getAcquiredPermits(), 0);
+ } finally {
+ channel.finish();
+ limiter.close();
+ }
+ }
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java
index e39ab45a0a84..8c2ade341fb9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java
@@ -29,8 +29,10 @@ import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@@ -52,8 +54,11 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.IntStream;
@@ -63,6 +68,7 @@ import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.namespace.NamespaceService;
import org.apache.pulsar.broker.resources.PulsarResources;
+import org.apache.pulsar.broker.resources.TopicListener;
import org.apache.pulsar.broker.resources.TopicResources;
import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
@@ -75,6 +81,7 @@ import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
import org.apache.pulsar.common.semaphore.AsyncSemaphore;
import org.apache.pulsar.common.topics.TopicList;
import org.apache.pulsar.common.topics.TopicsPattern;
+import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.Notification;
import org.apache.pulsar.metadata.api.NotificationType;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
@@ -104,6 +111,8 @@ public class TopicListServiceTest {
private ScheduledExecutorService scheduledExecutorService;
private PulsarService pulsar;
private NamespaceService namespaceService;
+ private TopicListSizeResultCache.ResultHolder resultHolder;
+ private AtomicBoolean connectionActive;
@BeforeMethod(alwaysRun = true)
public void setup() throws Exception {
@@ -132,7 +141,7 @@ public class TopicListServiceTest {
when(pulsar.getBrokerService()).thenReturn(brokerService);
TopicListSizeResultCache topicListSizeResultCache = mock(TopicListSizeResultCache.class);
when(brokerService.getTopicListSizeResultCache()).thenReturn(topicListSizeResultCache);
- TopicListSizeResultCache.ResultHolder resultHolder = mock(TopicListSizeResultCache.ResultHolder.class);
+ resultHolder = mock(TopicListSizeResultCache.ResultHolder.class);
doReturn(resultHolder).when(topicListSizeResultCache).getTopicListSize(anyString(), any());
doReturn(CompletableFuture.completedFuture(1L)).when(resultHolder).getSizeAsync();
@@ -143,10 +152,11 @@ public class TopicListServiceTest {
when(connection.getRemoteAddress()).thenReturn(new InetSocketAddress(10000));
pulsarCommandSender = mock(PulsarCommandSender.class);
when(connection.getCommandSender()).thenReturn(pulsarCommandSender);
- when(connection.isActive()).thenReturn(true);
- when(pulsarCommandSender.sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any()))
+ connectionActive = new AtomicBoolean(true);
+ when(connection.isActive()).thenAnswer(__ -> connectionActive.get());
+ when(pulsarCommandSender.sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any()))
.thenReturn(CompletableFuture.completedFuture(null));
- when(pulsarCommandSender.sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any()))
+ when(pulsarCommandSender.sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any()))
.thenReturn(CompletableFuture.completedFuture(null));
scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
@@ -210,7 +220,7 @@ public class TopicListServiceTest {
verify(topicResources).registerPersistentTopicListener(any(TopicListService.TopicListWatcher.class));
Collection<String> expectedTopics = new ArrayList<>(topics);
verify(connection.getCommandSender()).sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics),
- any());
+ any(), any());
}
@Test
@@ -238,7 +248,31 @@ public class TopicListServiceTest {
verify(topicResources).registerPersistentTopicListener(any(TopicListService.TopicListWatcher.class));
Collection<String> expectedTopics = new ArrayList<>(topics);
verify(connection.getCommandSender()).sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics),
- any());
+ any(), any());
+ }
+
+ @Test
+ public void testCancelledWatchResetsTopicListSizeEstimate() throws Exception {
+ // acquire all permits so that the watcher's permit request waits in the queue
+ AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit permit =
+ memoryLimiter.acquire(1_000_000, AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY,
+ Boolean.FALSE::booleanValue)
+ .get();
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ 7,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+ Awaitility.await().untilAsserted(() -> Assert.assertEquals(
+ memoryLimiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY).getQueueSize(), 1));
+ // the connection closes while the request waits for permits
+ connectionActive.set(false);
+ memoryLimiter.release(permit);
+ // concurrent requests waiting for the size estimate of the cancelled request must be able to proceed
+ Awaitility.await().untilAsserted(() -> verify(resultHolder).resetIfInitializing());
+ verify(namespaceService, never()).getListOfUserTopics(any(), any());
}
@Test
@@ -299,12 +333,12 @@ public class TopicListServiceTest {
} else {
return CompletableFuture.completedFuture(null);
}
- }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any());
+ }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any());
topicListFuture.complete(topics);
assertThat(topicListService.getWatcherFuture(13)).succeedsWithin(Duration.ofSeconds(2));
Collection<String> expectedTopics = new ArrayList<>(topics);
verify(connection.getCommandSender(), timeout(2000L).times(3))
- .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any());
+ .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any(), any());
}
@Test
@@ -325,13 +359,13 @@ public class TopicListServiceTest {
notificationConsumer.accept(
new Notification(NotificationType.Created, "/managed-ledgers/tenant/ns/persistent/topic2"));
verify(connection.getCommandSender(), timeout(2000L))
- .sendWatchTopicListUpdate(eq(13L), eq(newTopics), any(), eq(hash), any());
+ .sendWatchTopicListUpdate(eq(13L), eq(newTopics), any(), eq(hash), any(), any());
hash = TopicList.calculateHash(newTopics);
notificationConsumer.accept(
new Notification(NotificationType.Deleted, "/managed-ledgers/tenant/ns/persistent/topic1"));
verify(connection.getCommandSender(), timeout(2000L))
- .sendWatchTopicListUpdate(eq(13L), eq(List.of()), eq(topics), eq(hash), any());
+ .sendWatchTopicListUpdate(eq(13L), eq(List.of()), eq(topics), eq(hash), any(), any());
}
@Test
@@ -348,7 +382,7 @@ public class TopicListServiceTest {
} else {
return CompletableFuture.completedFuture(null);
}
- }).when(pulsarCommandSender).sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any());
+ }).when(pulsarCommandSender).sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
topicListService.handleWatchTopicList(
NamespaceName.get("tenant/ns"),
@@ -369,9 +403,9 @@ public class TopicListServiceTest {
new Notification(NotificationType.Deleted, "/managed-ledgers/tenant/ns/persistent/topic2"));
InOrder inOrder = inOrder(connection.getCommandSender());
inOrder.verify(connection.getCommandSender(), timeout(2000L).times(3))
- .sendWatchTopicListUpdate(eq(13L), eq(newTopics), eq(List.of()), eq(hash), any());
+ .sendWatchTopicListUpdate(eq(13L), eq(newTopics), eq(List.of()), eq(hash), any(), any());
inOrder.verify(connection.getCommandSender(), timeout(2000L).times(1))
- .sendWatchTopicListUpdate(eq(13L), eq(List.of()), eq(newTopics), any(), any());
+ .sendWatchTopicListUpdate(eq(13L), eq(List.of()), eq(newTopics), any(), any(), any());
}
@Test
@@ -420,12 +454,12 @@ public class TopicListServiceTest {
topicsPatternImplementation, hash,
lookupSemaphore);
doReturn(CompletableFuture.completedFuture(null)).when(pulsarCommandSender)
- .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any());
+ .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any());
topicListFuture.complete(topics);
assertThat(topicListService.getWatcherFuture(13)).succeedsWithin(Duration.ofSeconds(2));
Collection<String> expectedTopics = List.of();
verify(connection.getCommandSender(), timeout(2000L).times(1))
- .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any());
+ .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any(), any());
}
@Test
@@ -440,12 +474,12 @@ public class TopicListServiceTest {
topicsPatternImplementation, "INVALID_HASH",
lookupSemaphore);
doReturn(CompletableFuture.completedFuture(null)).when(pulsarCommandSender)
- .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any());
+ .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any());
topicListFuture.complete(topics);
assertThat(topicListService.getWatcherFuture(13)).succeedsWithin(Duration.ofSeconds(2));
Collection<String> expectedTopics = topics;
verify(connection.getCommandSender(), timeout(2000L).times(1))
- .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any());
+ .sendWatchTopicListSuccess(eq(7L), eq(13L), eq(hash), eq(expectedTopics), any(), any());
}
@Test
@@ -503,7 +537,7 @@ public class TopicListServiceTest {
.sendWatchTopicListUpdate(eq(13L),
argThat(newTopics -> new HashSet<>(newTopics).equals(new HashSet<>(expectedNew))),
argThat(deletedTopics -> new HashSet<>(deletedTopics).equals(new HashSet<>(expectedDeleted))),
- eq(expectedHash), any());
+ eq(expectedHash), any(), any());
// Verify the watcher's matching topics are updated
assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(topicsAfterReconnect);
@@ -560,7 +594,7 @@ public class TopicListServiceTest {
.sendWatchTopicListUpdate(eq(13L),
argThat(newTopics -> new HashSet<>(newTopics).equals(new HashSet<>(expectedNew))),
argThat(deletedTopics -> new HashSet<>(deletedTopics).equals(new HashSet<>(expectedDeleted))),
- eq(expectedHash), any());
+ eq(expectedHash), any(), any());
assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(topicsAfterReconnect);
}
@@ -604,7 +638,7 @@ public class TopicListServiceTest {
// Verify that sendWatchTopicListUpdate was NOT called (no changes to report)
verify(connection.getCommandSender(), timeout(500L).times(0))
- .sendWatchTopicListUpdate(eq(13L), any(), any(), any(), any());
+ .sendWatchTopicListUpdate(eq(13L), any(), any(), any(), any(), any());
}
@Test
@@ -734,4 +768,399 @@ public class TopicListServiceTest {
Awaitility.await().untilAsserted(() ->
assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(initialTopics));
}
+
+ private TopicListService.TopicListWatcher createWatcher(List<String> topics) throws Exception {
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ 7,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+ topicListFuture.complete(topics);
+ TopicListService.TopicListWatcher watcher =
+ topicListService.getWatcherFuture(13).get(2, TimeUnit.SECONDS);
+ Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+ return watcher;
+ }
+
+ @Test
+ public void testDuplicateWatchRequestCompletesWhenWatcherIsClosedDuringUpdate() throws Exception {
+ TopicListService.TopicListWatcher watcher =
+ createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+ // start an update that doesn't complete
+ topicListFuture = new CompletableFuture<>();
+ assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+ topicListService.updateTopicListWatcher(watcher);
+
+ // a duplicate watch request waits for the ongoing update
+ lookupSemaphore.acquire();
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ 8,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+ assertThat(lookupSemaphore.availablePermits()).isZero();
+
+ // closing the watcher completes the waiting request with an error and releases its lookup permit
+ topicListService.deleteTopicListWatcher(13L);
+ Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+ verify(pulsarCommandSender).sendErrorResponse(eq(8L), any(ServerError.class), anyString());
+ verify(pulsarCommandSender, never())
+ .sendWatchTopicListSuccess(eq(8L), anyLong(), anyString(), any(), any(), any());
+ }
+
+ @Test
+ public void testDuplicateWatchRequestGetsNonRetriableErrorWhenTopicListIsLargerThanMemoryLimit()
+ throws Exception {
+ TopicListService.TopicListWatcher watcher =
+ createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+ // start an update that doesn't complete yet
+ topicListFuture = new CompletableFuture<>();
+ assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+ topicListService.updateTopicListWatcher(watcher);
+
+ // a duplicate watch request waits for the ongoing update
+ lookupSemaphore.acquire();
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ 8,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+
+ // the estimated size of the topic list is larger than the 1_000_000 bytes heap memory limit
+ topicListFuture.complete(List.of("persistent://tenant/ns/topic1" + "x".repeat(1_000_000)));
+
+ // the waiting request gets the non-retriable error of the update, not the error of closing the watcher
+ verify(pulsarCommandSender, timeout(2000L))
+ .sendErrorResponse(eq(8L), eq(ServerError.NotAllowedError), anyString());
+ Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+ assertThat(topicListService.getWatcherFuture(13)).isNull();
+ }
+
+ @Test
+ public void testFailedWatcherUpdateIsRetried() throws Exception {
+ TopicListService.TopicListWatcher watcher =
+ createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+ List<String> refreshedTopics = List.of("persistent://tenant/ns/topic1", "persistent://tenant/ns/topic2");
+ AtomicInteger listingCalls = new AtomicInteger();
+ doAnswer(invocationOnMock -> listingCalls.incrementAndGet() == 1
+ ? CompletableFuture.failedFuture(new PulsarServerException("Metadata error"))
+ : CompletableFuture.completedFuture(refreshedTopics))
+ .when(namespaceService).getListOfUserTopics(any(), eq(CommandGetTopicsOfNamespace.Mode.PERSISTENT));
+
+ assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+ assertThat(topicListService.updateTopicListWatcher(watcher)).succeedsWithin(Duration.ofSeconds(10));
+
+ assertThat(listingCalls).hasValue(2);
+ assertThat(watcher.isUpdatingTopics()).isFalse();
+ assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(refreshedTopics);
+ }
+
+ @Test
+ public void testWatcherIsClosedWhenTopicListIsLargerThanMemoryLimit() throws Exception {
+ TopicListService.TopicListWatcher watcher =
+ createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+ // the estimated size of the topic list is larger than the 1_000_000 bytes heap memory limit
+ List<String> hugeTopicList = List.of("persistent://tenant/ns/topic1" + "x".repeat(1_000_000));
+ doReturn(CompletableFuture.completedFuture(hugeTopicList))
+ .when(namespaceService).getListOfUserTopics(any(), eq(CommandGetTopicsOfNamespace.Mode.PERSISTENT));
+
+ assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+ Throwable failure = topicListService.updateTopicListWatcher(watcher)
+ .handle((__, t) -> t).get(10, TimeUnit.SECONDS);
+ assertThat(FutureUtil.unwrapCompletionException(failure))
+ .isInstanceOf(AsyncSemaphore.PermitAcquireLimitExceededException.class);
+
+ assertThat(topicListService.getWatcherFuture(13)).isNull();
+ verify(topicResources).deregisterPersistentTopicListener(watcher);
+ assertThat(memoryLimiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY).getAcquiredPermits())
+ .isZero();
+ }
+
+ @Test
+ public void testWatchSendsAreCancelledWhenWatcherIsClosed() throws Exception {
+ AtomicReference<BooleanSupplier> successCancelledRef = new AtomicReference<>();
+ doAnswer(invocationOnMock -> {
+ successCancelledRef.set(invocationOnMock.getArgument(5));
+ return CompletableFuture.completedFuture(null);
+ }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+ any());
+ AtomicReference<BooleanSupplier> updateCancelledRef = new AtomicReference<>();
+ doAnswer(invocationOnMock -> {
+ updateCancelledRef.set(invocationOnMock.getArgument(5));
+ return CompletableFuture.completedFuture(null);
+ }).when(pulsarCommandSender).sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
+ createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+ notificationConsumer.accept(
+ new Notification(NotificationType.Created, "/managed-ledgers/tenant/ns/persistent/topic2"));
+ Awaitility.await().untilAsserted(() -> assertThat(updateCancelledRef.get()).isNotNull());
+ assertThat(successCancelledRef.get().getAsBoolean()).isFalse();
+ assertThat(updateCancelledRef.get().getAsBoolean()).isFalse();
+
+ topicListService.deleteTopicListWatcher(13L);
+
+ // sends that are still waiting for direct memory permits are cancelled
+ assertThat(successCancelledRef.get().getAsBoolean()).isTrue();
+ assertThat(updateCancelledRef.get().getAsBoolean()).isTrue();
+ }
+
+ @Test
+ public void testWatcherIsDeletedWhenSendingSuccessResponseFails() throws Exception {
+ doReturn(CompletableFuture.failedFuture(new RuntimeException("Sending failed")))
+ .when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+ any());
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ 7,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+ topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+ Awaitility.await().untilAsserted(() -> assertThat(topicListService.getWatcherFuture(13)).isNull());
+ verify(topicResources).deregisterPersistentTopicListener(any(TopicListService.TopicListWatcher.class));
+ // the request gets an error response instead of waiting for a response that never comes
+ verify(pulsarCommandSender).sendErrorResponse(eq(7L), any(ServerError.class), eq("Sending failed"));
+ assertThat(lookupSemaphore.availablePermits()).isEqualTo(1);
+ }
+
+ @Test
+ public void testWatchRequestGetsErrorWhenSuccessResponseIsLargerThanDirectMemoryLimit() throws Exception {
+ doAnswer(invocationOnMock -> {
+ Function<Throwable, CompletableFuture<Void>> permitAcquireErrorHandler = invocationOnMock.getArgument(4);
+ return permitAcquireErrorHandler.apply(new AsyncSemaphore.PermitAcquireLimitExceededException(
+ "Requested permits are larger than the limit", new IllegalArgumentException()));
+ }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+ any());
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ 7,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+ topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+ Awaitility.await().untilAsserted(() -> assertThat(topicListService.getWatcherFuture(13)).isNull());
+ verify(pulsarCommandSender, timeout(2000L))
+ .sendErrorResponse(eq(7L), eq(ServerError.NotAllowedError), anyString());
+ // the success response isn't retried
+ verify(pulsarCommandSender, times(1))
+ .sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(), any());
+ assertThat(lookupSemaphore.availablePermits()).isEqualTo(1);
+ }
+
+ @Test
+ public void testWatchRequestGetsErrorWhenWatcherIsClosedBeforeSuccessResponseIsSent() throws Exception {
+ doAnswer(invocationOnMock -> {
+ // the watcher is closed while the success response waits for direct memory permits, the permit
+ // request is then cancelled
+ topicListService.deleteTopicListWatcher(13L);
+ BooleanSupplier isCancelled = invocationOnMock.getArgument(5);
+ assertThat(isCancelled.getAsBoolean()).isTrue();
+ Function<Throwable, CompletableFuture<Void>> permitAcquireErrorHandler = invocationOnMock.getArgument(4);
+ return permitAcquireErrorHandler.apply(
+ new AsyncSemaphore.PermitAcquireCancelledException("Permit acquisition was cancelled"));
+ }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+ any());
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ 7,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+ topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+ // the connection is still active, so the request gets an error response instead of no response
+ verify(pulsarCommandSender, timeout(2000L)).sendErrorResponse(eq(7L), any(ServerError.class), anyString());
+ Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+ }
+
+ @Test
+ public void testDuplicateWatchRequestGetsErrorWhenUpdateFails() throws Exception {
+ TopicListService.TopicListWatcher watcher =
+ createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+ // the update keeps failing until the listing is allowed to succeed
+ AtomicBoolean listingFails = new AtomicBoolean(true);
+ List<String> refreshedTopics = List.of("persistent://tenant/ns/topic1", "persistent://tenant/ns/topic2");
+ doAnswer(invocationOnMock -> listingFails.get()
+ ? CompletableFuture.failedFuture(new PulsarServerException("Metadata error"))
+ : CompletableFuture.completedFuture(refreshedTopics))
+ .when(namespaceService).getListOfUserTopics(any(), eq(CommandGetTopicsOfNamespace.Mode.PERSISTENT));
+
+ // a duplicate watch request triggers an update and waits for it
+ lookupSemaphore.acquire();
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ 8,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+
+ // the request gets an error response after the first failure instead of waiting for the retries
+ verify(pulsarCommandSender, timeout(5000L)).sendErrorResponse(eq(8L), any(ServerError.class), anyString());
+ Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+ assertThat(watcher.isUpdatingTopics()).isTrue();
+
+ // the update keeps being retried and completes once the listing succeeds
+ listingFails.set(false);
+ Awaitility.await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> {
+ assertThat(watcher.isUpdatingTopics()).isFalse();
+ assertThat(watcher.getMatchingTopics()).containsExactlyInAnyOrderElementsOf(refreshedTopics);
+ });
+ }
+
+ @Test
+ public void testWatchRequestGetsErrorWhenWatcherIsRemovedBeforeFirstSendAttempt() throws Exception {
+ createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+ // the update triggered by the duplicate watch requests waits for the listing
+ topicListFuture = new CompletableFuture<>();
+ doAnswer(invocationOnMock -> {
+ long requestId = invocationOnMock.getArgument(0);
+ if (requestId == 8L) {
+ // the watcher is closed while the next waiting request is about to send its success response
+ topicListService.deleteTopicListWatcher(13L);
+ }
+ return CompletableFuture.completedFuture(null);
+ }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+ any());
+ lookupSemaphore.release(2);
+ for (long requestId = 8; requestId <= 9; requestId++) {
+ lookupSemaphore.acquire();
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ requestId,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+ }
+
+ topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+ // request 9 finds the watcher removed before its first send attempt and gets an error response
+ verify(pulsarCommandSender, timeout(2000L)).sendErrorResponse(eq(9L), any(ServerError.class), anyString());
+ verify(pulsarCommandSender, never())
+ .sendWatchTopicListSuccess(eq(9L), anyLong(), anyString(), any(), any(), any());
+ Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(3));
+ }
+
+ @Test
+ public void testWaitingWatchRequestRespondsWithSnapshotOfItsUpdate() throws Exception {
+ List<String> topics = Collections.singletonList("persistent://tenant/ns/topic1");
+ TopicListService.TopicListWatcher watcher = createWatcher(topics);
+ // the update triggered by the duplicate watch requests waits for the listing
+ topicListFuture = new CompletableFuture<>();
+ doAnswer(invocationOnMock -> {
+ long requestId = invocationOnMock.getArgument(0);
+ if (requestId == 8L) {
+ // the metadata store connection is lost after the update has completed, which clears the matching
+ // topics until the next update, while the next waiting request is about to respond
+ watcher.onSessionEvent(SessionEvent.ConnectionLost);
+ }
+ return CompletableFuture.completedFuture(null);
+ }).when(pulsarCommandSender).sendWatchTopicListSuccess(anyLong(), anyLong(), anyString(), any(), any(),
+ any());
+ lookupSemaphore.release(2);
+ for (long requestId = 8; requestId <= 9; requestId++) {
+ lookupSemaphore.acquire();
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ requestId,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+ }
+
+ topicListFuture.complete(topics);
+
+ // request 9 responds with the matching topics of the completed update, not with the cleared list
+ verify(pulsarCommandSender, timeout(2000L)).sendWatchTopicListSuccess(eq(9L), eq(13L),
+ eq(TopicList.calculateHash(topics)), eq(new ArrayList<>(topics)), any(), any());
+ }
+
+ @Test
+ public void testFailedWatchRequestClosesTheWatcherItCreated() throws Exception {
+ AtomicReference<TopicListService.TopicListWatcher> createdWatcher = new AtomicReference<>();
+ doAnswer(invocationOnMock -> {
+ invocationOnMock.callRealMethod();
+ TopicListService.TopicListWatcher watcher = invocationOnMock.getArgument(0);
+ createdWatcher.set(watcher);
+ // the metadata store connection is lost while the watcher is being created, which starts an update
+ watcher.onSessionEvent(SessionEvent.ConnectionLost);
+ return null;
+ }).when(topicResources).registerPersistentTopicListener(any(TopicListener.class));
+ topicListService.handleWatchTopicList(
+ NamespaceName.get("tenant/ns"),
+ 13,
+ 7,
+ "persistent://tenant/ns/topic\\d",
+ topicsPatternImplementation, null,
+ lookupSemaphore);
+ topicListFuture.complete(Collections.singletonList("persistent://tenant/ns/topic1"));
+ // the permits are granted and the watcher is created asynchronously
+ Awaitility.await().untilAsserted(() -> assertThat(createdWatcher.get()).isNotNull());
+ // the watch request waits for the update since the matching topics are incomplete
+ Awaitility.await().untilAsserted(() -> assertThat(topicListService.getWatcherFuture(13)).isCompleted());
+ assertThat(lookupSemaphore.availablePermits()).isZero();
+
+ // the update fails, which fails the waiting watch request
+ doReturn(CompletableFuture.failedFuture(new PulsarServerException("Metadata error")))
+ .when(namespaceService).getListOfUserTopics(any(), eq(CommandGetTopicsOfNamespace.Mode.PERSISTENT));
+ topicListService.updateTopicListWatcher(createdWatcher.get());
+
+ // the request gets an error response, and the watcher that it created is removed, deregistered and closed
+ verify(pulsarCommandSender, timeout(2000L)).sendErrorResponse(eq(7L), any(ServerError.class), anyString());
+ Awaitility.await().untilAsserted(() -> assertThat(lookupSemaphore.availablePermits()).isEqualTo(1));
+ assertThat(topicListService.getWatcherFuture(13)).isNull();
+ verify(topicResources, timeout(2000L)).deregisterPersistentTopicListener(createdWatcher.get());
+ }
+
+ @Test
+ public void testTopicChangesAreSentAfterPendingSuccessResponse() throws Exception {
+ TopicListService.TopicListWatcher watcher =
+ createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+ // the success response of a watch request that waited for an update is being sent
+ watcher.successResponseSendingStarted();
+ assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+ // completing the update doesn't send the queued topic changes before the success response
+ watcher.updateTopics(Collections.singletonList("persistent://tenant/ns/topic1"));
+
+ notificationConsumer.accept(
+ new Notification(NotificationType.Created, "/managed-ledgers/tenant/ns/persistent/topic2"));
+ Thread.sleep(500);
+ verify(pulsarCommandSender, never())
+ .sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
+
+ // the topic change is sent after the success response has been sent
+ watcher.successResponseSendingCompleted();
+ verify(pulsarCommandSender, timeout(2000L))
+ .sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
+ }
+
+ @Test
+ public void testTopicChangesAreQueuedWhileUpdateIsInProgress() throws Exception {
+ TopicListService.TopicListWatcher watcher =
+ createWatcher(Collections.singletonList("persistent://tenant/ns/topic1"));
+ // an update starts, for example after a metadata store session event
+ assertThat(watcher.prepareUpdateTopics(null)).isTrue();
+ // a send that was in progress before the update completes
+ watcher.sendingCompleted();
+
+ // a topic change during the update isn't sent, since the matching topics are being rebuilt
+ notificationConsumer.accept(
+ new Notification(NotificationType.Created, "/managed-ledgers/tenant/ns/persistent/topic2"));
+ Thread.sleep(500);
+ verify(pulsarCommandSender, never())
+ .sendWatchTopicListUpdate(anyLong(), any(), any(), anyString(), any(), any());
+ }
}
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiter.java b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiter.java
index 59be8db89239..444ddb199454 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiter.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiter.java
@@ -70,8 +70,13 @@ public interface AsyncDualMemoryLimiter {
void release(AsyncDualMemoryLimiterPermit permit);
/**
* Execute the specified function with acquired permits and release the permits after the returned future completes.
+ * When the permits can't be acquired, including when the memory size is larger than the limit
+ * ({@link AsyncSemaphore.PermitAcquireLimitExceededException}), permitAcquireErrorHandler is called instead.
* @param memorySize memory size to acquire permits for
* @param limitType memory limit type to acquire permits for
+ * @param isCancelled supplier that returns true if the request has been cancelled. When it returns true after
+ * the permits have been acquired, the function isn't executed, the permits are released and
+ * permitAcquireErrorHandler is called with AsyncSemaphore.PermitAcquireCancelledException
* @param function function to execute with acquired permits
* @return result of the function
* @param <T> type of the CompletableFuture returned by the function
@@ -82,14 +87,18 @@ public interface AsyncDualMemoryLimiter {
CompletableFuture<T>> function,
Function<Throwable, CompletableFuture<T>>
permitAcquireErrorHandler) {
- return AsyncDualMemoryLimiterUtil.withPermitsFuture(acquire(memorySize, limitType, isCancelled), function,
- permitAcquireErrorHandler, this::release);
+ return AsyncDualMemoryLimiterUtil.withPermitsFuture(
+ AsyncDualMemoryLimiterUtil.requestPermits(() -> acquire(memorySize, limitType, isCancelled)),
+ isCancelled, function, permitAcquireErrorHandler, this::release);
}
/**
* Executed the specified function with updated permits and release the permits after the returned future completes.
* @param initialPermit initial permit to update
* @param newMemorySize new memory size to update to
+ * @param isCancelled supplier that returns true if the request has been cancelled. When it returns true after
+ * the permits have been updated, the function isn't executed, the permits are released and
+ * permitAcquireErrorHandler is called with AsyncSemaphore.PermitAcquireCancelledException
* @param function function to execute with updated permits
* @return result of the function
* @param <T> type of the CompletableFuture returned by the function
@@ -100,8 +109,9 @@ public interface AsyncDualMemoryLimiter {
CompletableFuture<T>> function,
Function<Throwable, CompletableFuture<T>>
permitAcquireErrorHandler) {
- return AsyncDualMemoryLimiterUtil.withPermitsFuture(update(initialPermit, newMemorySize, isCancelled), function,
- permitAcquireErrorHandler, this::release);
+ return AsyncDualMemoryLimiterUtil.withPermitsFuture(
+ AsyncDualMemoryLimiterUtil.requestPermits(() -> update(initialPermit, newMemorySize, isCancelled)),
+ isCancelled, function, permitAcquireErrorHandler, this::release);
}
/**
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtil.java
index 9f3880a2c09b..e7233de492de 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtil.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtil.java
@@ -24,11 +24,13 @@ import java.util.concurrent.CompletableFuture;
import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Function;
+import java.util.function.Supplier;
import lombok.experimental.UtilityClass;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.common.api.proto.BaseCommand;
import org.apache.pulsar.common.protocol.Commands;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
+import org.apache.pulsar.common.util.FutureUtil;
@UtilityClass
public class AsyncDualMemoryLimiterUtil {
@@ -52,6 +54,33 @@ public class AsyncDualMemoryLimiterUtil {
Function<Throwable, CompletableFuture<T>>
permitAcquireErrorHandler,
Consumer<AsyncDualMemoryLimiterPermit> releaser) {
+ return withPermitsFuture(permitsFuture, () -> false, function, permitAcquireErrorHandler, releaser);
+ }
+
+ /**
+ * Execute a function with acquired permits and ensure permits are released after completion.
+ * This method handles the lifecycle of permits - acquisition, usage, and release, including error cases.
+ * If the request has been cancelled by the time the permits are acquired, the permits are released
+ * without executing the function, and the permit acquire error handler is called with
+ * {@link AsyncSemaphore.PermitAcquireCancelledException}.
+ *
+ * @param permitsFuture Future that will complete with the required permits
+ * @param isCancelled supplier that returns true if the request has been cancelled
+ * @param function Function to execute once permits are acquired that returns a CompletableFuture
+ * @param permitAcquireErrorHandler Handler for permit acquisition errors that returns a CompletableFuture
+ * @param releaser Consumer that handles releasing the permits
+ * @param <T> The type of result returned by the function
+ * @return CompletableFuture that completes with the result of the function execution
+ */
+ public static <T> CompletableFuture<T> withPermitsFuture(
+ CompletableFuture<AsyncDualMemoryLimiterPermit>
+ permitsFuture,
+ BooleanSupplier isCancelled,
+ Function<AsyncDualMemoryLimiterPermit,
+ CompletableFuture<T>> function,
+ Function<Throwable, CompletableFuture<T>>
+ permitAcquireErrorHandler,
+ Consumer<AsyncDualMemoryLimiterPermit> releaser) {
return permitsFuture
// combine the permits and error into a single pair so that it can be used in thenCompose
.handle((permits, permitAcquireError) ->
@@ -63,6 +92,11 @@ public class AsyncDualMemoryLimiterUtil {
} else {
// permits were acquired
AsyncDualMemoryLimiterPermit permits = permitsAndError.getLeft();
+ if (isCancelled.getAsBoolean()) {
+ // the request was cancelled while waiting for the permits, skip the operation
+ releaser.accept(permits);
+ return permitAcquireErrorHandler.apply(newCancelledException());
+ }
try {
return function.apply(permits)
.whenComplete((__, ___) ->
@@ -83,7 +117,10 @@ public class AsyncDualMemoryLimiterUtil {
*
* @param ctx the channel handler context used for writing the response
* @param dualMemoryLimiter the memory limiter used to acquire and release memory permits
- * @param isCancelled supplier that indicates if the permit acquisition should be cancelled
+ * @param isCancelled supplier that indicates if the permit acquisition should be cancelled. When it returns true
+ * after the permits have been acquired, the command isn't serialized or written, the permits
+ * are released and permitAcquireErrorHandler is called with
+ * AsyncSemaphore.PermitAcquireCancelledException
* @param command the base command to serialize and write to the channel
* @param permitAcquireErrorHandler handler for errors that occur during permit acquisition
* @return a future that completes when the command has been written to the channel's outbound buffer
@@ -100,11 +137,17 @@ public class AsyncDualMemoryLimiterUtil {
// Calculate serialized size before acquiring permits
int serializedSize = command.getSerializedSize();
// Acquire permits
- return dualMemoryLimiter.acquire(serializedSize, AsyncDualMemoryLimiter.LimitType.DIRECT_MEMORY, isCancelled)
+ return requestPermits(() -> dualMemoryLimiter.acquire(serializedSize,
+ AsyncDualMemoryLimiter.LimitType.DIRECT_MEMORY, isCancelled))
.handle((permits, t) -> {
if (t != null) {
return permitAcquireErrorHandler.apply(t);
}
+ if (isCancelled.getAsBoolean()) {
+ // the request was cancelled while waiting for the permits, skip serializing and writing
+ dualMemoryLimiter.release(permits);
+ return permitAcquireErrorHandler.apply(newCancelledException());
+ }
try {
// Serialize the response
ByteBuf outBuf = Commands.serializeWithPrecalculatedSerializedSize(command, serializedSize);
@@ -121,4 +164,50 @@ public class AsyncDualMemoryLimiterUtil {
return CompletableFuture.<Void>completedFuture(null);
}).thenCompose(Function.identity());
}
+
+ /**
+ * Requests permits so that all failures are reported through the returned future. The limiter rejects a request
+ * that is larger than the limit by throwing {@link IllegalArgumentException}, which is converted to a future
+ * that completes exceptionally with {@link AsyncSemaphore.PermitAcquireLimitExceededException}. Callers that
+ * handle failures only through the future would otherwise never complete the request.
+ *
+ * @param permitRequest supplier that requests the permits from the limiter
+ * @return future that completes with the permits or exceptionally when the permits can't be acquired
+ */
+ public static CompletableFuture<AsyncDualMemoryLimiterPermit> requestPermits(
+ Supplier<CompletableFuture<AsyncDualMemoryLimiterPermit>> permitRequest) {
+ try {
+ return permitRequest.get();
+ } catch (IllegalArgumentException e) {
+ return CompletableFuture.failedFuture(
+ new AsyncSemaphore.PermitAcquireLimitExceededException(e.getMessage(), e));
+ }
+ }
+
+ /**
+ * Checks whether the permit request failed because the request was cancelled.
+ *
+ * @param throwable the failure, possibly wrapped in a {@link java.util.concurrent.CompletionException}
+ * @return true if the request was cancelled
+ */
+ public static boolean isPermitAcquireCancelled(Throwable throwable) {
+ return FutureUtil.unwrapCompletionException(throwable)
+ instanceof AsyncSemaphore.PermitAcquireCancelledException;
+ }
+
+ /**
+ * Checks whether the permit request failed because more permits were requested than the limit. Unlike a permit
+ * timeout or a full queue, retrying such a request fails the same way.
+ *
+ * @param throwable the failure, possibly wrapped in a {@link java.util.concurrent.CompletionException}
+ * @return true if the request was larger than the limit
+ */
+ public static boolean isPermitAcquireLimitExceeded(Throwable throwable) {
+ return FutureUtil.unwrapCompletionException(throwable)
+ instanceof AsyncSemaphore.PermitAcquireLimitExceededException;
+ }
+
+ private static AsyncSemaphore.PermitAcquireCancelledException newCancelledException() {
+ return new AsyncSemaphore.PermitAcquireCancelledException("Request was cancelled before permits were used");
+ }
}
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncSemaphore.java b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncSemaphore.java
index 99479d9b3e54..dd15be67cdb3 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncSemaphore.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/semaphore/AsyncSemaphore.java
@@ -84,6 +84,10 @@ public interface AsyncSemaphore {
public PermitAcquireException(String message) {
super(message);
}
+
+ public PermitAcquireException(String message, Throwable cause) {
+ super(message, cause);
+ }
}
/**
@@ -119,6 +123,18 @@ public interface AsyncSemaphore {
}
}
+ /**
+ * Exception used when the requested permits are larger than the maximum number of permits, so the request
+ * can never be granted.
+ */
+ class PermitAcquireLimitExceededException extends PermitAcquireException {
+ private static final long serialVersionUID = 1L;
+
+ public PermitAcquireLimitExceededException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ }
+
/**
* Exception thrown when permit acquisition is cancelled.
*/
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtilTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtilTest.java
index cf202397cb77..a5f78e3eec8c 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtilTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/semaphore/AsyncDualMemoryLimiterUtilTest.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.common.semaphore;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -36,6 +37,7 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.util.concurrent.GenericFutureListener;
import java.time.Duration;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -45,6 +47,7 @@ import org.apache.pulsar.common.api.proto.BaseCommand;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.AsyncDualMemoryLimiterPermit;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter.LimitType;
import org.apache.pulsar.common.semaphore.AsyncSemaphore.PermitAcquireCancelledException;
+import org.apache.pulsar.common.semaphore.AsyncSemaphore.PermitAcquireLimitExceededException;
import org.awaitility.Awaitility;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
@@ -600,6 +603,163 @@ public class AsyncDualMemoryLimiterUtilTest {
assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAvailablePermits()).isEqualTo(10000);
}
+ @Test
+ public void testWithAcquiredPermitsSkipsFunctionWhenCancelledAfterPermitsAreGranted() throws Exception {
+ // the request gets cancelled after the permits have been granted, but before the function is executed
+ AtomicBoolean cancelled = new AtomicBoolean(false);
+ CompletableFuture<AsyncDualMemoryLimiterPermit> permitFuture =
+ limiter.acquire(100, LimitType.HEAP_MEMORY, cancelled::get);
+ assertThat(permitFuture).succeedsWithin(Duration.ofSeconds(1));
+ cancelled.set(true);
+
+ AtomicBoolean functionExecuted = new AtomicBoolean(false);
+ AtomicReference<Throwable> capturedError = new AtomicReference<>();
+ CompletableFuture<String> result = AsyncDualMemoryLimiterUtil.withPermitsFuture(
+ permitFuture,
+ cancelled::get,
+ permit -> {
+ functionExecuted.set(true);
+ return CompletableFuture.completedFuture("success");
+ },
+ throwable -> {
+ capturedError.set(throwable);
+ return CompletableFuture.completedFuture("cancelled");
+ },
+ limiter::release
+ );
+
+ assertThat(result).succeedsWithin(Duration.ofSeconds(1)).isEqualTo("cancelled");
+ assertThat(functionExecuted.get()).isFalse();
+ assertThat(capturedError.get()).isInstanceOf(PermitAcquireCancelledException.class);
+ assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isEqualTo(0);
+ }
+
+ @Test
+ public void testWithUpdatedPermitsSkipsFunctionWhenCancelledAfterPermitsAreGranted() throws Exception {
+ AtomicBoolean cancelled = new AtomicBoolean(false);
+ AtomicBoolean innerFunctionExecuted = new AtomicBoolean(false);
+ AtomicReference<Throwable> capturedError = new AtomicReference<>();
+ CompletableFuture<String> result = limiter.withAcquiredPermits(100, LimitType.HEAP_MEMORY, cancelled::get,
+ initialPermit -> {
+ // the request is cancelled while the listing is in progress
+ cancelled.set(true);
+ return limiter.withUpdatedPermits(initialPermit, 50, cancelled::get, updatedPermit -> {
+ innerFunctionExecuted.set(true);
+ return CompletableFuture.completedFuture("success");
+ }, throwable -> {
+ capturedError.set(throwable);
+ return CompletableFuture.completedFuture("cancelled");
+ });
+ }, CompletableFuture::failedFuture);
+
+ assertThat(result).succeedsWithin(Duration.ofSeconds(1)).isEqualTo("cancelled");
+ assertThat(innerFunctionExecuted.get()).isFalse();
+ assertThat(capturedError.get()).isInstanceOf(PermitAcquireCancelledException.class);
+ assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isEqualTo(0);
+ }
+
+ @Test
+ public void testAcquireDirectMemoryPermitsAndWriteAndFlushSkipsWriteWhenCancelledAfterPermitsAreGranted()
+ throws Exception {
+ // simulate a permit grant that races with the cancellation of the request
+ AsyncDualMemoryLimiter dualMemoryLimiter = mock(AsyncDualMemoryLimiter.class);
+ AsyncDualMemoryLimiterPermit permit = mock(AsyncDualMemoryLimiterPermit.class);
+ when(dualMemoryLimiter.acquire(anyLong(), any(), any())).thenReturn(CompletableFuture.completedFuture(permit));
+ ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+ AtomicReference<Throwable> capturedError = new AtomicReference<>();
+
+ CompletableFuture<Void> result = AsyncDualMemoryLimiterUtil.acquireDirectMemoryPermitsAndWriteAndFlush(
+ ctx,
+ dualMemoryLimiter,
+ () -> true,
+ createTestCommand(),
+ throwable -> {
+ capturedError.set(throwable);
+ return CompletableFuture.completedFuture(null);
+ }
+ );
+
+ assertThat(result).succeedsWithin(Duration.ofSeconds(1));
+ verify(ctx, never()).writeAndFlush(any());
+ verify(dualMemoryLimiter, times(1)).release(permit);
+ assertThat(capturedError.get()).isInstanceOf(PermitAcquireCancelledException.class);
+ }
+
+ @Test
+ public void testWithAcquiredPermitsCallsErrorHandlerWhenRequestIsLargerThanLimit() {
+ AtomicBoolean functionExecuted = new AtomicBoolean(false);
+ AtomicReference<Throwable> capturedError = new AtomicReference<>();
+
+ // the limiter rejects the request synchronously, the failure is passed to the error handler
+ CompletableFuture<String> result = limiter.withAcquiredPermits(10001, LimitType.HEAP_MEMORY, () -> false,
+ permit -> {
+ functionExecuted.set(true);
+ return CompletableFuture.completedFuture("success");
+ }, throwable -> {
+ capturedError.set(throwable);
+ return CompletableFuture.completedFuture("too large");
+ });
+
+ assertThat(result).succeedsWithin(Duration.ofSeconds(1)).isEqualTo("too large");
+ assertThat(functionExecuted.get()).isFalse();
+ assertThat(capturedError.get()).isInstanceOf(PermitAcquireLimitExceededException.class);
+ assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isEqualTo(0);
+ }
+
+ @Test
+ public void testWithUpdatedPermitsCallsErrorHandlerWhenRequestIsLargerThanLimit() {
+ AtomicBoolean innerFunctionExecuted = new AtomicBoolean(false);
+ AtomicReference<Throwable> capturedError = new AtomicReference<>();
+
+ CompletableFuture<String> result = limiter.withAcquiredPermits(100, LimitType.HEAP_MEMORY, () -> false,
+ initialPermit -> limiter.withUpdatedPermits(initialPermit, 10001, () -> false, updatedPermit -> {
+ innerFunctionExecuted.set(true);
+ return CompletableFuture.completedFuture("success");
+ }, throwable -> {
+ capturedError.set(throwable);
+ return CompletableFuture.completedFuture("too large");
+ }), CompletableFuture::failedFuture);
+
+ assertThat(result).succeedsWithin(Duration.ofSeconds(1)).isEqualTo("too large");
+ assertThat(innerFunctionExecuted.get()).isFalse();
+ assertThat(capturedError.get()).isInstanceOf(PermitAcquireLimitExceededException.class);
+ // the initial permits are released after the outer function completes
+ assertThat(limiter.getLimiter(LimitType.HEAP_MEMORY).getAcquiredPermits()).isEqualTo(0);
+ }
+
+ @Test
+ public void testAcquireDirectMemoryPermitsAndWriteAndFlushCallsErrorHandlerWhenCommandIsLargerThanLimit() {
+ limiter.close();
+ // a direct memory limit smaller than the serialized command
+ limiter = new AsyncDualMemoryLimiterImpl(10000, 100, 5000, 1, 100, 5000);
+ ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+ AtomicReference<Throwable> capturedError = new AtomicReference<>();
+
+ CompletableFuture<Void> result = AsyncDualMemoryLimiterUtil.acquireDirectMemoryPermitsAndWriteAndFlush(
+ ctx, limiter, () -> false, createTestCommand(), throwable -> {
+ capturedError.set(throwable);
+ return CompletableFuture.completedFuture(null);
+ });
+
+ assertThat(result).succeedsWithin(Duration.ofSeconds(1));
+ verify(ctx, never()).writeAndFlush(any());
+ assertThat(capturedError.get()).isInstanceOf(PermitAcquireLimitExceededException.class);
+ }
+
+ @Test
+ public void testIsPermitAcquireLimitExceeded() {
+ PermitAcquireLimitExceededException limitExceeded =
+ new PermitAcquireLimitExceededException("larger than the limit", new IllegalArgumentException());
+ assertThat(AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(limitExceeded)).isTrue();
+ assertThat(AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(new CompletionException(limitExceeded)))
+ .isTrue();
+ // a permit timeout or a full queue can succeed when retried
+ assertThat(AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(
+ new AsyncSemaphore.PermitAcquireTimeoutException("timeout"))).isFalse();
+ assertThat(AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(
+ new AsyncSemaphore.PermitAcquireQueueFullException("queue full"))).isFalse();
+ }
+
private BaseCommand createTestCommand() {
BaseCommand command = new BaseCommand().setType(BaseCommand.Type.PING);
command.setPing();
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
index 9d07c5fd203b..75dd59e86210 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.proxy.server;
import static org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil.acquireDirectMemoryPermitsAndWriteAndFlush;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
+import io.netty.util.ReferenceCountUtil;
import io.prometheus.client.Counter;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
@@ -55,6 +56,8 @@ import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion;
import org.apache.pulsar.common.protocol.schema.SchemaVersion;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterUtil;
+import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.netty.NettyChannelUtil;
import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData;
@@ -374,7 +377,8 @@ public class LookupProxyHandler {
command = Commands.newGetTopicsOfNamespaceRequest(namespaceName, requestId, mode,
topicsPattern, topicsHash, propertiesMap);
- internalPerformGetTopicsOfNamespace(clientRequestId, namespaceName, mode, clientCnx, command, requestId);
+ internalPerformGetTopicsOfNamespace(clientRequestId, namespaceName, mode, propertiesMap, clientCnx,
+ command, requestId);
proxyConnection.getConnectionPool().releaseConnection(clientCnx);
}).exceptionally(ex -> {
// Failed to connect to backend broker
@@ -385,11 +389,12 @@ public class LookupProxyHandler {
}
private void internalPerformGetTopicsOfNamespace(long clientRequestId, String namespaceName,
- CommandGetTopicsOfNamespace.Mode mode, ClientCnx clientCnx,
+ CommandGetTopicsOfNamespace.Mode mode,
+ Map<String, String> properties, ClientCnx clientCnx,
ByteBuf command, long requestId) {
BooleanSupplier isPermitRequestCancelled = () -> !proxyConnection.ctx().channel().isActive();
TopicListSizeResultCache.ResultHolder
- listSizeHolder = topicListSizeResultCache.getTopicListSize(namespaceName, mode);
+ listSizeHolder = topicListSizeResultCache.getTopicListSize(namespaceName, mode, properties);
listSizeHolder.getSizeAsync().thenAccept(initialSize -> {
maxTopicListInFlightLimiter.withAcquiredPermits(initialSize,
AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY, isPermitRequestCancelled, initialPermits -> {
@@ -401,7 +406,8 @@ public class LookupProxyHandler {
.exceptionMessage(t)
.log("Failed to get TopicsOfNamespace");
listSizeHolder.resetIfInitializing();
- writeAndFlush(Commands.newError(clientRequestId, getServerError(t), t.getMessage()));
+ writeAndFlush(Commands.newError(clientRequestId, getTopicListServerError(t),
+ t.getMessage()));
return CompletableFuture.completedFuture(null);
} else {
long actualSize = TopicListMemoryLimiter.estimateTopicListSize(
@@ -412,27 +418,23 @@ public class LookupProxyHandler {
return handleWritingGetTopicsResponse(clientRequestId, r,
isPermitRequestCancelled);
}, t2 -> {
- log.warn()
- .attr("clientAddress", clientAddress)
- .attr("GetTopicsOfNamespace", t2.getMessage())
- .log("Failed to acquire actual heap memory permits for " + "");
- writeAndFlush(
- Commands.newError(clientRequestId, ServerError.TooManyRequests,
- "Failed due to heap memory limit exceeded"));
-
+ if (!isTopicListRequestCancelled(t2,
+ "Failed to acquire actual heap memory permits for "
+ + "GetTopicsOfNamespace")) {
+ writeTopicListPermitsError(clientRequestId, t2, true, "heap");
+ }
return CompletableFuture.completedFuture(null);
});
}
});
}, t -> {
- log.warn()
- .attr("clientAddress", clientAddress)
- .attr("GetTopicsOfNamespace", t.getMessage())
- .log("Failed to acquire initial heap memory permits for");
+ // the command wasn't sent to the broker, release it
+ ReferenceCountUtil.safeRelease(command);
listSizeHolder.resetIfInitializing();
- writeAndFlush(Commands.newError(clientRequestId, ServerError.TooManyRequests,
- "Failed due to heap memory limit exceeded"));
-
+ if (!isTopicListRequestCancelled(t,
+ "Failed to acquire initial heap memory permits for GetTopicsOfNamespace")) {
+ writeTopicListPermitsError(clientRequestId, t, listSizeHolder.isMeasured(), "heap");
+ }
return CompletableFuture.completedFuture(null);
}).exceptionally(ex -> {
listSizeHolder.resetIfInitializing();
@@ -449,16 +451,51 @@ public class LookupProxyHandler {
r.isChanged(), clientRequestId);
return acquireDirectMemoryPermitsAndWriteAndFlush(proxyConnection.ctx(), maxTopicListInFlightLimiter,
isCancelled, responseCommand, t -> {
- log.warn()
- .attr("clientAddress", clientAddress)
- .attr("GetTopicsOfNamespace", t.getMessage())
- .log("Failed to acquire actual direct memory permits for");
- writeAndFlush(Commands.newError(clientRequestId, ServerError.TooManyRequests,
- "Failed due to direct memory limit exceeded"));
+ if (!isTopicListRequestCancelled(t,
+ "Failed to acquire actual direct memory permits for GetTopicsOfNamespace")) {
+ writeTopicListPermitsError(clientRequestId, t, true, "direct");
+ }
return CompletableFuture.completedFuture(null);
});
}
+ /**
+ * Responds to a GetTopicsOfNamespace request that couldn't acquire topic list memory permits. A topic list that
+ * has been measured to be larger than the limit gets a non-retriable error, since a retry fails the same way
+ * until the list is measured again. A permit timeout or a full queue, or an estimate larger than the limit for
+ * a topic list that hasn't been measured, gets {@link ServerError#TooManyRequests}, which the client retries.
+ */
+ private void writeTopicListPermitsError(long clientRequestId, Throwable t, boolean topicListMeasured,
+ String memoryType) {
+ if (topicListMeasured && AsyncDualMemoryLimiterUtil.isPermitAcquireLimitExceeded(t)) {
+ writeAndFlush(Commands.newError(clientRequestId, ServerError.NotAllowedError,
+ "Failed since the topic list is larger than the topic list " + memoryType + " memory limit"));
+ } else {
+ writeAndFlush(Commands.newError(clientRequestId, ServerError.TooManyRequests,
+ "Failed due to " + memoryType + " memory limit exceeded"));
+ }
+ }
+
+ /**
+ * Logs a failure to acquire topic list memory permits. A request that was cancelled, because the client
+ * connection was closed, is logged at debug level, since there is no client to respond to.
+ *
+ * @return true if the request was cancelled and no error response should be sent
+ */
+ private boolean isTopicListRequestCancelled(Throwable t, String failureMessage) {
+ if (AsyncDualMemoryLimiterUtil.isPermitAcquireCancelled(t)) {
+ log.debug()
+ .attr("clientAddress", clientAddress)
+ .log("GetTopicsOfNamespace was cancelled since the client connection was closed");
+ return true;
+ }
+ log.warn()
+ .attr("clientAddress", clientAddress)
+ .exceptionMessage(t)
+ .log(failureMessage);
+ return false;
+ }
+
public void handleGetSchema(CommandGetSchema commandGetSchema) {
GET_SCHEMA_REQUESTS.inc();
log.debug()
@@ -568,6 +605,18 @@ public class LookupProxyHandler {
return responseError;
}
+ /**
+ * Maps a broker's GetTopicsOfNamespace failure to the error relayed to the client. The broker rejects a topic
+ * list that is larger than its topic list memory limit with {@link ServerError#NotAllowedError}, which is passed
+ * through so that the client doesn't retry it.
+ */
+ private ServerError getTopicListServerError(Throwable error) {
+ if (FutureUtil.unwrapCompletionException(error) instanceof PulsarClientException.NotAllowedException) {
+ return ServerError.NotAllowedError;
+ }
+ return getServerError(error);
+ }
+
private void writeAndFlush(ByteBuf cmd) {
final ChannelHandlerContext ctx = proxyConnection.ctx();
NettyChannelUtil.writeAndFlushWithVoidPromise(ctx, cmd);
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
index 84bb62ac79b2..a15f15614880 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
@@ -77,7 +77,9 @@ import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
import org.apache.pulsar.client.impl.tls.ClientTlsFactorySupport;
import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
+import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiter;
import org.apache.pulsar.common.semaphore.AsyncDualMemoryLimiterImpl;
+import org.apache.pulsar.common.semaphore.AsyncSemaphore;
import org.apache.pulsar.common.tls.impl.TlsContextAcquisition;
import org.apache.pulsar.common.tls.impl.TlsSynthesisSpec;
import org.apache.pulsar.common.util.netty.DnsResolverUtil;
@@ -190,7 +192,7 @@ public class ProxyService implements Closeable {
private final AsyncDualMemoryLimiterImpl maxTopicListInFlightLimiter;
@Getter
- private final TopicListSizeResultCache topicListSizeResultCache = new TopicListSizeResultCache();
+ private final TopicListSizeResultCache topicListSizeResultCache;
private boolean gracefulShutdown = true;
@@ -272,6 +274,11 @@ public class ProxyService implements Closeable {
proxyConfig.getMaxTopicListInFlightDirectMemSizeMB() * 1024L * 1024L,
proxyConfig.getMaxTopicListInFlightDirectMemSizePermitsAcquireQueueSize(),
proxyConfig.getMaxTopicListInFlightDirectMemSizePermitsAcquireTimeoutMillis());
+ AsyncSemaphore topicListHeapLimiter =
+ maxTopicListInFlightLimiter.getLimiter(AsyncDualMemoryLimiter.LimitType.HEAP_MEMORY);
+ this.topicListSizeResultCache = new TopicListSizeResultCache(
+ proxyConfig.getMaxTopicListInFlightHeapMemSizeMB() * 1024L * 1024L,
+ () -> topicListHeapLimiter.getQueueSize() == 0 && topicListHeapLimiter.getAcquiredPermits() == 0);
}
public void start() throws Exception {