You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/17 06:12:19 UTC
(pulsar) branch branch-5.0 updated (b4cbff32c39d -> 77a257d4b1fd)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a change to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
from b4cbff32c39d [improve][broker] Reduce the lookups and allocations of an individual ack (#26948)
new 9e5578c7f193 [fix][client] Surface V5 client failures as the typed exception of their error category (#26950)
new 71f852c56561 [fix][broker] Apply dedup disable after recovery (#25982)
new 12defe357161 [fix][misc] Fix Backoff mandatory-stop timer tracking (#25282)
new 6668da0b53e9 [fix][broker] Authorize scalable topic segments with the permissions of the parent topic (#26949)
new 8722988f2d41 [fix][fn] Fix Go function incorrectly using nanoseconds for timestamp (#25869)
new 45bbdbb17136 [fix][test] Split and fix flaky SASL authentication first-stage auth cache tests (#25949)
new 73d53f53968f [fix] [broker] Fix typo in broker.conf (#23295)
new 9a3100cd7c68 [fix][broker] Handle flow_or_qps bundle split requests in ELM (#25442)
new db4ed3fa895c [fix][broker]The partitions of the topic was wrongly created even though this cluster was not allowed to access it (#25443)
new 35227b68bd53 [fix][admin] Encode a space in a path component as %20, not as '+' (#26957)
new d79b840ad08a [improve][broker] Keep each ledger's pending acks in sorted blocks instead of an AVL tree (#26955)
new 5f320772644f [improve][fn] Provide a way to set Pulsar client memory limit in Pulsar Function (#23959)
new ecbf908eb0f9 [fix][proxy] Fix connection and topic stats that stop updating after the initial update (#26961)
new 77a257d4b1fd [improve][ml] Bound the requeues of size-based cache eviction to one cycle through the removal queue (#26933)
The 14 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
conf/broker.conf | 19 +-
conf/functions_worker.yml | 9 +
conf/standalone.conf | 15 +-
.../mledger/ManagedLedgerFactoryConfig.java | 15 +-
.../impl/cache/RangeCacheRemovalCounters.java | 4 +
.../mledger/impl/cache/RangeCacheRemovalQueue.java | 11 +-
.../impl/cache/RangeEntryCacheManagerImpl.java | 3 +-
.../impl/cache/RangeCacheRemovalQueueTest.java | 123 +++++
.../service/PendingAcksLedgerMapBenchmark.java | 373 ++++++++++++++
...k.java => PendingAcksMapOneEntryBenchmark.java} | 39 +-
pip/pip-460.md | 2 +-
pip/pip-466.md | 6 +-
pip/pip-468.md | 5 +-
.../authentication/SaslAuthenticateTest.java | 78 +--
.../apache/pulsar/broker/ServiceConfiguration.java | 17 +-
.../authorization/PulsarAuthorizationProvider.java | 12 +-
.../PulsarAuthorizationProviderTest.java | 87 ++++
.../broker/admin/impl/PersistentTopicsBase.java | 50 +-
.../pulsar/broker/admin/v2/ScalableTopics.java | 188 +++++++-
.../extensions/ExtensibleLoadManagerImpl.java | 68 ++-
.../pulsar/broker/namespace/NamespaceService.java | 11 +-
.../pulsar/broker/service/BrokerService.java | 32 ++
.../broker/service/PendingAcksLedgerMap.java | 535 +++++++++++++++++++++
.../pulsar/broker/service/PendingAcksMap.java | 103 ++--
.../service/persistent/MessageDeduplication.java | 5 +-
.../broker/BrokerMessageDeduplicationTest.java | 84 +++-
.../broker/admin/AdminApiPathEncodingTest.java | 162 +++++++
.../apache/pulsar/broker/admin/AdminApiTest.java | 18 +
.../broker/admin/ScalableTopicPermissionsTest.java | 370 ++++++++++++++
.../broker/admin/ScalableTopicsAuthZTest.java | 52 +-
.../extensions/ExtensibleLoadManagerImplTest.java | 16 +
.../service/OneWayReplicatorUsingGlobalZKTest.java | 33 ++
.../broker/service/PendingAcksLedgerMapTest.java | 501 +++++++++++++++++++
.../pulsar/broker/service/PendingAcksMapTest.java | 179 +++++++
.../pulsar/client/api/v5/V5AuthErrorTest.java | 191 ++++++++
.../client/api/v5/V5ProducerAccessModeTest.java | 20 +-
.../api/v5/V5ScalableTopicAutoCreateTest.java | 6 +
.../client/api/v5/V5SchemaEvolutionTest.java | 17 +-
.../pulsar/client/api/v5/V5TransactionTest.java | 43 ++
.../apache/pulsar/client/admin/ScalableTopics.java | 83 ++++
.../pulsar/common/functions/FunctionConfig.java | 3 +
.../pulsar/common/functions/MemoryLimit.java | 22 +-
.../pulsar/client/admin/internal/BrokersImpl.java | 5 +-
.../client/admin/internal/ScalableTopicsImpl.java | 55 +++
.../pulsar/client/admin/internal/WebTargets.java | 7 +-
.../admin/internal/ScalableTopicsImplTest.java | 50 ++
.../apache/pulsar/admin/cli/CmdFunctionsTest.java | 22 +
.../org/apache/pulsar/admin/cli/CmdFunctions.java | 15 +
.../apache/pulsar/admin/cli/CmdScalableTopics.java | 51 ++
.../pulsar/admin/cli/TestCmdScalableTopics.java | 19 +
.../impl/v5/CheckpointConsumerBuilderV5.java | 11 +-
.../pulsar/client/impl/v5/DagWatchClient.java | 2 +-
.../client/impl/v5/MultiTopicQueueConsumer.java | 2 +-
.../client/impl/v5/MultiTopicStreamConsumer.java | 2 +-
.../pulsar/client/impl/v5/ProducerBuilderV5.java | 12 +-
.../client/impl/v5/PulsarClientBuilderV5.java | 4 +-
.../client/impl/v5/PulsarClientProviderV5.java | 4 +-
.../pulsar/client/impl/v5/PulsarClientV5.java | 13 +-
.../client/impl/v5/QueueConsumerBuilderV5.java | 11 +-
.../client/impl/v5/ScalableCheckpointConsumer.java | 5 +-
.../client/impl/v5/ScalableQueueConsumer.java | 5 +-
.../client/impl/v5/ScalableStreamConsumer.java | 5 +-
.../client/impl/v5/ScalableTopicProducer.java | 52 +-
.../client/impl/v5/ScalableTopicsWatcher.java | 2 +-
.../client/impl/v5/StreamConsumerBuilderV5.java | 11 +-
.../pulsar/client/impl/v5/TransactionV5.java | 10 +-
.../pulsar/client/impl/v5/V5ExceptionMappings.java | 160 ++++++
.../pulsar/client/impl/v5/DagWatchClientTest.java | 26 +
.../client/impl/v5/PulsarClientBuilderV5Test.java | 13 +-
.../client/impl/v5/V5ExceptionMappingsTest.java | 135 ++++++
.../org/apache/pulsar/client/impl/ClientCnx.java | 2 +-
.../org/apache/pulsar/common/util/Backoff.java | 7 +-
.../pulsar/common/util/DirectMemoryUtils.java | 3 +
.../org/apache/pulsar/common/util/BackoffTest.java | 93 ++++
pulsar-function-go/pf/instance.go | 5 +-
pulsar-function-go/pf/stats.go | 6 +-
pulsar-function-go/pf/stats_test.go | 25 +
.../pulsar/functions/instance/InstanceConfig.java | 3 +-
.../org/apache/pulsar/functions/LocalRunner.java | 8 +-
.../proto/src/main/proto/Function.proto | 9 +
.../functions/runtime/JavaInstanceStarter.java | 19 +-
.../pulsar/functions/runtime/RuntimeUtils.java | 11 +
.../runtime/thread/ThreadRuntimeFactory.java | 35 +-
.../pulsar/functions/worker/WorkerConfig.java | 10 +
.../pulsar/functions/runtime/RuntimeUtilsTest.java | 42 ++
.../runtime/thread/ThreadRuntimeFactoryTest.java | 43 +-
.../functions/utils/FunctionConfigUtils.java | 45 ++
.../functions/utils/FunctionConfigUtilsTest.java | 61 +++
.../pulsar/functions/worker/FunctionActioner.java | 4 +
.../functions/worker/FunctionActionerTest.java | 27 ++
.../apache/pulsar/proxy/server/ProxyService.java | 14 +-
.../apache/pulsar/proxy/server/ProxyStatsTest.java | 14 +-
spec/scalable-topics/compatibility.md | 12 +-
spec/scalable-topics/wire-protocol.md | 3 +-
94 files changed, 4462 insertions(+), 358 deletions(-)
create mode 100644 microbench/src/main/java/org/apache/pulsar/broker/service/PendingAcksLedgerMapBenchmark.java
copy microbench/src/main/java/org/apache/pulsar/broker/service/{PendingAckMapCycleBenchmark.java => PendingAcksMapOneEntryBenchmark.java} (63%)
create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksLedgerMap.java
create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiPathEncodingTest.java
create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicPermissionsTest.java
create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksLedgerMapTest.java
create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5AuthErrorTest.java
copy pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeInfo.java => pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/MemoryLimit.java (64%)
create mode 100644 pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/ScalableTopicsImplTest.java
create mode 100644 pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/V5ExceptionMappings.java
create mode 100644 pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/V5ExceptionMappingsTest.java
(pulsar) 10/14: [fix][admin] Encode a space in a path component as %20, not as '+' (#26957)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 35227b68bd5330fc57957aa7b5ec1e751415dac7
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Fri Oct 16 20:17:25 2026 -0700
[fix][admin] Encode a space in a path component as %20, not as '+' (#26957)
(cherry picked from commit f3c69c85ad3c86725b80755c13e398d332192871)
---
.../broker/admin/AdminApiPathEncodingTest.java | 162 +++++++++++++++++++++
.../pulsar/client/admin/internal/BrokersImpl.java | 5 +-
.../pulsar/client/admin/internal/WebTargets.java | 7 +-
3 files changed, 170 insertions(+), 4 deletions(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiPathEncodingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiPathEncodingTest.java
new file mode 100644
index 000000000000..b55f573eca4f
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiPathEncodingTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.apache.pulsar.common.policies.data.DispatchRate;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Roles, subscription names, namespace properties and dynamic configuration values that the admin client sends as
+ * path components reach the broker unchanged: a space and a '+' make two different names, as "team app" and
+ * "team+app".
+ */
+@Test(groups = "broker-admin")
+public class AdminApiPathEncodingTest extends MockedPulsarServiceBaseTest {
+
+ @BeforeClass
+ @Override
+ protected void setup() throws Exception {
+ super.internalSetup();
+ setupDefaultTenantAndNamespace();
+ }
+
+ @AfterClass(alwaysRun = true)
+ @Override
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ private String newNamespace() throws Exception {
+ String namespace = "public/ns-" + UUID.randomUUID();
+ admin.namespaces().createNamespace(namespace);
+ return namespace;
+ }
+
+ private String newTopic(String namespace) throws Exception {
+ String topic = "persistent://" + namespace + "/" + UUID.randomUUID();
+ admin.topics().createNonPartitionedTopic(topic);
+ return topic;
+ }
+
+ @Test
+ public void testTopicPermissions() throws Exception {
+ String topic = newTopic(newNamespace());
+ admin.topics().grantPermission(topic, "team app", Set.of(AuthAction.consume));
+ admin.topics().grantPermission(topic, "team+app", Set.of(AuthAction.produce));
+ assertThat(admin.topics().getPermissions(topic))
+ .containsEntry("team app", Set.of(AuthAction.consume))
+ .containsEntry("team+app", Set.of(AuthAction.produce));
+
+ admin.topics().revokePermissions(topic, "team app");
+ assertThat(admin.topics().getPermissions(topic))
+ .doesNotContainKey("team app")
+ .containsEntry("team+app", Set.of(AuthAction.produce));
+ }
+
+ @Test
+ public void testNamespacePermissions() throws Exception {
+ String namespace = newNamespace();
+ admin.namespaces().grantPermissionOnNamespace(namespace, "team app", Set.of(AuthAction.consume));
+ admin.namespaces().grantPermissionOnNamespace(namespace, "team+app", Set.of(AuthAction.produce));
+ assertThat(admin.namespaces().getPermissions(namespace))
+ .containsEntry("team app", Set.of(AuthAction.consume))
+ .containsEntry("team+app", Set.of(AuthAction.produce));
+
+ admin.namespaces().revokePermissionsOnNamespace(namespace, "team app");
+ assertThat(admin.namespaces().getPermissions(namespace))
+ .doesNotContainKey("team app")
+ .containsEntry("team+app", Set.of(AuthAction.produce));
+ }
+
+ @Test
+ public void testSubscriptionPermissions() throws Exception {
+ String namespace = newNamespace();
+ admin.namespaces().grantPermissionOnSubscription(namespace, "team sub", Set.of("team app", "team+app"));
+ admin.namespaces().grantPermissionOnSubscription(namespace, "team+sub", Set.of("team app", "team+app"));
+ assertThat(admin.namespaces().getPermissionOnSubscription(namespace))
+ .containsEntry("team sub", Set.of("team app", "team+app"))
+ .containsEntry("team+sub", Set.of("team app", "team+app"));
+
+ admin.namespaces().revokePermissionOnSubscription(namespace, "team sub", "team app");
+ assertThat(admin.namespaces().getPermissionOnSubscription(namespace))
+ .containsEntry("team sub", Set.of("team+app"))
+ .containsEntry("team+sub", Set.of("team app", "team+app"));
+ }
+
+ @Test
+ public void testNamespaceProperties() throws Exception {
+ String namespace = newNamespace();
+ admin.namespaces().setProperty(namespace, "team key", "team value");
+ admin.namespaces().setProperty(namespace, "team+key", "team+value");
+ assertThat(admin.namespaces().getProperties(namespace))
+ .containsEntry("team key", "team value")
+ .containsEntry("team+key", "team+value");
+ assertThat(admin.namespaces().getProperty(namespace, "team key")).isEqualTo("team value");
+
+ assertThat(admin.namespaces().removeProperty(namespace, "team key")).isEqualTo("team value");
+ assertThat(admin.namespaces().getProperties(namespace))
+ .doesNotContainKey("team key")
+ .containsEntry("team+key", "team+value");
+ }
+
+ @Test
+ public void testUnsubscribeNamespace() throws Exception {
+ String namespace = newNamespace();
+ String topic = newTopic(namespace);
+ admin.topics().createSubscription(topic, "team sub", MessageId.earliest);
+ admin.topics().createSubscription(topic, "team+sub", MessageId.earliest);
+
+ admin.namespaces().unsubscribeNamespace(namespace, "team sub");
+ assertThat(admin.topics().getSubscriptions(topic)).containsExactly("team+sub");
+ }
+
+ @Test
+ public void testDynamicConfigurationValue() throws Exception {
+ admin.brokers().updateDynamicConfiguration("superUserRoles", "team app,team+app");
+ assertThat(admin.brokers().getAllDynamicConfigurations())
+ .containsEntry("superUserRoles", "team app,team+app");
+ Awaitility.await().untilAsserted(() ->
+ assertThat(pulsar.getConfiguration().getSuperUserRoles())
+ .containsExactlyInAnyOrder("team app", "team+app"));
+ }
+
+ /** These endpoints read the subscription name with {@code @Encoded} and decode it themselves. */
+ @Test
+ public void testSubscriptionDispatchRate() throws Exception {
+ String topic = newTopic(newNamespace());
+ DispatchRate rate = DispatchRate.builder()
+ .dispatchThrottlingRateInMsg(100)
+ .dispatchThrottlingRateInByte(1024)
+ .ratePeriodInSecond(1)
+ .build();
+ admin.topicPolicies().setSubscriptionDispatchRate(topic, "team sub", rate);
+ Awaitility.await().untilAsserted(() ->
+ assertThat(admin.topicPolicies().getSubscriptionDispatchRate(topic, "team sub")).isEqualTo(rate));
+ assertThat(admin.topicPolicies().getSubscriptionDispatchRate(topic, "team+sub")).isNull();
+ }
+}
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java
index 56046474d7c9..44217c0466bb 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java
@@ -32,7 +32,6 @@ import org.apache.pulsar.client.api.Authentication;
import org.apache.pulsar.common.conf.InternalConfigurationData;
import org.apache.pulsar.common.policies.data.BrokerInfo;
import org.apache.pulsar.common.policies.data.NamespaceOwnershipStatus;
-import org.apache.pulsar.common.util.Codec;
public class BrokersImpl extends BaseResource implements Brokers {
private final WebTarget adminBrokers;
@@ -94,8 +93,8 @@ public class BrokersImpl extends BaseResource implements Brokers {
@Override
public CompletableFuture<Void> updateDynamicConfigurationAsync(String configName, String configValue) {
- String value = Codec.encode(configValue);
- WebTarget path = adminBrokers.path("configuration").path(configName).path(value);
+ WebTarget path = WebTargets.addParts(adminBrokers.path("configuration").path(configName),
+ new String[] {configValue});
return asyncPostRequest(path, Entity.entity("", MediaType.APPLICATION_JSON));
}
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java
index 1ddb14aeeec5..05783db651a7 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java
@@ -26,10 +26,15 @@ import lombok.CustomLog;
@CustomLog
class WebTargets {
+ /**
+ * Appends each part as one path segment, even if it contains a '/'. A space is encoded as "%20", not as the '+'
+ * of the form encoding of {@link URLEncoder}: a plain {@code @PathParam} keeps a '+' of the path as a '+', while
+ * "%20" is a space both there and for the endpoints that read the part {@code @Encoded} and URL-decode it.
+ */
static WebTarget addParts(WebTarget target, String[] parts) {
if (parts != null && parts.length > 0) {
for (String part : parts) {
- String encode = URLEncoder.encode(part, StandardCharsets.UTF_8);
+ String encode = URLEncoder.encode(part, StandardCharsets.UTF_8).replace("+", "%20");
target = target.path(encode);
}
}
(pulsar) 03/14: [fix][misc] Fix Backoff mandatory-stop timer tracking (#25282)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 12defe357161e7c68a9e681c4f3fc7aaabc8e9e6
Author: sinan liu <li...@gmail.com>
AuthorDate: Sat Oct 17 08:31:25 2026 +0800
[fix][misc] Fix Backoff mandatory-stop timer tracking (#25282)
Co-authored-by: Matteo Merli <mm...@apache.org>
(cherry picked from commit 7c7efeed3cc6a19976cdd7616a868e00e56cd760)
---
.../org/apache/pulsar/common/util/Backoff.java | 7 +-
.../org/apache/pulsar/common/util/BackoffTest.java | 93 ++++++++++++++++++++++
2 files changed, 99 insertions(+), 1 deletion(-)
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/Backoff.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/Backoff.java
index 8bcc165dd612..05ff43462870 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/Backoff.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/Backoff.java
@@ -68,6 +68,7 @@ public class Backoff {
private Duration next;
@Getter
private Instant firstBackoffTime;
+ private boolean firstBackoffTimeSet;
@Getter
private boolean mandatoryStopMade;
@@ -79,6 +80,7 @@ public class Backoff {
this.next = initial;
this.clock = clock;
this.firstBackoffTime = Instant.EPOCH;
+ this.firstBackoffTimeSet = false;
if (initial.isZero() && max.isZero() && mandatoryStop.isZero()) {
this.mandatoryStopMade = true;
}
@@ -124,8 +126,9 @@ public class Backoff {
if (!mandatoryStopMade) {
Instant now = clock.instant();
Duration timeElapsedSinceFirstBackoff = Duration.ZERO;
- if (initial.equals(current)) {
+ if (!firstBackoffTimeSet) {
firstBackoffTime = now;
+ firstBackoffTimeSet = true;
} else {
timeElapsedSinceFirstBackoff = Duration.between(firstBackoffTime, now);
}
@@ -163,6 +166,8 @@ public class Backoff {
*/
public void reset() {
this.next = this.initial;
+ this.firstBackoffTime = Instant.EPOCH;
+ this.firstBackoffTimeSet = false;
this.mandatoryStopMade = initial.isZero() && max.isZero() && mandatoryStop.isZero();
}
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/BackoffTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/BackoffTest.java
index 41c2de63c104..3b9cdf9d0d7a 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/BackoffTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/BackoffTest.java
@@ -73,6 +73,99 @@ public class BackoffTest {
assertEquals(diffBackOffTime, 300);
}
+ @Test
+ public void mandatoryStopShouldWorkWithConstantBackoff() {
+ Clock mockClock = Mockito.mock(Clock.class);
+ Mockito.when(mockClock.instant())
+ .thenReturn(Instant.ofEpochMilli(0))
+ .thenReturn(Instant.ofEpochMilli(400));
+
+ Backoff backoff = Backoff.builder()
+ .initialDelay(Duration.ofMillis(100))
+ .maxBackoff(Duration.ofMillis(100))
+ .mandatoryStop(Duration.ofMillis(300))
+ .clock(mockClock)
+ .build();
+
+ // first call starts the mandatory-stop timer
+ backoff.next();
+ Instant firstBackoffTime = backoff.getFirstBackoffTime();
+
+ // elapsed wall-clock time should exceed mandatory stop
+ backoff.next();
+ assertEquals(backoff.getFirstBackoffTime(), firstBackoffTime,
+ "firstBackoffTime should not be updated after the first call to next()");
+ assertTrue(backoff.isMandatoryStopMade(),
+ "mandatory stop should be reached even when initial == max (constant backoff)");
+ }
+
+ @Test
+ public void reduceToHalfShouldNotResetMandatoryStopTimer() {
+ Clock mockClock = Mockito.mock(Clock.class);
+ Mockito.when(mockClock.instant())
+ .thenReturn(Instant.ofEpochMilli(0))
+ .thenReturn(Instant.ofEpochMilli(200));
+
+ Backoff backoff = Backoff.builder()
+ .initialDelay(Duration.ofMillis(100))
+ .maxBackoff(Duration.ofMillis(400))
+ .mandatoryStop(Duration.ofMillis(250))
+ .clock(mockClock)
+ .build();
+
+ // first call starts the mandatory-stop timer (next becomes 200)
+ backoff.next();
+ Instant firstBackoffTime = backoff.getFirstBackoffTime();
+
+ // This can bring the next delay back to initial, but it should not reset firstBackoffTime.
+ backoff.reduceToHalf();
+
+ backoff.next();
+ assertEquals(backoff.getFirstBackoffTime(), firstBackoffTime,
+ "reduceToHalf should not reset firstBackoffTime");
+ assertTrue(backoff.isMandatoryStopMade(),
+ "mandatory stop should be reached based on wall-clock time since the first call to next()");
+ }
+
+ @Test
+ public void resetShouldStartNewMandatoryStopCycle() {
+ Clock mockClock = Mockito.mock(Clock.class);
+ Mockito.when(mockClock.instant())
+ .thenReturn(Instant.ofEpochMilli(0))
+ .thenReturn(Instant.ofEpochMilli(400))
+ .thenReturn(Instant.ofEpochMilli(1000))
+ .thenReturn(Instant.ofEpochMilli(1400));
+
+ Backoff backoff = Backoff.builder()
+ .initialDelay(Duration.ofMillis(100))
+ .maxBackoff(Duration.ofMillis(100))
+ .mandatoryStop(Duration.ofMillis(300))
+ .clock(mockClock)
+ .build();
+
+ // Cycle 1: reach mandatory stop
+ backoff.next();
+ Instant firstCycleStart = backoff.getFirstBackoffTime();
+ backoff.next();
+ assertTrue(backoff.isMandatoryStopMade(), "mandatory stop should be reached in cycle-1");
+
+ // Reset should clear mandatory-stop state so a new cycle can start.
+ backoff.reset();
+ assertFalse(backoff.isMandatoryStopMade(), "reset should clear mandatoryStopMade");
+ assertEquals(backoff.getFirstBackoffTime(), Instant.EPOCH, "reset should clear firstBackoffTime");
+
+ // Cycle 2: should start timing again and be able to reach mandatory stop again.
+ backoff.next();
+ Instant secondCycleStart = backoff.getFirstBackoffTime();
+ assertEquals(secondCycleStart, Instant.ofEpochMilli(1000),
+ "reset should start a new mandatory-stop timing window");
+ assertEquals(Duration.between(firstCycleStart, secondCycleStart).toMillis(), 1000,
+ "reset should not reuse the old timing window");
+
+ backoff.next();
+ assertTrue(backoff.isMandatoryStopMade(), "mandatory stop should be reached in cycle-2");
+ }
+
@Test
public void basicTest() {
Clock mockClock = Clock.fixed(Instant.EPOCH, ZoneId.systemDefault());
(pulsar) 12/14: [improve][fn] Provide a way to set Pulsar client memory limit in Pulsar Function (#23959)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 5f320772644f30ad7cfc1caa3c58ca058a9581b7
Author: Lee hong <40...@users.noreply.github.com>
AuthorDate: Sat Oct 17 12:43:33 2026 +0800
[improve][fn] Provide a way to set Pulsar client memory limit in Pulsar Function (#23959)
Co-authored-by: Lari Hotari <lh...@users.noreply.github.com>
Co-authored-by: Matteo Merli <mm...@apache.org>
(cherry picked from commit 2fbd8153e09508c23e18b2e9514a9e44c67e794c)
---
conf/functions_worker.yml | 9 ++++
.../pulsar/common/functions/FunctionConfig.java | 3 ++
.../pulsar/common/functions/MemoryLimit.java | 31 ++++++-----
.../apache/pulsar/admin/cli/CmdFunctionsTest.java | 22 ++++++++
.../org/apache/pulsar/admin/cli/CmdFunctions.java | 15 ++++++
.../pulsar/common/util/DirectMemoryUtils.java | 3 ++
.../pulsar/functions/instance/InstanceConfig.java | 3 +-
.../org/apache/pulsar/functions/LocalRunner.java | 8 ++-
.../proto/src/main/proto/Function.proto | 9 ++++
.../functions/runtime/JavaInstanceStarter.java | 19 ++++++-
.../pulsar/functions/runtime/RuntimeUtils.java | 11 ++++
.../runtime/thread/ThreadRuntimeFactory.java | 35 ++++++++-----
.../pulsar/functions/worker/WorkerConfig.java | 10 ++++
.../pulsar/functions/runtime/RuntimeUtilsTest.java | 42 +++++++++++++++
.../runtime/thread/ThreadRuntimeFactoryTest.java | 43 +++++++++++----
.../functions/utils/FunctionConfigUtils.java | 45 ++++++++++++++++
.../functions/utils/FunctionConfigUtilsTest.java | 61 ++++++++++++++++++++++
.../pulsar/functions/worker/FunctionActioner.java | 4 ++
.../functions/worker/FunctionActionerTest.java | 27 ++++++++++
19 files changed, 358 insertions(+), 42 deletions(-)
diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
index 426ae802913c..1a13e361b13d 100644
--- a/conf/functions_worker.yml
+++ b/conf/functions_worker.yml
@@ -169,6 +169,15 @@ functionRuntimeFactoryConfigs:
# This arguments will be added to the command line execution of 'java'
#additionalJavaRuntimeArguments: ['-XX:+ExitOnOutOfMemoryError']
+#### Pulsar client memory limit of Java function instances ####
+# Default for functions that don't set their own limit. If both values are set, the smaller limit is used.
+# The thread runtime uses it when functionRuntimeFactoryConfigs.pulsarClientMemoryLimit is not set.
+#pulsarClientMemoryLimit:
+# # the max memory in bytes the pulsar client can use
+# limitInBytes:
+# # the max memory the pulsar client can use as a percentage of max direct memory set for JVM
+# percentOfMaxDirectMemory:
+
#### Thread Runtime ####
# Pulsar function instances are run as threads
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java
index 9d6ce1fcc0d3..ce791caea02a 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java
@@ -147,6 +147,9 @@ public class FunctionConfig {
private Boolean skipToLatest;
// Pulsar client API for the function's own topics; null picks it from the topic domains
private ClientApi clientApi;
+ // Memory limit of the Pulsar client used by each Java function instance; null uses the worker default.
+ // Ignored by the worker's thread runtime, where all instances share one client configured by the worker.
+ private MemoryLimit pulsarClientMemoryLimit;
@Builder.Default
private SubscriptionInitialPosition subscriptionPosition = SubscriptionInitialPosition.Latest;
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/MemoryLimit.java
similarity index 57%
copy from pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
copy to pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/MemoryLimit.java
index 0e5a450aa6bf..01cf706acda1 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/MemoryLimit.java
@@ -16,22 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pulsar.common.util;
+package org.apache.pulsar.common.functions;
-// CHECKSTYLE.OFF: IllegalImport
-import io.netty.util.internal.PlatformDependent;
-// CHECKSTYLE.ON: IllegalImport
-
-public class DirectMemoryUtils {
-
- /**
- * PlatformDependent.maxDirectMemory can be inaccurate if java property `io.netty.maxDirectMemory` are setted.
- * Cache the result in this field.
- */
- private static final long JVM_MAX_DIRECT_MEMORY = PlatformDependent.estimateMaxDirectMemory();
-
- public static long jvmMaxDirectMemory() {
- return JVM_MAX_DIRECT_MEMORY;
- }
+import lombok.Data;
+import lombok.experimental.Accessors;
+/**
+ * Memory limit of the Pulsar client used by a function instance.
+ * If both {@code limitInBytes} and {@code percentOfMaxDirectMemory} are set,
+ * the smaller of the two limits is used.
+ */
+@Data
+@Accessors(chain = true)
+public class MemoryLimit {
+ // The max memory in bytes the Pulsar client can use
+ private Long limitInBytes;
+ // The max memory the Pulsar client can use, as a percentage (0, 100] of the JVM max direct memory
+ private Double percentOfMaxDirectMemory;
}
diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
index 9950bf257b7f..5488f0bd1654 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
@@ -45,6 +45,7 @@ import org.apache.pulsar.admin.cli.CmdFunctions.UpdateFunction;
import org.apache.pulsar.client.admin.Functions;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.common.functions.FunctionConfig;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.UpdateOptionsImpl;
import org.apache.pulsar.functions.api.Context;
import org.apache.pulsar.functions.api.Function;
@@ -519,6 +520,27 @@ public class CmdFunctionsTest {
verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
}
+ @Test
+ public void testCreateFunctionWithPulsarClientMemoryLimit() throws Exception {
+ cmd.run(new String[] {
+ "create",
+ "--name", FN_NAME,
+ "--inputs", "persistent://sample/ns1/in",
+ "--output", "persistent://sample/ns1/out",
+ "--client-memory-limit", "64M",
+ "--client-memory-limit-percent", "25",
+ "--jar", JAR_NAME,
+ "--tenant", "sample",
+ "--namespace", "ns1",
+ "--className", DummyFunction.class.getName(),
+ });
+
+ CreateFunction creater = cmd.getCreater();
+ assertEquals(new MemoryLimit().setLimitInBytes(64L * 1024 * 1024).setPercentOfMaxDirectMemory(25.0),
+ creater.getFunctionConfig().getPulsarClientMemoryLimit());
+ verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
+ }
+
@Test
public void testCreateUsingFullyQualifiedFunctionName() throws Exception {
String tenant = "sample";
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index 330c1e0f1c5f..8b57f1ae1bac 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -42,6 +42,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.WordUtils;
import org.apache.pulsar.admin.cli.utils.CmdUtils;
import org.apache.pulsar.cli.ClientApi;
+import org.apache.pulsar.cli.converters.picocli.ByteUnitToLongConverter;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.PulsarClientException;
@@ -49,6 +50,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
import org.apache.pulsar.common.functions.FunctionState;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.functions.UpdateOptionsImpl;
@@ -325,6 +327,13 @@ public class CmdFunctions extends CmdBase {
protected Boolean skipToLatest;
@Option(names = ClientApi.OPTION_NAME, description = CmdFunctions.CLIENT_API_DESCRIPTION)
protected FunctionConfig.ClientApi clientApi;
+ @Option(names = "--client-memory-limit", description = "The max memory the Pulsar client of each function "
+ + "instance can use (eg: 32M, 64M) #Java(Process & K8s)", converter = ByteUnitToLongConverter.class)
+ protected Long clientMemoryLimit;
+ @Option(names = "--client-memory-limit-percent", description = "The max memory the Pulsar client of each "
+ + "function instance can use, as a percentage of the instance JVM max direct memory. If "
+ + "--client-memory-limit is also set, the smaller of the two limits is used #Java(Process & K8s)")
+ protected Double clientMemoryLimitPercent;
@Option(names = "--parallelism", description = "The parallelism factor of a Pulsar Function "
+ "(i.e. the number of function instances to run) #Java")
protected Integer parallelism;
@@ -571,6 +580,12 @@ public class CmdFunctions extends CmdBase {
functionConfig.setClientApi(clientApi);
}
+ if (null != clientMemoryLimit || null != clientMemoryLimitPercent) {
+ functionConfig.setPulsarClientMemoryLimit(new MemoryLimit()
+ .setLimitInBytes(clientMemoryLimit)
+ .setPercentOfMaxDirectMemory(clientMemoryLimitPercent));
+ }
+
if (null != userConfigString) {
Type type = new TypeToken<Map<String, Object>>() {}.getType();
Map<String, Object> userConfigMap = new Gson().fromJson(userConfigString, type);
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
index 0e5a450aa6bf..abc1466922ea 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
@@ -34,4 +34,7 @@ public class DirectMemoryUtils {
return JVM_MAX_DIRECT_MEMORY;
}
+ public static long jvmMaxDirectMemoryPercentage(double percent) {
+ return (long) (DirectMemoryUtils.jvmMaxDirectMemory() * (percent / 100));
+ }
}
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
index f4c5ca380860..204ebadb6418 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.List;
import lombok.Data;
import lombok.Getter;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.functions.proto.FunctionAuthenticationSpec;
import org.apache.pulsar.functions.proto.FunctionDetails;
@@ -49,7 +50,7 @@ public class InstanceConfig {
private int metricsPort;
private List<String> additionalJavaRuntimeArguments = Collections.emptyList();
private boolean ignoreUnknownConfigFields;
-
+ private MemoryLimit pulsarClientMemoryLimit;
/**
* Get the string representation of {@link #getInstanceId()}.
*
diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java
index f7f524aa495a..ad169a160970 100644
--- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java
+++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java
@@ -49,6 +49,7 @@ import lombok.CustomLog;
import lombok.Value;
import org.apache.pulsar.common.functions.FunctionConfig;
import org.apache.pulsar.common.functions.FunctionDefinition;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.Utils;
import org.apache.pulsar.common.io.ConnectorDefinition;
import org.apache.pulsar.common.io.SinkConfig;
@@ -582,6 +583,7 @@ public class LocalRunner implements AutoCloseable {
instanceConfig
.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
}
+ instanceConfig.setPulsarClientMemoryLimit(functionConfig.getPulsarClientMemoryLimit());
}
RuntimeSpawner runtimeSpawner = new RuntimeSpawner(
@@ -662,6 +664,9 @@ public class LocalRunner implements AutoCloseable {
if (userCodeClassLoader != null && userCodeClassLoader.getClassLoader() != null) {
Thread.currentThread().setContextClassLoader(userCodeClassLoader.getClassLoader());
}
+ Optional<MemoryLimit> pulsarClientMemoryLimit = functionConfig != null
+ ? Optional.ofNullable(functionConfig.getPulsarClientMemoryLimit())
+ : Optional.empty();
runtimeFactory = new ThreadRuntimeFactory("LocalRunnerThreadGroup",
serviceUrl,
stateStorageImplClass,
@@ -670,7 +675,7 @@ public class LocalRunner implements AutoCloseable {
secretsProvider,
collectorRegistry, narExtractionDirectory,
null,
- exposePulsarAdminClientEnabled, webServiceUrl);
+ exposePulsarAdminClientEnabled, webServiceUrl, pulsarClientMemoryLimit);
} finally {
Thread.currentThread().setContextClassLoader(originalClassLoader);
}
@@ -693,6 +698,7 @@ public class LocalRunner implements AutoCloseable {
instanceConfig
.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
}
+ instanceConfig.setPulsarClientMemoryLimit(functionConfig.getPulsarClientMemoryLimit());
}
RuntimeSpawner runtimeSpawner = new RuntimeSpawner(
diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto
index 8742b4740ac7..da545179ed9e 100644
--- a/pulsar-functions/proto/src/main/proto/Function.proto
+++ b/pulsar-functions/proto/src/main/proto/Function.proto
@@ -106,6 +106,15 @@ message FunctionDetails {
bool retainKeyOrdering = 22;
SubscriptionPosition subscriptionPosition = 23;
ClientApi clientApi = 24;
+ MemoryLimit pulsarClientMemoryLimit = 25;
+}
+
+// Memory limit of the Pulsar client used by a function instance. A 0 value means unset.
+message MemoryLimit {
+ // The max memory in bytes the Pulsar client can use
+ int64 limitInBytes = 1;
+ // The max memory the Pulsar client can use, as a percentage of the JVM max direct memory
+ double percentOfMaxDirectMemory = 2;
}
message ConsumerSpec {
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
index 9eae1997e51d..194deff1050d 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
@@ -29,6 +29,7 @@ import io.prometheus.client.exporter.HTTPServer;
import java.lang.reflect.Type;
import java.net.InetSocketAddress;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@@ -37,6 +38,7 @@ import net.bytebuddy.description.type.TypeDefinition;
import net.bytebuddy.dynamic.ClassFileLocator;
import net.bytebuddy.pool.TypePool;
import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.WindowConfig;
import org.apache.pulsar.common.nar.NarClassLoader;
import org.apache.pulsar.common.util.Reflections;
@@ -161,6 +163,15 @@ public class JavaInstanceStarter implements AutoCloseable {
required = false)
public Boolean ignoreUnknownConfigFields = false;
+ @Option(names = "--pulsar_client_memory_limit_bytes",
+ description = "The max memory in bytes the Pulsar client can use",
+ required = false)
+ public Long pulsarClientMemoryLimitBytes;
+
+ @Option(names = "--pulsar_client_memory_limit_percent",
+ description = "The max memory the Pulsar client can use, as a percentage of the JVM max direct memory",
+ required = false)
+ public Double pulsarClientMemoryLimitPercent;
private Server server;
private RuntimeSpawner runtimeSpawner;
@@ -189,6 +200,11 @@ public class JavaInstanceStarter implements AutoCloseable {
instanceConfig.setMaxPendingAsyncRequests(maxPendingAsyncRequests);
instanceConfig.setExposePulsarAdminClientEnabled(exposePulsarAdminClientEnabled);
instanceConfig.setIgnoreUnknownConfigFields(ignoreUnknownConfigFields);
+ if (pulsarClientMemoryLimitBytes != null || pulsarClientMemoryLimitPercent != null) {
+ instanceConfig.setPulsarClientMemoryLimit(new MemoryLimit()
+ .setLimitInBytes(pulsarClientMemoryLimitBytes)
+ .setPercentOfMaxDirectMemory(pulsarClientMemoryLimitPercent));
+ }
if (functionDetailsJsonString.charAt(0) == '\'') {
functionDetailsJsonString = functionDetailsJsonString.substring(1);
}
@@ -244,7 +260,8 @@ public class JavaInstanceStarter implements AutoCloseable {
.tlsHostnameVerificationEnable(isTrue(tlsHostNameVerificationEnabled))
.tlsTrustCertsFilePath(tlsTrustCertFilePath).build(),
secretsProvider, collectorRegistry, narExtractionDirectory, rootClassLoader,
- exposePulsarAdminClientEnabled, webServiceUrl, fnCache);
+ exposePulsarAdminClientEnabled, webServiceUrl, fnCache,
+ Optional.ofNullable(instanceConfig.getPulsarClientMemoryLimit()));
runtimeSpawner = new RuntimeSpawner(
instanceConfig,
jarFile,
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
index 45f687b27ca9..3074c9543fa4 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
@@ -47,6 +47,7 @@ import lombok.CustomLog;
import org.apache.commons.lang3.JavaVersion;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.functions.instance.AuthenticationConfig;
import org.apache.pulsar.functions.instance.InstanceConfig;
@@ -507,6 +508,16 @@ public class RuntimeUtils {
if (instanceConfig.isIgnoreUnknownConfigFields()) {
args.add("--ignore_unknown_config_fields");
}
+
+ MemoryLimit clientMemoryLimit = instanceConfig.getPulsarClientMemoryLimit();
+ if (clientMemoryLimit != null && clientMemoryLimit.getLimitInBytes() != null) {
+ args.add("--pulsar_client_memory_limit_bytes");
+ args.add(String.valueOf(clientMemoryLimit.getLimitInBytes()));
+ }
+ if (clientMemoryLimit != null && clientMemoryLimit.getPercentOfMaxDirectMemory() != null) {
+ args.add("--pulsar_client_memory_limit_percent");
+ args.add(String.valueOf(clientMemoryLimit.getPercentOfMaxDirectMemory()));
+ }
}
// state storage configs
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
index 032fda7b00e4..93765b175e47 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
@@ -19,6 +19,7 @@
package org.apache.pulsar.functions.runtime.thread;
import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.pulsar.common.util.DirectMemoryUtils.jvmMaxDirectMemoryPercentage;
import java.util.Optional;
import lombok.CustomLog;
import lombok.Getter;
@@ -27,7 +28,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.common.util.DirectMemoryUtils;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.util.Reflections;
import org.apache.pulsar.functions.auth.FunctionAuthProvider;
import org.apache.pulsar.functions.instance.AuthenticationConfig;
@@ -84,8 +85,8 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
AuthenticationConfig authConfig, SecretsProvider secretsProvider,
FunctionCollectorRegistry collectorRegistry, String narExtractionDirectory,
ClassLoader rootClassLoader, boolean exposePulsarAdminClientEnabled,
- String pulsarWebServiceUrl) throws Exception {
- initialize(threadGroupName, Optional.empty(), pulsarServiceUrl, authConfig,
+ String pulsarWebServiceUrl, Optional<MemoryLimit> memoryLimit) throws Exception {
+ initialize(threadGroupName, memoryLimit, pulsarServiceUrl, authConfig,
stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry,
narExtractionDirectory,
rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(),
@@ -98,15 +99,16 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
AuthenticationConfig authConfig, SecretsProvider secretsProvider,
FunctionCollectorRegistry collectorRegistry, String narExtractionDirectory,
ClassLoader rootClassLoader, boolean exposePulsarAdminClientEnabled,
- String pulsarWebServiceUrl, FunctionCacheManager fnCache) throws Exception {
- initialize(threadGroupName, Optional.empty(), pulsarServiceUrl, authConfig,
+ String pulsarWebServiceUrl, FunctionCacheManager fnCache,
+ Optional<MemoryLimit> memoryLimit) throws Exception {
+ initialize(threadGroupName, memoryLimit, pulsarServiceUrl, authConfig,
stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry,
narExtractionDirectory,
rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(),
Optional.empty(), fnCache);
}
- private void initialize(String threadGroupName, Optional<ThreadRuntimeFactoryConfig.MemoryLimit> memoryLimit,
+ private void initialize(String threadGroupName, Optional<MemoryLimit> memoryLimit,
String pulsarServiceUrl, AuthenticationConfig authConfig, String stateStorageImplClass,
String storageServiceUrl,
SecretsProviderConfigurator secretsProviderConfigurator, SecretsProvider secretsProvider,
@@ -145,10 +147,10 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
this.functionsManager = functionsManager;
}
- private Optional<Long> calculateClientMemoryLimit(Optional<ThreadRuntimeFactoryConfig.MemoryLimit> memoryLimit) {
+ private Optional<Long> calculateClientMemoryLimit(Optional<MemoryLimit> memoryLimit) {
if (memoryLimit.isPresent()) {
- Long absolute = memoryLimit.get().getAbsoluteValue();
+ Long absolute = memoryLimit.get().getLimitInBytes();
Double percentOfDirectMem = memoryLimit.get().getPercentOfMaxDirectMemory();
if (absolute != null) {
checkArgument(absolute > 0, "Absolute memory limit for Pulsar client has to be positive");
@@ -159,7 +161,7 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
}
if (absolute != null && percentOfDirectMem != null) {
- return Optional.of(Math.min(absolute, getBytesPercentDirectMem(percentOfDirectMem)));
+ return Optional.of(Math.min(absolute, jvmMaxDirectMemoryPercentage(percentOfDirectMem)));
}
if (absolute != null) {
@@ -167,14 +169,21 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
}
if (percentOfDirectMem != null) {
- return Optional.of(getBytesPercentDirectMem(percentOfDirectMem));
+ return Optional.of(jvmMaxDirectMemoryPercentage(percentOfDirectMem));
}
}
return Optional.empty();
}
- private long getBytesPercentDirectMem(double percent) {
- return (long) (DirectMemoryUtils.jvmMaxDirectMemory() * (percent / 100));
+ private static Optional<MemoryLimit> getPulsarClientMemoryLimit(ThreadRuntimeFactoryConfig factoryConfig,
+ WorkerConfig workerConfig) {
+ ThreadRuntimeFactoryConfig.MemoryLimit memoryLimit = factoryConfig.getPulsarClientMemoryLimit();
+ if (memoryLimit == null) {
+ return Optional.ofNullable(workerConfig.getPulsarClientMemoryLimit());
+ }
+ return Optional.of(new MemoryLimit()
+ .setLimitInBytes(memoryLimit.getAbsoluteValue())
+ .setPercentOfMaxDirectMemory(memoryLimit.getPercentOfMaxDirectMemory()));
}
@@ -188,7 +197,7 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
ThreadRuntimeFactoryConfig factoryConfig = RuntimeUtils.getRuntimeFunctionConfig(
workerConfig.getFunctionRuntimeFactoryConfigs(), ThreadRuntimeFactoryConfig.class);
- initialize(factoryConfig.getThreadGroupName(), Optional.ofNullable(factoryConfig.getPulsarClientMemoryLimit()),
+ initialize(factoryConfig.getThreadGroupName(), getPulsarClientMemoryLimit(factoryConfig, workerConfig),
workerConfig.getPulsarServiceUrl(), authenticationConfig,
workerConfig.getStateStorageProviderImplementation(),
workerConfig.getStateStorageServiceUrl(), secretsProviderConfigurator, null,
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
index b4adbb005ec9..66d97108a479 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
@@ -48,6 +48,7 @@ import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider;
import org.apache.pulsar.common.configuration.Category;
import org.apache.pulsar.common.configuration.FieldContext;
import org.apache.pulsar.common.configuration.PulsarConfiguration;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.nar.NarClassLoader;
import org.apache.pulsar.common.sasl.SaslConstants;
@@ -876,6 +877,15 @@ public class WorkerConfig implements Serializable, PulsarConfiguration {
)
private int maxPendingAsyncRequests = 1000;
+ @FieldContext(
+ category = CATEGORY_FUNC_RUNTIME_MNG,
+ doc = "Default memory limit of the Pulsar client used by each Java function instance, for functions"
+ + " that don't set their own. If both `limitInBytes` and `percentOfMaxDirectMemory` are set,"
+ + " the smaller of the two limits is used. The thread runtime uses it when"
+ + " `functionRuntimeFactoryConfigs.pulsarClientMemoryLimit` is not set."
+ )
+ private MemoryLimit pulsarClientMemoryLimit;
+
@FieldContext(
doc = "Whether to forward the source message properties to the output message"
)
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
index 4c8214e6e6a6..0dbc12a24bf8 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import lombok.CustomLog;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.functions.instance.AuthenticationConfig;
import org.apache.pulsar.functions.instance.InstanceConfig;
import org.apache.pulsar.functions.proto.FunctionDetails;
@@ -38,6 +39,7 @@ import org.jose4j.json.internal.json_simple.JSONObject;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
+import picocli.CommandLine;
@CustomLog
public class RuntimeUtilsTest {
@@ -239,6 +241,46 @@ public class RuntimeUtilsTest {
assertTrue(indexAdditionalArguments < indexJavaClass);
}
+ @DataProvider(name = "pulsarClientMemoryLimit")
+ public Object[][] pulsarClientMemoryLimit() {
+ return new Object[][] {
+ {null},
+ {new MemoryLimit().setLimitInBytes(64L * 1024 * 1024)},
+ {new MemoryLimit().setPercentOfMaxDirectMemory(25.0)},
+ {new MemoryLimit().setLimitInBytes(64L * 1024 * 1024).setPercentOfMaxDirectMemory(25.0)},
+ };
+ }
+
+ @Test(dataProvider = "pulsarClientMemoryLimit")
+ public void testPulsarClientMemoryLimitReachesJavaInstance(MemoryLimit memoryLimit) throws Exception {
+ InstanceConfig instanceConfig = new InstanceConfig();
+ instanceConfig.setClusterName("kluster");
+ instanceConfig.setInstanceId(3000);
+ instanceConfig.setFunctionId("func-7734");
+ instanceConfig.setFunctionVersion("1.0.0");
+ instanceConfig.setMaxBufferedTuples(5);
+ instanceConfig.setPort(1337);
+ instanceConfig.setFunctionDetails(new FunctionDetails());
+ instanceConfig.setPulsarClientMemoryLimit(memoryLimit);
+
+ List<String> cmd = RuntimeUtils.getCmd(instanceConfig, "instanceFile",
+ null, "logDirectory", "originalCodeFileName", null, "pulsarServiceUrl",
+ null, AuthenticationConfig.builder().build(), "0", 23, 1234L, "logConfigFile",
+ null, null, false, null, null, "narExtractionDirectory", "functionInstanceClassPath",
+ false, "");
+
+ // the arguments after the main class are the ones JavaInstanceStarter parses in the instance process
+ int indexJavaClass = cmd.indexOf("org.apache.pulsar.functions.instance.JavaInstanceMain");
+ JavaInstanceStarter javaInstanceStarter = new JavaInstanceStarter();
+ new CommandLine(javaInstanceStarter).parseArgs(
+ cmd.subList(indexJavaClass + 1, cmd.size()).toArray(new String[0]));
+
+ assertEquals(javaInstanceStarter.pulsarClientMemoryLimitBytes,
+ memoryLimit != null ? memoryLimit.getLimitInBytes() : null);
+ assertEquals(javaInstanceStarter.pulsarClientMemoryLimitPercent,
+ memoryLimit != null ? memoryLimit.getPercentOfMaxDirectMemory() : null);
+ }
+
@Test
public void testSanitizeFileName() {
assertEquals(RuntimeUtils.sanitizeFileName("file(1).jar"), "file_1_.jar");
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java
index 044b291cfd91..ab089db7202a 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java
@@ -29,6 +29,7 @@ import lombok.CustomLog;
import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.SizeUnit;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.util.DirectMemoryUtils;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.functions.instance.AuthenticationConfig;
@@ -101,8 +102,37 @@ public class ThreadRuntimeFactoryTest {
}
- @SuppressWarnings("unchecked")
+ @Test
+ public void testWorkerMemoryLimitUsedWhenFactoryLimitNotSet() throws Exception {
+ ClientBuilder clientBuilder = initWithMemoryLimits(null, new MemoryLimit().setPercentOfMaxDirectMemory(50.0));
+ Mockito.verify(clientBuilder, Mockito.times(1))
+ .memoryLimit(Mockito.eq(JVM_MAX_DIRECT_MEMORY / 2), Mockito.eq(SizeUnit.BYTES));
+ }
+
+ @Test
+ public void testFactoryMemoryLimitOverridesWorkerMemoryLimit() throws Exception {
+ ClientBuilder clientBuilder = initWithMemoryLimits(
+ new ThreadRuntimeFactoryConfig.MemoryLimit().setAbsoluteValue(JVM_MAX_DIRECT_MEMORY / 4),
+ new MemoryLimit().setPercentOfMaxDirectMemory(50.0));
+ Mockito.verify(clientBuilder, Mockito.times(1))
+ .memoryLimit(Mockito.eq(JVM_MAX_DIRECT_MEMORY / 4), Mockito.eq(SizeUnit.BYTES));
+ }
+
private ClientBuilder testMemoryLimit(Long absolute, Double percent) throws Exception {
+ ThreadRuntimeFactoryConfig.MemoryLimit memoryLimit = new ThreadRuntimeFactoryConfig.MemoryLimit();
+ if (percent != null) {
+ memoryLimit.setPercentOfMaxDirectMemory(percent);
+ }
+
+ if (absolute != null) {
+ memoryLimit.setAbsoluteValue(absolute);
+ }
+ return initWithMemoryLimits(memoryLimit, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ private ClientBuilder initWithMemoryLimits(ThreadRuntimeFactoryConfig.MemoryLimit factoryMemoryLimit,
+ MemoryLimit workerMemoryLimit) throws Exception {
try (MockedStatic<PulsarClient> mockedPulsarClient = mockStatic(PulsarClient.class)) {
ClientBuilder clientBuilder = Mockito.mock(ClientBuilder.class);
mockedPulsarClient.when(PulsarClient::builder).thenAnswer(i -> clientBuilder);
@@ -111,21 +141,14 @@ public class ThreadRuntimeFactoryTest {
ThreadRuntimeFactoryConfig threadRuntimeFactoryConfig = new ThreadRuntimeFactoryConfig();
threadRuntimeFactoryConfig.setThreadGroupName("foo");
- ThreadRuntimeFactoryConfig.MemoryLimit memoryLimit = new ThreadRuntimeFactoryConfig.MemoryLimit();
- if (percent != null) {
- memoryLimit.setPercentOfMaxDirectMemory(percent);
- }
-
- if (absolute != null) {
- memoryLimit.setAbsoluteValue(absolute);
- }
- threadRuntimeFactoryConfig.setPulsarClientMemoryLimit(memoryLimit);
+ threadRuntimeFactoryConfig.setPulsarClientMemoryLimit(factoryMemoryLimit);
WorkerConfig workerConfig = new WorkerConfig();
workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getMapper().getObjectMapper()
.convertValue(threadRuntimeFactoryConfig, Map.class));
workerConfig.setPulsarServiceUrl("pulsar://broker.pulsar:6650");
+ workerConfig.setPulsarClientMemoryLimit(workerMemoryLimit);
ThreadRuntimeFactory threadRuntimeFactory = new ThreadRuntimeFactory();
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
index 55bc1b6c0b73..095ea904d00f 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
@@ -48,6 +48,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
import org.apache.pulsar.common.functions.FunctionDefinition;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.functions.WindowConfig;
@@ -378,6 +379,16 @@ public class FunctionConfigUtils {
String builtin = functionConfig.getJar().replaceFirst("^builtin://", "");
functionDetails.setBuiltin(builtin);
}
+ MemoryLimit clientMemoryLimit = functionConfig.getPulsarClientMemoryLimit();
+ if (clientMemoryLimit != null) {
+ org.apache.pulsar.functions.proto.MemoryLimit memoryLimit = functionDetails.setPulsarClientMemoryLimit();
+ if (clientMemoryLimit.getLimitInBytes() != null) {
+ memoryLimit.setLimitInBytes(clientMemoryLimit.getLimitInBytes());
+ }
+ if (clientMemoryLimit.getPercentOfMaxDirectMemory() != null) {
+ memoryLimit.setPercentOfMaxDirectMemory(clientMemoryLimit.getPercentOfMaxDirectMemory());
+ }
+ }
functionDetails.setClientApi(FunctionCommon.convertClientApi(functionConfig.getClientApi()));
ClientApiResolver.resolve(functionDetails);
@@ -401,6 +412,21 @@ public class FunctionConfigUtils {
return functionDetails;
}
+ /**
+ * Convert the proto memory limit, where a 0 value means unset, to the config one.
+ */
+ public static MemoryLimit convertPulsarClientMemoryLimit(
+ org.apache.pulsar.functions.proto.MemoryLimit memoryLimit) {
+ MemoryLimit result = new MemoryLimit();
+ if (memoryLimit.getLimitInBytes() != 0) {
+ result.setLimitInBytes(memoryLimit.getLimitInBytes());
+ }
+ if (memoryLimit.getPercentOfMaxDirectMemory() != 0) {
+ result.setPercentOfMaxDirectMemory(memoryLimit.getPercentOfMaxDirectMemory());
+ }
+ return result;
+ }
+
@SuppressWarnings("deprecation")
public static FunctionConfig convertFromDetails(FunctionDetails functionDetails) {
functionDetails = validateFunctionDetails(functionDetails);
@@ -528,6 +554,10 @@ public class FunctionConfigUtils {
if (!isEmpty(functionDetails.getCustomRuntimeOptions())) {
functionConfig.setCustomRuntimeOptions(functionDetails.getCustomRuntimeOptions());
}
+ if (functionDetails.hasPulsarClientMemoryLimit()) {
+ functionConfig.setPulsarClientMemoryLimit(
+ convertPulsarClientMemoryLimit(functionDetails.getPulsarClientMemoryLimit()));
+ }
return functionConfig;
}
@@ -867,6 +897,18 @@ public class FunctionConfigUtils {
ResourceConfigUtils.validate(functionConfig.getResources());
}
+ MemoryLimit clientMemoryLimit = functionConfig.getPulsarClientMemoryLimit();
+ if (clientMemoryLimit != null) {
+ if (clientMemoryLimit.getLimitInBytes() != null && clientMemoryLimit.getLimitInBytes() <= 0) {
+ throw new IllegalArgumentException("Pulsar client memory limit in bytes must be a positive number");
+ }
+ Double percent = clientMemoryLimit.getPercentOfMaxDirectMemory();
+ if (percent != null && (percent <= 0 || percent > 100)) {
+ throw new IllegalArgumentException(
+ "Pulsar client memory limit percent of max direct memory must be in (0, 100]");
+ }
+ }
+
if (functionConfig.getTimeoutMs() != null && functionConfig.getTimeoutMs() <= 0) {
throw new IllegalArgumentException("Function timeout must be a positive number");
}
@@ -1141,6 +1183,9 @@ public class FunctionConfigUtils {
if (newConfig.getProducerConfig() != null) {
mergedConfig.setProducerConfig(newConfig.getProducerConfig());
}
+ if (newConfig.getPulsarClientMemoryLimit() != null) {
+ mergedConfig.setPulsarClientMemoryLimit(newConfig.getPulsarClientMemoryLimit());
+ }
return mergedConfig;
}
}
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
index eec410c062e4..4e54177c068b 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
@@ -46,6 +46,7 @@ import org.apache.pulsar.client.impl.schema.JSONSchema;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.CryptoConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.functions.WindowConfig;
@@ -62,6 +63,7 @@ import org.apache.pulsar.functions.proto.RetryDetails;
import org.apache.pulsar.functions.proto.SinkSpec;
import org.apache.pulsar.functions.proto.SourceSpec;
import org.apache.pulsar.functions.proto.SubscriptionType;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
/**
@@ -139,6 +141,65 @@ public class FunctionConfigUtilsTest {
);
}
+ @DataProvider(name = "pulsarClientMemoryLimit")
+ public Object[][] pulsarClientMemoryLimit() {
+ return new Object[][] {
+ {new MemoryLimit().setLimitInBytes(1024L)},
+ {new MemoryLimit().setPercentOfMaxDirectMemory(25.0)},
+ {new MemoryLimit().setLimitInBytes(1024L).setPercentOfMaxDirectMemory(25.0)},
+ };
+ }
+
+ @Test(dataProvider = "pulsarClientMemoryLimit")
+ public void testConvertPulsarClientMemoryLimit(MemoryLimit memoryLimit) {
+ FunctionConfig functionConfig = createFunctionConfig();
+ functionConfig.setPulsarClientMemoryLimit(memoryLimit);
+
+ FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig);
+ assertEquals(FunctionConfigUtils.convertFromDetails(functionDetails).getPulsarClientMemoryLimit(),
+ memoryLimit);
+ }
+
+ @Test
+ public void testConvertWithoutPulsarClientMemoryLimit() {
+ FunctionDetails functionDetails = FunctionConfigUtils.convert(createFunctionConfig());
+ assertFalse(functionDetails.hasPulsarClientMemoryLimit());
+ assertNull(FunctionConfigUtils.convertFromDetails(functionDetails).getPulsarClientMemoryLimit());
+ }
+
+ @DataProvider(name = "invalidPulsarClientMemoryLimit")
+ public Object[][] invalidPulsarClientMemoryLimit() {
+ return new Object[][] {
+ {new MemoryLimit().setLimitInBytes(0L)},
+ {new MemoryLimit().setLimitInBytes(-1L)},
+ {new MemoryLimit().setPercentOfMaxDirectMemory(0.0)},
+ {new MemoryLimit().setPercentOfMaxDirectMemory(100.1)},
+ };
+ }
+
+ @Test(dataProvider = "invalidPulsarClientMemoryLimit", expectedExceptions = IllegalArgumentException.class,
+ expectedExceptionsMessageRegExp = "Pulsar client memory limit .*")
+ public void testValidateInvalidPulsarClientMemoryLimit(MemoryLimit memoryLimit) {
+ FunctionConfig functionConfig = createFunctionConfig();
+ functionConfig.setPulsarClientMemoryLimit(memoryLimit);
+ FunctionConfigUtils.doCommonChecks(functionConfig);
+ }
+
+ @Test
+ public void testMergePulsarClientMemoryLimit() {
+ FunctionConfig functionConfig = createFunctionConfig();
+ functionConfig.setPulsarClientMemoryLimit(new MemoryLimit().setLimitInBytes(1024L));
+
+ // an update without a limit keeps the existing one
+ FunctionConfig mergedConfig = FunctionConfigUtils.validateUpdate(functionConfig, createFunctionConfig());
+ assertEquals(mergedConfig.getPulsarClientMemoryLimit(), new MemoryLimit().setLimitInBytes(1024L));
+
+ FunctionConfig newFunctionConfig = createFunctionConfig();
+ newFunctionConfig.setPulsarClientMemoryLimit(new MemoryLimit().setPercentOfMaxDirectMemory(25.0));
+ mergedConfig = FunctionConfigUtils.validateUpdate(functionConfig, newFunctionConfig);
+ assertEquals(mergedConfig.getPulsarClientMemoryLimit(), new MemoryLimit().setPercentOfMaxDirectMemory(25.0));
+ }
+
@SuppressWarnings("deprecation")
@Test
public void testConvertWindow() {
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
index 8023aa349e2f..e5fe9f325841 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
@@ -73,6 +73,7 @@ import org.apache.pulsar.functions.runtime.RuntimeSpawner;
import org.apache.pulsar.functions.utils.Actions;
import org.apache.pulsar.functions.utils.ClientApiResolver;
import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.apache.pulsar.functions.utils.FunctionConfigUtils;
import org.apache.pulsar.functions.utils.SourceConfigUtils;
import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
import org.apache.pulsar.functions.utils.io.Connector;
@@ -237,6 +238,9 @@ public class FunctionActioner {
instanceConfig.setAdditionalJavaRuntimeArguments(workerConfig.getAdditionalJavaRuntimeArguments());
}
instanceConfig.setIgnoreUnknownConfigFields(workerConfig.isIgnoreUnknownConfigFields());
+ instanceConfig.setPulsarClientMemoryLimit(functionDetails.hasPulsarClientMemoryLimit()
+ ? FunctionConfigUtils.convertPulsarClientMemoryLimit(functionDetails.getPulsarClientMemoryLimit())
+ : workerConfig.getPulsarClientMemoryLimit());
return instanceConfig;
}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java
index 5526c1e621cc..a55e64953d6b 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java
@@ -36,8 +36,11 @@ import java.util.Optional;
import org.apache.distributedlog.api.namespace.Namespace;
import org.apache.pulsar.client.admin.Packages;
import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.functions.auth.FunctionAuthProvider;
+import org.apache.pulsar.functions.instance.InstanceConfig;
+import org.apache.pulsar.functions.proto.FunctionDetails;
import org.apache.pulsar.functions.proto.FunctionMetaData;
import org.apache.pulsar.functions.proto.Instance;
import org.apache.pulsar.functions.runtime.Runtime;
@@ -270,4 +273,28 @@ public class FunctionActionerTest {
verify(runtime, times(1)).start();
}
+ @Test
+ public void testCreateInstanceConfigPulsarClientMemoryLimit() {
+ WorkerConfig workerConfig = new WorkerConfig();
+ @SuppressWarnings("resource")
+ FunctionActioner actioner = new FunctionActioner(workerConfig, mock(RuntimeFactory.class),
+ mock(Namespace.class), mock(ConnectorsManager.class), mock(FunctionsManager.class),
+ mock(PulsarAdmin.class), mock(PackageUrlValidator.class));
+
+ // neither the worker nor the function sets a limit
+ InstanceConfig instanceConfig = actioner.createInstanceConfig(new FunctionDetails(), null, 0, "c");
+ assertNull(instanceConfig.getPulsarClientMemoryLimit());
+
+ // the worker default applies to a function without its own limit
+ workerConfig.setPulsarClientMemoryLimit(new MemoryLimit().setPercentOfMaxDirectMemory(50.0));
+ instanceConfig = actioner.createInstanceConfig(new FunctionDetails(), null, 0, "c");
+ assertEquals(instanceConfig.getPulsarClientMemoryLimit(),
+ new MemoryLimit().setPercentOfMaxDirectMemory(50.0));
+
+ // the function's own limit overrides the worker default
+ FunctionDetails functionDetails = new FunctionDetails();
+ functionDetails.setPulsarClientMemoryLimit().setLimitInBytes(1024);
+ instanceConfig = actioner.createInstanceConfig(functionDetails, null, 0, "c");
+ assertEquals(instanceConfig.getPulsarClientMemoryLimit(), new MemoryLimit().setLimitInBytes(1024L));
+ }
}
(pulsar) 09/14: [fix][broker]The partitions of the topic was wrongly created even though this cluster was not allowed to access it (#25443)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit db4ed3fa895cde4ad62263451e372ee2963189af
Author: fengyubiao <yu...@streamnative.io>
AuthorDate: Sat Oct 17 10:17:35 2026 +0800
[fix][broker]The partitions of the topic was wrongly created even though this cluster was not allowed to access it (#25443)
Co-authored-by: Matteo Merli <mm...@apache.org>
(cherry picked from commit 1cd3457c0a52753a5e995b1b4a17e024cf41952f)
---
.../broker/admin/impl/PersistentTopicsBase.java | 50 ++++++++++++++--------
.../pulsar/broker/service/BrokerService.java | 32 ++++++++++++++
.../service/OneWayReplicatorUsingGlobalZKTest.java | 33 ++++++++++++++
3 files changed, 98 insertions(+), 17 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index c1bb46628752..ae5b247ab18b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -55,6 +55,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
@@ -567,23 +568,7 @@ public class PersistentTopicsBase extends AdminResource {
}
protected void internalCreateMissedPartitions(AsyncResponse asyncResponse) {
- getPartitionedTopicMetadataAsync(topicName, false, false).thenAccept(metadata -> {
- if (metadata != null && metadata.partitions > 0) {
- validateNamespaceOperationAsync(topicName.getNamespaceObject(),
- NamespaceOperation.CREATE_TOPIC)
- .thenCompose(__ -> tryCreatePartitionsAsync(metadata.partitions)).thenAccept(v -> {
- asyncResponse.resume(Response.noContent().build());
- }).exceptionally(e -> {
- log.error()
- .attr("topic", topicName)
- .log("Failed to create partitions for topic");
- resumeAsyncResponseExceptionally(asyncResponse, e);
- return null;
- });
- } else {
- throw new RestException(Status.NOT_FOUND, String.format("Topic %s does not exist", topicName));
- }
- }).exceptionally(ex -> {
+ Consumer<Throwable> errorHandler = ex -> {
// If the exception is not redirect exception we need to log it.
if (!isRedirectException(ex)) {
log.error()
@@ -591,6 +576,37 @@ public class PersistentTopicsBase extends AdminResource {
.log("Failed to create partitions for topic");
}
resumeAsyncResponseExceptionally(asyncResponse, ex);
+ };
+ pulsar().getBrokerService().isCurrentClusterAllowed(topicName).thenAccept(allowed -> {
+ if (!allowed) {
+ resumeAsyncResponseExceptionally(asyncResponse,
+ new RestException(Status.BAD_REQUEST, String.format("Topic [%s] is not allowed to be loaded"
+ + " up on the current cluster, please recheck replication polices.", topicName)));
+ return;
+ }
+ getPartitionedTopicMetadataAsync(topicName, false, false).thenAccept(metadata -> {
+ if (metadata != null && metadata.partitions > 0) {
+ validateNamespaceOperationAsync(topicName.getNamespaceObject(),
+ NamespaceOperation.CREATE_TOPIC)
+ .thenCompose(__ -> tryCreatePartitionsAsync(metadata.partitions)).thenAccept(v -> {
+ asyncResponse.resume(Response.noContent().build());
+ }).exceptionally(e -> {
+ log.error()
+ .attr("topic", topicName)
+ .log("Failed to create partitions for topic");
+ resumeAsyncResponseExceptionally(asyncResponse, e);
+ return null;
+ });
+ } else {
+ resumeAsyncResponseExceptionally(asyncResponse,
+ new RestException(Status.NOT_FOUND, String.format("Topic %s does not exist", topicName)));
+ }
+ }).exceptionally(ex -> {
+ errorHandler.accept(ex);
+ return null;
+ });
+ }).exceptionally(ex -> {
+ errorHandler.accept(ex);
return null;
});
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 536f1e6f920c..68259df9a471 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -55,6 +55,7 @@ import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -4832,6 +4833,37 @@ public class BrokerService implements Closeable {
this.pulsarChannelInitFactory = factory;
}
+ /**
+ * Whether the current cluster is allowed to access the topic, by the same rule that is applied when the topic is
+ * loaded: the effective replication clusters (topic level, then global topic level, then namespace level) or the
+ * namespace-level allowed clusters contain the current cluster.
+ */
+ public CompletableFuture<Boolean> isCurrentClusterAllowed(@NonNull TopicName topicName) {
+ final String cluster = pulsar.getConfig().getClusterName();
+ final CompletableFuture<Optional<TopicPolicies>> topicPoliciesFuture =
+ getTopicPoliciesBypassSystemTopic(topicName, TopicPoliciesService.GetType.LOCAL_ONLY);
+ final CompletableFuture<Optional<TopicPolicies>> globalTopicPoliciesFuture =
+ getTopicPoliciesBypassSystemTopic(topicName, TopicPoliciesService.GetType.GLOBAL_ONLY);
+ final CompletableFuture<Optional<Policies>> nsPoliciesFuture = pulsar.getPulsarResources()
+ .getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject());
+ return CompletableFuture.allOf(topicPoliciesFuture, globalTopicPoliciesFuture, nsPoliciesFuture)
+ .thenApply(__ -> {
+ Optional<Policies> nsPolicies = nsPoliciesFuture.join();
+ List<String> topicClusters = topicPoliciesFuture.join()
+ .map(TopicPolicies::getReplicationClusters)
+ .or(() -> globalTopicPoliciesFuture.join().map(TopicPolicies::getReplicationClusters))
+ .orElse(null);
+ Collection<String> replicationClusters = topicClusters != null ? topicClusters
+ : nsPolicies.map(p -> p.replication_clusters).orElse(null);
+ // A topic without replication clusters is loaded without checking the cluster either.
+ if (isEmpty(replicationClusters)) {
+ return true;
+ }
+ return replicationClusters.contains(cluster)
+ || nsPolicies.map(p -> p.allowed_clusters.contains(cluster)).orElse(false);
+ });
+ }
+
/***
* After PIP-321 Introduce allowed-cluster at the namespace level, the condition that whether the cluster is
* allowed to access by the current cluster was defined by two fields:
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java
index 53bea469d366..6ae881126c3d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java
@@ -202,6 +202,15 @@ public class OneWayReplicatorUsingGlobalZKTest extends OneWayReplicatorTest {
});
waitReplicatorStopped(subTopic, pulsar1, pulsar2, true);
+ try {
+ admin2.topics().createMissedPartitions(topicName);
+ fail("The action that creates mission partitions should have thrown exception");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("is not allowed to be loaded up"));
+ }
+ assertFalse(admin2.topics().getList(replicatedNamespace)
+ .contains(TopicName.get(topicName).getPartition(0).toString()));
+
// Remove global policy.
admin1.topicPolicies(true).removeReplicationClusters(topicName);
Producer<byte[]> producer2 = client1.newProducer().topic(topicName).create();
@@ -965,4 +974,28 @@ public class OneWayReplicatorUsingGlobalZKTest extends OneWayReplicatorTest {
p.close();
admin1.topics().delete(topic);
}
+
+ /**
+ * A cluster in the namespace-level allowed clusters can load the topic even when the topic-level replication
+ * clusters do not contain it, so it is allowed to create the missed partitions too.
+ */
+ @Test
+ public void testCreateMissedPartitionsOnAllowedClusterOutsideTopicReplication() throws Exception {
+ final String ns1 = defaultTenant + "/ns_" + UUID.randomUUID().toString().replace("-", "");
+ final String topicName = BrokerTestUtil.newUniqueName("persistent://" + ns1 + "/tp");
+ final String subTopic = TopicName.get(topicName).getPartition(0).toString();
+ admin1.namespaces().createNamespace(ns1);
+ admin1.namespaces().setNamespaceAllowedClusters(ns1, new HashSet<>(Arrays.asList(cluster1, cluster2)));
+ admin1.namespaces().setNamespaceReplicationClusters(ns1,
+ new HashSet<>(Arrays.asList(cluster1, cluster2)), true);
+ admin1.topics().createPartitionedTopic(topicName, 1);
+ admin1.topicPolicies(true).setReplicationClusters(topicName, Arrays.asList(cluster1));
+ Awaitility.await().untilAsserted(() -> {
+ Set<String> global2 = admin2.topicPolicies(true).getReplicationClusters(topicName, false);
+ assertEquals(global2, Collections.singleton(cluster1));
+ });
+
+ admin2.topics().createMissedPartitions(topicName);
+ assertTrue(admin2.topics().getList(ns1).contains(subTopic));
+ }
}
(pulsar) 14/14: [improve][ml] Bound the requeues of size-based cache eviction to one cycle through the removal queue (#26933)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 77a257d4b1fda94968f8c774ccdf584479a5e71f
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Oct 16 23:05:34 2026 -0700
[improve][ml] Bound the requeues of size-based cache eviction to one cycle through the removal queue (#26933)
(cherry picked from commit 23a7fb66989a7e2483d925ad7c16b0904b321315)
---
conf/broker.conf | 15 ++-
conf/standalone.conf | 15 ++-
.../mledger/ManagedLedgerFactoryConfig.java | 15 ++-
.../impl/cache/RangeCacheRemovalCounters.java | 4 +
.../mledger/impl/cache/RangeCacheRemovalQueue.java | 11 +-
.../impl/cache/RangeEntryCacheManagerImpl.java | 3 +-
.../impl/cache/RangeCacheRemovalQueueTest.java | 123 +++++++++++++++++++++
.../apache/pulsar/broker/ServiceConfiguration.java | 17 ++-
8 files changed, 193 insertions(+), 10 deletions(-)
diff --git a/conf/broker.conf b/conf/broker.conf
index 05b954f85828..e5b63a644f2d 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1482,7 +1482,12 @@ cacheEvictionByMarkDeletedPosition=false
# This setting will override the cacheEvictionByMarkDeletedPosition setting.
cacheEvictionByExpectedReadCount=true
-# Whether we should make a copy of the entry payloads when inserting in cache
+# Whether we should make a copy of the entry payloads when inserting in cache. Without a copy, a cached entry keeps
+# the buffer that it was received in allocated while it stays in the cache. That is a buffer of the Netty connection
+# that the entry arrived on, from a client or from a bookie, such as a receive buffer or one that the TLS handler or
+# the frame decoder allocated, and it can be much larger than the entry.
+# The cache size limit counts only the entry's own size, so size-based eviction doesn't bound that memory, and
+# managedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes extends how long it stays allocated.
managedLedgerCacheCopyEntries=false
# Threshold to which bring down the cache level when eviction is triggered
@@ -1509,7 +1514,13 @@ managedLedgerCacheEvictionTimeThresholdMillis=1000
# - Consumers temporarily falling behind the tail
# Entries with remaining expected reads will have their TTL extended up to this many times
# before being eligible for eviction. The TTL will be extended by
-# managedLedgerCacheEvictionTimeThresholdMillis each time.
+# managedLedgerCacheEvictionTimeThresholdMillis each time. When the cache needs memory, size-based eviction
+# overrides this protection after requeuing an entry at most once per eviction pass. Both this protection and the
+# one of managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed apply only in a pass's first cycle through the
+# cache's entries; after it, the pass removes entries in their order in the queue. Entries added while a pass is
+# running are past its first cycle, so that pass doesn't requeue them.
+# Without managedLedgerCacheCopyEntries, a cached entry keeps the larger buffer that it was received in allocated
+# while its time to live is extended, so consider enabling copying before raising this value.
# Default is 5.
managedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes=5
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 4e48a012d182..17fc3df4dd9a 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -976,7 +976,12 @@ cacheEvictionByMarkDeletedPosition=false
# This setting will override the cacheEvictionByMarkDeletedPosition setting.
cacheEvictionByExpectedReadCount=true
-# Whether we should make a copy of the entry payloads when inserting in cache
+# Whether we should make a copy of the entry payloads when inserting in cache. Without a copy, a cached entry keeps
+# the buffer that it was received in allocated while it stays in the cache. That is a buffer of the Netty connection
+# that the entry arrived on, from a client or from a bookie, such as a receive buffer or one that the TLS handler or
+# the frame decoder allocated, and it can be much larger than the entry.
+# The cache size limit counts only the entry's own size, so size-based eviction doesn't bound that memory, and
+# managedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes extends how long it stays allocated.
managedLedgerCacheCopyEntries=false
# Threshold to which bring down the cache level when eviction is triggered
@@ -1003,7 +1008,13 @@ managedLedgerCacheEvictionTimeThresholdMillis=1000
# - Consumers temporarily falling behind the tail
# Entries with remaining expected reads will have their TTL extended up to this many times
# before being eligible for eviction. The TTL will be extended by
-# managedLedgerCacheEvictionTimeThresholdMillis each time.
+# managedLedgerCacheEvictionTimeThresholdMillis each time. When the cache needs memory, size-based eviction
+# overrides this protection after requeuing an entry at most once per eviction pass. Both this protection and the
+# one of managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed apply only in a pass's first cycle through the
+# cache's entries; after it, the pass removes entries in their order in the queue. Entries added while a pass is
+# running are past its first cycle, so that pass doesn't requeue them.
+# Without managedLedgerCacheCopyEntries, a cached entry keeps the larger buffer that it was received in allocated
+# while its time to live is extended, so consider enabling copying before raising this value.
# Default is 5.
managedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes=5
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
index 25fac086fd54..f410fe057e00 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
@@ -66,7 +66,13 @@ public class ManagedLedgerFactoryConfig {
* - Consumers temporarily falling behind the tail
* Entries with remaining expected reads will have their TTL extended up to this many times
* before being eligible for eviction. The TTL will be extended by
- * managedLedgerCacheEvictionTimeThresholdMillis each time.
+ * managedLedgerCacheEvictionTimeThresholdMillis each time. When the cache needs memory, size-based eviction
+ * overrides this protection after requeuing an entry at most once per eviction pass. Both this protection and the
+ * one of cacheEvictionExtendTTLOfRecentlyAccessed apply only in a pass's first cycle through the cache's entries;
+ * after it, the pass removes entries in their order in the queue. Entries added while a pass is running are past
+ * its first cycle, so that pass doesn't requeue them.
+ * Without copyEntriesInCache, a cached entry keeps the larger buffer that it was received in allocated while its
+ * time to live is extended, so consider enabling copying before raising this value.
*/
private int cacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes = 5;
@@ -77,7 +83,12 @@ public class ManagedLedgerFactoryConfig {
private boolean cacheEvictionExtendTTLOfRecentlyAccessed = true;
/**
- * Whether we should make a copy of the entry payloads when inserting in cache.
+ * Whether we should make a copy of the entry payloads when inserting in cache. Without a copy, a cached entry keeps
+ * the buffer that it was received in allocated while it stays in the cache. That is a buffer of the Netty
+ * connection that the entry arrived on, from a client or from a bookie, such as a receive buffer or one that the
+ * TLS handler or the frame decoder allocated, and it can be much larger than the entry.
+ * The cache size limit counts only the entry's own size, so size-based eviction doesn't bound that memory, and
+ * cacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes extends how long it stays allocated.
*/
private boolean copyEntriesInCache = false;
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalCounters.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalCounters.java
index bbbdb9dc619d..ac4fadd4916e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalCounters.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalCounters.java
@@ -34,6 +34,8 @@ class RangeCacheRemovalCounters {
};
int removedEntries;
long removedSize;
+ // the queue slots that an eviction pass has examined, to tell the queue's first cycle from the next ones
+ int evaluatedEntries;
private RangeCacheRemovalCounters(Recycler.Handle<RangeCacheRemovalCounters> recyclerHandle) {
this.recyclerHandle = recyclerHandle;
@@ -43,12 +45,14 @@ class RangeCacheRemovalCounters {
RangeCacheRemovalCounters results = RECYCLER.get();
results.removedEntries = 0;
results.removedSize = 0;
+ results.evaluatedEntries = 0;
return results;
}
void recycle() {
removedEntries = 0;
removedSize = 0;
+ evaluatedEntries = 0;
recyclerHandle.recycle(this);
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalQueue.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalQueue.java
index 46446599bdbc..2b315a567029 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalQueue.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalQueue.java
@@ -83,12 +83,19 @@ class RangeCacheRemovalQueue {
public synchronized Pair<Integer, Long> evictLeastAccessedEntries(long sizeToFree) {
checkArgument(sizeToFree > 0);
+ // A requeued entry goes to the end of the queue, where this pass meets it again. An entry is requeued only in
+ // the pass's first cycle through the queue, so that freeing memory takes at most two cycles, whatever the
+ // maximum requeue count for entries with expected reads is. After the first cycle, entries are removed in
+ // their order in the queue, so an entry with expected reads ahead of a recently accessed entry goes first.
+ // Inserting into the cache doesn't wait for a running pass, so entries added during the pass land behind
+ // the first cycle and are removed without a requeue when the pass reaches them.
+ int firstCycleEntries = removalQueue.size();
return evictEntries((e, c) -> {
// stop eviction if we have already removed enough entries
if (c.removedSize >= sizeToFree) {
return EvictionResult.STOP;
}
- if (shouldRequeue(e)) {
+ if (c.evaluatedEntries <= firstCycleEntries && shouldRequeue(e)) {
return EvictionResult.REQUEUE;
}
// remove by default since there's a need to free up more memory
@@ -189,6 +196,8 @@ class RangeCacheRemovalQueue {
private static EvictionResult evaluateEvictionPredicate(EvictionPredicate evictionPredicate,
RangeCacheRemovalCounters counters,
RangeCacheEntryWrapper entry) {
+ // every examined slot of the queue counts, also an entry that was already removed from the cache
+ counters.evaluatedEntries++;
if (entry.key == null) {
// entry has been removed by another thread
return EvictionResult.MISSING;
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
index 39d90bb189e5..7e760004a171 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
@@ -136,7 +136,8 @@ public class RangeEntryCacheManagerImpl implements EntryCacheManager {
Optional<CompletableFuture<Void>> triggerEvictionWhenNeeded() {
long currentSize = this.currentSize.get();
- // Trigger a single eviction in background. While the eviction is running we stop inserting entries in the cache
+ // Trigger a single eviction in background. While it is running, a new trigger returns the running eviction's
+ // future instead of starting another one. Inserting entries in the cache doesn't wait for it.
if (currentSize > evictionTriggerThreshold) {
CompletableFuture<Void> evictionCompletionFuture = null;
while (evictionCompletionFuture == null) {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalQueueTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalQueueTest.java
index 35b7d548e3cc..b059125d6225 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalQueueTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheRemovalQueueTest.java
@@ -28,7 +28,10 @@ import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
+import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.bookkeeper.mledger.Position;
import org.apache.bookkeeper.mledger.PositionFactory;
import org.apache.bookkeeper.mledger.impl.EntryImpl;
@@ -174,6 +177,105 @@ public class RangeCacheRemovalQueueTest {
assertThat(e3.key).isNull();
}
+ @Test
+ public void testSizeBasedEvictionRequeuesAtMostOncePerPass() {
+ // A high requeue limit for entries with expected reads keeps them in the cache longer on time-based eviction,
+ // but size-based eviction must still free memory in one pass, without cycling through the queue up to the
+ // limit
+ RangeCacheRemovalQueue q = new RangeCacheRemovalQueue(
+ /*maxRequeueCountWhenHasExpectedReads*/ 1000,
+ /*extendTTLOfRecentlyAccessed*/ true);
+ List<RangeCacheEntryWrapper> entries = new ArrayList<>();
+ for (long entryId = 0; entryId < 100; entryId++) {
+ RangeCacheEntryWrapper e = addWrappedEntry(q, 12L, entryId, 2 /* has expected reads */, 10);
+ e.accessed = true;
+ entries.add(e);
+ }
+
+ Pair<Integer, Long> result = q.evictLeastAccessedEntries(500);
+
+ assertThat(result.getRight().longValue()).isEqualTo(500L);
+ assertThat(entries.stream().filter(e -> e.key == null).count()).isEqualTo(50);
+ // every remaining entry was requeued once, in the pass's first cycle, before the second cycle removed the
+ // oldest
+ assertThat(entries.stream().filter(e -> e.key != null).mapToInt(e -> e.requeueCount))
+ .hasSize(50).containsOnly(1);
+
+ // the remaining entries keep their other extensions: time-based eviction requeues them instead of removing
+ // them, although they have expired
+ long now = System.nanoTime();
+ entries.stream().filter(e -> e.key != null).forEach(e -> e.timestampNanos = now - 10_000_000L);
+ Pair<Integer, Long> timeBasedResult = q.evictLEntriesBeforeTimestamp(now - 1L);
+ assertThat(timeBasedResult.getLeft()).isZero();
+ assertThat(entries.stream().filter(e -> e.key != null).mapToInt(e -> e.requeueCount))
+ .hasSize(50).containsOnly(2);
+ }
+
+ @Test
+ public void testSizeBasedEvictionReleasesBuffersOfEntriesWithExpectedReads() {
+ // As testSizeBasedEvictionRequeuesAtMostOncePerPass, with a real cache and buffers: the entries that the
+ // pass's second cycle removes are removed from the cache, and their buffers are released
+ RangeCacheRemovalQueue q = new RangeCacheRemovalQueue(
+ /*maxRequeueCountWhenHasExpectedReads*/ 1000,
+ /*extendTTLOfRecentlyAccessed*/ false);
+ RangeCache cache = new RangeCache(q);
+ List<ByteBuf> buffers = new ArrayList<>();
+ for (long entryId = 0; entryId < 100; entryId++) {
+ ByteBuf buffer = Unpooled.directBuffer(10).writeZero(10);
+ Position position = PositionFactory.create(14L, entryId);
+ // the entry retains the buffer, and the cache takes the entry's reference
+ assertTrue(cache.put(position, EntryImpl.create(position, buffer, 2 /* has expected reads */)));
+ buffer.release();
+ buffers.add(buffer);
+ }
+ try {
+ assertThat(cache.getSize()).isEqualTo(1000L);
+
+ Pair<Integer, Long> result = q.evictLeastAccessedEntries(500);
+
+ assertThat(result.getLeft()).isEqualTo(50);
+ assertThat(result.getRight()).isEqualTo(500L);
+ assertThat(cache.getSize()).isEqualTo(500L);
+ assertThat(cache.getNumberOfEntries()).isEqualTo(50);
+ // the oldest half was requeued once and removed in the second cycle
+ for (int i = 0; i < 100; i++) {
+ Position position = PositionFactory.create(14L, i);
+ if (i < 50) {
+ assertThat(cache.exists(position)).as("entry %d", i).isFalse();
+ assertThat(buffers.get(i).refCnt()).as("entry %d", i).isZero();
+ } else {
+ assertThat(cache.exists(position)).as("entry %d", i).isTrue();
+ assertThat(buffers.get(i).refCnt()).as("entry %d", i).isEqualTo(1);
+ }
+ }
+ } finally {
+ cache.clear();
+ }
+ assertThat(buffers).allSatisfy(buffer -> assertThat(buffer.refCnt()).isZero());
+ }
+
+ @Test
+ public void testSizeBasedEvictionCountsRemovedEntriesInTheFirstCycle() {
+ // Entries that were removed from the cache stay in the queue until a pass meets them. They count toward the
+ // pass's first cycle, so that the live entries behind them are requeued only once.
+ RangeCacheRemovalQueue q = new RangeCacheRemovalQueue(
+ /*maxRequeueCountWhenHasExpectedReads*/ 1000,
+ /*extendTTLOfRecentlyAccessed*/ false);
+ for (long entryId = 0; entryId < 98; entryId++) {
+ RangeCacheEntryWrapper removed = addWrappedEntry(q, 13L, entryId, 0, 10);
+ removed.markRemoved(removed.key, removed.value);
+ }
+ RangeCacheEntryWrapper first = addWrappedEntry(q, 13L, 98L, 2 /* has expected reads */, 10);
+ RangeCacheEntryWrapper second = addWrappedEntry(q, 13L, 99L, 2 /* has expected reads */, 10);
+
+ Pair<Integer, Long> result = q.evictLeastAccessedEntries(10);
+
+ assertThat(result.getRight().longValue()).isEqualTo(10L);
+ assertThat(first.key).isNull();
+ assertThat(second.key).isNotNull();
+ assertThat(second.requeueCount).isEqualTo(1);
+ }
+
@Test
public void testSizeBasedEvictionEvictsEasiestFirstAsLongAsNeeded() {
RangeCacheRemovalQueue q = new RangeCacheRemovalQueue(
@@ -216,6 +318,27 @@ public class RangeCacheRemovalQueueTest {
assertThat(e3.key).isNull();
}
+ @Test
+ public void testSizeBasedEvictionRemovesInQueueOrderAfterTheFirstCycle() {
+ // Both protections apply only in the pass's first cycle through the queue. After it, entries are removed in
+ // their order in the queue, so an entry with expected reads ahead of a recently accessed entry goes first.
+ RangeCacheRemovalQueue q = new RangeCacheRemovalQueue(
+ /*maxRequeueCountWhenHasExpectedReads*/ 5,
+ /*extendTTLOfRecentlyAccessed*/ true);
+
+ RangeCacheEntryWrapper e1 = addWrappedEntry(q, 11L, 1L, 2, 50); // has expected reads
+ RangeCacheEntryWrapper e2 = addWrappedEntry(q, 11L, 2L, 0, 40); // recently accessed
+ e2.accessed = true;
+ RangeCacheEntryWrapper e3 = addWrappedEntry(q, 11L, 3L, 0, 30); // easiest to evict
+
+ Pair<Integer, Long> r1 = q.evictLeastAccessedEntries(35);
+ assertNotNull(r1);
+ assertThat(r1.getRight().longValue()).isEqualTo(80L);
+ assertThat(e1.key).isNull();
+ assertThat(e2.key).isNotNull();
+ assertThat(e3.key).isNull();
+ }
+
@Test
public void testGetNonEvictableSizeCountsEntriesWithExpectedReads() {
RangeCacheRemovalQueue q = new RangeCacheRemovalQueue(
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 1db070a76dff..ec8fe5780840 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2635,7 +2635,13 @@ public class ServiceConfiguration implements PulsarConfiguration {
(int) (DirectMemoryUtils.jvmMaxDirectMemory() / 5 / (1024 * 1024)));
@FieldContext(category = CATEGORY_STORAGE_ML, doc = "Whether we should make a copy of the entry payloads when "
- + "inserting in cache")
+ + "inserting in cache. Without a copy, a cached entry keeps the buffer that it was received in allocated\n"
+ + "while it stays in the cache. That is a buffer of the Netty connection that the entry arrived on, from\n"
+ + "a client or from a bookie, such as a receive buffer or one that the TLS handler or the frame decoder\n"
+ + "allocated, and it can be much larger than the entry. The cache size limit counts only\n"
+ + "the entry's own size, so size-based eviction doesn't bound that memory, and\n"
+ + "managedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes extends how long it\n"
+ + "stays allocated.")
private boolean managedLedgerCacheCopyEntries = false;
@FieldContext(category = CATEGORY_STORAGE_ML, doc = "Maximum buffer size in MB for bytes read from storage"
@@ -2695,7 +2701,14 @@ public class ServiceConfiguration implements PulsarConfiguration {
+ "- Consumers temporarily falling behind the tail\n"
+ "Entries with remaining expected reads will have their TTL extended up to this many times\n"
+ "before being eligible for eviction. The TTL will be extended by\n"
- + "managedLedgerCacheEvictionTimeThresholdMillis each time.\n"
+ + "managedLedgerCacheEvictionTimeThresholdMillis each time. When the cache needs memory, size-based\n"
+ + "eviction overrides this protection after requeuing an entry at most once per eviction pass. Both this\n"
+ + "protection and the one of managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed apply only in a\n"
+ + "pass's first cycle through the cache's entries; after it, the pass removes entries in their order in\n"
+ + "the queue. Entries added while a pass is running are past its first cycle, so that pass doesn't\n"
+ + "requeue them.\n"
+ + "Without managedLedgerCacheCopyEntries, a cached entry keeps the larger buffer that it was received in\n"
+ + "allocated while its time to live is extended, so consider enabling copying before raising this value.\n"
+ "Default is 5.")
private int managedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes = 5;
(pulsar) 04/14: [fix][broker] Authorize scalable topic segments with the permissions of the parent topic (#26949)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 6668da0b53e9240d71cd74c04869aba5134fc4ea
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Fri Oct 16 17:54:07 2026 -0700
[fix][broker] Authorize scalable topic segments with the permissions of the parent topic (#26949)
(cherry picked from commit 745b216475c6a705d2096acf58676b468263aeb0)
---
pip/pip-460.md | 2 +-
pip/pip-466.md | 6 +-
pip/pip-468.md | 5 +-
.../authorization/PulsarAuthorizationProvider.java | 12 +-
.../PulsarAuthorizationProviderTest.java | 87 +++++
.../pulsar/broker/admin/v2/ScalableTopics.java | 188 ++++++++++-
.../broker/admin/ScalableTopicPermissionsTest.java | 370 +++++++++++++++++++++
.../broker/admin/ScalableTopicsAuthZTest.java | 52 ++-
.../apache/pulsar/client/admin/ScalableTopics.java | 83 +++++
.../client/admin/internal/ScalableTopicsImpl.java | 55 +++
.../admin/internal/ScalableTopicsImplTest.java | 50 +++
.../apache/pulsar/admin/cli/CmdScalableTopics.java | 51 +++
.../pulsar/admin/cli/TestCmdScalableTopics.java | 19 ++
spec/scalable-topics/compatibility.md | 12 +-
spec/scalable-topics/wire-protocol.md | 3 +-
15 files changed, 971 insertions(+), 24 deletions(-)
diff --git a/pip/pip-460.md b/pip/pip-460.md
index 2dbb25af74fa..89336f9df491 100644
--- a/pip/pip-460.md
+++ b/pip/pip-460.md
@@ -373,7 +373,7 @@ Geo-replication for scalable topics is a new protocol path. Clusters participati
## Security Considerations
-Scalable topics follow the same tenant/namespace/topic authorization model as existing Pulsar topics. The new admin API endpoints for segment operations (split, merge, metadata read) will require the same permissions as equivalent topic-level admin operations.
+Scalable topics follow the same tenant/namespace/topic authorization model as existing Pulsar topics. Topic-level permissions are granted on the scalable topic's `topic://` name and apply to all its segments: the broker authorizes the per-segment producers and consumers, which attach to `segment://` names, with the permissions of the parent topic (and, as for any topic, the permissions of its namespace and the tenant admin roles). The new admin API endpoints for segment operations requir [...]
The watch session and the consumer controller stream are authenticated using the same mechanisms as existing Pulsar binary protocol connections (TLS, token authentication, mutual TLS).
diff --git a/pip/pip-466.md b/pip/pip-466.md
index a8781e73514a..af494a5e3355 100644
--- a/pip/pip-466.md
+++ b/pip/pip-466.md
@@ -406,8 +406,10 @@ and authorization to the underlying v4 client:
- Authentication is configured via `PulsarClientBuilder.authentication()` and delegated to the
v4 `AuthenticationProvider` framework via `AuthenticationAdapter`
-- Topic-level authorization applies to the parent `topic://` name — accessing the underlying
- `segment://` topics uses the same tenant/namespace permissions
+- Topic-level authorization applies to the parent `topic://` name — the broker authorizes access to
+ the underlying `segment://` topics as access to the parent topic (its own and its namespace's
+ permissions, and the tenant admin roles), so a permission granted on the scalable topic covers all
+ its segments
- End-to-end encryption is supported via `EncryptionPolicy` on `ProducerBuilder`, delegated to
the v4 `CryptoKeyReader` framework via `CryptoKeyReaderAdapter`
- The new `CommandScalableTopicLookup` protocol command is sent only after the connection is
diff --git a/pip/pip-468.md b/pip/pip-468.md
index 3cb9e4f32934..1fef35a20c9b 100644
--- a/pip/pip-468.md
+++ b/pip/pip-468.md
@@ -599,10 +599,11 @@ The existing Pulsar client SDK rejects `topic://` and `segment://` domains with
The controller follows the same tenant/namespace/topic authorization model as existing Pulsar topics:
-- Admin API operations require topic-level admin permissions.
+- Admin API operations require the same permissions as the equivalent operations on a regular topic. Split, merge and rebucket require the permission to change the partition count of a partitioned topic (`PolicyName.PARTITION` write, which the default authorization provider grants to tenant admins).
- DAG watch sessions require topic-level lookup permissions.
- Consumer registration requires topic-level consume permissions.
-- Segment operations are internal and authenticated via the broker's internal admin client.
+- The per-segment producers and consumers are authorized with the permissions of the parent `topic://`, so a permission granted on the scalable topic covers all its segments.
+- Segment operations are internal and authenticated via the broker's internal admin client; their admin endpoints require super-user access.
The controller leader lock in the metadata store is accessible only to authenticated brokers.
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
index 0b8681efb639..ab10e7ce3a7a 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
@@ -52,6 +52,7 @@ import org.apache.pulsar.common.policies.data.PolicyOperation;
import org.apache.pulsar.common.policies.data.SubscriptionAuthMode;
import org.apache.pulsar.common.policies.data.TenantOperation;
import org.apache.pulsar.common.policies.data.TopicOperation;
+import org.apache.pulsar.common.scalable.SegmentTopicName;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.RestException;
import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException;
@@ -538,6 +539,10 @@ public class PulsarAuthorizationProvider implements AuthorizationProvider {
}
public CompletableFuture<Boolean> checkPermission(TopicName topicName, String role, AuthAction action) {
+ // A segment of a scalable topic has no permissions of its own: it is authorized with the
+ // topic-level permissions granted on its parent topic://.
+ TopicName permissionsTopicName =
+ topicName.isSegment() ? SegmentTopicName.getParentTopicName(topicName) : topicName;
return pulsarResources.getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject())
.thenApply(policies -> {
if (!policies.isPresent()) {
@@ -552,7 +557,7 @@ public class PulsarAuthorizationProvider implements AuthorizationProvider {
}
Map<String, Set<AuthAction>> topicRoles = policies.get().auth_policies.getTopicAuthentication()
- .get(topicName.toString());
+ .get(permissionsTopicName.toString());
if (topicRoles != null && role != null) {
// Topic has custom policy
Set<AuthAction> topicActions = topicRoles.get(role);
@@ -579,9 +584,10 @@ public class PulsarAuthorizationProvider implements AuthorizationProvider {
// the new sub partitions may not inherit the policy of the partition topic.
// We can also check the permission of partitioned topic.
// For https://github.com/apache/pulsar/issues/10300
- if (topicName.isPartitioned()) {
+ // A scalable topic has no partitions: a "-partition-N" suffix is part of its name.
+ if (permissionsTopicName.isPartitioned() && !permissionsTopicName.isScalable()) {
topicRoles = policies.get().auth_policies
- .getTopicAuthentication().get(topicName.getPartitionedTopicName());
+ .getTopicAuthentication().get(permissionsTopicName.getPartitionedTopicName());
if (topicRoles != null) {
// Topic has custom policy
Set<AuthAction> topicActions = topicRoles.get(role);
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java
index fed32e74e74b..6396f9e73bf4 100644
--- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java
@@ -18,21 +18,29 @@
*/
package org.apache.pulsar.broker.authorization;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
import static org.testng.Assert.expectThrows;
import jakarta.ws.rs.core.Response;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import org.apache.logging.log4j.Level;
import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.resources.NamespaceResources;
import org.apache.pulsar.broker.resources.PulsarResources;
import org.apache.pulsar.broker.resources.TenantResources;
import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.apache.pulsar.common.policies.data.Policies;
import org.apache.pulsar.common.policies.data.TenantInfo;
import org.apache.pulsar.common.policies.data.TopicOperation;
import org.apache.pulsar.common.util.RestException;
@@ -43,6 +51,7 @@ public class PulsarAuthorizationProviderTest {
private static final String MISSING_TENANT = "non-existent-tenant";
private static final String ROLE = "some-role";
+ private static final TopicName SEGMENT = TopicName.get("segment://tenant/ns/orders/0000-7fff-1");
private static PulsarAuthorizationProvider providerReturning(
CompletableFuture<Optional<TenantInfo>> tenantLookup) throws Exception {
@@ -122,4 +131,82 @@ public class PulsarAuthorizationProviderTest {
"A metadata store failure must still be logged at ERROR");
}
}
+
+ private static PulsarAuthorizationProvider providerWithPolicies(Policies policies, ServiceConfiguration conf)
+ throws Exception {
+ NamespaceResources namespaceResources = mock(NamespaceResources.class);
+ when(namespaceResources.getPoliciesAsync(any()))
+ .thenReturn(CompletableFuture.completedFuture(Optional.of(policies)));
+ PulsarResources pulsarResources = mock(PulsarResources.class);
+ when(pulsarResources.getNamespaceResources()).thenReturn(namespaceResources);
+
+ PulsarAuthorizationProvider provider = new PulsarAuthorizationProvider();
+ provider.initialize(conf, pulsarResources);
+ return provider;
+ }
+
+ private static Policies policiesGrantingOnTopic(String topic, String role, AuthAction action) {
+ Policies policies = new Policies();
+ policies.auth_policies.getTopicAuthentication()
+ .computeIfAbsent(topic, __ -> new HashMap<>())
+ .put(role, Set.of(action));
+ return policies;
+ }
+
+ /** A segment of a scalable topic is authorized with the permissions granted on its parent topic://. */
+ @Test
+ public void testSegmentIsAuthorizedWithParentTopicPermissions() throws Exception {
+ PulsarAuthorizationProvider provider = providerWithPolicies(
+ policiesGrantingOnTopic("topic://tenant/ns/orders", ROLE, AuthAction.produce),
+ new ServiceConfiguration());
+
+ assertTrue(provider.canProduceAsync(SEGMENT, ROLE, null).get());
+ assertTrue(provider.canLookupAsync(SEGMENT, ROLE, null).get());
+ assertFalse(provider.canConsumeAsync(SEGMENT, ROLE, null, null).get(),
+ "Only produce was granted on the parent topic");
+ assertFalse(provider.canProduceAsync(SEGMENT, "other-role", null).get());
+ assertFalse(provider.canProduceAsync(
+ TopicName.get("segment://tenant/ns/payments/0000-7fff-1"), ROLE, null).get(),
+ "A segment of another scalable topic must not get the permissions of this one");
+ }
+
+ @Test
+ public void testSegmentIsAuthorizedWithNamespacePermissions() throws Exception {
+ Policies policies = new Policies();
+ policies.auth_policies.getNamespaceAuthentication().put(ROLE, Set.of(AuthAction.consume));
+ PulsarAuthorizationProvider provider = providerWithPolicies(policies, new ServiceConfiguration());
+
+ assertTrue(provider.canConsumeAsync(SEGMENT, ROLE, null, null).get());
+ assertFalse(provider.canProduceAsync(SEGMENT, ROLE, null).get());
+ }
+
+ @Test
+ public void testSegmentIsAuthorizedWithParentTopicWildcardPermissions() throws Exception {
+ ServiceConfiguration conf = new ServiceConfiguration();
+ conf.setAuthorizationAllowWildcardsMatching(true);
+ PulsarAuthorizationProvider provider = providerWithPolicies(
+ policiesGrantingOnTopic("topic://tenant/ns/orders", "app-*", AuthAction.produce), conf);
+
+ assertTrue(provider.canProduceAsync(SEGMENT, "app-1", null).get());
+ assertFalse(provider.canProduceAsync(SEGMENT, "other-role", null).get());
+ }
+
+ /**
+ * A scalable topic has no partitions: unlike a partition of a partitioned topic, a scalable topic whose name
+ * ends with "-partition-N" is a topic of its own, and the permissions of the name without the suffix don't
+ * apply to it or to its segments.
+ */
+ @Test
+ public void testScalableTopicNameWithPartitionSuffixIsNotAPartition() throws Exception {
+ Policies policies = policiesGrantingOnTopic("topic://tenant/ns/orders", ROLE, AuthAction.produce);
+ policies.auth_policies.getTopicAuthentication()
+ .put("persistent://tenant/ns/orders", Map.of(ROLE, Set.of(AuthAction.produce)));
+ PulsarAuthorizationProvider provider = providerWithPolicies(policies, new ServiceConfiguration());
+
+ assertFalse(provider.canProduceAsync(TopicName.get("topic://tenant/ns/orders-partition-1"), ROLE, null).get());
+ assertFalse(provider.canProduceAsync(
+ TopicName.get("segment://tenant/ns/orders-partition-1/0000-ffff-0"), ROLE, null).get());
+ assertTrue(provider.canProduceAsync(TopicName.get("persistent://tenant/ns/orders-partition-1"), ROLE, null)
+ .get(), "A partition of a partitioned topic still gets the permissions of the partitioned topic");
+ }
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
index 388eacd8e5a9..39033688ac89 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
@@ -49,6 +49,7 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
import lombok.CustomLog;
@@ -63,6 +64,7 @@ import org.apache.pulsar.broker.web.RestException;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.common.naming.TopicDomain;
import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.AuthAction;
import org.apache.pulsar.common.policies.data.AutoScalePolicyOverride;
import org.apache.pulsar.common.policies.data.NamespaceOperation;
import org.apache.pulsar.common.policies.data.PolicyName;
@@ -82,7 +84,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreException;
* Admin REST API for scalable topics (topic:// domain).
*
* <p>Provides operations to create, delete, list, and inspect scalable topics,
- * as well as segment split/merge operations.
+ * to manage their permissions, as well as segment split/merge operations.
*/
@CustomLog
@Path("/scalable")
@@ -640,6 +642,151 @@ public class ScalableTopics extends AdminResource {
});
}
+ // --- Permissions ---
+
+ @GET
+ @Path("/{tenant}/{namespace}/{topic}/permissions")
+ @Operation(summary = "Get permissions on a scalable topic.",
+ description = "Retrieve the effective permissions for a scalable topic. These permissions are defined"
+ + " by the permissions set at the namespace level combined (union) with any eventual specific"
+ + " permission set on the topic, and apply to every segment of the topic."
+ + " Returns a map structure: Map<String, Set<AuthAction>>.")
+ @ApiResponses(value = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Get permissions on a scalable topic.",
+ content = @Content(schema = @Schema(type = "object"),
+ additionalPropertiesArraySchema = @ArraySchema(
+ schema = @Schema(implementation = AuthAction.class), uniqueItems = true))),
+ @ApiResponse(responseCode = "401",
+ description = "Don't have permission to administrate resources on this tenant"),
+ @ApiResponse(responseCode = "403", description = "Don't have admin permission"),
+ @ApiResponse(responseCode = "404", description = "Tenant, namespace or scalable topic doesn't exist"),
+ @ApiResponse(responseCode = "500", description = "Internal server error")})
+ public void getPermissions(
+ @Suspended final AsyncResponse asyncResponse,
+ @Parameter(description = "Specify the tenant", required = true)
+ @PathParam("tenant") String tenant,
+ @Parameter(description = "Specify the namespace", required = true)
+ @PathParam("namespace") String namespace,
+ @Parameter(description = "Specify topic name", required = true)
+ @PathParam("topic") @Encoded String encodedTopic) {
+ validateNamespaceName(tenant, namespace);
+ TopicName tn = TopicName.get(TopicDomain.topic.value(), namespaceName, Codec.decode(encodedTopic));
+
+ validateAdminAccessForTenantAsync(tenant)
+ .thenCompose(__ -> checkTopicExistsForPermissionsAsync(tn))
+ .thenCompose(__ -> getAuthorizationService().getPermissionsAsync(tn))
+ .thenAccept(asyncResponse::resume)
+ .exceptionally(ex -> {
+ if (isNot307And404Exception(ex)) {
+ log.error().attr("clientAppId", clientAppId()).attr("topic", tn)
+ .exception(ex).log("Failed to get permissions on scalable topic");
+ }
+ resumeAsyncResponseExceptionally(asyncResponse, ex);
+ return null;
+ });
+ }
+
+ @POST
+ @Path("/{tenant}/{namespace}/{topic}/permissions/{role}")
+ @Operation(summary = "Grant a new permission to a role on a scalable topic.",
+ description = "The permission applies to every segment of the topic.")
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "204", description = "Operation successful"),
+ @ApiResponse(responseCode = "401",
+ description = "Don't have permission to administrate resources on this tenant"),
+ @ApiResponse(responseCode = "403", description = "Don't have admin permission"),
+ @ApiResponse(responseCode = "404", description = "Tenant, namespace or scalable topic doesn't exist"),
+ @ApiResponse(responseCode = "409", description = "Concurrent modification"),
+ @ApiResponse(responseCode = "500", description = "Internal server error")})
+ public void grantPermissions(
+ @Suspended final AsyncResponse asyncResponse,
+ @Parameter(description = "Specify the tenant", required = true)
+ @PathParam("tenant") String tenant,
+ @Parameter(description = "Specify the namespace", required = true)
+ @PathParam("namespace") String namespace,
+ @Parameter(description = "Specify topic name", required = true)
+ @PathParam("topic") @Encoded String encodedTopic,
+ @Parameter(description = "Client role to which grant permissions", required = true)
+ @PathParam("role") String role,
+ @RequestBody(description = "Actions to be granted (produce,functions,consume)",
+ content = @Content(schema = @Schema(allowableValues = {"produce", "functions", "consume"})))
+ Set<AuthAction> actions) {
+ validateNamespaceName(tenant, namespace);
+ TopicName tn = TopicName.get(TopicDomain.topic.value(), namespaceName, Codec.decode(encodedTopic));
+
+ validateAdminAccessForTenantAsync(tenant)
+ .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync())
+ .thenCompose(__ -> checkTopicExistsForPermissionsAsync(tn))
+ .thenCompose(__ -> getAuthorizationService().grantPermissionAsync(tn, actions, role, null))
+ .thenAccept(__ -> asyncResponse.resume(Response.noContent().build()))
+ .exceptionally(ex -> {
+ if (isNot307And404Exception(ex)) {
+ log.error().attr("clientAppId", clientAppId()).attr("topic", tn).attr("role", role)
+ .exception(ex).log("Failed to grant permissions on scalable topic");
+ }
+ resumeAsyncResponseExceptionally(asyncResponse, ex);
+ return null;
+ });
+ }
+
+ @DELETE
+ @Path("/{tenant}/{namespace}/{topic}/permissions/{role}")
+ @Operation(summary = "Revoke permissions on a scalable topic.",
+ description = "Revoke the permissions granted to a role on the topic itself. Permissions granted at"
+ + " the namespace level are not affected.")
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "204", description = "Operation successful"),
+ @ApiResponse(responseCode = "401",
+ description = "Don't have permission to administrate resources on this tenant"),
+ @ApiResponse(responseCode = "403", description = "Don't have admin permission"),
+ @ApiResponse(responseCode = "404", description = "Tenant, namespace or scalable topic doesn't exist"),
+ @ApiResponse(responseCode = "409", description = "Concurrent modification"),
+ @ApiResponse(responseCode = "500", description = "Internal server error")})
+ public void revokePermissions(
+ @Suspended final AsyncResponse asyncResponse,
+ @Parameter(description = "Specify the tenant", required = true)
+ @PathParam("tenant") String tenant,
+ @Parameter(description = "Specify the namespace", required = true)
+ @PathParam("namespace") String namespace,
+ @Parameter(description = "Specify topic name", required = true)
+ @PathParam("topic") @Encoded String encodedTopic,
+ @Parameter(description = "Client role to which revoke permissions", required = true)
+ @PathParam("role") String role) {
+ validateNamespaceName(tenant, namespace);
+ TopicName tn = TopicName.get(TopicDomain.topic.value(), namespaceName, Codec.decode(encodedTopic));
+
+ validateAdminAccessForTenantAsync(tenant)
+ .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync())
+ .thenCompose(__ -> checkTopicExistsForPermissionsAsync(tn))
+ .thenCompose(__ -> getAuthorizationService().revokePermissionAsync(tn, role))
+ .thenAccept(__ -> asyncResponse.resume(Response.noContent().build()))
+ .exceptionally(ex -> {
+ if (isNot307And404Exception(ex)) {
+ log.error().attr("clientAppId", clientAppId()).attr("topic", tn).attr("role", role)
+ .exception(ex).log("Failed to revoke permissions on scalable topic");
+ }
+ resumeAsyncResponseExceptionally(asyncResponse, ex);
+ return null;
+ });
+ }
+
+ /**
+ * Like the permission operations on a regular topic, fail with a 404 if the scalable topic doesn't exist,
+ * unless {@code allowAclChangesOnNonExistentTopics} is set.
+ */
+ private CompletableFuture<Void> checkTopicExistsForPermissionsAsync(TopicName tn) {
+ if (pulsar().getConfiguration().isAllowAclChangesOnNonExistentTopics()) {
+ return CompletableFuture.completedFuture(null);
+ }
+ return resources().scalableTopicExistsAsync(tn).thenAccept(exists -> {
+ if (!exists) {
+ throw new RestException(Response.Status.NOT_FOUND, "Scalable topic not found: " + tn);
+ }
+ });
+ }
+
// --- Delete ---
@DELETE
@@ -681,14 +828,18 @@ public class ScalableTopics extends AdminResource {
// leader election re-acquires the lock the delete removes, under the deleted
// path, and it keeps serving consumers from the deleted layout. Mark the
// topic deleted first, so the controllers following it on other brokers
- // don't take the lock over once it is released. Then delete metadata, then
- // best-effort clean up segment topics and the schema they share (deleting a
- // segment topic leaves the schema in place). The segments are read again
- // after the release, to include those of a split that was still in flight.
+ // don't take the lock over once it is released. Then remove the permissions
+ // granted on the topic, as deleting a regular topic does, so a topic created
+ // again with the same name doesn't inherit them (a failure leaves the topic
+ // marked, for a retried delete). Then delete metadata, then best-effort clean
+ // up segment topics and the schema they share (deleting a segment topic
+ // leaves the schema in place). The segments are read again after the
+ // release, to include those of a split that was still in flight.
return resources().markScalableTopicDeletedAsync(tn)
.thenCompose(__ -> releaseController(tn))
.thenCompose(__ -> resources().getScalableTopicMetadataAsync(tn, true))
- .thenCompose(latest -> resources().deleteScalableTopicAsync(tn)
+ .thenCompose(latest -> getAuthorizationService().removePermissionsAsync(tn)
+ .thenCompose(__ -> resources().deleteScalableTopicAsync(tn))
.thenCompose(__ -> deleteSegmentTopics(tn, latest.orElse(optMd.get()),
force)))
.thenCompose(__ -> pulsar().getBrokerService().deleteSchema(tn)
@@ -1004,6 +1155,9 @@ public class ScalableTopics extends AdminResource {
@Operation(summary = "Split a segment into two halves.")
@ApiResponses(value = {
@ApiResponse(responseCode = "204", description = "Segment split successfully"),
+ @ApiResponse(responseCode = "401",
+ description = "Don't have permission to administrate resources on this tenant"),
+ @ApiResponse(responseCode = "403", description = "Don't have admin permission"),
@ApiResponse(responseCode = "404", description = "Scalable topic or segment doesn't exist"),
@ApiResponse(responseCode = "412", description = "Segment is not active or cannot be split"),
@ApiResponse(responseCode = "500", description = "Internal server error")})
@@ -1020,7 +1174,7 @@ public class ScalableTopics extends AdminResource {
validateNamespaceName(tenant, namespace);
TopicName tn = TopicName.get(TopicDomain.topic.value(), namespaceName, encodedTopic);
- validateSuperUserAccessAsync()
+ validateSegmentLayoutOperationAsync(tn)
.thenCompose(__ -> onControllerLeader(tn, svc -> svc.splitSegment(tn, segmentId)))
.thenAccept(__ -> {
log.info().attr("clientAppId", clientAppId())
@@ -1043,6 +1197,9 @@ public class ScalableTopics extends AdminResource {
+ "entry-bucket count.")
@ApiResponses(value = {
@ApiResponse(responseCode = "204", description = "Segment rebucketed successfully"),
+ @ApiResponse(responseCode = "401",
+ description = "Don't have permission to administrate resources on this tenant"),
+ @ApiResponse(responseCode = "403", description = "Don't have admin permission"),
@ApiResponse(responseCode = "404", description = "Scalable topic doesn't exist"),
@ApiResponse(responseCode = "412", description = "Segment is unknown, not active, "
+ "or the bucket count is invalid or unchanged"),
@@ -1062,7 +1219,7 @@ public class ScalableTopics extends AdminResource {
validateNamespaceName(tenant, namespace);
TopicName tn = TopicName.get(TopicDomain.topic.value(), namespaceName, encodedTopic);
- validateSuperUserAccessAsync()
+ validateSegmentLayoutOperationAsync(tn)
.thenCompose(__ -> onControllerLeader(tn,
svc -> svc.rebucketSegment(tn, segmentId, bucketCount)))
.thenAccept(__ -> {
@@ -1097,6 +1254,9 @@ public class ScalableTopics extends AdminResource {
@Operation(summary = "Merge two adjacent segments into one.")
@ApiResponses(value = {
@ApiResponse(responseCode = "204", description = "Segments merged successfully"),
+ @ApiResponse(responseCode = "401",
+ description = "Don't have permission to administrate resources on this tenant"),
+ @ApiResponse(responseCode = "403", description = "Don't have admin permission"),
@ApiResponse(responseCode = "404", description = "Scalable topic or segment doesn't exist"),
@ApiResponse(responseCode = "412", description = "Segments are not active or not adjacent"),
@ApiResponse(responseCode = "500", description = "Internal server error")})
@@ -1115,7 +1275,7 @@ public class ScalableTopics extends AdminResource {
validateNamespaceName(tenant, namespace);
TopicName tn = TopicName.get(TopicDomain.topic.value(), namespaceName, encodedTopic);
- validateSuperUserAccessAsync()
+ validateSegmentLayoutOperationAsync(tn)
.thenCompose(__ -> onControllerLeader(tn,
svc -> svc.mergeSegments(tn, segmentId1, segmentId2)))
.thenAccept(__ -> {
@@ -1136,6 +1296,16 @@ public class ScalableTopics extends AdminResource {
// --- Internal helpers ---
+ /**
+ * Authorize a manual change of the segment layout (split, merge, rebucket) as the equivalent operation on a
+ * partitioned topic, a change of its partition count: by default, tenant admins and super users may make it.
+ * Tenant admins already shape the layout through the number of initial segments and the auto split/merge
+ * policy override.
+ */
+ private CompletableFuture<Void> validateSegmentLayoutOperationAsync(TopicName tn) {
+ return validateTopicPolicyOperationAsync(tn, PolicyName.PARTITION, PolicyOperation.WRITE);
+ }
+
/**
* Resolve the local scalable-topic service and pass it to {@code op}. Surfaces a 503
* RestException if the service is not available on this broker.
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicPermissionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicPermissionsTest.java
new file mode 100644
index 000000000000..3b27fb768308
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicPermissionsTest.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.awaitility.Awaitility.await;
+import io.jsonwebtoken.Jwts;
+import java.time.Duration;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import lombok.Cleanup;
+import lombok.SneakyThrows;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.v5.Message;
+import org.apache.pulsar.client.api.v5.Producer;
+import org.apache.pulsar.client.api.v5.PulsarClient;
+import org.apache.pulsar.client.api.v5.PulsarClientException;
+import org.apache.pulsar.client.api.v5.QueueConsumer;
+import org.apache.pulsar.client.api.v5.StreamConsumer;
+import org.apache.pulsar.client.api.v5.auth.AuthenticationFactory;
+import org.apache.pulsar.client.api.v5.config.SubscriptionInitialPosition;
+import org.apache.pulsar.client.api.v5.schema.Schema;
+import org.apache.pulsar.client.impl.auth.AuthenticationToken;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.apache.pulsar.common.policies.data.ScalableSubscriptionType;
+import org.apache.pulsar.common.policies.data.ScalableTopicMetadata;
+import org.apache.pulsar.common.policies.data.ScalableTopicStats.SegmentSubscriptionStats;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Topic-level permissions on scalable topics, with the default authorization provider. The V5 client produces and
+ * consumes through per-segment producers and consumers attached to the {@code segment://} topics of a scalable
+ * topic: they must be authorized by the permissions granted on the scalable topic's {@code topic://} name.
+ */
+@Test(groups = "broker-admin")
+public class ScalableTopicPermissionsTest extends AuthZTest {
+
+ private static final String NAMESPACE = "public/default";
+ private static final String CLIENT_ROLE = "scalable-topic-client";
+ private static final String CLIENT_TOKEN = Jwts.builder()
+ .claim("sub", CLIENT_ROLE).signWith(SECRET_KEY).compact();
+ private static final int NUM_MESSAGES = 20;
+
+ @SneakyThrows
+ @BeforeClass(alwaysRun = true)
+ public void setup() {
+ configureTokenAuthentication();
+ configureDefaultAuthorization();
+ start();
+ this.superUserAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(SUPER_USER_TOKEN))
+ .build();
+ }
+
+ @SneakyThrows
+ @AfterClass(alwaysRun = true)
+ public void cleanup() {
+ close();
+ }
+
+ /** A scalable topic with two segments, so the messages of a test spread over more than one segment. */
+ private String newScalableTopic() throws Exception {
+ String topic = "topic://" + NAMESPACE + "/" + UUID.randomUUID();
+ superUserAdmin.scalableTopics().createScalableTopic(topic, 2);
+ return topic;
+ }
+
+ private PulsarClient newClient(String token) throws Exception {
+ return PulsarClient.builder()
+ .serviceUrl(getPulsarService().getBrokerServiceUrl())
+ .authentication(AuthenticationFactory.token(token))
+ .build();
+ }
+
+ @FunctionalInterface
+ private interface Receiver {
+ Message<String> receive(Duration timeout) throws PulsarClientException;
+ }
+
+ private static Set<String> receiveValues(Receiver receiver, int count) throws Exception {
+ Set<String> values = new HashSet<>();
+ for (int i = 0; i < count; i++) {
+ Message<String> message = receiver.receive(Duration.ofSeconds(10));
+ assertThat(message).as("message %s of %s", i, count).isNotNull();
+ values.add(message.value());
+ }
+ return values;
+ }
+
+ @FunctionalInterface
+ private interface ThrowingRunnable {
+ void run() throws Exception;
+ }
+
+ private static void assertNotAuthorized(ThrowingRunnable call) {
+ assertThatThrownBy(call::run)
+ .isInstanceOf(PulsarClientException.class)
+ .hasStackTraceContaining("is not authorized");
+ }
+
+ @Test
+ public void testTopicPermissionsAuthorizeProduceAndConsume() throws Exception {
+ String topic = newScalableTopic();
+ superUserAdmin.scalableTopics().grantPermission(topic, CLIENT_ROLE,
+ Set.of(AuthAction.produce, AuthAction.consume));
+
+ @Cleanup
+ PulsarClient client = newClient(CLIENT_TOKEN);
+ @Cleanup
+ StreamConsumer<String> streamConsumer = client.newStreamConsumer(Schema.string())
+ .topic(topic)
+ .subscriptionName("stream-sub")
+ .subscriptionInitialPosition(SubscriptionInitialPosition.EARLIEST)
+ .subscribe();
+ @Cleanup
+ QueueConsumer<String> queueConsumer = client.newQueueConsumer(Schema.string())
+ .topic(topic)
+ .subscriptionName("queue-sub")
+ .subscriptionInitialPosition(SubscriptionInitialPosition.EARLIEST)
+ .subscribe();
+ @Cleanup
+ Producer<String> producer = client.newProducer(Schema.string())
+ .topic(topic)
+ .create();
+
+ Set<String> sent = new HashSet<>();
+ for (int i = 0; i < NUM_MESSAGES; i++) {
+ String value = "msg-" + i;
+ // Keyed, so the messages are routed to both segments.
+ producer.newMessage().key("key-" + i).value(value).send();
+ sent.add(value);
+ }
+
+ assertThat(receiveValues(streamConsumer::receive, NUM_MESSAGES)).isEqualTo(sent);
+ assertThat(receiveValues(queueConsumer::receive, NUM_MESSAGES)).isEqualTo(sent);
+ }
+
+ @Test
+ public void testNoTopicPermissionsCannotProduceOrConsume() throws Exception {
+ String topic = newScalableTopic();
+ // The permissions granted on another scalable topic don't apply to this one.
+ superUserAdmin.scalableTopics().grantPermission(newScalableTopic(), CLIENT_ROLE,
+ Set.of(AuthAction.produce, AuthAction.consume));
+
+ @Cleanup
+ PulsarClient client = newClient(CLIENT_TOKEN);
+ assertNotAuthorized(() -> client.newProducer(Schema.string()).topic(topic).create());
+ assertNotAuthorized(() -> client.newStreamConsumer(Schema.string())
+ .topic(topic)
+ .subscriptionName("stream-sub")
+ .subscribe());
+ assertNotAuthorized(() -> client.newQueueConsumer(Schema.string())
+ .topic(topic)
+ .subscriptionName("queue-sub")
+ .subscribe());
+ }
+
+ /**
+ * Either action is enough to watch the topic's layout, so these are refused by the segments themselves: they
+ * only allow the actions granted on the parent topic.
+ */
+ @Test
+ public void testSegmentsAllowOnlyTheActionsGrantedOnTheTopic() throws Exception {
+ String consumeOnlyTopic = newScalableTopic();
+ superUserAdmin.scalableTopics().grantPermission(consumeOnlyTopic, CLIENT_ROLE, Set.of(AuthAction.consume));
+ String produceOnlyTopic = newScalableTopic();
+ superUserAdmin.scalableTopics().grantPermission(produceOnlyTopic, CLIENT_ROLE, Set.of(AuthAction.produce));
+
+ @Cleanup
+ PulsarClient client = newClient(CLIENT_TOKEN);
+
+ @Cleanup
+ Producer<String> refusedProducer = client.newProducer(Schema.string())
+ .topic(consumeOnlyTopic)
+ .create();
+ assertNotAuthorized(() -> refusedProducer.newMessage().value("refused").send());
+
+ @Cleanup
+ Producer<String> producer = client.newProducer(Schema.string())
+ .topic(produceOnlyTopic)
+ .create();
+ producer.newMessage().value("allowed").send();
+ assertNotAuthorized(() -> client.newQueueConsumer(Schema.string())
+ .topic(produceOnlyTopic)
+ .subscriptionName("queue-sub")
+ .subscribe());
+ }
+
+ /** The segments that a split creates are authorized with the permissions of the topic, like the first ones. */
+ @Test
+ public void testSegmentsCreatedBySplitAreAuthorizedWithTheTopicPermissions() throws Exception {
+ String topic = "topic://" + NAMESPACE + "/" + UUID.randomUUID();
+ superUserAdmin.scalableTopics().createScalableTopic(topic, 1);
+ superUserAdmin.scalableTopics().grantPermission(topic, CLIENT_ROLE,
+ Set.of(AuthAction.produce, AuthAction.consume));
+
+ @Cleanup
+ PulsarClient client = newClient(CLIENT_TOKEN);
+ @Cleanup
+ QueueConsumer<String> consumer = client.newQueueConsumer(Schema.string())
+ .topic(topic)
+ .subscriptionName("queue-sub")
+ .subscriptionInitialPosition(SubscriptionInitialPosition.EARLIEST)
+ .subscribe();
+ @Cleanup
+ Producer<String> producer = client.newProducer(Schema.string())
+ .topic(topic)
+ .create();
+
+ Set<String> sent = new HashSet<>();
+ for (int i = 0; i < NUM_MESSAGES; i++) {
+ producer.newMessage().key("key-" + i).value("before-" + i).send();
+ sent.add("before-" + i);
+ }
+ superUserAdmin.scalableTopics().splitSegment(topic, 0);
+ await().untilAsserted(() -> assertThat(superUserAdmin.scalableTopics().getMetadata(topic).getSegments()
+ .values().stream().filter(ScalableTopicMetadata.SegmentInfo::isActive).count()).isEqualTo(2));
+ // Routed to the two segments the split created.
+ for (int i = 0; i < NUM_MESSAGES; i++) {
+ producer.newMessage().key("key-" + i).value("after-" + i).send();
+ sent.add("after-" + i);
+ }
+
+ assertThat(receiveValues(consumer::receive, sent.size())).isEqualTo(sent);
+ }
+
+ @Test
+ public void testGrantGetAndRevokePermissions() throws Exception {
+ String topic = newScalableTopic();
+ assertThat(superUserAdmin.scalableTopics().getPermissions(topic)).doesNotContainKey(CLIENT_ROLE);
+
+ superUserAdmin.scalableTopics().grantPermission(topic, CLIENT_ROLE,
+ Set.of(AuthAction.produce, AuthAction.consume));
+ assertThat(superUserAdmin.scalableTopics().getPermissions(topic))
+ .containsEntry(CLIENT_ROLE, Set.of(AuthAction.produce, AuthAction.consume));
+
+ superUserAdmin.scalableTopics().revokePermissions(topic, CLIENT_ROLE);
+ assertThat(superUserAdmin.scalableTopics().getPermissions(topic)).doesNotContainKey(CLIENT_ROLE);
+ @Cleanup
+ PulsarClient client = newClient(CLIENT_TOKEN);
+ assertNotAuthorized(() -> client.newProducer(Schema.string()).topic(topic).create());
+ }
+
+ /**
+ * A role reaches the permission endpoints unchanged, as a single path component: a '/' doesn't split it, a '%'
+ * doesn't escape anything, and a space and a '+' are different roles.
+ */
+ @Test
+ public void testGrantGetAndRevokeRolesWithReservedCharacters() throws Exception {
+ String topic = newScalableTopic();
+ Map<String, Set<AuthAction>> grants = Map.of(
+ "group/app", Set.of(AuthAction.consume),
+ "100%app", Set.of(AuthAction.produce),
+ "team app", Set.of(AuthAction.consume),
+ "team+app", Set.of(AuthAction.produce, AuthAction.consume));
+ for (Map.Entry<String, Set<AuthAction>> grant : grants.entrySet()) {
+ superUserAdmin.scalableTopics().grantPermission(topic, grant.getKey(), grant.getValue());
+ }
+ assertThat(superUserAdmin.scalableTopics().getPermissions(topic)).containsAllEntriesOf(grants);
+
+ superUserAdmin.scalableTopics().revokePermissions(topic, "team app");
+ assertThat(superUserAdmin.scalableTopics().getPermissions(topic))
+ .doesNotContainKey("team app")
+ .containsEntry("team+app", Set.of(AuthAction.produce, AuthAction.consume));
+
+ for (String role : List.of("group/app", "100%app", "team+app")) {
+ superUserAdmin.scalableTopics().revokePermissions(topic, role);
+ }
+ assertThat(superUserAdmin.scalableTopics().getPermissions(topic)).doesNotContainKeys("group/app", "100%app",
+ "team app", "team+app");
+ }
+
+ /** Revoking the permission granted on the topic disconnects the producers already attached to its segments. */
+ @Test
+ public void testRevokeDisconnectsSegmentProducers() throws Exception {
+ String topic = newScalableTopic();
+ superUserAdmin.scalableTopics().grantPermission(topic, CLIENT_ROLE, Set.of(AuthAction.produce));
+
+ @Cleanup
+ PulsarClient client = newClient(CLIENT_TOKEN);
+ @Cleanup
+ Producer<String> producer = client.newProducer(Schema.string())
+ .topic(topic)
+ .create();
+ for (int i = 0; i < NUM_MESSAGES; i++) {
+ producer.newMessage().key("key-" + i).value("msg-" + i).send();
+ }
+ assertThat(superUserAdmin.scalableTopics().getStats(topic).getProducers()).isNotEmpty();
+
+ superUserAdmin.scalableTopics().revokePermissions(topic, CLIENT_ROLE);
+
+ await().untilAsserted(() ->
+ assertThat(superUserAdmin.scalableTopics().getStats(topic).getProducers()).isEmpty());
+ }
+
+ /** Revoking the permission granted on the topic disconnects the consumers already attached to its segments. */
+ @Test
+ public void testRevokeDisconnectsSegmentConsumers() throws Exception {
+ String topic = newScalableTopic();
+ superUserAdmin.scalableTopics().createSubscription(topic, "queue-sub", ScalableSubscriptionType.QUEUE);
+ superUserAdmin.scalableTopics().grantPermission(topic, CLIENT_ROLE, Set.of(AuthAction.consume));
+
+ @Cleanup
+ PulsarClient client = newClient(CLIENT_TOKEN);
+ @Cleanup
+ QueueConsumer<String> consumer = client.newQueueConsumer(Schema.string())
+ .topic(topic)
+ .subscriptionName("queue-sub")
+ .subscribe();
+ // One per-segment consumer on each of the two segments.
+ await().untilAsserted(() -> assertThat(segmentConsumerCount(topic, "queue-sub")).isEqualTo(2));
+
+ superUserAdmin.scalableTopics().revokePermissions(topic, CLIENT_ROLE);
+
+ await().untilAsserted(() -> assertThat(segmentConsumerCount(topic, "queue-sub")).isZero());
+ }
+
+ private int segmentConsumerCount(String topic, String subscription) throws Exception {
+ return superUserAdmin.scalableTopics().getStats(topic).getSubscriptions().get(subscription).getSegments()
+ .values().stream().mapToInt(SegmentSubscriptionStats::getConsumerCount).sum();
+ }
+
+ @Test
+ public void testPermissionsOfMissingTopic() {
+ String topic = "topic://" + NAMESPACE + "/" + UUID.randomUUID();
+ assertThatThrownBy(() -> superUserAdmin.scalableTopics().getPermissions(topic))
+ .isInstanceOf(PulsarAdminException.NotFoundException.class);
+ assertThatThrownBy(() -> superUserAdmin.scalableTopics().grantPermission(topic, CLIENT_ROLE,
+ Set.of(AuthAction.produce)))
+ .isInstanceOf(PulsarAdminException.NotFoundException.class);
+ assertThatThrownBy(() -> superUserAdmin.scalableTopics().revokePermissions(topic, CLIENT_ROLE))
+ .isInstanceOf(PulsarAdminException.NotFoundException.class);
+ }
+
+ /** As for a regular topic, a topic created again with the name of a deleted one doesn't inherit its grants. */
+ @Test
+ public void testDeleteRemovesTopicPermissions() throws Exception {
+ String topic = newScalableTopic();
+ superUserAdmin.scalableTopics().grantPermission(topic, CLIENT_ROLE, Set.of(AuthAction.produce));
+
+ superUserAdmin.scalableTopics().deleteScalableTopic(topic, true);
+ superUserAdmin.scalableTopics().createScalableTopic(topic, 1);
+
+ assertThat(superUserAdmin.scalableTopics().getPermissions(topic)).doesNotContainKey(CLIENT_ROLE);
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicsAuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicsAuthZTest.java
index 7b1b9dfe121e..4bcc0eaef3c4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicsAuthZTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicsAuthZTest.java
@@ -19,12 +19,14 @@
package org.apache.pulsar.broker.admin;
import java.util.List;
+import java.util.Set;
import java.util.UUID;
import lombok.Cleanup;
import lombok.SneakyThrows;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.impl.auth.AuthenticationToken;
+import org.apache.pulsar.common.policies.data.AuthAction;
import org.apache.pulsar.common.policies.data.ScalableSubscriptionType;
import org.apache.pulsar.common.policies.data.TenantInfo;
import org.testng.Assert;
@@ -185,7 +187,7 @@ public class ScalableTopicsAuthZTest extends AuthZTest {
assertAuthorized(() -> superUserAdmin.scalableTopics().deleteSubscription(topic, "sub-x"));
}
- // -------- ScalableTopics: super-user-only endpoints --------
+ // -------- ScalableTopics: segment layout changes, tenant-admin-allowed like partition count changes --------
@Test
public void testSplitSegmentAuthZ() throws Exception {
@@ -193,7 +195,7 @@ public class ScalableTopicsAuthZTest extends AuthZTest {
superUserAdmin.scalableTopics().createScalableTopic(topic, 1);
@Cleanup PulsarAdmin nobody = nobodyAdmin();
assertNotAuthorized(() -> nobody.scalableTopics().splitSegment(topic, 1L));
- assertNotAuthorized(() -> tenantManagerAdmin.scalableTopics().splitSegment(topic, 1L));
+ assertAuthorized(() -> tenantManagerAdmin.scalableTopics().splitSegment(topic, 1L));
assertAuthorized(() -> superUserAdmin.scalableTopics().splitSegment(topic, 1L));
}
@@ -203,10 +205,54 @@ public class ScalableTopicsAuthZTest extends AuthZTest {
superUserAdmin.scalableTopics().createScalableTopic(topic, 1);
@Cleanup PulsarAdmin nobody = nobodyAdmin();
assertNotAuthorized(() -> nobody.scalableTopics().mergeSegments(topic, 1L, 2L));
- assertNotAuthorized(() -> tenantManagerAdmin.scalableTopics().mergeSegments(topic, 1L, 2L));
+ assertAuthorized(() -> tenantManagerAdmin.scalableTopics().mergeSegments(topic, 1L, 2L));
assertAuthorized(() -> superUserAdmin.scalableTopics().mergeSegments(topic, 1L, 2L));
}
+ @Test
+ public void testRebucketSegmentAuthZ() throws Exception {
+ final String topic = randomTopic();
+ superUserAdmin.scalableTopics().createScalableTopic(topic, 1);
+ @Cleanup PulsarAdmin nobody = nobodyAdmin();
+ assertNotAuthorized(() -> nobody.scalableTopics().rebucketSegment(topic, 1L, 2));
+ assertAuthorized(() -> tenantManagerAdmin.scalableTopics().rebucketSegment(topic, 1L, 2));
+ assertAuthorized(() -> superUserAdmin.scalableTopics().rebucketSegment(topic, 1L, 2));
+ }
+
+ // -------- ScalableTopics: permissions, managed by tenant admins --------
+
+ @Test
+ public void testGetPermissionsAuthZ() throws Exception {
+ final String topic = randomTopic();
+ superUserAdmin.scalableTopics().createScalableTopic(topic, 1);
+ @Cleanup PulsarAdmin nobody = nobodyAdmin();
+ assertNotAuthorized(() -> nobody.scalableTopics().getPermissions(topic));
+ assertAuthorized(() -> tenantManagerAdmin.scalableTopics().getPermissions(topic));
+ assertAuthorized(() -> superUserAdmin.scalableTopics().getPermissions(topic));
+ }
+
+ @Test
+ public void testGrantPermissionAuthZ() throws Exception {
+ final String topic = randomTopic();
+ superUserAdmin.scalableTopics().createScalableTopic(topic, 1);
+ @Cleanup PulsarAdmin nobody = nobodyAdmin();
+ assertNotAuthorized(() -> nobody.scalableTopics().grantPermission(topic, "role", Set.of(AuthAction.consume)));
+ assertAuthorized(() -> tenantManagerAdmin.scalableTopics()
+ .grantPermission(topic, "role", Set.of(AuthAction.consume)));
+ assertAuthorized(() -> superUserAdmin.scalableTopics()
+ .grantPermission(topic, "role", Set.of(AuthAction.consume)));
+ }
+
+ @Test
+ public void testRevokePermissionAuthZ() throws Exception {
+ final String topic = randomTopic();
+ superUserAdmin.scalableTopics().createScalableTopic(topic, 1);
+ @Cleanup PulsarAdmin nobody = nobodyAdmin();
+ assertNotAuthorized(() -> nobody.scalableTopics().revokePermissions(topic, "role"));
+ assertAuthorized(() -> tenantManagerAdmin.scalableTopics().revokePermissions(topic, "role"));
+ assertAuthorized(() -> superUserAdmin.scalableTopics().revokePermissions(topic, "role"));
+ }
+
// -------- Segments resource: super-user only on all endpoints --------
@Test
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ScalableTopics.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ScalableTopics.java
index 88d72275e878..5ff5ba6c51a5 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ScalableTopics.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ScalableTopics.java
@@ -20,7 +20,9 @@ package org.apache.pulsar.client.admin;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.common.policies.data.AuthAction;
import org.apache.pulsar.common.policies.data.AutoScalePolicyOverride;
import org.apache.pulsar.common.policies.data.ScalableSubscriptionType;
import org.apache.pulsar.common.policies.data.ScalableTopicMetadata;
@@ -195,6 +197,87 @@ public interface ScalableTopics {
*/
CompletableFuture<Void> removeAutoScalePolicyAsync(String topic);
+ /**
+ * Get permissions on a scalable topic.
+ *
+ * <p>Retrieve the effective permissions for a scalable topic. These permissions are defined by the
+ * permissions set at the namespace level combined (union) with any eventual specific permission set on
+ * the topic, and apply to every segment of the topic.
+ *
+ * <p>Response Example:
+ *
+ * <pre>
+ * <code>{
+ * "role-1" : [ "produce" ],
+ * "role-2" : [ "consume" ]
+ * }</code>
+ * </pre>
+ *
+ * @param topic Topic name in the format "tenant/namespace/topic"
+ * @return a map of roles and their permissions
+ * @throws PulsarAdminException.NotAuthorizedException Don't have admin permission
+ * @throws PulsarAdminException.NotFoundException Scalable topic does not exist
+ * @throws PulsarAdminException Unexpected error
+ */
+ Map<String, Set<AuthAction>> getPermissions(String topic) throws PulsarAdminException;
+
+ /**
+ * Get permissions on a scalable topic asynchronously.
+ *
+ * @param topic Topic name in the format "tenant/namespace/topic"
+ * @return a map of roles and their permissions
+ */
+ CompletableFuture<Map<String, Set<AuthAction>>> getPermissionsAsync(String topic);
+
+ /**
+ * Grant a new permission to a client role on a scalable topic. The permission applies to every segment
+ * of the topic.
+ *
+ * <p>Request parameter example:
+ *
+ * <pre>
+ * <code>["produce", "consume"]</code>
+ * </pre>
+ *
+ * @param topic Topic name in the format "tenant/namespace/topic"
+ * @param role Client role to which grant permission
+ * @param actions Auth actions (produce and consume)
+ * @throws PulsarAdminException.NotAuthorizedException Don't have admin permission
+ * @throws PulsarAdminException.NotFoundException Scalable topic does not exist
+ * @throws PulsarAdminException.ConflictException Concurrent modification
+ * @throws PulsarAdminException Unexpected error
+ */
+ void grantPermission(String topic, String role, Set<AuthAction> actions) throws PulsarAdminException;
+
+ /**
+ * Grant a new permission to a client role on a scalable topic asynchronously.
+ *
+ * @param topic Topic name in the format "tenant/namespace/topic"
+ * @param role Client role to which grant permission
+ * @param actions Auth actions (produce and consume)
+ */
+ CompletableFuture<Void> grantPermissionAsync(String topic, String role, Set<AuthAction> actions);
+
+ /**
+ * Revoke the permissions granted to a client role on a scalable topic. Permissions granted at the
+ * namespace level are not affected.
+ *
+ * @param topic Topic name in the format "tenant/namespace/topic"
+ * @param role Client role to which remove permission
+ * @throws PulsarAdminException.NotAuthorizedException Don't have admin permission
+ * @throws PulsarAdminException.NotFoundException Scalable topic does not exist
+ * @throws PulsarAdminException Unexpected error
+ */
+ void revokePermissions(String topic, String role) throws PulsarAdminException;
+
+ /**
+ * Revoke the permissions granted to a client role on a scalable topic asynchronously.
+ *
+ * @param topic Topic name in the format "tenant/namespace/topic"
+ * @param role Client role to which remove permission
+ */
+ CompletableFuture<Void> revokePermissionsAsync(String topic, String role);
+
/**
* Delete a scalable topic and all its underlying segment topics.
*
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ScalableTopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ScalableTopicsImpl.java
index 7fb9ab0c36f5..ae529455e261 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ScalableTopicsImpl.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ScalableTopicsImpl.java
@@ -23,8 +23,11 @@ import jakarta.ws.rs.client.WebTarget;
import jakarta.ws.rs.core.GenericType;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response.Status;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException;
@@ -32,6 +35,7 @@ import org.apache.pulsar.client.admin.ScalableTopics;
import org.apache.pulsar.client.api.Authentication;
import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.AuthAction;
import org.apache.pulsar.common.policies.data.AutoScalePolicyOverride;
import org.apache.pulsar.common.policies.data.ScalableTopicMetadata;
import org.apache.pulsar.common.policies.data.ScalableTopicStats;
@@ -190,6 +194,57 @@ public class ScalableTopicsImpl extends BaseResource implements ScalableTopics {
return asyncDeleteRequest(topicPath(tn).path("autoScalePolicy"));
}
+ // --- Permissions ---
+
+ @Override
+ public Map<String, Set<AuthAction>> getPermissions(String topic) throws PulsarAdminException {
+ return sync(() -> getPermissionsAsync(topic));
+ }
+
+ @Override
+ public CompletableFuture<Map<String, Set<AuthAction>>> getPermissionsAsync(String topic) {
+ return FutureUtil.supplySafely(() -> {
+ TopicName tn = validateTopic(topic);
+ return asyncGetRequest(topicPath(tn).path("permissions"),
+ new GenericType<Map<String, Set<AuthAction>>>() {});
+ });
+ }
+
+ @Override
+ public void grantPermission(String topic, String role, Set<AuthAction> actions) throws PulsarAdminException {
+ sync(() -> grantPermissionAsync(topic, role, actions));
+ }
+
+ @Override
+ public CompletableFuture<Void> grantPermissionAsync(String topic, String role, Set<AuthAction> actions) {
+ return FutureUtil.supplySafely(() -> {
+ TopicName tn = validateTopic(topic);
+ return asyncPostRequest(permissionsPath(tn, role), Entity.entity(actions, MediaType.APPLICATION_JSON));
+ });
+ }
+
+ @Override
+ public void revokePermissions(String topic, String role) throws PulsarAdminException {
+ sync(() -> revokePermissionsAsync(topic, role));
+ }
+
+ @Override
+ public CompletableFuture<Void> revokePermissionsAsync(String topic, String role) {
+ return FutureUtil.supplySafely(() -> {
+ TopicName tn = validateTopic(topic);
+ return asyncDeleteRequest(permissionsPath(tn, role));
+ });
+ }
+
+ /**
+ * The role as a single path component, even if it contains a '/'. Unlike the form encoding of
+ * {@link URLEncoder}, a space must be "%20" in a path: the broker reads a '+' of the path as a '+'.
+ */
+ private WebTarget permissionsPath(TopicName tn, String role) {
+ String encodedRole = URLEncoder.encode(role, StandardCharsets.UTF_8).replace("+", "%20");
+ return topicPath(tn).path("permissions").path(encodedRole);
+ }
+
// --- Delete ---
@Override
diff --git a/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/ScalableTopicsImplTest.java b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/ScalableTopicsImplTest.java
new file mode 100644
index 000000000000..661e3899e0f7
--- /dev/null
+++ b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/ScalableTopicsImplTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.admin.internal;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
+import jakarta.ws.rs.client.WebTarget;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.testng.annotations.Test;
+
+public class ScalableTopicsImplTest {
+
+ /** Missing the local name: not a valid topic name. */
+ private static final String MALFORMED_TOPIC = "topic://tenant/ns";
+
+ /** The asynchronous permission methods report an invalid topic name through the future, never by throwing. */
+ @Test
+ public void testPermissionMethodsFailTheFutureOnMalformedTopic() {
+ ScalableTopicsImpl scalableTopics = new ScalableTopicsImpl(mock(WebTarget.class), null, 0);
+
+ assertFailsWithIllegalArgument(scalableTopics.getPermissionsAsync(MALFORMED_TOPIC));
+ assertFailsWithIllegalArgument(
+ scalableTopics.grantPermissionAsync(MALFORMED_TOPIC, "role", Set.of(AuthAction.produce)));
+ assertFailsWithIllegalArgument(scalableTopics.revokePermissionsAsync(MALFORMED_TOPIC, "role"));
+ }
+
+ private static void assertFailsWithIllegalArgument(CompletableFuture<?> future) {
+ assertThat(future).isCompletedExceptionally();
+ assertThatThrownBy(future::join).hasCauseInstanceOf(IllegalArgumentException.class);
+ }
+}
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdScalableTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdScalableTopics.java
index 4c642e9c12db..eb99f24c0aec 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdScalableTopics.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdScalableTopics.java
@@ -235,6 +235,54 @@ public class CmdScalableTopics extends CmdBase {
}
}
+ @Command(description = "Get the permissions on a scalable topic: the permissions set at the namespace"
+ + " level combined (union) with any eventual specific permission set on the topic")
+ private class PermissionsCmd extends CliCommand {
+ @Parameters(description = "tenant/namespace/topic", arity = "1")
+ private String topic;
+
+ @Override
+ void run() throws Exception {
+ print(scalableTopics().getPermissions(topic));
+ }
+ }
+
+ @Command(description = "Grant a new permission to a client role on a scalable topic. The permission"
+ + " applies to every segment of the topic")
+ private class GrantPermissionCmd extends CliCommand {
+ @Parameters(description = "tenant/namespace/topic", arity = "1")
+ private String topic;
+
+ @Option(names = {"-r", "--role"}, description = "Client role to which grant permissions", required = true)
+ private String role;
+
+ @Option(names = {"-a", "--actions"}, description = "Actions to be granted (produce,consume,sources,sinks,"
+ + "functions,packages)", required = true, split = ",")
+ private List<String> actions;
+
+ @Override
+ void run() throws Exception {
+ scalableTopics().grantPermission(topic, role, getAuthActions(actions));
+ print("Granted " + actions + " to role " + role + " on topic " + topic);
+ }
+ }
+
+ @Command(description = "Revoke the permissions granted to a client role on a scalable topic. Permissions"
+ + " granted at the namespace level are not affected")
+ private class RevokePermissionCmd extends CliCommand {
+ @Parameters(description = "tenant/namespace/topic", arity = "1")
+ private String topic;
+
+ @Option(names = {"-r", "--role"}, description = "Client role to which revoke permissions", required = true)
+ private String role;
+
+ @Override
+ void run() throws Exception {
+ scalableTopics().revokePermissions(topic, role);
+ print("Revoked the permissions of role " + role + " on topic " + topic);
+ }
+ }
+
public CmdScalableTopics(Supplier<PulsarAdmin> admin) {
super("scalable-topics", admin);
addCommand("list", new ListCmd());
@@ -248,6 +296,9 @@ public class CmdScalableTopics extends CmdBase {
addCommand("merge-segments", new MergeSegmentsCmd());
addCommand("seek", new SeekSubscriptionCmd());
addCommand("clear-backlog", new ClearBacklogCmd());
+ addCommand("permissions", new PermissionsCmd());
+ addCommand("grant-permission", new GrantPermissionCmd());
+ addCommand("revoke-permission", new RevokePermissionCmd());
}
private Map<String, String> parseProperties(List<String> properties) {
diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdScalableTopics.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdScalableTopics.java
index dae672cce438..a1a33565445c 100644
--- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdScalableTopics.java
+++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdScalableTopics.java
@@ -25,8 +25,10 @@ import static org.mockito.Mockito.when;
import static org.testng.Assert.assertTrue;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.ScalableTopics;
+import org.apache.pulsar.common.policies.data.AuthAction;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -150,4 +152,21 @@ public class TestCmdScalableTopics {
verify(scalableTopics).listScalableTopicsByProperties(eq(namespace),
eq(Map.of("app", "cli", "mode", "blackbox")));
}
+
+ @Test
+ public void testPermissionCommands() throws Exception {
+ String topic = "topic://public/default/v5-cli-permissions";
+
+ assertTrue(cmdScalableTopics.run(new String[] {
+ "grant-permission", topic, "--role", "app", "--actions", "produce,consume"
+ }));
+ verify(scalableTopics).grantPermission(eq(topic), eq("app"),
+ eq(Set.of(AuthAction.produce, AuthAction.consume)));
+
+ assertTrue(cmdScalableTopics.run(new String[] {"permissions", topic}));
+ verify(scalableTopics).getPermissions(eq(topic));
+
+ assertTrue(cmdScalableTopics.run(new String[] {"revoke-permission", topic, "--role", "app"}));
+ verify(scalableTopics).revokePermissions(eq(topic), eq("app"));
+ }
}
diff --git a/spec/scalable-topics/compatibility.md b/spec/scalable-topics/compatibility.md
index 42076486cee4..7787d88142d5 100644
--- a/spec/scalable-topics/compatibility.md
+++ b/spec/scalable-topics/compatibility.md
@@ -65,6 +65,12 @@ This bridge exists only for migration. New applications use canonical `topic://`
## 5. Schema and policies
-Schema negotiation, topic-level policies, authentication/authorization, and TLS are the standard Pulsar
-mechanisms, applied at the segment level. A scalable topic has a single topic-level schema regardless of
-segment count ([Data Model](data-model.md) §6).
+Schema negotiation, topic-level policies, authentication, and TLS are the standard Pulsar mechanisms,
+applied at the segment level. A scalable topic has a single topic-level schema regardless of segment count
+([Data Model](data-model.md) §6).
+
+Authorization is also the standard Pulsar mechanism, but it is evaluated against the scalable topic: the
+broker authorizes a client's commands on a `segment://` name as it authorizes the parent `topic://`, with
+the permissions granted on that topic or on its namespace. An application needs no per-segment grant, and
+a grant on the topic covers the segments that splits and merges create later. A legacy segment (§1) is
+authorized as the wrapped `persistent://` topic.
diff --git a/spec/scalable-topics/wire-protocol.md b/spec/scalable-topics/wire-protocol.md
index 0d7edcd8f652..eb9b208ca8a0 100644
--- a/spec/scalable-topics/wire-protocol.md
+++ b/spec/scalable-topics/wire-protocol.md
@@ -382,7 +382,8 @@ sequenceDiagram
`CommandConnect`/auth, TLS, schema negotiation (`CommandGetSchema`/`CommandGetOrCreateSchema`), broker
lookup, and `CommandSend`/`CommandAck` framing are the standard Pulsar protocol, applied at the
**segment** topic level (`segment://`, or the wrapped `persistent://` for legacy segments). No
-scalable-topic-specific changes are required to them.
+scalable-topic-specific changes are required to them. The broker authorizes these segment-level commands
+with the permissions of the parent scalable topic ([Compatibility](compatibility.md) §5).
## 11. Cross-cutting requirements
(pulsar) 08/14: [fix][broker] Handle flow_or_qps bundle split requests in ELM (#25442)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 9a3100cd7c6822586994cdc1f3ac22cc7f0bcd6d
Author: sinan liu <li...@gmail.com>
AuthorDate: Sat Oct 17 10:15:31 2026 +0800
[fix][broker] Handle flow_or_qps bundle split requests in ELM (#25442)
Co-authored-by: Lari Hotari <lh...@users.noreply.github.com>
Co-authored-by: Matteo Merli <mm...@apache.org>
(cherry picked from commit c78a99e6b46c553330c1bf1d774e760793effe1d)
---
.../extensions/ExtensibleLoadManagerImpl.java | 68 +++++++++++++---------
.../pulsar/broker/namespace/NamespaceService.java | 11 ++--
.../apache/pulsar/broker/admin/AdminApiTest.java | 18 ++++++
.../extensions/ExtensibleLoadManagerImplTest.java | 16 +++++
4 files changed, 82 insertions(+), 31 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 0e78fd682f0a..d49b1a6347f6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -759,34 +759,50 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager, BrokerS
final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle.toString());
NamespaceBundle namespaceBundle =
pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange);
- return pulsar.getNamespaceService().getSplitBoundary(namespaceBundle, splitAlgorithm, boundaries)
- .thenCompose(splitBundlesPair -> {
- if (splitBundlesPair == null) {
- String msg = format("Bundle %s not found under namespace", namespaceBundle);
- log.error(msg);
- return FutureUtil.failedFuture(new IllegalStateException(msg));
+ return pulsar.getNamespaceService().getSplitBoundary(namespaceBundle, boundaries, splitAlgorithm)
+ .thenCompose(splitBoundaries -> {
+ if (splitBoundaries == null || splitBoundaries.isEmpty()) {
+ log.info()
+ .attr("namespaceObject", namespaceBundle.getNamespaceObject().toString())
+ .attr("boundaries", boundaries)
+ .attr("bundle", namespaceBundle.getBundleRange())
+ .log("No valid boundary found to split bundle");
+ return CompletableFuture.completedFuture(null);
}
-
- return getOwnershipAsync(Optional.empty(), bundle)
- .thenCompose(brokerOpt -> {
- if (brokerOpt.isEmpty()) {
- String msg = String.format("Namespace bundle: %s is not owned by any broker.",
- bundle);
- log.warn(msg);
- throw new IllegalStateException(msg);
+ return pulsar.getNamespaceService().getNamespaceBundleFactory()
+ .splitBundles(namespaceBundle, splitBoundaries.size() + 1, splitBoundaries)
+ .thenCompose(splitBundlesPair -> {
+ if (splitBundlesPair == null) {
+ String msg = format("Bundle %s not found under namespace", namespaceBundle);
+ log.error(msg);
+ return FutureUtil.failedFuture(new IllegalStateException(msg));
}
- String sourceBroker = brokerOpt.get();
- SplitDecision splitDecision = new SplitDecision();
- List<NamespaceBundle> splitBundles = splitBundlesPair.getRight();
- Map<String, Optional<String>> splitServiceUnitToDestBroker = new HashMap<>();
- splitBundles.forEach(splitBundle -> splitServiceUnitToDestBroker
- .put(splitBundle.getBundleRange(), Optional.empty()));
- splitDecision.setSplit(
- new Split(bundle.toString(), sourceBroker, splitServiceUnitToDestBroker));
- splitDecision.setLabel(Success);
- splitDecision.setReason(Admin);
- return splitAsync(splitDecision,
- conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS);
+
+ return getOwnershipAsync(Optional.empty(), bundle)
+ .thenCompose(brokerOpt -> {
+ if (brokerOpt.isEmpty()) {
+ String msg = String.format(
+ "Namespace bundle: %s is not owned by any broker.",
+ bundle);
+ log.warn(msg);
+ throw new IllegalStateException(msg);
+ }
+ String sourceBroker = brokerOpt.get();
+ SplitDecision splitDecision = new SplitDecision();
+ List<NamespaceBundle> splitBundles = splitBundlesPair.getRight();
+ Map<String, Optional<String>> splitServiceUnitToDestBroker =
+ new HashMap<>();
+ splitBundles.forEach(splitBundle -> splitServiceUnitToDestBroker
+ .put(splitBundle.getBundleRange(), Optional.empty()));
+ splitDecision.setSplit(
+ new Split(bundle.toString(), sourceBroker,
+ splitServiceUnitToDestBroker));
+ splitDecision.setLabel(Success);
+ splitDecision.setReason(Admin);
+ return splitAsync(splitDecision,
+ conf.getNamespaceBundleUnloadingTimeoutMs(),
+ TimeUnit.MILLISECONDS);
+ });
});
});
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index 4d39c226de22..a3351a8621b3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -88,6 +88,7 @@ import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode;
import org.apache.pulsar.common.lookup.GetTopicsResult;
import org.apache.pulsar.common.lookup.data.LookupData;
import org.apache.pulsar.common.naming.BundleSplitOption;
+import org.apache.pulsar.common.naming.FlowOrQpsEquallyDivideBundleSplitAlgorithm;
import org.apache.pulsar.common.naming.FlowOrQpsEquallyDivideBundleSplitOption;
import org.apache.pulsar.common.naming.NamespaceBundle;
import org.apache.pulsar.common.naming.NamespaceBundleFactory;
@@ -981,7 +982,7 @@ public class NamespaceService implements AutoCloseable {
CompletableFuture<Void> completionFuture,
NamespaceBundleSplitAlgorithm splitAlgorithm,
List<Long> boundaries) {
- BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config);
+ BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config, splitAlgorithm);
splitAlgorithm.getSplitBoundary(bundleSplitOption).whenComplete((splitBoundaries, ex) -> {
CompletableFuture<List<NamespaceBundle>> updateFuture = new CompletableFuture<>();
@@ -1180,16 +1181,16 @@ public class NamespaceService implements AutoCloseable {
public CompletableFuture<List<Long>> getSplitBoundary(
NamespaceBundle bundle, List<Long> boundaries, NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm) {
- BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config);
+ BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config, nsBundleSplitAlgorithm);
return nsBundleSplitAlgorithm.getSplitBoundary(bundleSplitOption);
}
private BundleSplitOption getBundleSplitOption(NamespaceBundle bundle,
List<Long> boundaries,
- ServiceConfiguration config) {
+ ServiceConfiguration config,
+ NamespaceBundleSplitAlgorithm splitAlgorithm) {
BundleSplitOption bundleSplitOption;
- if (config.getDefaultNamespaceBundleSplitAlgorithm()
- .equals(NamespaceBundleSplitAlgorithm.FLOW_OR_QPS_EQUALLY_DIVIDE)) {
+ if (splitAlgorithm instanceof FlowOrQpsEquallyDivideBundleSplitAlgorithm) {
Map<String, TopicStatsImpl> topicStatsMap = pulsar.getBrokerService().getTopicStats(bundle);
bundleSplitOption = new FlowOrQpsEquallyDivideBundleSplitOption(this, bundle, boundaries,
topicStatsMap,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
index 88f36d2bacec..beadfc0649ef 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
@@ -1749,6 +1749,24 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest {
}
}
+ @Test
+ public void testNamespaceSplitBundleWithFlowOrQpsAlgorithmUsesRequestedAlgorithm() throws Exception {
+ final String namespace = "prop-xyz/flow-or-qps-split";
+ final String topicName = "persistent://" + namespace + "/topic-1";
+ admin.namespaces().createNamespace(namespace, 1);
+ admin.topics().createNonPartitionedTopic(topicName);
+ admin.lookups().lookupTopic(topicName);
+
+ try {
+ admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", true,
+ NamespaceBundleSplitAlgorithm.FLOW_OR_QPS_EQUALLY_DIVIDE);
+ } catch (Exception e) {
+ fail("split bundle with flow_or_qps_equally_divide shouldn't have thrown exception", e);
+ }
+
+ assertEquals(admin.namespaces().getBundles(namespace).getNumBundles(), 1);
+ }
+
@Test
public void testNamespacesGetTopicHashPositions() throws Exception {
// Force to create a namespace with only one bundle and create a topic
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index c93bc8ec75b8..fbfb3ac5b9ca 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -123,6 +123,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.LookupService;
import org.apache.pulsar.common.naming.NamespaceBundle;
+import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.ServiceUnitId;
import org.apache.pulsar.common.naming.TopicDomain;
@@ -1034,6 +1035,21 @@ public class ExtensibleLoadManagerImplTest extends ExtensibleLoadManagerImplBase
assertTrue(bundlesData.getBoundaries().contains(midBundle));
assertTrue(bundlesData.getBoundaries().contains(highBundle));
}
+
+ @Test(timeOut = 30 * 1000)
+ public void testSplitBundleWithFlowOrQpsAdminAPINoValidBoundary() throws Exception {
+ String namespace = "public/test-split-with-flow-or-qps";
+ String topic = "persistent://" + namespace + "/test-split-with-flow-or-qps";
+ admin.namespaces().createNamespace(namespace, 1);
+ admin.topics().createNonPartitionedTopic(topic);
+ admin.lookups().lookupTopic(topic);
+
+ admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", true,
+ NamespaceBundleSplitAlgorithm.FLOW_OR_QPS_EQUALLY_DIVIDE);
+
+ assertEquals(admin.namespaces().getBundles(namespace).getNumBundles(), 1);
+ }
+
@Test(timeOut = 30 * 1000)
public void testDeleteNamespaceBundle() throws Exception {
final String namespace = "public/testDeleteNamespaceBundle";
(pulsar) 06/14: [fix][test] Split and fix flaky SASL authentication first-stage auth cache tests (#25949)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 45bbdbb17136c269aae380707bf978c3e3a6101c
Author: Oneby Wang <89...@qq.com>
AuthorDate: Sat Oct 17 09:15:23 2026 +0800
[fix][test] Split and fix flaky SASL authentication first-stage auth cache tests (#25949)
Co-authored-by: Matteo Merli <mm...@apache.org>
(cherry picked from commit d51c364806ced01846727b318a102932bd6f53b6)
---
.../authentication/SaslAuthenticateTest.java | 78 +++++++++++++---------
1 file changed, 46 insertions(+), 32 deletions(-)
diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java
index 24d895dffca4..ea58cc8d6d4b 100644
--- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java
+++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java
@@ -18,8 +18,10 @@
*/
package org.apache.pulsar.broker.authentication;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNull;
@@ -33,7 +35,6 @@ import jakarta.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
-import java.lang.reflect.Field;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -65,13 +66,13 @@ import org.apache.pulsar.client.impl.auth.AuthenticationSasl;
import org.apache.pulsar.common.api.AuthData;
import org.apache.pulsar.common.sasl.SaslConstants;
import org.apache.pulsar.common.util.ObjectMapperFactory;
-import org.testng.Assert;
+import org.awaitility.Awaitility;
+import org.mockito.ArgumentCaptor;
import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-import org.testng.collections.CollectionUtils;
@CustomLog
public class SaslAuthenticateTest extends ProducerConsumerBase {
@@ -224,7 +225,7 @@ public class SaslAuthenticateTest extends ProducerConsumerBase {
@Override
protected void cleanup() throws Exception {
FileUtils.deleteQuietly(secretKeyFile);
- Assert.assertFalse(secretKeyFile.exists());
+ assertFalse(secretKeyFile.exists());
super.internalCleanup();
}
@@ -313,50 +314,63 @@ public class SaslAuthenticateTest extends ProducerConsumerBase {
}
@Test
- @SuppressWarnings("unchecked")
- public void testSaslOnlyAuthFirstStage() throws Exception {
+ public void testSaslOnlyAuthFirstStageKeepsInflightContextsBeforeExpiry() throws Exception {
@Cleanup
AuthenticationProviderSasl saslServer = new AuthenticationProviderSasl();
- // The cache expiration time is set to 500ms. Residual auth info should be cleaned up
- conf.setInflightSaslContextExpiryMs(500);
+ conf.setInflightSaslContextExpiryMs(Integer.MAX_VALUE);
saslServer.initialize(AuthenticationProvider.Context.builder().config(conf).build());
HttpServletRequest servletRequest = mock(HttpServletRequest.class);
- doReturn("Init").when(servletRequest).getHeader("State");
- // 10 clients only do one-stage verification, resulting in 10 auth info remaining in memory
+ doReturn(SaslConstants.SASL_STATE_CLIENT_INIT).when(servletRequest).getHeader(SaslConstants.SASL_HEADER_STATE);
for (int i = 0; i < 10; i++) {
- AuthenticationDataProvider dataProvider = authSasl.getAuthData("localhost");
+ AuthenticationDataProvider dataProvider = authSasl.getAuthData(localHostname);
AuthData initData1 = dataProvider.authenticate(AuthData.INIT_AUTH_DATA);
doReturn(Base64.getEncoder().encodeToString(initData1.getBytes())).when(
- servletRequest).getHeader("SASL-Token");
- doReturn(String.valueOf(i)).when(servletRequest).getHeader("SASL-Server-ID");
+ servletRequest).getHeader(SaslConstants.SASL_AUTH_TOKEN);
+ doReturn(String.valueOf(i)).when(servletRequest).getHeader(SaslConstants.SASL_STATE_SERVER);
saslServer.authenticateHttpRequest(servletRequest, mock(HttpServletResponse.class));
}
- Field field = AuthenticationProviderSasl.class.getDeclaredField("authStates");
- field.setAccessible(true);
- Cache<Long, AuthenticationState> cache = (Cache<Long, AuthenticationState>) field.get(saslServer);
+
+ Cache<Long, AuthenticationState> cache = saslServer.getAuthStates();
assertEquals(cache.asMap().size(), 10);
- // Add more auth info into memory
+ }
+
+ @Test
+ public void testSaslOnlyAuthFirstStageExpiresResidualContexts() throws Exception {
+ @Cleanup
+ AuthenticationProviderSasl saslServer = new AuthenticationProviderSasl();
+ conf.setInflightSaslContextExpiryMs(50);
+ saslServer.initialize(AuthenticationProvider.Context.builder().config(conf).build());
+
+ HttpServletRequest servletRequest = mock(HttpServletRequest.class);
+ doReturn(SaslConstants.SASL_STATE_CLIENT_INIT).when(servletRequest).getHeader(SaslConstants.SASL_HEADER_STATE);
+ // The cache is keyed by the server-generated state id, which the broker returns in the
+ // SASL-Server-ID response header, not by the SASL-Server-ID request header.
+ Set<Long> stateIds = new HashSet<>();
for (int i = 0; i < 10; i++) {
- AuthenticationDataProvider dataProvider = authSasl.getAuthData("localhost");
+ AuthenticationDataProvider dataProvider = authSasl.getAuthData(localHostname);
AuthData initData1 = dataProvider.authenticate(AuthData.INIT_AUTH_DATA);
doReturn(Base64.getEncoder().encodeToString(initData1.getBytes())).when(
- servletRequest).getHeader("SASL-Token");
- doReturn(String.valueOf(10 + i)).when(servletRequest).getHeader("SASL-Server-ID");
- saslServer.authenticateHttpRequest(servletRequest, mock(HttpServletResponse.class));
+ servletRequest).getHeader(SaslConstants.SASL_AUTH_TOKEN);
+ doReturn(String.valueOf(i)).when(servletRequest).getHeader(SaslConstants.SASL_STATE_SERVER);
+ HttpServletResponse response = mock(HttpServletResponse.class);
+ saslServer.authenticateHttpRequest(servletRequest, response);
+ ArgumentCaptor<String> stateId = ArgumentCaptor.forClass(String.class);
+ verify(response).setHeader(eq(SaslConstants.SASL_STATE_SERVER), stateId.capture());
+ stateIds.add(Long.parseLong(stateId.getValue()));
}
- long start = System.currentTimeMillis();
- while (true) {
- if (System.currentTimeMillis() - start > 5000) {
- fail();
- }
- cache = (Cache<Long, AuthenticationState>) field.get(saslServer);
- // Residual auth info should be cleaned up
- if (CollectionUtils.hasElements(cache.asMap())) {
- break;
+ assertEquals(stateIds.size(), 10);
+
+ Cache<Long, AuthenticationState> cache = saslServer.getAuthStates();
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+ for (long stateId : stateIds) {
+ assertNull(cache.getIfPresent(stateId));
}
- Thread.sleep(5);
- }
+ // Expired entries are physically removed only by Caffeine maintenance, so run it before
+ // checking that no residual context is left in memory.
+ cache.cleanUp();
+ assertTrue(cache.asMap().isEmpty());
+ });
}
@Test
(pulsar) 13/14: [fix][proxy] Fix connection and topic stats that stop updating after the initial update (#26961)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit ecbf908eb0f9761c87f812d6059b668fb630672e
Author: void-ptr974 <vo...@protonmail.com>
AuthorDate: Sat Oct 17 13:30:45 2026 +0800
[fix][proxy] Fix connection and topic stats that stop updating after the initial update (#26961)
(cherry picked from commit c5518ccf84acfccf3dac0f711872fd83d63e2da8)
---
.../java/org/apache/pulsar/proxy/server/ProxyService.java | 14 ++++++++++++--
.../org/apache/pulsar/proxy/server/ProxyStatsTest.java | 14 +++++++++++++-
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
index c16a3b8a04cc..84bb62ac79b2 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.proxy.server;
import static java.util.Objects.requireNonNull;
import static org.apache.commons.lang3.StringUtils.isBlank;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.AdaptiveRecvByteBufAllocator;
@@ -197,6 +198,15 @@ public class ProxyService implements Closeable {
public ProxyService(ProxyConfiguration proxyConfig,
AuthenticationService authenticationService,
Authentication proxyClientAuthentication) throws Exception {
+ this(proxyConfig, authenticationService, proxyClientAuthentication, 60, TimeUnit.SECONDS);
+ }
+
+ @VisibleForTesting
+ ProxyService(ProxyConfiguration proxyConfig,
+ AuthenticationService authenticationService,
+ Authentication proxyClientAuthentication,
+ long statsUpdateInterval,
+ TimeUnit statsUpdateIntervalUnit) throws Exception {
requireNonNull(proxyConfig);
this.proxyConfig = proxyConfig;
this.clientCnxs = Sets.newConcurrentHashSet();
@@ -234,7 +244,7 @@ public class ProxyService implements Closeable {
statsExecutor = Executors
.newSingleThreadScheduledExecutor(new DefaultThreadFactory("proxy-stats-executor"));
- statsExecutor.schedule(() -> {
+ statsExecutor.scheduleAtFixedRate(() -> {
this.clientCnxs.forEach(cnx -> {
if (cnx.getDirectProxyHandler() != null
&& cnx.getDirectProxyHandler().getInboundChannelRequestsRate() != null) {
@@ -244,7 +254,7 @@ public class ProxyService implements Closeable {
this.topicStats.forEach((topic, stats) -> {
stats.calculate();
});
- }, 60, TimeUnit.SECONDS);
+ }, statsUpdateInterval, statsUpdateInterval, statsUpdateIntervalUnit);
this.proxyAdditionalServlets = AdditionalServlets.load(proxyConfig);
this.proxyClientAuthentication = proxyClientAuthentication;
this.connectionController = new ConnectionController.DefaultConnectionController(
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java
index 02b22bafacaf..c272613f0729 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java
@@ -87,7 +87,7 @@ public class ProxyStatsTest extends MockedPulsarServiceBaseTest {
proxyService = Mockito.spy(new ProxyService(proxyConfig,
new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)),
- proxyClientAuthentication));
+ proxyClientAuthentication, 10, TimeUnit.MILLISECONDS));
doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper)))
.when(proxyService).createLocalMetadataStore();
doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService)
@@ -224,6 +224,18 @@ public class ProxyStatsTest extends MockedPulsarServiceBaseTest {
assertTrue(proxyService.getTopicStats().isEmpty());
}
+ @Test
+ public void testStatsUpdatedPeriodically() {
+ String topicName = "periodic-stats-update";
+ TopicStats stats = Mockito.spy(new TopicStats());
+ proxyService.getTopicStats().put(topicName, stats);
+ try {
+ Mockito.verify(stats, Mockito.timeout(5000).atLeast(2)).calculate();
+ } finally {
+ proxyService.getTopicStats().remove(topicName);
+ }
+ }
+
@Test
public void testMemoryLeakFixed() throws Exception {
proxyService.setProxyLogLevel(2);
(pulsar) 05/14: [fix][fn] Fix Go function incorrectly using nanoseconds for timestamp (#25869)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 8722988f2d416872f038d2ed1d3d28841e3d7778
Author: Dream95 <86...@qq.com>
AuthorDate: Sat Oct 17 09:13:13 2026 +0800
[fix][fn] Fix Go function incorrectly using nanoseconds for timestamp (#25869)
Signed-off-by: Dream95 <zh...@163.com>
Signed-off-by: Matteo Merli <mm...@apache.org>
Co-authored-by: Matteo Merli <mm...@apache.org>
(cherry picked from commit efe1465d7eacc89c01dd595888dee340423433b6)
---
pulsar-function-go/pf/instance.go | 5 +++--
pulsar-function-go/pf/stats.go | 6 +++---
pulsar-function-go/pf/stats_test.go | 25 +++++++++++++++++++++++++
3 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index f2b5dfd50b80..581c6d371b55 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -725,11 +725,12 @@ func (gi *goInstance) getAvgProcessLatency() float32 {
return float32(*sum) / float32(*count)
}
-func (gi *goInstance) getLastInvocation() float32 {
+func (gi *goInstance) getLastInvocation() float64 {
metric := gi.getMatchingMetricFromRegistry(PulsarFunctionMetricsPrefix + LastInvocation)
// "pulsar_function_" + "last_invocation", GaugeVec.
+ // float64, as a float32 would round an epoch-millis timestamp to ~2 minutes.
val := metric.GetGauge().Value
- return float32(*val)
+ return *val
}
func (gi *goInstance) getTotalProcessedSuccessfully1min() float32 {
diff --git a/pulsar-function-go/pf/stats.go b/pulsar-function-go/pf/stats.go
index d25424b37047..a1b7035b3b2a 100644
--- a/pulsar-function-go/pf/stats.go
+++ b/pulsar-function-go/pf/stats.go
@@ -248,7 +248,7 @@ func getFirstMatch(
func (stat *StatWithLabelValues) setLastInvocation() {
now := time.Now()
- stat.statLastInvocation.Set(float64(now.UnixNano()))
+ stat.statLastInvocation.Set(float64(now.UnixMilli()))
}
func (stat *StatWithLabelValues) processTimeStart() {
@@ -272,7 +272,7 @@ func (stat *StatWithLabelValues) incrTotalUserExceptions(err error) {
func (stat *StatWithLabelValues) addUserException(err error) {
now := time.Now()
- ts := now.UnixNano()
+ ts := now.UnixMilli()
errorTS := LatestException{err, ts}
stat.latestUserException = append(stat.latestUserException, errorTS)
if len(stat.latestUserException) > 10 {
@@ -302,7 +302,7 @@ func (stat *StatWithLabelValues) incrTotalSysExceptions(exception error) {
func (stat *StatWithLabelValues) addSysException(exception error) {
now := time.Now()
- ts := now.UnixNano()
+ ts := now.UnixMilli()
errorTS := LatestException{exception, ts}
stat.latestSysException = append(stat.latestSysException, errorTS)
if len(stat.latestSysException) > 10 {
diff --git a/pulsar-function-go/pf/stats_test.go b/pulsar-function-go/pf/stats_test.go
index 550ef18aa8f0..2c0f5b61778d 100644
--- a/pulsar-function-go/pf/stats_test.go
+++ b/pulsar-function-go/pf/stats_test.go
@@ -21,6 +21,7 @@ package pf
import (
"context"
+ "errors"
"fmt"
"io"
"math"
@@ -216,3 +217,27 @@ func TestInstanceControlMetrics(t *testing.T) {
assert.EqualValuesf(t, value+1, metrics.UserMetrics[label], "user metric %s != %d", label, value+1)
}
}
+
+func TestStatsTimestampsAreMillis(t *testing.T) {
+ gi := newGoInstance()
+ t.Cleanup(gi.close)
+
+ before := time.Now().UnixMilli()
+ gi.stats.setLastInvocation()
+ gi.stats.incrTotalUserExceptions(errors.New("user error"))
+ gi.stats.incrTotalSysExceptions(errors.New("system error"))
+ after := time.Now().UnixMilli()
+
+ status := gi.getFunctionStatus()
+ assert.GreaterOrEqual(t, status.LastInvocationTime, before)
+ assert.LessOrEqual(t, status.LastInvocationTime, after)
+ assert.Len(t, status.LatestUserExceptions, 1)
+ assert.GreaterOrEqual(t, status.LatestUserExceptions[0].MsSinceEpoch, before)
+ assert.LessOrEqual(t, status.LatestUserExceptions[0].MsSinceEpoch, after)
+ assert.Len(t, status.LatestSystemExceptions, 1)
+ assert.GreaterOrEqual(t, status.LatestSystemExceptions[0].MsSinceEpoch, before)
+ assert.LessOrEqual(t, status.LatestSystemExceptions[0].MsSinceEpoch, after)
+
+ metrics := gi.getMetrics()
+ assert.Equal(t, status.LastInvocationTime, metrics.LastInvocation)
+}
(pulsar) 11/14: [improve][broker] Keep each ledger's pending acks in sorted blocks instead of an AVL tree (#26955)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit d79b840ad08a43823d3e9cdfdff1e43b6431b860
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Oct 16 21:42:20 2026 -0700
[improve][broker] Keep each ledger's pending acks in sorted blocks instead of an AVL tree (#26955)
(cherry picked from commit 8fd72345a2a389e07c29cd10b23c46c73d4cbbaa)
---
.../service/PendingAcksLedgerMapBenchmark.java | 373 ++++++++++++++
.../service/PendingAcksMapOneEntryBenchmark.java | 62 +++
.../broker/service/PendingAcksLedgerMap.java | 535 +++++++++++++++++++++
.../pulsar/broker/service/PendingAcksMap.java | 103 ++--
.../broker/service/PendingAcksLedgerMapTest.java | 501 +++++++++++++++++++
.../pulsar/broker/service/PendingAcksMapTest.java | 179 +++++++
6 files changed, 1708 insertions(+), 45 deletions(-)
diff --git a/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAcksLedgerMapBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAcksLedgerMapBenchmark.java
new file mode 100644
index 000000000000..4d193dde5ceb
--- /dev/null
+++ b/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAcksLedgerMapBenchmark.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import it.unimi.dsi.fastutil.longs.Long2LongAVLTreeMap;
+import it.unimi.dsi.fastutil.longs.Long2LongMap;
+import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
+import java.util.Arrays;
+import java.util.SplittableRandom;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * Compares the per-ledger structures of a consumer's pending acks: the AVL tree that {@link PendingAcksMap} used
+ * before, {@code Long2LongAVLTreeMap}, and {@link PendingAcksLedgerMap}, in the patterns of dispatch and acks. It
+ * excludes {@link PendingAcksMap}'s lock and outer ledger map, which are the same for both and which
+ * {@link PendingAckMapCycleBenchmark} includes.
+ *
+ * <p>The window is the number of pending acks. Every operation that removes an entry adds a new one after the others,
+ * as a dispatch does, so that the window stays the same. The results are per entry, except for {@code traverse},
+ * which is per traversal of the window.
+ *
+ * <p>{@link PendingAcksMap} reuses an emptied ledger map, where it created a new AVL tree before:
+ * {@code oneEntryAtATime} does the same with each structure.
+ */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+public class PendingAcksLedgerMapBenchmark {
+ // the entries that an ack command acknowledges together, and that a mark-delete removes together
+ private static final int GROUP = 64;
+ // one in this many acked entries is redelivered: removed and added again later, out of order
+ private static final int REDELIVERY_INTERVAL = 16;
+ // a redelivered entry is added again this many acks later
+ private static final int REDELIVERY_DELAY = 64;
+ // the number of maps in tinyMaps, each with a window of TINY_WINDOW entries
+ private static final int TINY_MAPS = 8192;
+ private static final int TINY_WINDOW = 8;
+ // the length of the precomputed sequence of random indexes that ackShuffled acks
+ private static final int RANDOM_INDEXES = 1 << 20;
+
+ /** The operations of a ledger's pending acks that the patterns use. */
+ interface LedgerMap {
+ long get(long entryId);
+
+ void put(long entryId, long value);
+
+ long remove(long entryId);
+
+ /** Removes the entries up to the entry ID, inclusive, as a mark-delete does; returns their count. */
+ int removeUpTo(long entryId);
+
+ void forEach(Blackhole blackhole);
+ }
+
+ /** The previous structure, as {@link PendingAcksMap} used it. */
+ static final class AvlTreeLedgerMap implements LedgerMap {
+ private final Long2LongAVLTreeMap map = new Long2LongAVLTreeMap();
+
+ AvlTreeLedgerMap() {
+ map.defaultReturnValue(PendingAckValues.PACKED_NOT_FOUND);
+ }
+
+ @Override
+ public long get(long entryId) {
+ return map.get(entryId);
+ }
+
+ @Override
+ public void put(long entryId, long value) {
+ map.put(entryId, value);
+ }
+
+ @Override
+ public long remove(long entryId) {
+ return map.remove(entryId);
+ }
+
+ @Override
+ public int removeUpTo(long entryId) {
+ int removed = 0;
+ ObjectBidirectionalIterator<Long2LongMap.Entry> iterator =
+ map.headMap(entryId + 1).long2LongEntrySet().iterator();
+ while (iterator.hasNext()) {
+ iterator.next();
+ iterator.remove();
+ removed++;
+ }
+ return removed;
+ }
+
+ @Override
+ public void forEach(Blackhole blackhole) {
+ for (Long2LongMap.Entry entry : map.long2LongEntrySet()) {
+ blackhole.consume(entry.getLongKey());
+ blackhole.consume(entry.getLongValue());
+ }
+ }
+ }
+
+ /** The new structure, as {@link PendingAcksMap} uses it. */
+ static final class BlockListLedgerMap implements LedgerMap {
+ private final PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+
+ @Override
+ public long get(long entryId) {
+ return map.get(entryId);
+ }
+
+ @Override
+ public void put(long entryId, long value) {
+ map.put(entryId, value);
+ }
+
+ @Override
+ public long remove(long entryId) {
+ return map.remove(entryId);
+ }
+
+ @Override
+ public int removeUpTo(long entryId) {
+ // as PendingAcksMap.removeAllUpTo does, which tolerates callbacks that change the map
+ int removed = 0;
+ long next = map.hasEntriesUpTo(entryId) ? map.firstEntryId() : PendingAcksLedgerMap.NO_ENTRY;
+ while (next != PendingAcksLedgerMap.NO_ENTRY && next <= entryId) {
+ if (!PendingAckValues.isNotFound(map.get(next))) {
+ map.remove(next);
+ removed++;
+ }
+ next = map.higherEntryId(next);
+ }
+ return removed;
+ }
+
+ @Override
+ public void forEach(Blackhole blackhole) {
+ map.forEach((entryId, value) -> {
+ blackhole.consume(entryId);
+ blackhole.consume(value);
+ });
+ }
+ }
+
+ @Param({"AVL_TREE", "BLOCK_LIST"})
+ public String implementation;
+
+ @Param({"1024", "65536"})
+ public int window;
+
+ private final long packedValue = PendingAckValues.pack(1, 0);
+
+ private LedgerMap map;
+ // the window's entry IDs; an operation acks the entry at an index and replaces it with a new one
+ private long[] entryIds;
+ private long nextEntryId;
+ private int cursor;
+ // the negatively acked entry IDs, added again REDELIVERY_DELAY acks after their removal
+ private long[] negativelyAcked;
+ // the redelivered entry IDs, acked REDELIVERY_DELAY acks after they were added again
+ private long[] redelivered;
+ private int acks;
+
+ private LedgerMap[] tinyMaps;
+ private long[] tinyNextEntryIds;
+ // the index of a random entry of the window, for each ack of ackShuffled
+ private int[] randomIndexes;
+ // the emptied map that oneEntryAtATime reuses with the block list
+ private LedgerMap emptyMap;
+ private long nextSingleEntryId;
+
+ private LedgerMap newLedgerMap() {
+ return "AVL_TREE".equals(implementation) ? new AvlTreeLedgerMap() : new BlockListLedgerMap();
+ }
+
+ @Setup(Level.Trial)
+ public void setup() {
+ map = newLedgerMap();
+ entryIds = new long[window];
+ for (int i = 0; i < window; i++) {
+ entryIds[i] = i;
+ map.put(i, packedValue);
+ }
+ nextEntryId = window;
+ negativelyAcked = new long[REDELIVERY_DELAY];
+ Arrays.fill(negativelyAcked, -1);
+ redelivered = new long[REDELIVERY_DELAY];
+ Arrays.fill(redelivered, -1);
+ tinyMaps = new LedgerMap[TINY_MAPS];
+ tinyNextEntryIds = new long[TINY_MAPS];
+ for (int m = 0; m < TINY_MAPS; m++) {
+ tinyMaps[m] = newLedgerMap();
+ for (int i = 0; i < TINY_WINDOW; i++) {
+ tinyMaps[m].put(i, packedValue);
+ }
+ tinyNextEntryIds[m] = TINY_WINDOW;
+ }
+ SplittableRandom random = new SplittableRandom(1);
+ randomIndexes = new int[RANDOM_INDEXES];
+ for (int i = 0; i < RANDOM_INDEXES; i++) {
+ randomIndexes[i] = random.nextInt(window);
+ }
+ emptyMap = newLedgerMap();
+ }
+
+ /** The index of the next entry to ack: in order, or a random one of the window. */
+ private int nextIndex(boolean shuffled) {
+ int next = cursor++;
+ return shuffled ? randomIndexes[next & (RANDOM_INDEXES - 1)] : next & (window - 1);
+ }
+
+ // Acks the entry at the index: looks it up and removes it, and dispatches a new entry in its place
+ private long ackAndReplace(int index) {
+ long entryId = entryIds[index];
+ long value = map.get(entryId);
+ map.remove(entryId);
+ long added = nextEntryId++;
+ map.put(added, packedValue);
+ entryIds[index] = added;
+ return value;
+ }
+
+ /** Acks arrive in the order of the entries, the common case: lookup, removal and a new entry. */
+ @Benchmark
+ public long ackInOrder() {
+ return ackAndReplace(nextIndex(false));
+ }
+
+ /**
+ * Acks arrive in a random order over the window, as when a consumer processes messages at different speeds: each
+ * ack is for a random entry of the window, which is replaced by a new one.
+ */
+ @Benchmark
+ public long ackShuffled() {
+ return ackAndReplace(nextIndex(true));
+ }
+
+ /**
+ * An ack command acknowledges a group of entries in order: they are all looked up before any of them is removed
+ * after the ack is persisted, and new entries are dispatched after that.
+ */
+ @Benchmark
+ @OperationsPerInvocation(GROUP)
+ public long ackGroupInOrder() {
+ int first = cursor;
+ cursor += GROUP;
+ long sum = 0;
+ for (int i = 0; i < GROUP; i++) {
+ sum += map.get(entryIds[(first + i) & (window - 1)]);
+ }
+ for (int i = 0; i < GROUP; i++) {
+ map.remove(entryIds[(first + i) & (window - 1)]);
+ }
+ for (int i = 0; i < GROUP; i++) {
+ long added = nextEntryId++;
+ map.put(added, packedValue);
+ entryIds[(first + i) & (window - 1)] = added;
+ }
+ return sum;
+ }
+
+ /**
+ * Acks in order, where one in {@value #REDELIVERY_INTERVAL} entries is negatively acked: it is removed, added again
+ * {@value #REDELIVERY_DELAY} acks later, older than the others, and acked another {@value #REDELIVERY_DELAY} acks
+ * later. The result is per ack of a dispatched entry, including the redeliveries' share.
+ */
+ @Benchmark
+ public long ackInOrderWithRedeliveries() {
+ int slot = acks++ & (REDELIVERY_DELAY - 1);
+ long value = 0;
+ if (redelivered[slot] >= 0) {
+ // the redelivered entry is acked
+ value += map.get(redelivered[slot]);
+ map.remove(redelivered[slot]);
+ redelivered[slot] = -1;
+ }
+ if (negativelyAcked[slot] >= 0) {
+ // the negatively acked entry is dispatched again, out of order
+ map.put(negativelyAcked[slot], packedValue);
+ redelivered[slot] = negativelyAcked[slot];
+ negativelyAcked[slot] = -1;
+ }
+ int index = nextIndex(false);
+ long entryId = entryIds[index];
+ value += map.get(entryId);
+ map.remove(entryId);
+ if (acks % REDELIVERY_INTERVAL == 0) {
+ // negatively acked instead
+ negativelyAcked[slot] = entryId;
+ }
+ long added = nextEntryId++;
+ map.put(added, packedValue);
+ entryIds[index] = added;
+ return value;
+ }
+
+ /** A mark-delete removes the oldest entries up to a position, and new entries are dispatched. */
+ @Benchmark
+ @OperationsPerInvocation(GROUP)
+ public int removeUpToMarkDelete() {
+ long upTo = nextEntryId - window + GROUP - 1;
+ int removed = map.removeUpTo(upTo);
+ for (int i = 0; i < GROUP; i++) {
+ map.put(nextEntryId++, packedValue);
+ }
+ return removed;
+ }
+
+ /** A traversal of the window, as a redelivery of all pending acks or a consumer's close does. */
+ @Benchmark
+ public void traverse(Blackhole blackhole) {
+ map.forEach(blackhole);
+ }
+
+ /**
+ * A consumer whose pending acks drop to none between entries, as on a topic with a low rate: the ledger's map is
+ * removed when it empties, and a map is taken for the next entry. {@link PendingAcksMap} created a new AVL tree
+ * for it, and reuses the emptied block list.
+ */
+ @Benchmark
+ public long oneEntryAtATime() {
+ LedgerMap single = "AVL_TREE".equals(implementation) ? new AvlTreeLedgerMap() : emptyMap;
+ long entryId = nextSingleEntryId++;
+ single.put(entryId, packedValue);
+ long value = single.get(entryId);
+ single.remove(entryId);
+ return value;
+ }
+
+ /** Many consumers with a few pending acks each, as with many slow or idle consumers: an ack on a random one. */
+ @Benchmark
+ public long tinyMaps() {
+ int m = (cursor++ * 4051) & (TINY_MAPS - 1);
+ LedgerMap tiny = tinyMaps[m];
+ long next = tinyNextEntryIds[m]++;
+ long entryId = next - TINY_WINDOW;
+ long value = tiny.get(entryId);
+ tiny.remove(entryId);
+ tiny.put(next, packedValue);
+ return value;
+ }
+}
diff --git a/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAcksMapOneEntryBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAcksMapOneEntryBenchmark.java
new file mode 100644
index 000000000000..6cf77b76c3e3
--- /dev/null
+++ b/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAcksMapOneEntryBenchmark.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * A consumer whose pending acks drop to none between messages, as on a topic with a low rate, with the real
+ * pending-ACK map's locks and outer ledger map: each message is added, looked up when its ack arrives and removed,
+ * which empties the map.
+ */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+public class PendingAcksMapOneEntryBenchmark {
+ private PendingAcksMap pendingAcks;
+ private long nextEntryId;
+
+ @Setup
+ public void setup() {
+ Consumer owner = new Consumer("benchmark-owner", 0);
+ pendingAcks = new PendingAcksMap(owner, () -> null, () -> null);
+ }
+
+ @Benchmark
+ public int oneEntryAtATime() {
+ long entryId = nextEntryId++;
+ pendingAcks.addPendingAckIfAllowed(1, entryId, 1, 0);
+ int remainingUnacked = pendingAcks.getRemainingUnacked(1, entryId);
+ pendingAcks.removeAndGetRemainingUnacked(1, entryId);
+ return remainingUnacked;
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksLedgerMap.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksLedgerMap.java
new file mode 100644
index 000000000000..2a0948bd86e0
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksLedgerMap.java
@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/**
+ * The pending acks of one ledger: a map from entry ID to a packed {@link PendingAckValues} value, in the order of the
+ * entry IDs. Not thread-safe; {@link PendingAcksMap} guards it with its lock.
+ *
+ * <p>The entries are kept in a list of blocks, each a sorted array of up to {@value #BLOCK_CAPACITY} entries, in the
+ * order of their entry IDs, with each block's entries in a window of its arrays. A consumer is sent a ledger's entries
+ * mostly in increasing order and acks them mostly in the same order, so that adding an entry appends it to the last
+ * block and removing one advances the first block's window; a lookup is a binary search over the blocks' first
+ * entry IDs, kept in one array, and over one block's consecutive memory, without the pointer chasing of a tree. An
+ * entry added out of order, as a redelivery adds it, shifts the entries of one block at most, which splits when it is
+ * full. Empty blocks are dropped and a block that becomes sparse merges with its neighbor, so that the memory follows
+ * the number of entries. A lone block starts small, grows as entries are added and shrinks as they are removed, so
+ * that a map of a few entries stays small. While the map has more than one block, the last emptied block is kept for
+ * the next one, so that a window of entries that moves through the map doesn't allocate.
+ *
+ * <p>The methods that read the map don't change it: {@link PendingAcksMap} calls them concurrently under its read
+ * lock.
+ */
+final class PendingAcksLedgerMap {
+ static final int BLOCK_CAPACITY = 128;
+ // The smallest capacity of a block: a lone block starts with it, doubles up to BLOCK_CAPACITY as entries are added
+ // and halves as they are removed
+ static final int INITIAL_BLOCK_CAPACITY = 16;
+ // A block with fewer entries than this merges with a neighbor when they fit in half a block together
+ private static final int MERGE_THRESHOLD = BLOCK_CAPACITY / 4;
+
+ private static final class Block {
+ long[] entryIds;
+ long[] values;
+ // the entries are in [start, end)
+ int start;
+ int end;
+
+ Block(int capacity) {
+ entryIds = new long[capacity];
+ values = new long[capacity];
+ }
+
+ int capacity() {
+ return entryIds.length;
+ }
+
+ int size() {
+ return end - start;
+ }
+
+ long firstEntryId() {
+ return entryIds[start];
+ }
+
+ long lastEntryId() {
+ return entryIds[end - 1];
+ }
+
+ int indexOf(long entryId) {
+ return Arrays.binarySearch(entryIds, start, end, entryId);
+ }
+
+ /** Inserts at the insertion point, which keeps the order; the block's arrays mustn't be full. */
+ void insertAt(int insertionPoint, long entryId, long value) {
+ boolean roomBefore = start > 0;
+ boolean roomAfter = end < capacity();
+ if (roomBefore && (!roomAfter || insertionPoint - start < end - insertionPoint)) {
+ // shift the entries before the insertion point to the left
+ System.arraycopy(entryIds, start, entryIds, start - 1, insertionPoint - start);
+ System.arraycopy(values, start, values, start - 1, insertionPoint - start);
+ start--;
+ entryIds[insertionPoint - 1] = entryId;
+ values[insertionPoint - 1] = value;
+ } else {
+ System.arraycopy(entryIds, insertionPoint, entryIds, insertionPoint + 1, end - insertionPoint);
+ System.arraycopy(values, insertionPoint, values, insertionPoint + 1, end - insertionPoint);
+ end++;
+ entryIds[insertionPoint] = entryId;
+ values[insertionPoint] = value;
+ }
+ }
+
+ void removeAt(int index) {
+ if (index - start < end - 1 - index) {
+ System.arraycopy(entryIds, start, entryIds, start + 1, index - start);
+ System.arraycopy(values, start, values, start + 1, index - start);
+ start++;
+ } else {
+ System.arraycopy(entryIds, index + 1, entryIds, index, end - 1 - index);
+ System.arraycopy(values, index + 1, values, index, end - 1 - index);
+ end--;
+ }
+ if (start == end) {
+ start = 0;
+ end = 0;
+ }
+ }
+
+ /** Moves the entries to the start of the arrays. */
+ void compact() {
+ if (start > 0) {
+ System.arraycopy(entryIds, start, entryIds, 0, size());
+ System.arraycopy(values, start, values, 0, size());
+ end -= start;
+ start = 0;
+ }
+ }
+
+ /** Grows the arrays to at least the capacity, doubling, up to {@link #BLOCK_CAPACITY}; compacts them. */
+ void grow(int minCapacity) {
+ int capacity = capacity();
+ while (capacity < minCapacity) {
+ capacity *= 2;
+ }
+ resize(Math.min(capacity, BLOCK_CAPACITY));
+ }
+
+ /** Moves the entries to new arrays of the capacity, which must hold them. */
+ void resize(int capacity) {
+ int size = size();
+ entryIds = Arrays.copyOfRange(entryIds, start, start + capacity);
+ values = Arrays.copyOfRange(values, start, start + capacity);
+ start = 0;
+ end = size;
+ }
+ }
+
+ /** A callback for each entry, in the order of the entry IDs. */
+ interface EntryConsumer {
+ void accept(long entryId, long value);
+ }
+
+ /**
+ * Returned by {@link #higherEntryId} when there's no entry after the entry ID. It can't be an entry ID, since
+ * BookKeeper's entry IDs aren't negative.
+ */
+ static final long NO_ENTRY = Long.MIN_VALUE;
+
+ private final ArrayList<Block> blocks = new ArrayList<>(1);
+ // the first entry ID of each block, in step with the blocks, for the binary search over the blocks
+ private long[] firstEntryIds = new long[1];
+ // The last emptied block, reused for the next block: one of the full capacity while the map has more than one
+ // block, so that a window that moves through the map doesn't allocate, or the lone block of the smallest capacity
+ // that an empty map had, so that a map that empties between entries, which PendingAcksMap reuses, doesn't allocate
+ private Block spareBlock;
+ // changed when an entry is added or removed, so that a traversal notices when its callback changed the map
+ private int modCount;
+ // the most blocks since the list's capacity was last trimmed, to release the capacity after the map shrinks
+ private int peakBlocks;
+ private int size;
+ // the blocks allocated by this map, for tests
+ private int allocatedBlocks;
+
+ int size() {
+ return size;
+ }
+
+ boolean isEmpty() {
+ return size == 0;
+ }
+
+ @VisibleForTesting
+ int blockCount() {
+ return blocks.size();
+ }
+
+ @VisibleForTesting
+ int allocatedBlocks() {
+ return allocatedBlocks;
+ }
+
+ @VisibleForTesting
+ int blockCapacity(int blockIndex) {
+ return blocks.get(blockIndex).capacity();
+ }
+
+ /** Checks the map's internal consistency, for tests. */
+ @VisibleForTesting
+ void checkInvariants() {
+ int entries = 0;
+ long previous = Long.MIN_VALUE;
+ for (int b = 0; b < blocks.size(); b++) {
+ Block block = blocks.get(b);
+ if (block.size() == 0) {
+ throw new IllegalStateException("empty block " + b);
+ }
+ if (firstEntryIds[b] != block.firstEntryId()) {
+ throw new IllegalStateException("first entry ID of block " + b + " is " + block.firstEntryId()
+ + ", indexed as " + firstEntryIds[b]);
+ }
+ for (int i = block.start; i < block.end; i++) {
+ if (block.entryIds[i] <= previous) {
+ throw new IllegalStateException("entry " + block.entryIds[i] + " after " + previous);
+ }
+ previous = block.entryIds[i];
+ }
+ entries += block.size();
+ }
+ if (entries != size) {
+ throw new IllegalStateException(entries + " entries, size " + size);
+ }
+ if (spareBlock != null && spareBlock.size() != 0) {
+ throw new IllegalStateException("a spare block with entries");
+ }
+ if (spareBlock != null && (blocks.size() == 1
+ || blocks.isEmpty() && spareBlock.capacity() != INITIAL_BLOCK_CAPACITY)) {
+ throw new IllegalStateException("a spare block of capacity " + spareBlock.capacity() + " with "
+ + blocks.size() + " blocks");
+ }
+ if (blocks.size() > 1) {
+ for (Block block : blocks) {
+ if (block.capacity() != BLOCK_CAPACITY) {
+ throw new IllegalStateException("a block of capacity " + block.capacity() + " among others");
+ }
+ }
+ }
+ }
+
+ /** The entry's value, or {@link PendingAckValues#PACKED_NOT_FOUND}. */
+ long get(long entryId) {
+ if (isFirst(entryId)) {
+ return firstValue();
+ }
+ int blockIndex = blockIndexFor(entryId);
+ if (blockIndex < 0) {
+ return PendingAckValues.PACKED_NOT_FOUND;
+ }
+ Block block = blocks.get(blockIndex);
+ int index = block.indexOf(entryId);
+ return index >= 0 ? block.values[index] : PendingAckValues.PACKED_NOT_FOUND;
+ }
+
+ // Whether the entry is the first one: acks mostly come in the order of the entries, and both the ack's lookup and
+ // its removal then find it without a search
+ private boolean isFirst(long entryId) {
+ return size > 0 && firstEntryId() == entryId;
+ }
+
+ boolean containsKey(long entryId) {
+ return !PendingAckValues.isNotFound(get(entryId));
+ }
+
+ /**
+ * Sets the entry's value, which mustn't be {@link PendingAckValues#PACKED_NOT_FOUND}.
+ *
+ * @return the previous value, or {@link PendingAckValues#PACKED_NOT_FOUND}
+ */
+ long put(long entryId, long value) {
+ Block last = blocks.isEmpty() ? null : blocks.get(blocks.size() - 1);
+ if (last == null || entryId > last.lastEntryId()) {
+ // the common case: an entry after the others
+ boolean newBlock = false;
+ if (last == null || last.end == last.capacity()) {
+ if (last != null && last.start >= last.capacity() / 4) {
+ // compacting frees a quarter of the block at least, so that it doesn't happen on every append
+ last.compact();
+ } else if (last != null && last.capacity() < BLOCK_CAPACITY) {
+ last.grow(last.capacity() + 1);
+ } else {
+ // a map's first block starts small, the blocks after it as full ones
+ last = newBlock(last == null ? INITIAL_BLOCK_CAPACITY : BLOCK_CAPACITY);
+ newBlock = true;
+ }
+ }
+ last.entryIds[last.end] = entryId;
+ last.values[last.end] = value;
+ last.end++;
+ if (newBlock) {
+ addBlock(blocks.size(), last);
+ }
+ size++;
+ modCount++;
+ return PendingAckValues.PACKED_NOT_FOUND;
+ }
+ // the first block whose entries reach the entry ID
+ int blockIndex = Math.max(blockIndexFor(entryId), 0);
+ Block block = blocks.get(blockIndex);
+ int index = block.indexOf(entryId);
+ if (index >= 0) {
+ long previous = block.values[index];
+ block.values[index] = value;
+ return previous;
+ }
+ if (block.size() == BLOCK_CAPACITY) {
+ Block upper = split(blockIndex);
+ if (entryId > block.lastEntryId()) {
+ block = upper;
+ blockIndex++;
+ }
+ index = block.indexOf(entryId);
+ } else if (block.size() == block.capacity()) {
+ // a small first block that is full
+ block.grow(block.capacity() + 1);
+ index = block.indexOf(entryId);
+ }
+ block.insertAt(-index - 1, entryId, value);
+ firstEntryIds[blockIndex] = block.firstEntryId();
+ size++;
+ modCount++;
+ return PendingAckValues.PACKED_NOT_FOUND;
+ }
+
+ /**
+ * Removes the entry.
+ *
+ * @return its value, or {@link PendingAckValues#PACKED_NOT_FOUND} if it wasn't in the map
+ */
+ long remove(long entryId) {
+ if (isFirst(entryId)) {
+ long previous = firstValue();
+ removeFirst();
+ return previous;
+ }
+ int blockIndex = blockIndexFor(entryId);
+ if (blockIndex < 0) {
+ return PendingAckValues.PACKED_NOT_FOUND;
+ }
+ Block block = blocks.get(blockIndex);
+ int index = block.indexOf(entryId);
+ if (index < 0) {
+ return PendingAckValues.PACKED_NOT_FOUND;
+ }
+ long previous = block.values[index];
+ removeAt(blockIndex, block, index);
+ return previous;
+ }
+
+ /** The first entry's ID; the map mustn't be empty. */
+ long firstEntryId() {
+ return firstEntryIds[0];
+ }
+
+ /** The first entry's value; the map mustn't be empty. */
+ long firstValue() {
+ Block first = blocks.get(0);
+ return first.values[first.start];
+ }
+
+ /** Removes the first entry; the map mustn't be empty. */
+ void removeFirst() {
+ Block first = blocks.get(0);
+ removeAt(0, first, first.start);
+ }
+
+ /** Whether the map has an entry with an ID up to {@code maxEntryId}, inclusive. */
+ boolean hasEntriesUpTo(long maxEntryId) {
+ return size > 0 && firstEntryId() <= maxEntryId;
+ }
+
+ /**
+ * Calls the consumer for each entry, in the order of the entry IDs. The consumer may add and remove entries: the
+ * traversal continues with the first entry after the current one as the map is then, so it visits the entries
+ * added after the current one and not those added before it, and never an entry that was removed.
+ */
+ void forEach(EntryConsumer consumer) {
+ for (int b = 0; b < blocks.size(); b++) {
+ Block block = blocks.get(b);
+ for (int i = block.start; i < block.end; i++) {
+ long entryId = block.entryIds[i];
+ int expectedModCount = modCount;
+ consumer.accept(entryId, block.values[i]);
+ if (modCount != expectedModCount) {
+ // the entries may have moved: continue after the entry, wherever the next one is now
+ for (long next = higherEntryId(entryId); next != NO_ENTRY; next = higherEntryId(next)) {
+ consumer.accept(next, get(next));
+ }
+ return;
+ }
+ }
+ }
+ }
+
+ /** The ID of the first entry after the entry ID, or {@link #NO_ENTRY}. */
+ long higherEntryId(long entryId) {
+ if (size == 0) {
+ return NO_ENTRY;
+ }
+ if (firstEntryId() > entryId) {
+ // the common case when entries are removed in order
+ return firstEntryId();
+ }
+ int blockIndex = blockIndexFor(entryId);
+ Block block = blocks.get(blockIndex);
+ int index = block.indexOf(entryId);
+ int next = index >= 0 ? index + 1 : -index - 1;
+ if (next < block.end) {
+ return block.entryIds[next];
+ }
+ return blockIndex + 1 < blocks.size() ? firstEntryIds[blockIndex + 1] : NO_ENTRY;
+ }
+
+ /** The index of the last block whose first entry ID is at most the entry ID, or -1. */
+ private int blockIndexFor(long entryId) {
+ int low = 0;
+ int high = blocks.size() - 1;
+ int found = -1;
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ if (firstEntryIds[mid] <= entryId) {
+ found = mid;
+ low = mid + 1;
+ } else {
+ high = mid - 1;
+ }
+ }
+ return found;
+ }
+
+ /** A block for the next entries: the spare one, or a new one of the capacity. */
+ private Block newBlock(int capacity) {
+ Block block = spareBlock;
+ if (block != null && block.capacity() >= capacity) {
+ spareBlock = null;
+ return block;
+ }
+ allocatedBlocks++;
+ return new Block(capacity);
+ }
+
+ /** Adds a block that has entries at the index, with its first entry ID. */
+ private void addBlock(int blockIndex, Block block) {
+ int count = blocks.size();
+ if (count == firstEntryIds.length) {
+ firstEntryIds = Arrays.copyOf(firstEntryIds, count * 2);
+ }
+ System.arraycopy(firstEntryIds, blockIndex, firstEntryIds, blockIndex + 1, count - blockIndex);
+ firstEntryIds[blockIndex] = block.firstEntryId();
+ blocks.add(blockIndex, block);
+ peakBlocks = Math.max(peakBlocks, blocks.size());
+ }
+
+ /** Removes the block at the index, keeping it as the spare one. */
+ private void removeBlock(int blockIndex) {
+ Block block = blocks.remove(blockIndex);
+ System.arraycopy(firstEntryIds, blockIndex + 1, firstEntryIds, blockIndex, blocks.size() - blockIndex);
+ // a block merged into another one still has its window
+ block.start = 0;
+ block.end = 0;
+ spareBlock = block;
+ }
+
+ /** Splits a full block in two halves; returns the new upper half, which follows it in the list. */
+ private Block split(int blockIndex) {
+ Block block = blocks.get(blockIndex);
+ Block upper = newBlock(BLOCK_CAPACITY);
+ int half = block.size() / 2;
+ int from = block.start + half;
+ int count = block.end - from;
+ System.arraycopy(block.entryIds, from, upper.entryIds, 0, count);
+ System.arraycopy(block.values, from, upper.values, 0, count);
+ upper.end = count;
+ block.end = from;
+ addBlock(blockIndex + 1, upper);
+ return upper;
+ }
+
+ private void removeAt(int blockIndex, Block block, int index) {
+ block.removeAt(index);
+ size--;
+ modCount++;
+ if (block.size() == 0) {
+ // Removing the first block shifts the references to the others and the first entry IDs: draining N
+ // entries in order moves about (N / BLOCK_CAPACITY)^2 references in all. That is a small part of the
+ // entries' own handling up to about a million pending acks of one consumer in one ledger, and the AVL
+ // tree's cost at about 8 million, far above the default maxUnackedMessagesPerConsumer of 50,000
+ removeBlock(blockIndex);
+ } else {
+ firstEntryIds[blockIndex] = block.firstEntryId();
+ if (block.size() < MERGE_THRESHOLD) {
+ mergeWithNeighbor(blockIndex, block);
+ }
+ }
+ if (blocks.size() == 1) {
+ // a lone block shrinks with its entries, and the map keeps no spare block of the full capacity for it
+ spareBlock = null;
+ Block lone = blocks.get(0);
+ if (lone.capacity() > INITIAL_BLOCK_CAPACITY && lone.size() < lone.capacity() / 4) {
+ lone.resize(lone.capacity() / 2);
+ }
+ } else if (blocks.isEmpty() && (spareBlock == null || spareBlock.capacity() != INITIAL_BLOCK_CAPACITY)) {
+ // an empty map keeps a spare block of the smallest capacity only
+ spareBlock = null;
+ }
+ if (!blocks.isEmpty() && blocks.size() * 4 < peakBlocks) {
+ // the list keeps the capacity of its largest size: release it once a quarter of it is in use
+ blocks.trimToSize();
+ firstEntryIds = Arrays.copyOf(firstEntryIds, Math.max(blocks.size(), 1));
+ peakBlocks = blocks.size();
+ }
+ }
+
+ /** Merges a sparse block with the next or the previous one, when they fit in one block together. */
+ private void mergeWithNeighbor(int blockIndex, Block block) {
+ if (blockIndex + 1 < blocks.size() && block.size() + blocks.get(blockIndex + 1).size() <= BLOCK_CAPACITY / 2) {
+ moveInto(block, blocks.get(blockIndex + 1));
+ removeBlock(blockIndex + 1);
+ } else if (blockIndex > 0 && block.size() + blocks.get(blockIndex - 1).size() <= BLOCK_CAPACITY / 2) {
+ moveInto(blocks.get(blockIndex - 1), block);
+ removeBlock(blockIndex);
+ }
+ }
+
+ /** Appends the entries of the following block to the target block. */
+ private static void moveInto(Block target, Block following) {
+ int count = target.size() + following.size();
+ if (count > target.capacity()) {
+ target.grow(count);
+ } else {
+ target.compact();
+ }
+ System.arraycopy(following.entryIds, following.start, target.entryIds, target.end, following.size());
+ System.arraycopy(following.values, following.start, target.values, target.end, following.size());
+ target.end += following.size();
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java
index 80345266db7e..4fc842a587f0 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java
@@ -18,9 +18,6 @@
*/
package org.apache.pulsar.broker.service;
-import it.unimi.dsi.fastutil.longs.Long2LongAVLTreeMap;
-import it.unimi.dsi.fastutil.longs.Long2LongMap;
-import it.unimi.dsi.fastutil.longs.Long2LongSortedMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectRBTreeMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectSortedMap;
@@ -107,13 +104,19 @@ public class PendingAcksMap {
* Pending acknowledgments grouped by ledger id. The outer map key is ledgerId and each inner map key is entryId.
* The inner map value is packed by {@link PendingAckValues} with the remaining unacked count and sticky key hash.
*/
- private final Long2ObjectSortedMap<Long2LongSortedMap> pendingAcks;
+ private final Long2ObjectSortedMap<PendingAcksLedgerMap> pendingAcks;
private final Supplier<PendingAcksAddHandler> pendingAcksAddHandlerSupplier;
private final Supplier<PendingAcksRemoveHandler> pendingAcksRemoveHandlerSupplier;
private final Lock readLock;
private final Lock writeLock;
private boolean closed = false;
private volatile long size;
+ // A ledger map that became empty, reused for the next ledger: a consumer whose pending acks drop to none between
+ // messages, as on a topic with a low rate, doesn't allocate a ledger map for each message. Guarded by the lock
+ private PendingAcksLedgerMap emptyLedgerMap;
+ // The number of traversals whose callbacks may change the map. While one runs, a ledger map that a callback empties
+ // isn't reused: the traversal may still be iterating it. Guarded by the write lock
+ private int mutableTraversals;
PendingAcksMap(Consumer consumer, Supplier<PendingAcksAddHandler> pendingAcksAddHandlerSupplier,
Supplier<PendingAcksRemoveHandler> pendingAcksRemoveHandlerSupplier) {
@@ -154,8 +157,8 @@ public class PendingAcksMap {
&& !pendingAcksAddHandler.handleAdding(consumer, ledgerId, entryId, stickyKeyHash)) {
return false;
}
- Long2LongSortedMap ledgerPendingAcks =
- pendingAcks.computeIfAbsent(ledgerId, k -> newLedgerPendingAcks());
+ PendingAcksLedgerMap ledgerPendingAcks =
+ pendingAcks.computeIfAbsent(ledgerId, k -> takeEmptyLedgerMap());
long packedValue = PendingAckValues.pack(remainingUnacked, stickyKeyHash);
long previous = ledgerPendingAcks.put(entryId, packedValue);
if (PendingAckValues.isNotFound(previous)) {
@@ -190,15 +193,11 @@ public class PendingAcksMap {
}
}
- // this code uses for loops intentionally, don't refactor to use forEach
private void processPendingAcks(PendingAcksConsumer processor) {
- for (Long2ObjectMap.Entry<Long2LongSortedMap> entry : pendingAcks.long2ObjectEntrySet()) {
+ for (Long2ObjectMap.Entry<PendingAcksLedgerMap> entry : pendingAcks.long2ObjectEntrySet()) {
long ledgerId = entry.getLongKey();
- for (Long2LongMap.Entry pendingAckEntry : entry.getValue().long2LongEntrySet()) {
- long packedValue = pendingAckEntry.getLongValue();
- processor.accept(ledgerId, pendingAckEntry.getLongKey(),
- PendingAckValues.remainingUnacked(packedValue), PendingAckValues.stickyKeyHash(packedValue));
- }
+ entry.getValue().forEach((entryId, packedValue) -> processor.accept(ledgerId, entryId,
+ PendingAckValues.remainingUnacked(packedValue), PendingAckValues.stickyKeyHash(packedValue)));
}
}
@@ -226,6 +225,7 @@ public class PendingAcksMap {
private void internalForEachAndClear(PendingAcksConsumer processor, boolean close) {
try {
writeLock.lock();
+ mutableTraversals++;
if (close) {
closed = true;
}
@@ -246,6 +246,7 @@ public class PendingAcksMap {
pendingAcks.clear();
size = 0;
} finally {
+ mutableTraversals--;
writeLock.unlock();
}
}
@@ -260,7 +261,7 @@ public class PendingAcksMap {
public boolean contains(long ledgerId, long entryId) {
try {
readLock.lock();
- Long2LongSortedMap ledgerMap = pendingAcks.get(ledgerId);
+ PendingAcksLedgerMap ledgerMap = pendingAcks.get(ledgerId);
if (ledgerMap == null) {
return false;
}
@@ -278,7 +279,7 @@ public class PendingAcksMap {
int getRemainingUnacked(long ledgerId, long entryId) {
try {
readLock.lock();
- Long2LongSortedMap ledgerMap = pendingAcks.get(ledgerId);
+ PendingAcksLedgerMap ledgerMap = pendingAcks.get(ledgerId);
if (ledgerMap == null) {
return PENDING_ACK_NOT_FOUND;
}
@@ -302,7 +303,7 @@ public class PendingAcksMap {
public boolean remove(long ledgerId, long entryId, int batchSize, int stickyKeyHash) {
try {
writeLock.lock();
- Long2LongSortedMap ledgerMap = pendingAcks.get(ledgerId);
+ PendingAcksLedgerMap ledgerMap = pendingAcks.get(ledgerId);
if (batchSize < 0) {
return false;
}
@@ -315,6 +316,7 @@ public class PendingAcksMap {
handleRemovePendingAck(ledgerId, entryId, stickyKeyHash);
if (ledgerMap.isEmpty()) {
pendingAcks.remove(ledgerId);
+ recycleEmptyLedgerMap(ledgerMap);
}
return true;
} finally {
@@ -334,7 +336,7 @@ public class PendingAcksMap {
public boolean updateRemainingUnacked(long ledgerId, long entryId, int ackedDelta) {
try {
writeLock.lock();
- Long2LongSortedMap ledgerMap = pendingAcks.get(ledgerId);
+ PendingAcksLedgerMap ledgerMap = pendingAcks.get(ledgerId);
if (ledgerMap == null) {
return false;
}
@@ -363,7 +365,7 @@ public class PendingAcksMap {
public boolean remove(long ledgerId, long entryId) {
try {
writeLock.lock();
- Long2LongSortedMap ledgerMap = pendingAcks.get(ledgerId);
+ PendingAcksLedgerMap ledgerMap = pendingAcks.get(ledgerId);
if (ledgerMap == null) {
return false;
}
@@ -375,6 +377,7 @@ public class PendingAcksMap {
handleRemovePendingAck(ledgerId, entryId, PendingAckValues.stickyKeyHash(removedEntry));
if (ledgerMap.isEmpty()) {
pendingAcks.remove(ledgerId);
+ recycleEmptyLedgerMap(ledgerMap);
}
return true;
} finally {
@@ -390,7 +393,7 @@ public class PendingAcksMap {
int removeAndGetRemainingUnacked(long ledgerId, long entryId) {
try {
writeLock.lock();
- Long2LongSortedMap ledgerMap = pendingAcks.get(ledgerId);
+ PendingAcksLedgerMap ledgerMap = pendingAcks.get(ledgerId);
if (ledgerMap == null) {
return PENDING_ACK_NOT_FOUND;
}
@@ -402,6 +405,7 @@ public class PendingAcksMap {
handleRemovePendingAck(ledgerId, entryId, PendingAckValues.stickyKeyHash(removedEntry));
if (ledgerMap.isEmpty()) {
pendingAcks.remove(ledgerId);
+ recycleEmptyLedgerMap(ledgerMap);
}
return PendingAckValues.remainingUnacked(removedEntry);
} finally {
@@ -447,31 +451,31 @@ public class PendingAcksMap {
if (useWriteLock) {
writeLock.lock();
acquiredWriteLock = true;
+ mutableTraversals++;
} else {
readLock.lock();
}
- ObjectBidirectionalIterator<Long2ObjectMap.Entry<Long2LongSortedMap>> ledgerMapIterator =
+ ObjectBidirectionalIterator<Long2ObjectMap.Entry<PendingAcksLedgerMap>> ledgerMapIterator =
pendingAcks.headMap(markDeleteLedgerId + 1).long2ObjectEntrySet().iterator();
while (ledgerMapIterator.hasNext()) {
- Long2ObjectMap.Entry<Long2LongSortedMap> entry = ledgerMapIterator.next();
+ Long2ObjectMap.Entry<PendingAcksLedgerMap> entry = ledgerMapIterator.next();
long ledgerId = entry.getLongKey();
- Long2LongSortedMap ledgerMap = entry.getValue();
- Long2LongSortedMap ledgerMapHead;
- if (ledgerId == markDeleteLedgerId) {
- ledgerMapHead = ledgerMap.headMap(markDeleteEntryId + 1);
- } else {
- ledgerMapHead = ledgerMap;
- }
- ObjectBidirectionalIterator<Long2LongMap.Entry> entryMapIterator =
- ledgerMapHead.long2LongEntrySet().iterator();
- while (entryMapIterator.hasNext()) {
- Long2LongMap.Entry pendingAckEntry = entryMapIterator.next();
- long entryId = pendingAckEntry.getLongKey();
- if (!acquiredWriteLock) {
+ PendingAcksLedgerMap ledgerMap = entry.getValue();
+ long maxEntryId = ledgerId == markDeleteLedgerId ? markDeleteEntryId : Long.MAX_VALUE;
+ if (!acquiredWriteLock) {
+ if (ledgerMap.hasEntriesUpTo(maxEntryId) || ledgerMap.isEmpty()) {
retryWithWriteLock = true;
return;
}
- long packedValue = pendingAckEntry.getLongValue();
+ continue;
+ }
+ // One entry at a time, the callbacks before its removal, as an iterator would. The callbacks may add
+ // and remove entries: the next entry is the first one after the removed one, so an entry added before
+ // it isn't removed, as with the tree map's iterator before, and the removal ends
+ long entryId = ledgerMap.hasEntriesUpTo(maxEntryId) ? ledgerMap.firstEntryId()
+ : PendingAcksLedgerMap.NO_ENTRY;
+ while (entryId != PendingAcksLedgerMap.NO_ENTRY && entryId <= maxEntryId) {
+ long packedValue = ledgerMap.get(entryId);
int batchSize = PendingAckValues.remainingUnacked(packedValue);
int stickyKeyHash = PendingAckValues.stickyKeyHash(packedValue);
if (pendingAcksRemoveHandler != null) {
@@ -484,15 +488,14 @@ public class PendingAcksMap {
if (removedEntryCallback != null) {
removedEntryCallback.accept(ledgerId, entryId, batchSize, stickyKeyHash);
}
- entryMapIterator.remove();
- size--;
+ if (!PendingAckValues.isNotFound(ledgerMap.remove(entryId))) {
+ size--;
+ }
+ entryId = ledgerMap.higherEntryId(entryId);
}
if (ledgerMap.isEmpty()) {
- if (!acquiredWriteLock) {
- retryWithWriteLock = true;
- return;
- }
ledgerMapIterator.remove();
+ recycleEmptyLedgerMap(ledgerMap);
}
}
} finally {
@@ -500,6 +503,7 @@ public class PendingAcksMap {
pendingAcksRemoveHandler.endBatch();
}
if (acquiredWriteLock) {
+ mutableTraversals--;
writeLock.unlock();
} else {
readLock.unlock();
@@ -510,10 +514,19 @@ public class PendingAcksMap {
}
}
- private static Long2LongSortedMap newLedgerPendingAcks() {
- Long2LongAVLTreeMap ledgerPendingAcks = new Long2LongAVLTreeMap();
- ledgerPendingAcks.defaultReturnValue(PendingAckValues.PACKED_NOT_FOUND);
- return ledgerPendingAcks;
+ private void recycleEmptyLedgerMap(PendingAcksLedgerMap ledgerMap) {
+ if (mutableTraversals == 0) {
+ emptyLedgerMap = ledgerMap;
+ }
+ }
+
+ private PendingAcksLedgerMap takeEmptyLedgerMap() {
+ PendingAcksLedgerMap ledgerMap = emptyLedgerMap;
+ if (ledgerMap == null) {
+ return new PendingAcksLedgerMap();
+ }
+ emptyLedgerMap = null;
+ return ledgerMap;
}
private void handleRemovePendingAck(long ledgerId, long entryId, int stickyKeyHash) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksLedgerMapTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksLedgerMapTest.java
new file mode 100644
index 000000000000..bd73c9a5c0cb
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksLedgerMapTest.java
@@ -0,0 +1,501 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import org.testng.annotations.Test;
+
+public class PendingAcksLedgerMapTest {
+
+ @Test
+ public void keepsEntriesInOrderWithAppendsInsertsAndRemovals() {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ assertThat(map.put(5, value(1))).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ assertThat(map.put(7, value(2))).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ // out of order, as a redelivery adds it
+ assertThat(map.put(6, value(3))).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ assertThat(map.put(1, value(4))).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ assertThat(map.put(6, value(5))).isEqualTo(value(3));
+ assertThat(entries(map)).containsExactly(1L, 5L, 6L, 7L);
+ assertThat(map.get(6)).isEqualTo(value(5));
+ assertThat(map.get(2)).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+
+ assertThat(map.remove(5)).isEqualTo(value(1));
+ assertThat(map.remove(5)).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ assertThat(map.containsKey(5)).isFalse();
+ // a removed entry can come back
+ assertThat(map.put(5, value(6))).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ assertThat(entries(map)).containsExactly(1L, 5L, 6L, 7L);
+
+ List<Long> removed = new ArrayList<>();
+ assertThat(map.hasEntriesUpTo(0)).isFalse();
+ assertThat(map.hasEntriesUpTo(6)).isTrue();
+ while (map.hasEntriesUpTo(6)) {
+ removed.add(map.firstEntryId());
+ map.removeFirst();
+ }
+ assertThat(removed).containsExactly(1L, 5L, 6L);
+ assertThat(entries(map)).containsExactly(7L);
+ assertThat(map.size()).isEqualTo(1);
+ assertThat(map.remove(7)).isEqualTo(value(2));
+ assertThat(map.isEmpty()).isTrue();
+ assertThat(map.hasEntriesUpTo(Long.MAX_VALUE)).isFalse();
+ }
+
+ @Test
+ public void findsAndRemovesEntriesAckedInOrder() {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ int count = PendingAcksLedgerMap.BLOCK_CAPACITY * 3 + 5;
+ for (int i = 0; i < count; i++) {
+ map.put(i * 2, value(i));
+ }
+ // acked in the order they were sent: each is the first entry, across the blocks
+ for (int i = 0; i < count; i++) {
+ assertThat(map.get(i * 2)).isEqualTo(value(i));
+ // not an entry, just after the first
+ assertThat(map.get(i * 2 + 1)).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ assertThat(map.remove(i * 2)).isEqualTo(value(i));
+ assertThat(map.get(i * 2)).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ assertThat(map.size()).isEqualTo(count - i - 1);
+ }
+ assertThat(map.isEmpty()).isTrue();
+ assertThat(map.remove(0)).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ assertThat(map.get(0)).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ }
+
+ @Test
+ public void behavesAsASortedMap() {
+ // random operations in the patterns of dispatch and acks: mostly increasing entry IDs, some redeliveries,
+ // removals near the start and range removals up to a mark-delete position
+ Random random = new Random(1);
+ for (int round = 0; round < 50; round++) {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ TreeMap<Long, Long> reference = new TreeMap<>();
+ long nextEntryId = 0;
+ for (int step = 0; step < 5000; step++) {
+ int operation = random.nextInt(100);
+ if (operation < 45) {
+ long entryId = random.nextInt(10) == 0 && nextEntryId > 0
+ ? random.nextLong(nextEntryId) : nextEntryId++;
+ long v = value(random.nextInt(1000));
+ Long previous = reference.put(entryId, v);
+ assertThat(map.put(entryId, v))
+ .isEqualTo(previous != null ? previous : PendingAckValues.PACKED_NOT_FOUND);
+ } else if (operation < 90) {
+ long entryId = reference.isEmpty() || random.nextInt(4) == 0
+ ? random.nextLong(nextEntryId + 1) : nthKey(reference, random.nextInt(
+ Math.min(reference.size(), 20)));
+ Long previous = reference.remove(entryId);
+ assertThat(map.remove(entryId))
+ .isEqualTo(previous != null ? previous : PendingAckValues.PACKED_NOT_FOUND);
+ } else if (operation < 95) {
+ long upTo = reference.isEmpty() ? nextEntryId : reference.firstKey() + random.nextInt(30);
+ Map<Long, Long> head = new TreeMap<>(reference.headMap(upTo, true));
+ List<Long> removedIds = new ArrayList<>();
+ List<Long> removedValues = new ArrayList<>();
+ assertThat(map.hasEntriesUpTo(upTo)).isEqualTo(!head.isEmpty());
+ while (map.hasEntriesUpTo(upTo)) {
+ removedIds.add(map.firstEntryId());
+ removedValues.add(map.firstValue());
+ map.removeFirst();
+ }
+ assertThat(removedIds).containsExactlyElementsOf(head.keySet());
+ assertThat(removedValues).containsExactlyElementsOf(head.values());
+ reference.headMap(upTo, true).clear();
+ } else {
+ long entryId = random.nextLong(nextEntryId + 1);
+ Long expected = reference.get(entryId);
+ assertThat(map.get(entryId))
+ .isEqualTo(expected != null ? expected : PendingAckValues.PACKED_NOT_FOUND);
+ assertThat(map.containsKey(entryId)).isEqualTo(expected != null);
+ }
+ assertThat(map.size()).isEqualTo(reference.size());
+ map.checkInvariants();
+ if (step % 97 == 0) {
+ assertThat(contents(map)).isEqualTo(reference);
+ }
+ }
+ List<Long> ids = new ArrayList<>();
+ List<Long> values = new ArrayList<>();
+ map.forEach((entryId, v) -> {
+ ids.add(entryId);
+ values.add(v);
+ });
+ assertThat(ids).containsExactlyElementsOf(reference.keySet());
+ assertThat(values).containsExactlyElementsOf(reference.values());
+ }
+ }
+
+ @Test(timeOut = 30_000)
+ public void reinsertsARemovedPrefixInAscendingOrderWithoutCopyingTheRest() {
+ // explicit redelivery removes a consumer's pending acks and dispatches them again, in the order of their
+ // positions: re-adding them mustn't shift every following entry each time
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ int count = 200_000;
+ for (int entryId = 0; entryId < count; entryId++) {
+ map.put(entryId, value(1));
+ }
+ for (int entryId = 0; entryId < count / 2; entryId++) {
+ assertThat(map.remove(entryId)).isEqualTo(value(1));
+ }
+ for (int entryId = 0; entryId < count / 2; entryId++) {
+ assertThat(map.put(entryId, value(2))).isEqualTo(PendingAckValues.PACKED_NOT_FOUND);
+ }
+ assertThat(map.size()).isEqualTo(count);
+ long[] expected = {0};
+ map.forEach((entryId, v) -> {
+ assertThat(entryId).isEqualTo(expected[0]++);
+ assertThat(v).isEqualTo(entryId < count / 2 ? value(2) : value(1));
+ });
+ }
+
+ @Test
+ public void releasesTheMemoryOfRemovedEntries() {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ int count = 64 * PendingAcksLedgerMap.BLOCK_CAPACITY;
+ for (int entryId = 0; entryId < count; entryId++) {
+ map.put(entryId, value(1));
+ }
+ assertThat(map.blockCount()).isEqualTo(64);
+ // keep every 64th entry: the sparse blocks merge with their neighbors
+ for (int entryId = 0; entryId < count; entryId++) {
+ if (entryId % 64 != 0) {
+ map.remove(entryId);
+ }
+ }
+ // 128 entries, in blocks of at most half a block after the merges
+ assertThat(map.size()).isEqualTo(128);
+ assertThat(map.blockCount()).isLessThanOrEqualTo(4);
+ for (int entryId = 0; entryId < count; entryId += 64) {
+ assertThat(map.get(entryId)).isEqualTo(value(1));
+ }
+ while (!map.isEmpty()) {
+ map.removeFirst();
+ }
+ assertThat(map.blockCount()).isZero();
+ }
+
+ @Test
+ public void splitsAFullBlockForAnEntryInsertedInIt() {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ for (int entryId = 0; entryId < PendingAcksLedgerMap.BLOCK_CAPACITY; entryId++) {
+ map.put(entryId * 2L, value(1));
+ }
+ assertThat(map.blockCount()).isEqualTo(1);
+ map.put(1, value(2));
+ map.put(2L * PendingAcksLedgerMap.BLOCK_CAPACITY - 3, value(3));
+ assertThat(map.blockCount()).isEqualTo(2);
+ assertThat(map.get(1)).isEqualTo(value(2));
+ assertThat(map.get(2L * PendingAcksLedgerMap.BLOCK_CAPACITY - 3)).isEqualTo(value(3));
+ assertThat(map.size()).isEqualTo(PendingAcksLedgerMap.BLOCK_CAPACITY + 2);
+ List<Long> ids = entries(map);
+ assertThat(ids).isSorted().doesNotHaveDuplicates();
+ }
+
+ @Test
+ public void insertsIntoTheUpperHalfOfAFullBlock() {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ for (int entryId = 0; entryId < PendingAcksLedgerMap.BLOCK_CAPACITY; entryId++) {
+ map.put(entryId * 2L, value(1));
+ }
+ // after the first block's middle, so that it goes into the new upper block of the split
+ long entryId = 2L * PendingAcksLedgerMap.BLOCK_CAPACITY - 5;
+ map.put(entryId, value(2));
+ assertThat(map.blockCount()).isEqualTo(2);
+ assertThat(map.get(entryId)).isEqualTo(value(2));
+ assertThat(entries(map)).isSorted().doesNotHaveDuplicates().hasSize(PendingAcksLedgerMap.BLOCK_CAPACITY + 1);
+ }
+
+ @Test
+ public void forEachContinuesAfterTheCurrentEntryWhenTheConsumerChangesTheMap() {
+ // The consumer removes the current entry, removes or adds other entries before and after it, in blocks that
+ // shift and split. The traversal visits the entries after the current one as they are after the consumer's
+ // changes, as TreeMap.higherKey finds them
+ for (int round = 0; round < 20; round++) {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ TreeMap<Long, Long> reference = new TreeMap<>();
+ for (long entryId = 0; entryId < 1000; entryId += 2) {
+ map.put(entryId, value(1));
+ reference.put(entryId, value(1));
+ }
+ long seed = round;
+ List<Long> visited = new ArrayList<>();
+ map.forEach((entryId, v) -> {
+ visited.add(entryId);
+ changeAt(entryId, seed, (id, add) -> {
+ if (add) {
+ map.put(id, value(1));
+ } else {
+ map.remove(id);
+ }
+ });
+ });
+
+ List<Long> expected = new ArrayList<>();
+ for (Long entryId = reference.firstKey(); entryId != null; entryId = reference.higherKey(entryId)) {
+ expected.add(entryId);
+ changeAt(entryId, seed, (id, add) -> {
+ if (add) {
+ reference.put(id, value(1));
+ } else {
+ reference.remove(id);
+ }
+ });
+ }
+ assertThat(visited).as("round %d", round).isEqualTo(expected);
+ assertThat(contents(map)).as("round %d", round).isEqualTo(reference);
+ map.checkInvariants();
+ }
+ }
+
+ @Test
+ public void forEachVisitsEveryEntryWhileTheConsumerDrainsTheMap() {
+ // Removing the current entry, and every other one after it, empties blocks, which are removed, and leaves
+ // sparse blocks, which merge: the traversal still visits each remaining entry once, in order
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ int count = 5 * PendingAcksLedgerMap.BLOCK_CAPACITY;
+ for (long entryId = 0; entryId < count; entryId++) {
+ map.put(entryId, value(1));
+ }
+ int blocksBefore = map.blockCount();
+ List<Long> visited = new ArrayList<>();
+ int[] minBlocks = {blocksBefore};
+ map.forEach((entryId, v) -> {
+ visited.add(entryId);
+ map.remove(entryId);
+ if (entryId % 64 == 0) {
+ // thin out the entries ahead, which leaves sparse blocks that merge
+ for (long ahead = entryId + 2; ahead < entryId + 64; ahead += 2) {
+ map.remove(ahead);
+ }
+ }
+ minBlocks[0] = Math.min(minBlocks[0], map.blockCount());
+ });
+
+ List<Long> expected = new ArrayList<>();
+ for (long entryId = 0; entryId < count; entryId++) {
+ if (entryId % 64 == 0 || entryId % 2 == 1) {
+ expected.add(entryId);
+ }
+ }
+ assertThat(visited).isEqualTo(expected);
+ assertThat(map.isEmpty()).isTrue();
+ assertThat(map.blockCount()).isZero();
+ assertThat(minBlocks[0]).isLessThan(blocksBefore);
+ map.checkInvariants();
+ }
+
+ @Test
+ public void startsWithASmallFirstBlockThatGrows() {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ map.put(0, value(1));
+ assertThat(map.blockCapacity(0)).isEqualTo(PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY);
+ for (long entryId = 1; entryId <= PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY; entryId++) {
+ map.put(entryId, value(1));
+ }
+ assertThat(map.blockCapacity(0)).isEqualTo(2 * PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY);
+ for (long entryId = PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY + 1;
+ entryId < PendingAcksLedgerMap.BLOCK_CAPACITY; entryId++) {
+ map.put(entryId, value(1));
+ }
+ // one block of the full capacity, and then a new one of the full capacity
+ assertThat(map.blockCount()).isEqualTo(1);
+ assertThat(map.blockCapacity(0)).isEqualTo(PendingAcksLedgerMap.BLOCK_CAPACITY);
+ map.put(PendingAcksLedgerMap.BLOCK_CAPACITY, value(1));
+ assertThat(map.blockCount()).isEqualTo(2);
+ assertThat(map.blockCapacity(1)).isEqualTo(PendingAcksLedgerMap.BLOCK_CAPACITY);
+ map.checkInvariants();
+ }
+
+ @Test
+ public void growsASmallFullBlockForAnEntryInsertedInIt() {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ for (long entryId = 0; entryId < 2L * PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY; entryId += 2) {
+ map.put(entryId, value(1));
+ }
+ assertThat(map.blockCapacity(0)).isEqualTo(PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY);
+ // before the first entry, and between two entries
+ map.put(-1, value(2));
+ map.put(5, value(3));
+ assertThat(map.blockCount()).isEqualTo(1);
+ assertThat(map.blockCapacity(0)).isEqualTo(2 * PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY);
+ assertThat(map.firstEntryId()).isEqualTo(-1);
+ assertThat(map.get(-1)).isEqualTo(value(2));
+ assertThat(map.get(5)).isEqualTo(value(3));
+ assertThat(entries(map)).isSorted().doesNotHaveDuplicates()
+ .hasSize(PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY + 2);
+ map.checkInvariants();
+ }
+
+ @Test
+ public void reusesTheLastEmptiedBlockForAWindowThatMoves() {
+ // a consumer's window of pending acks moving through a ledger: acks remove the oldest entries and dispatches
+ // append new ones
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ int window = 4 * PendingAcksLedgerMap.BLOCK_CAPACITY;
+ for (long entryId = 0; entryId < window; entryId++) {
+ map.put(entryId, value(1));
+ }
+ int allocated = map.allocatedBlocks();
+ for (long entryId = 0; entryId < 100L * PendingAcksLedgerMap.BLOCK_CAPACITY; entryId++) {
+ assertThat(map.remove(entryId)).isEqualTo(value(1));
+ map.put(entryId + window, value(1));
+ }
+ // a block more than the window's, while the oldest block still has entries when a new one starts
+ assertThat(map.allocatedBlocks()).isLessThanOrEqualTo(allocated + 1);
+ assertThat(map.size()).isEqualTo(window);
+ map.checkInvariants();
+ // an empty map keeps a spare block of the smallest capacity only, which it starts again with
+ while (!map.isEmpty()) {
+ map.removeFirst();
+ map.checkInvariants();
+ }
+ int allocatedWhenEmpty = map.allocatedBlocks();
+ map.put(0, value(1));
+ assertThat(map.blockCapacity(0)).isEqualTo(PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY);
+ assertThat(map.allocatedBlocks()).isEqualTo(allocatedWhenEmpty);
+ map.checkInvariants();
+ }
+
+ @Test
+ public void shrinksALoneBlockAsItsEntriesAreRemoved() {
+ // a map that held many entries and now holds a few keeps a small block, and no spare one
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ int count = 5 * PendingAcksLedgerMap.BLOCK_CAPACITY;
+ for (long entryId = 0; entryId < count; entryId++) {
+ map.put(entryId, value(1));
+ }
+ // ack holes: every 100th entry stays pending
+ for (long entryId = 0; entryId < count; entryId++) {
+ if (entryId % 100 != 0) {
+ map.remove(entryId);
+ map.checkInvariants();
+ }
+ }
+ assertThat(map.size()).isEqualTo(7);
+ assertThat(map.blockCount()).isEqualTo(1);
+ assertThat(map.blockCapacity(0)).isEqualTo(PendingAcksLedgerMap.INITIAL_BLOCK_CAPACITY);
+ assertThat(entries(map)).containsExactly(0L, 100L, 200L, 300L, 400L, 500L, 600L);
+ }
+
+ @Test
+ public void staysASortedMapWithRemovalsAcrossTheWindowAndAppendsAfterThem() {
+ // Removals anywhere in a large window empty, merge and shrink blocks, whose arrays are reused as spare
+ // blocks for the appends, inserts and splits that follow
+ Random random = new Random(2);
+ for (int round = 0; round < 20; round++) {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ TreeMap<Long, Long> reference = new TreeMap<>();
+ long nextEntryId = 0;
+ for (; nextEntryId < 1000; nextEntryId++) {
+ map.put(nextEntryId, value(1));
+ reference.put(nextEntryId, value(1));
+ }
+ for (int step = 0; step < 4000; step++) {
+ int operation = random.nextInt(10);
+ if (operation < 5 && !reference.isEmpty()) {
+ // a removal anywhere in the window
+ long entryId = random.nextLong(nextEntryId);
+ Long previous = reference.remove(entryId);
+ assertThat(map.remove(entryId))
+ .isEqualTo(previous != null ? previous : PendingAckValues.PACKED_NOT_FOUND);
+ } else if (operation < 8) {
+ long v = value(1 + random.nextInt(100));
+ reference.put(nextEntryId, v);
+ map.put(nextEntryId++, v);
+ } else {
+ // a redelivery, out of order
+ long entryId = random.nextLong(nextEntryId);
+ long v = value(1 + random.nextInt(100));
+ Long previous = reference.put(entryId, v);
+ assertThat(map.put(entryId, v))
+ .isEqualTo(previous != null ? previous : PendingAckValues.PACKED_NOT_FOUND);
+ }
+ map.checkInvariants();
+ assertThat(map.size()).isEqualTo(reference.size());
+ }
+ assertThat(contents(map)).as("round %d", round).isEqualTo(reference);
+ }
+ }
+
+ @Test
+ public void higherEntryIdFindsTheNextEntryAcrossBlocks() {
+ PendingAcksLedgerMap map = new PendingAcksLedgerMap();
+ assertThat(map.higherEntryId(0)).isEqualTo(PendingAcksLedgerMap.NO_ENTRY);
+ for (long entryId = 10; entryId < 10 + 3 * PendingAcksLedgerMap.BLOCK_CAPACITY; entryId += 2) {
+ map.put(entryId, value(1));
+ }
+ assertThat(map.blockCount()).isGreaterThan(1);
+ long last = 10 + 3 * PendingAcksLedgerMap.BLOCK_CAPACITY - 2;
+ assertThat(map.higherEntryId(Long.MIN_VALUE + 1)).isEqualTo(10);
+ assertThat(map.higherEntryId(10)).isEqualTo(12);
+ assertThat(map.higherEntryId(11)).isEqualTo(12);
+ for (long entryId = 10; entryId < last; entryId++) {
+ assertThat(map.higherEntryId(entryId)).as("after %d", entryId).isEqualTo(entryId + 2 - entryId % 2);
+ }
+ assertThat(map.higherEntryId(last)).isEqualTo(PendingAcksLedgerMap.NO_ENTRY);
+ assertThat(map.higherEntryId(Long.MAX_VALUE)).isEqualTo(PendingAcksLedgerMap.NO_ENTRY);
+ }
+
+ private interface Change {
+ void apply(long entryId, boolean add);
+ }
+
+ // The changes that a consumer makes when it visits the entry, the same for the map and the reference
+ private static void changeAt(long entryId, long seed, Change change) {
+ Random random = new Random(seed * 1_000_003 + entryId);
+ if (random.nextInt(3) == 0) {
+ change.apply(entryId, false);
+ }
+ for (int i = random.nextInt(4); i > 0; i--) {
+ change.apply(random.nextInt(1200), random.nextBoolean());
+ }
+ }
+
+ private static TreeMap<Long, Long> contents(PendingAcksLedgerMap map) {
+ TreeMap<Long, Long> contents = new TreeMap<>();
+ map.forEach(contents::put);
+ return contents;
+ }
+
+ private static long value(int remainingUnacked) {
+ return PendingAckValues.pack(remainingUnacked, remainingUnacked * 31);
+ }
+
+ private static long nthKey(TreeMap<Long, Long> map, int n) {
+ int i = 0;
+ for (Long key : map.keySet()) {
+ if (i++ == n) {
+ return key;
+ }
+ }
+ throw new IllegalArgumentException();
+ }
+
+ private static List<Long> entries(PendingAcksLedgerMap map) {
+ List<Long> ids = new ArrayList<>();
+ map.forEach((entryId, v) -> ids.add(entryId));
+ return ids;
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java
index 59aa302ebb15..ae8b432fb31f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java
@@ -131,6 +131,185 @@ public class PendingAcksMapTest {
assertEquals(pendingAcksMap.size(), 1);
}
+ @Test
+ public void forEachAndClear_VisitsTheEntriesAfterOneThatTheCallbackRemoves() {
+ Consumer consumer = createMockConsumer("consumer1");
+ PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 3L, 1, 125);
+ List<Long> visited = new ArrayList<>();
+
+ pendingAcksMap.forEachAndClear((ledgerId, entryId, batchSize, stickyKeyHash) -> {
+ visited.add(entryId);
+ if (entryId == 2L) {
+ pendingAcksMap.remove(1L, 2L);
+ }
+ });
+
+ assertEquals(visited, List.of(1L, 2L, 3L));
+ assertEquals(pendingAcksMap.size(), 0);
+ }
+
+ @Test(timeOut = 10_000)
+ public void removeAllUpTo_DoesntRemoveAnEntryThatTheCallbackAddsBeforeTheCurrentOne() {
+ Consumer consumer = createMockConsumer("consumer1");
+ PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 3L, 1, 125);
+ List<Long> removed = new ArrayList<>();
+
+ // adding an entry before the current one, and one after it, which is removed too
+ pendingAcksMap.removeAllUpTo(1L, 3L, (ledgerId, entryId, batchSize, stickyKeyHash) -> {
+ removed.add(entryId);
+ if (entryId == 1L) {
+ pendingAcksMap.addPendingAckIfAllowed(1L, 0L, 1, 122);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124);
+ } else if (entryId == 2L) {
+ // re-adding the removed first entry, which would be removed again without an end
+ pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123);
+ }
+ });
+
+ assertEquals(removed, List.of(1L, 2L, 3L));
+ assertTrue(pendingAcksMap.contains(1L, 0L));
+ assertTrue(pendingAcksMap.contains(1L, 1L));
+ assertEquals(pendingAcksMap.size(), 2);
+ }
+
+ @Test
+ public void removeAllUpTo_KeepsTheSizeWhenTheCallbackRemovesTheCurrentEntry() {
+ Consumer consumer = createMockConsumer("consumer1");
+ PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 3L, 1, 125);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 4L, 1, 126);
+
+ pendingAcksMap.removeAllUpTo(1L, 2L, (ledgerId, entryId, batchSize, stickyKeyHash) ->
+ pendingAcksMap.remove(ledgerId, entryId));
+
+ // each removed entry is counted once
+ assertEquals(pendingAcksMap.size(), 2);
+ assertTrue(pendingAcksMap.contains(1L, 3L));
+ assertTrue(pendingAcksMap.contains(1L, 4L));
+ }
+
+ @Test
+ public void forEachAndClear_DoesntVisitAnEntryThatTheCallbackRemoves() {
+ Consumer consumer = createMockConsumer("consumer1");
+ PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null);
+ for (long entryId = 1; entryId <= 5; entryId++) {
+ pendingAcksMap.addPendingAckIfAllowed(1L, entryId, 1, 120 + (int) entryId);
+ }
+ List<Long> visited = new ArrayList<>();
+
+ pendingAcksMap.forEachAndClear((ledgerId, entryId, batchSize, stickyKeyHash) -> {
+ visited.add(entryId);
+ if (entryId == 2L) {
+ pendingAcksMap.remove(1L, 3L);
+ }
+ });
+
+ assertEquals(visited, List.of(1L, 2L, 4L, 5L));
+ assertEquals(pendingAcksMap.size(), 0);
+ }
+
+ @Test
+ public void reusesAnEmptiedLedgerMapForTheNextEntries() {
+ // a consumer whose pending acks drop to none between messages
+ Consumer consumer = createMockConsumer("consumer1");
+ PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null);
+ for (long entryId = 0; entryId < 100; entryId++) {
+ assertTrue(pendingAcksMap.addPendingAckIfAllowed(entryId / 10, entryId, 1, 123));
+ assertEquals(pendingAcksMap.getRemainingUnacked(entryId / 10, entryId), 1);
+ assertEquals(pendingAcksMap.removeAndGetRemainingUnacked(entryId / 10, entryId), 1);
+ assertEquals(pendingAcksMap.size(), 0);
+ assertFalse(pendingAcksMap.contains(entryId / 10, entryId));
+ }
+ pendingAcksMap.addPendingAckIfAllowed(20L, 1L, 2, 123);
+ pendingAcksMap.addPendingAckIfAllowed(20L, 2L, 3, 124);
+ assertEquals(pendingAcksMap.getRemainingUnacked(20L, 1L), 2);
+ assertEquals(pendingAcksMap.getRemainingUnacked(20L, 2L), 3);
+ assertEquals(pendingAcksMap.size(), 2);
+ }
+
+ @Test
+ public void removeAllUpTo_KeepsAnEntryThatTheCallbackAddsToAnotherLedgerAfterEmptyingTheCurrentOne() {
+ Consumer consumer = createMockConsumer("consumer1");
+ PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123);
+
+ pendingAcksMap.removeAllUpTo(1L, 1L, (ledgerId, entryId, batchSize, stickyKeyHash) -> {
+ // emptying the ledger map that the removal iterates, then adding an entry to another ledger
+ pendingAcksMap.remove(1L, 1L);
+ pendingAcksMap.addPendingAckIfAllowed(2L, 1L, 1, 124);
+ });
+
+ assertFalse(pendingAcksMap.contains(1L, 1L));
+ assertTrue(pendingAcksMap.contains(2L, 1L));
+ assertEquals(pendingAcksMap.size(), 1);
+ }
+
+ @Test
+ public void forEachAndClear_DoesntVisitAnEntryThatTheCallbackAddsToAnotherLedgerAfterEmptyingTheCurrentOne() {
+ Consumer consumer = createMockConsumer("consumer1");
+ PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123);
+ List<String> visited = new ArrayList<>();
+
+ pendingAcksMap.forEachAndClear((ledgerId, entryId, batchSize, stickyKeyHash) -> {
+ visited.add(ledgerId + ":" + entryId);
+ if (ledgerId == 1L) {
+ pendingAcksMap.remove(1L, 1L);
+ pendingAcksMap.addPendingAckIfAllowed(2L, 5L, 1, 124);
+ }
+ });
+
+ assertEquals(visited, List.of("1:1"));
+ assertEquals(pendingAcksMap.size(), 0);
+ }
+
+ @Test
+ public void removeAllUpTo_KeepsTheSizeWhenTheCallbackThrows() {
+ Consumer consumer = createMockConsumer("consumer1");
+ PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124);
+
+ try {
+ pendingAcksMap.removeAllUpTo(1L, 2L, (ledgerId, entryId, batchSize, stickyKeyHash) -> {
+ if (entryId == 2L) {
+ throw new IllegalStateException("failing callback");
+ }
+ });
+ } catch (IllegalStateException expected) {
+ // the entries before the failing one were removed, the failing one wasn't
+ }
+
+ assertFalse(pendingAcksMap.contains(1L, 1L));
+ assertTrue(pendingAcksMap.contains(1L, 2L));
+ assertEquals(pendingAcksMap.size(), 1);
+ assertTrue(pendingAcksMap.remove(1L, 2L));
+ assertEquals(pendingAcksMap.size(), 0);
+ }
+
+ @Test
+ public void removeAllUpTo_KeepsTheSizeWhenTheCallbackAddsAnEntry() {
+ Consumer consumer = createMockConsumer("consumer1");
+ PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123);
+ pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124);
+
+ pendingAcksMap.removeAllUpTo(1L, 1L, (ledgerId, entryId, batchSize, stickyKeyHash) ->
+ pendingAcksMap.addPendingAckIfAllowed(1L, 3L, 1, 125));
+
+ assertFalse(pendingAcksMap.contains(1L, 1L));
+ assertTrue(pendingAcksMap.contains(1L, 2L));
+ assertTrue(pendingAcksMap.contains(1L, 3L));
+ assertEquals(pendingAcksMap.size(), 2);
+ }
+
@Test
public void removeAllUpTo_RemovesAllPendingAcksUpToSpecifiedEntryAcrossMultipleLedgers() {
Consumer consumer = createMockConsumer("consumer1");
(pulsar) 02/14: [fix][broker] Apply dedup disable after recovery (#25982)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 71f852c56561cb1a09a1fb1af27632afb03fb7e1
Author: sinan liu <li...@gmail.com>
AuthorDate: Sat Oct 17 08:07:03 2026 +0800
[fix][broker] Apply dedup disable after recovery (#25982)
(cherry picked from commit 31bb46606d6729ed5c1216639db3174b28532144)
---
.../service/persistent/MessageDeduplication.java | 5 +-
.../broker/BrokerMessageDeduplicationTest.java | 84 +++++++++++++++++++++-
2 files changed, 85 insertions(+), 4 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
index a9ced74e432f..9bb2eb5d52d8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
@@ -162,6 +162,9 @@ public class MessageDeduplication {
boolean shouldBeEnabled = topic.isDeduplicationEnabled();
synchronized (this) {
if (status == Status.Recovering) {
+ if (!shouldBeEnabled) {
+ return statusChangeFuture.handle((__, e) -> null).thenCompose(__ -> checkStatus());
+ }
return statusChangeFuture;
}
if (status == Status.Removing) {
@@ -192,7 +195,7 @@ public class MessageDeduplication {
}, null);
}
- if (status == Status.Enabled && !shouldBeEnabled) {
+ if ((status == Status.Enabled || status == Status.Failed) && !shouldBeEnabled) {
// Disabled deduping
CompletableFuture<Void> future = new CompletableFuture<>();
status = Status.Removing;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerMessageDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerMessageDeduplicationTest.java
index f36caf87e6c7..bc8af54570d1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerMessageDeduplicationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerMessageDeduplicationTest.java
@@ -56,6 +56,7 @@ import org.testng.annotations.Test;
public class BrokerMessageDeduplicationTest {
private ManagedLedger managedLedger;
+ private PersistentTopic topic;
private MessageDeduplication deduplication;
private ScheduledExecutorService executor;
@@ -68,9 +69,9 @@ public class BrokerMessageDeduplicationTest {
executor = Executors.newScheduledThreadPool(1, new ExecutorProvider.ExtendedThreadFactory("pulsar"));
doReturn(executor).when(pulsarService).getExecutor();
managedLedger = mock(ManagedLedger.class);
- final var mockTopic = mock(PersistentTopic.class);
- doReturn(true).when(mockTopic).isDeduplicationEnabled();
- deduplication = spy(new MessageDeduplication(pulsarService, mockTopic, managedLedger));
+ topic = mock(PersistentTopic.class);
+ doReturn(true).when(topic).isDeduplicationEnabled();
+ deduplication = spy(new MessageDeduplication(pulsarService, topic, managedLedger));
doReturn(true).when(deduplication).isEnabled();
}
@@ -163,6 +164,83 @@ public class BrokerMessageDeduplicationTest {
verify(managedLedger, times(1)).asyncOpenCursor(any(), any(), any());
}
+ @Test
+ public void checkStatusDisablesAfterRecoveryCompletesWhenPolicyChanges() throws Exception {
+ final var cursor = mock(ManagedCursor.class);
+ final var shouldEnableDeduplication = new AtomicReference<>(true);
+ final var openCursorCallback = new AtomicReference<AsyncCallbacks.OpenCursorCallback>();
+ doAnswer(invocation -> shouldEnableDeduplication.get()).when(topic).isDeduplicationEnabled();
+ doAnswer(invocation -> {
+ openCursorCallback.set(invocation.getArgument(1));
+ return null;
+ }).when(managedLedger).asyncOpenCursor(any(), any(), any());
+ doAnswer(invocation -> {
+ ((AsyncCallbacks.DeleteCursorCallback) invocation.getArgument(1)).deleteCursorComplete(null);
+ return null;
+ }).when(managedLedger).asyncDeleteCursor(any(), any(), any());
+ doReturn(Map.of("from-snapshot", 10L)).when(cursor).getProperties();
+ doReturn(false).when(cursor).hasMoreEntries();
+
+ final var enableFuture = deduplication.checkStatus();
+ assertFalse(enableFuture.isDone());
+ assertEquals(String.valueOf(deduplication.getStatus()), "Recovering");
+
+ shouldEnableDeduplication.set(false);
+ final var disableFuture = deduplication.checkStatus();
+ assertFalse(disableFuture.isDone());
+ verify(managedLedger, times(1)).asyncOpenCursor(any(), any(), any());
+
+ openCursorCallback.get().openCursorComplete(cursor, null);
+ enableFuture.get(3, TimeUnit.SECONDS);
+ disableFuture.get(3, TimeUnit.SECONDS);
+
+ assertEquals(String.valueOf(deduplication.getStatus()), "Disabled");
+ assertEquals(deduplication.getLastPublishedSequenceId("from-snapshot"), -1L);
+ verify(managedLedger, times(1)).asyncDeleteCursor(any(), any(), any());
+ }
+
+ @Test
+ public void checkStatusDisablesAfterRecoveryFailsWhenPolicyChanges() throws Exception {
+ final var cursor = mock(ManagedCursor.class);
+ final var shouldEnableDeduplication = new AtomicReference<>(true);
+ final var openCursorCallback = new AtomicReference<AsyncCallbacks.OpenCursorCallback>();
+ doAnswer(invocation -> shouldEnableDeduplication.get()).when(topic).isDeduplicationEnabled();
+ doAnswer(invocation -> {
+ openCursorCallback.set(invocation.getArgument(1));
+ return null;
+ }).when(managedLedger).asyncOpenCursor(any(), any(), any());
+ doAnswer(invocation -> {
+ ((AsyncCallbacks.DeleteCursorCallback) invocation.getArgument(1)).deleteCursorComplete(null);
+ return null;
+ }).when(managedLedger).asyncDeleteCursor(any(), any(), any());
+ doReturn(Map.of()).when(cursor).getProperties();
+ doReturn(true).when(cursor).hasMoreEntries();
+ doAnswer(invocation -> {
+ throw new RuntimeException("asyncReadEntries failed");
+ }).when(cursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any());
+
+ final var enableFuture = deduplication.checkStatus();
+ assertFalse(enableFuture.isDone());
+ assertEquals(String.valueOf(deduplication.getStatus()), "Recovering");
+
+ shouldEnableDeduplication.set(false);
+ final var disableFuture = deduplication.checkStatus();
+ assertFalse(disableFuture.isDone());
+
+ openCursorCallback.get().openCursorComplete(cursor, null);
+ try {
+ enableFuture.get(3, TimeUnit.SECONDS);
+ fail();
+ } catch (ExecutionException e) {
+ assertTrue(e.getCause() instanceof RuntimeException);
+ assertTrue(e.getMessage().contains("asyncReadEntries failed"));
+ }
+ disableFuture.get(3, TimeUnit.SECONDS);
+
+ assertEquals(String.valueOf(deduplication.getStatus()), "Disabled");
+ verify(managedLedger, times(1)).asyncDeleteCursor(any(), any(), any());
+ }
+
@Test
public void checkStatusRetriesAfterFailedEnable() throws Exception {
final var cursor = mock(ManagedCursor.class);
(pulsar) 01/14: [fix][client] Surface V5 client failures as the typed exception of their error category (#26950)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 9e5578c7f193e2892f9f53824f451cb7b6fb7d16
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Fri Oct 16 16:16:34 2026 -0700
[fix][client] Surface V5 client failures as the typed exception of their error category (#26950)
(cherry picked from commit cb89efaed4f9a84de6e9b86a523dbc0305a0ef7d)
---
.../pulsar/client/api/v5/V5AuthErrorTest.java | 191 +++++++++++++++++++++
.../client/api/v5/V5ProducerAccessModeTest.java | 20 +--
.../api/v5/V5ScalableTopicAutoCreateTest.java | 6 +
.../client/api/v5/V5SchemaEvolutionTest.java | 17 +-
.../pulsar/client/api/v5/V5TransactionTest.java | 43 +++++
.../impl/v5/CheckpointConsumerBuilderV5.java | 11 +-
.../pulsar/client/impl/v5/DagWatchClient.java | 2 +-
.../client/impl/v5/MultiTopicQueueConsumer.java | 2 +-
.../client/impl/v5/MultiTopicStreamConsumer.java | 2 +-
.../pulsar/client/impl/v5/ProducerBuilderV5.java | 12 +-
.../client/impl/v5/PulsarClientBuilderV5.java | 4 +-
.../client/impl/v5/PulsarClientProviderV5.java | 4 +-
.../pulsar/client/impl/v5/PulsarClientV5.java | 13 +-
.../client/impl/v5/QueueConsumerBuilderV5.java | 11 +-
.../client/impl/v5/ScalableCheckpointConsumer.java | 5 +-
.../client/impl/v5/ScalableQueueConsumer.java | 5 +-
.../client/impl/v5/ScalableStreamConsumer.java | 5 +-
.../client/impl/v5/ScalableTopicProducer.java | 52 +++---
.../client/impl/v5/ScalableTopicsWatcher.java | 2 +-
.../client/impl/v5/StreamConsumerBuilderV5.java | 11 +-
.../pulsar/client/impl/v5/TransactionV5.java | 10 +-
.../pulsar/client/impl/v5/V5ExceptionMappings.java | 160 +++++++++++++++++
.../pulsar/client/impl/v5/DagWatchClientTest.java | 26 +++
.../client/impl/v5/PulsarClientBuilderV5Test.java | 13 +-
.../client/impl/v5/V5ExceptionMappingsTest.java | 135 +++++++++++++++
.../org/apache/pulsar/client/impl/ClientCnx.java | 2 +-
26 files changed, 648 insertions(+), 116 deletions(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5AuthErrorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5AuthErrorTest.java
new file mode 100644
index 000000000000..2e802472a2c4
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5AuthErrorTest.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api.v5;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import io.jsonwebtoken.Jwts;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.api.v5.auth.AuthenticationFactory;
+import org.apache.pulsar.client.api.v5.config.ProducerAccessMode;
+import org.apache.pulsar.client.api.v5.schema.Schema;
+import org.apache.pulsar.client.impl.auth.AuthenticationToken;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.apache.pulsar.security.MockedPulsarStandalone;
+import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * A V5 client whose role is authenticated but not permitted gets a typed
+ * {@link PulsarClientException.AuthorizationException}, whichever request the broker rejects: the topic
+ * lookup (DAG watch), the namespace watch, the subscribe to the topic's controller, or a per-segment producer,
+ * consumer or reader. A client whose credentials are rejected gets a typed
+ * {@link PulsarClientException.AuthenticationException}. See error-model.md §2 and §4.
+ */
+public class V5AuthErrorTest extends MockedPulsarStandalone {
+
+ private static final String NAMESPACE = "public/default";
+ private static final String PRODUCER_ROLE = "v5-producer";
+ private static final String CONSUMER_ROLE = "v5-consumer";
+
+ private PulsarAdmin superUserAdmin;
+ /** No permission at all. */
+ private PulsarClient nobodyClient;
+ /** Produce permission on the namespace only. */
+ private PulsarClient producerClient;
+ /** Consume permission on the namespace only. */
+ private PulsarClient consumerClient;
+
+ @BeforeClass(alwaysRun = true)
+ public void setup() throws Exception {
+ configureTokenAuthentication();
+ configureDefaultAuthorization();
+ start();
+ superUserAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(SUPER_USER_TOKEN))
+ .build();
+ superUserAdmin.namespaces().grantPermissionOnNamespace(NAMESPACE, PRODUCER_ROLE, Set.of(AuthAction.produce));
+ superUserAdmin.namespaces().grantPermissionOnNamespace(NAMESPACE, CONSUMER_ROLE, Set.of(AuthAction.consume));
+ nobodyClient = newClient(NOBODY_TOKEN);
+ producerClient = newClient(tokenOf(PRODUCER_ROLE));
+ consumerClient = newClient(tokenOf(CONSUMER_ROLE));
+ }
+
+ @AfterClass(alwaysRun = true)
+ public void cleanup() throws Exception {
+ for (AutoCloseable c : new AutoCloseable[] {nobodyClient, producerClient, consumerClient, superUserAdmin}) {
+ if (c != null) {
+ c.close();
+ }
+ }
+ close();
+ }
+
+ @Test
+ public void testLookupNotAuthorized() throws Exception {
+ String topic = newScalableTopic();
+
+ assertNotAuthorized(() -> nobodyClient.newProducer(Schema.string()).topic(topic).create());
+ assertNotAuthorized(nobodyClient.newProducer(Schema.string()).topic(topic).createAsync());
+ assertNotAuthorized(() -> nobodyClient.newQueueConsumer(Schema.string()).topic(topic)
+ .subscriptionName("queue").subscribe());
+ assertNotAuthorized(() -> nobodyClient.newStreamConsumer(Schema.string()).topic(topic)
+ .subscriptionName("stream").subscribe());
+ assertNotAuthorized(nobodyClient.newStreamConsumer(Schema.string()).topic(topic)
+ .subscriptionName("stream").subscribeAsync());
+ assertNotAuthorized(() -> nobodyClient.newCheckpointConsumer(Schema.string()).topic(topic).create());
+ assertNotAuthorized(() -> nobodyClient.newCheckpointConsumer(Schema.string()).topic(topic)
+ .consumerGroup("group").create());
+ }
+
+ @Test
+ public void testNamespaceWatchNotAuthorized() {
+ assertNotAuthorized(() -> nobodyClient.newStreamConsumer(Schema.string()).namespace(NAMESPACE)
+ .subscriptionName("stream").subscribe());
+ assertNotAuthorized(() -> nobodyClient.newQueueConsumer(Schema.string()).namespace(NAMESPACE)
+ .subscriptionName("queue").subscribe());
+ }
+
+ @Test
+ public void testSubscribeNotAuthorized() throws Exception {
+ String topic = newScalableTopic();
+ // The role can look the topic up and produce to it...
+ @Cleanup
+ Producer<String> producer = producerClient.newProducer(Schema.string()).topic(topic).create();
+ producer.newMessage().value("allowed").send();
+
+ // ...but neither the controller (stream, consumer group) nor a segment (queue, checkpoint) lets it consume.
+ assertNotAuthorized(() -> producerClient.newStreamConsumer(Schema.string()).topic(topic)
+ .subscriptionName("stream").subscribe());
+ assertNotAuthorized(() -> producerClient.newCheckpointConsumer(Schema.string()).topic(topic)
+ .consumerGroup("group").create());
+ assertNotAuthorized(() -> producerClient.newQueueConsumer(Schema.string()).topic(topic)
+ .subscriptionName("queue").subscribe());
+ assertNotAuthorized(producerClient.newQueueConsumer(Schema.string()).topic(topic)
+ .subscriptionName("queue").subscribeAsync());
+ assertNotAuthorized(() -> producerClient.newCheckpointConsumer(Schema.string()).topic(topic).create());
+ }
+
+ @Test
+ public void testProduceNotAuthorized() throws Exception {
+ String topic = newScalableTopic();
+ // The role can look the topic up and subscribe to it...
+ @Cleanup
+ StreamConsumer<String> consumer = consumerClient.newStreamConsumer(Schema.string()).topic(topic)
+ .subscriptionName("stream").subscribe();
+ // ...and create a producer, which attaches to the segments on the first send...
+ @Cleanup
+ Producer<String> producer = consumerClient.newProducer(Schema.string()).topic(topic).create();
+
+ // ...where the segment rejects it.
+ assertNotAuthorized(() -> producer.newMessage().value("rejected").send());
+ assertNotAuthorized(producer.async().newMessage().value("rejected").send());
+ // An exclusive producer attaches to the segments on create.
+ assertNotAuthorized(() -> consumerClient.newProducer(Schema.string()).topic(topic)
+ .accessMode(ProducerAccessMode.EXCLUSIVE).create());
+ }
+
+ @Test
+ public void testCredentialsRejected() throws Exception {
+ String topic = newScalableTopic();
+ @Cleanup
+ PulsarClient client = newClient("not-a-valid-token");
+
+ assertThatThrownBy(() -> client.newProducer(Schema.string()).topic(topic).create())
+ .isInstanceOf(PulsarClientException.AuthenticationException.class);
+ assertThatThrownBy(() -> client.newStreamConsumer(Schema.string()).topic(topic)
+ .subscriptionName("stream").subscribe())
+ .isInstanceOf(PulsarClientException.AuthenticationException.class);
+ }
+
+ private static void assertNotAuthorized(ThrowingCallable call) {
+ assertThatThrownBy(call).isInstanceOf(PulsarClientException.AuthorizationException.class);
+ }
+
+ private static void assertNotAuthorized(CompletableFuture<?> future) {
+ assertThat(future).failsWithin(30, TimeUnit.SECONDS)
+ .withThrowableThat()
+ .withCauseInstanceOf(PulsarClientException.AuthorizationException.class);
+ }
+
+ private String newScalableTopic() throws Exception {
+ String topic = "topic://" + NAMESPACE + "/" + UUID.randomUUID();
+ superUserAdmin.scalableTopics().createScalableTopic(topic, 1);
+ return topic;
+ }
+
+ private PulsarClient newClient(String token) throws PulsarClientException {
+ return PulsarClient.builder()
+ .serviceUrl(getPulsarService().getBrokerServiceUrl())
+ .authentication(AuthenticationFactory.token(token))
+ .build();
+ }
+
+ private static String tokenOf(String role) {
+ return Jwts.builder().claim("sub", role).signWith(SECRET_KEY).compact();
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ProducerAccessModeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ProducerAccessModeTest.java
index 7424ae30fa2f..b4b466ab97fa 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ProducerAccessModeTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ProducerAccessModeTest.java
@@ -19,8 +19,7 @@
package org.apache.pulsar.client.api.v5;
import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
+import static org.testng.Assert.expectThrows;
import java.time.Duration;
import lombok.Cleanup;
import org.apache.pulsar.client.api.v5.config.ProducerAccessMode;
@@ -84,18 +83,11 @@ public class V5ProducerAccessModeTest extends V5ClientBaseTest {
// Second EXCLUSIVE on the same topic must fail at create() — the V5 builder
// eagerly claims every active segment up front for exclusive access modes.
- try {
- v5Client.newProducer(Schema.string())
- .topic(topic)
- .accessMode(ProducerAccessMode.EXCLUSIVE)
- .create();
- fail("a second EXCLUSIVE producer should not be able to attach");
- } catch (PulsarClientException expected) {
- String msg = expected.getMessage() == null ? "" : expected.getMessage();
- assertTrue(msg.contains("Busy") || msg.contains("Fenced")
- || msg.contains("ProducerFenced") || msg.contains("ProducerBusy"),
- "unexpected error message for second EXCLUSIVE producer: " + msg);
- }
+ expectThrows(PulsarClientException.ProducerBusyException.class, () ->
+ v5Client.newProducer(Schema.string())
+ .topic(topic)
+ .accessMode(ProducerAccessMode.EXCLUSIVE)
+ .create());
// The first producer is still healthy and can send.
first.newMessage().value("still-here").send();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ScalableTopicAutoCreateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ScalableTopicAutoCreateTest.java
index a9c31ebdde9c..1c8493ec71d7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ScalableTopicAutoCreateTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ScalableTopicAutoCreateTest.java
@@ -99,6 +99,12 @@ public class V5ScalableTopicAutoCreateTest extends V5ClientBaseTest {
// topic does — not a generic PulsarClientException.
assertThrows(PulsarClientException.NotFoundException.class, () ->
v5Client.newProducer(Schema.string()).topic(topic).create());
+ assertThrows(PulsarClientException.NotFoundException.class, () ->
+ v5Client.newQueueConsumer(Schema.string()).topic(topic).subscriptionName("sub").subscribe());
+ assertThrows(PulsarClientException.NotFoundException.class, () ->
+ v5Client.newStreamConsumer(Schema.string()).topic(topic).subscriptionName("sub").subscribe());
+ assertThrows(PulsarClientException.NotFoundException.class, () ->
+ v5Client.newCheckpointConsumer(Schema.string()).topic(topic).create());
} finally {
admin.namespaces().removeAutoTopicCreation(getNamespace());
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5SchemaEvolutionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5SchemaEvolutionTest.java
index 7c8ac2d669ee..86aba2345792 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5SchemaEvolutionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5SchemaEvolutionTest.java
@@ -21,10 +21,9 @@ package org.apache.pulsar.client.api.v5;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
+import static org.testng.Assert.expectThrows;
import java.time.Duration;
import java.util.Objects;
-import java.util.concurrent.CompletionException;
import lombok.Cleanup;
import org.apache.pulsar.client.api.v5.config.SubscriptionInitialPosition;
import org.apache.pulsar.client.api.v5.schema.Schema;
@@ -178,17 +177,9 @@ public class V5SchemaEvolutionTest extends V5ClientBaseTest {
Producer<VBad> bad = v5Client.newProducer(Schema.avro(VBad.class))
.topic(topic)
.create();
- try {
- bad.newMessage().key("bad").value(new VBad(5, 50)).send();
- fail("expected incompatible schema to be rejected post-split");
- } catch (PulsarClientException expected) {
- // good — the broker preserved v1 history across the split and rejected
- // the registration.
- } catch (CompletionException ce) {
- // Some V5 builders may surface the error wrapped — accept either form.
- assertTrue(ce.getCause() instanceof PulsarClientException,
- "unexpected wrapped error: " + ce);
- }
+ // The broker preserved v1 history across the split and rejects the registration.
+ expectThrows(PulsarClientException.IncompatibleSchemaException.class,
+ () -> bad.newMessage().key("bad").value(new VBad(5, 50)).send());
// The registry still shows just v1 — the failed registration didn't leak through.
var schemas = admin.schemas().getAllSchemas(topic);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5TransactionTest.java
index 59057de8bf36..3e91bfe26ab2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5TransactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5TransactionTest.java
@@ -22,11 +22,13 @@ import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.expectThrows;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.ExecutionException;
import lombok.Cleanup;
import org.apache.pulsar.client.api.v5.config.SubscriptionInitialPosition;
import org.apache.pulsar.client.api.v5.config.TransactionPolicy;
@@ -71,6 +73,47 @@ public class V5TransactionTest extends V5ClientBaseTest {
assertEquals(txn.state(), Transaction.State.ABORTED);
}
+ @Test
+ public void testCommitOfAbortedTransactionFailsWithTransactionConflict() throws Exception {
+ // A transaction that is no longer open cannot be committed: the commit fails with a
+ // typed TransactionConflict, blocking and async alike.
+ PulsarClient client = newTxnClient();
+ Transaction txn = client.newTransaction();
+ txn.abort();
+
+ expectThrows(PulsarClientException.TransactionConflictException.class, txn::commit);
+ ExecutionException e = expectThrows(ExecutionException.class, () -> txn.async().commit().get());
+ assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException, e.toString());
+ }
+
+ @Test
+ public void testSendInAbortedTransactionFailsWithTransactionConflict() throws Exception {
+ // A send in a transaction that is no longer open fails with a typed TransactionConflict,
+ // blocking and async alike.
+ PulsarClient client = newTxnClient();
+ String topic = newScalableTopic(1);
+
+ @Cleanup
+ Producer<String> producer = client.newProducer(Schema.string())
+ .topic(topic)
+ .create();
+
+ Transaction txn = client.newTransaction();
+ txn.abort();
+
+ expectThrows(PulsarClientException.TransactionConflictException.class,
+ () -> producer.newMessage().transaction(txn).value("blocking").send());
+ ExecutionException e = expectThrows(ExecutionException.class,
+ () -> producer.async().newMessage().transaction(txn).value("async").send().get());
+ assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException, e.toString());
+ }
+
+ @Test
+ public void testNewTransactionWithoutTransactionPolicyFailsWithInvalidConfiguration() {
+ // The shared client has no transaction policy, so it has transactions disabled.
+ expectThrows(PulsarClientException.InvalidConfigurationException.class, v5Client::newTransaction);
+ }
+
@Test
public void testCommitMakesProducedMessagesVisible() throws Exception {
PulsarClient client = newTxnClient();
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/CheckpointConsumerBuilderV5.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/CheckpointConsumerBuilderV5.java
index e87a41231dde..5859cc1b5fad 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/CheckpointConsumerBuilderV5.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/CheckpointConsumerBuilderV5.java
@@ -52,10 +52,7 @@ final class CheckpointConsumerBuilderV5<T> implements CheckpointConsumerBuilder<
try {
return createAsync().join();
} catch (java.util.concurrent.CompletionException e) {
- if (e.getCause() instanceof PulsarClientException pce) {
- throw pce;
- }
- throw new PulsarClientException(e.getCause());
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -88,14 +85,16 @@ final class CheckpointConsumerBuilderV5<T> implements CheckpointConsumerBuilder<
// its clean unsubscribe once the in-flight attempt settles.
session.close();
}
- });
+ })
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
// Unmanaged: read every active segment, no broker-side state.
DagWatchClient dagWatch = new DagWatchClient(client.v4Client(), topic);
return dagWatch.start()
.thenCompose(initialLayout -> ScalableCheckpointConsumer.createUnmanagedAsync(
- client, v5Schema, dagWatch, initialLayout, startPosition, consumerName));
+ client, v5Schema, dagWatch, initialLayout, startPosition, consumerName))
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
@Override
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/DagWatchClient.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/DagWatchClient.java
index dcb152a24720..dd01b80922ad 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/DagWatchClient.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/DagWatchClient.java
@@ -236,7 +236,7 @@ final class DagWatchClient implements DagWatchSession, AutoCloseable {
initialLayoutFuture.completeExceptionally(
error == ServerError.TopicNotFound
? new PulsarClientException.NotFoundException(detail)
- : new PulsarClientException(detail));
+ : ClientCnx.getPulsarClientException(error, detail));
}
// After the initial layout has arrived, broker-side errors on this session
// (e.g., metadata unavailable) are transient — a reconnect typically clears
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/MultiTopicQueueConsumer.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/MultiTopicQueueConsumer.java
index b808536d4e02..11cf0a5de578 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/MultiTopicQueueConsumer.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/MultiTopicQueueConsumer.java
@@ -375,7 +375,7 @@ final class MultiTopicQueueConsumer<T> implements QueueConsumerImpl<T> {
Thread.currentThread().interrupt();
throw new PulsarClientException("Close interrupted", e);
} catch (ExecutionException e) {
- throw new PulsarClientException(e.getCause());
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/MultiTopicStreamConsumer.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/MultiTopicStreamConsumer.java
index 4b6d40c5eac8..1e4903adc0f6 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/MultiTopicStreamConsumer.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/MultiTopicStreamConsumer.java
@@ -381,7 +381,7 @@ final class MultiTopicStreamConsumer<T> implements StreamConsumer<T> {
Thread.currentThread().interrupt();
throw new PulsarClientException("Close interrupted", e);
} catch (ExecutionException e) {
- throw new PulsarClientException(e.getCause());
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ProducerBuilderV5.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ProducerBuilderV5.java
index db04b64994a2..c244fe9d970a 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ProducerBuilderV5.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ProducerBuilderV5.java
@@ -58,14 +58,7 @@ final class ProducerBuilderV5<T> implements ProducerBuilder<T> {
try {
return createAsync().join();
} catch (java.util.concurrent.CompletionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof PulsarClientException pce) {
- throw pce;
- }
- if (cause instanceof org.apache.pulsar.client.api.PulsarClientException.NotFoundException) {
- throw new PulsarClientException.NotFoundException(cause.getMessage());
- }
- throw new PulsarClientException(cause);
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -94,7 +87,8 @@ final class ProducerBuilderV5<T> implements ProducerBuilder<T> {
throw ex instanceof java.util.concurrent.CompletionException ce
? ce : new java.util.concurrent.CompletionException(ex);
}));
- });
+ })
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
@Override
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientBuilderV5.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientBuilderV5.java
index 94183dfd381c..b6a6c2ca4ed7 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientBuilderV5.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientBuilderV5.java
@@ -117,7 +117,7 @@ final class PulsarClientBuilderV5 implements PulsarClientBuilder {
var v4Client = new PulsarClientImpl(clientConf);
return new PulsarClientV5(v4Client, description, transactionTimeout);
} catch (org.apache.pulsar.client.api.PulsarClientException e) {
- throw new PulsarClientException(e.getMessage(), e);
+ throw V5ExceptionMappings.toV5Exception(e);
} finally {
// Whether the factory was consumed is read off the copy the client was given rather than
// inferred from how the build ended: a build that failed after the framework took the instance
@@ -222,7 +222,7 @@ final class PulsarClientBuilderV5 implements PulsarClientBuilder {
try {
this.v5Authentication = V5AuthenticationLoader.create(authPluginClassName, authParamsString);
} catch (org.apache.pulsar.client.api.PulsarClientException e) {
- throw new PulsarClientException(e.getMessage(), e);
+ throw V5ExceptionMappings.toV5Exception(e);
}
return this;
}
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientProviderV5.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientProviderV5.java
index eb05fc8dea15..79809a120b26 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientProviderV5.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientProviderV5.java
@@ -194,7 +194,7 @@ public final class PulsarClientProviderV5 implements PulsarClientProvider {
try {
return V5AuthenticationLoader.create(className, params);
} catch (org.apache.pulsar.client.api.PulsarClientException e) {
- throw new PulsarClientException(e.getMessage(), e);
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -204,7 +204,7 @@ public final class PulsarClientProviderV5 implements PulsarClientProvider {
try {
return V5AuthenticationLoader.create(className, params);
} catch (org.apache.pulsar.client.api.PulsarClientException e) {
- throw new PulsarClientException(e.getMessage(), e);
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
}
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientV5.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientV5.java
index 792d9ea060ce..599d1fddbd2b 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientV5.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/PulsarClientV5.java
@@ -20,7 +20,6 @@ package org.apache.pulsar.client.impl.v5;
import java.time.Duration;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.pulsar.client.api.v5.CheckpointConsumerBuilder;
@@ -81,8 +80,7 @@ final class PulsarClientV5 implements PulsarClient {
try {
return newTransactionAsync().get();
} catch (ExecutionException e) {
- Throwable cause = e.getCause() != null ? e.getCause() : e;
- throw new PulsarClientException(cause.getMessage(), cause);
+ throw V5ExceptionMappings.toV5Exception(e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new PulsarClientException("Interrupted while creating transaction", e);
@@ -95,7 +93,8 @@ final class PulsarClientV5 implements PulsarClient {
if (transactionTimeout != null) {
builder.withTransactionTimeout(transactionTimeout.toMillis(), TimeUnit.MILLISECONDS);
}
- return builder.build().thenApply(v4Txn -> (Transaction) new TransactionV5(v4Txn));
+ return builder.build().thenApply(v4Txn -> (Transaction) new TransactionV5(v4Txn))
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
@Override
@@ -103,15 +102,13 @@ final class PulsarClientV5 implements PulsarClient {
try {
v4Client.close();
} catch (org.apache.pulsar.client.api.PulsarClientException e) {
- throw new PulsarClientException(e.getMessage(), e);
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@Override
public CompletableFuture<Void> closeAsync() {
- return v4Client.closeAsync().exceptionally(ex -> {
- throw new CompletionException(new PulsarClientException(ex.getMessage(), ex));
- });
+ return v4Client.closeAsync().exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
@Override
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/QueueConsumerBuilderV5.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/QueueConsumerBuilderV5.java
index c1d8bdf261fb..b24626d5720e 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/QueueConsumerBuilderV5.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/QueueConsumerBuilderV5.java
@@ -66,10 +66,7 @@ final class QueueConsumerBuilderV5<T> implements QueueConsumerBuilder<T> {
try {
return subscribeAsync().join();
} catch (java.util.concurrent.CompletionException e) {
- if (e.getCause() instanceof PulsarClientException pce) {
- throw pce;
- }
- throw new PulsarClientException(e.getCause());
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -89,13 +86,15 @@ final class QueueConsumerBuilderV5<T> implements QueueConsumerBuilder<T> {
if (namespaceSet) {
return MultiTopicQueueConsumer.createAsync(
- client, v5Schema, conf, namespaceName, propertyFilters);
+ client, v5Schema, conf, namespaceName, propertyFilters)
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
TopicName topic = V5Utils.parseScalableTopicInput(topicName);
DagWatchClient dagWatch = new DagWatchClient(client.v4Client(), topic);
return dagWatch.start()
.thenCompose(initialLayout -> ScalableQueueConsumer.createAsync(
- client, v5Schema, conf, dagWatch, initialLayout, dlqPolicy));
+ client, v5Schema, conf, dagWatch, initialLayout, dlqPolicy))
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
@Override
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableCheckpointConsumer.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableCheckpointConsumer.java
index 7ce0b1ad694f..8e434c2f8000 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableCheckpointConsumer.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableCheckpointConsumer.java
@@ -277,7 +277,7 @@ final class ScalableCheckpointConsumer<T> implements CheckpointConsumer<T> {
Thread.currentThread().interrupt();
throw new PulsarClientException("Close interrupted", e);
} catch (ExecutionException e) {
- throw new PulsarClientException(e.getCause());
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -319,7 +319,8 @@ final class ScalableCheckpointConsumer<T> implements CheckpointConsumer<T> {
: CompletableFuture.completedFuture(null)));
}
return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new))
- .whenComplete((__, ___) -> segmentReaders.clear());
+ .whenComplete((__, ___) -> segmentReaders.clear())
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
// --- Assignment change handling ---
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumer.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumer.java
index 02e40c15078d..2e6623759ee5 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumer.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumer.java
@@ -277,7 +277,7 @@ final class ScalableQueueConsumer<T> implements QueueConsumerImpl<T>, DagWatchCl
Thread.currentThread().interrupt();
throw new PulsarClientException("Close interrupted", e);
} catch (ExecutionException e) {
- throw new PulsarClientException(e.getCause());
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -309,7 +309,8 @@ final class ScalableQueueConsumer<T> implements QueueConsumerImpl<T>, DagWatchCl
: CompletableFuture.completedFuture(null)));
}
return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new))
- .whenComplete((__, ___) -> segmentConsumers.clear());
+ .whenComplete((__, ___) -> segmentConsumers.clear())
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
// --- Layout change handling ---
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumer.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumer.java
index 9764b27876c0..2ff846b67c46 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumer.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumer.java
@@ -467,7 +467,7 @@ final class ScalableStreamConsumer<T>
Thread.currentThread().interrupt();
throw new PulsarClientException("Close interrupted", e);
} catch (ExecutionException e) {
- throw new PulsarClientException(e.getCause());
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -515,7 +515,8 @@ final class ScalableStreamConsumer<T>
sharedSegmentUnacked.clear();
lastCumulativeAcked.clear();
cumulativeAckWatermark.clear();
- });
+ })
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
// --- Assignment change handling ---
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableTopicProducer.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableTopicProducer.java
index 62e468edfce4..9d5010d8b259 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableTopicProducer.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableTopicProducer.java
@@ -431,18 +431,13 @@ final class ScalableTopicProducer<T> implements Producer<T>, DagWatchClient.Layo
var v4MsgId = sendV4(producer, send);
return new MessageIdV5(v4MsgId, segmentId);
} catch (PulsarClientException e) {
- // Thrown while (re)creating the per-segment producer — already a V5 exception
- // (it may wrap a v4 TopicTerminated/AlreadyClosed cause).
+ // Thrown while (re)creating the per-segment producer or by its send(): the V5
+ // exception for the v4 failure, which is its cause (a v4 TopicTerminated or
+ // AlreadyClosed one when the segment is gone).
if (!isSegmentGoneError(e)) {
throw e;
}
lastError = e;
- } catch (org.apache.pulsar.client.api.PulsarClientException e) {
- // Thrown by the v4 producer's send().
- if (!isSegmentGoneError(e)) {
- throw new PulsarClientException(e.getMessage(), e);
- }
- lastError = new PulsarClientException(e.getMessage(), e);
}
// The target segment is gone: sealed by a split/merge, or terminated by a
// regular-to-scalable migration. Wait for the DAG watch to deliver the new
@@ -471,29 +466,33 @@ final class ScalableTopicProducer<T> implements Producer<T>, DagWatchClient.Layo
/**
* The v4 {@code send()} with the payload accounting handed over in between its steps: enqueue,
* drop this layer's share of the payload, flush so that a batched message does not wait out the
- * batching delay, then wait for the acknowledgement.
+ * batching delay, then wait for the acknowledgement. It fails with the V5 exception for the
+ * send's own failure, as the async send does, not for its v4 {@code PulsarClientException.unwrap},
+ * which turns a failure of another type, such as a transaction that is no longer open, into a
+ * plain v4 exception.
*/
private org.apache.pulsar.client.api.MessageId sendV4(org.apache.pulsar.client.api.Producer<T> producer,
- PendingSend<T> send)
- throws org.apache.pulsar.client.api.PulsarClientException {
+ PendingSend<T> send) throws PulsarClientException {
+ var ackFuture = buildV4Message(producer, send).sendAsync();
+ if (!ackFuture.isDone()) {
+ releasePayloadShare(send);
+ producer.flushAsync();
+ }
try {
- var ackFuture = buildV4Message(producer, send).sendAsync();
- if (!ackFuture.isDone()) {
- releasePayloadShare(send);
- producer.flushAsync();
- }
return ackFuture.get();
- } catch (Exception e) {
- throw org.apache.pulsar.client.api.PulsarClientException.unwrap(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new PulsarClientException("Interrupted while waiting for the send to be acknowledged", e);
+ } catch (ExecutionException e) {
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
/**
* True if {@code t} (or one of its causes) signals that the target segment is gone —
* sealed by a split/merge or terminated by a regular-to-scalable migration — so the send
- * should be retried once the new layout arrives. Handles both the v4 exceptions thrown by
- * {@code send()} and the V5-wrapped exceptions thrown while (re)creating the per-segment
- * producer on a now-terminated topic.
+ * should be retried once the new layout arrives. Handles both the v4 exceptions that fail an
+ * async send and the V5 exceptions, caused by them, that fail a blocking one.
*/
private static boolean isSegmentGoneError(Throwable t) {
for (Throwable cause = t; cause != null; cause = cause.getCause()) {
@@ -647,7 +646,7 @@ final class ScalableTopicProducer<T> implements Producer<T>, DagWatchClient.Layo
.log("Target segment gone, retrying async send after layout update");
retry.run();
} else {
- finish(send, null, ex);
+ finish(send, null, V5ExceptionMappings.toV5Exception(ex));
}
}
@@ -995,14 +994,7 @@ final class ScalableTopicProducer<T> implements Producer<T>, DagWatchClient.Layo
Thread.currentThread().interrupt();
throw new PulsarClientException("Interrupted while creating segment producer", e);
} catch (ExecutionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof org.apache.pulsar.client.api.PulsarClientException v4Exc) {
- throw new PulsarClientException(v4Exc.getMessage(), v4Exc);
- }
- if (cause instanceof PulsarClientException v5Exc) {
- throw v5Exc;
- }
- throw new PulsarClientException(cause != null ? cause : e);
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
}
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableTopicsWatcher.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableTopicsWatcher.java
index 984a55a8afd6..21755ddbf566 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableTopicsWatcher.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableTopicsWatcher.java
@@ -298,7 +298,7 @@ final class ScalableTopicsWatcher implements ScalableTopicsWatcherSession, AutoC
.log("WatchScalableTopics rejected");
if (!initialSnapshotFuture.isDone()) {
initialSnapshotFuture.completeExceptionally(
- new PulsarClientException("WatchScalableTopics failed: " + error
+ ClientCnx.getPulsarClientException(error, "WatchScalableTopics failed: " + error
+ (message != null ? " - " + message : "")));
}
}
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/StreamConsumerBuilderV5.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/StreamConsumerBuilderV5.java
index a408fdf9fa69..d4dab5d38c16 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/StreamConsumerBuilderV5.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/StreamConsumerBuilderV5.java
@@ -56,10 +56,7 @@ final class StreamConsumerBuilderV5<T> implements StreamConsumerBuilder<T> {
try {
return subscribeAsync().join();
} catch (java.util.concurrent.CompletionException e) {
- if (e.getCause() instanceof PulsarClientException pce) {
- throw pce;
- }
- throw new PulsarClientException(e.getCause());
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -84,7 +81,8 @@ final class StreamConsumerBuilderV5<T> implements StreamConsumerBuilder<T> {
if (namespaceSet) {
return MultiTopicStreamConsumer.createAsync(
- client, v5Schema, conf, namespaceName, propertyFilters);
+ client, v5Schema, conf, namespaceName, propertyFilters)
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
TopicName topic = V5Utils.parseScalableTopicInput(topicName);
@@ -107,7 +105,8 @@ final class StreamConsumerBuilderV5<T> implements StreamConsumerBuilder<T> {
// path already closed the session.
session.close();
}
- });
+ })
+ .exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
@Override
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/TransactionV5.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/TransactionV5.java
index ff7fa5223c15..2cbe476afa17 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/TransactionV5.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/TransactionV5.java
@@ -50,8 +50,7 @@ final class TransactionV5 implements Transaction {
Thread.currentThread().interrupt();
throw new PulsarClientException("Commit interrupted", e);
} catch (ExecutionException e) {
- Throwable cause = e.getCause() != null ? e.getCause() : e;
- throw new PulsarClientException(cause.getMessage(), cause);
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -63,8 +62,7 @@ final class TransactionV5 implements Transaction {
Thread.currentThread().interrupt();
throw new PulsarClientException("Abort interrupted", e);
} catch (ExecutionException e) {
- Throwable cause = e.getCause() != null ? e.getCause() : e;
- throw new PulsarClientException(cause.getMessage(), cause);
+ throw V5ExceptionMappings.toV5Exception(e);
}
}
@@ -103,12 +101,12 @@ final class TransactionV5 implements Transaction {
private final class AsyncView implements AsyncTransaction {
@Override
public CompletableFuture<Void> commit() {
- return v4Transaction.commit();
+ return v4Transaction.commit().exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
@Override
public CompletableFuture<Void> abort() {
- return v4Transaction.abort();
+ return v4Transaction.abort().exceptionallyCompose(V5ExceptionMappings::failWithV5Exception);
}
}
}
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/V5ExceptionMappings.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/V5ExceptionMappings.java
new file mode 100644
index 000000000000..d3089ea2b486
--- /dev/null
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/V5ExceptionMappings.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl.v5;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException;
+import org.apache.pulsar.client.api.v5.PulsarClientException;
+import org.apache.pulsar.common.util.FutureUtil;
+
+/**
+ * Converts the failures of the operations that the V5 client runs on the v4 client into the V5 exceptions that the
+ * application gets, typed by their error category (error-model.md §2).
+ */
+final class V5ExceptionMappings {
+
+ /** The V5 type of each error category of error-model.md §2, for the v4 exceptions that report it. */
+ private static final List<V5Type> V5_TYPES = List.of(
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.NotFoundException.class,
+ PulsarClientException.NotFoundException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.TopicDoesNotExistException.class,
+ PulsarClientException.NotFoundException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.AuthenticationException.class,
+ PulsarClientException.AuthenticationException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.GettingAuthenticationDataException.class,
+ PulsarClientException.GettingAuthenticationDataException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.UnsupportedAuthenticationException.class,
+ PulsarClientException.UnsupportedAuthenticationException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.AuthorizationException.class,
+ PulsarClientException.AuthorizationException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.TimeoutException.class,
+ PulsarClientException.TimeoutException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.AlreadyClosedException.class,
+ PulsarClientException.AlreadyClosedException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.ProducerBusyException.class,
+ PulsarClientException.ProducerBusyException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.ProducerFencedException.class,
+ PulsarClientException.ProducerBusyException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.ConsumerBusyException.class,
+ PulsarClientException.ConsumerBusyException::new),
+ // Overlapping Key_Shared hash ranges: a subscription constraint that keeps the consumer out.
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.ConsumerAssignException.class,
+ PulsarClientException.ConsumerBusyException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.IncompatibleSchemaException.class,
+ PulsarClientException.IncompatibleSchemaException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.TopicTerminatedException.class,
+ PulsarClientException.TopicTerminatedException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.ConnectException.class,
+ PulsarClientException.ConnectException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.NotConnectedException.class,
+ PulsarClientException.NotConnectedException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.ProducerQueueIsFullError.class,
+ PulsarClientException.ProducerQueueIsFullException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.MemoryBufferIsFullError.class,
+ PulsarClientException.MemoryBufferIsFullException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.TransactionConflictException.class,
+ PulsarClientException.TransactionConflictException::new),
+ // A transaction with a failed operation, no longer open, or no longer known to its coordinator (timed
+ // out) cannot be committed or aborted (client-api.md).
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.TransactionHasOperationFailedException.class,
+ PulsarClientException.TransactionConflictException::new),
+ new V5Type(TransactionCoordinatorClientException.InvalidTxnStatusException.class,
+ PulsarClientException.TransactionConflictException::new),
+ new V5Type(TransactionCoordinatorClientException.TransactionNotFoundException.class,
+ PulsarClientException.TransactionConflictException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.CryptoException.class,
+ PulsarClientException.CryptoException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.InvalidConfigurationException.class,
+ PulsarClientException.InvalidConfigurationException::new),
+ new V5Type(org.apache.pulsar.client.api.PulsarClientException.InvalidServiceURL.class,
+ PulsarClientException.InvalidServiceURLException::new));
+
+ /** {@link #V5_TYPES} by their v4 type, for a failure of exactly one of these types. */
+ private static final Map<Class<?>, V5Type> V5_TYPES_BY_V4_TYPE = V5_TYPES.stream()
+ .collect(Collectors.toUnmodifiableMap(V5Type::v4Type, Function.identity()));
+
+ /**
+ * {@link #V5_TYPES} with each v4 type before its supertypes, for a failure of a subtype of them: the closest
+ * supertype decides, whatever the order of {@link #V5_TYPES}.
+ */
+ private static final List<V5Type> V5_TYPES_SUBTYPES_FIRST = V5_TYPES.stream()
+ .sorted(Comparator.comparingInt(V5Type::depth).reversed())
+ .toList();
+
+ private V5ExceptionMappings() {
+ }
+
+ /**
+ * The V5 exception to hand the application for the failure of an operation that the V5 client runs on the
+ * v4 client, so that it can branch on the error category (error-model.md §2): the V5 type of its category,
+ * or the plain {@link PulsarClientException} for a failure outside them, either one caused by the original
+ * failure. A V5 exception is returned as is. A {@code CompletionException} or {@code ExecutionException} is
+ * unwrapped first.
+ */
+ static PulsarClientException toV5Exception(Throwable failure) {
+ Throwable cause = FutureUtil.unwrapCompletionException(failure);
+ if (cause instanceof PulsarClientException v5Exception) {
+ return v5Exception;
+ }
+ V5Type type = v5TypeOf(cause);
+ if (type == null) {
+ return new PulsarClientException(cause.getMessage(), cause);
+ }
+ PulsarClientException typed = type.constructor().apply(cause.getMessage());
+ // Unlike the plain exception, most of these types have no constructor that takes the cause.
+ typed.initCause(cause);
+ return typed;
+ }
+
+ /** A future failed with the {@link #toV5Exception V5 exception} for {@code failure}. */
+ static <T> CompletableFuture<T> failWithV5Exception(Throwable failure) {
+ return CompletableFuture.failedFuture(toV5Exception(failure));
+ }
+
+ private static V5Type v5TypeOf(Throwable failure) {
+ V5Type type = V5_TYPES_BY_V4_TYPE.get(failure.getClass());
+ if (type != null) {
+ return type;
+ }
+ for (V5Type candidate : V5_TYPES_SUBTYPES_FIRST) {
+ if (candidate.v4Type().isInstance(failure)) {
+ return candidate;
+ }
+ }
+ return null;
+ }
+
+ /** The V5 exception type, by its message constructor, for the v4 exceptions of {@code v4Type}. */
+ private record V5Type(Class<? extends Exception> v4Type, Function<String, PulsarClientException> constructor) {
+
+ /** How deep {@link #v4Type} is in its class hierarchy: a subtype is deeper than its supertypes. */
+ int depth() {
+ int depth = 0;
+ for (Class<?> type = v4Type; type != null; type = type.getSuperclass()) {
+ depth++;
+ }
+ return depth;
+ }
+ }
+}
diff --git a/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/DagWatchClientTest.java b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/DagWatchClientTest.java
index 7f8922312b62..cec11c676fa4 100644
--- a/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/DagWatchClientTest.java
+++ b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/DagWatchClientTest.java
@@ -30,11 +30,14 @@ import io.netty.util.Timer;
import io.netty.util.TimerTask;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.impl.PulsarClientImpl;
import org.apache.pulsar.common.api.proto.ScalableTopicDAG;
+import org.apache.pulsar.common.api.proto.ServerError;
import org.apache.pulsar.common.naming.TopicName;
import org.testng.annotations.Test;
@@ -142,6 +145,29 @@ public class DagWatchClientTest {
assertThat(changes).containsExactly("1->2", "2->3");
}
+ @Test
+ public void testRejectedLookupFailsStartWithItsErrorCategory() {
+ assertThat(startFailure(ServerError.TopicNotFound))
+ .hasCauseInstanceOf(PulsarClientException.NotFoundException.class);
+ assertThat(startFailure(ServerError.AuthorizationError))
+ .hasCauseInstanceOf(PulsarClientException.AuthorizationException.class);
+ assertThat(startFailure(ServerError.AuthenticationError))
+ .hasCauseInstanceOf(PulsarClientException.AuthenticationException.class);
+ }
+
+ /** How {@link DagWatchClient#start} fails when the broker rejects the lookup with {@code error}. */
+ private static Throwable startFailure(ServerError error) {
+ PulsarClientImpl v4Client = mock(PulsarClientImpl.class);
+ // The broker's answer is fed in by the test: the connection never opens.
+ when(v4Client.getConnection(TOPIC)).thenReturn(new CompletableFuture<>());
+ DagWatchClient watch = new DagWatchClient(v4Client, TopicName.get(TOPIC));
+ CompletableFuture<ClientSegmentLayout> start = watch.start();
+
+ watch.onError(error, "rejected");
+
+ return catchThrowable(start::join);
+ }
+
private static void awaitUninterruptibly(CountDownLatch latch) {
try {
latch.await();
diff --git a/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/PulsarClientBuilderV5Test.java b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/PulsarClientBuilderV5Test.java
index 25f691b6ba28..dd17fc9236cd 100644
--- a/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/PulsarClientBuilderV5Test.java
+++ b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/PulsarClientBuilderV5Test.java
@@ -798,13 +798,20 @@ public class PulsarClientBuilderV5Test {
@Test
public void testAuthenticationPluginNotFoundIsWrapped() {
- // A bad plugin class name should surface as V5 PulsarClientException (not a v4 exception
- // type leaking through the surface).
+ // A bad plugin class name should surface as the V5 PulsarClientException of its category (not a
+ // v4 exception type leaking through the surface).
PulsarClientBuilderV5 builder = new PulsarClientBuilderV5();
- assertThrows(PulsarClientException.class, () ->
+ assertThrows(PulsarClientException.UnsupportedAuthenticationException.class, () ->
builder.authentication("com.example.NoSuchAuth", ""));
}
+ @Test
+ public void testBuildWithMalformedServiceUrlFailsWithInvalidServiceUrl() {
+ // The pulsar:// scheme passes the builder's check; the client rejects the port when it is built.
+ PulsarClientBuilder builder = PulsarClient.builder().serviceUrl("pulsar://localhost:not-a-port");
+ assertThrows(PulsarClientException.InvalidServiceURLException.class, builder::build);
+ }
+
/**
* PIP-478: a cross-format TLS-material fold must fail loud, not silently drop trust anchors. A PEM
* {@code tlsPolicy(...)} carrying a private-CA {@code trustCertsFilePath} combined with a keystore-format
diff --git a/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/V5ExceptionMappingsTest.java b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/V5ExceptionMappingsTest.java
new file mode 100644
index 000000000000..b7f34056a3b8
--- /dev/null
+++ b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/V5ExceptionMappingsTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl.v5;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertSame;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException;
+import org.apache.pulsar.client.api.v5.PulsarClientException;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class V5ExceptionMappingsTest {
+
+ /** A v4 exception, and the V5 type of its error category (error-model.md §2). */
+ @DataProvider
+ public static Object[][] errorCategories() {
+ return new Object[][] {
+ {new org.apache.pulsar.client.api.PulsarClientException.NotFoundException("error"),
+ PulsarClientException.NotFoundException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.TopicDoesNotExistException("error"),
+ PulsarClientException.NotFoundException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.AuthenticationException("error"),
+ PulsarClientException.AuthenticationException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.GettingAuthenticationDataException("error"),
+ PulsarClientException.GettingAuthenticationDataException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.UnsupportedAuthenticationException("error"),
+ PulsarClientException.UnsupportedAuthenticationException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.AuthorizationException("error"),
+ PulsarClientException.AuthorizationException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.TimeoutException("error"),
+ PulsarClientException.TimeoutException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.AlreadyClosedException("error"),
+ PulsarClientException.AlreadyClosedException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.ProducerBusyException("error"),
+ PulsarClientException.ProducerBusyException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.ProducerFencedException("error"),
+ PulsarClientException.ProducerBusyException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.ConsumerBusyException("error"),
+ PulsarClientException.ConsumerBusyException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.ConsumerAssignException("error"),
+ PulsarClientException.ConsumerBusyException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.IncompatibleSchemaException("error"),
+ PulsarClientException.IncompatibleSchemaException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.TopicTerminatedException("error"),
+ PulsarClientException.TopicTerminatedException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.ConnectException("error"),
+ PulsarClientException.ConnectException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.NotConnectedException("error"),
+ PulsarClientException.NotConnectedException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.ProducerQueueIsFullError("error"),
+ PulsarClientException.ProducerQueueIsFullException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.MemoryBufferIsFullError("error"),
+ PulsarClientException.MemoryBufferIsFullException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.TransactionConflictException("error"),
+ PulsarClientException.TransactionConflictException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.TransactionHasOperationFailedException("error"),
+ PulsarClientException.TransactionConflictException.class},
+ {new TransactionCoordinatorClientException.InvalidTxnStatusException("error"),
+ PulsarClientException.TransactionConflictException.class},
+ {new TransactionCoordinatorClientException.TransactionNotFoundException("error"),
+ PulsarClientException.TransactionConflictException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.CryptoException("error"),
+ PulsarClientException.CryptoException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.InvalidConfigurationException("error"),
+ PulsarClientException.InvalidConfigurationException.class},
+ {new org.apache.pulsar.client.api.PulsarClientException.InvalidServiceURL("error", null),
+ PulsarClientException.InvalidServiceURLException.class},
+ };
+ }
+
+ @Test(dataProvider = "errorCategories")
+ public void testToV5ExceptionKeepsTheErrorCategory(Exception v4Exception, Class<?> v5Type) {
+ assertConverted(V5ExceptionMappings.toV5Exception(v4Exception), v5Type, v4Exception);
+ assertConverted(V5ExceptionMappings.toV5Exception(new CompletionException(v4Exception)), v5Type,
+ v4Exception);
+ assertConverted(V5ExceptionMappings.toV5Exception(new ExecutionException(v4Exception)), v5Type,
+ v4Exception);
+ }
+
+ /** A v4 exception type that the mappings don't list. */
+ private static class CustomTimeoutException
+ extends org.apache.pulsar.client.api.PulsarClientException.TimeoutException {
+ CustomTimeoutException(String message) {
+ super(message);
+ }
+ }
+
+ @Test
+ public void testToV5ExceptionTypesASubtypeByItsClosestMappedSupertype() {
+ var timeout = new CustomTimeoutException("timed out");
+
+ assertConverted(V5ExceptionMappings.toV5Exception(timeout), PulsarClientException.TimeoutException.class,
+ timeout);
+ }
+
+ @Test
+ public void testToV5ExceptionWrapsOtherFailuresInThePlainException() {
+ var notAllowed = new org.apache.pulsar.client.api.PulsarClientException.NotAllowedException("not allowed");
+
+ assertConverted(V5ExceptionMappings.toV5Exception(new CompletionException(notAllowed)),
+ PulsarClientException.class, notAllowed);
+ }
+
+ @Test
+ public void testToV5ExceptionReturnsV5ExceptionsAsIs() {
+ var closed = new PulsarClientException.AlreadyClosedException("closed");
+
+ assertSame(V5ExceptionMappings.toV5Exception(new CompletionException(closed)), closed);
+ }
+
+ private static void assertConverted(PulsarClientException converted, Class<?> expectedType,
+ Throwable v4Exception) {
+ assertEquals(converted.getClass(), expectedType);
+ assertEquals(converted.getMessage(), v4Exception.getMessage());
+ assertSame(converted.getCause(), v4Exception);
+ }
+}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
index 0015cd7bacc1..120830bf8292 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
@@ -1753,7 +1753,7 @@ public class ClientCnx extends PulsarHandler {
if (cmd.hasError()) {
CompletableFuture<? extends Object> requestFuture = pendingRequests.get(requestId);
if (requestFuture != null && removePendingRequest(requestId, requestFuture)) {
- requestFuture.completeExceptionally(new PulsarClientException(
+ requestFuture.completeExceptionally(getPulsarClientException(cmd.getError(),
"Scalable topic subscribe failed: " + cmd.getError()
+ (cmd.hasMessage() ? " - " + cmd.getMessage() : "")));
}
(pulsar) 07/14: [fix] [broker] Fix typo in broker.conf (#23295)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 73d53f53968f5f6c99b9784b619f3b825b45446c
Author: atomchen <at...@tencent.com>
AuthorDate: Sat Oct 17 10:15:11 2026 +0800
[fix] [broker] Fix typo in broker.conf (#23295)
Co-authored-by: Matteo Merli <mm...@apache.org>
(cherry picked from commit 5228e4eff433dfda36ca218107203d0cb26bbfc7)
---
conf/broker.conf | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/conf/broker.conf b/conf/broker.conf
index b9f268a4dfcd..05b954f85828 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -719,8 +719,8 @@ maxMessageSize=5242880
# Interval between checks to see if topics with compaction policies need to be compacted
brokerServiceCompactionMonitorIntervalInSeconds=60
-# The estimated backlog size is greater than this threshold, compression will be triggered.
-# Using a value of 0, is disabling compression check.
+# The estimated backlog size is greater than this threshold, compaction will be triggered.
+# Using a value of 0, is disabling compaction check.
brokerServiceCompactionThresholdInBytes=0
# Timeout for each read request in the compaction phase one loop.