You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2024/10/22 05:50:39 UTC
(pulsar) branch branch-4.0 updated (8c20e64651b -> 347840463b2)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a change to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
from 8c20e64651b Bump version to 4.0.1-SNAPSHOT
new b0ef03c609a [fix][client] Use dedicated executor for requests in BinaryProtoLookupService (#23378)
new cf337257d42 [fix][client] Fix ConsumerStats.getRateMsgsReceived javadoc (#21618)
new 0630b1332ef [fix][client] Prevent embedding protobuf-java class files in pulsar-client-admin and pulsar-client-all (#23468)
new 9e65b087935 [feat][monitor] Add offloader stats grafana dashboard (#23479)
new 403d138d56c [improve][monitor] Upgrade OTel to 1.41.0 (#23484)
new 72a21e3a9be [fix][test] Prevent OOM in test by not spying invocations in SimpleProducerConsumerTest (#23486)
new 337389ce1ba [fix] [proxy] Fix pattern consumer does not work when using Proxy (#23489)
new 197fb923fc3 [fix][test] Fix running ClusterMetadataSetupTest in IDE (#23492)
new 347840463b2 [fix][broker] Key_Shared subscription: Reject consumers with incompatible policy (#23449)
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
distribution/server/src/assemble/LICENSE.bin.txt | 54 +-
distribution/shell/src/assemble/LICENSE.bin.txt | 6 +-
grafana/dashboards/offloader.json | 599 +++++++++++++++++++++
pom.xml | 6 +-
.../org/apache/pulsar/broker/PulsarService.java | 5 +
.../broker/service/AbstractSubscription.java | 46 ++
.../nonpersistent/NonPersistentSubscription.java | 7 +-
.../service/persistent/PersistentSubscription.java | 7 +-
.../SubscriptionConsumerCompatibilityTest.java | 175 ++++++
.../persistent/PersistentSubscriptionTest.java | 9 -
.../broker/zookeeper/ClusterMetadataSetupTest.java | 6 +-
.../client/api/SimpleProducerConsumerTest.java | 11 -
pulsar-client-admin-shaded/pom.xml | 2 +-
pulsar-client-all/pom.xml | 2 +-
.../apache/pulsar/client/api/ConsumerStats.java | 2 +-
.../client/impl/BinaryProtoLookupService.java | 65 ++-
.../pulsar/client/impl/PulsarClientImpl.java | 38 +-
.../client/impl/BinaryProtoLookupServiceTest.java | 55 +-
.../pulsar/proxy/server/LookupProxyHandler.java | 3 +-
.../org/apache/pulsar/proxy/server/ProxyTest.java | 92 +++-
20 files changed, 1080 insertions(+), 110 deletions(-)
create mode 100644 grafana/dashboards/offloader.json
create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionConsumerCompatibilityTest.java
(pulsar) 03/09: [fix][client] Prevent embedding protobuf-java class files in pulsar-client-admin and pulsar-client-all (#23468)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 0630b1332effc80de249912b9497a4a236a47dd8
Author: 萧易客 <km...@live.com>
AuthorDate: Fri Oct 18 02:07:38 2024 +0800
[fix][client] Prevent embedding protobuf-java class files in pulsar-client-admin and pulsar-client-all (#23468)
(cherry picked from commit 63c6a25718f27528ea0c7da690c479ffab17c455)
---
pulsar-client-admin-shaded/pom.xml | 2 +-
pulsar-client-all/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml
index a3519185442..70754b4ea51 100644
--- a/pulsar-client-admin-shaded/pom.xml
+++ b/pulsar-client-admin-shaded/pom.xml
@@ -119,7 +119,6 @@
<include>org.reactivestreams:reactive-streams</include>
<include>com.typesafe.netty:netty-reactive-streams</include>
<include>org.javassist:javassist</include>
- <include>com.google.protobuf:protobuf-java</include>
<include>com.google.guava:guava</include>
<include>com.google.code.gson:gson</include>
<include>com.google.re2j:re2j</include>
@@ -153,6 +152,7 @@
</includes>
<excludes>
<exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
+ <exclude>com.google.protobuf:protobuf-java</exclude>
</excludes>
</artifactSet>
<filters>
diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
index d3d204b6a51..dbdc72401d3 100644
--- a/pulsar-client-all/pom.xml
+++ b/pulsar-client-all/pom.xml
@@ -159,7 +159,6 @@
<include>org.reactivestreams:reactive-streams</include>
<include>com.typesafe.netty:netty-reactive-streams</include>
<include>org.javassist:javassist</include>
- <include>com.google.protobuf:protobuf-java</include>
<include>com.google.guava:*</include>
<include>org.checkerframework:*</include>
<include>com.google.code.findbugs:*</include>
@@ -210,6 +209,7 @@
</includes>
<excludes>
<exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
+ <exclude>com.google.protobuf:protobuf-java</exclude>
</excludes>
</artifactSet>
<filters>
(pulsar) 07/09: [fix] [proxy] Fix pattern consumer does not work when using Proxy (#23489)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 337389ce1baae01d196d2bc98a11cf95aff9b80e
Author: fengyubiao <yu...@streamnative.io>
AuthorDate: Mon Oct 21 20:32:35 2024 +0800
[fix] [proxy] Fix pattern consumer does not work when using Proxy (#23489)
(cherry picked from commit a0beab029545a4822ec5d3687493aec5efb4a44f)
---
.../pulsar/proxy/server/LookupProxyHandler.java | 3 +-
.../org/apache/pulsar/proxy/server/ProxyTest.java | 92 +++++++++++++++++-----
2 files changed, 73 insertions(+), 22 deletions(-)
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
index 03975e153ac..db10c37786f 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
@@ -341,7 +341,8 @@ public class LookupProxyHandler {
Commands.newError(clientRequestId, getServerError(t), t.getMessage()));
} else {
writeAndFlush(
- Commands.newGetTopicsOfNamespaceResponse(r.getTopics(), r.getTopicsHash(), r.isFiltered(),
+ Commands.newGetTopicsOfNamespaceResponse(r.getNonPartitionedOrPartitionTopics(),
+ r.getTopicsHash(), r.isFiltered(),
r.isChanged(), clientRequestId));
}
});
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java
index 4c0cd49d74f..ee770e6edee 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java
@@ -23,8 +23,11 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static org.mockito.Mockito.doReturn;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
import io.netty.channel.EventLoopGroup;
import io.netty.util.concurrent.DefaultThreadFactory;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@@ -36,6 +39,7 @@ import lombok.EqualsAndHashCode;
import lombok.ToString;
import org.apache.avro.reflect.Nullable;
import org.apache.pulsar.PulsarVersion;
+import org.apache.pulsar.broker.BrokerTestUtil;
import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
import org.apache.pulsar.broker.authentication.AuthenticationService;
import org.apache.pulsar.client.api.Authentication;
@@ -57,6 +61,9 @@ import org.apache.pulsar.common.api.proto.CommandActiveConsumerChange;
import org.apache.pulsar.common.api.proto.ProtocolVersion;
import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.RetentionPolicies;
+import org.apache.pulsar.common.policies.data.TenantInfo;
import org.apache.pulsar.common.policies.data.TenantInfoImpl;
import org.apache.pulsar.common.policies.data.TopicType;
import org.apache.pulsar.common.schema.SchemaInfo;
@@ -68,6 +75,7 @@ import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class ProxyTest extends MockedPulsarServiceBaseTest {
@@ -97,12 +105,19 @@ public class ProxyTest extends MockedPulsarServiceBaseTest {
initializeProxyConfig();
proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService(
- PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication));
+ PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication));
doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore();
doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService)
.createConfigurationMetadataStore();
proxyService.start();
+
+ // create default resources.
+ admin.clusters().createCluster(conf.getClusterName(),
+ ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build());
+ TenantInfo tenantInfo = new TenantInfoImpl(Collections.emptySet(), Collections.singleton(conf.getClusterName()));
+ admin.tenants().createTenant("public", tenantInfo);
+ admin.namespaces().createNamespace("public/default");
}
protected void initializeProxyConfig() throws Exception {
@@ -276,30 +291,65 @@ public class ProxyTest extends MockedPulsarServiceBaseTest {
}
}
- @Test(timeOut = 60_000)
- public void testRegexSubscriptionWithTopicDiscovery() throws Exception {
- @Cleanup
- PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
- String subName = "regex-proxy-test-" + System.currentTimeMillis();
- String regexSubscriptionPattern = "persistent://sample/test/local/regex-topic-.*";
- try (Consumer<byte[]> consumer = client.newConsumer()
+ @DataProvider(name = "topicTypes")
+ public Object[][] topicTypes() {
+ return new Object[][]{
+ {TopicType.PARTITIONED},
+ {TopicType.NON_PARTITIONED}
+ };
+ }
+
+ @Test(timeOut = 60_000, dataProvider = "topicTypes", invocationCount = 100)
+ public void testRegexSubscriptionWithTopicDiscovery(TopicType topicType) throws Exception {
+ final PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
+ final int topics = 10;
+ final String subName = "s1";
+ final String namespace = "public/default";
+ final String topicPrefix = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp");
+ final String regexSubscriptionPattern = topicPrefix + ".*";
+ // Set retention policy to avoid flaky.
+ RetentionPolicies retentionPolicies = new RetentionPolicies(3600, 1024);
+ admin.namespaces().setRetention(namespace, retentionPolicies);
+ final List<String> topicList = new ArrayList<>();
+ // create topics
+ for (int i = 0; i < topics; i++) {
+ String topic = topicPrefix + i;
+ topicList.add(topic);
+ if (TopicType.PARTITIONED.equals(topicType)) {
+ admin.topics().createPartitionedTopic(topic, 2);
+ } else {
+ admin.topics().createNonPartitionedTopic(topic);
+ }
+ }
+ // Create consumer.
+ Consumer<String> consumer = client.newConsumer(Schema.STRING)
.topicsPattern(regexSubscriptionPattern)
.subscriptionName(subName)
.subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
.patternAutoDiscoveryPeriod(10, TimeUnit.MINUTES)
- .subscribe()) {
- final int topics = 10;
- final String topicPrefix = "persistent://sample/test/local/regex-topic-";
- for (int i = 0; i < topics; i++) {
- Producer<byte[]> producer = client.newProducer(Schema.BYTES)
- .topic(topicPrefix + i)
- .create();
- producer.send(("" + i).getBytes(UTF_8));
- producer.close();
- }
- for (int i = 0; i < topics; i++) {
- Message<byte[]> msg = consumer.receive();
- assertEquals(topicPrefix + new String(msg.getValue(), UTF_8), msg.getTopicName());
+ .isAckReceiptEnabled(true)
+ .subscribe();
+ // Pub & Sub -> Verify
+ for (String topic : topicList) {
+ String msgPublished = topic + " -> msg";
+ Producer<String> producer = client.newProducer(Schema.STRING)
+ .topic(topic)
+ .create();
+ producer.send(msgPublished);
+ producer.close();
+ // Verify.
+ Message<String> msg = consumer.receive(10, TimeUnit.SECONDS);
+ assertNotNull(msg);
+ assertEquals(msg.getValue(), msgPublished);
+ consumer.acknowledge(msg);
+ }
+ // cleanup.
+ consumer.close();
+ for (String topic : topicList) {
+ if (TopicType.PARTITIONED.equals(topicType)) {
+ admin.topics().deletePartitionedTopic(topic);
+ } else {
+ admin.topics().delete(topic);
}
}
}
(pulsar) 01/09: [fix][client] Use dedicated executor for requests in BinaryProtoLookupService (#23378)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit b0ef03c609aab9790b9cc97eed767da1c10645f5
Author: Zixuan Liu <no...@gmail.com>
AuthorDate: Tue Oct 15 21:41:26 2024 +0800
[fix][client] Use dedicated executor for requests in BinaryProtoLookupService (#23378)
Signed-off-by: Zixuan Liu <no...@gmail.com>
(cherry picked from commit f98297f3c9c052d7ddd8444bc0ef876ceeb924b1)
---
.../org/apache/pulsar/broker/PulsarService.java | 5 ++
.../client/impl/BinaryProtoLookupService.java | 65 +++++++++++++++++-----
.../pulsar/client/impl/PulsarClientImpl.java | 38 +++++++++++--
.../client/impl/BinaryProtoLookupServiceTest.java | 55 +++++++++++++++++-
4 files changed, 140 insertions(+), 23 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index dcc0e961275..05491d9c281 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -263,6 +263,7 @@ public class PulsarService implements AutoCloseable, ShutdownService {
private final ExecutorProvider brokerClientSharedExternalExecutorProvider;
private final ScheduledExecutorProvider brokerClientSharedScheduledExecutorProvider;
private final Timer brokerClientSharedTimer;
+ private final ExecutorProvider brokerClientSharedLookupExecutorProvider;
private MetricsGenerator metricsGenerator;
private final PulsarBrokerOpenTelemetry openTelemetry;
@@ -388,6 +389,8 @@ public class PulsarService implements AutoCloseable, ShutdownService {
new ScheduledExecutorProvider(1, "broker-client-shared-scheduled-executor");
this.brokerClientSharedTimer =
new HashedWheelTimer(new DefaultThreadFactory("broker-client-shared-timer"), 1, TimeUnit.MILLISECONDS);
+ this.brokerClientSharedLookupExecutorProvider =
+ new ScheduledExecutorProvider(1, "broker-client-shared-lookup-executor");
// here in the constructor we don't have the offloader scheduler yet
this.offloaderStats = LedgerOffloaderStats.create(false, false, null, 0);
@@ -696,6 +699,7 @@ public class PulsarService implements AutoCloseable, ShutdownService {
brokerClientSharedExternalExecutorProvider.shutdownNow();
brokerClientSharedInternalExecutorProvider.shutdownNow();
brokerClientSharedScheduledExecutorProvider.shutdownNow();
+ brokerClientSharedLookupExecutorProvider.shutdownNow();
brokerClientSharedTimer.stop();
monotonicSnapshotClock.close();
@@ -1687,6 +1691,7 @@ public class PulsarService implements AutoCloseable, ShutdownService {
.internalExecutorProvider(brokerClientSharedInternalExecutorProvider)
.externalExecutorProvider(brokerClientSharedExternalExecutorProvider)
.scheduledExecutorProvider(brokerClientSharedScheduledExecutorProvider)
+ .lookupExecutorProvider(brokerClientSharedLookupExecutorProvider)
.build();
}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java
index b45d6e9f6a8..795cdc6d693 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.client.impl;
import static java.lang.String.format;
import static org.apache.pulsar.client.api.PulsarClientException.FailedFeatureCheck.SupportsGetPartitionedMetadataWithoutAutoCreation;
import io.netty.buffer.ByteBuf;
+import io.netty.util.concurrent.DefaultThreadFactory;
import io.opentelemetry.api.common.Attributes;
import java.net.InetSocketAddress;
import java.net.URI;
@@ -29,6 +30,7 @@ import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -58,9 +60,11 @@ public class BinaryProtoLookupService implements LookupService {
private final PulsarClientImpl client;
private final ServiceNameResolver serviceNameResolver;
private final boolean useTls;
- private final ExecutorService executor;
+ private final ExecutorService scheduleExecutor;
private final String listenerName;
private final int maxLookupRedirects;
+ private final ExecutorService lookupPinnedExecutor;
+ private final boolean createdLookupPinnedExecutor;
private final ConcurrentHashMap<Pair<TopicName, Map<String, String>>, CompletableFuture<LookupTopicResult>>
lookupInProgress = new ConcurrentHashMap<>();
@@ -73,23 +77,43 @@ public class BinaryProtoLookupService implements LookupService {
private final LatencyHistogram histoGetSchema;
private final LatencyHistogram histoListTopics;
+ /**
+ * @deprecated use {@link
+ * #BinaryProtoLookupService(PulsarClientImpl, String, String, boolean, ExecutorService, ExecutorService)} instead.
+ */
+ @Deprecated
+ public BinaryProtoLookupService(PulsarClientImpl client,
+ String serviceUrl,
+ boolean useTls,
+ ExecutorService scheduleExecutor)
+ throws PulsarClientException {
+ this(client, serviceUrl, null, useTls, scheduleExecutor);
+ }
+
+ /**
+ * @deprecated use {@link
+ * #BinaryProtoLookupService(PulsarClientImpl, String, String, boolean, ExecutorService, ExecutorService)} instead.
+ */
+ @Deprecated
public BinaryProtoLookupService(PulsarClientImpl client,
String serviceUrl,
+ String listenerName,
boolean useTls,
- ExecutorService executor)
+ ExecutorService scheduleExecutor)
throws PulsarClientException {
- this(client, serviceUrl, null, useTls, executor);
+ this(client, serviceUrl, listenerName, useTls, scheduleExecutor, null);
}
public BinaryProtoLookupService(PulsarClientImpl client,
String serviceUrl,
String listenerName,
boolean useTls,
- ExecutorService executor)
+ ExecutorService scheduleExecutor,
+ ExecutorService lookupPinnedExecutor)
throws PulsarClientException {
this.client = client;
this.useTls = useTls;
- this.executor = executor;
+ this.scheduleExecutor = scheduleExecutor;
this.maxLookupRedirects = client.getConfiguration().getMaxLookupRedirects();
this.serviceNameResolver = new PulsarServiceNameResolver();
this.listenerName = listenerName;
@@ -103,6 +127,15 @@ public class BinaryProtoLookupService implements LookupService {
histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "metadata").build());
histoGetSchema = histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "schema").build());
histoListTopics = histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "list-topics").build());
+
+ if (lookupPinnedExecutor == null) {
+ this.createdLookupPinnedExecutor = true;
+ this.lookupPinnedExecutor =
+ Executors.newSingleThreadExecutor(new DefaultThreadFactory("pulsar-client-binary-proto-lookup"));
+ } else {
+ this.createdLookupPinnedExecutor = false;
+ this.lookupPinnedExecutor = lookupPinnedExecutor;
+ }
}
@Override
@@ -180,7 +213,7 @@ public class BinaryProtoLookupService implements LookupService {
return addressFuture;
}
- client.getCnxPool().getConnection(socketAddress).thenAccept(clientCnx -> {
+ client.getCnxPool().getConnection(socketAddress).thenAcceptAsync(clientCnx -> {
long requestId = client.newRequestId();
ByteBuf request = Commands.newLookup(topicName.toString(), listenerName, authoritative, requestId,
properties);
@@ -247,7 +280,7 @@ public class BinaryProtoLookupService implements LookupService {
}
client.getCnxPool().releaseConnection(clientCnx);
});
- }).exceptionally(connectionException -> {
+ }, lookupPinnedExecutor).exceptionally(connectionException -> {
addressFuture.completeExceptionally(FutureUtil.unwrapCompletionException(connectionException));
return null;
});
@@ -260,7 +293,7 @@ public class BinaryProtoLookupService implements LookupService {
long startTime = System.nanoTime();
CompletableFuture<PartitionedTopicMetadata> partitionFuture = new CompletableFuture<>();
- client.getCnxPool().getConnection(socketAddress).thenAccept(clientCnx -> {
+ client.getCnxPool().getConnection(socketAddress).thenAcceptAsync(clientCnx -> {
boolean finalAutoCreationEnabled = metadataAutoCreationEnabled;
if (!metadataAutoCreationEnabled && !clientCnx.isSupportsGetPartitionedMetadataWithoutAutoCreation()) {
if (useFallbackForNonPIP344Brokers) {
@@ -301,7 +334,7 @@ public class BinaryProtoLookupService implements LookupService {
}
client.getCnxPool().releaseConnection(clientCnx);
});
- }).exceptionally(connectionException -> {
+ }, lookupPinnedExecutor).exceptionally(connectionException -> {
partitionFuture.completeExceptionally(FutureUtil.unwrapCompletionException(connectionException));
return null;
});
@@ -324,7 +357,7 @@ public class BinaryProtoLookupService implements LookupService {
return schemaFuture;
}
InetSocketAddress socketAddress = serviceNameResolver.resolveHost();
- client.getCnxPool().getConnection(socketAddress).thenAccept(clientCnx -> {
+ client.getCnxPool().getConnection(socketAddress).thenAcceptAsync(clientCnx -> {
long requestId = client.newRequestId();
ByteBuf request = Commands.newGetSchema(requestId, topicName.toString(),
Optional.ofNullable(BytesSchemaVersion.of(version)));
@@ -340,7 +373,7 @@ public class BinaryProtoLookupService implements LookupService {
}
client.getCnxPool().releaseConnection(clientCnx);
});
- }).exceptionally(ex -> {
+ }, lookupPinnedExecutor).exceptionally(ex -> {
schemaFuture.completeExceptionally(FutureUtil.unwrapCompletionException(ex));
return null;
});
@@ -385,7 +418,7 @@ public class BinaryProtoLookupService implements LookupService {
String topicsHash) {
long startTime = System.nanoTime();
- client.getCnxPool().getConnection(socketAddress).thenAccept(clientCnx -> {
+ client.getCnxPool().getConnection(socketAddress).thenAcceptAsync(clientCnx -> {
long requestId = client.newRequestId();
ByteBuf request = Commands.newGetTopicsOfNamespaceRequest(
namespace.toString(), requestId, mode, topicsPattern, topicsHash);
@@ -404,7 +437,7 @@ public class BinaryProtoLookupService implements LookupService {
}
client.getCnxPool().releaseConnection(clientCnx);
});
- }).exceptionally((e) -> {
+ }, lookupPinnedExecutor).exceptionally((e) -> {
long nextDelay = Math.min(backoff.next(), remainingTime.get());
if (nextDelay <= 0) {
getTopicsResultFuture.completeExceptionally(
@@ -414,7 +447,7 @@ public class BinaryProtoLookupService implements LookupService {
return null;
}
- ((ScheduledExecutorService) executor).schedule(() -> {
+ ((ScheduledExecutorService) scheduleExecutor).schedule(() -> {
log.warn("[namespace: {}] Could not get connection while getTopicsUnderNamespace -- Will try again in"
+ " {} ms", namespace, nextDelay);
remainingTime.addAndGet(-nextDelay);
@@ -428,7 +461,9 @@ public class BinaryProtoLookupService implements LookupService {
@Override
public void close() throws Exception {
- // no-op
+ if (createdLookupPinnedExecutor && lookupPinnedExecutor != null && !lookupPinnedExecutor.isShutdown()) {
+ lookupPinnedExecutor.shutdown();
+ }
}
public static class LookupDataResult {
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
index e0d4bf35f8a..603844eeb78 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
@@ -113,6 +113,7 @@ public class PulsarClientImpl implements PulsarClient {
private final boolean createdExecutorProviders;
private final boolean createdScheduledProviders;
+ private final boolean createdLookupProviders;
private LookupService lookup;
private Map<String, LookupService> urlLookupMap = new ConcurrentHashMap<>();
private final ConnectionPool cnxPool;
@@ -121,6 +122,7 @@ public class PulsarClientImpl implements PulsarClient {
private boolean needStopTimer;
private final ExecutorProvider externalExecutorProvider;
private final ExecutorProvider internalExecutorProvider;
+ private final ExecutorProvider lookupExecutorProvider;
private final ScheduledExecutorProvider scheduledExecutorProvider;
private final boolean createdEventLoopGroup;
@@ -163,29 +165,39 @@ public class PulsarClientImpl implements PulsarClient {
private TransactionCoordinatorClientImpl tcClient;
public PulsarClientImpl(ClientConfigurationData conf) throws PulsarClientException {
- this(conf, null, null, null, null, null, null);
+ this(conf, null, null, null, null, null, null, null);
}
public PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) throws PulsarClientException {
- this(conf, eventLoopGroup, null, null, null, null, null);
+ this(conf, eventLoopGroup, null, null, null, null, null, null);
}
public PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, ConnectionPool cnxPool)
throws PulsarClientException {
- this(conf, eventLoopGroup, cnxPool, null, null, null, null);
+ this(conf, eventLoopGroup, cnxPool, null, null, null, null, null);
}
public PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, ConnectionPool cnxPool,
Timer timer)
throws PulsarClientException {
- this(conf, eventLoopGroup, cnxPool, timer, null, null, null);
+ this(conf, eventLoopGroup, cnxPool, timer, null, null, null, null);
+ }
+
+ public PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, ConnectionPool connectionPool,
+ Timer timer, ExecutorProvider externalExecutorProvider,
+ ExecutorProvider internalExecutorProvider,
+ ScheduledExecutorProvider scheduledExecutorProvider)
+ throws PulsarClientException {
+ this(conf, eventLoopGroup, connectionPool, timer, externalExecutorProvider, internalExecutorProvider,
+ scheduledExecutorProvider, null);
}
@Builder(builderClassName = "PulsarClientImplBuilder")
private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, ConnectionPool connectionPool,
Timer timer, ExecutorProvider externalExecutorProvider,
ExecutorProvider internalExecutorProvider,
- ScheduledExecutorProvider scheduledExecutorProvider) throws PulsarClientException {
+ ScheduledExecutorProvider scheduledExecutorProvider,
+ ExecutorProvider lookupExecutorProvider) throws PulsarClientException {
EventLoopGroup eventLoopGroupReference = null;
ConnectionPool connectionPoolReference = null;
@@ -198,6 +210,7 @@ public class PulsarClientImpl implements PulsarClient {
}
this.createdExecutorProviders = externalExecutorProvider == null;
this.createdScheduledProviders = scheduledExecutorProvider == null;
+ this.createdLookupProviders = lookupExecutorProvider == null;
eventLoopGroupReference = eventLoopGroup != null ? eventLoopGroup : getEventLoopGroup(conf);
this.eventLoopGroup = eventLoopGroupReference;
if (conf == null || isBlank(conf.getServiceUrl())) {
@@ -218,11 +231,14 @@ public class PulsarClientImpl implements PulsarClient {
new ExecutorProvider(conf.getNumListenerThreads(), "pulsar-external-listener");
this.internalExecutorProvider = internalExecutorProvider != null ? internalExecutorProvider :
new ExecutorProvider(conf.getNumIoThreads(), "pulsar-client-internal");
+ this.lookupExecutorProvider = lookupExecutorProvider != null ? lookupExecutorProvider :
+ new ExecutorProvider(1, "pulsar-client-lookup");
if (conf.getServiceUrl().startsWith("http")) {
lookup = new HttpLookupService(instrumentProvider, conf, this.eventLoopGroup);
} else {
lookup = new BinaryProtoLookupService(this, conf.getServiceUrl(), conf.getListenerName(),
- conf.isUseTls(), this.scheduledExecutorProvider.getExecutor());
+ conf.isUseTls(), this.scheduledExecutorProvider.getExecutor(),
+ this.lookupExecutorProvider.getExecutor());
}
if (timer == null) {
this.timer = new HashedWheelTimer(getThreadFactory("pulsar-timer"), 1, TimeUnit.MILLISECONDS);
@@ -976,6 +992,16 @@ public class PulsarClientImpl implements PulsarClient {
pulsarClientException = PulsarClientException.unwrap(t);
}
}
+
+ if (createdLookupProviders && lookupExecutorProvider != null && !lookupExecutorProvider.isShutdown()) {
+ try {
+ lookupExecutorProvider.shutdownNow();
+ } catch (Throwable t) {
+ log.warn("Failed to shutdown lookupExecutorProvider", t);
+ pulsarClientException = PulsarClientException.unwrap(t);
+ }
+ }
+
if (pulsarClientException != null) {
throw pulsarClientException;
}
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BinaryProtoLookupServiceTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BinaryProtoLookupServiceTest.java
index f691215b04e..11e00eefcfd 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BinaryProtoLookupServiceTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BinaryProtoLookupServiceTest.java
@@ -25,25 +25,41 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import io.netty.buffer.ByteBuf;
+import io.netty.util.concurrent.DefaultThreadFactory;
import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.pulsar.client.api.PulsarClientException.LookupException;
import org.apache.pulsar.client.impl.BinaryProtoLookupService.LookupDataResult;
import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
import org.apache.pulsar.client.impl.metrics.InstrumentProvider;
+import org.apache.pulsar.common.api.proto.BaseCommand;
+import org.apache.pulsar.common.api.proto.BaseCommand.Type;
import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.protocol.Commands;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class BinaryProtoLookupServiceTest {
private BinaryProtoLookupService lookup;
private TopicName topicName;
+ private ExecutorService internalExecutor;
+
+ @AfterMethod
+ public void cleanup() throws Exception {
+ internalExecutor.shutdown();
+ lookup.close();
+ }
@BeforeMethod
public void setup() throws Exception {
@@ -72,9 +88,13 @@ public class BinaryProtoLookupServiceTest {
doReturn(1L).when(client).newRequestId();
ClientConfigurationData data = new ClientConfigurationData();
doReturn(data).when(client).getConfiguration();
+ internalExecutor =
+ Executors.newSingleThreadExecutor(new DefaultThreadFactory("pulsar-client-test-internal-executor"));
+ doReturn(internalExecutor).when(client).getInternalExecutorService();
+
+ lookup = spy(new BinaryProtoLookupService(client, "pulsar://localhost:6650", null, false,
+ mock(ExecutorService.class), internalExecutor));
- lookup = spy(
- new BinaryProtoLookupService(client, "pulsar://localhost:6650", false, mock(ExecutorService.class)));
topicName = TopicName.get("persistent://tenant1/ns1/t1");
}
@@ -118,6 +138,37 @@ public class BinaryProtoLookupServiceTest {
}
}
+ @Test
+ public void testCommandUnChangedInDifferentThread() throws Exception {
+ BaseCommand successCommand = Commands.newSuccessCommand(10000);
+ lookup.getBroker(topicName).get();
+ assertEquals(successCommand.getType(), Type.SUCCESS);
+ lookup.getPartitionedTopicMetadata(topicName, true, true).get();
+ assertEquals(successCommand.getType(), Type.SUCCESS);
+ }
+
+ @Test
+ public void testCommandChangedInSameThread() throws Exception {
+ AtomicReference<BaseCommand> successCommand = new AtomicReference<>();
+ internalExecutor.execute(() -> successCommand.set(Commands.newSuccessCommand(10000)));
+ Awaitility.await().untilAsserted(() -> {
+ BaseCommand baseCommand = successCommand.get();
+ assertNotNull(baseCommand);
+ assertEquals(baseCommand.getType(), Type.SUCCESS);
+ });
+ lookup.getBroker(topicName).get();
+ assertEquals(successCommand.get().getType(), Type.LOOKUP);
+
+ internalExecutor.execute(() -> successCommand.set(Commands.newSuccessCommand(10000)));
+ Awaitility.await().untilAsserted(() -> {
+ BaseCommand baseCommand = successCommand.get();
+ assertNotNull(baseCommand);
+ assertEquals(baseCommand.getType(), Type.SUCCESS);
+ });
+ lookup.getPartitionedTopicMetadata(topicName, true, true).get();
+ assertEquals(successCommand.get().getType(), Type.PARTITIONED_METADATA);
+ }
+
private static LookupDataResult createLookupDataResult(String brokerUrl, boolean redirect) throws Exception {
LookupDataResult lookupResult = new LookupDataResult(-1);
(pulsar) 09/09: [fix][broker] Key_Shared subscription: Reject consumers with incompatible policy (#23449)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 347840463b2122351c9e6a3446129e5483b39621
Author: Philipp Dolif <52...@users.noreply.github.com>
AuthorDate: Mon Oct 21 23:32:31 2024 +0200
[fix][broker] Key_Shared subscription: Reject consumers with incompatible policy (#23449)
(cherry picked from commit 1344167328c31ea39054ec2a6019f003fb8bab50)
---
.../broker/service/AbstractSubscription.java | 46 ++++++
.../nonpersistent/NonPersistentSubscription.java | 7 +-
.../service/persistent/PersistentSubscription.java | 7 +-
.../SubscriptionConsumerCompatibilityTest.java | 175 +++++++++++++++++++++
.../persistent/PersistentSubscriptionTest.java | 9 --
5 files changed, 229 insertions(+), 15 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java
index 10e6b796097..ae108646532 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractSubscription.java
@@ -19,8 +19,12 @@
package org.apache.pulsar.broker.service;
import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.LongAdder;
import java.util.function.ToLongFunction;
+import org.apache.pulsar.common.api.proto.CommandSubscribe;
+import org.apache.pulsar.common.api.proto.KeySharedMeta;
+import org.apache.pulsar.common.util.FutureUtil;
public abstract class AbstractSubscription implements Subscription {
protected final LongAdder bytesOutFromRemovedConsumers = new LongAdder();
@@ -39,4 +43,46 @@ public abstract class AbstractSubscription implements Subscription {
.map(dispatcher -> dispatcher.getConsumers().stream().mapToLong(toCounter).sum())
.orElse(0L);
}
+
+ /**
+ * Checks if the given consumer is compatible with the given dispatcher. They are incompatible if
+ * <ul>
+ * <li>the subscription type of the consumer differs from the subscription type of the dispatcher or</li>
+ * <li>if both the consumer and dispatcher are of {@link CommandSubscribe.SubType#Key_Shared} type but
+ * their policies differ ({@link org.apache.pulsar.common.api.proto.KeySharedMode#AUTO_SPLIT}/
+ * {@link org.apache.pulsar.common.api.proto.KeySharedMode#STICKY} or allowOutOfOrderDelivery true/false).</li>
+ * </ul>
+ * @param dispatcher The dispatcher of the subscription
+ * @param consumer New consumer to be added to the subscription
+ * @return Optional containing failed future with {@link BrokerServiceException.SubscriptionBusyException} if
+ * consumer and dispatcher are incompatible or empty optional otherwise.
+ */
+ protected Optional<CompletableFuture<Void>> checkForConsumerCompatibilityErrorWithDispatcher(Dispatcher dispatcher,
+ Consumer consumer) {
+ if (consumer.subType() != dispatcher.getType()) {
+ return Optional.of(FutureUtil.failedFuture(new BrokerServiceException.SubscriptionBusyException(
+ String.format("Subscription is of different type. Active subscription type of '%s' is different "
+ + "than the connecting consumer's type '%s'.",
+ dispatcher.getType(), consumer.subType()))));
+ } else if (dispatcher.getType() == CommandSubscribe.SubType.Key_Shared) {
+ KeySharedMeta dispatcherKsm = dispatcher.getConsumers().get(0).getKeySharedMeta();
+ KeySharedMeta consumerKsm = consumer.getKeySharedMeta();
+ if (dispatcherKsm.getKeySharedMode() != consumerKsm.getKeySharedMode()) {
+ return Optional.of(FutureUtil.failedFuture(new BrokerServiceException.SubscriptionBusyException(
+ String.format("Subscription is of different type. Active subscription key_shared mode of '%s' "
+ + "is different than the connecting consumer's key_shared mode '%s'.",
+ dispatcherKsm.getKeySharedMode(), consumerKsm.getKeySharedMode()))));
+ }
+ if (dispatcherKsm.isAllowOutOfOrderDelivery() != consumerKsm.isAllowOutOfOrderDelivery()) {
+ return Optional.of(FutureUtil.failedFuture(new BrokerServiceException.SubscriptionBusyException(
+ String.format("Subscription is of different type. %s",
+ dispatcherKsm.isAllowOutOfOrderDelivery()
+ ? "Active subscription allows out of order delivery while the connecting "
+ + "consumer does not allow it." :
+ "Active subscription does not allow out of order delivery while the connecting "
+ + "consumer allows it."))));
+ }
+ }
+ return Optional.empty();
+ }
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java
index e92eef5cb7b..55639868e0a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java
@@ -34,7 +34,6 @@ import org.apache.pulsar.broker.service.AbstractSubscription;
import org.apache.pulsar.broker.service.AnalyzeBacklogResult;
import org.apache.pulsar.broker.service.BrokerServiceException;
import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException;
-import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException;
import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionFencedException;
import org.apache.pulsar.broker.service.Consumer;
import org.apache.pulsar.broker.service.Dispatcher;
@@ -159,8 +158,10 @@ public class NonPersistentSubscription extends AbstractSubscription {
});
}
} else {
- if (consumer.subType() != dispatcher.getType()) {
- return FutureUtil.failedFuture(new SubscriptionBusyException("Subscription is of different type"));
+ Optional<CompletableFuture<Void>> compatibilityError =
+ checkForConsumerCompatibilityErrorWithDispatcher(dispatcher, consumer);
+ if (compatibilityError.isPresent()) {
+ return compatibilityError.get();
}
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index df1c23cbbcb..0096f398ada 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -307,9 +307,10 @@ public class PersistentSubscription extends AbstractSubscription {
});
}
} else {
- if (consumer.subType() != dispatcher.getType()) {
- return FutureUtil.failedFuture(
- new SubscriptionBusyException("Subscription is of different type"));
+ Optional<CompletableFuture<Void>> compatibilityError =
+ checkForConsumerCompatibilityErrorWithDispatcher(dispatcher, consumer);
+ if (compatibilityError.isPresent()) {
+ return compatibilityError.get();
}
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionConsumerCompatibilityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionConsumerCompatibilityTest.java
new file mode 100644
index 00000000000..dc1d0170f9d
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionConsumerCompatibilityTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.mledger.ManagedLedger;
+import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
+import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
+import org.apache.pulsar.broker.BrokerTestUtil;
+import org.apache.pulsar.broker.service.nonpersistent.NonPersistentSubscription;
+import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.testcontext.PulsarTestContext;
+import org.apache.pulsar.common.api.proto.CommandSubscribe;
+import org.apache.pulsar.common.api.proto.KeySharedMeta;
+import org.apache.pulsar.common.api.proto.KeySharedMode;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class SubscriptionConsumerCompatibilityTest {
+
+ private PulsarTestContext pulsarTestContext;
+ private ManagedLedger ledgerMock;
+ private ManagedCursorImpl cursorMock;
+ private final String successTopicName = "persistent://prop/use/ns-abc/successTopic";
+ private final String subName = "subscriptionName";
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ pulsarTestContext = PulsarTestContext.builderForNonStartableContext().build();
+
+ ledgerMock = mock(ManagedLedgerImpl.class);
+ ManagedLedgerConfig managedLedgerConfigMock = mock(ManagedLedgerConfig.class);
+ doReturn(managedLedgerConfigMock).when(ledgerMock).getConfig();
+
+ cursorMock = mock(ManagedCursorImpl.class);
+ doReturn("mockCursor").when(cursorMock).getName();
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void teardown() throws Exception {
+ if (pulsarTestContext != null) {
+ pulsarTestContext.close();
+ pulsarTestContext = null;
+ }
+ }
+
+ @DataProvider(name = "incompatibleKeySharedPolicies")
+ public Object[][] incompatibleKeySharedPolicies() {
+ KeySharedMeta ksmSticky = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY);
+ ksmSticky.addHashRange().setStart(0).setEnd(2);
+
+ KeySharedMeta ksmStickyAllowOutOfOrder = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY)
+ .setAllowOutOfOrderDelivery(true);
+ ksmStickyAllowOutOfOrder.addHashRange().setStart(3).setEnd(5);
+
+ KeySharedMeta ksmAutoSplit = new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT);
+ KeySharedMeta ksmAutoSplitAllowOutOfOrder = new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT)
+ .setAllowOutOfOrderDelivery(true);
+
+ String errorMessagePrefix = "Subscription is of different type. ";
+ String errorMessageSubscriptionModeSticky = errorMessagePrefix + "Active subscription key_shared mode of "
+ + "'STICKY' is different than the connecting consumer's key_shared mode 'AUTO_SPLIT'.";
+ String errorMessageSubscriptionModeAutoSplit = errorMessagePrefix + "Active subscription key_shared mode of "
+ + "'AUTO_SPLIT' is different than the connecting consumer's key_shared mode 'STICKY'.";
+ String errorMessageOutOfOrderNotAllowed = errorMessagePrefix + "Active subscription does not allow out of "
+ + "order delivery while the connecting consumer allows it.";
+ String errorMessageOutOfOrderAllowed = errorMessagePrefix + "Active subscription allows out of order delivery "
+ + "while the connecting consumer does not allow it.";
+
+ return new Object[][] {
+ { ksmAutoSplit, ksmSticky, errorMessageSubscriptionModeAutoSplit },
+ { ksmAutoSplit, ksmStickyAllowOutOfOrder, errorMessageSubscriptionModeAutoSplit },
+ { ksmAutoSplit, ksmAutoSplitAllowOutOfOrder, errorMessageOutOfOrderNotAllowed },
+
+ { ksmAutoSplitAllowOutOfOrder, ksmSticky, errorMessageSubscriptionModeAutoSplit },
+ { ksmAutoSplitAllowOutOfOrder, ksmStickyAllowOutOfOrder, errorMessageSubscriptionModeAutoSplit },
+ { ksmAutoSplitAllowOutOfOrder, ksmAutoSplit, errorMessageOutOfOrderAllowed },
+
+ { ksmSticky, ksmStickyAllowOutOfOrder, errorMessageOutOfOrderNotAllowed },
+ { ksmSticky, ksmAutoSplit, errorMessageSubscriptionModeSticky },
+ { ksmSticky, ksmAutoSplitAllowOutOfOrder, errorMessageSubscriptionModeSticky },
+
+ { ksmStickyAllowOutOfOrder, ksmSticky, errorMessageOutOfOrderAllowed },
+ { ksmStickyAllowOutOfOrder, ksmAutoSplit, errorMessageSubscriptionModeSticky },
+ { ksmStickyAllowOutOfOrder, ksmAutoSplitAllowOutOfOrder, errorMessageSubscriptionModeSticky }
+ };
+ }
+
+ @Test(dataProvider = "incompatibleKeySharedPolicies")
+ public void testIncompatibleKeySharedPoliciesNotAllowedInNonPersistentSub(KeySharedMeta consumer1Ksm, KeySharedMeta consumer2Ksm,
+ String expectedErrorMessage) throws Exception {
+ NonPersistentTopic topic = new NonPersistentTopic(successTopicName, pulsarTestContext.getBrokerService());
+ NonPersistentSubscription sub = new NonPersistentSubscription(topic, subName, Map.of());
+
+ // two consumers with incompatible key_shared policies
+ Consumer keySharedConsumerMock1 = createKeySharedMockConsumer("consumer-1", consumer1Ksm);
+ Consumer keySharedConsumerMock2 = createKeySharedMockConsumer("consumer-2", consumer2Ksm);
+
+ // first consumer defines key_shared mode of subscription and whether out of order delivery is allowed
+ sub.addConsumer(keySharedConsumerMock1).get(5, TimeUnit.SECONDS);
+
+ try {
+ // add second consumer with incompatible key_shared policy
+ sub.addConsumer(keySharedConsumerMock2).get(5, TimeUnit.SECONDS);
+ fail(BrokerServiceException.SubscriptionBusyException.class.getSimpleName() + " not thrown");
+ } catch (Exception e) {
+ // subscription throws exception when consumer with incompatible key_shared policy is added
+ Throwable cause = e.getCause();
+ assertTrue(cause instanceof BrokerServiceException.SubscriptionBusyException);
+ assertEquals(cause.getMessage(), expectedErrorMessage);
+ }
+ }
+
+ @Test(dataProvider = "incompatibleKeySharedPolicies")
+ public void testIncompatibleKeySharedPoliciesNotAllowedInPersistentSub(KeySharedMeta consumer1Ksm, KeySharedMeta consumer2Ksm,
+ String expectedErrorMessage) throws Exception {
+ PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, pulsarTestContext.getBrokerService());
+ PersistentSubscription sub = new PersistentSubscription(topic, subName, cursorMock, false);
+
+ // two consumers with incompatible key_shared policies
+ Consumer keySharedConsumerMock1 = createKeySharedMockConsumer("consumer-1", consumer1Ksm);
+ Consumer keySharedConsumerMock2 = createKeySharedMockConsumer("consumer-2", consumer2Ksm);
+
+ // first consumer defines key_shared mode of subscription and whether out of order delivery is allowed
+ sub.addConsumer(keySharedConsumerMock1).get(5, TimeUnit.SECONDS);
+
+ try {
+ // add second consumer with incompatible key_shared policy
+ sub.addConsumer(keySharedConsumerMock2).get(5, TimeUnit.SECONDS);
+ fail(BrokerServiceException.SubscriptionBusyException.class.getSimpleName() + " not thrown");
+ } catch (Exception e) {
+ // subscription throws exception when consumer with incompatible key_shared policy is added
+ Throwable cause = e.getCause();
+ assertTrue(cause instanceof BrokerServiceException.SubscriptionBusyException);
+ assertEquals(cause.getMessage(), expectedErrorMessage);
+ }
+ }
+
+ protected Consumer createKeySharedMockConsumer(String name, KeySharedMeta ksm) {
+ Consumer consumer = BrokerTestUtil.createMockConsumer(name);
+ doReturn(CommandSubscribe.SubType.Key_Shared).when(consumer).subType();
+ doReturn(ksm).when(consumer).getKeySharedMeta();
+ doReturn(mock(PendingAcksMap.class)).when(consumer).getPendingAcks();
+ return consumer;
+ }
+
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
index 309cd7b55ac..9a46a2919d1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
@@ -26,7 +26,6 @@ import static org.mockito.Mockito.mock;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
-import io.netty.channel.EventLoopGroup;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collections;
@@ -35,7 +34,6 @@ import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
-import org.apache.bookkeeper.common.util.OrderedExecutor;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.ManagedLedger;
import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
@@ -60,8 +58,6 @@ import org.apache.pulsar.common.api.proto.TxnAction;
import org.apache.pulsar.common.policies.data.Policies;
import org.apache.pulsar.transaction.common.exception.TransactionConflictException;
import org.awaitility.Awaitility;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -83,11 +79,6 @@ public class PersistentSubscriptionTest {
final TxnID txnID1 = new TxnID(1,1);
final TxnID txnID2 = new TxnID(1,2);
- private static final Logger log = LoggerFactory.getLogger(PersistentTopicTest.class);
-
- private OrderedExecutor executor;
- private EventLoopGroup eventLoopGroup;
-
@BeforeMethod
public void setup() throws Exception {
pulsarTestContext = PulsarTestContext.builderForNonStartableContext()
(pulsar) 05/09: [improve][monitor] Upgrade OTel to 1.41.0 (#23484)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 403d138d56c297bc5edc3d2a7db8821bc3cd9355
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Oct 18 21:24:25 2024 +0300
[improve][monitor] Upgrade OTel to 1.41.0 (#23484)
(cherry picked from commit 2ef8ef0a711daa3ad25f6ebbb2bf2716d353b29b)
---
distribution/server/src/assemble/LICENSE.bin.txt | 54 ++++++++++++------------
distribution/shell/src/assemble/LICENSE.bin.txt | 6 +--
pom.xml | 6 +--
3 files changed, 33 insertions(+), 33 deletions(-)
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 271f6dc6ceb..133182b9616 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -338,12 +338,12 @@ The Apache Software License, Version 2.0
- io.prometheus-simpleclient_tracer_otel-0.16.0.jar
- io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar
* Prometheus exporter
- - io.prometheus-prometheus-metrics-config-1.2.1.jar
- - io.prometheus-prometheus-metrics-exporter-common-1.2.1.jar
- - io.prometheus-prometheus-metrics-exporter-httpserver-1.2.1.jar
- - io.prometheus-prometheus-metrics-exposition-formats-1.2.1.jar
- - io.prometheus-prometheus-metrics-model-1.2.1.jar
- - io.prometheus-prometheus-metrics-shaded-protobuf-1.2.1.jar
+ - io.prometheus-prometheus-metrics-config-1.3.1.jar
+ - io.prometheus-prometheus-metrics-exporter-common-1.3.1.jar
+ - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.1.jar
+ - io.prometheus-prometheus-metrics-exposition-formats-1.3.1.jar
+ - io.prometheus-prometheus-metrics-model-1.3.1.jar
+ - io.prometheus-prometheus-metrics-shaded-protobuf-1.3.1.jar
* Jakarta Bean Validation API
- jakarta.validation-jakarta.validation-api-2.0.2.jar
- javax.validation-validation-api-1.1.0.Final.jar
@@ -516,27 +516,27 @@ The Apache Software License, Version 2.0
* RoaringBitmap
- org.roaringbitmap-RoaringBitmap-1.2.0.jar
* OpenTelemetry
- - io.opentelemetry-opentelemetry-api-1.38.0.jar
- - io.opentelemetry-opentelemetry-api-incubator-1.38.0-alpha.jar
- - io.opentelemetry-opentelemetry-context-1.38.0.jar
- - io.opentelemetry-opentelemetry-exporter-common-1.38.0.jar
- - io.opentelemetry-opentelemetry-exporter-otlp-1.38.0.jar
- - io.opentelemetry-opentelemetry-exporter-otlp-common-1.38.0.jar
- - io.opentelemetry-opentelemetry-exporter-prometheus-1.38.0-alpha.jar
- - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-common-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-logs-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-metrics-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-trace-1.38.0.jar
- - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.3.jar
- - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.3-alpha.jar
- - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.3-alpha.jar
- - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.3-alpha.jar
- - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.3-alpha.jar
- - io.opentelemetry.semconv-opentelemetry-semconv-1.25.0-alpha.jar
+ - io.opentelemetry-opentelemetry-api-1.41.0.jar
+ - io.opentelemetry-opentelemetry-api-incubator-1.41.0-alpha.jar
+ - io.opentelemetry-opentelemetry-context-1.41.0.jar
+ - io.opentelemetry-opentelemetry-exporter-common-1.41.0.jar
+ - io.opentelemetry-opentelemetry-exporter-otlp-1.41.0.jar
+ - io.opentelemetry-opentelemetry-exporter-otlp-common-1.41.0.jar
+ - io.opentelemetry-opentelemetry-exporter-prometheus-1.41.0-alpha.jar
+ - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.41.0.jar
+ - io.opentelemetry-opentelemetry-sdk-1.41.0.jar
+ - io.opentelemetry-opentelemetry-sdk-common-1.41.0.jar
+ - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.41.0.jar
+ - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.41.0.jar
+ - io.opentelemetry-opentelemetry-sdk-logs-1.41.0.jar
+ - io.opentelemetry-opentelemetry-sdk-metrics-1.41.0.jar
+ - io.opentelemetry-opentelemetry-sdk-trace-1.41.0.jar
+ - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.6.jar
+ - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.6-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.6-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.6-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.6-alpha.jar
+ - io.opentelemetry.semconv-opentelemetry-semconv-1.28.0-alpha.jar
* Spotify completable-futures
- com.spotify-completable-futures-0.3.6.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 15b2a918b9e..9befa2a803d 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -388,9 +388,9 @@ The Apache Software License, Version 2.0
- log4j-slf4j2-impl-2.23.1.jar
- log4j-web-2.23.1.jar
* OpenTelemetry
- - opentelemetry-api-1.38.0.jar
- - opentelemetry-api-incubator-1.38.0-alpha.jar
- - opentelemetry-context-1.38.0.jar
+ - opentelemetry-api-1.41.0.jar
+ - opentelemetry-api-incubator-1.41.0-alpha.jar
+ - opentelemetry-context-1.41.0.jar
* BookKeeper
- bookkeeper-common-allocator-4.17.1.jar
diff --git a/pom.xml b/pom.xml
index 16b6950b2ff..2e04f4f87b3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -258,11 +258,11 @@ flexible messaging model and an intuitive client API.</description>
<disruptor.version>3.4.3</disruptor.version>
<zstd-jni.version>1.5.2-3</zstd-jni.version>
<netty-reactive-streams.version>2.0.6</netty-reactive-streams.version>
- <opentelemetry.version>1.38.0</opentelemetry.version>
+ <opentelemetry.version>1.41.0</opentelemetry.version>
<opentelemetry.alpha.version>${opentelemetry.version}-alpha</opentelemetry.alpha.version>
- <opentelemetry.instrumentation.version>1.33.3</opentelemetry.instrumentation.version>
+ <opentelemetry.instrumentation.version>1.33.6</opentelemetry.instrumentation.version>
<opentelemetry.instrumentation.alpha.version>${opentelemetry.instrumentation.version}-alpha</opentelemetry.instrumentation.alpha.version>
- <opentelemetry.semconv.version>1.25.0-alpha</opentelemetry.semconv.version>
+ <opentelemetry.semconv.version>1.28.0-alpha</opentelemetry.semconv.version>
<picocli.version>4.7.5</picocli.version>
<re2j.version>1.7</re2j.version>
<completable-futures.version>0.3.6</completable-futures.version>
(pulsar) 08/09: [fix][test] Fix running ClusterMetadataSetupTest in IDE (#23492)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 197fb923fc3741e392527fd86774667977e2c94b
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Mon Oct 21 23:36:57 2024 +0300
[fix][test] Fix running ClusterMetadataSetupTest in IDE (#23492)
(cherry picked from commit 06471dd3054ed47262b09fe0b79263e5abb55580)
---
.../apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java
index 0c402a83e42..6a7d54dbf7c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java
@@ -506,11 +506,9 @@ public class ClusterMetadataSetupTest {
private ZooKeeperServer zks;
private NIOServerCnxnFactory serverFactory;
private final int zkPort;
- private final String hostPort;
public ZookeeperServerTest(int zkPort) throws IOException {
this.zkPort = zkPort;
- this.hostPort = "127.0.0.1:" + zkPort;
this.zkTmpDir = File.createTempFile("zookeeper", "test");
log.info("**** Start GZK on {} ****", zkTmpDir);
if (!zkTmpDir.delete() || !zkTmpDir.mkdir()) {
@@ -520,15 +518,17 @@ public class ClusterMetadataSetupTest {
public void start() throws IOException {
try {
+ System.setProperty("zookeeper.4lw.commands.whitelist", "*");
zks = new ZooKeeperServer(zkTmpDir, zkTmpDir, ZooKeeperServer.DEFAULT_TICK_TIME);
zks.setMaxSessionTimeout(20000);
serverFactory = new NIOServerCnxnFactory();
- serverFactory.configure(new InetSocketAddress(zkPort), 1000);
+ serverFactory.configure(new InetSocketAddress("127.0.0.1", zkPort), 1000);
serverFactory.startup(zks);
} catch (Exception e) {
log.error("Exception while instantiating ZooKeeper", e);
}
+ String hostPort = "127.0.0.1:" + serverFactory.getLocalPort();
LocalBookkeeperEnsemble.waitForServerUp(hostPort, 30000);
log.info("ZooKeeper started at {}", hostPort);
}
(pulsar) 02/09: [fix][client] Fix ConsumerStats.getRateMsgsReceived javadoc (#21618)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit cf337257d42b0b92a044fab1a51e356ed646c0ac
Author: ahscuml <ah...@163.com>
AuthorDate: Thu Oct 17 15:21:25 2024 +0800
[fix][client] Fix ConsumerStats.getRateMsgsReceived javadoc (#21618)
Co-authored-by: leoyxli <le...@tencent.com>
(cherry picked from commit 842f0ceb26217cdf9c110a47d2de45c0cf6ddff4)
---
.../src/main/java/org/apache/pulsar/client/api/ConsumerStats.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerStats.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerStats.java
index e488aa81151..71234f3809c 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerStats.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerStats.java
@@ -49,7 +49,7 @@ public interface ConsumerStats extends Serializable {
long getNumBytesReceived();
/**
- * @return Rate of bytes per second received in the last interval
+ * @return Rate of messages per second received in the last interval
*/
double getRateMsgsReceived();
(pulsar) 04/09: [feat][monitor] Add offloader stats grafana dashboard (#23479)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 9e65b0879357b4ad53ace3a2af634826115cab64
Author: 道君 <da...@apache.org>
AuthorDate: Fri Oct 18 14:47:50 2024 +0800
[feat][monitor] Add offloader stats grafana dashboard (#23479)
(cherry picked from commit 6ebca5e4a4b7aa41d2eac3419762bd2f2a57b793)
---
grafana/dashboards/offloader.json | 599 ++++++++++++++++++++++++++++++++++++++
1 file changed, 599 insertions(+)
diff --git a/grafana/dashboards/offloader.json b/grafana/dashboards/offloader.json
new file mode 100644
index 00000000000..12cf21cfcc3
--- /dev/null
+++ b/grafana/dashboards/offloader.json
@@ -0,0 +1,599 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 2,
+ "iteration": 1644809446689,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "id": 14,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_writeError{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_writeError",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "id": 18,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.3.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_readLedgerLatencyBuckets{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_readLedgerLatencyBuckets",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "id": 6,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_readOffloadError{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_readOffloadError",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ },
+ "id": 16,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_writeRate{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_writeRate",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 16
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_offloadError{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_offloadError",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 16
+ },
+ "id": 8,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_readOffloadRate{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_readOffloadRate",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 34,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "standalone",
+ "value": "standalone"
+ },
+ "definition": "{cluster=~\".+\"}",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [],
+ "query": {
+ "query": "{cluster=~\".+\"}",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "/.*[^_]cluster=\\\"([^\\\"]+)\\\".*/",
+ "skipUrlSync": false,
+ "sort": 1,
+ "type": "query"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "public/functions",
+ "value": "public/functions"
+ },
+ "definition": "pulsar_rate_in{namespace=~\".+\", cluster=~\"$cluster\"}",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [],
+ "query": {
+ "query": "pulsar_rate_in{namespace=~\".+\", cluster=~\"$cluster\"}",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "/.*namespace=\\\"([^\\\"]+)\\\".*/",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "Pulsar",
+ "value": "Pulsar"
+ },
+ "definition": "pulsar_storage_size{job=~\".+\"}",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Job",
+ "multi": false,
+ "name": "job",
+ "options": [],
+ "query": {
+ "query": "pulsar_storage_size{job=~\".+\"}",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "/.*[^_]job=\\\"([^\\\"]+)\\\".*/",
+ "skipUrlSync": false,
+ "sort": 1,
+ "type": "query"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Pulsar Offloader",
+ "uid": "_qAaz517k",
+ "version": 25,
+ "weekStart": ""
+}
\ No newline at end of file
(pulsar) 06/09: [fix][test] Prevent OOM in test by not spying invocations in SimpleProducerConsumerTest (#23486)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 72a21e3a9bebd8a65706073b1fc9472b3ac8e8aa
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Mon Oct 21 06:08:46 2024 +0300
[fix][test] Prevent OOM in test by not spying invocations in SimpleProducerConsumerTest (#23486)
(cherry picked from commit 7367f1c6553c83d7d335977b86ed38494c9485b7)
---
.../apache/pulsar/client/api/SimpleProducerConsumerTest.java | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index e76c3d8fb84..a867dc0cd35 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -91,13 +91,10 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.PulsarVersion;
import org.apache.pulsar.broker.BrokerTestUtil;
import org.apache.pulsar.broker.PulsarService;
-import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.service.ServerCnx;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.broker.storage.ManagedLedgerStorage;
import org.apache.pulsar.broker.storage.ManagedLedgerStorageClass;
-import org.apache.pulsar.broker.testcontext.PulsarTestContext;
-import org.apache.pulsar.broker.testcontext.SpyConfig;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.schema.GenericRecord;
import org.apache.pulsar.client.impl.BatchMessageIdImpl;
@@ -155,14 +152,6 @@ public class SimpleProducerConsumerTest extends ProducerConsumerBase {
super.producerBaseSetup();
}
- @Override
- protected PulsarTestContext.Builder createPulsarTestContextBuilder(ServiceConfiguration conf) {
- return super.createPulsarTestContextBuilder(conf)
- .spyConfig(SpyConfig.builder()
- .managedLedgerStorage(SpyConfig.SpyType.SPY_ALSO_INVOCATIONS)
- .build());
- }
-
@AfterMethod(alwaysRun = true)
public void cleanupAfterMethod() throws Exception {
try {