You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by te...@apache.org on 2023/06/05 08:36:39 UTC
[pulsar] branch branch-2.11 updated (373294bfa04 -> 76b6e30b7b7)
This is an automated email from the ASF dual-hosted git repository.
technoboy pushed a change to branch branch-2.11
in repository https://gitbox.apache.org/repos/asf/pulsar.git
from 373294bfa04 Fix cherry-pick #19929 cause license header issue
new d78747b5f8b [fix][client] Fix where the function getMsgNumInReceiverQueue always returns 0 when using message listener (#20245)
new 76b6e30b7b7 [fix] [broker] do not filter system topic while shedding. (#18949)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/pulsar/broker/loadbalance/LoadData.java | 2 +-
.../pulsar/broker/namespace/NamespaceService.java | 11 ++++++++
.../client/api/SimpleProducerConsumerTest.java | 29 ++++++++++++++++++++++
.../apache/pulsar/client/impl/ConsumerBase.java | 8 ++++++
.../client/impl/ConsumerStatsRecorderImpl.java | 6 ++++-
5 files changed, 54 insertions(+), 2 deletions(-)
[pulsar] 01/02: [fix][client] Fix where the function getMsgNumInReceiverQueue always returns 0 when using message listener (#20245)
Posted by te...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
technoboy pushed a commit to branch branch-2.11
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit d78747b5f8bb1fedd1e84b68ce5e3c08d8b49af8
Author: crossoverJie <cr...@gmail.com>
AuthorDate: Mon Jun 5 16:10:44 2023 +0800
[fix][client] Fix where the function getMsgNumInReceiverQueue always returns 0 when using message listener (#20245)
---
.../client/api/SimpleProducerConsumerTest.java | 29 ++++++++++++++++++++++
.../apache/pulsar/client/impl/ConsumerBase.java | 8 ++++++
.../client/impl/ConsumerStatsRecorderImpl.java | 6 ++++-
3 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index 2f572a841b0..21ec7e3e5f3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -4024,6 +4024,35 @@ public class SimpleProducerConsumerTest extends ProducerConsumerBase {
consumer.close();
producer.close();
}
+ @Test(timeOut = 100000)
+ public void testMessageListenerGetStats() throws Exception {
+ final String topicName = "persistent://my-property/my-ns/testGetStats" + UUID.randomUUID();
+ final String subName = "my-sub";
+ final int receiveQueueSize = 100;
+ @Cleanup
+ PulsarClient client = newPulsarClient(lookupUrl.toString(), 100);
+ Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+ .enableBatching(false).topic(topicName).create();
+ ConsumerImpl<String> consumer = (ConsumerImpl<String>) client.newConsumer(Schema.STRING)
+ .messageListener((MessageListener<String>) (consumer1, msg) -> {
+ try {
+ TimeUnit.SECONDS.sleep(10);
+ } catch (InterruptedException igr) {
+ }
+ })
+ .topic(topicName).receiverQueueSize(receiveQueueSize).subscriptionName(subName).subscribe();
+ Assert.assertNull(consumer.getStats().getMsgNumInSubReceiverQueue());
+ Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), 0);
+
+ for (int i = 0; i < receiveQueueSize; i++) {
+ producer.sendAsync("msg" + i);
+ }
+ //Give some time to consume
+ Awaitility.await()
+ .untilAsserted(() -> Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), receiveQueueSize));
+ consumer.close();
+ producer.close();
+ }
@Test(timeOut = 100000)
public void testGetStatsForPartitionedTopic() throws Exception {
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
index ebd6a2e0ebf..13f8e735e7b 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
@@ -86,6 +86,11 @@ public abstract class ConsumerBase<T> extends HandlerState implements Consumer<T
protected final ConcurrentLinkedQueue<CompletableFuture<Message<T>>> pendingReceives;
protected final int maxReceiverQueueSize;
private volatile int currentReceiverQueueSize;
+
+ protected static final AtomicIntegerFieldUpdater<ConsumerBase> MESSAGE_LISTENER_QUEUE_SIZE_UPDATER =
+ AtomicIntegerFieldUpdater.newUpdater(ConsumerBase.class, "messageListenerQueueSize");
+ protected volatile int messageListenerQueueSize = 0;
+
protected static final AtomicIntegerFieldUpdater<ConsumerBase> CURRENT_RECEIVER_QUEUE_SIZE_UPDATER =
AtomicIntegerFieldUpdater.newUpdater(ConsumerBase.class, "currentReceiverQueueSize");
protected final Schema<T> schema;
@@ -1050,6 +1055,7 @@ public abstract class ConsumerBase<T> extends HandlerState implements Consumer<T
// Trigger the notification on the message listener in a separate thread to avoid blocking the
// internal pinned executor thread while the message processing happens
final Message<T> finalMsg = msg;
+ MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.incrementAndGet(this);
if (SubscriptionType.Key_Shared == conf.getSubscriptionType()) {
executorProvider.getExecutor(peekMessageKey(msg)).execute(() ->
callMessageListener(finalMsg));
@@ -1086,6 +1092,8 @@ public abstract class ConsumerBase<T> extends HandlerState implements Consumer<T
} catch (Throwable t) {
log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription,
msg.getMessageId(), t);
+ } finally {
+ MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.decrementAndGet(this);
}
}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java
index a41e036e516..789c7f9ea63 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java
@@ -239,7 +239,11 @@ public class ConsumerStatsRecorderImpl implements ConsumerStatsRecorder {
@Override
public Integer getMsgNumInReceiverQueue() {
if (consumer instanceof ConsumerBase) {
- return ((ConsumerBase<?>) consumer).incomingMessages.size();
+ ConsumerBase<?> consumerBase = (ConsumerBase<?>) consumer;
+ if (consumerBase.listener != null){
+ return ConsumerBase.MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.get(consumerBase);
+ }
+ return consumerBase.incomingMessages.size();
}
return null;
}
[pulsar] 02/02: [fix] [broker] do not filter system topic while shedding. (#18949)
Posted by te...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
technoboy pushed a commit to branch branch-2.11
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 76b6e30b7b7933c236a75cdd16664bc3e943683a
Author: thetumbled <52...@users.noreply.github.com>
AuthorDate: Wed May 31 14:34:32 2023 +0800
[fix] [broker] do not filter system topic while shedding. (#18949)
### Motivation
Currently, topics/bundles in `pulsar/system` will be filter while doing shedding, which is introduced by mistake by pr https://github.com/apache/pulsar/pull/15252.
But we need to unload topics/bundles in `pulsar/system` for load balancing.
### Modifications
do not filter topics/bundles in `pulsar/system`.
---
.../java/org/apache/pulsar/broker/loadbalance/LoadData.java | 2 +-
.../org/apache/pulsar/broker/namespace/NamespaceService.java | 11 +++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java
index 066b656a9dd..5b7875078d9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java
@@ -64,7 +64,7 @@ public class LoadData {
public Map<String, BundleData> getBundleDataForLoadShedding() {
return bundleData.entrySet().stream()
- .filter(e -> !NamespaceService.isSystemServiceNamespace(
+ .filter(e -> !NamespaceService.filterNamespaceForShedding(
NamespaceBundle.getBundleNamespace(e.getKey())))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index 4b2df9c5487..5c288af7b8a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -1408,6 +1408,17 @@ public class NamespaceService implements AutoCloseable {
|| HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches();
}
+ /**
+ * used for filtering bundles in special namespace.
+ * @param namespace
+ * @return True if namespace is HEARTBEAT_NAMESPACE or SLA_NAMESPACE
+ */
+ public static boolean filterNamespaceForShedding(String namespace) {
+ return SLA_NAMESPACE_PATTERN.matcher(namespace).matches()
+ || HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches()
+ || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches();
+ }
+
public static boolean isHeartbeatNamespace(ServiceUnitId ns) {
String namespace = ns.getNamespaceObject().toString();
return HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches()