You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by xi...@apache.org on 2025/03/25 13:33:12 UTC
(pulsar) branch master updated: [fix][client] Fix inconsistent compression threshold behavior across batching modes (#24102)
This is an automated email from the ASF dual-hosted git repository.
xiangying pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new b42beef7e9c [fix][client] Fix inconsistent compression threshold behavior across batching modes (#24102)
b42beef7e9c is described below
commit b42beef7e9cc3239f741e4e114e7657f821ed824
Author: Xiangying Meng <55...@users.noreply.github.com>
AuthorDate: Tue Mar 25 21:33:05 2025 +0800
[fix][client] Fix inconsistent compression threshold behavior across batching modes (#24102)
Co-authored-by: xiangying <me...@xiaohongshu.com>
https://github.com/apache/pulsar/pull/23526
## Motivation
Fix inconsistent compression threshold behavior across batching modes. For single message sending, compression is enabled when the message size is greater than or equal to the threshold, while for batch sending, compression occurs when the size is greater than the threshold.
## Modifications
1. Standardize the criteria for determining the threshold to enable compression.
2. Optimize formatting.
3. Enhance testing.
---
.../client/impl/ProducerConsumerInternalTest.java | 21 ++++++++++++++++++---
.../org/apache/pulsar/client/impl/ProducerImpl.java | 6 ++----
2 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java
index 4617f631207..503afbefb78 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java
@@ -237,7 +237,7 @@ public class ProducerConsumerInternalTest extends ProducerConsumerBase {
@Test
public void testProducerCompressionMinMsgBodySize() throws PulsarClientException {
- byte[] msg1022 = new byte[1022];
+ byte[] msg1024 = new byte[1024];
byte[] msg1025 = new byte[1025];
final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_");
@Cleanup
@@ -256,7 +256,7 @@ public class ProducerConsumerInternalTest extends ProducerConsumerBase {
producer.conf.setCompressionType(CompressionType.LZ4);
// disable batch
producer.conf.setBatchingEnabled(false);
- producer.newMessage().value(msg1022).send();
+ producer.newMessage().value(msg1024).send();
MessageImpl<byte[]> message = (MessageImpl<byte[]>) consumer.receive();
CompressionType compressionType = message.getCompressionType();
assertEquals(compressionType, CompressionType.NONE);
@@ -267,7 +267,7 @@ public class ProducerConsumerInternalTest extends ProducerConsumerBase {
// enable batch
producer.conf.setBatchingEnabled(true);
- producer.newMessage().value(msg1022).send();
+ producer.newMessage().value(msg1024).send();
message = (MessageImpl<byte[]>) consumer.receive();
compressionType = message.getCompressionType();
assertEquals(compressionType, CompressionType.NONE);
@@ -275,5 +275,20 @@ public class ProducerConsumerInternalTest extends ProducerConsumerBase {
message = (MessageImpl<byte[]>) consumer.receive();
compressionType = message.getCompressionType();
assertEquals(compressionType, CompressionType.LZ4);
+
+ // Verify data integrity
+ String data = "compression test message";
+ producer.conf.setBatchingEnabled(true);
+ producer.getConfiguration().setCompressMinMsgBodySize(1);
+ producer.newMessage().value(data.getBytes()).send();
+ message = (MessageImpl<byte[]>) consumer.receive();
+ assertEquals(new String(message.getData()), data);
+
+ producer.conf.setBatchingEnabled(false);
+ producer.getConfiguration().setCompressMinMsgBodySize(1);
+ producer.newMessage().value(data.getBytes()).send();
+ message = (MessageImpl<byte[]>) consumer.receive();
+ assertEquals(new String(message.getData()), data);
+
}
}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
index 2752dce945f..41df75d9b3e 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
@@ -542,10 +542,8 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
boolean compressed = false;
// Batch will be compressed when closed
// If a message has a delayed delivery time, we'll always send it individually
- if (((!isBatchMessagingEnabled() || msgMetadata.hasDeliverAtTime()))) {
- if (payload.readableBytes() < conf.getCompressMinMsgBodySize()) {
-
- } else {
+ if (!isBatchMessagingEnabled() || msgMetadata.hasDeliverAtTime()) {
+ if (payload.readableBytes() > conf.getCompressMinMsgBodySize()) {
compressedPayload = applyCompression(payload);
compressed = true;