You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/04/29 02:03:43 UTC
[GitHub] sijie closed pull request #1672: Support read-compacted on Reader
sijie closed pull request #1672: Support read-compacted on Reader
URL: https://github.com/apache/incubator-pulsar/pull/1672
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java
index fee60b7795..6451b0a8ef 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java
@@ -22,6 +22,9 @@
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -41,8 +44,8 @@
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
-import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.CompressionType;
+import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.CryptoKeyReader;
import org.apache.pulsar.client.api.EncryptionKeyInfo;
import org.apache.pulsar.client.api.Message;
@@ -51,24 +54,17 @@
import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.api.Reader;
import org.apache.pulsar.client.impl.BatchMessageIdImpl;
import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.RetentionPolicies;
import org.apache.pulsar.common.policies.data.TenantInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
public class CompactionTest extends MockedPulsarServiceBaseTest {
- private static final Logger log = LoggerFactory.getLogger(CompactionTest.class);
-
private ScheduledExecutorService compactionScheduler;
private BookKeeper bk;
@@ -154,6 +150,69 @@ public void testCompaction() throws Exception {
}
}
+ @Test
+ public void testCompactionWithReader() throws Exception {
+ String topic = "persistent://my-property/use/my-ns/my-topic1";
+ final int numMessages = 20;
+ final int maxKeys = 10;
+
+ // Configure retention to ensue data is retained for reader
+ admin.namespaces().setRetention("my-property/use/my-ns", new RetentionPolicies(-1, -1));
+
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic(topic)
+ .enableBatching(false)
+ .messageRoutingMode(MessageRoutingMode.SinglePartition)
+ .create();
+
+ Map<String, String> expected = new HashMap<>();
+ List<Pair<String, String>> all = new ArrayList<>();
+ Random r = new Random(0);
+
+ for (int j = 0; j < numMessages; j++) {
+ int keyIndex = r.nextInt(maxKeys);
+ String key = "key" + keyIndex;
+ String value = "my-message-" + key + "-" + j;
+ producer.send(MessageBuilder.create().setKey(key).setContent(value.getBytes()).build());
+ expected.put(key, value);
+ all.add(Pair.of(key, value));
+ }
+
+ Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
+ compactor.compact(topic).get();
+
+
+
+ // consumer with readCompacted enabled only get compacted entries
+ try (Reader<byte[]> reader = pulsarClient.newReader().topic(topic).readCompacted(true)
+ .startMessageId(MessageId.earliest).create()) {
+ while (true) {
+ Message<byte[]> m = reader.readNext(2, TimeUnit.SECONDS);
+ Assert.assertEquals(expected.remove(m.getKey()), new String(m.getData()));
+ if (expected.isEmpty()) {
+ break;
+ }
+ }
+ Assert.assertTrue(expected.isEmpty());
+ }
+
+ // can get full backlog if read compacted disabled
+ try (Reader<byte[]> reader = pulsarClient.newReader().topic(topic).readCompacted(false)
+ .startMessageId(MessageId.earliest).create()) {
+ while (true) {
+ Message<byte[]> m = reader.readNext(2, TimeUnit.SECONDS);
+ Pair<String, String> expectedMessage = all.remove(0);
+ Assert.assertEquals(expectedMessage.getLeft(), m.getKey());
+ Assert.assertEquals(expectedMessage.getRight(), new String(m.getData()));
+ if (all.isEmpty()) {
+ break;
+ }
+ }
+ Assert.assertTrue(all.isEmpty());
+ }
+ }
+
+
@Test
public void testReadCompactedBeforeCompaction() throws Exception {
String topic = "persistent://my-property/use/my-ns/my-topic1";
@@ -343,7 +402,7 @@ public void testFirstMessageRetained() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producer = pulsarClient.newProducer().topic(topic).enableBatching(false).create()) {
+ try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).enableBatching(false).create()) {
producer.sendAsync(MessageBuilder.create()
.setKey("key1")
.setContent("my-message-1".getBytes()).build());
@@ -356,8 +415,8 @@ public void testFirstMessageRetained() throws Exception {
}
// Read messages before compaction to get ids
- List<Message> messages = new ArrayList<>();
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ List<Message<byte[]>> messages = new ArrayList<>();
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()) {
messages.add(consumer.receive());
messages.add(consumer.receive());
@@ -369,14 +428,14 @@ public void testFirstMessageRetained() throws Exception {
compactor.compact(topic).get();
// Check that messages after compaction have same ids
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key1");
Assert.assertEquals(new String(message1.getData()), "my-message-1");
Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId());
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key2");
Assert.assertEquals(new String(message2.getData()), "my-message-3");
Assert.assertEquals(message2.getMessageId(), messages.get(2).getMessageId());
@@ -391,7 +450,7 @@ public void testBatchMessageIdsDontChange() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producer = pulsarClient.newProducer().topic(topic)
+ try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
.maxPendingMessages(3)
.enableBatching(true)
.batchingMaxMessages(3)
@@ -411,8 +470,8 @@ public void testBatchMessageIdsDontChange() throws Exception {
}
// Read messages before compaction to get ids
- List<Message> messages = new ArrayList<>();
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ List<Message<byte[]>> messages = new ArrayList<>();
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()) {
messages.add(consumer.receive());
messages.add(consumer.receive());
@@ -434,14 +493,14 @@ public void testBatchMessageIdsDontChange() throws Exception {
compactor.compact(topic).get();
// Check that messages after compaction have same ids
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key1");
Assert.assertEquals(new String(message1.getData()), "my-message-1");
Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId());
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key2");
Assert.assertEquals(new String(message2.getData()), "my-message-3");
Assert.assertEquals(message2.getMessageId(), messages.get(2).getMessageId());
@@ -456,11 +515,11 @@ public void testWholeBatchCompactedOut() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producerNormal = pulsarClient.newProducer().topic(topic)
+ try (Producer<byte[]> producerNormal = pulsarClient.newProducer().topic(topic)
.enableBatching(false)
.messageRoutingMode(MessageRoutingMode.SinglePartition)
.create();
- Producer producerBatch = pulsarClient.newProducer().topic(topic)
+ Producer<byte[]> producerBatch = pulsarClient.newProducer().topic(topic)
.maxPendingMessages(3)
.enableBatching(true)
.batchingMaxMessages(3)
@@ -485,9 +544,9 @@ public void testWholeBatchCompactedOut() throws Exception {
Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
compactor.compact(topic).get();
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()){
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
Assert.assertEquals(message.getKey(), "key1");
Assert.assertEquals(new String(message.getData()), "my-message-4");
}
@@ -501,8 +560,8 @@ public void testKeyLessMessagesPassThrough() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producerNormal = pulsarClient.newProducer().topic(topic).create();
- Producer producerBatch = pulsarClient.newProducer().topic(topic).maxPendingMessages(3)
+ try (Producer<byte[]> producerNormal = pulsarClient.newProducer().topic(topic).create();
+ Producer<byte[]> producerBatch = pulsarClient.newProducer().topic(topic).maxPendingMessages(3)
.enableBatching(true).batchingMaxMessages(3)
.batchingMaxPublishDelay(1, TimeUnit.HOURS).create()) {
producerNormal.sendAsync(MessageBuilder.create()
@@ -534,29 +593,29 @@ public void testKeyLessMessagesPassThrough() throws Exception {
Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
compactor.compact(topic).get();
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertFalse(message1.hasKey());
Assert.assertEquals(new String(message1.getData()), "my-message-1");
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertFalse(message2.hasKey());
Assert.assertEquals(new String(message2.getData()), "my-message-2");
- Message message3 = consumer.receive();
+ Message<byte[]> message3 = consumer.receive();
Assert.assertEquals(message3.getKey(), "key1");
Assert.assertEquals(new String(message3.getData()), "my-message-4");
- Message message4 = consumer.receive();
+ Message<byte[]> message4 = consumer.receive();
Assert.assertEquals(message4.getKey(), "key2");
Assert.assertEquals(new String(message4.getData()), "my-message-6");
- Message message5 = consumer.receive();
+ Message<byte[]> message5 = consumer.receive();
Assert.assertFalse(message5.hasKey());
Assert.assertEquals(new String(message5.getData()), "my-message-7");
- Message message6 = consumer.receive();
+ Message<byte[]> message6 = consumer.receive();
Assert.assertFalse(message6.hasKey());
Assert.assertEquals(new String(message6.getData()), "my-message-8");
}
@@ -571,11 +630,11 @@ public void testEmptyPayloadDeletes() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producerNormal = pulsarClient.newProducer()
+ try (Producer<byte[]> producerNormal = pulsarClient.newProducer()
.topic(topic)
.enableBatching(false)
.create();
- Producer producerBatch = pulsarClient.newProducer()
+ Producer<byte[]> producerBatch = pulsarClient.newProducer()
.topic(topic)
.maxPendingMessages(3)
.enableBatching(true)
@@ -625,13 +684,13 @@ public void testEmptyPayloadDeletes() throws Exception {
Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
compactor.compact(topic).get();
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key0");
Assert.assertEquals(new String(message1.getData()), "my-message-0");
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key4");
Assert.assertEquals(new String(message2.getData()), "my-message-4");
}
@@ -659,7 +718,7 @@ public void testCompactorReadsCompacted() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").subscribe().close();
// create the topic on the broker
- try (Producer producerNormal = pulsarClient.newProducer().topic(topic).create()) {
+ try (Producer<byte[]> producerNormal = pulsarClient.newProducer().topic(topic).create()) {
producerNormal.send(MessageBuilder.create()
.setKey("key0")
.setContent("my-message-0".getBytes()).build());
@@ -669,7 +728,7 @@ public void testCompactorReadsCompacted() throws Exception {
pulsar.getBrokerService().getTopicReference(topic).get().close().get();
// write a message to avoid issue #1517
- try (Producer producerNormal = pulsarClient.newProducer().topic(topic).create()) {
+ try (Producer<byte[]> producerNormal = pulsarClient.newProducer().topic(topic).create()) {
producerNormal.send(MessageBuilder.create()
.setKey("key1")
.setContent("my-message-1".getBytes()).build());
@@ -695,7 +754,7 @@ public void testCompactorReadsCompacted() throws Exception {
pulsar.getBrokerService().getTopicReference(topic).get().close().get();
// write a message to avoid issue #1517
- try (Producer producerNormal = pulsarClient.newProducer().topic(topic).create()) {
+ try (Producer<byte[]> producerNormal = pulsarClient.newProducer().topic(topic).create()) {
producerNormal.send(MessageBuilder.create()
.setKey("key2")
.setContent("my-message-2".getBytes()).build());
@@ -720,17 +779,17 @@ public void testCompactorReadsCompacted() throws Exception {
Assert.assertFalse(ledgersOpened.contains(info.ledgers.get(2).ledgerId));
// all three messages should be there when we read compacted
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key0");
Assert.assertEquals(new String(message1.getData()), "my-message-0");
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key1");
Assert.assertEquals(new String(message2.getData()), "my-message-1");
- Message message3 = consumer.receive();
+ Message<byte[]> message3 = consumer.receive();
Assert.assertEquals(message3.getKey(), "key2");
Assert.assertEquals(new String(message3.getData()), "my-message-2");
}
@@ -744,7 +803,7 @@ public void testCompactCompressedNoBatch() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producer = pulsarClient.newProducer().topic(topic)
+ try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
.compressionType(CompressionType.LZ4).enableBatching(false).create()) {
producer.sendAsync(MessageBuilder.create()
.setKey("key1")
@@ -761,13 +820,13 @@ public void testCompactCompressedNoBatch() throws Exception {
Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
compactor.compact(topic).get();
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key1");
Assert.assertEquals(new String(message1.getData()), "my-message-1");
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key2");
Assert.assertEquals(new String(message2.getData()), "my-message-3");
}
@@ -781,7 +840,7 @@ public void testCompactCompressedBatching() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producer = pulsarClient.newProducer().topic(topic)
+ try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
.compressionType(CompressionType.LZ4)
.maxPendingMessages(3)
.enableBatching(true)
@@ -802,13 +861,13 @@ public void testCompactCompressedBatching() throws Exception {
Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
compactor.compact(topic).get();
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key1");
Assert.assertEquals(new String(message1.getData()), "my-message-1");
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key2");
Assert.assertEquals(new String(message2.getData()), "my-message-3");
}
@@ -858,7 +917,7 @@ public void testCompactEncryptedNoBatch() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producer = pulsarClient.newProducer().topic(topic)
+ try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
.addEncryptionKey("client-ecdsa.pem").cryptoKeyReader(new EncKeyReader())
.enableBatching(false).create()) {
producer.sendAsync(MessageBuilder.create()
@@ -877,14 +936,14 @@ public void testCompactEncryptedNoBatch() throws Exception {
compactor.compact(topic).get();
// Check that messages after compaction have same ids
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").cryptoKeyReader(new EncKeyReader())
.readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key1");
Assert.assertEquals(new String(message1.getData()), "my-message-1");
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key2");
Assert.assertEquals(new String(message2.getData()), "my-message-3");
}
@@ -898,7 +957,7 @@ public void testCompactEncryptedBatching() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producer = pulsarClient.newProducer().topic(topic)
+ try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
.addEncryptionKey("client-ecdsa.pem").cryptoKeyReader(new EncKeyReader())
.maxPendingMessages(3)
.enableBatching(true)
@@ -921,18 +980,18 @@ public void testCompactEncryptedBatching() throws Exception {
// with encryption, all messages are passed through compaction as it doesn't
// have the keys to decrypt the batch payload
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").cryptoKeyReader(new EncKeyReader())
.readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key1");
Assert.assertEquals(new String(message1.getData()), "my-message-1");
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key2");
Assert.assertEquals(new String(message2.getData()), "my-message-2");
- Message message3 = consumer.receive();
+ Message<byte[]> message3 = consumer.receive();
Assert.assertEquals(message3.getKey(), "key2");
Assert.assertEquals(new String(message3.getData()), "my-message-3");
}
@@ -946,7 +1005,7 @@ public void testCompactEncryptedAndCompressedNoBatch() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producer = pulsarClient.newProducer().topic(topic)
+ try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
.addEncryptionKey("client-ecdsa.pem").cryptoKeyReader(new EncKeyReader())
.compressionType(CompressionType.LZ4)
.enableBatching(false).create()) {
@@ -966,14 +1025,14 @@ public void testCompactEncryptedAndCompressedNoBatch() throws Exception {
compactor.compact(topic).get();
// Check that messages after compaction have same ids
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").cryptoKeyReader(new EncKeyReader())
.readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key1");
Assert.assertEquals(new String(message1.getData()), "my-message-1");
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key2");
Assert.assertEquals(new String(message2.getData()), "my-message-3");
}
@@ -987,7 +1046,7 @@ public void testCompactEncryptedAndCompressedBatching() throws Exception {
pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
.readCompacted(true).subscribe().close();
- try (Producer producer = pulsarClient.newProducer().topic(topic)
+ try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
.addEncryptionKey("client-ecdsa.pem").cryptoKeyReader(new EncKeyReader())
.compressionType(CompressionType.LZ4)
.maxPendingMessages(3)
@@ -1011,18 +1070,18 @@ public void testCompactEncryptedAndCompressedBatching() throws Exception {
// with encryption, all messages are passed through compaction as it doesn't
// have the keys to decrypt the batch payload
- try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic)
.subscriptionName("sub1").cryptoKeyReader(new EncKeyReader())
.readCompacted(true).subscribe()){
- Message message1 = consumer.receive();
+ Message<byte[]> message1 = consumer.receive();
Assert.assertEquals(message1.getKey(), "key1");
Assert.assertEquals(new String(message1.getData()), "my-message-1");
- Message message2 = consumer.receive();
+ Message<byte[]> message2 = consumer.receive();
Assert.assertEquals(message2.getKey(), "key2");
Assert.assertEquals(new String(message2.getData()), "my-message-2");
- Message message3 = consumer.receive();
+ Message<byte[]> message3 = consumer.receive();
Assert.assertEquals(message3.getKey(), "key2");
Assert.assertEquals(new String(message3.getData()), "my-message-3");
}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/api/ReaderBuilder.java b/pulsar-client/src/main/java/org/apache/pulsar/client/api/ReaderBuilder.java
index f40a8a4552..314af7b14a 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/api/ReaderBuilder.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/api/ReaderBuilder.java
@@ -144,4 +144,18 @@
* @param subscriptionRolePrefix
*/
ReaderBuilder<T> subscriptionRolePrefix(String subscriptionRolePrefix);
+
+ /**
+ * If enabled, the reader will read messages from the compacted topic rather than reading the full message backlog
+ * of the topic. This means that, if the topic has been compacted, the reader will only see the latest value for
+ * each key in the topic, up until the point in the topic message backlog that has been compacted. Beyond that
+ * point, the messages will be sent as normal.
+ *
+ * readCompacted can only be enabled when reading from a persistent topic. Attempting to enable it on non-persistent
+ * topics will lead to the reader create call throwing a PulsarClientException.
+ *
+ * @param readCompacted
+ * whether to read from the compacted topic
+ */
+ ReaderBuilder<T> readCompacted(boolean readCompacted);
}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java
index 4ab5f423b2..9e8bce82aa 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java
@@ -141,4 +141,10 @@ private ReaderBuilderImpl(PulsarClientImpl client, ReaderConfigurationData<T> co
conf.setSubscriptionRolePrefix(subscriptionRolePrefix);
return this;
}
+
+ @Override
+ public ReaderBuilder<T> readCompacted(boolean readCompacted) {
+ conf.setReadCompacted(readCompacted);
+ return this;
+ }
}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java
index ed374f6632..9d0ed93d68 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java
@@ -54,6 +54,7 @@ public ReaderImpl(PulsarClientImpl client, ReaderConfigurationData<T> readerConf
consumerConfiguration.setSubscriptionName(subscription);
consumerConfiguration.setSubscriptionType(SubscriptionType.Exclusive);
consumerConfiguration.setReceiverQueueSize(readerConfiguration.getReceiverQueueSize());
+ consumerConfiguration.setReadCompacted(readerConfiguration.isReadCompacted());
if (readerConfiguration.getReaderName() != null) {
consumerConfiguration.setConsumerName(readerConfiguration.getReaderName());
}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ReaderConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ReaderConfigurationData.java
index 7d2219da16..362c8f96c5 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ReaderConfigurationData.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ReaderConfigurationData.java
@@ -43,6 +43,8 @@
private CryptoKeyReader cryptoKeyReader = null;
private ConsumerCryptoFailureAction cryptoFailureAction = ConsumerCryptoFailureAction.FAIL;
+ private boolean readCompacted = false;
+
@SuppressWarnings("unchecked")
public ReaderConfigurationData<T> clone() {
try {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services