You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2022/06/27 20:13:51 UTC
[pinot] branch master updated: KafkaConsumer to reset offset to earliest when fetch position out of range (#8309)
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new db07f2928a KafkaConsumer to reset offset to earliest when fetch position out of range (#8309)
db07f2928a is described below
commit db07f2928a10907badd6b4c1ed583020e1b2d9d2
Author: Neha Pawar <ne...@gmail.com>
AuthorDate: Mon Jun 27 13:13:46 2022 -0700
KafkaConsumer to reset offset to earliest when fetch position out of range (#8309)
* KafkaConsumer to reset offset to earliest when fetch position out of range
* Don't override auto.offset.reset if already present in configs
* Default behavior unchanged
---
.../kafka20/KafkaPartitionLevelConsumerTest.java | 60 ++++++++++++++++++++--
.../stream/kafka20/utils/MiniKafkaCluster.java | 10 ++++
2 files changed, 67 insertions(+), 3 deletions(-)
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumerTest.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumerTest.java
index e3f2c5a460..c8adbff28e 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumerTest.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumerTest.java
@@ -50,6 +50,7 @@ public class KafkaPartitionLevelConsumerTest {
private static final long STABILIZE_SLEEP_DELAYS = 3000;
private static final String TEST_TOPIC_1 = "foo";
private static final String TEST_TOPIC_2 = "bar";
+ private static final String TEST_TOPIC_3 = "expired";
private static final int NUM_MSG_PRODUCED_PER_PARTITION = 1000;
private static final long TIMESTAMP = Instant.now().toEpochMilli();
@@ -64,9 +65,11 @@ public class KafkaPartitionLevelConsumerTest {
_kafkaBrokerAddress = _kafkaCluster.getKafkaServerAddress();
_kafkaCluster.createTopic(TEST_TOPIC_1, 1, 1);
_kafkaCluster.createTopic(TEST_TOPIC_2, 2, 1);
+ _kafkaCluster.createTopic(TEST_TOPIC_3, 1, 1);
Thread.sleep(STABILIZE_SLEEP_DELAYS);
produceMsgToKafka();
Thread.sleep(STABILIZE_SLEEP_DELAYS);
+ _kafkaCluster.deleteRecordsBeforeOffset(TEST_TOPIC_3, 0, 200);
}
private void produceMsgToKafka() {
@@ -81,6 +84,7 @@ public class KafkaPartitionLevelConsumerTest {
// TEST_TOPIC_2 has 2 partitions
producer.send(new ProducerRecord<>(TEST_TOPIC_2, 0, TIMESTAMP + i, null, "sample_msg_" + i));
producer.send(new ProducerRecord<>(TEST_TOPIC_2, 1, TIMESTAMP + i, null, "sample_msg_" + i));
+ producer.send(new ProducerRecord<>(TEST_TOPIC_3, "sample_msg_" + i));
}
producer.flush();
}
@@ -89,9 +93,13 @@ public class KafkaPartitionLevelConsumerTest {
@AfterClass
public void tearDown()
throws Exception {
- _kafkaCluster.deleteTopic(TEST_TOPIC_1);
- _kafkaCluster.deleteTopic(TEST_TOPIC_2);
- _kafkaCluster.close();
+ try {
+ _kafkaCluster.deleteTopic(TEST_TOPIC_1);
+ _kafkaCluster.deleteTopic(TEST_TOPIC_2);
+ _kafkaCluster.deleteTopic(TEST_TOPIC_3);
+ } finally {
+ _kafkaCluster.close();
+ }
}
@Test
@@ -354,4 +362,50 @@ public class KafkaPartitionLevelConsumerTest {
protected String getKafkaConsumerFactoryName() {
return KafkaConsumerFactory.class.getName();
}
+
+ @Test
+ public void testOffsetsExpired()
+ throws TimeoutException {
+ Map<String, String> streamConfigMap = new HashMap<>();
+ streamConfigMap.put("streamType", "kafka");
+ streamConfigMap.put("stream.kafka.topic.name", TEST_TOPIC_3);
+ streamConfigMap.put("stream.kafka.broker.list", _kafkaBrokerAddress);
+ streamConfigMap.put("stream.kafka.consumer.type", "lowlevel");
+ streamConfigMap.put("stream.kafka.consumer.factory.class.name", getKafkaConsumerFactoryName());
+ streamConfigMap.put("stream.kafka.decoder.class.name", "decoderClass");
+ streamConfigMap.put("auto.offset.reset", "earliest");
+ StreamConfig streamConfig = new StreamConfig("tableName_REALTIME", streamConfigMap);
+
+ StreamConsumerFactory streamConsumerFactory = StreamConsumerFactoryProvider.create(streamConfig);
+ PartitionLevelConsumer consumer = streamConsumerFactory.createPartitionLevelConsumer("clientId", 0);
+
+ // Start offset has expired. Automatically reset to earliest available and fetch whatever available
+ MessageBatch batch1 = consumer.fetchMessages(new LongMsgOffset(0), new LongMsgOffset(400), 10000);
+ Assert.assertEquals(batch1.getMessageCount(), 200);
+ for (int i = 0; i < batch1.getMessageCount(); i++) {
+ byte[] msg = (byte[]) batch1.getMessageAtIndex(i);
+ Assert.assertEquals(new String(msg), "sample_msg_" + (i + 200));
+ }
+ Assert.assertEquals(batch1.getOffsetOfNextBatch().toString(), "400");
+
+ // Start and end offset has expired
+ MessageBatch batch2 = consumer.fetchMessages(new LongMsgOffset(0), new LongMsgOffset(100), 10000);
+ Assert.assertEquals(batch2.getMessageCount(), 0);
+
+ MessageBatch batch3 = consumer.fetchMessages(new LongMsgOffset(201), new LongMsgOffset(401), 10000);
+ Assert.assertEquals(batch3.getMessageCount(), 200);
+ for (int i = 0; i < batch3.getMessageCount(); i++) {
+ byte[] msg = (byte[]) batch3.getMessageAtIndex(i);
+ Assert.assertEquals(new String(msg), "sample_msg_" + (i + 201));
+ }
+ Assert.assertEquals(batch3.getOffsetOfNextBatch().toString(), "401");
+
+ MessageBatch batch4 = consumer.fetchMessages(new LongMsgOffset(0), null, 10000);
+ Assert.assertEquals(batch4.getMessageCount(), 500);
+ for (int i = 0; i < batch4.getMessageCount(); i++) {
+ byte[] msg = (byte[]) batch4.getMessageAtIndex(i);
+ Assert.assertEquals(new String(msg), "sample_msg_" + (i + 200));
+ }
+ Assert.assertEquals(batch4.getOffsetOfNextBatch().toString(), "700");
+ }
}
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/utils/MiniKafkaCluster.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/utils/MiniKafkaCluster.java
index 29a8fbe6d1..6551a13e44 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/utils/MiniKafkaCluster.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/utils/MiniKafkaCluster.java
@@ -23,6 +23,8 @@ import java.io.File;
import java.io.IOException;
import java.net.ServerSocket;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
@@ -31,7 +33,9 @@ import kafka.server.KafkaServer;
import org.apache.commons.io.FileUtils;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.RecordsToDelete;
import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.utils.Time;
import scala.Option;
@@ -112,4 +116,10 @@ public final class MiniKafkaCluster implements Closeable {
throws ExecutionException, InterruptedException {
_adminClient.deleteTopics(Collections.singletonList(topicName)).all().get();
}
+
+ public void deleteRecordsBeforeOffset(String topicName, int partitionId, long offset) {
+ Map<TopicPartition, RecordsToDelete> recordsToDelete = new HashMap<>();
+ recordsToDelete.put(new TopicPartition(topicName, partitionId), RecordsToDelete.beforeOffset(offset));
+ _adminClient.deleteRecords(recordsToDelete);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org