You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2022/06/27 20:13:51 UTC

[pinot] branch master updated: KafkaConsumer to reset offset to earliest when fetch position out of range (#8309)

This is an automated email from the ASF dual-hosted git repository.

nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new db07f2928a KafkaConsumer to reset offset to earliest when fetch position out of range (#8309)
db07f2928a is described below

commit db07f2928a10907badd6b4c1ed583020e1b2d9d2
Author: Neha Pawar <ne...@gmail.com>
AuthorDate: Mon Jun 27 13:13:46 2022 -0700

    KafkaConsumer to reset offset to earliest when fetch position out of range (#8309)
    
    * KafkaConsumer to reset offset to earliest when fetch position out of range
    
    * Don't override auto.offset.reset if already present in configs
    
    * Default behavior unchanged
---
 .../kafka20/KafkaPartitionLevelConsumerTest.java   | 60 ++++++++++++++++++++--
 .../stream/kafka20/utils/MiniKafkaCluster.java     | 10 ++++
 2 files changed, 67 insertions(+), 3 deletions(-)

diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumerTest.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumerTest.java
index e3f2c5a460..c8adbff28e 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumerTest.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumerTest.java
@@ -50,6 +50,7 @@ public class KafkaPartitionLevelConsumerTest {
   private static final long STABILIZE_SLEEP_DELAYS = 3000;
   private static final String TEST_TOPIC_1 = "foo";
   private static final String TEST_TOPIC_2 = "bar";
+  private static final String TEST_TOPIC_3 = "expired";
   private static final int NUM_MSG_PRODUCED_PER_PARTITION = 1000;
   private static final long TIMESTAMP = Instant.now().toEpochMilli();
 
@@ -64,9 +65,11 @@ public class KafkaPartitionLevelConsumerTest {
     _kafkaBrokerAddress = _kafkaCluster.getKafkaServerAddress();
     _kafkaCluster.createTopic(TEST_TOPIC_1, 1, 1);
     _kafkaCluster.createTopic(TEST_TOPIC_2, 2, 1);
+    _kafkaCluster.createTopic(TEST_TOPIC_3, 1, 1);
     Thread.sleep(STABILIZE_SLEEP_DELAYS);
     produceMsgToKafka();
     Thread.sleep(STABILIZE_SLEEP_DELAYS);
+    _kafkaCluster.deleteRecordsBeforeOffset(TEST_TOPIC_3, 0, 200);
   }
 
   private void produceMsgToKafka() {
@@ -81,6 +84,7 @@ public class KafkaPartitionLevelConsumerTest {
         // TEST_TOPIC_2 has 2 partitions
         producer.send(new ProducerRecord<>(TEST_TOPIC_2, 0, TIMESTAMP + i, null, "sample_msg_" + i));
         producer.send(new ProducerRecord<>(TEST_TOPIC_2, 1, TIMESTAMP + i, null, "sample_msg_" + i));
+        producer.send(new ProducerRecord<>(TEST_TOPIC_3, "sample_msg_" + i));
       }
       producer.flush();
     }
@@ -89,9 +93,13 @@ public class KafkaPartitionLevelConsumerTest {
   @AfterClass
   public void tearDown()
       throws Exception {
-    _kafkaCluster.deleteTopic(TEST_TOPIC_1);
-    _kafkaCluster.deleteTopic(TEST_TOPIC_2);
-    _kafkaCluster.close();
+    try {
+      _kafkaCluster.deleteTopic(TEST_TOPIC_1);
+      _kafkaCluster.deleteTopic(TEST_TOPIC_2);
+      _kafkaCluster.deleteTopic(TEST_TOPIC_3);
+    } finally {
+      _kafkaCluster.close();
+    }
   }
 
   @Test
@@ -354,4 +362,50 @@ public class KafkaPartitionLevelConsumerTest {
   protected String getKafkaConsumerFactoryName() {
     return KafkaConsumerFactory.class.getName();
   }
+
+  @Test
+  public void testOffsetsExpired()
+      throws TimeoutException {
+    Map<String, String> streamConfigMap = new HashMap<>();
+    streamConfigMap.put("streamType", "kafka");
+    streamConfigMap.put("stream.kafka.topic.name", TEST_TOPIC_3);
+    streamConfigMap.put("stream.kafka.broker.list", _kafkaBrokerAddress);
+    streamConfigMap.put("stream.kafka.consumer.type", "lowlevel");
+    streamConfigMap.put("stream.kafka.consumer.factory.class.name", getKafkaConsumerFactoryName());
+    streamConfigMap.put("stream.kafka.decoder.class.name", "decoderClass");
+    streamConfigMap.put("auto.offset.reset", "earliest");
+    StreamConfig streamConfig = new StreamConfig("tableName_REALTIME", streamConfigMap);
+
+    StreamConsumerFactory streamConsumerFactory = StreamConsumerFactoryProvider.create(streamConfig);
+    PartitionLevelConsumer consumer = streamConsumerFactory.createPartitionLevelConsumer("clientId", 0);
+
+    // Start offset has expired. Automatically reset to earliest available and fetch whatever available
+    MessageBatch batch1 = consumer.fetchMessages(new LongMsgOffset(0), new LongMsgOffset(400), 10000);
+    Assert.assertEquals(batch1.getMessageCount(), 200);
+    for (int i = 0; i < batch1.getMessageCount(); i++) {
+      byte[] msg = (byte[]) batch1.getMessageAtIndex(i);
+      Assert.assertEquals(new String(msg), "sample_msg_" + (i + 200));
+    }
+    Assert.assertEquals(batch1.getOffsetOfNextBatch().toString(), "400");
+
+    // Start and end offset has expired
+    MessageBatch batch2 = consumer.fetchMessages(new LongMsgOffset(0), new LongMsgOffset(100), 10000);
+    Assert.assertEquals(batch2.getMessageCount(), 0);
+
+    MessageBatch batch3 = consumer.fetchMessages(new LongMsgOffset(201), new LongMsgOffset(401), 10000);
+    Assert.assertEquals(batch3.getMessageCount(), 200);
+    for (int i = 0; i < batch3.getMessageCount(); i++) {
+      byte[] msg = (byte[]) batch3.getMessageAtIndex(i);
+      Assert.assertEquals(new String(msg), "sample_msg_" + (i + 201));
+    }
+    Assert.assertEquals(batch3.getOffsetOfNextBatch().toString(), "401");
+
+    MessageBatch batch4 = consumer.fetchMessages(new LongMsgOffset(0), null, 10000);
+    Assert.assertEquals(batch4.getMessageCount(), 500);
+    for (int i = 0; i < batch4.getMessageCount(); i++) {
+      byte[] msg = (byte[]) batch4.getMessageAtIndex(i);
+      Assert.assertEquals(new String(msg), "sample_msg_" + (i + 200));
+    }
+    Assert.assertEquals(batch4.getOffsetOfNextBatch().toString(), "700");
+  }
 }
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/utils/MiniKafkaCluster.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/utils/MiniKafkaCluster.java
index 29a8fbe6d1..6551a13e44 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/utils/MiniKafkaCluster.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/utils/MiniKafkaCluster.java
@@ -23,6 +23,8 @@ import java.io.File;
 import java.io.IOException;
 import java.net.ServerSocket;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Properties;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
@@ -31,7 +33,9 @@ import kafka.server.KafkaServer;
 import org.apache.commons.io.FileUtils;
 import org.apache.kafka.clients.admin.AdminClient;
 import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.RecordsToDelete;
 import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.utils.Time;
 import scala.Option;
 
@@ -112,4 +116,10 @@ public final class MiniKafkaCluster implements Closeable {
       throws ExecutionException, InterruptedException {
     _adminClient.deleteTopics(Collections.singletonList(topicName)).all().get();
   }
+
+  public void deleteRecordsBeforeOffset(String topicName, int partitionId, long offset) {
+    Map<TopicPartition, RecordsToDelete> recordsToDelete = new HashMap<>();
+    recordsToDelete.put(new TopicPartition(topicName, partitionId), RecordsToDelete.beforeOffset(offset));
+    _adminClient.deleteRecords(recordsToDelete);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org