You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2026/03/18 18:27:31 UTC

(pinot) branch master updated: Use BytesDeserializer for Kafka record keys to preserve binary key format (#17899)

This is an automated email from the ASF dual-hosted git repository.

xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 10237512dd4 Use BytesDeserializer for Kafka record keys to preserve binary key format (#17899)
10237512dd4 is described below

commit 10237512dd43fe02b84e1b9ba4570bca5d716331
Author: NOOB <43...@users.noreply.github.com>
AuthorDate: Wed Mar 18 23:57:25 2026 +0530

    Use BytesDeserializer for Kafka record keys to preserve binary key format (#17899)
---
 .../KafkaPartitionLevelConnectionHandler.java       | 11 +++++------
 .../stream/kafka30/KafkaPartitionLevelConsumer.java | 13 ++++++-------
 .../kafka30/KafkaPartitionLevelConsumerTest.java    | 21 ++++++++++++---------
 .../KafkaPartitionLevelConnectionHandler.java       | 11 +++++------
 .../stream/kafka40/KafkaPartitionLevelConsumer.java | 13 ++++++-------
 .../kafka40/KafkaPartitionLevelConsumerTest.java    | 21 ++++++++++++---------
 6 files changed, 46 insertions(+), 44 deletions(-)

diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConnectionHandler.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConnectionHandler.java
index c469cc75e48..ffa36d3c06c 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConnectionHandler.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConnectionHandler.java
@@ -36,7 +36,6 @@ import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.serialization.BytesDeserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.pinot.plugin.stream.kafka.KafkaAdminClientManager;
 import org.apache.pinot.plugin.stream.kafka.KafkaPartitionLevelStreamConfig;
@@ -63,7 +62,7 @@ public abstract class KafkaPartitionLevelConnectionHandler {
   protected final String _clientId;
   protected final int _partition;
   protected final String _topic;
-  protected final Consumer<String, Bytes> _consumer;
+  protected final Consumer<Bytes, Bytes> _consumer;
   protected final TopicPartition _topicPartition;
   protected final Properties _consumerProp;
   protected volatile KafkaAdminClientManager.AdminClientReference _sharedAdminClientRef;
@@ -93,7 +92,7 @@ public abstract class KafkaPartitionLevelConnectionHandler {
     Properties consumerProp = new Properties();
     consumerProp.putAll(streamConfig.getStreamConfigsMap());
     consumerProp.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, _config.getBootstrapHosts());
-    consumerProp.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+    consumerProp.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
     consumerProp.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
     if (_config.getKafkaIsolationLevel() != null) {
       consumerProp.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, _config.getKafkaIsolationLevel());
@@ -120,8 +119,8 @@ public abstract class KafkaPartitionLevelConnectionHandler {
     return filteredProps;
   }
 
-  private Consumer<String, Bytes> createConsumer(Properties consumerProp, RetryPolicy retryPolicy) {
-    AtomicReference<Consumer<String, Bytes>> consumer = new AtomicReference<>();
+  private Consumer<Bytes, Bytes> createConsumer(Properties consumerProp, RetryPolicy retryPolicy) {
+    AtomicReference<Consumer<Bytes, Bytes>> consumer = new AtomicReference<>();
     try {
       retryPolicy.attempt(() -> {
         try {
@@ -140,7 +139,7 @@ public abstract class KafkaPartitionLevelConnectionHandler {
   }
 
   @VisibleForTesting
-  protected Consumer<String, Bytes> createConsumer(Properties consumerProp) {
+  protected Consumer<Bytes, Bytes> createConsumer(Properties consumerProp) {
     return retry(() -> new KafkaConsumer<>(filterKafkaProperties(consumerProp, CONSUMER_CONFIG_NAMES)), 5);
   }
 
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumer.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumer.java
index 262b8a7996e..a91545dd6f8 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumer.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumer.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pinot.plugin.stream.kafka30;
 
-import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
@@ -71,8 +70,8 @@ public class KafkaPartitionLevelConsumer extends KafkaPartitionLevelConnectionHa
       _consumer.seek(_topicPartition, startOffset);
     }
 
-    ConsumerRecords<String, Bytes> consumerRecords = _consumer.poll(Duration.ofMillis(timeoutMs));
-    List<ConsumerRecord<String, Bytes>> records = consumerRecords.records(_topicPartition);
+    ConsumerRecords<Bytes, Bytes> consumerRecords = _consumer.poll(Duration.ofMillis(timeoutMs));
+    List<ConsumerRecord<Bytes, Bytes>> records = consumerRecords.records(_topicPartition);
     List<BytesStreamMessage> filteredRecords = new ArrayList<>(records.size());
     long firstOffset = -1;
     long offsetOfNextBatch = startOffset;
@@ -82,12 +81,12 @@ public class KafkaPartitionLevelConsumer extends KafkaPartitionLevelConnectionHa
       firstOffset = records.get(0).offset();
       _lastFetchedOffset = records.get(records.size() - 1).offset();
       offsetOfNextBatch = _lastFetchedOffset + 1;
-      for (ConsumerRecord<String, Bytes> record : records) {
+      for (ConsumerRecord<Bytes, Bytes> record : records) {
         StreamMessageMetadata messageMetadata = extractMessageMetadata(record);
         Bytes message = record.value();
         if (message != null) {
-          String key = record.key();
-          byte[] keyBytes = key != null ? key.getBytes(StandardCharsets.UTF_8) : null;
+          Bytes key = record.key();
+          byte[] keyBytes = key != null ? key.get() : null;
           filteredRecords.add(new BytesStreamMessage(keyBytes, message.get(), messageMetadata));
           if (messageMetadata.getRecordSerializedSize() > 0) {
             batchSizeInBytes += messageMetadata.getRecordSerializedSize();
@@ -111,7 +110,7 @@ public class KafkaPartitionLevelConsumer extends KafkaPartitionLevelConnectionHa
         hasDataLoss, batchSizeInBytes);
   }
 
-  private StreamMessageMetadata extractMessageMetadata(ConsumerRecord<String, Bytes> record) {
+  private StreamMessageMetadata extractMessageMetadata(ConsumerRecord<Bytes, Bytes> record) {
     long timestamp = record.timestamp();
     long offset = record.offset();
 
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumerTest.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumerTest.java
index 38423b0de4a..96afe08e070 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumerTest.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumerTest.java
@@ -520,20 +520,23 @@ public class KafkaPartitionLevelConsumerTest {
       }
 
       @Override
-      protected Consumer<String, Bytes> createConsumer(Properties consumerProp) {
-        Consumer<String, Bytes> mockConsumer = mock(Consumer.class);
+      protected Consumer<Bytes, Bytes> createConsumer(Properties consumerProp) {
+        Consumer<Bytes, Bytes> mockConsumer = mock(Consumer.class);
         // Mock records
-        ConsumerRecord<String, Bytes> record1 =
+        ConsumerRecord<Bytes, Bytes> record1 =
             new ConsumerRecord<>("test-topic", 0, 0L, NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE, NULL_CHECKSUM, 4,
-                5, "key1", new Bytes("value1".getBytes(StandardCharsets.UTF_8)));
-        ConsumerRecord<String, Bytes> record2 =
+                5, new Bytes("key1".getBytes(StandardCharsets.UTF_8)),
+                new Bytes("value1".getBytes(StandardCharsets.UTF_8)));
+        ConsumerRecord<Bytes, Bytes> record2 =
             new ConsumerRecord<>("test-topic", 0, 0L, NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE, NULL_CHECKSUM, 4,
-                9, "key2", new Bytes("value2".getBytes(StandardCharsets.UTF_8)));
-        ConsumerRecord<String, Bytes> record3 =
+                9, new Bytes("key2".getBytes(StandardCharsets.UTF_8)),
+                new Bytes("value2".getBytes(StandardCharsets.UTF_8)));
+        ConsumerRecord<Bytes, Bytes> record3 =
             new ConsumerRecord<>("test-topic", 0, 0L, NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE, NULL_CHECKSUM, 4,
-                -1, "key2", new Bytes("value2".getBytes(StandardCharsets.UTF_8)));
+                -1, new Bytes("key2".getBytes(StandardCharsets.UTF_8)),
+                new Bytes("value2".getBytes(StandardCharsets.UTF_8)));
         // Mock return of poll()
-        ConsumerRecords<String, Bytes> consumerRecords = new ConsumerRecords<>(
+        ConsumerRecords<Bytes, Bytes> consumerRecords = new ConsumerRecords<>(
             Map.of(topicPartition, List.of(record1, record2, record3))
         );
         when(mockConsumer.poll(any(Duration.class))).thenReturn(consumerRecords);
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConnectionHandler.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConnectionHandler.java
index 36c24f43771..ae5d8b753b0 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConnectionHandler.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConnectionHandler.java
@@ -36,7 +36,6 @@ import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.serialization.BytesDeserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.pinot.plugin.stream.kafka.KafkaAdminClientManager;
 import org.apache.pinot.plugin.stream.kafka.KafkaPartitionLevelStreamConfig;
@@ -63,7 +62,7 @@ public abstract class KafkaPartitionLevelConnectionHandler {
   protected final String _clientId;
   protected final int _partition;
   protected final String _topic;
-  protected final Consumer<String, Bytes> _consumer;
+  protected final Consumer<Bytes, Bytes> _consumer;
   protected final TopicPartition _topicPartition;
   protected final Properties _consumerProp;
   protected volatile KafkaAdminClientManager.AdminClientReference _sharedAdminClientRef;
@@ -93,7 +92,7 @@ public abstract class KafkaPartitionLevelConnectionHandler {
     Properties consumerProp = new Properties();
     consumerProp.putAll(streamConfig.getStreamConfigsMap());
     consumerProp.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, _config.getBootstrapHosts());
-    consumerProp.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+    consumerProp.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
     consumerProp.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
     if (_config.getKafkaIsolationLevel() != null) {
       consumerProp.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, _config.getKafkaIsolationLevel());
@@ -120,8 +119,8 @@ public abstract class KafkaPartitionLevelConnectionHandler {
     return filteredProps;
   }
 
-  private Consumer<String, Bytes> createConsumer(Properties consumerProp, RetryPolicy retryPolicy) {
-    AtomicReference<Consumer<String, Bytes>> consumer = new AtomicReference<>();
+  private Consumer<Bytes, Bytes> createConsumer(Properties consumerProp, RetryPolicy retryPolicy) {
+    AtomicReference<Consumer<Bytes, Bytes>> consumer = new AtomicReference<>();
     try {
       retryPolicy.attempt(() -> {
         try {
@@ -140,7 +139,7 @@ public abstract class KafkaPartitionLevelConnectionHandler {
   }
 
   @VisibleForTesting
-  protected Consumer<String, Bytes> createConsumer(Properties consumerProp) {
+  protected Consumer<Bytes, Bytes> createConsumer(Properties consumerProp) {
     return retry(() -> new KafkaConsumer<>(filterKafkaProperties(consumerProp, CONSUMER_CONFIG_NAMES)), 5);
   }
 
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConsumer.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConsumer.java
index 589736ef444..e5d28be8031 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConsumer.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConsumer.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pinot.plugin.stream.kafka40;
 
-import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
@@ -71,8 +70,8 @@ public class KafkaPartitionLevelConsumer extends KafkaPartitionLevelConnectionHa
       _consumer.seek(_topicPartition, startOffset);
     }
 
-    ConsumerRecords<String, Bytes> consumerRecords = _consumer.poll(Duration.ofMillis(timeoutMs));
-    List<ConsumerRecord<String, Bytes>> records = consumerRecords.records(_topicPartition);
+    ConsumerRecords<Bytes, Bytes> consumerRecords = _consumer.poll(Duration.ofMillis(timeoutMs));
+    List<ConsumerRecord<Bytes, Bytes>> records = consumerRecords.records(_topicPartition);
     List<BytesStreamMessage> filteredRecords = new ArrayList<>(records.size());
     long firstOffset = -1;
     long offsetOfNextBatch = startOffset;
@@ -82,12 +81,12 @@ public class KafkaPartitionLevelConsumer extends KafkaPartitionLevelConnectionHa
       firstOffset = records.get(0).offset();
       _lastFetchedOffset = records.get(records.size() - 1).offset();
       offsetOfNextBatch = _lastFetchedOffset + 1;
-      for (ConsumerRecord<String, Bytes> record : records) {
+      for (ConsumerRecord<Bytes, Bytes> record : records) {
         StreamMessageMetadata messageMetadata = extractMessageMetadata(record);
         Bytes message = record.value();
         if (message != null) {
-          String key = record.key();
-          byte[] keyBytes = key != null ? key.getBytes(StandardCharsets.UTF_8) : null;
+          Bytes key = record.key();
+          byte[] keyBytes = key != null ? key.get() : null;
           filteredRecords.add(new BytesStreamMessage(keyBytes, message.get(), messageMetadata));
           if (messageMetadata.getRecordSerializedSize() > 0) {
             batchSizeInBytes += messageMetadata.getRecordSerializedSize();
@@ -111,7 +110,7 @@ public class KafkaPartitionLevelConsumer extends KafkaPartitionLevelConnectionHa
         hasDataLoss, batchSizeInBytes);
   }
 
-  private StreamMessageMetadata extractMessageMetadata(ConsumerRecord<String, Bytes> record) {
+  private StreamMessageMetadata extractMessageMetadata(ConsumerRecord<Bytes, Bytes> record) {
     long timestamp = record.timestamp();
     long offset = record.offset();
 
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/test/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConsumerTest.java b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/test/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConsumerTest.java
index 34852964e6e..3584595f870 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/test/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConsumerTest.java
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/test/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConsumerTest.java
@@ -524,20 +524,23 @@ public class KafkaPartitionLevelConsumerTest {
       }
 
       @Override
-      protected Consumer<String, Bytes> createConsumer(Properties consumerProp) {
-        Consumer<String, Bytes> mockConsumer = mock(Consumer.class);
+      protected Consumer<Bytes, Bytes> createConsumer(Properties consumerProp) {
+        Consumer<Bytes, Bytes> mockConsumer = mock(Consumer.class);
         // Mock records using Kafka 4.x compatible constructor (headers cannot be null in Kafka 4.x)
-        ConsumerRecord<String, Bytes> record1 =
+        ConsumerRecord<Bytes, Bytes> record1 =
             new ConsumerRecord<>("test-topic", 0, 0L, NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE, 4,
-                5, "key1", new Bytes("value1".getBytes(StandardCharsets.UTF_8)), new RecordHeaders(), null);
-        ConsumerRecord<String, Bytes> record2 =
+                5, new Bytes("key1".getBytes(StandardCharsets.UTF_8)),
+                new Bytes("value1".getBytes(StandardCharsets.UTF_8)), new RecordHeaders(), null);
+        ConsumerRecord<Bytes, Bytes> record2 =
             new ConsumerRecord<>("test-topic", 0, 0L, NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE, 4,
-                9, "key2", new Bytes("value2".getBytes(StandardCharsets.UTF_8)), new RecordHeaders(), null);
-        ConsumerRecord<String, Bytes> record3 =
+                9, new Bytes("key2".getBytes(StandardCharsets.UTF_8)),
+                new Bytes("value2".getBytes(StandardCharsets.UTF_8)), new RecordHeaders(), null);
+        ConsumerRecord<Bytes, Bytes> record3 =
             new ConsumerRecord<>("test-topic", 0, 0L, NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE, 4,
-                -1, "key2", new Bytes("value2".getBytes(StandardCharsets.UTF_8)), new RecordHeaders(), null);
+                -1, new Bytes("key2".getBytes(StandardCharsets.UTF_8)),
+                new Bytes("value2".getBytes(StandardCharsets.UTF_8)), new RecordHeaders(), null);
         // Mock return of poll()
-        ConsumerRecords<String, Bytes> consumerRecords = new ConsumerRecords<>(
+        ConsumerRecords<Bytes, Bytes> consumerRecords = new ConsumerRecords<>(
             Map.of(topicPartition, List.of(record1, record2, record3))
         );
         when(mockConsumer.poll(any(Duration.class))).thenReturn(consumerRecords);


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org