[spark] branch master updated: [SPARK-41197][BUILD] Upgrade Kafka to 3.3.1

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 60ab363183b [SPARK-41197][BUILD] Upgrade Kafka to 3.3.1
60ab363183b is described below

commit 60ab363183b8b9565ebadfd1b1e826d1d64ae212
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Mon Nov 28 12:49:43 2022 -0800

    [SPARK-41197][BUILD] Upgrade Kafka to 3.3.1
    
    ### What changes were proposed in this pull request?
    This PR upgrades Kafka to 3.3.1 release.
    
    The new default partitioner keeps track of how many bytes are produced per-partition and once the amount exceeds `batch.size`, it switches to the next partition. For spark kafka tests, this will result in records being sent to only one partition in some tests.
    `KafkaTestUtils.producerConfiguration` is modified to use `DefaultPartitioner`.
    
    ### Why are the changes needed?
    Kafka 3.3.1 release has new features along with bug fixes: https://www.confluent.io/blog/apache-kafka-3-3-0-new-features-and-updates/
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    Existing test suite
    
    Closes #38715 from tedyu/k-33.
    
    Lead-authored-by: Dongjoon Hyun <do...@apache.org>
    Co-authored-by: Ted Yu <yu...@gmail.com>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala   | 2 ++
 .../test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala | 2 ++
 pom.xml                                                                 | 2 +-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 431d9d6b278..7c9c40883a5 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -537,6 +537,8 @@ class KafkaTestUtils(
     props.put("key.serializer", classOf[StringSerializer].getName)
     // wait for all in-sync replicas to ack sends
     props.put("acks", "all")
+    props.put("partitioner.class",
+      classOf[org.apache.kafka.clients.producer.internals.DefaultPartitioner].getName)
     setAuthenticationConfigIfNeeded(props)
     props
   }
diff --git a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
index d341b6977b2..91fecacb6e7 100644
--- a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
+++ b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
@@ -263,6 +263,8 @@ private[kafka010] class KafkaTestUtils extends Logging {
     props.put("key.serializer", classOf[StringSerializer].getName)
     // wait for all in-sync replicas to ack sends
     props.put("acks", "all")
+    props.put("partitioner.class",
+      classOf[org.apache.kafka.clients.producer.internals.DefaultPartitioner].getName)
     props
   }
 
diff --git a/pom.xml b/pom.xml
index f47bedb18e7..691ea8f563f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,7 +133,7 @@
     <!-- Version used for internal directory structure -->
     <hive.version.short>2.3</hive.version.short>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
-    <kafka.version>3.2.3</kafka.version>
+    <kafka.version>3.3.1</kafka.version>
     <!-- After 10.15.1.3, the minimum required version is JDK9 -->
     <derby.version>10.14.2.0</derby.version>
     <parquet.version>1.12.3</parquet.version>


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org