You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/11/28 20:57:44 UTC
[spark] branch master updated (60ab363183b -> 0ff201cc219)
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
from 60ab363183b [SPARK-41197][BUILD] Upgrade Kafka to 3.3.1
new 8e3e9575099 Revert "[SPARK-41197][BUILD] Upgrade Kafka to 3.3.1"
new 0ff201cc219 [SPARK-41197][BUILD] Upgrade Kafka to 3.3.1
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[spark] 02/02: [SPARK-41197][BUILD] Upgrade Kafka to 3.3.1
Posted by do...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
commit 0ff201cc219884d3cbb6844732c681546e53f4d4
Author: Ted Yu <yu...@gmail.com>
AuthorDate: Mon Nov 28 12:49:43 2022 -0800
[SPARK-41197][BUILD] Upgrade Kafka to 3.3.1
This PR upgrades Kafka to 3.3.1 release.
The new default partitioner keeps track of how many bytes are produced per-partition and once the amount exceeds `batch.size`, it switches to the next partition. For spark kafka tests, this will result in records being sent to only one partition in some tests.
`KafkaTestUtils.producerConfiguration` is modified to use `DefaultPartitioner`.
Kafka 3.3.1 release has new features along with bug fixes: https://www.confluent.io/blog/apache-kafka-3-3-0-new-features-and-updates/
No
Existing test suite
Closes #38715 from tedyu/k-33.
Authored-by: Ted Yu <yu...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala | 2 ++
.../test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala | 2 ++
pom.xml | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 431d9d6b278..7c9c40883a5 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -537,6 +537,8 @@ class KafkaTestUtils(
props.put("key.serializer", classOf[StringSerializer].getName)
// wait for all in-sync replicas to ack sends
props.put("acks", "all")
+ props.put("partitioner.class",
+ classOf[org.apache.kafka.clients.producer.internals.DefaultPartitioner].getName)
setAuthenticationConfigIfNeeded(props)
props
}
diff --git a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
index d341b6977b2..91fecacb6e7 100644
--- a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
+++ b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
@@ -263,6 +263,8 @@ private[kafka010] class KafkaTestUtils extends Logging {
props.put("key.serializer", classOf[StringSerializer].getName)
// wait for all in-sync replicas to ack sends
props.put("acks", "all")
+ props.put("partitioner.class",
+ classOf[org.apache.kafka.clients.producer.internals.DefaultPartitioner].getName)
props
}
diff --git a/pom.xml b/pom.xml
index f47bedb18e7..691ea8f563f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,7 +133,7 @@
<!-- Version used for internal directory structure -->
<hive.version.short>2.3</hive.version.short>
<!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
- <kafka.version>3.2.3</kafka.version>
+ <kafka.version>3.3.1</kafka.version>
<!-- After 10.15.1.3, the minimum required version is JDK9 -->
<derby.version>10.14.2.0</derby.version>
<parquet.version>1.12.3</parquet.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[spark] 01/02: Revert "[SPARK-41197][BUILD] Upgrade Kafka to 3.3.1"
Posted by do...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
commit 8e3e95750993277568010df9a53f56104ced7304
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Mon Nov 28 12:56:30 2022 -0800
Revert "[SPARK-41197][BUILD] Upgrade Kafka to 3.3.1"
This reverts commit 60ab363183b8b9565ebadfd1b1e826d1d64ae212.
---
.../src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala | 2 --
.../test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala | 2 --
pom.xml | 2 +-
3 files changed, 1 insertion(+), 5 deletions(-)
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 7c9c40883a5..431d9d6b278 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -537,8 +537,6 @@ class KafkaTestUtils(
props.put("key.serializer", classOf[StringSerializer].getName)
// wait for all in-sync replicas to ack sends
props.put("acks", "all")
- props.put("partitioner.class",
- classOf[org.apache.kafka.clients.producer.internals.DefaultPartitioner].getName)
setAuthenticationConfigIfNeeded(props)
props
}
diff --git a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
index 91fecacb6e7..d341b6977b2 100644
--- a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
+++ b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
@@ -263,8 +263,6 @@ private[kafka010] class KafkaTestUtils extends Logging {
props.put("key.serializer", classOf[StringSerializer].getName)
// wait for all in-sync replicas to ack sends
props.put("acks", "all")
- props.put("partitioner.class",
- classOf[org.apache.kafka.clients.producer.internals.DefaultPartitioner].getName)
props
}
diff --git a/pom.xml b/pom.xml
index 691ea8f563f..f47bedb18e7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,7 +133,7 @@
<!-- Version used for internal directory structure -->
<hive.version.short>2.3</hive.version.short>
<!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
- <kafka.version>3.3.1</kafka.version>
+ <kafka.version>3.2.3</kafka.version>
<!-- After 10.15.1.3, the minimum required version is JDK9 -->
<derby.version>10.14.2.0</derby.version>
<parquet.version>1.12.3</parquet.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org