You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2026/07/08 02:22:12 UTC
(kyuubi) branch master updated: [KYUUBI #7543] Bump Spark 3.5.8, 4.0.3, 4.1.2
This is an automated email from the ASF dual-hosted git repository.
pan3793 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 016db7f3b0 [KYUUBI #7543] Bump Spark 3.5.8, 4.0.3, 4.1.2
016db7f3b0 is described below
commit 016db7f3b05bd94e98da2ee112536ac0babbf464
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Wed Jul 8 10:22:03 2026 +0800
[KYUUBI #7543] Bump Spark 3.5.8, 4.0.3, 4.1.2
### Why are the changes needed?
Bump Spark to the latest patched versions.
- Spark 3.5.7 to 3.5.8
- Spark 4.0.1 to 4.0.3
- Spark 4.1.1 to 4.1.2
### How was this patch tested?
Pass GHA.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #7543 from pan3793/spark-upgrade.
Closes #7543
04b1f303c [Cheng Pan] [KYUUBI] Bump Spark versions
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.github/workflows/master.yml | 10 +++++-----
bin/docker-image-tool.sh | 4 ++--
docker/playground/.env | 2 +-
docs/deployment/kyuubi_on_kubernetes.md | 4 ++--
docs/extensions/engines/spark/lineage.md | 2 +-
.../test/deployment/KyuubiOnKubernetesTestsSuite.scala | 2 +-
.../kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala | 2 +-
pom.xml | 8 ++++----
8 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 37966c80a1..7af7e041b1 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -89,13 +89,13 @@ jobs:
- java: 17
python: '3.11'
spark: '3.5'
- spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.0.1 -Dspark.archive.name=spark-4.0.1-bin-hadoop3.tgz'
+ spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.0.3 -Dspark.archive.name=spark-4.0.3-bin-hadoop3.tgz'
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-4.0-binary'
- java: 17
python: '3.11'
spark: '3.5'
- spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.1.1 -Dspark.archive.name=spark-4.1.1-bin-hadoop3.tgz'
+ spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.1.2 -Dspark.archive.name=spark-4.1.2-bin-hadoop3.tgz'
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-4.1-binary'
env:
@@ -447,14 +447,14 @@ jobs:
cache-binary: false
- name: Pull Spark image
run: |
- docker pull apache/spark:3.5.7
+ docker pull apache/spark:3.5.8
- name: Build Kyuubi Docker Image
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
# passthrough CI into build container
build-args: |
CI=${CI}
- BASE_IMAGE=apache/spark:3.5.7
+ BASE_IMAGE=apache/spark:3.5.8
MVN_ARG=--spark-provided --flink-provided --hive-provided
context: .
file: build/Dockerfile.CI
@@ -471,7 +471,7 @@ jobs:
# https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
minikube image load apache/kyuubi:ci
# pre-install spark into minikube
- minikube image load apache/spark:3.5.7
+ minikube image load apache/spark:3.5.8
- name: kubectl pre-check
run: |
kubectl get nodes
diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
index f4684ab09e..69c882f27c 100755
--- a/bin/docker-image-tool.sh
+++ b/bin/docker-image-tool.sh
@@ -181,8 +181,8 @@ Examples:
$0 -r docker.io/myrepo -t v1.8.1 build
$0 -r docker.io/myrepo -t v1.8.1 push
- - Build and push with tag "v1.8.1" and Spark-3.5.7 as base image to docker.io/myrepo
- $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.7 build
+ - Build and push with tag "v1.8.1" and Spark-3.5.8 as base image to docker.io/myrepo
+ $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.8 build
$0 -r docker.io/myrepo -t v1.8.1 push
- Build and push for multiple archs to docker.io/myrepo
diff --git a/docker/playground/.env b/docker/playground/.env
index 435e585392..3f39d1dbd7 100644
--- a/docker/playground/.env
+++ b/docker/playground/.env
@@ -24,7 +24,7 @@ KYUUBI_HADOOP_VERSION=3.3.6
POSTGRES_VERSION=12
POSTGRES_JDBC_VERSION=42.3.4
SCALA_BINARY_VERSION=2.12
-SPARK_VERSION=3.5.7
+SPARK_VERSION=3.5.8
SPARK_BINARY_VERSION=3.5
SPARK_HADOOP_VERSION=3.3.4
ZOOKEEPER_VERSION=3.6.3
diff --git a/docs/deployment/kyuubi_on_kubernetes.md b/docs/deployment/kyuubi_on_kubernetes.md
index a853a922c9..0453d64e94 100644
--- a/docs/deployment/kyuubi_on_kubernetes.md
+++ b/docs/deployment/kyuubi_on_kubernetes.md
@@ -42,8 +42,8 @@ Examples:
$0 -r docker.io/myrepo -t v1.8.1 build
$0 -r docker.io/myrepo -t v1.8.1 push
- - Build and push with tag "v1.8.1" and Spark-3.5.7 as base image to docker.io/myrepo
- $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.7 build
+ - Build and push with tag "v1.8.1" and Spark-3.5.8 as base image to docker.io/myrepo
+ $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.8 build
$0 -r docker.io/myrepo -t v1.8.1 push
- Build and push for multiple archs to docker.io/myrepo
diff --git a/docs/extensions/engines/spark/lineage.md b/docs/extensions/engines/spark/lineage.md
index 083ca37111..73b9f6af05 100644
--- a/docs/extensions/engines/spark/lineage.md
+++ b/docs/extensions/engines/spark/lineage.md
@@ -117,7 +117,7 @@ Sometimes, it may be incompatible with other Spark distributions, then you may n
For example,
```shell
-build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.5.7
+build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.5.8
```
The available `spark.version`s are shown in the following table.
diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
index 9313494c6a..cf04cd2ce4 100644
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
@@ -56,7 +56,7 @@ class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes
Map(
"spark.master" -> s"k8s://$miniKubeApiMaster",
// We should update spark docker image in ./github/workflows/master.yml at the same time
- "spark.kubernetes.container.image" -> "apache/spark:3.5.7",
+ "spark.kubernetes.container.image" -> "apache/spark:3.5.8",
"spark.kubernetes.container.image.pullPolicy" -> "IfNotPresent",
"spark.executor.memory" -> "512M",
"spark.driver.memory" -> "1024M",
diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
index 092e799617..b31aa2e31f 100644
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
@@ -51,7 +51,7 @@ abstract class SparkOnKubernetesSuiteBase
// TODO Support more Spark version
// Spark official docker image: https://hub.docker.com/r/apache/spark/tags
KyuubiConf().set("spark.master", s"k8s://$apiServerAddress")
- .set("spark.kubernetes.container.image", "apache/spark:3.5.7")
+ .set("spark.kubernetes.container.image", "apache/spark:3.5.8")
.set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent")
.set("spark.executor.instances", "1")
.set("spark.executor.memory", "512M")
diff --git a/pom.xml b/pom.xml
index 98835abba5..917ee8fb6f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -210,7 +210,7 @@
DO NOT forget to change the following properties when change the minor version of Spark:
`delta.version`, `delta.artifact`, `maven.plugin.scalatest.exclude.tags`
-->
- <spark.version>3.5.7</spark.version>
+ <spark.version>3.5.8</spark.version>
<spark.binary.version>3.5</spark.binary.version>
<spark.archive.scala.suffix></spark.archive.scala.suffix>
<spark.archive.name>spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz</spark.archive.name>
@@ -2057,7 +2057,7 @@
<module>extensions/spark/kyuubi-spark-connector-hive</module>
</modules>
<properties>
- <spark.version>3.5.7</spark.version>
+ <spark.version>3.5.8</spark.version>
<spark.binary.version>3.5</spark.binary.version>
<delta.version>3.3.1</delta.version>
<delta.artifact>delta-spark_${scala.binary.version}</delta.artifact>
@@ -2073,7 +2073,7 @@
</modules>
<properties>
<enforcer.maxJdkVersion>17</enforcer.maxJdkVersion>
- <spark.version>4.0.1</spark.version>
+ <spark.version>4.0.3</spark.version>
<spark.binary.version>4.0</spark.binary.version>
<antlr4.version>4.13.1</antlr4.version>
<delta.version>4.0.0</delta.version>
@@ -2098,7 +2098,7 @@
<properties>
<maven.compiler.release>17</maven.compiler.release>
<enforcer.maxJdkVersion>17</enforcer.maxJdkVersion>
- <spark.version>4.1.1</spark.version>
+ <spark.version>4.1.2</spark.version>
<spark.binary.version>4.0</spark.binary.version>
<antlr4.version>4.13.1</antlr4.version>
<delta.version>4.0.0</delta.version>