You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ma...@apache.org on 2026/09/10 04:02:50 UTC
(iceberg) branch ci-fix-spark-release-config created (now baa09f2f46)
This is an automated email from the ASF dual-hosted git repository.
manuzhang pushed a change to branch ci-fix-spark-release-config
in repository https://gitbox.apache.org/repos/asf/iceberg.git
at baa09f2f46 Build: Fix Spark publication and CI configuration
This branch includes the following new commits:
new baa09f2f46 Build: Fix Spark publication and CI configuration
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iceberg) 01/01: Build: Fix Spark publication and CI configuration
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
manuzhang pushed a commit to branch ci-fix-spark-release-config
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit baa09f2f461c7333c9c94fc3152f86bd776f1563
Author: Manu Zhang <ow...@gmail.com>
AuthorDate: Thu Sep 10 13:02:26 2026 +0900
Build: Fix Spark publication and CI configuration
Avoid republishing unrelated snapshot artifacts and preserve complete release BOM constraints. Correct stale benchmark workflow filters and clarify the supported Spark matrix.
Generated-by: Codex
---
.github/workflows/delta-conversion-ci.yml | 2 +-
.github/workflows/flink-ci.yml | 2 +-
.github/workflows/hive-ci.yml | 2 +-
.github/workflows/java-ci.yml | 2 +-
.github/workflows/kafka-connect-ci.yml | 2 +-
.github/workflows/publish-snapshot.yml | 6 +++++-
.github/workflows/spark-ci.yml | 7 ++++---
build.gradle | 3 +++
dev/stage-binaries.sh | 2 +-
9 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml
index 639d48837b..1a91e1bdf2 100644
--- a/.github/workflows/delta-conversion-ci.yml
+++ b/.github/workflows/delta-conversion-ci.yml
@@ -35,7 +35,7 @@ on:
- '.github/workflows/flink-ci.yml'
- '.github/workflows/hive-ci.yml'
- '.github/workflows/java-ci.yml'
- - '.github/workflows/jmh-benchmarks-ci.yml'
+ - '.github/workflows/jmh-benchmarks.yml'
- '.github/workflows/kafka-connect-ci.yml'
- '.github/workflows/cve-scan.yml'
- '.github/workflows/labeler.yml'
diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml
index ae4b8554cd..86d2ef6c46 100644
--- a/.github/workflows/flink-ci.yml
+++ b/.github/workflows/flink-ci.yml
@@ -35,7 +35,7 @@ on:
- '.github/workflows/docs-ci.yml'
- '.github/workflows/hive-ci.yml'
- '.github/workflows/java-ci.yml'
- - '.github/workflows/jmh-benchmarks-ci.yml'
+ - '.github/workflows/jmh-benchmarks.yml'
- '.github/workflows/kafka-connect-ci.yml'
- '.github/workflows/cve-scan.yml'
- '.github/workflows/labeler.yml'
diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml
index 592de5a211..046d346f4a 100644
--- a/.github/workflows/hive-ci.yml
+++ b/.github/workflows/hive-ci.yml
@@ -35,7 +35,7 @@ on:
- '.github/workflows/docs-ci.yml'
- '.github/workflows/flink-ci.yml'
- '.github/workflows/java-ci.yml'
- - '.github/workflows/jmh-benchmarks-ci.yml'
+ - '.github/workflows/jmh-benchmarks.yml'
- '.github/workflows/kafka-connect-ci.yml'
- '.github/workflows/cve-scan.yml'
- '.github/workflows/labeler.yml'
diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml
index 850bd4b961..bfe0aaa454 100644
--- a/.github/workflows/java-ci.yml
+++ b/.github/workflows/java-ci.yml
@@ -35,7 +35,7 @@ on:
- '.github/workflows/docs-ci.yml'
- '.github/workflows/flink-ci.yml'
- '.github/workflows/hive-ci.yml'
- - '.github/workflows/jmh-benchmarks-ci.yml'
+ - '.github/workflows/jmh-benchmarks.yml'
- '.github/workflows/kafka-connect-ci.yml'
- '.github/workflows/cve-scan.yml'
- '.github/workflows/labeler.yml'
diff --git a/.github/workflows/kafka-connect-ci.yml b/.github/workflows/kafka-connect-ci.yml
index c1318fb4b9..3506e41208 100644
--- a/.github/workflows/kafka-connect-ci.yml
+++ b/.github/workflows/kafka-connect-ci.yml
@@ -36,7 +36,7 @@ on:
- '.github/workflows/flink-ci.yml'
- '.github/workflows/hive-ci.yml'
- '.github/workflows/java-ci.yml'
- - '.github/workflows/jmh-benchmarks-ci.yml'
+ - '.github/workflows/jmh-benchmarks.yml'
- '.github/workflows/labeler.yml'
- '.github/workflows/license-check.yml'
- '.github/workflows/open-api.yml'
diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml
index e9b06b9ffb..b97d6ffe43 100644
--- a/.github/workflows/publish-snapshot.yml
+++ b/.github/workflows/publish-snapshot.yml
@@ -53,4 +53,8 @@ jobs:
run: |
./gradlew printVersion
./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser="$NEXUS_USER" -PmavenPassword="$NEXUS_PW"
- ./gradlew -DflinkVersions= -DsparkVersions=3.5,4.0 -DscalaVersion=2.13 -DkafkaVersions=3 publishApachePublicationToMavenRepository -PmavenUser="$NEXUS_USER" -PmavenPassword="$NEXUS_PW"
+ ./gradlew -DflinkVersions= -DsparkVersions=3.5 -DscalaVersion=2.13 -DkafkaVersions= \
+ :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository \
+ :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository \
+ :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository \
+ -PmavenUser="$NEXUS_USER" -PmavenPassword="$NEXUS_PW"
diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml
index f23ad33478..52e28184e4 100644
--- a/.github/workflows/spark-ci.yml
+++ b/.github/workflows/spark-ci.yml
@@ -36,7 +36,7 @@ on:
- '.github/workflows/flink-ci.yml'
- '.github/workflows/hive-ci.yml'
- '.github/workflows/java-ci.yml'
- - '.github/workflows/jmh-benchmarks-ci.yml'
+ - '.github/workflows/jmh-benchmarks.yml'
- '.github/workflows/kafka-connect-ci.yml'
- '.github/workflows/cve-scan.yml'
- '.github/workflows/labeler.yml'
@@ -81,6 +81,8 @@ jobs:
# Keep matrix <= 20 jobs; exceeding it queues a second wave and slows CI.
max-parallel: 20
matrix:
+ # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
+ # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831)
jvm: [17, 21]
event_name: ['${{ github.event_name }}']
spark: ['3.5', '4.0', '4.1', '4.2']
@@ -89,8 +91,7 @@ jobs:
# so they run as concurrent jobs rather than serially in one job.
tests: [core, extensions]
exclude:
- # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
- # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831)
+ # Spark 4.0+ only supports Scala 2.13.
- spark: '4.0'
scala: '2.12'
- spark: '4.1'
diff --git a/build.gradle b/build.gradle
index f98d46bc59..38bdb467c6 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1258,8 +1258,11 @@ project(':iceberg-bom') {
// only contain artifacts for that single Scala version. The following code ensures that
// the BOM references the artifacts for all Scala versions.
def sparkScalaPattern = ~"(.*)-([0-9][.][0-9]+)_([0-9][.][0-9]+)"
+ // Only include Spark versions published in binary releases.
def sparkScalaVersions = [
"3.5": ["2.12", "2.13"],
+ "4.0": ["2.13"],
+ "4.1": ["2.13"],
]
rootProject.allprojects.forEach {
// Do not include ':iceberg-spark', the bom itself and the root project.
diff --git a/dev/stage-binaries.sh b/dev/stage-binaries.sh
index bf8576d95f..fedbee2f1c 100755
--- a/dev/stage-binaries.sh
+++ b/dev/stage-binaries.sh
@@ -20,6 +20,7 @@
SCALA_VERSION=2.12
FLINK_VERSIONS=1.20,2.1,2.2,2.3
+# Spark 4.2 is intentionally excluded because it is not part of binary releases.
SPARK_VERSIONS=3.5,4.0,4.1
KAFKA_VERSIONS=3
@@ -29,4 +30,3 @@ KAFKA_VERSIONS=3
# Flink does not yet support 2.13 (and is largely dropping a user-facing dependency on Scala). Hive doesn't need a Scala specification.
./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.5 :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache
# Spark 4.0+ only supports Scala 2.13. no need to specify scalaVersion
-