You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/09/28 04:46:04 UTC
(datafusion-comet) branch branch-1.0 updated: fix: build the Spark 3.4 and 3.5 jars for Java 11 on any JDK (#6285)
This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new ff8b9d9d2d fix: build the Spark 3.4 and 3.5 jars for Java 11 on any JDK (#6285)
ff8b9d9d2d is described below
commit ff8b9d9d2d379f61cefa3c66f1c6a0a6dfe03f83
Author: Andy Grove <ag...@apache.org>
AuthorDate: Sun Sep 27 22:45:58 2026 -0600
fix: build the Spark 3.4 and 3.5 jars for Java 11 on any JDK (#6285)
---
dev/release/build-release-comet.sh | 4 ++
dev/release/check-class-versions.py | 63 ++++++++++++++++++++++++++++
docs/source/contributor-guide/development.md | 4 +-
pom.xml | 29 ++-----------
4 files changed, 72 insertions(+), 28 deletions(-)
diff --git a/dev/release/build-release-comet.sh b/dev/release/build-release-comet.sh
index 159394e0f5..ac63a3d382 100755
--- a/dev/release/build-release-comet.sh
+++ b/dev/release/build-release-comet.sh
@@ -209,6 +209,10 @@ LOCAL_REPO=$(mktemp -d /tmp/comet-staging-repo-XXXXX)
./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.4 -P scala-2.13 -DskipTests install
./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.5 -P scala-2.12 -DskipTests install
./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.5 -P scala-2.13 -DskipTests install
+# Spark 3.4 and 3.5 are supported on Java 11, so nothing in their jars may need a newer JVM.
+# Releases 0.11.0 to 1.0.0 shipped these jars built for Java 17 (#6283).
+python3 "$SCRIPT_DIR/check-class-versions.py" --max 55 \
+ "${LOCAL_REPO}"/org/apache/datafusion/*-spark3.[45]_*/*/*.jar
# The spark-4.x profiles pin their own Scala 2.13.x patch versions to match the
# corresponding Spark release, so the scala-2.13 profile is not used here.
./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-4.0 -DskipTests install
diff --git a/dev/release/check-class-versions.py b/dev/release/check-class-versions.py
new file mode 100644
index 0000000000..c73e840ef4
--- /dev/null
+++ b/dev/release/check-class-versions.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+##############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##############################################################################
+
+# Fails if any class in the given jars, shaded dependencies included, has a
+# class file version newer than --max (55 is Java 11, 61 is Java 17). Entries
+# under META-INF/versions/ are skipped, because the JVM only loads a
+# multi-release entry on the Java version it is for.
+
+import argparse
+import struct
+import sys
+import zipfile
+
+
+def versions_above(jar, limit):
+ """Returns {class file version: first class seen} for versions above limit."""
+ found = {}
+ with zipfile.ZipFile(jar) as z:
+ for name in z.namelist():
+ if not name.endswith(".class") or name.startswith("META-INF/versions/"):
+ continue
+ with z.open(name) as f:
+ major = struct.unpack(">H", f.read(8)[6:8])[0]
+ if major > limit:
+ found.setdefault(major, name)
+ return found
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--max", type=int, required=True,
+ help="newest class file version allowed, e.g. 55 for Java 11")
+ parser.add_argument("jars", nargs="+")
+ args = parser.parse_args()
+ failed = False
+ for jar in args.jars:
+ for major, name in sorted(versions_above(jar, args.max).items()):
+ print(f"{jar}: {name} has class file version {major}, newer than {args.max}")
+ failed = True
+ if failed:
+ sys.exit(1)
+ print(f"Checked {len(args.jars)} jars: no class file version is newer than {args.max}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md
index 51a46ab3b9..ab17b3f295 100644
--- a/docs/source/contributor-guide/development.md
+++ b/docs/source/contributor-guide/development.md
@@ -413,9 +413,7 @@ from a clean IntelliJ configuration:
```
The `spark-4.0` profile sets Scala 2.13 and Java 17 properties. If you need to be explicit, use
- `PROFILES="-Pspark-4.0 -Pscala-2.13 -Pjdk17" make release`.
-
- The Maven profile is named `jdk17` in this project.
+ `PROFILES="-Pspark-4.0 -Pscala-2.13" make release`.
If the native build previously used a different JDK, clear Cargo's cached JNI link path before
rebuilding:
diff --git a/pom.xml b/pom.xml
index 6178e82cd2..fe8f86f78a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -43,7 +43,10 @@ under the License.
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <java.version>11</java.version>
+ <!-- Each Spark profile sets the Java version its jars target, whatever JDK runs the build, so
+ the Spark 3.x jars run on Java 11 even when built on JDK 17. This default is for the
+ default Spark version, 4.1. -->
+ <java.version>17</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
@@ -770,30 +773,6 @@ under the License.
</properties>
</profile>
- <profile>
- <id>jdk11</id>
- <activation>
- <jdk>11</jdk>
- </activation>
- <properties>
- <java.version>11</java.version>
- <maven.compiler.source>${java.version}</maven.compiler.source>
- <maven.compiler.target>${java.version}</maven.compiler.target>
- </properties>
- </profile>
-
- <profile>
- <id>jdk17</id>
- <activation>
- <jdk>17</jdk>
- </activation>
- <properties>
- <java.version>17</java.version>
- <maven.compiler.source>${java.version}</maven.compiler.source>
- <maven.compiler.target>${java.version}</maven.compiler.target>
- </properties>
- </profile>
-
<profile>
<id>semanticdb</id>
<properties>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org