You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2025/03/27 08:44:34 UTC
(spark) branch branch-4.0 updated: [SPARK-51623][BUILD] Remove class files in source releases
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new ff8f55948cae [SPARK-51623][BUILD] Remove class files in source releases
ff8f55948cae is described below
commit ff8f55948cae36676b9aa0e0a733c5ecb67aec05
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Thu Mar 27 17:42:59 2025 +0900
[SPARK-51623][BUILD] Remove class files in source releases
This PR proposes to remove test class files in source releases during release process.
Apache source releases must not contained class files.
The issue is discussed on https://lists.apache.org/thread/0ro5yn6lbbpmvmqp2px3s2pf7cwljlc4
To end users, no.
Will be tested in CI.
No.
Closes #50422 from HyukjinKwon/SPARK-51623.
Lead-authored-by: Hyukjin Kwon <gu...@apache.org>
Co-authored-by: Hyukjin Kwon <gu...@gmail.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit 0528ac76144a0a56a48c238582c6dc100a04e00b)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
dev/create-release/release-tag.sh | 3 ++-
dev/is-changed.py | 29 +++++++++++++++++++++
dev/test-classes.txt | 8 ++++++
.../scala/org/apache/spark/repl/ReplSuite.scala | 2 ++
.../spark/sql/connect/client/ArtifactSuite.scala | 21 ++++++++++-----
.../sql/connect/client/ClassFinderSuite.scala | 1 +
.../connect/client/SparkConnectClientSuite.scala | 4 ++-
.../connect/service/AddArtifactsHandlerSuite.scala | 11 ++++----
.../spark/sql/artifact/ArtifactManagerSuite.scala | 30 ++++++++++++++++++----
9 files changed, 90 insertions(+), 19 deletions(-)
diff --git a/dev/create-release/release-tag.sh b/dev/create-release/release-tag.sh
index 8d3ea88796a4..d53b3224ba6a 100755
--- a/dev/create-release/release-tag.sh
+++ b/dev/create-release/release-tag.sh
@@ -75,7 +75,8 @@ git config user.email "$GIT_EMAIL"
# Remove test jars that do not belong to source releases.
rm $(<dev/test-jars.txt)
-git commit -a -m "Removing test jars"
+rm $(<dev/test-classes.txt)
+git commit -a -m "Removing test jars and class files"
JAR_RM_REF=$(git rev-parse HEAD)
# Create release version
diff --git a/dev/is-changed.py b/dev/is-changed.py
index 1962e244d5dd..4c3adf691327 100755
--- a/dev/is-changed.py
+++ b/dev/is-changed.py
@@ -65,6 +65,35 @@ def main():
changed_files = identify_changed_files_from_git_commits(
os.environ["GITHUB_SHA"], target_ref=os.environ["GITHUB_PREV_SHA"]
)
+
+ if any(f.endswith(".jar") for f in changed_files):
+ with open(
+ os.path.join(os.path.dirname(os.path.realpath(__file__)), "test-jars.txt")
+ ) as jarlist:
+ itrsect = set((line.strip() for line in jarlist.readlines())).intersection(
+ set(changed_files)
+ )
+ if len(itrsect) > 0:
+ raise SystemExit(
+ f"Cannot include jars in source codes ({', '.join(itrsect)}). "
+ "If they have to be added temporarily, "
+ "please add the file name into dev/test-jars.txt."
+ )
+
+ if any(f.endswith(".class") for f in changed_files):
+ with open(
+ os.path.join(os.path.dirname(os.path.realpath(__file__)), "test-classes.txt")
+ ) as clslist:
+ itrsect = set((line.strip() for line in clslist.readlines())).intersection(
+ set(changed_files)
+ )
+ if len(itrsect) > 0:
+ raise SystemExit(
+ f"Cannot include class files in source codes ({', '.join(itrsect)}). "
+ "If they have to be added temporarily, "
+ "please add the file name into dev/test-classes.txt."
+ )
+
changed_modules = determine_modules_to_test(
determine_modules_for_files(changed_files), deduplicated=False
)
diff --git a/dev/test-classes.txt b/dev/test-classes.txt
new file mode 100644
index 000000000000..5315c970c5ba
--- /dev/null
+++ b/dev/test-classes.txt
@@ -0,0 +1,8 @@
+repl/src/test/resources/IntSumUdf.class
+sql/core/src/test/resources/artifact-tests/Hello.class
+sql/core/src/test/resources/artifact-tests/IntSumUdf.class
+sql/core/src/test/resources/artifact-tests/smallClassFile.class
+sql/connect/common/src/test/resources/artifact-tests/Hello.class
+sql/core/src/test/resources/artifact-tests/HelloWithPackage.class
+sql/connect/common/src/test/resources/artifact-tests/smallClassFile.class
+sql/connect/common/src/test/resources/artifact-tests/smallClassFileDup.class
diff --git a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
index 8969bc8b5e2b..9bf716e52fff 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
@@ -400,6 +400,7 @@ class ReplSuite extends SparkFunSuite {
test("register UDF via SparkSession.addArtifact") {
val artifactPath = new File("src/test/resources").toPath
val intSumUdfPath = artifactPath.resolve("IntSumUdf.class")
+ assume(intSumUdfPath.toFile.exists)
val output = runInterpreterInPasteMode("local",
s"""
|import org.apache.spark.sql.api.java.UDF2
@@ -438,6 +439,7 @@ class ReplSuite extends SparkFunSuite {
test("register a class via SparkSession.addArtifact") {
val artifactPath = new File("src/test/resources").toPath
val intSumUdfPath = artifactPath.resolve("IntSumUdf.class")
+ assume(intSumUdfPath.toFile.exists)
val output = runInterpreterInPasteMode("local",
s"""
|import org.apache.spark.sql.functions.udf
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala
index 4cb43d6623fc..dcf3b91fece2 100644
--- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala
+++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala
@@ -199,10 +199,12 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
}
test("Batched SingleChunkArtifacts") {
- val file1 = artifactFilePath.resolve("smallClassFile.class").toUri
- val path = artifactFilePath.resolve("smallJar.jar")
- val file2 = path.toUri
- assume(path.toFile.exists)
+ val path1 = artifactFilePath.resolve("smallClassFile.class")
+ assume(path1.toFile.exists)
+ val file1 = path1.toUri
+ val path2 = artifactFilePath.resolve("smallJar.jar")
+ assume(path2.toFile.exists)
+ val file2 = path2.toUri
artifactManager.addArtifacts(Seq(file1, file2))
val receivedRequests = service.getAndClearLatestAddArtifactRequests()
// Single request containing 2 artifacts.
@@ -223,11 +225,15 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
}
test("Mix of SingleChunkArtifact and chunked artifact") {
- val file1 = artifactFilePath.resolve("smallClassFile.class").toUri
+ val path1 = artifactFilePath.resolve("smallClassFile.class")
+ assume(path1.toFile.exists)
+ val file1 = path1.toUri
val path2 = artifactFilePath.resolve("junitLargeJar.jar")
assume(path2.toFile.exists)
val file2 = path2.toUri
- val file3 = artifactFilePath.resolve("smallClassFileDup.class").toUri
+ val path3 = artifactFilePath.resolve("smallClassFileDup.class")
+ assume(path3.toFile.exists)
+ val file3 = path3.toUri
val path4 = artifactFilePath.resolve("smallJar.jar")
assume(path4.toFile.exists)
val file4 = path4.toUri
@@ -298,6 +304,7 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
test("artifact with custom target") {
val artifactPath = artifactFilePath.resolve("smallClassFile.class")
+ assume(artifactPath.toFile.exists)
val target = "sub/package/smallClassFile.class"
artifactManager.addArtifact(artifactPath.toString, target)
val receivedRequests = service.getAndClearLatestAddArtifactRequests()
@@ -318,6 +325,7 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
test("in-memory artifact with custom target") {
val artifactPath = artifactFilePath.resolve("smallClassFile.class")
+ assume(artifactPath.toFile.exists)
val artifactBytes = Files.readAllBytes(artifactPath)
val target = "sub/package/smallClassFile.class"
artifactManager.addArtifact(artifactBytes, target)
@@ -341,6 +349,7 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
"When both source and target paths are given, extension conditions are checked " +
"on target path") {
val artifactPath = artifactFilePath.resolve("smallClassFile.class")
+ assume(artifactPath.toFile.exists)
assertThrows[UnsupportedOperationException] {
artifactManager.addArtifact(artifactPath.toString, "dummy.extension")
}
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ClassFinderSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ClassFinderSuite.scala
index 2f8332878bbf..6de37827967f 100644
--- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ClassFinderSuite.scala
+++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ClassFinderSuite.scala
@@ -37,6 +37,7 @@ class ClassFinderSuite extends ConnectFunSuite {
"Hello.class",
"smallClassFile.class",
"smallClassFileDup.class") ++ additionalClasses).map(name => Paths.get(name))
+ expectedClassFiles.foreach(p => assume(p.toFile.exists))
val foundArtifacts = monitor.findClasses().toSeq
assert(expectedClassFiles.forall { classPath =>
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala
index ab07cf72f768..3f90c4b0bab7 100644
--- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala
+++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala
@@ -487,7 +487,9 @@ class SparkConnectClientSuite extends ConnectFunSuite with BeforeAndAfterEach {
val session = SparkSession.builder().client(client).create()
val artifactFilePath = commonResourcePath.resolve("artifact-tests")
- session.addArtifact(artifactFilePath.resolve("smallClassFile.class").toString)
+ val path = artifactFilePath.resolve("smallClassFile.class")
+ assume(path.toFile.exists)
+ session.addArtifact(path.toString)
}
private def buildPlan(query: String): proto.Plan = {
diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala
index d931d18f3a12..a158ca9fad8c 100644
--- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala
+++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala
@@ -193,6 +193,7 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {
try {
val name = "classes/smallClassFile.class"
val artifactPath = inputFilePath.resolve("smallClassFile.class")
+ assume(artifactPath.toFile.exists)
addSingleChunkArtifact(handler, name, artifactPath)
handler.onCompleted()
val response = ThreadUtils.awaitResult(promise.future, 5.seconds)
@@ -246,15 +247,12 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {
"classes/smallClassFileDup.class",
"jars/smallJar.jar")
- val path1 = inputFilePath.resolve("junitLargeJar.jar")
- val path2 = inputFilePath.resolve("smallJar.jar")
val artifactPaths = Seq(
inputFilePath.resolve("smallClassFile.class"),
- path1,
+ inputFilePath.resolve("junitLargeJar.jar"),
inputFilePath.resolve("smallClassFileDup.class"),
- path2)
- assume(path1.toFile.exists)
- assume(path2.toFile.exists)
+ inputFilePath.resolve("smallJar.jar"))
+ artifactPaths.foreach(p => assume(p.toFile.exists))
addSingleChunkArtifact(handler, names.head, artifactPaths.head)
addChunkedArtifact(handler, names(1), artifactPaths(1))
@@ -286,6 +284,7 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {
try {
val name = "classes/smallClassFile.class"
val artifactPath = inputFilePath.resolve("smallClassFile.class")
+ assume(artifactPath.toFile.exists)
val dataChunks = getDataChunks(artifactPath)
assert(dataChunks.size == 1)
val bytes = dataChunks.head
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala
index e2f1ebba590a..d23f483f9ddf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala
@@ -53,11 +53,13 @@ class ArtifactManagerSuite extends SharedSparkSession {
}
test("Class artifacts are added to the correct directory.") {
+ assume(artifactPath.resolve("smallClassFile.class").toFile.exists)
+
val copyDir = Utils.createTempDir().toPath
FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
val stagingPath = copyDir.resolve("smallClassFile.class")
- val remotePath = Paths.get("classes/smallClassFile.class")
assert(stagingPath.toFile.exists())
+ val remotePath = Paths.get("classes/smallClassFile.class")
artifactManager.addArtifact(remotePath, stagingPath, None)
val movedClassFile = ArtifactManager.artifactRootDirectory
@@ -67,11 +69,13 @@ class ArtifactManagerSuite extends SharedSparkSession {
}
test("Class file artifacts are added to SC classloader") {
+ assume(artifactPath.resolve("Hello.class").toFile.exists)
+
val copyDir = Utils.createTempDir().toPath
FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
val stagingPath = copyDir.resolve("Hello.class")
- val remotePath = Paths.get("classes/Hello.class")
assert(stagingPath.toFile.exists())
+ val remotePath = Paths.get("classes/Hello.class")
artifactManager.addArtifact(remotePath, stagingPath, None)
val movedClassFile = ArtifactManager.artifactRootDirectory
@@ -91,11 +95,13 @@ class ArtifactManagerSuite extends SharedSparkSession {
}
test("UDF can reference added class file") {
+ assume(artifactPath.resolve("Hello.class").toFile.exists)
+
val copyDir = Utils.createTempDir().toPath
FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
val stagingPath = copyDir.resolve("Hello.class")
- val remotePath = Paths.get("classes/Hello.class")
assert(stagingPath.toFile.exists())
+ val remotePath = Paths.get("classes/Hello.class")
artifactManager.addArtifact(remotePath, stagingPath, None)
@@ -170,6 +176,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
}
test("SPARK-43790: Forward artifact file to cloud storage path") {
+ assume(artifactPath.resolve("smallClassFile.class").toFile.exists)
+
val copyDir = Utils.createTempDir().toPath
val destFSDir = Utils.createTempDir().toPath
FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
@@ -184,6 +192,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
}
test("Removal of resources") {
+ assume(artifactPath.resolve("smallClassFile.class").toFile.exists)
+
withTempPath { path =>
// Setup cache
val stagingPath = path.toPath
@@ -228,6 +238,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
}
test("Classloaders for spark sessions are isolated") {
+ assume(artifactPath.resolve("Hello.class").toFile.exists)
+
val session1 = spark.newSession()
val session2 = spark.newSession()
val session3 = spark.newSession()
@@ -284,6 +296,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
}
test("SPARK-44300: Cleaning up resources only deletes session-specific resources") {
+ assume(artifactPath.resolve("Hello.class").toFile.exists)
+
val copyDir = Utils.createTempDir().toPath
FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
val stagingPath = copyDir.resolve("Hello.class")
@@ -335,7 +349,9 @@ class ArtifactManagerSuite extends SharedSparkSession {
}
test("Added artifact can be loaded by the current SparkSession") {
- val buffer = Files.readAllBytes(artifactPath.resolve("IntSumUdf.class"))
+ val path = artifactPath.resolve("IntSumUdf.class")
+ assume(path.toFile.exists)
+ val buffer = Files.readAllBytes(path)
spark.addArtifact(buffer, "IntSumUdf.class")
spark.udf.registerJava("intSum", "IntSumUdf", DataTypes.LongType)
@@ -350,6 +366,8 @@ class ArtifactManagerSuite extends SharedSparkSession {
private def testAddArtifactToLocalSession(
classFileToUse: String, binaryName: String)(addFunc: Path => String): Unit = {
val copyDir = Utils.createTempDir().toPath
+ assume(artifactPath.resolve(classFileToUse).toFile.exists)
+
FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
val classPath = copyDir.resolve(classFileToUse)
assert(classPath.toFile.exists())
@@ -395,8 +413,10 @@ class ArtifactManagerSuite extends SharedSparkSession {
Files.write(randomFilePath, testBytes)
// Register multiple kinds of artifacts
+ val clsPath = path.resolve("Hello.class")
+ assume(clsPath.toFile.exists)
artifactManager.addArtifact( // Class
- Paths.get("classes/Hello.class"), path.resolve("Hello.class"), None)
+ Paths.get("classes/Hello.class"), clsPath, None)
artifactManager.addArtifact( // Python
Paths.get("pyfiles/abc.zip"), randomFilePath, None, deleteStagedFile = false)
val jarPath = Paths.get("jars/udf_noA.jar")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org