(spark) branch branch-4.x updated: [SPARK-59821][CORE] Make Maven utility tests safe for concurrent runs

[gu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new b1a57d5ec476 [SPARK-59821][CORE] Make Maven utility tests safe for concurrent runs
b1a57d5ec476 is described below

commit b1a57d5ec476535ac894ed1d800ece4ec008094b
Author: qianlan717 <qi...@databricks.com>
AuthorDate: Tue Sep 29 06:52:36 2026 +0900

    [SPARK-59821][CORE] Make Maven utility tests safe for concurrent runs
    
    ### What changes were proposed in this pull request?
    
    This follow-up to #58874 makes the Maven test utilities safe when the same suite runs concurrently.
    
    - Compile generated Java classes in an invocation-specific temporary directory instead of the process working directory.
    - Give each test JVM its own stable fake local Maven repository.
    - Add a regression test that compiles the same class name concurrently into separate destinations.
    
    ### Why are the changes needed?
    
    `SparkTestUtils.createCompiledClass` currently asks javac to emit a fixed class name, such as `MyLib.class`, in the process working directory before moving it to the requested destination. `MavenUtils.m2Path` also uses the fixed relative path `dummy/.m2/repository` during tests.
    
    When a test target is repeated concurrently, the JVMs can move or delete files needed by another JVM. This causes failures such as:
    
    ```
    java.nio.file.NoSuchFileException:
      MyLib.class -> dummy/.m2/repository/.../MyLib.class
    ```
    
    Using isolated temporary paths removes this cross-JVM interference without changing production Maven resolution.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added a `SparkTestUtilsSuite` regression test that starts eight concurrent compilations of the same class name and verifies every output exists.
    
    `git diff --check` passes.
    
    The following focused command was attempted locally:
    
    ```
    /usr/bin/sbt \
      "common-utils/testOnly org.apache.spark.util.SparkTestUtilsSuite" \
      "common-utils/testOnly org.apache.spark.util.MavenUtilsSuite"
    ```
    
    The suites could not start because the local environment could not resolve Maven Central or the SBT plugin repositories (`UnknownHostException`). GitHub Actions will provide the executable test result.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-5)
    
    Closes #59096 from qianlan717/SPARK-59553-concurrent-ivy-test-isolation.
    
    Authored-by: qianlan717 <qi...@databricks.com>
    Signed-off-by: Hyukjin Kwon <hy...@databricks.com>
    (cherry picked from commit aeda06559238be417f32a11bb5814af59b2bace7)
    Signed-off-by: Hyukjin Kwon <hy...@databricks.com>
---
 .../scala/org/apache/spark/util/MavenUtils.scala   |  8 ++++--
 .../org/apache/spark/util/SparkTestUtils.scala     | 31 +++++++++++++---------
 .../apache/spark/util/SparkTestUtilsSuite.scala    | 28 +++++++++++++++++++
 3 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
index 8ca1f9efe649..0f5b94fe5f87 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
@@ -46,6 +46,10 @@ import org.apache.spark.util.ArrayImplicits._
 private[spark] object MavenUtils extends Logging {
   val JAR_IVY_SETTING_PATH_KEY: String = "spark.jars.ivySettings"
 
+  private lazy val testM2Path = new File(
+    SparkFileUtils.createTempDir(namePrefix = "spark-test-m2"),
+    "repository")
+
   private val ivyLock = new ReentrantLock()
   private val ivyLockPollIntervalMs = 100L
 
@@ -152,8 +156,8 @@ private[spark] object MavenUtils extends Logging {
   /** Path of the local Maven cache. */
   private[util] def m2Path: File = {
     if (SparkEnvUtils.isTesting) {
-      // test builds delete the maven cache, and this can cause flakiness
-      new File("dummy", ".m2" + File.separator + "repository")
+      // Keep concurrent test JVMs from deleting each other's local Maven repository.
+      testM2Path
     } else {
       new File(System.getProperty("user.home"), ".m2" + File.separator + "repository")
     }
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
index 0af0e0f6de45..c5832706e977 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
@@ -46,10 +46,8 @@ private[spark] trait SparkTestUtils {
       sourceFile: JavaSourceFromString,
       classpathUrls: Seq[URL]): File = {
     val compiler = ToolProvider.getSystemJavaCompiler
-
-    // Calling this outputs a class file in pwd. It's easier to just rename the files than
-    // build a custom FileManager that controls the output location.
-    val options = if (classpathUrls.nonEmpty) {
+    val compileDir = SparkFileUtils.createTempDir(namePrefix = "spark-javac")
+    val classpathOptions = if (classpathUrls.nonEmpty) {
       Seq(
         "-classpath",
         classpathUrls
@@ -60,17 +58,24 @@ private[spark] trait SparkTestUtils {
     } else {
       Seq.empty
     }
-    compiler.getTask(null, null, null, options.asJava, null, Arrays.asList(sourceFile)).call()
-
-    val fileName = className + ".class"
-    val result = new File(fileName)
-    assert(result.exists(), "Compiled file not found: " + result.getAbsolutePath())
-    val out = new File(destDir, fileName)
+    val options = Seq("-d", compileDir.getAbsolutePath) ++ classpathOptions
 
-    Files.move(result.toPath, out.toPath)
+    try {
+      val compiled = compiler
+        .getTask(null, null, null, options.asJava, null, Arrays.asList(sourceFile))
+        .call()
+      assert(compiled, s"Failed to compile $className")
 
-    assert(out.exists(), "Destination file not moved: " + out.getAbsolutePath())
-    out
+      val fileName = className + ".class"
+      val result = SparkFileUtils.recursiveList(compileDir)
+        .find(file => file.isFile && file.getName == fileName)
+        .getOrElse(throw new IllegalStateException(s"Compiled file not found: $fileName"))
+      val out = new File(destDir, fileName)
+      Files.move(result.toPath, out.toPath)
+      out
+    } finally {
+      SparkFileUtils.deleteRecursively(compileDir)
+    }
   }
 
   /** Creates a compiled class with the given name. Class file will be placed in destDir. */
diff --git a/common/utils/src/test/scala/org/apache/spark/util/SparkTestUtilsSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/SparkTestUtilsSuite.scala
index 10a599739f6c..2b099fdc11a9 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/SparkTestUtilsSuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/SparkTestUtilsSuite.scala
@@ -17,6 +17,11 @@
 
 package org.apache.spark.util
 
+import java.util.concurrent.{CountDownLatch, Executors}
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
+
 import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
 
 class SparkTestUtilsSuite extends AnyFunSuite with SparkTestUtils { // scalastyle:ignore funsuite
@@ -27,4 +32,27 @@ class SparkTestUtilsSuite extends AnyFunSuite with SparkTestUtils { // scalastyl
     val result = createCompiledClass("Hello", dir, sourceFile, Seq(dir.toURI.toURL))
     assert(result.exists(), s"Compiled class file should exist at ${result.getPath}")
   }
+
+  test("createCompiledClass supports concurrent compilation of the same class name") {
+    val executor = Executors.newFixedThreadPool(8)
+    implicit val executionContext: ExecutionContext =
+      ExecutionContext.fromExecutorService(executor)
+    val start = new CountDownLatch(1)
+    try {
+      val compiledClasses = (1 to 8).map { _ =>
+        Future {
+          val dir = SparkFileUtils.createTempDir()
+          val sourceFile = new JavaSourceFromString("Hello", "public class Hello {}")
+          start.await()
+          createCompiledClass("Hello", dir, sourceFile, Seq.empty)
+        }
+      }
+      start.countDown()
+
+      val results = SparkThreadUtils.awaitResult(Future.sequence(compiledClasses), 30.seconds)
+      assert(results.forall(_.exists()))
+    } finally {
+      executor.shutdownNow()
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org