You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/10/09 06:22:03 UTC
(spark) branch master updated: [SPARK-60013][CONNECT] Ensure Ivy contexts are popped after Maven resolution failures
This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 2589965d96e4 [SPARK-60013][CONNECT] Ensure Ivy contexts are popped after Maven resolution failures
2589965d96e4 is described below
commit 2589965d96e46f641c31619f0d68a75805100e7e
Author: qianlan717 <qi...@databricks.com>
AuthorDate: Fri Oct 9 06:21:50 2026 +0000
[SPARK-60013][CONNECT] Ensure Ivy contexts are popped after Maven resolution failures
### What changes were proposed in this pull request?
Add a scoped `withIvyContext` helper that always pops an Ivy context in a `finally` block. Use it for both the primary Maven resolution and the no-cache fallback resolution.
Add regression tests for failures in both resolution paths and verify that the original Ivy context is restored. The primary-path test also verifies that a subsequent resolution on the same thread succeeds.
### Why are the changes needed?
The existing `popContext()` calls only run on successful paths. Exceptions from resolve or retrieve can leave Ivy's thread-local context stack unbalanced, allowing a later Maven resolution on the same thread to inherit stale Ivy state or settings.
### Does this PR introduce _any_ user-facing change?
Yes. A Maven resolution failure no longer leaves stale Ivy context state that can affect a later resolution on the same thread.
### How was this patch tested?
Added regression coverage to `MavenUtilsSuite` for primary and no-cache fallback failures.
Ran:
```
build/sbt 'common-utils/testOnly *MavenUtilsSuite'
build/sbt 'common-utils/scalastyle' 'common-utils/Test/scalastyle'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #59262 from qianlan717/SPARK-60013-context.
Authored-by: qianlan717 <qi...@databricks.com>
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
.../scala/org/apache/spark/util/MavenUtils.scala | 127 +++++++++++----------
.../org/apache/spark/util/MavenUtilsSuite.scala | 84 +++++++++++++-
2 files changed, 145 insertions(+), 66 deletions(-)
diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
index 0f5b94fe5f87..f7e37cc3ab68 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
@@ -53,6 +53,15 @@ private[spark] object MavenUtils extends Logging {
private val ivyLock = new ReentrantLock()
private val ivyLockPollIntervalMs = 100L
+ private def withIvyContext[T](ivy: Ivy)(f: => T): T = {
+ ivy.pushContext()
+ try {
+ f
+ } finally {
+ ivy.popContext()
+ }
+ }
+
private def acquireIvyLock(isCancelled: () => Boolean): Unit = {
def checkCancelled(): Unit = {
if (isCancelled() || Thread.currentThread().isInterrupted) {
@@ -559,71 +568,65 @@ private[spark] object MavenUtils extends Logging {
// scalastyle:on println
val ivy = Ivy.newInstance(ivySettings)
- ivy.pushContext()
-
- // Set resolve options to download transitive dependencies as well
- val resolveOptions = new ResolveOptions
- resolveOptions.setTransitive(transitive)
- val retrieveOptions = new RetrieveOptions
- // Turn downloading and logging off for testing
- if (isTest) {
- resolveOptions.setDownload(false)
- resolveOptions.setLog(LogOptions.LOG_QUIET)
- retrieveOptions.setLog(LogOptions.LOG_QUIET)
- } else {
- resolveOptions.setDownload(true)
- }
- // retrieve all resolved dependencies
- retrieveOptions.setDestArtifactPattern(
- packagesDirectory.getAbsolutePath + File.separator +
- "[organization]_[artifact]-[revision](-[classifier]).[ext]")
- retrieveOptions.setConfs(Array(ivyConfName))
-
- // Add exclusion rules for Spark and Scala Library
- addExclusionRules(ivySettings, ivyConfName, md)
- // add all supplied maven artifacts as dependencies
- addDependenciesToIvy(md, artifacts, ivyConfName)
- exclusions.foreach { e =>
- md.addExcludeRule(createExclusion(e + ":*", ivySettings, ivyConfName))
- }
- // resolve dependencies
- val rr: ResolveReport = ivy.resolve(md, resolveOptions)
- if (rr.hasError) {
- // SPARK-46302: When there are some corrupted jars in the local maven repo,
- // we try to continue without the cache
- val failedReports = rr.getArtifactsReports(DownloadStatus.FAILED, true)
- if (failedReports.nonEmpty && noCacheIvySettings.isDefined) {
- val failedArtifacts = failedReports.map(r => r.getArtifact)
- logInfo(log"Download failed: " +
- log"${MDC(LogKeys.ARTIFACTS, failedArtifacts.mkString("[", ", ", "]"))}, " +
- log"attempt to retry while skipping local-m2-cache.")
- failedArtifacts.foreach(artifact => {
- clearInvalidIvyCacheFiles(artifact.getModuleRevisionId, ivySettings.getDefaultCache)
- })
- ivy.popContext()
-
- val noCacheIvy = Ivy.newInstance(noCacheIvySettings.get)
- noCacheIvy.pushContext()
-
- val noCacheRr = noCacheIvy.resolve(md, resolveOptions)
- if (noCacheRr.hasError) {
- throw new RuntimeException(noCacheRr.getAllProblemMessages.toString)
+ withIvyContext(ivy) {
+ // Set resolve options to download transitive dependencies as well
+ val resolveOptions = new ResolveOptions
+ resolveOptions.setTransitive(transitive)
+ val retrieveOptions = new RetrieveOptions
+ // Turn downloading and logging off for testing
+ if (isTest) {
+ resolveOptions.setDownload(false)
+ resolveOptions.setLog(LogOptions.LOG_QUIET)
+ retrieveOptions.setLog(LogOptions.LOG_QUIET)
+ } else {
+ resolveOptions.setDownload(true)
+ }
+ // retrieve all resolved dependencies
+ retrieveOptions.setDestArtifactPattern(
+ packagesDirectory.getAbsolutePath + File.separator +
+ "[organization]_[artifact]-[revision](-[classifier]).[ext]")
+ retrieveOptions.setConfs(Array(ivyConfName))
+
+ // Add exclusion rules for Spark and Scala Library
+ addExclusionRules(ivySettings, ivyConfName, md)
+ // add all supplied maven artifacts as dependencies
+ addDependenciesToIvy(md, artifacts, ivyConfName)
+ exclusions.foreach { e =>
+ md.addExcludeRule(createExclusion(e + ":*", ivySettings, ivyConfName))
+ }
+ // resolve dependencies
+ val rr: ResolveReport = ivy.resolve(md, resolveOptions)
+ if (rr.hasError) {
+ // SPARK-46302: When there are some corrupted jars in the local maven repo,
+ // we try to continue without the cache
+ val failedReports = rr.getArtifactsReports(DownloadStatus.FAILED, true)
+ if (failedReports.nonEmpty && noCacheIvySettings.isDefined) {
+ val failedArtifacts = failedReports.map(r => r.getArtifact)
+ logInfo(log"Download failed: " +
+ log"${MDC(LogKeys.ARTIFACTS, failedArtifacts.mkString("[", ", ", "]"))}, " +
+ log"attempt to retry while skipping local-m2-cache.")
+ failedArtifacts.foreach(artifact => {
+ clearInvalidIvyCacheFiles(
+ artifact.getModuleRevisionId, ivySettings.getDefaultCache)
+ })
+
+ val noCacheIvy = Ivy.newInstance(noCacheIvySettings.get)
+ withIvyContext(noCacheIvy) {
+ val noCacheRr = noCacheIvy.resolve(md, resolveOptions)
+ if (noCacheRr.hasError) {
+ throw new RuntimeException(noCacheRr.getAllProblemMessages.toString)
+ }
+ noCacheIvy.retrieve(
+ noCacheRr.getModuleDescriptor.getModuleRevisionId, retrieveOptions)
+ resolveDependencyPaths(noCacheRr.getArtifacts.toArray, packagesDirectory)
+ }
+ } else {
+ throw new RuntimeException(rr.getAllProblemMessages.toString)
}
- noCacheIvy.retrieve(noCacheRr.getModuleDescriptor.getModuleRevisionId, retrieveOptions)
- val dependencyPaths = resolveDependencyPaths(
- noCacheRr.getArtifacts.toArray, packagesDirectory)
- noCacheIvy.popContext()
-
- dependencyPaths
} else {
- throw new RuntimeException(rr.getAllProblemMessages.toString)
+ ivy.retrieve(rr.getModuleDescriptor.getModuleRevisionId, retrieveOptions)
+ resolveDependencyPaths(rr.getArtifacts.toArray, packagesDirectory)
}
- } else {
- ivy.retrieve(rr.getModuleDescriptor.getModuleRevisionId, retrieveOptions)
- val dependencyPaths = resolveDependencyPaths(rr.getArtifacts.toArray, packagesDirectory)
- ivy.popContext()
-
- dependencyPaths
}
} finally {
System.setOut(sysOut)
diff --git a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala
index e473a90110cd..b0e249b5a9f1 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala
@@ -29,7 +29,9 @@ import scala.concurrent.{ExecutionContext, Future}
import scala.concurrent.duration._
import scala.jdk.CollectionConverters._
-import org.apache.ivy.core.module.descriptor.MDArtifact
+import org.apache.ivy.core.IvyContext
+import org.apache.ivy.core.module.descriptor.{DependencyDescriptor, MDArtifact}
+import org.apache.ivy.core.resolve.{ResolveData, ResolvedModuleRevision}
import org.apache.ivy.core.settings.IvySettings
import org.apache.ivy.plugins.resolver.{AbstractResolver, ChainResolver, FileSystemResolver, IBiblioResolver}
import org.scalatest.BeforeAndAfterEach
@@ -60,6 +62,29 @@ class MavenUtilsSuite
// scalastyle:on println
}
+ private def buildTestIvySettings(
+ repo: File,
+ ivyPath: String,
+ resolutionStarted: Option[AtomicBoolean] = None): IvySettings = {
+ val settings = new IvySettings
+ MavenUtils.processIvyPathArg(settings, Some(ivyPath))
+ val resolver = new IBiblioResolver {
+ override def getDependency(
+ descriptor: DependencyDescriptor,
+ data: ResolveData): ResolvedModuleRevision = {
+ resolutionStarted.foreach(_.set(true))
+ super.getDependency(descriptor, data)
+ }
+ }
+ resolver.setM2compatible(true)
+ resolver.setUsepoms(true)
+ resolver.setRoot(repo.toURI.toString)
+ resolver.setName("test-repo")
+ settings.addResolver(resolver)
+ settings.setDefaultResolver(resolver.getName)
+ settings
+ }
+
override def beforeEach(): Unit = {
super.beforeEach()
tempIvyPath = SparkFileUtils.createTempDir(namePrefix = "ivy").getAbsolutePath()
@@ -348,14 +373,65 @@ class MavenUtilsSuite
}
}
- test("dependency not found throws RuntimeException") {
+ test("primary Ivy context is restored after a resolution failure") {
+ val missingRepo = new File(tempIvyPath, "missing-repo")
+ assert(missingRepo.mkdirs())
+ val initialContext = IvyContext.getContext
+
intercept[RuntimeException] {
MavenUtils.resolveMavenCoordinates(
- "a:b:c",
- MavenUtils.buildIvySettings(None, Some(tempIvyPath)),
+ "a:b:c",
+ buildTestIvySettings(
+ missingRepo,
+ Paths.get(tempIvyPath, "missing-cache").toString),
transitive = true,
+ isTest = true)
+ }
+ assert(IvyContext.getContext eq initialContext)
+
+ val valid = MavenCoordinate("my.context.after.failure", "mylib", "0.1")
+ val validRepo = IvyTestUtils.createLocalRepositoryForTests(
+ valid,
+ dependencies = None,
+ rootDir = Some(new File(tempIvyPath, "valid-repo")))
+ val resolved = MavenUtils.resolveMavenCoordinates(
+ valid.toString,
+ buildTestIvySettings(validRepo, Paths.get(tempIvyPath, "valid-cache").toString),
+ transitive = true,
isTest = true)
+ assert(resolved.nonEmpty)
+ assert(IvyContext.getContext eq initialContext)
+ }
+
+ test("no-cache Ivy context is restored after a resolution failure") {
+ val artifact = MavenCoordinate("my.context.no.cache", "mylib", "0.1")
+ val primaryRepo = IvyTestUtils.createLocalRepositoryForTests(
+ artifact,
+ dependencies = None,
+ rootDir = Some(new File(tempIvyPath, "primary-repo")))
+ val jar = new File(
+ IvyTestUtils.pathFromCoordinate(artifact, primaryRepo, "jar", useIvyLayout = false),
+ IvyTestUtils.artifactName(artifact, useIvyLayout = false))
+ assert(jar.delete())
+
+ val noCacheRepo = new File(tempIvyPath, "no-cache-repo")
+ assert(noCacheRepo.mkdirs())
+ val fallbackStarted = new AtomicBoolean(false)
+ val initialContext = IvyContext.getContext
+ intercept[RuntimeException] {
+ MavenUtils.resolveMavenCoordinates(
+ artifact.toString,
+ buildTestIvySettings(
+ primaryRepo,
+ Paths.get(tempIvyPath, "primary-cache").toString),
+ noCacheIvySettings = Some(buildTestIvySettings(
+ noCacheRepo,
+ Paths.get(tempIvyPath, "no-cache").toString,
+ resolutionStarted = Some(fallbackStarted))),
+ transitive = true)
}
+ assert(fallbackStarted.get())
+ assert(IvyContext.getContext eq initialContext)
}
test("neglects Spark and Spark's dependencies") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org