[GitHub] spark pull request #18801: SPARK-10878 Fix race condition when multiple clie...

[vanzin <gi...@git.apache.org>]

Github user vanzin commented on a diff in the pull request:

    https://github.com/apache/spark/pull/18801#discussion_r156240364
  
    --- Diff: core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala ---
    @@ -1216,7 +1216,36 @@ private[spark] object SparkSubmitUtils {
     
       /** A nice function to use in tests as well. Values are dummy strings. */
       def getModuleDescriptor: DefaultModuleDescriptor = DefaultModuleDescriptor.newDefaultInstance(
    -    ModuleRevisionId.newInstance("org.apache.spark", "spark-submit-parent", "1.0"))
    +    // Include timestamp in module name, so multiple clients resolving maven coordinates at the
    +    // same time do not modify the same resolution file concurrently.
    +    ModuleRevisionId.newInstance("org.apache.spark",
    +      "spark-submit-parent-" + System.currentTimeMillis().toString,
    +      "1.0"))
    +
    +  /**
    +   * clear ivy resolution from current launch. The resolution file is usually at
    +   * ~/.ivy2/org.apache.spark-spark-submit-parent-$timestamp-default.xml,
    +   * ~/.ivy2/resolved-org.apache.spark-spark-submit-parent-$timestamp-1.0.xml, and
    +   * ~/.ivy2/resolved-org.apache.spark-spark-submit-parent-$timestamp-1.0.properties.
    +   * Since each launch will have its own resolution files created, delete them after
    +   * each resolution to prevent accumulation of these files in the ivy cache dir.
    +   */
    +  private def clearIvyResolutionFiles(
    +      mdId: ModuleRevisionId,
    +      ivySettings: IvySettings,
    +      ivyConfName: String): Unit = {
    +    val currentResolutionFiles = Seq[File](
    +      new File(ivySettings.getDefaultCache,
    +        s"${mdId.getOrganisation}-${mdId.getName}-$ivyConfName.xml"),
    +      new File(ivySettings.getDefaultCache,
    +        s"resolved-${mdId.getOrganisation}-${mdId.getName}-${mdId.getRevision}.xml"),
    +      new File(ivySettings.getDefaultCache,
    +        s"resolved-${mdId.getOrganisation}-${mdId.getName}-${mdId.getRevision}.properties")
    +    )
    +    currentResolutionFiles.foreach{ file =>
    --- End diff --
    
    space before `{`


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org