(spark) branch master updated: [SPARK-46215][CORE] Improve `FileSystemPersistenceEngine` to allow nonexistent parents

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new d1311f75dea [SPARK-46215][CORE] Improve `FileSystemPersistenceEngine` to allow nonexistent parents
d1311f75dea is described below

commit d1311f75dea21201f2c37b8f92b111c91ebc9aad
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Sat Dec 2 20:24:11 2023 -0800

    [SPARK-46215][CORE] Improve `FileSystemPersistenceEngine` to allow nonexistent parents
    
    ### What changes were proposed in this pull request?
    
    This PR aims to improve `FileSystemPersistenceEngine` to allow non-exist parents
    
    ### Why are the changes needed?
    
    To prevent the following error,
    ```
    java.io.IOException: No such file or directory
      at java.base/java.io.UnixFileSystem.createFileExclusively(Native Method)
      at java.base/java.io.File.createNewFile(File.java:1043)
      at org.apache.spark.deploy.master.FileSystemPersistenceEngine.serializeIntoFile(FileSystemPersistenceEngine.scala:62)
      at org.apache.spark.deploy.master.FileSystemPersistenceEngine.persist(FileSystemPersistenceEngine.scala:45)
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test case.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #44127 from dongjoon-hyun/SPARK-46215.
    
    Authored-by: Dongjoon Hyun <dh...@apple.com>
    Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
 .../apache/spark/deploy/master/FileSystemPersistenceEngine.scala | 5 +++--
 .../org/apache/spark/deploy/master/PersistenceEngineSuite.scala  | 9 +++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
index 7f624816f25..094136ea274 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.deploy.master
 
 import java.io._
+import java.nio.file.{Files, Paths}
 
 import scala.reflect.ClassTag
 
@@ -31,7 +32,7 @@ import org.apache.spark.util.Utils
  * Stores data in a single on-disk directory with one file per application and worker.
  * Files are deleted when applications and workers are removed.
  *
- * @param dir Directory to store files. Created if non-existent (but not recursively).
+ * @param dir Directory to store files. Created if non-existent.
  * @param serializer Used to serialize our objects.
  */
 private[master] class FileSystemPersistenceEngine(
@@ -39,7 +40,7 @@ private[master] class FileSystemPersistenceEngine(
     val serializer: Serializer)
   extends PersistenceEngine with Logging {
 
-  new File(dir).mkdir()
+  Files.createDirectories(Paths.get(dir))
 
   override def persist(name: String, obj: Object): Unit = {
     serializeIntoFile(new File(dir + File.separator + name), obj)
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
index 66a61d80d2a..8546f4e01f3 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
@@ -53,6 +53,15 @@ class PersistenceEngineSuite extends SparkFunSuite {
     }
   }
 
+  test("SPARK-46215: FileSystemPersistenceEngine with a non-existent parent dir") {
+    withTempDir { dir =>
+      val conf = new SparkConf()
+      testPersistenceEngine(conf, serializer =>
+        new FileSystemPersistenceEngine(dir.getAbsolutePath + "/a/b/c/dir", serializer)
+      )
+    }
+  }
+
   test("SPARK-46205: Support KryoSerializer in FileSystemPersistenceEngine") {
     withTempDir { dir =>
       val conf = new SparkConf()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org