You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2021/06/25 00:11:34 UTC

[spark] branch master updated: [SPARK-33298][CORE] Introduce new API to FileCommitProtocol allow flexible file naming

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 2da42ca  [SPARK-33298][CORE] Introduce new API to FileCommitProtocol allow flexible file naming
2da42ca is described below

commit 2da42ca3b4ed50a73a0b0c56d57b0b6102650566
Author: Cheng Su <ch...@fb.com>
AuthorDate: Thu Jun 24 17:10:54 2021 -0700

    [SPARK-33298][CORE] Introduce new API to FileCommitProtocol allow flexible file naming
    
    ### What changes were proposed in this pull request?
    
    This PR is to introduce a new sets of APIs `newTaskTempFile` and `newTaskTempFileAbsPath` inside `FileCommitProtocol`, to allow more flexible file naming of Spark output. The major change is to pass `FileNameSpec` into `FileCommitProtocol`, instead of original `ext` (currently having `prefix` and `ext`), to allow individual `FileCommitProtocol` implementation comes up with more flexible file names (e.g. has a custom `prefix`) for Hive/Presto bucketing - https://github.com/apache/spark [...]
    
    ### Why are the changes needed?
    
    To make commit protocol more flexible in terms of Spark output file name.
    Pre-requisite of https://github.com/apache/spark/pull/30003.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes for developers  who implement/run custom implementation of `FileCommitProtocol`. They can choose to implement for the newly added API.
    
    ### How was this patch tested?
    
    Existing unit tests as this is just adding an API.
    
    Closes #33012 from c21/commit-protocol-api.
    
    Authored-by: Cheng Su <ch...@fb.com>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../spark/internal/io/FileCommitProtocol.scala     | 66 ++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
index d9d7b06..6465cc7 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
@@ -93,6 +93,35 @@ abstract class FileCommitProtocol extends Logging {
   def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], ext: String): String
 
   /**
+   * Notifies the commit protocol to add a new file, and gets back the full path that should be
+   * used. Must be called on the executors when running tasks.
+   *
+   * Note that the returned temp file may have an arbitrary path. The commit protocol only
+   * promises that the file will be at the location specified by the arguments after job commit.
+   *
+   * The "dir" parameter specifies the sub-directory within the base path, used to specify
+   * partitioning. The "spec" parameter specifies the file name. The rest are left to the commit
+   * protocol implementation to decide.
+   *
+   * Important: it is the caller's responsibility to add uniquely identifying content to "spec"
+   * if a task is going to write out multiple files to the same dir. The file commit protocol only
+   * guarantees that files written by different tasks will not conflict.
+   *
+   * This API should be implemented and called, instead of
+   * [[newTaskTempFile(taskContest, dir, ext)]]. Provide a default implementation here to be
+   * backward compatible with custom [[FileCommitProtocol]] implementations before Spark 3.2.0.
+   */
+  def newTaskTempFile(
+      taskContext: TaskAttemptContext, dir: Option[String], spec: FileNameSpec): String = {
+    if (spec.prefix.isEmpty) {
+      newTaskTempFile(taskContext, dir, spec.suffix)
+    } else {
+      throw new UnsupportedOperationException(s"${getClass.getSimpleName}.newTaskTempFile does " +
+        s"not support file name prefix: ${spec.prefix}")
+    }
+  }
+
+  /**
    * Similar to newTaskTempFile(), but allows files to committed to an absolute output location.
    * Depending on the implementation, there may be weaker guarantees around adding files this way.
    *
@@ -104,6 +133,34 @@ abstract class FileCommitProtocol extends Logging {
       taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String
 
   /**
+   * Similar to newTaskTempFile(), but allows files to committed to an absolute output location.
+   * Depending on the implementation, there may be weaker guarantees around adding files this way.
+   *
+   * The "absoluteDir" parameter specifies the final absolute directory of file. The "spec"
+   * parameter specifies the file name. The rest are left to the commit protocol implementation to
+   * decide.
+   *
+   * Important: it is the caller's responsibility to add uniquely identifying content to "spec"
+   * if a task is going to write out multiple files to the same dir. The file commit protocol only
+   * guarantees that files written by different tasks will not conflict.
+   *
+   * This API should be implemented and called, instead of
+   * [[newTaskTempFileAbsPath(taskContest, absoluteDir, ext)]]. Provide a default implementation
+   * here to be backward compatible with custom [[FileCommitProtocol]] implementations before
+   * Spark 3.2.0.
+   */
+  def newTaskTempFileAbsPath(
+      taskContext: TaskAttemptContext, absoluteDir: String, spec: FileNameSpec): String = {
+    if (spec.prefix.isEmpty) {
+      newTaskTempFileAbsPath(taskContext, absoluteDir, spec.suffix)
+    } else {
+      throw new UnsupportedOperationException(
+        s"${getClass.getSimpleName}.newTaskTempFileAbsPath does not support file name prefix: " +
+          s"${spec.prefix}")
+    }
+  }
+
+  /**
    * Commits a task after the writes succeed. Must be called on the executors when running tasks.
    */
   def commitTask(taskContext: TaskAttemptContext): TaskCommitMessage
@@ -174,3 +231,12 @@ object FileCommitProtocol extends Logging {
     new Path(path, ".spark-staging-" + jobId)
   }
 }
+
+/**
+ * The specification for Spark output file name.
+ * This is used by [[FileCommitProtocol]] to create full path of file.
+ *
+ * @param prefix Prefix of file.
+ * @param suffix Suffix of file.
+ */
+final case class FileNameSpec(prefix: String, suffix: String)

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org