You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2021/06/25 00:11:34 UTC
[spark] branch master updated: [SPARK-33298][CORE] Introduce new
API to FileCommitProtocol allow flexible file naming
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 2da42ca [SPARK-33298][CORE] Introduce new API to FileCommitProtocol allow flexible file naming
2da42ca is described below
commit 2da42ca3b4ed50a73a0b0c56d57b0b6102650566
Author: Cheng Su <ch...@fb.com>
AuthorDate: Thu Jun 24 17:10:54 2021 -0700
[SPARK-33298][CORE] Introduce new API to FileCommitProtocol allow flexible file naming
### What changes were proposed in this pull request?
This PR is to introduce a new sets of APIs `newTaskTempFile` and `newTaskTempFileAbsPath` inside `FileCommitProtocol`, to allow more flexible file naming of Spark output. The major change is to pass `FileNameSpec` into `FileCommitProtocol`, instead of original `ext` (currently having `prefix` and `ext`), to allow individual `FileCommitProtocol` implementation comes up with more flexible file names (e.g. has a custom `prefix`) for Hive/Presto bucketing - https://github.com/apache/spark [...]
### Why are the changes needed?
To make commit protocol more flexible in terms of Spark output file name.
Pre-requisite of https://github.com/apache/spark/pull/30003.
### Does this PR introduce _any_ user-facing change?
Yes for developers who implement/run custom implementation of `FileCommitProtocol`. They can choose to implement for the newly added API.
### How was this patch tested?
Existing unit tests as this is just adding an API.
Closes #33012 from c21/commit-protocol-api.
Authored-by: Cheng Su <ch...@fb.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../spark/internal/io/FileCommitProtocol.scala | 66 ++++++++++++++++++++++
1 file changed, 66 insertions(+)
diff --git a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
index d9d7b06..6465cc7 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
@@ -93,6 +93,35 @@ abstract class FileCommitProtocol extends Logging {
def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], ext: String): String
/**
+ * Notifies the commit protocol to add a new file, and gets back the full path that should be
+ * used. Must be called on the executors when running tasks.
+ *
+ * Note that the returned temp file may have an arbitrary path. The commit protocol only
+ * promises that the file will be at the location specified by the arguments after job commit.
+ *
+ * The "dir" parameter specifies the sub-directory within the base path, used to specify
+ * partitioning. The "spec" parameter specifies the file name. The rest are left to the commit
+ * protocol implementation to decide.
+ *
+ * Important: it is the caller's responsibility to add uniquely identifying content to "spec"
+ * if a task is going to write out multiple files to the same dir. The file commit protocol only
+ * guarantees that files written by different tasks will not conflict.
+ *
+ * This API should be implemented and called, instead of
+ * [[newTaskTempFile(taskContest, dir, ext)]]. Provide a default implementation here to be
+ * backward compatible with custom [[FileCommitProtocol]] implementations before Spark 3.2.0.
+ */
+ def newTaskTempFile(
+ taskContext: TaskAttemptContext, dir: Option[String], spec: FileNameSpec): String = {
+ if (spec.prefix.isEmpty) {
+ newTaskTempFile(taskContext, dir, spec.suffix)
+ } else {
+ throw new UnsupportedOperationException(s"${getClass.getSimpleName}.newTaskTempFile does " +
+ s"not support file name prefix: ${spec.prefix}")
+ }
+ }
+
+ /**
* Similar to newTaskTempFile(), but allows files to committed to an absolute output location.
* Depending on the implementation, there may be weaker guarantees around adding files this way.
*
@@ -104,6 +133,34 @@ abstract class FileCommitProtocol extends Logging {
taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String
/**
+ * Similar to newTaskTempFile(), but allows files to committed to an absolute output location.
+ * Depending on the implementation, there may be weaker guarantees around adding files this way.
+ *
+ * The "absoluteDir" parameter specifies the final absolute directory of file. The "spec"
+ * parameter specifies the file name. The rest are left to the commit protocol implementation to
+ * decide.
+ *
+ * Important: it is the caller's responsibility to add uniquely identifying content to "spec"
+ * if a task is going to write out multiple files to the same dir. The file commit protocol only
+ * guarantees that files written by different tasks will not conflict.
+ *
+ * This API should be implemented and called, instead of
+ * [[newTaskTempFileAbsPath(taskContest, absoluteDir, ext)]]. Provide a default implementation
+ * here to be backward compatible with custom [[FileCommitProtocol]] implementations before
+ * Spark 3.2.0.
+ */
+ def newTaskTempFileAbsPath(
+ taskContext: TaskAttemptContext, absoluteDir: String, spec: FileNameSpec): String = {
+ if (spec.prefix.isEmpty) {
+ newTaskTempFileAbsPath(taskContext, absoluteDir, spec.suffix)
+ } else {
+ throw new UnsupportedOperationException(
+ s"${getClass.getSimpleName}.newTaskTempFileAbsPath does not support file name prefix: " +
+ s"${spec.prefix}")
+ }
+ }
+
+ /**
* Commits a task after the writes succeed. Must be called on the executors when running tasks.
*/
def commitTask(taskContext: TaskAttemptContext): TaskCommitMessage
@@ -174,3 +231,12 @@ object FileCommitProtocol extends Logging {
new Path(path, ".spark-staging-" + jobId)
}
}
+
+/**
+ * The specification for Spark output file name.
+ * This is used by [[FileCommitProtocol]] to create full path of file.
+ *
+ * @param prefix Prefix of file.
+ * @param suffix Suffix of file.
+ */
+final case class FileNameSpec(prefix: String, suffix: String)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org