You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/12/23 11:22:47 UTC

[spark] branch master updated: [SPARK-41407][SQL][FOLLOW-UP] Use string jobTrackerID for FileFormatWriter.executeTask

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 3d5af030707 [SPARK-41407][SQL][FOLLOW-UP] Use string jobTrackerID for FileFormatWriter.executeTask
3d5af030707 is described below

commit 3d5af030707e1dd8bd14a1bee244350329303943
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Fri Dec 23 20:22:33 2022 +0900

    [SPARK-41407][SQL][FOLLOW-UP] Use string jobTrackerID for FileFormatWriter.executeTask
    
    ### What changes were proposed in this pull request?
    
    This PR is a followup of https://github.com/apache/spark/pull/38939 that fixes a logical conflict during merging PRs, see https://github.com/apache/spark/pull/38980 and https://github.com/apache/spark/pull/38939.
    
    ### Why are the changes needed?
    
    To recover the broken build.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, dev-only.
    
    ### How was this patch tested?
    
    Manually tested:
    
    ```
     ./build/sbt -Phive clean package
    ```
    
    Closes #39194 from HyukjinKwon/SPARK-41407.
    
    Authored-by: Hyukjin Kwon <gu...@apache.org>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 .../org/apache/spark/sql/execution/datasources/WriteFiles.scala     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala
index 39b7b252f6e..5bc8f9db32b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.datasources
 import java.util.Date
 
 import org.apache.spark.{SparkException, TaskContext}
-import org.apache.spark.internal.io.FileCommitProtocol
+import org.apache.spark.internal.io.{FileCommitProtocol, SparkHadoopWriterUtils}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -72,7 +72,7 @@ case class WriteFilesExec(child: SparkPlan) extends UnaryExecNode {
     val concurrentOutputWriterSpec = writeFilesSpec.concurrentOutputWriterSpecFunc(child)
     val description = writeFilesSpec.description
     val committer = writeFilesSpec.committer
-    val jobIdInstant = new Date().getTime
+    val jobTrackerID = SparkHadoopWriterUtils.createJobTrackerID(new Date())
     rddWithNonEmptyPartitions.mapPartitionsInternal { iterator =>
       val sparkStageId = TaskContext.get().stageId()
       val sparkPartitionId = TaskContext.get().partitionId()
@@ -80,7 +80,7 @@ case class WriteFilesExec(child: SparkPlan) extends UnaryExecNode {
 
       val ret = FileFormatWriter.executeTask(
         description,
-        jobIdInstant,
+        jobTrackerID,
         sparkStageId,
         sparkPartitionId,
         sparkAttemptNumber,


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org