You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2024/05/31 14:38:03 UTC
(spark) branch master updated: [SPARK-48484][SQL] Fix: V2Write use the same TaskAttemptId for different task attempts
This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 67d11b1992aa [SPARK-48484][SQL] Fix: V2Write use the same TaskAttemptId for different task attempts
67d11b1992aa is described below
commit 67d11b1992aaa100d0e1fa30b0e5c33684c93a89
Author: jackylee-ch <li...@baidu.com>
AuthorDate: Fri May 31 22:37:49 2024 +0800
[SPARK-48484][SQL] Fix: V2Write use the same TaskAttemptId for different task attempts
### What changes were proposed in this pull request?
After #40064 , we always get the same TaskAttemptId for different task attempts which has the same partitionId. This would lead different task attempts write to the same directory.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
GA
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #46811 from jackylee-ch/fix_v2write_use_same_directories_for_different_task_attempts.
Lead-authored-by: jackylee-ch <li...@baidu.com>
Co-authored-by: Kent Yao <ya...@apache.org>
Signed-off-by: yangjie01 <ya...@baidu.com>
---
.../datasources/v2/FileWriterFactory.scala | 8 ++--
.../datasources/v2/FileWriterFactorySuite.scala | 48 ++++++++++++++++++++++
2 files changed, 53 insertions(+), 3 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactory.scala
index 4b1a099d3bac..f18424b4bcb8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactory.scala
@@ -38,7 +38,7 @@ case class FileWriterFactory (
@transient private lazy val jobId = SparkHadoopWriterUtils.createJobID(jobTrackerID, 0)
override def createWriter(partitionId: Int, realTaskId: Long): DataWriter[InternalRow] = {
- val taskAttemptContext = createTaskAttemptContext(partitionId)
+ val taskAttemptContext = createTaskAttemptContext(partitionId, realTaskId.toInt & Int.MaxValue)
committer.setupTask(taskAttemptContext)
if (description.partitionColumns.isEmpty) {
new SingleDirectoryDataWriter(description, taskAttemptContext, committer)
@@ -47,9 +47,11 @@ case class FileWriterFactory (
}
}
- private def createTaskAttemptContext(partitionId: Int): TaskAttemptContextImpl = {
+ private def createTaskAttemptContext(
+ partitionId: Int,
+ realTaskId: Int): TaskAttemptContextImpl = {
val taskId = new TaskID(jobId, TaskType.MAP, partitionId)
- val taskAttemptId = new TaskAttemptID(taskId, 0)
+ val taskAttemptId = new TaskAttemptID(taskId, realTaskId)
// Set up the configuration object
val hadoopConf = description.serializableHadoopConf.value
hadoopConf.set("mapreduce.job.id", jobId.toString)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactorySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactorySuite.scala
new file mode 100644
index 000000000000..bd2030797441
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactorySuite.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
+import org.mockito.Mockito._
+import org.scalatest.PrivateMethodTester
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.internal.io.FileCommitProtocol
+import org.apache.spark.sql.execution.datasources.WriteJobDescription
+import org.apache.spark.util.SerializableConfiguration
+
+class FileWriterFactorySuite extends SparkFunSuite with PrivateMethodTester {
+
+ test("SPARK-48484: V2Write uses different TaskAttemptIds for different task attempts") {
+ val jobDescription = mock(classOf[WriteJobDescription])
+ when(jobDescription.serializableHadoopConf).thenReturn(
+ new SerializableConfiguration(new Configuration(false)))
+ val committer = mock(classOf[FileCommitProtocol])
+
+ val writerFactory = FileWriterFactory(jobDescription, committer)
+ val createTaskAttemptContext =
+ PrivateMethod[TaskAttemptContextImpl](Symbol("createTaskAttemptContext"))
+
+ val attemptContext =
+ writerFactory.invokePrivate(createTaskAttemptContext(0, 1))
+ val attemptContext1 =
+ writerFactory.invokePrivate(createTaskAttemptContext(0, 2))
+ assert(attemptContext.getTaskAttemptID.getTaskID == attemptContext1.getTaskAttemptID.getTaskID)
+ assert(attemptContext.getTaskAttemptID.getId != attemptContext1.getTaskAttemptID.getId)
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org