[GitHub] [spark] weixiuli commented on a change in pull request #30716: [SPARK-33747][CORE] Avoid calling unregisterMapOutput when the map stage is being rerunning.

[GitBox <gi...@apache.org>]

weixiuli commented on a change in pull request #30716:
URL: https://github.com/apache/spark/pull/30716#discussion_r546631296



##########
File path: core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
##########
@@ -2035,6 +2040,107 @@ class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with Ti
     assert(scheduler.activeJobs.isEmpty)
   }
 
+  def reInit(): Unit = {
+    assert(sc != null)
+    val dagOutputTracker = mapOutputTracker
+    val taskSchedulerImpl = new TaskSchedulerImpl(sc) {
+      override def submitTasks(taskSet: TaskSet) = {
+        super.submitTasks(taskSet)
+        taskSet.tasks.foreach(_.epoch = dagOutputTracker.getEpoch)
+        taskSets += taskSet
+      }
+
+      override def cancelTasks(stageId: Int, interruptThread: Boolean) {
+        cancelledStages += stageId
+      }
+    }
+    taskSchedulerImpl.initialize(new FakeSchedulerBackend)
+    scheduler = new DAGScheduler(
+      sc,
+      taskSchedulerImpl,
+      sc.listenerBus,
+      mapOutputTracker,
+      blockManagerMaster,
+      sc.env)
+    dagEventProcessLoopTester = new DAGSchedulerEventProcessLoopTester(scheduler)
+  }
+
+  test("Test dagScheduler.shouldUnregisterMapOutput with map stage not running") {
+    reInit()
+    val shuffleMapRdd = new MyRDD(sc, 2, Nil)
+    val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(2))
+    val shuffleId = shuffleDep.shuffleId
+    val reduceRdd = new MyRDD(sc, 2, List(shuffleDep), tracker = mapOutputTracker)
+    submit(reduceRdd, Array(0, 1))
+    complete(taskSets(0), Seq(
+      (Success, makeMapStatus("hostA", reduceRdd.partitions.length)),
+      (Success, makeMapStatus("hostB", reduceRdd.partitions.length))))
+    // The MapOutputTracker should know about both map output locations.
+    assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1.host).toSet ===
+      HashSet("hostA", "hostB"))
+
+    // The first result task fails, with a fetch failure for the output from the first mapper.
+    runEvent(makeCompletionEvent(
+      taskSets(1).tasks(0),
+      FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0, 0, 0, "ignored"),
+      null))
+    assert(sparkListener.failedStages.contains(1))
+
+    val mapStatuses = mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
+    // unregisterMapOutput with a fetchFailed.
+    assert(mapStatuses.count(_ != null) === 1)
+    assert(mapStatuses(1).location === makeBlockManagerId("hostB"))
+  }
+
+  test("Test dagScheduler.shouldUnregisterMapOutput with map stage re-running") {
+    reInit()
+    val shuffleMapRdd = new MyRDD(sc, 3, Nil)
+    val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(2))
+    val shuffleId = shuffleDep.shuffleId
+    val reduceRdd = new MyRDD(sc, 2, List(shuffleDep), tracker = mapOutputTracker)
+    submit(reduceRdd, Array(0, 1))
+    complete(taskSets(0), Seq(
+      (Success, makeMapStatus("hostA", reduceRdd.partitions.length)),
+      (Success, makeMapStatus("hostB", reduceRdd.partitions.length)),
+      (Success, makeMapStatus("hostC", reduceRdd.partitions.length))))
+    // The MapOutputTracker should know about both map output locations.
+    assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1.host).toSet ===
+      HashSet("hostA", "hostB", "hostC"))
+
+    runEvent(makeCompletionEvent(
+      taskSets(1).tasks(0),
+      FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0, 0, 0, "ignored"),
+      null))
+    runEvent(makeCompletionEvent(
+      taskSets(1).tasks(1),
+      FetchFailed(makeBlockManagerId("hostB"), shuffleId, 1, 1, 0, "ignored"),
+      null))
+
+    assert(sparkListener.failedStages.contains(1))
+
+    // Wait for a long time to make sure the map stage was resubmitted.
+    eventually(timeout(1000 milliseconds), interval(10 milliseconds)) {
+      assert(scheduler.runningStages.nonEmpty && taskSets.size == 3)
+    }
+
+    runEvent(makeCompletionEvent(
+      taskSets(2).tasks(0),
+      Success,
+      makeMapStatus("hostA", reduceRdd.partitions.length)))
+
+    runEvent(makeCompletionEvent(
+      taskSets(1).tasks(1),

Review comment:
        Oh, I'm sorry,i have update my ut, thanks.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org