[spark] branch master updated: [SPARK-40096][CORE][TESTS][FOLLOW-UP] Fix flaky test case

[mr...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mridulm80 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 0b770e18bd7 [SPARK-40096][CORE][TESTS][FOLLOW-UP] Fix flaky test case
0b770e18bd7 is described below

commit 0b770e18bd7250a8215f2e593fa38d3094f7106c
Author: Kun Wan <wa...@apache.org>
AuthorDate: Tue Nov 8 12:12:34 2022 -0600

    [SPARK-40096][CORE][TESTS][FOLLOW-UP] Fix flaky test case
    
    ### What changes were proposed in this pull request?
    
    This PR is a followup of https://github.com/apache/spark/pull/37533 that fix the flaky test case.
    
    ### Why are the changes needed?
    
    The test case is flaky, and will failure due to some unexpected error.
    
    https://github.com/apache/spark/pull/37989
    https://github.com/apache/spark/actions/runs/3145115911/jobs/5112006948
    https://github.com/apache/spark/actions/runs/3146198025/jobs/5114387367
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, test-only.
    
    ### How was this patch tested?
    
    CI in this PR should verify that.
    
    Closes #38091 from wankunde/SPARK-40096-2.
    
    Authored-by: Kun Wan <wa...@apache.org>
    Signed-off-by: Mridul <mridul<at>gmail.com>
---
 .../scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala     | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 59e725e2b75..f4e67eba40d 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -4514,14 +4514,13 @@ class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with Ti
       initPushBasedShuffleConfs(conf)
 
       sc.conf.set("spark.shuffle.push.results.timeout", "1s")
-      val myScheduler = new MyDAGScheduler(
+      val scheduler = new DAGScheduler(
         sc,
         taskScheduler,
         sc.listenerBus,
         mapOutputTracker,
         blockManagerMaster,
-        sc.env,
-        shuffleMergeFinalize = false)
+        sc.env)
 
       val mergerLocs = Seq(makeBlockManagerId("hostA"), makeBlockManagerId("hostB"))
       val timeoutSecs = 1
@@ -4554,9 +4553,9 @@ class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with Ti
       val shuffleMapRdd = new MyRDD(sc, 1, Nil)
       val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(2))
       shuffleDep.setMergerLocs(mergerLocs)
-      val shuffleStage = myScheduler.createShuffleMapStage(shuffleDep, 0)
+      val shuffleStage = scheduler.createShuffleMapStage(shuffleDep, 0)
 
-      myScheduler.finalizeShuffleMerge(shuffleStage, registerMergeResults)
+      scheduler.finalizeShuffleMerge(shuffleStage, registerMergeResults)
       sendRequestsLatch.await()
       verify(blockStoreClient, times(2))
         .finalizeShuffleMerge(any(), any(), any(), any(), any())


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org