You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2018/08/08 00:32:46 UTC

spark git commit: [SPARK-25045][CORE] Make `RDDBarrier.mapParititions` similar to `RDD.mapPartitions`

Repository: spark
Updated Branches:
  refs/heads/master 66699c5c3 -> d90f1336d


[SPARK-25045][CORE] Make `RDDBarrier.mapParititions` similar to `RDD.mapPartitions`

## What changes were proposed in this pull request?

Signature of the function passed to `RDDBarrier.mapPartitions()` is different from that of `RDD.mapPartitions`. The later doesn’t take a `TaskContext`. We shall make the function signature the same to avoid confusion and misusage.

This PR proposes the following API changes:
- In `RDDBarrier`, migrate `mapPartitions` from
   ```
        def mapPartitions[S: ClassTag](
            f: (Iterator[T], BarrierTaskContext) => Iterator[S],
            preservesPartitioning: Boolean = false): RDD[S]
        }
   ```
    to
   ```
        def mapPartitions[S: ClassTag](
            f: Iterator[T] => Iterator[S],
            preservesPartitioning: Boolean = false): RDD[S]
        }
   ```
- Add new static method to get a `BarrierTaskContext`:
   ```
        object BarrierTaskContext {
           def get(): BarrierTaskContext
        }
   ```

## How was this patch tested?

Existing test cases.

Author: Xingbo Jiang <xi...@databricks.com>

Closes #22026 from jiangxb1987/mapPartitions.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d90f1336
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d90f1336
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d90f1336

Branch: refs/heads/master
Commit: d90f1336d87199aac56fe227a0fe14ab0ae3a332
Parents: 66699c5
Author: Xingbo Jiang <xi...@databricks.com>
Authored: Tue Aug 7 17:32:41 2018 -0700
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Tue Aug 7 17:32:41 2018 -0700

----------------------------------------------------------------------
 .../org/apache/spark/BarrierTaskContext.scala   | 14 +++++++++++--
 .../scala/org/apache/spark/rdd/RDDBarrier.scala |  7 +++----
 .../spark/BarrierStageOnSubmittedSuite.scala    | 22 ++++++++++----------
 .../org/apache/spark/SparkContextSuite.scala    |  6 ++++--
 .../org/apache/spark/rdd/RDDBarrierSuite.scala  |  6 +++---
 .../scheduler/BarrierTaskContextSuite.scala     | 15 ++++++++-----
 .../spark/scheduler/DAGSchedulerSuite.scala     |  4 ++--
 7 files changed, 45 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/d90f1336/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 8e2b155..de82798 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -72,7 +72,8 @@ class BarrierTaskContext(
    * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
    * shall lead to timeout of the function call.
    * {{{
-   *   rdd.barrier().mapPartitions { (iter, context) =>
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
    *       if (context.partitionId() == 0) {
    *           // Do nothing.
    *       } else {
@@ -85,7 +86,8 @@ class BarrierTaskContext(
    * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
    * second function call.
    * {{{
-   *   rdd.barrier().mapPartitions { (iter, context) =>
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
    *       try {
    *           // Do something that might throw an Exception.
    *           doSomething()
@@ -152,3 +154,11 @@ class BarrierTaskContext(
     addressesStr.split(",").map(_.trim()).map(new BarrierTaskInfo(_))
   }
 }
+
+object BarrierTaskContext {
+  /**
+   * Return the currently active BarrierTaskContext. This can be called inside of user functions to
+   * access contextual information about running barrier tasks.
+   */
+  def get(): BarrierTaskContext = TaskContext.get().asInstanceOf[BarrierTaskContext]
+}

http://git-wip-us.apache.org/repos/asf/spark/blob/d90f1336/core/src/main/scala/org/apache/spark/rdd/RDDBarrier.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDBarrier.scala b/core/src/main/scala/org/apache/spark/rdd/RDDBarrier.scala
index 71f38bf..978e7c0 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDDBarrier.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDDBarrier.scala
@@ -28,7 +28,7 @@ class RDDBarrier[T: ClassTag](rdd: RDD[T]) {
 
   /**
    * :: Experimental ::
-   * Maps partitions together with a provided [[org.apache.spark.BarrierTaskContext]].
+   * Generate a new barrier RDD by applying a function to each partitions of the prev RDD.
    *
    * `preservesPartitioning` indicates whether the input function preserves the partitioner, which
    * should be `false` unless `rdd` is a pair RDD and the input function doesn't modify the keys.
@@ -36,13 +36,12 @@ class RDDBarrier[T: ClassTag](rdd: RDD[T]) {
   @Experimental
   @Since("2.4.0")
   def mapPartitions[S: ClassTag](
-      f: (Iterator[T], BarrierTaskContext) => Iterator[S],
+      f: Iterator[T] => Iterator[S],
       preservesPartitioning: Boolean = false): RDD[S] = rdd.withScope {
     val cleanedF = rdd.sparkContext.clean(f)
     new MapPartitionsRDD(
       rdd,
-      (context: TaskContext, index: Int, iter: Iterator[T]) =>
-        cleanedF(iter, context.asInstanceOf[BarrierTaskContext]),
+      (context: TaskContext, index: Int, iter: Iterator[T]) => cleanedF(iter),
       preservesPartitioning,
       isFromBarrier = true
     )

http://git-wip-us.apache.org/repos/asf/spark/blob/d90f1336/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
index 75e13a9..2f21e61 100644
--- a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
@@ -61,7 +61,7 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     val prunedRdd = new PartitionPruningRDD(sc.parallelize(1 to 10, 4), index => index > 1)
     val rdd = prunedRdd
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
     testSubmitJob(sc, rdd,
       message = DAGScheduler.ERROR_MESSAGE_RUN_BARRIER_WITH_UNSUPPORTED_RDD_CHAIN_PATTERN)
   }
@@ -71,7 +71,7 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     val prunedRdd = new PartitionPruningRDD(sc.parallelize(1 to 10, 4), index => index > 1)
     val rdd = prunedRdd
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
       .repartition(2)
       .map(x => x + 1)
     testSubmitJob(sc, rdd,
@@ -84,7 +84,7 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     val rdd = prunedRdd
       .repartition(2)
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
     // Should be able to submit job and run successfully.
     val result = rdd.collect().sorted
     assert(result === Seq(6, 7, 8, 9, 10))
@@ -94,7 +94,7 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     sc = createSparkContext()
     val rdd = sc.parallelize(1 to 10, 4)
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
     testSubmitJob(sc, rdd, Some(Seq(1, 3)),
       message = DAGScheduler.ERROR_MESSAGE_RUN_BARRIER_WITH_UNSUPPORTED_RDD_CHAIN_PATTERN)
   }
@@ -103,7 +103,7 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     sc = createSparkContext()
     val rdd1 = sc.parallelize(1 to 10, 2)
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
     val rdd2 = sc.parallelize(1 to 20, 2)
     val rdd3 = rdd1
       .union(rdd2)
@@ -117,7 +117,7 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     sc = createSparkContext()
     val rdd = sc.parallelize(1 to 10, 4)
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
       .coalesce(1)
     // Fail the job on submit because the barrier RDD requires to run on 4 tasks, but the stage
     // only launches 1 task.
@@ -129,10 +129,10 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     sc = createSparkContext()
     val rdd1 = sc.parallelize(1 to 10, 4)
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
     val rdd2 = sc.parallelize(11 to 20, 4)
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
     val rdd3 = rdd1
       .zip(rdd2)
       .map(x => x._1 + x._2)
@@ -144,7 +144,7 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     sc = createSparkContext()
     val rdd1 = sc.parallelize(1 to 10, 4)
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
     val rdd2 = sc.parallelize(11 to 20, 4)
     val rdd3 = rdd1
       .zip(rdd2)
@@ -164,7 +164,7 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
 
     val rdd = sc.parallelize(1 to 10, 4)
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
     testSubmitJob(sc, rdd,
       message = DAGScheduler.ERROR_MESSAGE_RUN_BARRIER_WITH_DYN_ALLOCATION)
   }
@@ -179,7 +179,7 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
 
     val rdd = sc.parallelize(1 to 10, 4)
       .barrier()
-      .mapPartitions((iter, context) => iter)
+      .mapPartitions(iter => iter)
       .repartition(2)
       .map(x => x + 1)
     testSubmitJob(sc, rdd,

http://git-wip-us.apache.org/repos/asf/spark/blob/d90f1336/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index e5f31a0..cb44110 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -632,7 +632,8 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     val conf = new SparkConf().setAppName("test").setMaster("local[2]")
     sc = new SparkContext(conf)
     val rdd = sc.makeRDD(Seq(1, 2, 3, 4), 2)
-    val rdd2 = rdd.barrier().mapPartitions { (it, context) =>
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
       // If we don't get the expected taskInfos, the job shall abort due to stage failure.
       if (context.getTaskInfos().length != 2) {
         throw new SparkException("Expected taksInfos length is 2, actual length is " +
@@ -654,7 +655,8 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
       .setAppName("test-cluster")
     sc = new SparkContext(conf)
     val rdd = sc.makeRDD(Seq(1, 2, 3, 4), 2)
-    val rdd2 = rdd.barrier().mapPartitions { (it, context) =>
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
       // If we don't get the expected taskInfos, the job shall abort due to stage failure.
       if (context.getTaskInfos().length != 2) {
         throw new SparkException("Expected taksInfos length is 2, actual length is " +

http://git-wip-us.apache.org/repos/asf/spark/blob/d90f1336/core/src/test/scala/org/apache/spark/rdd/RDDBarrierSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/rdd/RDDBarrierSuite.scala b/core/src/test/scala/org/apache/spark/rdd/RDDBarrierSuite.scala
index 39d4618..d57ea4d 100644
--- a/core/src/test/scala/org/apache/spark/rdd/RDDBarrierSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/RDDBarrierSuite.scala
@@ -25,19 +25,19 @@ class RDDBarrierSuite extends SparkFunSuite with SharedSparkContext {
     val rdd = sc.parallelize(1 to 10, 4)
     assert(rdd.isBarrier() === false)
 
-    val rdd2 = rdd.barrier().mapPartitions((iter, context) => iter)
+    val rdd2 = rdd.barrier().mapPartitions(iter => iter)
     assert(rdd2.isBarrier() === true)
   }
 
   test("create an RDDBarrier in the middle of a chain of RDDs") {
     val rdd = sc.parallelize(1 to 10, 4).map(x => x * 2)
-    val rdd2 = rdd.barrier().mapPartitions((iter, context) => iter).map(x => (x, x + 1))
+    val rdd2 = rdd.barrier().mapPartitions(iter => iter).map(x => (x, x + 1))
     assert(rdd2.isBarrier() === true)
   }
 
   test("RDDBarrier with shuffle") {
     val rdd = sc.parallelize(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions((iter, context) => iter).repartition(2)
+    val rdd2 = rdd.barrier().mapPartitions(iter => iter).repartition(2)
     assert(rdd2.isBarrier() === false)
   }
 }

http://git-wip-us.apache.org/repos/asf/spark/blob/d90f1336/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 5f96d6f..36dd620 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -31,7 +31,8 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
       .setAppName("test-cluster")
     sc = new SparkContext(conf)
     val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { (it, context) =>
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
       // Sleep for a random time before global sync.
       Thread.sleep(Random.nextInt(1000))
       context.barrier()
@@ -49,7 +50,8 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
       .setAppName("test-cluster")
     sc = new SparkContext(conf)
     val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { (it, context) =>
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
       // Sleep for a random time before global sync.
       Thread.sleep(Random.nextInt(1000))
       context.barrier()
@@ -79,7 +81,8 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
       .setAppName("test-cluster")
     sc = new SparkContext(conf)
     val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { (it, context) =>
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
       // Task 3 shall sleep 2000ms to ensure barrier() call timeout
       if (context.taskAttemptId == 3) {
         Thread.sleep(2000)
@@ -103,7 +106,8 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
       .setAppName("test-cluster")
     sc = new SparkContext(conf)
     val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { (it, context) =>
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
       if (context.taskAttemptId != 0) {
         context.barrier()
       }
@@ -125,7 +129,8 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
       .setAppName("test-cluster")
     sc = new SparkContext(conf)
     val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { (it, context) =>
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
       try {
         if (context.taskAttemptId == 0) {
           // Due to some non-obvious reason, the code can trigger an Exception and skip the

http://git-wip-us.apache.org/repos/asf/spark/blob/d90f1336/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 8b2b6b6..211002b 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -1062,7 +1062,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
   }
 
   test("Retry all the tasks on a resubmitted attempt of a barrier stage caused by FetchFailure") {
-    val shuffleMapRdd = new MyRDD(sc, 2, Nil).barrier().mapPartitions((it, context) => it)
+    val shuffleMapRdd = new MyRDD(sc, 2, Nil).barrier().mapPartitions(iter => iter)
     val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(2))
     val shuffleId = shuffleDep.shuffleId
     val reduceRdd = new MyRDD(sc, 2, List(shuffleDep), tracker = mapOutputTracker)
@@ -1091,7 +1091,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
   }
 
   test("Retry all the tasks on a resubmitted attempt of a barrier stage caused by TaskKilled") {
-    val shuffleMapRdd = new MyRDD(sc, 2, Nil).barrier().mapPartitions((it, context) => it)
+    val shuffleMapRdd = new MyRDD(sc, 2, Nil).barrier().mapPartitions(iter => iter)
     val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(2))
     val shuffleId = shuffleDep.shuffleId
     val reduceRdd = new MyRDD(sc, 2, List(shuffleDep), tracker = mapOutputTracker)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org