You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/05/12 10:40:58 UTC
spark git commit: [SPARK-7467] Dag visualization: treat checkpoint as
an RDD operation
Repository: spark
Updated Branches:
refs/heads/master 82e890fb1 -> f3e8e6006
[SPARK-7467] Dag visualization: treat checkpoint as an RDD operation
Such that a checkpoint RDD does not go into random scopes on the UI, e.g. `take`. We've seen this in streaming.
Author: Andrew Or <an...@databricks.com>
Closes #6004 from andrewor14/dag-viz-checkpoint and squashes the following commits:
9217439 [Andrew Or] Fix checkpoints
4ae8806 [Andrew Or] Merge branch 'master' of github.com:apache/spark into dag-viz-checkpoint
19bc07b [Andrew Or] Treat checkpoint as an RDD operation
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f3e8e600
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f3e8e600
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f3e8e600
Branch: refs/heads/master
Commit: f3e8e60063ccf0d713d03e671a3231560475f90d
Parents: 82e890f
Author: Andrew Or <an...@databricks.com>
Authored: Tue May 12 01:40:55 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Tue May 12 01:40:55 2015 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/rdd/RDD.scala | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f3e8e600/core/src/main/scala/org/apache/spark/rdd/RDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 7dad30e..02a94ba 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -1523,13 +1523,15 @@ abstract class RDD[T: ClassTag](
* has completed (therefore the RDD has been materialized and potentially stored in memory).
* doCheckpoint() is called recursively on the parent RDDs.
*/
- private[spark] def doCheckpoint() {
- if (!doCheckpointCalled) {
- doCheckpointCalled = true
- if (checkpointData.isDefined) {
- checkpointData.get.doCheckpoint()
- } else {
- dependencies.foreach(_.rdd.doCheckpoint())
+ private[spark] def doCheckpoint(): Unit = {
+ RDDOperationScope.withScope(sc, "checkpoint", false, true) {
+ if (!doCheckpointCalled) {
+ doCheckpointCalled = true
+ if (checkpointData.isDefined) {
+ checkpointData.get.doCheckpoint()
+ } else {
+ dependencies.foreach(_.rdd.doCheckpoint())
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org