You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/10/16 06:25:10 UTC

[1/2] git commit: Serialize and restore spark.cleaner.ttl to savepoint

Updated Branches:
  refs/heads/master 6dbd2208f -> b5346064d


Serialize and restore spark.cleaner.ttl to savepoint


Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/fbe40c58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/fbe40c58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/fbe40c58

Branch: refs/heads/master
Commit: fbe40c5806a01e38c56b12a09bc4b84681a99602
Parents: a106ed8
Author: Vadim Chekan <ko...@gmail.com>
Authored: Fri Sep 20 12:13:48 2013 -0700
Committer: Vadim Chekan <ko...@gmail.com>
Committed: Fri Sep 20 12:13:48 2013 -0700

----------------------------------------------------------------------
 .../src/main/scala/org/apache/spark/streaming/Checkpoint.scala   | 2 ++
 .../main/scala/org/apache/spark/streaming/StreamingContext.scala | 4 ++++
 2 files changed, 6 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/fbe40c58/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala b/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala
index 2d8f072..bb9feba 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration
 
 import org.apache.spark.Logging
 import org.apache.spark.io.CompressionCodec
+import org.apache.spark.util.MetadataCleaner
 
 
 private[streaming]
@@ -40,6 +41,7 @@ class Checkpoint(@transient ssc: StreamingContext, val checkpointTime: Time)
   val checkpointDir = ssc.checkpointDir
   val checkpointDuration = ssc.checkpointDuration
   val pendingTimes = ssc.scheduler.jobManager.getPendingTimes()
+  val delaySeconds = MetadataCleaner.getDelaySeconds
 
   def validate() {
     assert(master != null, "Checkpoint.master is null")

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/fbe40c58/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
index 878725c..098081d 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
@@ -100,6 +100,10 @@ class StreamingContext private (
       "both SparkContext and checkpoint as null")
   }
 
+  if(cp_ != null && cp_.delaySeconds >= 0 && MetadataCleaner.getDelaySeconds < 0) {
+    MetadataCleaner.setDelaySeconds(cp_.delaySeconds)
+  }
+
   if (MetadataCleaner.getDelaySeconds < 0) {
     throw new SparkException("Spark Streaming cannot be used without setting spark.cleaner.ttl; "
       + "set this property before creating a SparkContext (use SPARK_JAVA_OPTS for the shell)")


[2/2] git commit: Merge pull request #8 from vchekan/checkpoint-ttl-restore

Posted by ma...@apache.org.
Merge pull request #8 from vchekan/checkpoint-ttl-restore

Serialize and restore spark.cleaner.ttl to savepoint

In accordance to conversation in spark-dev maillist, preserve spark.cleaner.ttl parameter when serializing checkpoint.


Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/b5346064
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/b5346064
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/b5346064

Branch: refs/heads/master
Commit: b5346064d6a2b6858e77b718d2fbadd691374282
Parents: 6dbd220 fbe40c5
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Tue Oct 15 21:25:03 2013 -0700
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Tue Oct 15 21:25:03 2013 -0700

----------------------------------------------------------------------
 .../src/main/scala/org/apache/spark/streaming/Checkpoint.scala   | 2 ++
 .../main/scala/org/apache/spark/streaming/StreamingContext.scala | 4 ++++
 2 files changed, 6 insertions(+)
----------------------------------------------------------------------