You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by td...@apache.org on 2015/01/12 22:16:09 UTC
spark git commit: [SPARK-4999][Streaming] Change storeInBlockManager
to false by default
Repository: spark
Updated Branches:
refs/heads/master aff49a3ee -> 3aed3051c
[SPARK-4999][Streaming] Change storeInBlockManager to false by default
Currently WAL-backed block is read out from HDFS and put into BlockManger with storage level MEMORY_ONLY_SER by default, since WAL-backed block is already materialized in HDFS with fault-tolerance, no need to put into BlockManger again by default.
Author: jerryshao <sa...@intel.com>
Closes #3906 from jerryshao/SPARK-4999 and squashes the following commits:
b95f95e [jerryshao] Change storeInBlockManager to false by default
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3aed3051
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3aed3051
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3aed3051
Branch: refs/heads/master
Commit: 3aed3051c0b6cd5f38d7db7d20fa7a1680bfde6f
Parents: aff49a3
Author: jerryshao <sa...@intel.com>
Authored: Mon Jan 12 13:14:44 2015 -0800
Committer: Tathagata Das <ta...@gmail.com>
Committed: Mon Jan 12 13:14:44 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/streaming/dstream/ReceiverInputDStream.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3aed3051/streaming/src/main/scala/org/apache/spark/streaming/dstream/ReceiverInputDStream.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/ReceiverInputDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/ReceiverInputDStream.scala
index c834744..afd3c4b 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/ReceiverInputDStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/ReceiverInputDStream.scala
@@ -86,7 +86,7 @@ abstract class ReceiverInputDStream[T: ClassTag](@transient ssc_ : StreamingCont
}.toArray
// Since storeInBlockManager = false, the storage level does not matter.
new WriteAheadLogBackedBlockRDD[T](ssc.sparkContext,
- blockIds, logSegments, storeInBlockManager = true, StorageLevel.MEMORY_ONLY_SER)
+ blockIds, logSegments, storeInBlockManager = false, StorageLevel.MEMORY_ONLY_SER)
} else {
new BlockRDD[T](ssc.sc, blockIds)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org