You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/04/28 22:49:32 UTC
spark git commit: [MINOR] [CORE] Warn users who try to cache RDDs
with dynamic allocation on.
Repository: spark
Updated Branches:
refs/heads/master 53befacce -> 28b1af742
[MINOR] [CORE] Warn users who try to cache RDDs with dynamic allocation on.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #5751 from vanzin/cached-rdd-warning and squashes the following commits:
554cc07 [Marcelo Vanzin] Change message.
9efb9da [Marcelo Vanzin] [minor] [core] Warn users who try to cache RDDs with dynamic allocation on.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/28b1af74
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/28b1af74
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/28b1af74
Branch: refs/heads/master
Commit: 28b1af7420e0b5e7e2dfc09eafc45fe2ffcde5ec
Parents: 53befac
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Tue Apr 28 13:49:29 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Tue Apr 28 13:49:29 2015 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/SparkContext.scala | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/28b1af74/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 65b903a..d0cf2a8 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -1396,6 +1396,11 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* Register an RDD to be persisted in memory and/or disk storage
*/
private[spark] def persistRDD(rdd: RDD[_]) {
+ _executorAllocationManager.foreach { _ =>
+ logWarning(
+ s"Dynamic allocation currently does not support cached RDDs. Cached data for RDD " +
+ s"${rdd.id} will be lost when executors are removed.")
+ }
persistentRdds(rdd.id) = rdd
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org