You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/05/16 06:38:10 UTC
git commit: SPARK-1860: Do not cleanup application work/ directories
by default
Repository: spark
Updated Branches:
refs/heads/master 94c513960 -> bb98ecafc
SPARK-1860: Do not cleanup application work/ directories by default
This causes an unrecoverable error for applications that are running for longer
than 7 days that have jars added to the SparkContext, as the jars are cleaned up
even though the application is still running.
Author: Aaron Davidson <aa...@databricks.com>
Closes #800 from aarondav/shitty-defaults and squashes the following commits:
a573fbb [Aaron Davidson] SPARK-1860: Do not cleanup application work/ directories by default
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bb98ecaf
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bb98ecaf
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bb98ecaf
Branch: refs/heads/master
Commit: bb98ecafce196ecc5bc3a1e4cc9264df7b752c6a
Parents: 94c5139
Author: Aaron Davidson <aa...@databricks.com>
Authored: Thu May 15 21:37:58 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Thu May 15 21:37:58 2014 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala | 2 +-
docs/configuration.md | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bb98ecaf/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index 134624c..fb9cc11 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -65,7 +65,7 @@ private[spark] class Worker(
val REGISTRATION_TIMEOUT = 20.seconds
val REGISTRATION_RETRIES = 3
- val CLEANUP_ENABLED = conf.getBoolean("spark.worker.cleanup.enabled", true)
+ val CLEANUP_ENABLED = conf.getBoolean("spark.worker.cleanup.enabled", false)
// How often worker will clean up old app folders
val CLEANUP_INTERVAL_MILLIS = conf.getLong("spark.worker.cleanup.interval", 60 * 30) * 1000
// TTL for app folders/data; after TTL expires it will be cleaned up
http://git-wip-us.apache.org/repos/asf/spark/blob/bb98ecaf/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index 2eed96f..f89040d 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -390,10 +390,11 @@ Apart from these, the following properties are also available, and may be useful
</tr>
<tr>
<td>spark.worker.cleanup.enabled</td>
- <td>true</td>
+ <td>false</td>
<td>
Enable periodic cleanup of worker / application directories. Note that this only affects standalone
- mode, as YARN works differently.
+ mode, as YARN works differently. Applications directories are cleaned up regardless of whether
+ the application is still running.
</td>
</tr>
<tr>