You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2015/06/08 10:34:28 UTC
spark git commit: [SPARK-7705] [YARN] Cleanup of .sparkStaging
directory fails if application is killed
Repository: spark
Updated Branches:
refs/heads/master 10fc2f6f5 -> eacd4a929
[SPARK-7705] [YARN] Cleanup of .sparkStaging directory fails if application is killed
As I have tested, if we cancel or kill the app then the final status may be undefined, killed or succeeded, so clean up staging directory when appMaster exit at any final application status.
Author: linweizhong <li...@huawei.com>
Closes #6409 from Sephiroth-Lin/SPARK-7705 and squashes the following commits:
3a5a0a5 [linweizhong] Update
83dc274 [linweizhong] Update
923d44d [linweizhong] Update
0dd7c2d [linweizhong] Update
b76a102 [linweizhong] Update code style
7846b69 [linweizhong] Update
bd6cf0d [linweizhong] Refactor
aed9f18 [linweizhong] Clean up stagingDir when launch app on yarn
95595c3 [linweizhong] Cleanup of .sparkStaging directory when AppMaster exit at any final application status
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eacd4a92
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eacd4a92
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eacd4a92
Branch: refs/heads/master
Commit: eacd4a929bf5d697c33b1b705dcf958651cd20f4
Parents: 10fc2f6
Author: linweizhong <li...@huawei.com>
Authored: Mon Jun 8 09:34:16 2015 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Mon Jun 8 09:34:16 2015 +0100
----------------------------------------------------------------------
.../org/apache/spark/deploy/yarn/Client.scala | 34 ++++++++++++--------
1 file changed, 21 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/eacd4a92/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 234051e..f4d4321 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -121,25 +121,32 @@ private[spark] class Client(
} catch {
case e: Throwable =>
if (appId != null) {
- val appStagingDir = getAppStagingDir(appId)
- try {
- val preserveFiles = sparkConf.getBoolean("spark.yarn.preserve.staging.files", false)
- val stagingDirPath = new Path(appStagingDir)
- val fs = FileSystem.get(hadoopConf)
- if (!preserveFiles && fs.exists(stagingDirPath)) {
- logInfo("Deleting staging directory " + stagingDirPath)
- fs.delete(stagingDirPath, true)
- }
- } catch {
- case ioe: IOException =>
- logWarning("Failed to cleanup staging dir " + appStagingDir, ioe)
- }
+ cleanupStagingDir(appId)
}
throw e
}
}
/**
+ * Cleanup application staging directory.
+ */
+ private def cleanupStagingDir(appId: ApplicationId): Unit = {
+ val appStagingDir = getAppStagingDir(appId)
+ try {
+ val preserveFiles = sparkConf.getBoolean("spark.yarn.preserve.staging.files", false)
+ val stagingDirPath = new Path(appStagingDir)
+ val fs = FileSystem.get(hadoopConf)
+ if (!preserveFiles && fs.exists(stagingDirPath)) {
+ logInfo("Deleting staging directory " + stagingDirPath)
+ fs.delete(stagingDirPath, true)
+ }
+ } catch {
+ case ioe: IOException =>
+ logWarning("Failed to cleanup staging dir " + appStagingDir, ioe)
+ }
+ }
+
+ /**
* Set up the context for submitting our ApplicationMaster.
* This uses the YarnClientApplication not available in the Yarn alpha API.
*/
@@ -782,6 +789,7 @@ private[spark] class Client(
if (state == YarnApplicationState.FINISHED ||
state == YarnApplicationState.FAILED ||
state == YarnApplicationState.KILLED) {
+ cleanupStagingDir(appId)
return (state, report.getFinalApplicationStatus)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org