You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2014/10/23 00:08:34 UTC
git commit: [SPARK-3877][YARN] Throw an exception when application is
not successful so that the exit code wil be set to 1 (for branch-1.1)
Repository: spark
Updated Branches:
refs/heads/branch-1.1 457ef5955 -> eb62094e8
[SPARK-3877][YARN] Throw an exception when application is not successful so that the exit code wil be set to 1 (for branch-1.1)
This is a patch to fix SPARK-3877 in branch-1.1. See also #2732
Author: zsxwing <zs...@gmail.com>
Closes #2748 from zsxwing/SPARK-3877-branch-1.1 and squashes the following commits:
3701984 [zsxwing] Remove System.exit from Client.scala
8681881 [zsxwing] [SPARK-3877] Throw an exception when application is not successful so that the exit code wil be set to 1
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eb62094e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eb62094e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eb62094e
Branch: refs/heads/branch-1.1
Commit: eb62094e820992355e4cf8ba0e881417f8c6bbfa
Parents: 457ef59
Author: zsxwing <zs...@gmail.com>
Authored: Wed Oct 22 15:08:28 2014 -0700
Committer: Andrew Or <an...@gmail.com>
Committed: Wed Oct 22 15:08:28 2014 -0700
----------------------------------------------------------------------
.../org/apache/spark/deploy/yarn/Client.scala | 27 ++++++++------------
.../org/apache/spark/deploy/yarn/Client.scala | 27 ++++++++------------
2 files changed, 22 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/eb62094e/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 3607eed..c51ce62 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.{Apps, Records}
-import org.apache.spark.{Logging, SparkConf}
+import org.apache.spark.{Logging, SparkConf, SparkException}
/**
* Version of [[org.apache.spark.deploy.yarn.ClientBase]] tailored to YARN's alpha API.
@@ -84,7 +84,9 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
def run() {
val appId = runApp()
- monitorApplication(appId)
+ if (!monitorApplication(appId)) {
+ throw new SparkException("Application is not successful")
+ }
}
def logClusterResourceDetails() {
@@ -138,10 +140,12 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
)
val state = report.getYarnApplicationState()
- if (state == YarnApplicationState.FINISHED ||
- state == YarnApplicationState.FAILED ||
+ if (state == YarnApplicationState.FINISHED) {
+ return report.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED
+ }
+ if (state == YarnApplicationState.FAILED ||
state == YarnApplicationState.KILLED) {
- return true
+ return false
}
}
true
@@ -162,16 +166,7 @@ object Client {
val sparkConf = new SparkConf
- try {
- val args = new ClientArguments(argStrings, sparkConf)
- new Client(args, sparkConf).run()
- } catch {
- case e: Exception => {
- Console.err.println(e.getMessage)
- System.exit(1)
- }
- }
-
- System.exit(0)
+ val args = new ClientArguments(argStrings, sparkConf)
+ new Client(args, sparkConf).run()
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/eb62094e/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 0b55282..da05f7d 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.Records
-import org.apache.spark.{Logging, SparkConf}
+import org.apache.spark.{Logging, SparkConf, SparkException}
/**
@@ -94,7 +94,9 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
def run() {
val appId = runApp()
- monitorApplication(appId)
+ if (!monitorApplication(appId)) {
+ throw new SparkException("Application is not successful")
+ }
}
def logClusterResourceDetails() {
@@ -144,10 +146,12 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
)
val state = report.getYarnApplicationState()
- if (state == YarnApplicationState.FINISHED ||
- state == YarnApplicationState.FAILED ||
+ if (state == YarnApplicationState.FINISHED) {
+ return report.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED
+ }
+ if (state == YarnApplicationState.FAILED ||
state == YarnApplicationState.KILLED) {
- return true
+ return false
}
}
true
@@ -168,17 +172,8 @@ object Client {
System.setProperty("SPARK_YARN_MODE", "true")
val sparkConf = new SparkConf()
- try {
- val args = new ClientArguments(argStrings, sparkConf)
- new Client(args, sparkConf).run()
- } catch {
- case e: Exception => {
- Console.err.println(e.getMessage)
- System.exit(1)
- }
- }
-
- System.exit(0)
+ val args = new ClientArguments(argStrings, sparkConf)
+ new Client(args, sparkConf).run()
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org