You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/05/15 21:15:46 UTC
spark git commit: [SPARK-7504] [YARN] NullPointerException when
initializing SparkContext in YARN-cluster mode
Repository: spark
Updated Branches:
refs/heads/master ad92af9db -> 8e3822a07
[SPARK-7504] [YARN] NullPointerException when initializing SparkContext in YARN-cluster mode
Added a simple checking for SparkContext.
Also added two rational checking against null at AM object.
Author: ehnalis <zo...@gmail.com>
Closes #6083 from ehnalis/cluster and squashes the following commits:
926bd96 [ehnalis] Moved check to SparkContext.
7c89b6e [ehnalis] Remove false line.
ea2a5fe [ehnalis] [SPARK-7504] [YARN] NullPointerException when initializing SparkContext in YARN-cluster mode
4924e01 [ehnalis] [SPARK-7504] [YARN] NullPointerException when initializing SparkContext in YARN-cluster mode
39e4fa3 [ehnalis] SPARK-7504 [YARN] NullPointerException when initializing SparkContext in YARN-cluster mode
9f287c5 [ehnalis] [SPARK-7504] [YARN] NullPointerException when initializing SparkContext in YARN-cluster mode
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8e3822a0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8e3822a0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8e3822a0
Branch: refs/heads/master
Commit: 8e3822a0794b8b18436bd63d6859d40139a77090
Parents: ad92af9
Author: ehnalis <zo...@gmail.com>
Authored: Fri May 15 12:14:02 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Fri May 15 12:15:42 2015 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/SparkContext.scala | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8e3822a0/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index b59f562..af276e7 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -371,6 +371,14 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
throw new SparkException("An application name must be set in your configuration")
}
+ // System property spark.yarn.app.id must be set if user code ran by AM on a YARN cluster
+ // yarn-standalone is deprecated, but still supported
+ if ((master == "yarn-cluster" || master == "yarn-standalone") &&
+ !_conf.contains("spark.yarn.app.id")) {
+ throw new SparkException("Detected yarn-cluster mode, but isn't running on a cluster. " +
+ "Deployment to YARN is not supported directly by SparkContext. Please use spark-submit.")
+ }
+
if (_conf.getBoolean("spark.logConf", false)) {
logInfo("Spark configuration:\n" + _conf.toDebugString)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org