You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/04/18 04:17:10 UTC
spark git commit: [SPARK-6975][Yarn] Fix argument validation error
Repository: spark
Updated Branches:
refs/heads/master 199133733 -> d850b4bd3
[SPARK-6975][Yarn] Fix argument validation error
`numExecutors` checking is failed when dynamic allocation is enabled with default configuration. Details can be seen is [SPARK-6975](https://issues.apache.org/jira/browse/SPARK-6975). sryza, please help me to review this, not sure is this the correct way, I think previous you change this part :)
Author: jerryshao <sa...@intel.com>
Closes #5551 from jerryshao/SPARK-6975 and squashes the following commits:
4335da1 [jerryshao] Change according to the comments
77bdcbd [jerryshao] Fix argument validation error
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d850b4bd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d850b4bd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d850b4bd
Branch: refs/heads/master
Commit: d850b4bd3a294dd245881e03f7f94bf970a7ee79
Parents: 1991337
Author: jerryshao <sa...@intel.com>
Authored: Fri Apr 17 19:17:06 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Fri Apr 17 19:17:06 2015 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/deploy/yarn/ClientArguments.scala | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d850b4bd/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
index da6798c..1423533 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
@@ -103,9 +103,13 @@ private[spark] class ClientArguments(args: Array[String], sparkConf: SparkConf)
* This is intended to be called only after the provided arguments have been parsed.
*/
private def validateArgs(): Unit = {
- if (numExecutors <= 0) {
+ if (numExecutors < 0 || (!isDynamicAllocationEnabled && numExecutors == 0)) {
throw new IllegalArgumentException(
- "You must specify at least 1 executor!\n" + getUsageMessage())
+ s"""
+ |Number of executors was $numExecutors, but must be at least 1
+ |(or 0 if dynamic executor allocation is enabled).
+ |${getUsageMessage()}
+ """.stripMargin)
}
if (executorCores < sparkConf.getInt("spark.task.cpus", 1)) {
throw new SparkException("Executor cores must not be less than " +
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org