You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by js...@apache.org on 2018/04/27 07:25:11 UTC
spark git commit: [SPARK-23830][YARN] added check to ensure main
method is found
Repository: spark
Updated Branches:
refs/heads/master 8aa1d7b0e -> 109935fc5
[SPARK-23830][YARN] added check to ensure main method is found
## What changes were proposed in this pull request?
When a user specifies the wrong class -- or, in fact, a class instead of an object -- Spark throws an NPE which is not useful for debugging. This was reported in [SPARK-23830](https://issues.apache.org/jira/browse/SPARK-23830). This PR adds a check to ensure the main method was found and logs a useful error in the event that it's null.
## How was this patch tested?
* Unit tests + Manual testing
* The scope of the changes is very limited
Author: eric-maynard <em...@cloudera.com>
Author: Eric Maynard <em...@cloudera.com>
Closes #21168 from eric-maynard/feature/SPARK-23830.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/109935fc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/109935fc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/109935fc
Branch: refs/heads/master
Commit: 109935fc5d8b3d381bb1b09a4a570040a0a1846f
Parents: 8aa1d7b
Author: eric-maynard <em...@cloudera.com>
Authored: Fri Apr 27 15:25:07 2018 +0800
Committer: jerryshao <ss...@hortonworks.com>
Committed: Fri Apr 27 15:25:07 2018 +0800
----------------------------------------------------------------------
.../apache/spark/deploy/yarn/ApplicationMaster.scala | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/109935fc/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
----------------------------------------------------------------------
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 6508400..595077e 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -18,7 +18,7 @@
package org.apache.spark.deploy.yarn
import java.io.{File, IOException}
-import java.lang.reflect.InvocationTargetException
+import java.lang.reflect.{InvocationTargetException, Modifier}
import java.net.{Socket, URI, URL}
import java.security.PrivilegedExceptionAction
import java.util.concurrent.{TimeoutException, TimeUnit}
@@ -675,9 +675,14 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments) extends
val userThread = new Thread {
override def run() {
try {
- mainMethod.invoke(null, userArgs.toArray)
- finish(FinalApplicationStatus.SUCCEEDED, ApplicationMaster.EXIT_SUCCESS)
- logDebug("Done running users class")
+ if (!Modifier.isStatic(mainMethod.getModifiers)) {
+ logError(s"Could not find static main method in object ${args.userClass}")
+ finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_EXCEPTION_USER_CLASS)
+ } else {
+ mainMethod.invoke(null, userArgs.toArray)
+ finish(FinalApplicationStatus.SUCCEEDED, ApplicationMaster.EXIT_SUCCESS)
+ logDebug("Done running user class")
+ }
} catch {
case e: InvocationTargetException =>
e.getCause match {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org