You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2017/10/25 20:53:07 UTC
spark git commit: [SPARK-22341][YARN] Impersonate correct user when
preparing resources.
Repository: spark
Updated Branches:
refs/heads/master 6ea8a56ca -> d212ef14b
[SPARK-22341][YARN] Impersonate correct user when preparing resources.
The bug was introduced in SPARK-22290, which changed how the app's user
is impersonated in the AM. The changed missed an initialization function
that needs to be run as the app owner (who has the right credentials to
read from HDFS).
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #19566 from vanzin/SPARK-22341.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d212ef14
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d212ef14
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d212ef14
Branch: refs/heads/master
Commit: d212ef14be7c2864cc529e48a02a47584e46f7a5
Parents: 6ea8a56
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Wed Oct 25 13:53:01 2017 -0700
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Wed Oct 25 13:53:01 2017 -0700
----------------------------------------------------------------------
.../spark/deploy/yarn/ApplicationMaster.scala | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d212ef14/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
----------------------------------------------------------------------
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index f616723..244d912 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -97,9 +97,7 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments) extends
}
}
- private val client = ugi.doAs(new PrivilegedExceptionAction[YarnRMClient]() {
- def run: YarnRMClient = new YarnRMClient()
- })
+ private val client = doAsUser { new YarnRMClient() }
// Default to twice the number of executors (twice the maximum number of executors if dynamic
// allocation is enabled), with a minimum of 3.
@@ -178,7 +176,7 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments) extends
// Load the list of localized files set by the client. This is used when launching executors,
// and is loaded here so that these configs don't pollute the Web UI's environment page in
// cluster mode.
- private val localResources = {
+ private val localResources = doAsUser {
logInfo("Preparing Local resources")
val resources = HashMap[String, LocalResource]()
@@ -240,9 +238,9 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments) extends
}
final def run(): Int = {
- ugi.doAs(new PrivilegedExceptionAction[Unit]() {
- def run: Unit = runImpl()
- })
+ doAsUser {
+ runImpl()
+ }
exitCode
}
@@ -790,6 +788,12 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments) extends
}
}
+ private def doAsUser[T](fn: => T): T = {
+ ugi.doAs(new PrivilegedExceptionAction[T]() {
+ override def run: T = fn
+ })
+ }
+
}
object ApplicationMaster extends Logging {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org