You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by js...@apache.org on 2017/10/23 06:26:46 UTC
spark git commit: [SPARK-22319][CORE][BACKPORT-2.2] call
loginUserFromKeytab before accessing hdfs
Repository: spark
Updated Branches:
refs/heads/branch-2.2 f8c83fdc5 -> bf8163f5b
[SPARK-22319][CORE][BACKPORT-2.2] call loginUserFromKeytab before accessing hdfs
In SparkSubmit, call loginUserFromKeytab before attempting to make RPC calls to the NameNode.
Same as #https://github.com/apache/spark/pull/19540, but for branch-2.2.
Manually tested for master as described in https://github.com/apache/spark/pull/19540.
Author: Steven Rand <sr...@palantir.com>
Closes #19554 from sjrand/SPARK-22319-branch-2.2.
Change-Id: Ic550a818fd6a3f38b356ac48029942d463738458
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bf8163f5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bf8163f5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bf8163f5
Branch: refs/heads/branch-2.2
Commit: bf8163f5be55a94e02849ccbaf755702a2c6c68f
Parents: f8c83fd
Author: Steven Rand <sr...@palantir.com>
Authored: Mon Oct 23 14:26:03 2017 +0800
Committer: jerryshao <ss...@hortonworks.com>
Committed: Mon Oct 23 14:26:03 2017 +0800
----------------------------------------------------------------------
.../org/apache/spark/deploy/SparkSubmit.scala | 38 ++++++++++----------
1 file changed, 19 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bf8163f5/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 86d578e..4f2f2c1 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -316,6 +316,25 @@ object SparkSubmit extends CommandLineUtils {
RPackageUtils.checkAndBuildRPackage(args.jars, printStream, args.verbose)
}
+ // assure a keytab is available from any place in a JVM
+ if (clusterManager == YARN || clusterManager == LOCAL) {
+ if (args.principal != null) {
+ require(args.keytab != null, "Keytab must be specified when principal is specified")
+ if (!new File(args.keytab).exists()) {
+ throw new SparkException(s"Keytab file: ${args.keytab} does not exist")
+ } else {
+ // Add keytab and principal configurations in sysProps to make them available
+ // for later use; e.g. in spark sql, the isolated class loader used to talk
+ // to HiveMetastore will use these settings. They will be set as Java system
+ // properties and then loaded by SparkConf
+ sysProps.put("spark.yarn.keytab", args.keytab)
+ sysProps.put("spark.yarn.principal", args.principal)
+
+ UserGroupInformation.loginUserFromKeytab(args.principal, args.keytab)
+ }
+ }
+ }
+
// In client mode, download remote files.
var localPrimaryResource: String = null
var localJars: String = null
@@ -582,25 +601,6 @@ object SparkSubmit extends CommandLineUtils {
}
}
- // assure a keytab is available from any place in a JVM
- if (clusterManager == YARN || clusterManager == LOCAL) {
- if (args.principal != null) {
- require(args.keytab != null, "Keytab must be specified when principal is specified")
- if (!new File(args.keytab).exists()) {
- throw new SparkException(s"Keytab file: ${args.keytab} does not exist")
- } else {
- // Add keytab and principal configurations in sysProps to make them available
- // for later use; e.g. in spark sql, the isolated class loader used to talk
- // to HiveMetastore will use these settings. They will be set as Java system
- // properties and then loaded by SparkConf
- sysProps.put("spark.yarn.keytab", args.keytab)
- sysProps.put("spark.yarn.principal", args.principal)
-
- UserGroupInformation.loginUserFromKeytab(args.principal, args.keytab)
- }
- }
- }
-
// In yarn-cluster mode, use yarn.Client as a wrapper around the user class
if (isYarnCluster) {
childMainClass = "org.apache.spark.deploy.yarn.Client"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org