You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2015/04/29 15:58:57 UTC
spark git commit: [SPARK-6918] [YARN] Secure HBase support.
Repository: spark
Updated Branches:
refs/heads/master f49284b5b -> baed3f2c7
[SPARK-6918] [YARN] Secure HBase support.
Obtain HBase security token with Kerberos credentials locally to be sent to executors. Tested on eBay's secure HBase cluster.
Similar to obtainTokenForNamenodes and fails gracefully if HBase classes are not included in path.
Requires hbase-site.xml to be in the classpath(typically via conf dir) for the zookeeper configuration. Should that go in the docs somewhere? Did not see an HBase section.
Author: Dean Chen <de...@gmail.com>
Closes #5586 from deanchen/master and squashes the following commits:
0c190ef [Dean Chen] [SPARK-6918][YARN] Secure HBase support.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/baed3f2c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/baed3f2c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/baed3f2c
Branch: refs/heads/master
Commit: baed3f2c73afd9c7d9de34f0485c507ac6a498b0
Parents: f49284b
Author: Dean Chen <de...@gmail.com>
Authored: Wed Apr 29 08:58:33 2015 -0500
Committer: Thomas Graves <tg...@apache.org>
Committed: Wed Apr 29 08:58:33 2015 -0500
----------------------------------------------------------------------
.../org/apache/spark/deploy/yarn/Client.scala | 38 +++++++++++++++++++-
1 file changed, 37 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/baed3f2c/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 741239c..4abcf73 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -39,7 +39,7 @@ import org.apache.hadoop.io.Text
import org.apache.hadoop.mapred.Master
import org.apache.hadoop.mapreduce.MRJobConfig
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
-import org.apache.hadoop.security.token.Token
+import org.apache.hadoop.security.token.{TokenIdentifier, Token}
import org.apache.hadoop.util.StringUtils
import org.apache.hadoop.yarn.api._
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
@@ -226,6 +226,7 @@ private[spark] class Client(
val distributedUris = new HashSet[String]
obtainTokensForNamenodes(nns, hadoopConf, credentials)
obtainTokenForHiveMetastore(hadoopConf, credentials)
+ obtainTokenForHBase(hadoopConf, credentials)
val replication = sparkConf.getInt("spark.yarn.submit.file.replication",
fs.getDefaultReplication(dst)).toShort
@@ -1085,6 +1086,41 @@ object Client extends Logging {
}
/**
+ * Obtain security token for HBase.
+ */
+ def obtainTokenForHBase(conf: Configuration, credentials: Credentials): Unit = {
+ if (UserGroupInformation.isSecurityEnabled) {
+ val mirror = universe.runtimeMirror(getClass.getClassLoader)
+
+ try {
+ val confCreate = mirror.classLoader.
+ loadClass("org.apache.hadoop.hbase.HBaseConfiguration").
+ getMethod("create", classOf[Configuration])
+ val obtainToken = mirror.classLoader.
+ loadClass("org.apache.hadoop.hbase.security.token.TokenUtil").
+ getMethod("obtainToken", classOf[Configuration])
+
+ logDebug("Attempting to fetch HBase security token.")
+
+ val hbaseConf = confCreate.invoke(null, conf)
+ val token = obtainToken.invoke(null, hbaseConf).asInstanceOf[Token[TokenIdentifier]]
+ credentials.addToken(token.getService, token)
+
+ logInfo("Added HBase security token to credentials.")
+ } catch {
+ case e:java.lang.NoSuchMethodException =>
+ logInfo("HBase Method not found: " + e)
+ case e:java.lang.ClassNotFoundException =>
+ logDebug("HBase Class not found: " + e)
+ case e:java.lang.NoClassDefFoundError =>
+ logDebug("HBase Class not found: " + e)
+ case e:Exception =>
+ logError("Exception when obtaining HBase security token: " + e)
+ }
+ }
+ }
+
+ /**
* Return whether the two file systems are the same.
*/
private def compareFs(srcFs: FileSystem, destFs: FileSystem): Boolean = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org