You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2018/05/31 11:53:32 UTC
incubator-livy git commit: [LIVY-475] Support of Hadoop
CredentialProvider API
Repository: incubator-livy
Updated Branches:
refs/heads/master 8b3fcbc8d -> 02c5915f3
[LIVY-475] Support of Hadoop CredentialProvider API
In this PR I've added following option to livy.conf:
```
```
to allow to specify path to Hadoop Credential Provider, which than used in `WebServer.scala` to set Keystore password and Key password to JKS that used to enable SSL encryption.
Also before trying to use Hadoop CredentialProvider API I'm checking if it available (as it was not available in Hadoop < 2.6) using the same [method that used in Oozie](https://github.com/apache/oozie/commit/6a731f9926158da38d1e3b518671ada95a544fe8#diff-800f95e605f21c5aaf5edef13039c9b9R124).
To use this, you will need to generate Credential Provider containing "livy.keystore.password" and/or "livy.key-password" in the common way:
```bash
hadoop credential create "livy.keystore.password" -value "keystore_secret" -provider jceks://hdfsnn1.example.com/my/path/livy_creds.jceks
hadoop credential create "livy.key-password" -value "key_secret" -provider jceks://hdfsnn1.example.com/my/path/livy_creds.jceks
```
Author: Ivan Dzikovsky <id...@protonmail.com>
Closes #99 from idzikovsky/LIVY-475.
Change-Id: Iae60550900067e61a37a6f74acb5e299005f3397
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/02c5915f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/02c5915f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/02c5915f
Branch: refs/heads/master
Commit: 02c5915f38a0cd19acfc8222233cba37d73393f9
Parents: 8b3fcbc
Author: Ivan Dzikovsky <id...@protonmail.com>
Authored: Thu May 31 19:53:25 2018 +0800
Committer: jerryshao <ss...@hortonworks.com>
Committed: Thu May 31 19:53:25 2018 +0800
----------------------------------------------------------------------
conf/livy.conf.template | 5 ++++
.../main/scala/org/apache/livy/LivyConf.scala | 2 ++
.../org/apache/livy/server/WebServer.scala | 24 ++++++++++++++++----
3 files changed, 27 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/02c5915f/conf/livy.conf.template
----------------------------------------------------------------------
diff --git a/conf/livy.conf.template b/conf/livy.conf.template
index 6f50e2f..fede70a 100644
--- a/conf/livy.conf.template
+++ b/conf/livy.conf.template
@@ -23,6 +23,11 @@
# Specify the key password.
# livy.key-password =
+# Hadoop Credential Provider Path to get "livy.keystore.password" and "livy.key-password".
+# Credential Provider can be created using command as follow:
+# hadoop credential create "livy.keystore.password" -value "secret" -provider jceks://hdfs/path/to/livy.jceks
+# livy.hadoop.security.credential.provider.path =
+
# What host address to start the server on. By default, Livy will bind to all network interfaces.
# livy.server.host = 0.0.0.0
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/02c5915f/server/src/main/scala/org/apache/livy/LivyConf.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala
index 3f6a86c..edee79a 100644
--- a/server/src/main/scala/org/apache/livy/LivyConf.scala
+++ b/server/src/main/scala/org/apache/livy/LivyConf.scala
@@ -85,6 +85,8 @@ object LivyConf {
val SSL_KEYSTORE_PASSWORD = Entry("livy.keystore.password", null)
val SSL_KEY_PASSWORD = Entry("livy.key-password", null)
+ val HADOOP_CREDENTIAL_PROVIDER_PATH = Entry("livy.hadoop.security.credential.provider.path", null)
+
val AUTH_TYPE = Entry("livy.server.auth.type", null)
val AUTH_KERBEROS_PRINCIPAL = Entry("livy.server.auth.kerberos.principal", null)
val AUTH_KERBEROS_KEYTAB = Entry("livy.server.auth.kerberos.keytab", null)
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/02c5915f/server/src/main/scala/org/apache/livy/server/WebServer.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/WebServer.scala b/server/src/main/scala/org/apache/livy/server/WebServer.scala
index 549d6ab..1955974 100644
--- a/server/src/main/scala/org/apache/livy/server/WebServer.scala
+++ b/server/src/main/scala/org/apache/livy/server/WebServer.scala
@@ -20,6 +20,7 @@ package org.apache.livy.server
import java.net.InetAddress
import javax.servlet.ServletContextListener
+import org.apache.hadoop.conf.Configuration
import org.eclipse.jetty.server._
import org.eclipse.jetty.server.handler.{HandlerCollection, RequestLogHandler}
import org.eclipse.jetty.servlet.{DefaultServlet, ServletContextHandler}
@@ -48,10 +49,25 @@ class WebServer(livyConf: LivyConf, var host: String, var port: Int) extends Log
val sslContextFactory = new SslContextFactory()
sslContextFactory.setKeyStorePath(keystore)
- Option(livyConf.get(LivyConf.SSL_KEYSTORE_PASSWORD))
- .foreach(sslContextFactory.setKeyStorePassword)
- Option(livyConf.get(LivyConf.SSL_KEY_PASSWORD))
- .foreach(sslContextFactory.setKeyManagerPassword)
+
+ val credentialProviderPath = livyConf.get(LivyConf.HADOOP_CREDENTIAL_PROVIDER_PATH)
+ val hadoopConf = new Configuration()
+ if (credentialProviderPath != null) {
+ hadoopConf.set("hadoop.security.credential.provider.path", credentialProviderPath)
+ }
+
+ val keyStorePassword = Option(livyConf.get(LivyConf.SSL_KEYSTORE_PASSWORD))
+ .orElse {
+ Option(hadoopConf.getPassword(LivyConf.SSL_KEYSTORE_PASSWORD.key)).map(_.mkString)
+ }
+
+ val keyPassword = Option(livyConf.get(LivyConf.SSL_KEY_PASSWORD))
+ .orElse {
+ Option(hadoopConf.getPassword(LivyConf.SSL_KEY_PASSWORD.key)).map(_.mkString)
+ }
+
+ keyStorePassword.foreach(sslContextFactory.setKeyStorePassword)
+ keyPassword.foreach(sslContextFactory.setKeyManagerPassword)
(new ServerConnector(server,
new SslConnectionFactory(sslContextFactory, "http/1.1"),