You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2017/06/01 07:17:28 UTC
zeppelin git commit: [ZEPPELIN-2584]. Livy doesn't work under
kerberos and ssl
Repository: zeppelin
Updated Branches:
refs/heads/master 88c5c3ccd -> e3ba23a9a
[ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl
### What is this PR for?
We found that Livy doesn't work under kerberos and ssl, this is due to a bug when constructing KerberosRestTemplate. This PR fix this issue.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-2584
### How should this be tested?
Unfornatelly it is difficult to add system test, so I manually tested it in a secure cluster.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: root <ro...@ctr-e133-1493418528701-48413-01-000004.hwx.site>
Closes #2367 from zjffdu/ZEPPELIN-2584 and squashes the following commits:
18b23dc [root] [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/e3ba23a9
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/e3ba23a9
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/e3ba23a9
Branch: refs/heads/master
Commit: e3ba23a9aa4909c7a8d3231e4d363771e0245b53
Parents: 88c5c3c
Author: root <ro...@ctr-e133-1493418528701-48413-01-000004.hwx.site>
Authored: Fri May 26 08:44:09 2017 +0000
Committer: Jeff Zhang <zj...@apache.org>
Committed: Thu Jun 1 15:17:21 2017 +0800
----------------------------------------------------------------------
livy/pom.xml | 18 -------
.../zeppelin/livy/BaseLivyInterpreter.java | 56 +++++++++++++++++---
pom.xml | 4 +-
3 files changed, 52 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/livy/pom.xml
----------------------------------------------------------------------
diff --git a/livy/pom.xml b/livy/pom.xml
index 3c121fc..5f9dec7 100644
--- a/livy/pom.xml
+++ b/livy/pom.xml
@@ -37,14 +37,10 @@
<properties>
<!--library versions-->
<commons.exec.version>1.3</commons.exec.version>
- <httpcomponents.client.version>4.3.4</httpcomponents.client.version>
<spring.web.version>4.3.0.RELEASE</spring.web.version>
<spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient>
<!--test library versions-->
- <achilles.version>3.2.4-Zeppelin</achilles.version>
- <assertj.version>1.7.0</assertj.version>
- <mockito.version>1.9.5</mockito.version>
<livy.version>0.3.0</livy.version>
<spark.version>2.1.0</spark.version>
<hadoop.version>2.6.0</hadoop.version>
@@ -80,7 +76,6 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>${httpcomponents.client.version}</version>
</dependency>
<dependency>
@@ -107,19 +102,6 @@
</dependency>
<dependency>
- <groupId>org.assertj</groupId>
- <artifactId>assertj-core</artifactId>
- <version>${assertj.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>com.cloudera.livy</groupId>
<artifactId>livy-integration-test</artifactId>
<version>${livy.version}</version>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
index b52ba16..fcb7e9d 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
@@ -21,9 +21,20 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang.StringUtils;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContexts;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.zeppelin.interpreter.*;
@@ -38,11 +49,11 @@ import org.springframework.security.kerberos.client.KerberosRestTemplate;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
-
import javax.net.ssl.SSLContext;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.KeyStore;
+import java.security.Principal;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -52,7 +63,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-
/**
* Base class for livy interpreters.
*/
@@ -407,6 +417,11 @@ public abstract class BaseLivyInterpreter extends Interpreter {
private RestTemplate createRestTemplate() {
+ String keytabLocation = property.getProperty("zeppelin.livy.keytab");
+ String principal = property.getProperty("zeppelin.livy.principal");
+ boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) &&
+ StringUtils.isNotEmpty(principal);
+
HttpClient httpClient = null;
if (livyURL.startsWith("https:")) {
String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore");
@@ -427,7 +442,37 @@ public abstract class BaseLivyInterpreter extends Interpreter {
.loadTrustMaterial(trustStore)
.build();
SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
- httpClient = HttpClients.custom().setSSLSocketFactory(csf).build();
+ HttpClientBuilder httpClientBuilder = HttpClients.custom().setSSLSocketFactory(csf);
+ RequestConfig reqConfig = new RequestConfig() {
+ @Override
+ public boolean isAuthenticationEnabled() {
+ return true;
+ }
+ };
+ httpClientBuilder.setDefaultRequestConfig(reqConfig);
+ Credentials credentials = new Credentials() {
+ @Override
+ public String getPassword() {
+ return null;
+ }
+
+ @Override
+ public Principal getUserPrincipal() {
+ return null;
+ }
+ };
+ CredentialsProvider credsProvider = new BasicCredentialsProvider();
+ credsProvider.setCredentials(AuthScope.ANY, credentials);
+ httpClientBuilder.setDefaultCredentialsProvider(credsProvider);
+ if (isSpnegoEnabled) {
+ Registry<AuthSchemeProvider> authSchemeProviderRegistry =
+ RegistryBuilder.<AuthSchemeProvider>create()
+ .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
+ .build();
+ httpClientBuilder.setDefaultAuthSchemeRegistry(authSchemeProviderRegistry);
+ }
+
+ httpClient = httpClientBuilder.build();
} catch (Exception e) {
throw new RuntimeException("Failed to create SSL HttpClient", e);
} finally {
@@ -441,9 +486,8 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
}
- String keytabLocation = property.getProperty("zeppelin.livy.keytab");
- String principal = property.getProperty("zeppelin.livy.principal");
- if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) {
+
+ if (isSpnegoEnabled) {
if (httpClient == null) {
return new KerberosRestTemplate(keytabLocation, principal);
} else {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 87af323..9232b9e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,8 +96,8 @@
<gson-extras.version>0.2.1</gson-extras.version>
<guava.version>15.0</guava.version>
<jetty.version>9.2.15.v20160210</jetty.version>
- <httpcomponents.core.version>4.3.3</httpcomponents.core.version>
- <httpcomponents.client.version>4.3.6</httpcomponents.client.version>
+ <httpcomponents.core.version>4.4.1</httpcomponents.core.version>
+ <httpcomponents.client.version>4.5.1</httpcomponents.client.version>
<httpcomponents.asyncclient.version>4.0.2</httpcomponents.asyncclient.version>
<commons.lang.version>2.5</commons.lang.version>
<commons.configuration.version>1.9</commons.configuration.version>