You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by mg...@apache.org on 2019/10/13 07:42:28 UTC
[incubator-livy] branch master updated: [LIVY-356][SERVER] Add LDAP
authentication for livy-server.
This is an automated email from the ASF dual-hosted git repository.
mgaido pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-livy.git
The following commit(s) were added to refs/heads/master by this push:
new 2e7d691 [LIVY-356][SERVER] Add LDAP authentication for livy-server.
2e7d691 is described below
commit 2e7d691c03caa6eaa1d170b2ab03265a65ff12f2
Author: captainzmc <mi...@tencent.com>
AuthorDate: Sun Oct 13 09:42:23 2019 +0200
[LIVY-356][SERVER] Add LDAP authentication for livy-server.
## What changes were proposed in this pull request?
Currently, livy-server doesn't support LDAP Authentication from client to server(livy). We need to add LDAP authentication as that's preferable method due to security reasons.
Here we reimplement LdapAuthenticationHandle, which is new in hadoop2.8+.
## How was this patch tested?
UTs tests for this part have been added. We can test in UTs
Author: captainzmc <mi...@tencent.com>
Author: Janki Akhani <ja...@linkedin.com>
Closes #231 from captainzmc/add-ldap.
---
pom.xml | 4 +
server/pom.xml | 43 ++++
.../src/main/scala/org/apache/livy/LivyConf.scala | 7 +
.../scala/org/apache/livy/server/LivyServer.scala | 21 ++
.../auth/LdapAuthenticationHandlerImpl.scala | 219 +++++++++++++++++++++
.../auth/TestLdapAuthenticationHandlerImpl.scala | 156 +++++++++++++++
6 files changed, 450 insertions(+)
diff --git a/pom.xml b/pom.xml
index fa1600d..5bcf1a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -121,6 +121,10 @@
<!-- Set this to "true" to skip PySpark3 tests. -->
<skipPySpark3Tests>false</skipPySpark3Tests>
+ <!-- Required for testing LDAP integration -->
+ <apacheds.version>2.0.0-M21</apacheds.version>
+ <ldap-api.version>1.0.0-M33</ldap-api.version>
+
<!--
Properties for the copyright header style checks. Modules that use the ASF header
should override the "copyright.header" property.
diff --git a/server/pom.xml b/server/pom.xml
index e708964..8e797f2 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -223,6 +223,49 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.directory.server</groupId>
+ <artifactId>apacheds-core</artifactId>
+ <version>${apacheds.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.server</groupId>
+ <artifactId>apacheds-protocol-ldap</artifactId>
+ <version>${apacheds.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.server</groupId>
+ <artifactId>apacheds-ldif-partition</artifactId>
+ <version>${apacheds.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-ldap-codec-core</artifactId>
+ <version>${ldap-api.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-ldap-model</artifactId>
+ <version>${ldap-api.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.server</groupId>
+ <artifactId>apacheds-server-integ</artifactId>
+ <version>${apacheds.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.server</groupId>
+ <artifactId>apacheds-core-integ</artifactId>
+ <version>${apacheds.version}</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala
index dec8e4a..0a52d51 100644
--- a/server/src/main/scala/org/apache/livy/LivyConf.scala
+++ b/server/src/main/scala/org/apache/livy/LivyConf.scala
@@ -87,6 +87,13 @@ object LivyConf {
val HADOOP_CREDENTIAL_PROVIDER_PATH = Entry("livy.hadoop.security.credential.provider.path", null)
val AUTH_TYPE = Entry("livy.server.auth.type", null)
+ // Ldap configurations
+ val AUTH_LDAP_URL = Entry("livy.server.auth.ldap.url", null)
+ val AUTH_LDAP_BASE_DN = Entry("livy.server.auth.ldap.base-dn", null)
+ val AUTH_LDAP_USERNAME_DOMAIN = Entry("livy.server.auth.ldap.username-domain", null)
+ val AUTH_LDAP_ENABLE_START_TLS = Entry("livy.server.auth.ldap.enable-start-tls", "false")
+ val AUTH_LDAP_SECURITY_AUTH = Entry("livy.server.auth.ldap.security-authentication", "simple")
+ // kerberos configurations
val AUTH_KERBEROS_PRINCIPAL = Entry("livy.server.auth.kerberos.principal", null)
val AUTH_KERBEROS_KEYTAB = Entry("livy.server.auth.kerberos.keytab", null)
val AUTH_KERBEROS_NAME_RULES = Entry("livy.server.auth.kerberos.name-rules", "DEFAULT")
diff --git a/server/src/main/scala/org/apache/livy/server/LivyServer.scala b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
index 60f3961..b40a20e 100644
--- a/server/src/main/scala/org/apache/livy/server/LivyServer.scala
+++ b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
@@ -36,6 +36,7 @@ import org.scalatra.metrics.MetricsSupportExtensions._
import org.scalatra.servlet.{MultipartConfig, ServletApiImplicits}
import org.apache.livy._
+import org.apache.livy.server.auth.LdapAuthenticationHandlerImpl
import org.apache.livy.server.batch.BatchSessionServlet
import org.apache.livy.server.interactive.InteractiveSessionServlet
import org.apache.livy.server.recovery.{SessionStore, StateStore}
@@ -260,6 +261,26 @@ class LivyServer extends Logging {
server.context.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType]))
info(s"SPNEGO auth enabled (principal = $principal)")
+ case authType @ LdapAuthenticationHandlerImpl.TYPE =>
+ val holder = new FilterHolder(new AuthenticationFilter())
+ holder.setInitParameter(AuthenticationFilter.AUTH_TYPE,
+ LdapAuthenticationHandlerImpl.getClass.getCanonicalName.dropRight(1))
+ Option(livyConf.get(LivyConf.AUTH_LDAP_URL)).foreach { url =>
+ holder.setInitParameter(LdapAuthenticationHandlerImpl.PROVIDER_URL, url)
+ }
+ Option(livyConf.get(LivyConf.AUTH_LDAP_USERNAME_DOMAIN)).foreach { domain =>
+ holder.setInitParameter(LdapAuthenticationHandlerImpl.LDAP_BIND_DOMAIN, domain)
+ }
+ Option(livyConf.get(LivyConf.AUTH_LDAP_BASE_DN)).foreach { baseDN =>
+ holder.setInitParameter(LdapAuthenticationHandlerImpl.BASE_DN, baseDN)
+ }
+ holder.setInitParameter(LdapAuthenticationHandlerImpl.SECURITY_AUTHENTICATION,
+ livyConf.get(LivyConf.AUTH_LDAP_SECURITY_AUTH))
+ holder.setInitParameter(LdapAuthenticationHandlerImpl.ENABLE_START_TLS,
+ livyConf.get(LivyConf.AUTH_LDAP_ENABLE_START_TLS))
+ server.context.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType]))
+ info("LDAP auth enabled.")
+
case null =>
// Nothing to do.
diff --git a/server/src/main/scala/org/apache/livy/server/auth/LdapAuthenticationHandlerImpl.scala b/server/src/main/scala/org/apache/livy/server/auth/LdapAuthenticationHandlerImpl.scala
new file mode 100644
index 0000000..4fe8c01
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/auth/LdapAuthenticationHandlerImpl.scala
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.auth
+
+import java.io.IOException
+import java.nio.charset.StandardCharsets
+import java.util
+import java.util.Properties
+import javax.naming.NamingException
+import javax.naming.directory.InitialDirContext
+import javax.naming.ldap.{InitialLdapContext, StartTlsRequest, StartTlsResponse}
+import javax.net.ssl.{HostnameVerifier, SSLSession}
+import javax.servlet.ServletException
+import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
+
+import org.apache.commons.codec.binary.Base64
+import org.apache.hadoop.security.authentication.client.AuthenticationException
+import org.apache.hadoop.security.authentication.server.{AuthenticationHandler, AuthenticationToken}
+
+import org.apache.livy._
+
+object LdapAuthenticationHandlerImpl {
+
+ val AUTHORIZATION_SCHEME = "Basic"
+ val TYPE = "ldap"
+ val SECURITY_AUTHENTICATION = "simple"
+ val PROVIDER_URL = "ldap.providerurl"
+ val BASE_DN = "ldap.basedn"
+ val LDAP_BIND_DOMAIN = "ldap.binddomain"
+ val ENABLE_START_TLS = "ldap.enablestarttls"
+
+ private def hasDomain(userName: String): Boolean = {
+ indexOfDomainMatch(userName) > 0
+ }
+
+ /**
+ * Get the index separating the user name from domain name (the user's name up
+ * to the first '/' or '@').
+ */
+ private def indexOfDomainMatch(userName: String): Int = {
+ val idx = userName.indexOf('/')
+ val idx2 = userName.indexOf('@')
+ // Use the earlier match.
+ val endIdx = Math.min(idx, idx2)
+
+ // If neither '/' nor '@' was found, using the latter
+ if (endIdx == -1) Math.max(idx, idx2) else endIdx
+ }
+}
+
+class LdapAuthenticationHandlerImpl extends AuthenticationHandler with Logging {
+ private var ldapDomain = "null"
+ private var baseDN = "null"
+ private var providerUrl = "null"
+ private var enableStartTls = false
+ private var disableHostNameVerification = false
+
+ def getType: String = LdapAuthenticationHandlerImpl.TYPE
+
+ @throws[ServletException]
+ def init(config: Properties): Unit = {
+ this.baseDN = config.getProperty(LdapAuthenticationHandlerImpl.BASE_DN)
+ this.providerUrl = config.getProperty(LdapAuthenticationHandlerImpl.PROVIDER_URL)
+ this.ldapDomain = config.getProperty(LdapAuthenticationHandlerImpl.LDAP_BIND_DOMAIN)
+ this.enableStartTls = config.getProperty(
+ LdapAuthenticationHandlerImpl.ENABLE_START_TLS, "false").toBoolean
+ require(this.providerUrl != null, "The LDAP URI can not be null")
+
+ if (enableStartTls) {
+ require(!this.providerUrl.toLowerCase.startsWith("ldaps"),
+ "Can not use ldaps and StartTLS option at the same time")
+ }
+ }
+
+ def destroy(): Unit = { }
+
+ @throws[IOException]
+ @throws[AuthenticationException]
+ def managementOperation(
+ token: AuthenticationToken,
+ request: HttpServletRequest,
+ response: HttpServletResponse): Boolean = true
+
+ @throws[IOException]
+ @throws[AuthenticationException]
+ def authenticate(
+ request: HttpServletRequest,
+ response: HttpServletResponse): AuthenticationToken = {
+ var token: AuthenticationToken = null
+ var authorization = request.getHeader("Authorization")
+
+ if (authorization != null && authorization.regionMatches(true, 0,
+ LdapAuthenticationHandlerImpl.AUTHORIZATION_SCHEME, 0,
+ LdapAuthenticationHandlerImpl.AUTHORIZATION_SCHEME.length)) {
+ authorization = authorization.substring("Basic".length).trim
+ val base64 = new Base64(0)
+ val credentials = new String(base64.decode(authorization),
+ StandardCharsets.UTF_8).split(":", 2)
+
+ if (credentials.length == 2) {
+ debug(s"Authenticating [${credentials(0)}] user")
+ token = this.authenticateUser(credentials(0), credentials(1))
+ response.setStatus(HttpServletResponse.SC_OK)
+ }
+ } else {
+ response.setHeader("WWW-Authenticate", "Basic")
+ response.setStatus(HttpServletResponse.SC_UNAUTHORIZED)
+
+ if (authorization == null) {
+ trace("Basic auth starting")
+ } else {
+ warn(s"Authorization does not start with Basic : ${authorization} ")
+ }
+ }
+ token
+ }
+
+ @throws[AuthenticationException]
+ private def authenticateUser(userName: String, password: String): AuthenticationToken = {
+ if (userName == null || userName.isEmpty) {
+ throw new AuthenticationException(
+ "Error validating LDAP user: a null or blank username has been provided")
+ }
+ if (password == null || password.isEmpty) {
+ throw new AuthenticationException(
+ "Error validating LDAP user: a null or blank password has been provided")
+ }
+
+ var principle = userName
+ if (!LdapAuthenticationHandlerImpl.hasDomain(userName) && ldapDomain != null) {
+ principle = userName + "@" + ldapDomain
+ }
+ val bindDN = if (baseDN != null) {
+ "uid=" + principle + "," + baseDN
+ } else {
+ principle
+ }
+
+ if (enableStartTls) {
+ authenticateWithTlsExtension(bindDN, password)
+ } else {
+ authenticateWithoutTlsExtension(bindDN, password)
+ }
+
+ new AuthenticationToken(userName, userName, "ldap")
+ }
+
+ @throws[AuthenticationException]
+ private def authenticateWithTlsExtension(userDN: String, password: String): Unit = {
+ var ctx: InitialLdapContext = null
+ val env = new util.Hashtable[String, String]
+ env.put("java.naming.factory.initial", "com.sun.jndi.ldap.LdapCtxFactory")
+ env.put("java.naming.provider.url", providerUrl)
+
+ try {
+ ctx = new InitialLdapContext(env, null)
+ val ex = ctx.extendedOperation(new StartTlsRequest).asInstanceOf[StartTlsResponse]
+ if (this.disableHostNameVerification) {
+ ex.setHostnameVerifier(new HostnameVerifier() {
+ override def verify(hostname: String, session: SSLSession) = true
+ })
+ }
+ ex.negotiate
+
+ ctx.addToEnvironment("java.naming.security.authentication",
+ LdapAuthenticationHandlerImpl.SECURITY_AUTHENTICATION)
+ ctx.addToEnvironment("java.naming.security.principal", userDN)
+ ctx.addToEnvironment("java.naming.security.credentials", password)
+ ctx.lookup(userDN)
+ debug(s"Authentication successful for ${userDN}")
+ } catch {
+ case exception @ (_: IOException | _: NamingException) =>
+ throw new AuthenticationException("Error validating LDAP user", exception)
+ } finally {
+ if (ctx != null) {
+ try {
+ ctx.close()
+ } catch {
+ case exception: NamingException =>
+ }
+ }
+ }
+ }
+
+ @throws[AuthenticationException]
+ private def authenticateWithoutTlsExtension(userDN: String, password: String): Unit = {
+ val env = new util.Hashtable[String, String]
+ env.put("java.naming.factory.initial", "com.sun.jndi.ldap.LdapCtxFactory")
+ env.put("java.naming.provider.url", providerUrl)
+ env.put("java.naming.security.authentication",
+ LdapAuthenticationHandlerImpl.SECURITY_AUTHENTICATION)
+ env.put("java.naming.security.principal", userDN)
+ env.put("java.naming.security.credentials", password)
+
+ try {
+ val e = new InitialDirContext(env)
+ e.close()
+ debug(s"Authentication successful for ${userDN}")
+ } catch {
+ case exception: NamingException =>
+ throw new AuthenticationException("Error validating LDAP user", exception)
+ }
+ }
+}
diff --git a/server/src/test/scala/org/apache/livy/server/auth/TestLdapAuthenticationHandlerImpl.scala b/server/src/test/scala/org/apache/livy/server/auth/TestLdapAuthenticationHandlerImpl.scala
new file mode 100644
index 0000000..8008388
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/auth/TestLdapAuthenticationHandlerImpl.scala
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.auth
+
+import java.util.Properties
+import javax.servlet.http.HttpServletRequest
+import javax.servlet.http.HttpServletResponse
+
+import org.apache.commons.codec.binary.Base64
+import org.apache.directory.server.annotations.CreateLdapServer
+import org.apache.directory.server.annotations.CreateTransport
+import org.apache.directory.server.core.annotations.ApplyLdifs
+import org.apache.directory.server.core.annotations.ContextEntry
+import org.apache.directory.server.core.annotations.CreateDS
+import org.apache.directory.server.core.annotations.CreatePartition
+import org.apache.directory.server.core.integ.AbstractLdapTestUnit
+import org.apache.directory.server.core.integ.FrameworkRunner
+import org.apache.hadoop.security.authentication.client.AuthenticationException
+import org.junit.Assert
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mockito.Mockito
+
+/**
+ * This unit test verifies the functionality of LdapAuthenticationHandlerImpl.
+ */
+@RunWith(classOf[FrameworkRunner])
+@CreateLdapServer(transports = Array(
+ new CreateTransport(
+ protocol = "LDAP",
+ address = "localhost"
+ )))
+@CreateDS(
+ allowAnonAccess = true,
+ partitions = Array(
+ new CreatePartition(
+ name = "Test_Partition",
+ suffix = "dc=example,dc=com",
+ contextEntry = new ContextEntry(entryLdif = "dn: dc=example," +
+ "dc=com \ndc: example\nobjectClass: top\nobjectClass: domain\n\n")
+ )))
+@ApplyLdifs(Array(
+ "dn: uid=bjones,dc=example,dc=com",
+ "cn: Bob Jones",
+ "sn: Jones",
+ "objectClass: inetOrgPerson",
+ "uid: bjones",
+ "userPassword: p@ssw0rd"
+))
+class TestLdapAuthenticationHandlerImpl extends AbstractLdapTestUnit {
+ private val handler: LdapAuthenticationHandlerImpl = new LdapAuthenticationHandlerImpl
+ // HTTP header used by the server endpoint during an authentication sequence.
+ val WWW_AUTHENTICATE_HEADER = "WWW-Authenticate"
+ // HTTP header used by the client endpoint during an authentication sequence.
+ val AUTHORIZATION_HEADER = "Authorization"
+ // HTTP header prefix used during the Basic authentication sequence.
+ val BASIC = "Basic"
+
+ @Before
+ def setup(): Unit = {
+ handler.init(getDefaultProperties)
+ }
+
+ protected def getDefaultProperties: Properties = {
+ val p = new Properties
+ p.setProperty("ldap.basedn", "dc=example,dc=com")
+ p.setProperty("ldap.providerurl", String.format("ldap://%s:%s", "localhost",
+ AbstractLdapTestUnit.getLdapServer.getPort.toString))
+ p
+ }
+
+ @Test
+ def testRequestWithAuthorization(): Unit = {
+ val request = Mockito.mock(classOf[HttpServletRequest])
+ val response = Mockito.mock(classOf[HttpServletResponse])
+ val base64 = new Base64(0)
+ val credentials = base64.encodeToString("bjones:p@ssw0rd".getBytes)
+ val authHeader = BASIC + " " + credentials
+ Mockito.when(request.getHeader(AUTHORIZATION_HEADER)).thenReturn(authHeader)
+
+ val token = handler.authenticate(request, response)
+ Assert.assertNotNull(token)
+ Mockito.verify(response).setStatus(HttpServletResponse.SC_OK)
+ Assert.assertEquals("bjones", token.getUserName)
+ Assert.assertEquals("bjones", token.getName)
+ }
+
+ @Test
+ def testRequestWithoutAuthorization(): Unit = {
+ val request = Mockito.mock(classOf[HttpServletRequest])
+ val response = Mockito.mock(classOf[HttpServletResponse])
+
+ Assert.assertNull(handler.authenticate(request, response))
+ Mockito.verify(response).setHeader(WWW_AUTHENTICATE_HEADER, BASIC)
+ Mockito.verify(response).setStatus(HttpServletResponse.SC_UNAUTHORIZED)
+ }
+
+ @Test
+ def testRequestWithInvalidAuthorization(): Unit = {
+ val request = Mockito.mock(classOf[HttpServletRequest])
+ val response = Mockito.mock(classOf[HttpServletResponse])
+ val base64 = new Base64
+ val credentials = "bjones:invalidpassword"
+ Mockito.when(request.getHeader(AUTHORIZATION_HEADER)).
+ thenReturn(base64.encodeToString(credentials.getBytes))
+
+ Assert.assertNull(handler.authenticate(request, response))
+ Mockito.verify(response).setHeader(WWW_AUTHENTICATE_HEADER, BASIC)
+ Mockito.verify(response).setStatus(HttpServletResponse.SC_UNAUTHORIZED)
+ }
+
+ @Test
+ def testRequestWithIncompleteAuthorization(): Unit = {
+ val request = Mockito.mock(classOf[HttpServletRequest])
+ val response = Mockito.mock(classOf[HttpServletResponse])
+ Mockito.when(request.getHeader(AUTHORIZATION_HEADER)).thenReturn(BASIC)
+ Assert.assertNull(handler.authenticate(request, response))
+ }
+
+ @Test
+ def testRequestWithWrongCredentials(): Unit = {
+ val request = Mockito.mock(classOf[HttpServletRequest])
+ val response = Mockito.mock(classOf[HttpServletResponse])
+ val base64 = new Base64
+ val credentials = base64.encodeToString("bjones:foo123".getBytes)
+ val authHeader = BASIC + " " + credentials
+ Mockito.when(request.getHeader(AUTHORIZATION_HEADER)).thenReturn(authHeader)
+
+ try {
+ handler.authenticate(request, response)
+ Assert.fail
+ } catch {
+ case ex: AuthenticationException =>
+ // Expected
+ case ex: Exception =>
+ Assert.fail
+ }
+ }
+}
+