You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by vr...@apache.org on 2016/09/14 21:18:26 UTC
apex-core git commit: APEXCORE-517 Added BASIC authentication support
to be able to authenticate with Hadoop web services with BASIC authentication
enabled
Repository: apex-core
Updated Branches:
refs/heads/master 0670eb3e0 -> bfc1eb874
APEXCORE-517 Added BASIC authentication support to be able to authenticate with Hadoop web services with BASIC authentication enabled
Project: http://git-wip-us.apache.org/repos/asf/apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-core/commit/bfc1eb87
Tree: http://git-wip-us.apache.org/repos/asf/apex-core/tree/bfc1eb87
Diff: http://git-wip-us.apache.org/repos/asf/apex-core/diff/bfc1eb87
Branch: refs/heads/master
Commit: bfc1eb87436a324c1fdee130cb5c66c839fc76a2
Parents: 0670eb3
Author: Pramod Immaneni <pr...@datatorrent.com>
Authored: Mon Sep 5 13:59:13 2016 -0700
Committer: Pramod Immaneni <pr...@datatorrent.com>
Committed: Wed Sep 14 07:29:42 2016 -0700
----------------------------------------------------------------------
.../java/com/datatorrent/stram/cli/ApexCli.java | 2 +
.../datatorrent/stram/security/AuthScheme.java | 50 ++++++++++++++
.../datatorrent/stram/util/SecurityUtils.java | 25 ++++++-
.../stram/util/WebServicesClient.java | 71 +++++++++++++++-----
.../stram/util/SecurityUtilsTest.java | 23 ++++++-
.../stram/util/WebServicesClientTest.java | 32 +++++++++
.../test/resources/security/dt-site-basic.xml | 33 +++++++++
.../resources/security/dt-site-kerberos.xml | 33 +++++++++
engine/src/test/resources/security/user.keytab | 1 +
pom.xml | 2 +
10 files changed, 252 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java b/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java
index 9db488f..235ba58 100644
--- a/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java
+++ b/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java
@@ -123,6 +123,7 @@ import com.datatorrent.stram.plan.logical.requests.SetPortAttributeRequest;
import com.datatorrent.stram.plan.logical.requests.SetStreamAttributeRequest;
import com.datatorrent.stram.security.StramUserLogin;
import com.datatorrent.stram.util.JSONSerializationProvider;
+import com.datatorrent.stram.util.SecurityUtils;
import com.datatorrent.stram.util.VersionInfo;
import com.datatorrent.stram.util.WebServicesClient;
import com.datatorrent.stram.webapp.OperatorDiscoverer;
@@ -1131,6 +1132,7 @@ public class ApexCli
public void init() throws IOException
{
conf = StramClientUtils.addDTSiteResources(new YarnConfiguration());
+ SecurityUtils.init(conf);
fs = StramClientUtils.newFileSystemInstance(conf);
stramAgent = new StramAgent(fs, conf);
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/main/java/com/datatorrent/stram/security/AuthScheme.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/security/AuthScheme.java b/engine/src/main/java/com/datatorrent/stram/security/AuthScheme.java
new file mode 100644
index 0000000..f39dddf
--- /dev/null
+++ b/engine/src/main/java/com/datatorrent/stram/security/AuthScheme.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.security;
+
+/**
+ * Authentication scheme
+ *
+ * BASIC: BASIC HTTP authentication - RFC 2617
+ * SPNEGO: Kerberos based SPNEGO authentication - RFC 4559
+ * KERBEROS: Kerberos authentication - RFC 4120
+ */
+public enum AuthScheme
+{
+ BASIC("basic"), SPNEGO("kerberos"), KERBEROS("kerberos-standard");
+
+ String name;
+
+ AuthScheme(String name)
+ {
+ this.name = name;
+ }
+
+ public String getName()
+ {
+ return name;
+ }
+
+ @Override
+ public String toString()
+ {
+ return name;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java b/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java
index 0d54511..349d09a 100644
--- a/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java
+++ b/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import com.datatorrent.api.Context;
import com.datatorrent.api.Context.StramHTTPAuthentication;
+import com.datatorrent.stram.security.AuthScheme;
+import com.datatorrent.stram.security.StramUserLogin;
/**
*
@@ -43,15 +45,20 @@ public class SecurityUtils
hadoopWebSecurityEnabled = stramWebSecurityEnabled = UserGroupInformation.isSecurityEnabled();
}
+ public static void init(Configuration configuration)
+ {
+ init(configuration, null);
+ }
+
public static void init(Configuration configuration, StramHTTPAuthentication stramHTTPAuth)
{
hadoopWebSecurityEnabled = false;
String authValue = configuration.get(HADOOP_HTTP_AUTH_PROP);
if ((authValue != null) && !authValue.equals(HADOOP_HTTP_AUTH_VALUE_SIMPLE)) {
hadoopWebSecurityEnabled = true;
+ initAuth(configuration);
}
// Stram http auth may not be specified and is null but still set a default
- boolean authDefault = false;
if (stramHTTPAuth != null) {
if (stramHTTPAuth == Context.StramHTTPAuthentication.FOLLOW_HADOOP_HTTP_AUTH) {
stramWebSecurityEnabled = hadoopWebSecurityEnabled;
@@ -65,6 +72,22 @@ public class SecurityUtils
}
}
+ private static void initAuth(final Configuration configuration)
+ {
+ // Authentication scheme is not unambiguously known because in Hadoop authentication, authentication type can be
+ // specified as an implementation class, furthermore authentication types like SASL wrap other mechanisms like BASIC
+ // or SPNEGO underneath and the wrapped scheme is not known till the authentication negotiation process
+ WebServicesClient.initAuth(new WebServicesClient.ConfigProvider()
+ {
+ @Override
+ public String getProperty(AuthScheme scheme, String name)
+ {
+ StringBuilder propNamesb = new StringBuilder(StramUserLogin.DT_AUTH_PREFIX).append(scheme.getName()).append(".").append(name);
+ return configuration.get(propNamesb.toString());
+ }
+ });
+ }
+
public static boolean isHadoopWebSecurityEnabled()
{
return hadoopWebSecurityEnabled;
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java b/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java
index 1712080..73d5532 100644
--- a/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java
+++ b/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java
@@ -30,10 +30,13 @@ import org.slf4j.LoggerFactory;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.config.Lookup;
import org.apache.http.config.RegistryBuilder;
+import org.apache.http.impl.auth.BasicSchemeFactory;
+import org.apache.http.impl.auth.KerberosSchemeFactory;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.BasicCredentialsProvider;
@@ -49,6 +52,8 @@ import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.client.filter.ClientFilter;
import com.sun.jersey.client.apache4.ApacheHttpClient4Handler;
+import com.datatorrent.stram.security.AuthScheme;
+
/**
* <p>WebServicesClient class.</p>
*
@@ -61,6 +66,8 @@ public class WebServicesClient
private static final PoolingHttpClientConnectionManager connectionManager;
private static final CredentialsProvider credentialsProvider;
+ private static final RegistryBuilder<AuthSchemeProvider> registryBuilder;
+ private static Lookup<AuthSchemeProvider> authRegistry;
private static final int DEFAULT_CONNECT_TIMEOUT = 10000;
private static final int DEFAULT_READ_TIMEOUT = 10000;
@@ -68,27 +75,55 @@ public class WebServicesClient
private final Set<ClientFilter> clientFilters = new HashSet<>();
+ private static final Credentials DEFAULT_TOKEN_CREDENTIALS = new Credentials()
+ {
+ @Override
+ public Principal getUserPrincipal()
+ {
+ return null;
+ }
+
+ @Override
+ public String getPassword()
+ {
+ return null;
+ }
+ };
+
static {
connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(200);
connectionManager.setDefaultMaxPerRoute(5);
+ registryBuilder = RegistryBuilder.<AuthSchemeProvider>create();
credentialsProvider = new BasicCredentialsProvider();
- credentialsProvider.setCredentials(AuthScope.ANY, new Credentials()
- {
+ // By default add SPNEGO so that it works even if auth is not explictly configured like before, in future
+ // move it to auth setup below
+ registryBuilder.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true));
+ credentialsProvider.setCredentials(AuthScope.ANY, DEFAULT_TOKEN_CREDENTIALS);
+ authRegistry = registryBuilder.build();
+ }
- @Override
- public Principal getUserPrincipal()
- {
- return null;
- }
+ public static void initAuth(ConfigProvider configuration)
+ {
+ // Adding BASIC auth
+ AuthScheme scheme = AuthScheme.BASIC;
+ String username = configuration.getProperty(scheme, "username");
+ String password = configuration.getProperty(scheme, "password");
+ if ((username != null) && (password != null)) {
+ LOG.info("Setting up scheme {}", scheme);
+ registryBuilder.register(AuthSchemes.BASIC, new BasicSchemeFactory());
+ AuthScope authScope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.BASIC);
+ Credentials credentials = new UsernamePasswordCredentials(username, password);
+ credentialsProvider.setCredentials(authScope, credentials);
+ } else if ((username != null) || (password != null)) {
+ LOG.warn("Not setting up scheme {}, missing credentials {}", scheme, (username == null) ? "username" : "password");
+ }
- @Override
- public String getPassword()
- {
- return null;
- }
+ // Adding kerberos standard auth
+ registryBuilder.register(AuthSchemes.KERBEROS, new KerberosSchemeFactory());
+ credentialsProvider.setCredentials(AuthScope.ANY, DEFAULT_TOKEN_CREDENTIALS);
- });
+ authRegistry = registryBuilder.build();
}
public WebServicesClient()
@@ -105,10 +140,7 @@ public class WebServicesClient
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
httpClientBuilder.setConnectionManager(connectionManager);
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
- Lookup<AuthSchemeProvider> authProviders = RegistryBuilder.<AuthSchemeProvider>create()
- .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true))
- .build();
- httpClientBuilder.setDefaultAuthSchemeRegistry(authProviders);
+ httpClientBuilder.setDefaultAuthSchemeRegistry(authRegistry);
ApacheHttpClient4Handler httpClientHandler = new ApacheHttpClient4Handler(httpClientBuilder.build(), new BasicCookieStore(), false);
client = new Client(httpClientHandler, config);
} else {
@@ -183,6 +215,11 @@ public class WebServicesClient
});
}
+ public interface ConfigProvider
+ {
+ String getProperty(AuthScheme scheme, String name);
+ }
+
/**
*
* @param <T>
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java b/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java
index 86d4eae..a2623fe 100644
--- a/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java
@@ -24,6 +24,7 @@ import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import com.datatorrent.api.Context;
+import com.datatorrent.stram.security.AuthScheme;
/**
*
@@ -34,12 +35,30 @@ public class SecurityUtilsTest
public void testStramWebSecurity()
{
checkWebSecurity(false, false);
- Configuration conf = new Configuration();
+ Configuration conf = setupConfiguration(null);
checkSecurityConfiguration(conf, new boolean[][]{{false, false}, {false, true}, {false, false}, {false, false}, {false, false}});
- conf.set(SecurityUtils.HADOOP_HTTP_AUTH_PROP, "kerberos");
+ conf = setupConfiguration(AuthScheme.SPNEGO);
checkSecurityConfiguration(conf, new boolean[][]{{true, false}, {true, true}, {true, false}, {true, false}, {true, true}});
}
+ @Test
+ public void testInitAuth() throws NoSuchFieldException, IllegalAccessException
+ {
+ Configuration conf = setupConfiguration(AuthScheme.BASIC);
+ SecurityUtils.init(conf);
+ WebServicesClientTest.checkUserCredentials("testuser", "testpass");
+ }
+
+ private Configuration setupConfiguration(AuthScheme authScheme)
+ {
+ Configuration conf = new Configuration();
+ if (authScheme != null) {
+ conf.set(SecurityUtils.HADOOP_HTTP_AUTH_PROP, authScheme.getName());
+ conf.addResource("security/dt-site-" + authScheme.getName() + ".xml");
+ }
+ return conf;
+ }
+
private void checkSecurityConfiguration(Configuration conf, boolean[][] securityConf)
{
Assert.assertEquals("Number variations", 5, securityConf.length);
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java b/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java
index f07a7cf..fd422f0 100644
--- a/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java
@@ -18,15 +18,24 @@
*/
package com.datatorrent.stram.util;
+import java.lang.reflect.Field;
+
import org.junit.Assert;
import org.junit.Test;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.config.AuthSchemes;
/**
*
*/
public class WebServicesClientTest
{
+ private static final String CREDENTIALS_PROVIDER_FIELD = "credentialsProvider";
+
@Test
public void testFilterPresent()
{
@@ -35,4 +44,27 @@ public class WebServicesClientTest
webServicesClient.addFilter(clientFilter);
Assert.assertTrue("Filter present", webServicesClient.isFilterPresent(clientFilter));
}
+
+ public static void checkUserCredentials(String username, String password) throws NoSuchFieldException,
+ IllegalAccessException
+ {
+ CredentialsProvider provider = getCredentialsProvider();
+ AuthScope authScope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.BASIC);
+ Credentials credentials = provider.getCredentials(authScope);
+ Assert.assertNotNull("Credentials", credentials);
+ Assert.assertTrue("Credentials type is user", UsernamePasswordCredentials.class.isAssignableFrom(credentials.getClass()));
+ UsernamePasswordCredentials pwdCredentials = (UsernamePasswordCredentials)credentials;
+ Assert.assertEquals("Username", username, pwdCredentials.getUserName());
+ Assert.assertEquals("Password", password, pwdCredentials.getPassword());
+ }
+
+ private static CredentialsProvider getCredentialsProvider() throws NoSuchFieldException, IllegalAccessException
+ {
+ Field field = WebServicesClient.class.getDeclaredField(CREDENTIALS_PROVIDER_FIELD);
+ field.setAccessible(true);
+ CredentialsProvider credentials = (CredentialsProvider)field.get(null);
+ field.setAccessible(false);
+ return credentials;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/resources/security/dt-site-basic.xml
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/security/dt-site-basic.xml b/engine/src/test/resources/security/dt-site-basic.xml
new file mode 100644
index 0000000..e46e6a9
--- /dev/null
+++ b/engine/src/test/resources/security/dt-site-basic.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<configuration>
+ <property>
+ <name>dt.authentication.basic.username</name>
+ <value>testuser</value>
+ </property>
+ <property>
+ <name>dt.authentication.basic.password</name>
+ <value>testpass</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/resources/security/dt-site-kerberos.xml
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/security/dt-site-kerberos.xml b/engine/src/test/resources/security/dt-site-kerberos.xml
new file mode 100644
index 0000000..ab9ee1c
--- /dev/null
+++ b/engine/src/test/resources/security/dt-site-kerberos.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<configuration>
+ <property>
+ <name>dt.authentication.kerberos.principal</name>
+ <value>user/group@domain</value>
+ </property>
+ <property>
+ <name>dt.authentication.keberos.keytab</name>
+ <value>/security/user-keytab</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/resources/security/user.keytab
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/security/user.keytab b/engine/src/test/resources/security/user.keytab
new file mode 100644
index 0000000..f89b777
--- /dev/null
+++ b/engine/src/test/resources/security/user.keytab
@@ -0,0 +1 @@
+Fake keytab
http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e48adc3..6107df7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,6 +116,7 @@
<exclude>**/*.md</exclude>
<exclude>**/*.txt</exclude>
<exclude>**/*.yml</exclude>
+ <exclude>**/src/test/resources/**/*.keytab</exclude>
<exclude>**/*.importorder</exclude>
<exclude>**/archetype-resources/**</exclude>
<exclude>misc/ide-templates/**</exclude>
@@ -192,6 +193,7 @@
<exclude>.idea/**</exclude>
<exclude>**/src/test/resources/**/MANIFEST.MF</exclude>
<exclude>**/src/test/resources/**/*.json</exclude>
+ <exclude>**/src/test/resources/**/*.keytab</exclude>
<exclude>**/resources/META-INF/services/**</exclude>
<exclude>**/archetype-resources/**</exclude>
<exclude>**/*.md</exclude>