You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/08 02:03:31 UTC
[2/8] git commit: SLIDER-474 enable keytab-based security for AM
SLIDER-474 enable keytab-based security for AM
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2359c6dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2359c6dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2359c6dd
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2359c6ddbaab56c84a4c901b09c025593eee08ef
Parents: 80e8df0
Author: Jon Maron <jm...@hortonworks.com>
Authored: Mon Oct 6 15:50:36 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Mon Oct 6 15:50:36 2014 -0400
----------------------------------------------------------------------
.../org/apache/slider/common/SliderKeys.java | 4 +
.../slider/common/tools/CoreFileSystem.java | 15 ++
.../providers/agent/AgentProviderService.java | 27 +++
.../server/appmaster/SliderAppMaster.java | 161 ++++++++++-----
.../security/SecurityConfiguration.java | 201 +++++++++++++++++++
.../security/SecurityConfigurationTest.groovy | 159 +++++++++++++++
6 files changed, 520 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index ddb9ee0..4348fb0 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -86,6 +86,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
String HISTORY_DIR_NAME = "history";
String HISTORY_FILENAME_SUFFIX = "json";
String HISTORY_FILENAME_PREFIX = "rolehistory-";
+ String KEYTAB_DIR = "keytabs";
/**
* Filename pattern is required to save in strict temporal order.
@@ -170,6 +171,9 @@ public interface SliderKeys extends SliderXmlConfKeys {
String PASSPHRASE = "DEV";
String PASS_LEN = "50";
String KEYSTORE_LOCATION = "ssl.server.keystore.location";
+ String AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ String AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+ String KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
/**
* Python specific
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 955d991..b6e6ecf 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -134,6 +134,21 @@ public class CoreFileSystem {
}
/**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab installation location
+ */
+ public Path buildKeytabPath(String keytabName, String applicationName) {
+ Preconditions.checkNotNull(applicationName);
+ Path basePath = getBaseApplicationPath();
+ Path baseKeytabDir = new Path(basePath, SliderKeys.KEYTAB_DIR);
+ Path appKeytabDir = new Path(baseKeytabDir, applicationName);
+ return keytabName == null ? appKeytabDir :
+ new Path(appKeytabDir, keytabName);
+ }
+
+ /**
* Create the Slider cluster path for a named cluster and all its subdirs
* This is a directory; a mkdirs() operation is executed
* to ensure that it is there.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 67a268e..88c8709 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
import org.apache.curator.utils.ZKPaths;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -325,6 +326,32 @@ public class AgentProviderService extends AbstractProviderService implements
launcher.addLocalResource(AgentKeys.AGENT_VERSION_FILE, agentVerRes);
}
+ if (SliderUtils.isHadoopClusterSecure(getConfig())) {
+ String keytabFullPath = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderKeys.AM_KEYTAB_LOCAL_PATH);
+ String amKeytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ if (SliderUtils.isUnset(keytabFullPath)) {
+ // we need to localize the keytab files in the directory
+ Path keytabDir = fileSystem.buildKeytabPath(null,
+ getAmState().getApplicationName());
+ FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDir);
+ LocalResource keytabRes;
+ for (FileStatus keytab : keytabs) {
+ if (!amKeytabName.equals(keytab.getPath().getName())) {
+ log.info("Localizing keytab {}", keytab.getPath().getName());
+ keytabRes = fileSystem.createAmResource(keytab.getPath(),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" +
+ keytab.getPath().getName(),
+ keytabRes);
+ }
+ }
+ }
+ }
+
//add the configuration resources
launcher.addLocalResources(fileSystem.submitDirectory(
generatedConfPath,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 5676f3f..50fc265 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.security.Credentials;
@@ -121,6 +122,7 @@ import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider;
import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.security.SecurityConfiguration;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.ProviderAppState;
@@ -226,7 +228,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
/**
* token blob
*/
- private ByteBuffer allTokens;
+ private Credentials containerTokens;
private WorkflowRpcService rpcService;
@@ -430,19 +432,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
super.serviceStart();
}
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
-
- if (fsDelegationTokenManager != null) {
- try {
- fsDelegationTokenManager.cancelDelegationToken(getConfig());
- } catch (Exception e) {
- log.info("Error cancelling HDFS delegation token", e);
- }
- }
- }
-
/**
* Start the queue processing
*/
@@ -559,6 +548,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// triggers resolution and snapshotting in agent
appState.updateInstanceDefinition(instanceDefinition);
+
+ Configuration serviceConf = getConfig();
+
File confDir = getLocalConfDir();
if (!confDir.exists() || !confDir.isDirectory()) {
log.info("Conf dir {} does not exist.", confDir);
@@ -566,8 +558,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile));
}
- Configuration serviceConf = getConfig();
- // IP filtering
+ // IP filtering
serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS, AM_FILTER_NAME);
//get our provider
@@ -706,16 +697,49 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// set the RM-defined maximum cluster values
appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores));
appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory));
-
- boolean securityEnabled = UserGroupInformation.isSecurityEnabled();
+
+ // process the initial user to obtain the set of user
+ // supplied credentials (tokens were passed in by client). Remove AMRM
+ // token and HDFS delegation token, the latter because we will provide an
+ // up to date token for container launches (getContainerTokens()).
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ Credentials credentials = currentUser.getCredentials();
+ Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<?> token = iter.next();
+ log.info("Token {}", token.getKind());
+ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME) ||
+ token.getKind().equals(DelegationTokenIdentifier.HDFS_DELEGATION_KIND)) {
+ iter.remove();
+ }
+ }
+ // at this point this credentials map is probably clear, but leaving this
+ // code to allow for future tokens...
+ containerTokens = credentials;
+
+ SecurityConfiguration securityConfiguration = new SecurityConfiguration(
+ serviceConf, instanceDefinition, clustername);
+ // obtain security state
+ boolean securityEnabled = securityConfiguration.isSecurityEnabled();
if (securityEnabled) {
secretManager.setMasterKey(
amRegistrationData.getClientToAMTokenMasterKey().array());
applicationACLs = amRegistrationData.getApplicationACLs();
- //tell the server what the ACLs are
+ //tell the server what the ACLs are
rpcService.getServer().refreshServiceAcl(serviceConf,
new SliderAMPolicyProvider());
+ // perform keytab based login to establish kerberos authenticated
+ // principal. Can do so now since AM registration with RM above required
+ // tokens associated to principal
+ String principal = securityConfiguration.getPrincipal();
+ File localKeytabFile = securityConfiguration.getKeytabFile(
+ fs, instanceDefinition, principal);
+ // Now log in...
+ login(principal, localKeytabFile);
+ // obtain new FS reference that should be kerberos based and different
+ // than the previously cached reference
+ fs = getClusterFS();
}
// extract container list
@@ -795,27 +819,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
maybeStartMonkey();
// setup token renewal and expiry handling for long lived apps
- if (SliderUtils.isHadoopClusterSecure(getConfig())) {
- fsDelegationTokenManager = new FsDelegationTokenManager(actionQueues);
- fsDelegationTokenManager.acquireDelegationToken(getConfig());
- }
-
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- Credentials credentials =
- currentUser.getCredentials();
- DataOutputBuffer dob = new DataOutputBuffer();
- credentials.writeTokenStorageToStream(dob);
- dob.close();
- // Now remove the AM->RM token so that containers cannot access it.
- Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
- while (iter.hasNext()) {
- Token<?> token = iter.next();
- log.info("Token {}", token.getKind());
- if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
- iter.remove();
- }
- }
- allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+// if (SliderUtils.isHadoopClusterSecure(getConfig())) {
+// fsDelegationTokenManager = new FsDelegationTokenManager(actionQueues);
+// fsDelegationTokenManager.acquireDelegationToken(getConfig());
+// }
// if not a secure cluster, extract the username -it will be
// propagated to workers
@@ -861,6 +868,40 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return finish();
}
+ protected void login(String principal, File localKeytabFile)
+ throws IOException, SliderException {
+ UserGroupInformation.loginUserFromKeytab(principal,
+ localKeytabFile.getAbsolutePath());
+ validateLoginUser(UserGroupInformation.getLoginUser());
+ }
+
+ /**
+ * Ensure that the user is generated from a keytab and has no HDFS delegation
+ * tokens.
+ *
+ * @param user
+ * @throws SliderException
+ */
+ protected void validateLoginUser(UserGroupInformation user)
+ throws SliderException {
+ if (!user.isFromKeytab()) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, "User is "
+ + "not based on a keytab in a secure deployment.");
+ }
+ Credentials credentials =
+ user.getCredentials();
+ Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<?> token = iter.next();
+ log.info("Token {}", token.getKind());
+ if (token.getKind().equals(
+ DelegationTokenIdentifier.HDFS_DELEGATION_KIND)) {
+ log.info("Unexpected HDFS delegation token. Removing...");
+ iter.remove();
+ }
+ }
+ }
+
private void startAgentWebApp(MapOperations appInformation,
Configuration serviceConf) throws IOException {
URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs();
@@ -1387,9 +1428,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public void onShutdownRequest() {
LOG_YARN.info("Shutdown Request received");
signalAMComplete(new ActionStopSlider("stop",
- EXIT_SUCCESS,
- FinalApplicationStatus.SUCCEEDED,
- "Shutdown requested from RM"));
+ EXIT_SUCCESS,
+ FinalApplicationStatus.SUCCEEDED,
+ "Shutdown requested from RM"));
}
/**
@@ -1558,7 +1599,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
YarnException {
onRpcCall("getNode()");
RoleInstance instance = appState.getLiveInstanceByContainerID(
- request.getUuid());
+ request.getUuid());
return Messages.GetNodeResponseProto.newBuilder()
.setClusterNode(instance.toProtobuf())
.build();
@@ -1571,7 +1612,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
onRpcCall("getClusterNodes()");
List<RoleInstance>
clusterNodes = appState.getLiveInstancesByContainerIDs(
- request.getUuidList());
+ request.getUuidList());
Messages.GetClusterNodesResponseProto.Builder builder =
Messages.GetClusterNodesResponseProto.newBuilder();
@@ -1783,18 +1824,44 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
public void startContainer(Container container,
ContainerLaunchContext ctx,
- RoleInstance instance) {
+ RoleInstance instance) throws IOException {
// Set up tokens for the container too. Today, for normal shell commands,
// the container in distribute-shell doesn't need any tokens. We are
// populating them mainly for NodeManagers to be able to download any
// files in the distributed file-system. The tokens are otherwise also
// useful in cases, for e.g., when one is running a "hadoop dfs" command
// inside the distributed shell.
- ctx.setTokens(allTokens.duplicate());
+
+ // add current HDFS delegation token with an up to date token
+ ByteBuffer tokens = getContainerTokens();
+
+ if (tokens != null) {
+ ctx.setTokens(tokens);
+ } else {
+ log.warn("No delegation tokens obtained and set for launch context");
+ }
appState.containerStartSubmitted(container, instance);
nmClientAsync.startContainerAsync(container, ctx);
}
+ private ByteBuffer getContainerTokens() throws IOException {
+ // a delegation token can be retrieved from filesystem since
+ // the login is via a keytab (see above)
+ ByteBuffer tokens = null;
+ Token hdfsToken = getClusterFS().getFileSystem().getDelegationToken
+ (UserGroupInformation.getLoginUser().getShortUserName());
+ if (hdfsToken != null) {
+ Credentials credentials = new Credentials(containerTokens);
+ credentials.addToken(hdfsToken.getKind(), hdfsToken);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ credentials.writeTokenStorageToStream(dob);
+ dob.close();
+ tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ }
+
+ return tokens;
+ }
+
@Override // NMClientAsync.CallbackHandler
public void onContainerStopped(ContainerId containerId) {
// do nothing but log: container events from the AM
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
new file mode 100644
index 0000000..448d02f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.security;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ *
+ */
+public class SecurityConfiguration {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(SecurityConfiguration.class);
+ private final Configuration configuration;
+ private final AggregateConf instanceDefinition;
+ private String clusterName;
+
+ public SecurityConfiguration(Configuration configuration,
+ AggregateConf instanceDefinition,
+ String clusterName) throws SliderException {
+ Preconditions.checkNotNull(configuration);
+ Preconditions.checkNotNull(instanceDefinition);
+ Preconditions.checkNotNull(clusterName);
+ this.configuration = configuration;
+ this.instanceDefinition = instanceDefinition;
+ this.clusterName = clusterName;
+ validate();
+ }
+
+ private void validate() throws SliderException {
+ if (isSecurityEnabled()) {
+ String principal = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.KEYTAB_PRINCIPAL);
+ if(SliderUtils.isUnset(principal)) {
+ // if no login identity is available, fail
+ UserGroupInformation loginUser = null;
+ try {
+ loginUser = getLoginUser();
+ } catch (IOException e) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, e,
+ "No principal configured for the application and"
+ + "exception raised during retrieval of login user. "
+ + "Unable to proceed with application "
+ + "initialization. Please ensure a value "
+ + "for %s exists in the application "
+ + "configuration or the login issue is addressed",
+ SliderKeys.KEYTAB_PRINCIPAL);
+ }
+ if (loginUser == null) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "No principal configured for the application "
+ + "and no login user found. "
+ + "Unable to proceed with application "
+ + "initialization. Please ensure a value "
+ + "for %s exists in the application "
+ + "configuration or the login issue is addressed",
+ SliderKeys.KEYTAB_PRINCIPAL);
+ }
+ }
+ // ensure that either local or distributed keytab mechanism is enabled,
+ // but not both
+ String keytabFullPath = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM)
+ .get(SliderKeys.AM_KEYTAB_LOCAL_PATH);
+ String keytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM)
+ .get(SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ if (SliderUtils.isUnset(keytabFullPath) && SliderUtils.isUnset(keytabName)) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Either a keytab path on the cluster host (%s) or a"
+ + " keytab to be retrieved from HDFS (%s) are"
+ + " required. Please configure one of the keytab"
+ + " retrieval mechanisms.",
+ SliderKeys.AM_KEYTAB_LOCAL_PATH,
+ SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ }
+ if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Both a keytab on the cluster host (%s) and a"
+ + " keytab to be retrieved from HDFS (%s) are"
+ + " specified. Please configure only one keytab"
+ + " retrieval mechanism.",
+ SliderKeys.AM_KEYTAB_LOCAL_PATH,
+ SliderKeys.AM_LOGIN_KEYTAB_NAME);
+
+ }
+ }
+ }
+
+ protected UserGroupInformation getLoginUser() throws IOException {
+ return UserGroupInformation.getLoginUser();
+ }
+
+ public boolean isSecurityEnabled () {
+ return SliderUtils.isHadoopClusterSecure(configuration);
+ }
+
+ public String getPrincipal () throws IOException {
+ String principal = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.KEYTAB_PRINCIPAL);
+ if (SliderUtils.isUnset(principal)) {
+ principal = UserGroupInformation.getLoginUser().getShortUserName();
+ log.info("No principal set in the slider configuration. Will use AM login"
+ + " identity {} to attempt keytab-based login", principal);
+ }
+
+ return principal;
+ }
+
+ public File getKeytabFile(SliderFileSystem fs,
+ AggregateConf instanceDefinition, String principal)
+ throws SliderException, IOException {
+ String keytabFullPath = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM)
+ .get(SliderKeys.AM_KEYTAB_LOCAL_PATH);
+ File localKeytabFile;
+ if (SliderUtils.isUnset(keytabFullPath)) {
+ // get the keytab
+ String keytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ log.info("No host keytab file path specified. Downloading keytab {}"
+ + " from HDFS to perform login of using principal {}",
+ keytabName, principal);
+ // download keytab to local, protected directory
+ localKeytabFile = getFileFromFileSystem(fs, keytabName);
+ } else {
+ log.info("Leveraging host keytab file {} to login principal {}",
+ keytabFullPath, principal);
+ localKeytabFile = new File(keytabFullPath);
+ }
+ return localKeytabFile;
+ }
+
+ /**
+ * Download the keytab file from FileSystem to local file.
+ * @param fs
+ * @param keytabName
+ * @return
+ * @throws SliderException
+ * @throws IOException
+ */
+ protected File getFileFromFileSystem(SliderFileSystem fs, String keytabName)
+ throws SliderException, IOException {
+ File keytabDestinationDir = new File(
+ FileUtils.getTempDirectory().getAbsolutePath() +
+ "/keytab" + System.currentTimeMillis());
+ if (!keytabDestinationDir.mkdirs()) {
+ throw new SliderException("Unable to create local keytab directory");
+ }
+ RawLocalFileSystem fileSystem = new RawLocalFileSystem();
+ // allow app user to access local keytab dir
+ FsPermission permissions = new FsPermission(FsAction.ALL, FsAction.NONE,
+ FsAction.NONE);
+ fileSystem.setPermission(new Path(keytabDestinationDir.getAbsolutePath()),
+ permissions);
+
+ Path keytabPath = fs.buildKeytabPath(keytabName, clusterName);
+ File localKeytabFile = new File(keytabDestinationDir, keytabName);
+ FileUtil.copy(fs.getFileSystem(), keytabPath,
+ localKeytabFile,
+ false, configuration);
+ // set permissions on actual keytab file to be read-only for user
+ permissions = new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE);
+ fileSystem.setPermission(new Path(localKeytabFile.getAbsolutePath()),
+ permissions);
+ return localKeytabFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
new file mode 100644
index 0000000..1dcbd9c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.security
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.MapOperations
+import org.apache.slider.core.exceptions.SliderException;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class SecurityConfigurationTest {
+ final shouldFail = new GroovyTestCase().&shouldFail
+
+ @Test
+ public void testValidLocalConfiguration() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testValidDistributedConfiguration() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testMissingPrincipalNoLoginWithDistributedConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster") {
+ @Override
+ protected UserGroupInformation getLoginUser() throws IOException {
+ return null
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMissingPrincipalNoLoginWithLocalConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster") {
+ @Override
+ protected UserGroupInformation getLoginUser() throws IOException {
+ return null
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testBothKeytabMechanismsConfigured() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+ compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+ }
+
+ @Test
+ public void testNoKeytabMechanismConfigured() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+ }
+
+ @Test
+ public void testMissingPrincipalButLoginWithDistributedConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testMissingPrincipalButLoginWithLocalConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+}