You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/10 03:41:05 UTC
git commit: SLIDER-497 Secure_funtests_failing_no_keytab
Repository: incubator-slider
Updated Branches:
refs/heads/develop e39d99e19 -> 3b3886d4f
SLIDER-497 Secure_funtests_failing_no_keytab
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3b3886d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3b3886d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3b3886d4
Branch: refs/heads/develop
Commit: 3b3886d4ff5489daca9685f789038d53de16cfbf
Parents: e39d99e
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 9 18:40:27 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 9 18:40:27 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 10 ++-
.../org/apache/slider/common/SliderKeys.java | 5 --
.../common/SliderXMLConfKeysForTesting.java | 5 ++
.../apache/slider/common/SliderXmlConfKeys.java | 5 ++
.../providers/agent/AgentProviderService.java | 5 +-
.../server/appmaster/SliderAppMaster.java | 2 +-
.../security/SecurityConfiguration.java | 27 +++----
.../server/services/security/SecurityUtils.java | 5 +-
.../agent/TestAgentAMManagementWS.groovy | 4 +-
.../security/SecurityConfigurationTest.groovy | 25 +++----
.../security/TestCertificateManager.java | 3 +-
.../framework/AgentCommandTestBase.groovy | 15 ++--
.../funtest/framework/CommandTestBase.groovy | 75 +++++++++++++++++++-
.../funtest/lifecycle/AMFailuresIT.groovy | 9 +--
.../lifecycle/AgentClusterLifecycleIT.groovy | 10 +--
.../funtest/lifecycle/AgentFailures2IT.groovy | 12 ++--
.../funtest/lifecycle/AgentFailuresIT.groovy | 9 +--
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 9 +--
.../AppsThroughAgentQueueAndLabelsIT.groovy | 12 ++--
.../clusters/morzine/slider/slider-client.xml | 17 +++++
.../clusters/offline/slider/slider-client.xml | 16 +++++
.../clusters/sandbox/slider/slider-client.xml | 17 +++++
22 files changed, 206 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 476f300..78f214e 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -125,6 +124,7 @@ import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
+import org.apache.slider.server.appmaster.security.SecurityConfiguration;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -1106,6 +1106,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// will be valid.
propagatePrincipals(config, instanceDefinition);
+ // validate security data
+/*
+ // turned off until tested
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config,
+ instanceDefinition, clustername);
+
+*/
Configuration clientConfExtras = new Configuration(false);
// then build up the generated path.
FsPermission clusterPerms = getClusterDirectoryPermissions(config);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 68cef45..e75ec73 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -170,11 +170,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
String CRT_PASS_FILE_NAME = "pass.txt";
String PASSPHRASE = "DEV";
String PASS_LEN = "50";
- String KEYSTORE_LOCATION = "ssl.server.keystore.location";
- String AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
- String AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
- String KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
- String SECURITY_ENABLED = "site.global.security_enabled";
/**
* Python specific
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
index 8886cb7..bc1eaba 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
@@ -83,4 +83,9 @@ public interface SliderXMLConfKeysForTesting {
* security related keys
*/
String TEST_SECURITY_DIR = "/tmp/work/security";
+
+ /**
+ * Local path to AM keytab: {@value}
+ */
+ String KEY_TEST_AM_KEYTAB = "slider.test.am.keytab.local";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 243dd95..d82bbe8 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -142,4 +142,9 @@ public interface SliderXmlConfKeys {
"ipc.client.fallback-to-simple-auth-allowed";
String HADOOP_HTTP_FILTER_INITIALIZERS =
"hadoop.http.filter.initializers";
+ String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location";
+ String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+ String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
+ String KEY_SECURITY_ENABLED = "site.global.security_enabled";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 4fe4b6a..058a838 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -40,6 +40,7 @@ import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
@@ -358,10 +359,10 @@ public class AgentProviderService extends AbstractProviderService implements
if (SliderUtils.isHadoopClusterSecure(getConfig())) {
String keytabFullPath = instanceDefinition.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM).get(
- SliderKeys.AM_KEYTAB_LOCAL_PATH);
+ SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
String amKeytabName = instanceDefinition.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM).get(
- SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
if (SliderUtils.isUnset(keytabFullPath)) {
// we need to localize the keytab files in the directory
Path keytabDir = fileSystem.buildKeytabPath(null,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index fcc2802..3f52fd8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -560,7 +560,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
boolean securityEnabled = securityConfiguration.isSecurityEnabled();
// set the global security flag for the instance definition
instanceDefinition.getAppConfOperations().set(
- SECURITY_ENABLED, securityEnabled);
+ KEY_SECURITY_ENABLED, securityEnabled);
// triggers resolution and snapshotting in agent
appState.updateInstanceDefinition(instanceDefinition);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
index 448d02f..e5cdad2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
@@ -63,7 +64,7 @@ public class SecurityConfiguration {
private void validate() throws SliderException {
if (isSecurityEnabled()) {
String principal = instanceDefinition.getAppConfOperations()
- .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.KEYTAB_PRINCIPAL);
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
if(SliderUtils.isUnset(principal)) {
// if no login identity is available, fail
UserGroupInformation loginUser = null;
@@ -71,13 +72,13 @@ public class SecurityConfiguration {
loginUser = getLoginUser();
} catch (IOException e) {
throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, e,
- "No principal configured for the application and"
+ "No principal configured for the application and "
+ "exception raised during retrieval of login user. "
+ "Unable to proceed with application "
+ "initialization. Please ensure a value "
+ "for %s exists in the application "
+ "configuration or the login issue is addressed",
- SliderKeys.KEYTAB_PRINCIPAL);
+ SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
}
if (loginUser == null) {
throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
@@ -87,25 +88,25 @@ public class SecurityConfiguration {
+ "initialization. Please ensure a value "
+ "for %s exists in the application "
+ "configuration or the login issue is addressed",
- SliderKeys.KEYTAB_PRINCIPAL);
+ SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
}
}
// ensure that either local or distributed keytab mechanism is enabled,
// but not both
String keytabFullPath = instanceDefinition.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM)
- .get(SliderKeys.AM_KEYTAB_LOCAL_PATH);
+ .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
String keytabName = instanceDefinition.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM)
- .get(SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ .get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
if (SliderUtils.isUnset(keytabFullPath) && SliderUtils.isUnset(keytabName)) {
throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
"Either a keytab path on the cluster host (%s) or a"
+ " keytab to be retrieved from HDFS (%s) are"
+ " required. Please configure one of the keytab"
+ " retrieval mechanisms.",
- SliderKeys.AM_KEYTAB_LOCAL_PATH,
- SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
+ SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
}
if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) {
throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
@@ -113,8 +114,8 @@ public class SecurityConfiguration {
+ " keytab to be retrieved from HDFS (%s) are"
+ " specified. Please configure only one keytab"
+ " retrieval mechanism.",
- SliderKeys.AM_KEYTAB_LOCAL_PATH,
- SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
+ SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
}
}
@@ -130,7 +131,7 @@ public class SecurityConfiguration {
public String getPrincipal () throws IOException {
String principal = instanceDefinition.getAppConfOperations()
- .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.KEYTAB_PRINCIPAL);
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
if (SliderUtils.isUnset(principal)) {
principal = UserGroupInformation.getLoginUser().getShortUserName();
log.info("No principal set in the slider configuration. Will use AM login"
@@ -145,12 +146,12 @@ public class SecurityConfiguration {
throws SliderException, IOException {
String keytabFullPath = instanceDefinition.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM)
- .get(SliderKeys.AM_KEYTAB_LOCAL_PATH);
+ .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
File localKeytabFile;
if (SliderUtils.isUnset(keytabFullPath)) {
// get the keytab
String keytabName = instanceDefinition.getAppConfOperations()
- .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
log.info("No host keytab file path specified. Downloading keytab {}"
+ " from HDFS to perform login of using principal {}",
keytabName, principal);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
index 56ee199..c7ad8dd 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.conf.MapOperations;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,7 +35,7 @@ import java.io.IOException;
//import java.nio.file.Paths;
//import java.nio.file.attribute.PosixFilePermission;
//import java.nio.file.attribute.PosixFilePermissions;
-import java.util.Set;
+
/**
*
@@ -140,7 +141,7 @@ public class SecurityUtils {
public static void initializeSecurityParameters(MapOperations configMap) {
String keyStoreLocation = configMap.getOption(
- SliderKeys.KEYSTORE_LOCATION, getDefaultKeystoreLocation());
+ SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, getDefaultKeystoreLocation());
File secDirFile = new File(keyStoreLocation).getParentFile();
if (!secDirFile.exists()) {
// create entire required directory structure
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 7a39035..da62792 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -27,7 +27,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs
+import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.core.build.InstanceBuilder
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.MapOperations
@@ -90,7 +90,7 @@ class TestAgentAMManagementWS extends AgentTestBase {
void setup() {
super.setup()
MapOperations compOperations = new MapOperations();
- compOperations.put(SliderKeys.KEYSTORE_LOCATION, "/tmp/work/security/keystore.p12");
+ compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, "/tmp/work/security/keystore.p12");
SecurityUtils.initializeSecurityParameters(compOperations);
CertificateManager certificateManager = new CertificateManager();
certificateManager.initRootCert(compOperations);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
index 1dcbd9c..4ef142a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.CommonConfigurationKeysPublic
import org.apache.hadoop.security.UserGroupInformation
import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.MapOperations
import org.apache.slider.core.exceptions.SliderException;
@@ -38,8 +39,8 @@ public class SecurityConfigurationTest {
AggregateConf aggregateConf = new AggregateConf();
MapOperations compOps =
aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
- compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
- compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+ compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path")
SecurityConfiguration securityConfiguration =
new SecurityConfiguration(config, aggregateConf, "testCluster")
@@ -52,8 +53,8 @@ public class SecurityConfigurationTest {
AggregateConf aggregateConf = new AggregateConf();
MapOperations compOps =
aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
- compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
- compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+ compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab")
SecurityConfiguration securityConfiguration =
new SecurityConfiguration(config, aggregateConf, "testCluster")
@@ -66,7 +67,7 @@ public class SecurityConfigurationTest {
AggregateConf aggregateConf = new AggregateConf();
MapOperations compOps =
aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
- compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+ compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab")
shouldFail(SliderException) {
SecurityConfiguration securityConfiguration =
@@ -86,7 +87,7 @@ public class SecurityConfigurationTest {
AggregateConf aggregateConf = new AggregateConf();
MapOperations compOps =
aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
- compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+ compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path")
shouldFail(SliderException) {
SecurityConfiguration securityConfiguration =
@@ -106,9 +107,9 @@ public class SecurityConfigurationTest {
AggregateConf aggregateConf = new AggregateConf();
MapOperations compOps =
aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
- compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
- compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
- compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+ compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+ compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab")
shouldFail(SliderException) {
SecurityConfiguration securityConfiguration =
@@ -123,7 +124,7 @@ public class SecurityConfigurationTest {
AggregateConf aggregateConf = new AggregateConf();
MapOperations compOps =
aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
- compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test")
shouldFail(SliderException) {
SecurityConfiguration securityConfiguration =
@@ -138,7 +139,7 @@ public class SecurityConfigurationTest {
AggregateConf aggregateConf = new AggregateConf();
MapOperations compOps =
aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
- compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+ compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab")
SecurityConfiguration securityConfiguration =
new SecurityConfiguration(config, aggregateConf, "testCluster")
@@ -151,7 +152,7 @@ public class SecurityConfigurationTest {
AggregateConf aggregateConf = new AggregateConf();
MapOperations compOps =
aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
- compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+ compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path")
SecurityConfiguration securityConfiguration =
new SecurityConfiguration(config, aggregateConf, "testCluster")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
index 6d2d051..710757d 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
@@ -17,6 +17,7 @@
package org.apache.slider.server.services.security;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.conf.MapOperations;
import org.junit.Assert;
import org.junit.Before;
@@ -40,7 +41,7 @@ public class TestCertificateManager {
MapOperations compOperations = new MapOperations();
secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR);
File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
- compOperations.put(SliderKeys.KEYSTORE_LOCATION,
+ compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION,
keystoreFile.getAbsolutePath());
certMan.initRootCert(compOperations);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 602fe2c..ec7d3e3 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -21,7 +21,6 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderXMLConfKeysForTesting
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.common.tools.SliderUtils
@@ -111,7 +110,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
[
ACTION_INSTALL_PACKAGE,
Arguments.ARG_NAME, TEST_APP_PKG_NAME,
- Arguments.ARG_PACKAGE, zipFileName,
+ Arguments.ARG_PACKAGE, zipFileName.absolutePath,
Arguments.ARG_REPLACE_PKG
])
logShell(shell)
@@ -123,12 +122,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
}
public static void logShell(SliderShell shell) {
- for (String str in shell.out) {
- log.info str
- }
- for (String str in shell.err) {
- log.error str
- }
+ shell.dumpOutput();
}
public static void assertComponentCount(String component, int count, SliderShell shell) {
@@ -185,13 +179,14 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
}
public static void addDir(File dirObj, ZipOutputStream zipFile, String prefix) {
- dirObj.eachFile() { file ->
+ dirObj.eachFile() {File file ->
if (file.directory) {
addDir(file, zipFile, prefix + file.name + File.separator)
} else {
log.info("Adding to zip - " + prefix + file.getName())
zipFile.putNextEntry(new ZipEntry(prefix + file.getName()))
- file.eachByte(1024) { buffer, len -> zipFile.write(buffer, 0, len) }
+ file.eachByte(1024) {
+ byte[] buffer, int len -> zipFile.write(buffer, 0, len) }
zipFile.closeEntry()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index d780ac7..db1c562 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.util.ExitUtil
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
@@ -71,7 +72,12 @@ abstract class CommandTestBase extends SliderTestUtils {
public static final int SLIDER_TEST_TIMEOUT
public static final String YARN_RAM_REQUEST
-
+
+ /**
+ * Keytab for secure cluster
+ */
+ public static final String TEST_AM_KEYTAB
+ static File keytabFile
static {
@@ -86,10 +92,13 @@ abstract class CommandTestBase extends SliderTestUtils {
KEY_TEST_TIMEOUT,
1000 * DEFAULT_TEST_TIMEOUT_SECONDS)
- YARN_RAM_REQUEST = SLIDER_CONFIG.get(
+ YARN_RAM_REQUEST = SLIDER_CONFIG.getTrimmed(
KEY_TEST_YARN_RAM_REQUEST,
DEFAULT_YARN_RAM_REQUEST)
-
+
+ TEST_AM_KEYTAB = SLIDER_CONFIG.getTrimmed(
+ KEY_TEST_AM_KEYTAB)
+
}
@Rule
@@ -102,6 +111,18 @@ abstract class CommandTestBase extends SliderTestUtils {
if (SliderUtils.maybeInitSecurity(conf)) {
log.debug("Security enabled")
SliderUtils.forceLogin()
+ // now look for the security key
+/*
+ if (!TEST_AM_KEYTAB) {
+ fail("Security keytab is not defined in $KEY_TEST_AM_KEYTAB")
+ }
+ keytabFile = new File(TEST_AM_KEYTAB)
+ if (!keytabFile.exists()) {
+ throw new FileNotFoundException("Security keytab ${keytabFile} " +
+ " defined in $KEY_TEST_AM_KEYTAB")
+ }
+*/
+
} else {
log.info "Security is off"
}
@@ -465,6 +486,54 @@ abstract class CommandTestBase extends SliderTestUtils {
clusterOps)
}
+ /**
+ * Create a templated slider app
+ * @param name name
+ * @param appTemplate application template
+ * @param resourceTemplate resource template
+ * @return the shell
+ */
+ public SliderShell createTemplatedSliderApplication(
+ String name,
+ String appTemplate,
+ String resourceTemplate,
+ List<String> extraArgs=[]) {
+ List<String> commands = [
+ ACTION_CREATE, name,
+ ARG_TEMPLATE, appTemplate,
+ ARG_RESOURCES, resourceTemplate
+ ]
+
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME));
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH));
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL));
+ commands.addAll(extraArgs)
+ SliderShell shell = slider(LauncherExitCodes.EXIT_SUCCESS, commands)
+ return shell
+ }
+
+ /**
+ * If the option is not null/empty, add the command and the option
+ * @param args arg list being built up
+ * @param command command to add option
+ * @param option option to probe and use
+ * @return the (possibly extended) list
+ */
+ public List<String> maybeAddCommandOption(
+ List<String> args, List<String> commands, String option) {
+ if ( SliderUtils.isSet(option)) {
+ args.addAll(commands)
+ args << option
+ }
+ return args
+ }
+
public Path buildClusterPath(String clustername) {
return new Path(
clusterFS.homeDirectory,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 9db0fec..988d34d 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -60,12 +60,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
@Test
public void testAMKilledWithStateAMStartedAgentsStarted() throws Throwable {
cleanup(APPLICATION_NAME)
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, APPLICATION_NAME,
- ARG_TEMPLATE, APP_TEMPLATE,
- ARG_RESOURCES, APP_RESOURCE
- ])
+ SliderShell shell = createTemplatedSliderApplication(
+ APPLICATION_NAME, APP_TEMPLATE, APP_RESOURCE
+ )
logShell(shell)
ensureApplicationIsUp(APPLICATION_NAME)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 48ae266..d048aca 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -64,13 +64,9 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
def clusterpath = buildClusterPath(CLUSTER)
assert !clusterFS.exists(clusterpath)
-
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, CLUSTER,
- ARG_TEMPLATE, APP_TEMPLATE,
- ARG_RESOURCES, APP_RESOURCE2
- ])
+ SliderShell shell = createTemplatedSliderApplication(CLUSTER,
+ APP_TEMPLATE,
+ APP_RESOURCE2)
logShell(shell)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 9359521..a02fc0e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -53,12 +53,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
}
cleanup(APPLICATION_NAME)
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, APPLICATION_NAME,
- ARG_TEMPLATE, APP_TEMPLATE3,
- ARG_RESOURCES, APP_RESOURCE
- ])
+ SliderShell shell = createTemplatedSliderApplication(APP_TEMPLATE3,
+ ARG_RESOURCES,
+ APP_RESOURCE)
logShell(shell)
@@ -74,7 +71,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
APPLICATION_NAME])
assertComponentCount(COMMAND_LOGGER, 1, shell)
- String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
+ String requested = findLineEntryValue(shell,
+ ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
assert requested != null && requested.isInteger() && requested.toInteger() >= 3,
'At least 2 containers must be requested'
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index fcbfb3c..7575fc6 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -53,12 +53,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
}
cleanup(APPLICATION_NAME)
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, APPLICATION_NAME,
- ARG_TEMPLATE, APP_TEMPLATE2,
- ARG_RESOURCES, APP_RESOURCE
- ])
+ def shell = createTemplatedSliderApplication( APPLICATION_NAME,
+ APP_TEMPLATE2,
+ APP_RESOURCE)
logShell(shell)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 7e39791..234275a 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -47,12 +47,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assumeAgentTestsEnabled()
cleanup(APPLICATION_NAME)
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, APPLICATION_NAME,
- ARG_TEMPLATE, APP_TEMPLATE,
- ARG_RESOURCES, APP_RESOURCE
- ])
+ SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME,
+ APP_TEMPLATE,
+ APP_RESOURCE)
logShell(shell)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index a3b0ccb..6732691 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -78,13 +78,11 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assumeLabelsRedAndBlueAdded()
cleanup(APPLICATION_NAME)
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, APPLICATION_NAME,
- ARG_QUEUE, TARGET_QUEUE,
- ARG_TEMPLATE, APP_TEMPLATE,
- ARG_RESOURCES, APP_RESOURCE4
- ])
+ SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME,
+ APP_TEMPLATE,
+ APP_RESOURCE4,
+ [ARG_QUEUE, TARGET_QUEUE]
+ )
logShell(shell)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/src/test/clusters/morzine/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/morzine/slider/slider-client.xml b/src/test/clusters/morzine/slider/slider-client.xml
index 4d7ab41..dbf25bd 100644
--- a/src/test/clusters/morzine/slider/slider-client.xml
+++ b/src/test/clusters/morzine/slider/slider-client.xml
@@ -73,4 +73,21 @@
</property>
+ <property>
+ <name>slider.am.login.keytab.name</name>
+ <value>Location of keytab in HDFS</value>
+ </property>
+
+ <property>
+ <name>slider.am.keytab.local.path</name>
+ <description>absolute path to keytab</description>
+ <value></value>
+ </property>
+
+ <property>
+ <name>slider.keytab.principal.name</name>
+ <value>Optional principal name in keytab</value>
+ </property>
+
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/src/test/clusters/offline/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/offline/slider/slider-client.xml b/src/test/clusters/offline/slider/slider-client.xml
index 25c01cf..0f780f0 100644
--- a/src/test/clusters/offline/slider/slider-client.xml
+++ b/src/test/clusters/offline/slider/slider-client.xml
@@ -90,6 +90,22 @@
<value>file://${user.dir}/src/test/configs/sandbox/accumulo</value>
</property>
+ <property>
+ <name>slider.am.login.keytab.name</name>
+ <value>Location of keytab in HDFS</value>
+ </property>
+
+ <property>
+ <name>slider.am.keytab.local.path</name>
+ <description>absolute path to keytab</description>
+ <value></value>
+ </property>
+
+ <property>
+ <name>slider.keytab.principal.name</name>
+ <value>Optional principal name in keytab</value>
+ </property>
+
<property>
<name>zk.home</name>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/src/test/clusters/sandbox/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/sandbox/slider/slider-client.xml b/src/test/clusters/sandbox/slider/slider-client.xml
index 5ac5d59..41629ce 100644
--- a/src/test/clusters/sandbox/slider/slider-client.xml
+++ b/src/test/clusters/sandbox/slider/slider-client.xml
@@ -107,6 +107,23 @@
<property>
+ <name>slider.am.login.keytab.name</name>
+ <value>Location of keytab in HDFS</value>
+ </property>
+
+ <property>
+ <name>slider.am.keytab.local.path</name>
+ <description>absolute path to keytab</description>
+ <value></value>
+ </property>
+
+ <property>
+ <name>slider.keytab.principal.name</name>
+ <value>Optional principal name in keytab</value>
+ </property>
+
+
+ <property>
<name>zk.home</name>
<value>/usr/lib/zookeeper</value>
<description>Zookeeper home dir on target systems</description>