You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/01 12:49:23 UTC
[11/14] incubator-slider git commit: SLIDER-953 security auth
failures should return exit code EXIT_UNAUTHORIZED "41"
SLIDER-953 security auth failures should return exit code EXIT_UNAUTHORIZED "41"
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5ff77d06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5ff77d06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5ff77d06
Branch: refs/heads/develop
Commit: 5ff77d06da57981111e097d4891ea59434af2467
Parents: 88a7b34
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 26 16:26:57 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Oct 26 16:26:57 2015 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 32 ++++++++------------
.../apache/slider/common/tools/SliderUtils.java | 9 +++---
.../slideram/SliderAMClientProvider.java | 3 +-
.../server/appmaster/SliderAppMaster.java | 15 ++++-----
.../security/SecurityConfiguration.java | 20 ++++--------
5 files changed, 30 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ff77d06/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 16e5c9a..3404039 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -423,8 +423,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
break;
case ACTION_INSTALL_KEYTAB:
- exitCode =
- actionInstallKeytab(serviceArgs.getActionInstallKeytabArgs());
+ exitCode = actionInstallKeytab(serviceArgs.getActionInstallKeytabArgs());
break;
case ACTION_INSTALL_PACKAGE:
@@ -502,7 +501,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
yarnAppListClient =
new YarnAppListClient(yarnClient, getUsername(), getConfig());
// create the filesystem
- sliderFileSystem = new SliderFileSystem(getConfig());
+ sliderFileSystem = new SliderFileSystem(getConfig());
}
/**
@@ -573,7 +572,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (client != null) {
// set up the permissions. This must be done differently on a secure cluster from an insecure
// one
- List<ACL> zkperms = new ArrayList<ACL>();
+ List<ACL> zkperms = new ArrayList<>();
if (UserGroupInformation.isSecurityEnabled()) {
zkperms.add(new ACL(ZooDefs.Perms.ALL, ZooDefs.Ids.AUTH_IDS));
zkperms.add(new ACL(ZooDefs.Perms.READ, ZooDefs.Ids.ANYONE_ID_UNSAFE));
@@ -1972,13 +1971,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
instanceDefinition.resolve();
launchedInstanceDefinition = instanceDefinition;
- ConfTreeOperations internalOperations =
- instanceDefinition.getInternalOperations();
+ ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations();
MapOperations internalOptions = internalOperations.getGlobalOptions();
- ConfTreeOperations resourceOperations =
- instanceDefinition.getResourceOperations();
- ConfTreeOperations appOperations =
- instanceDefinition.getAppConfOperations();
+ ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations();
+ ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations();
Path generatedConfDirPath =
createPathThatMustExist(internalOptions.getMandatoryOption(
InternalKeys.INTERNAL_GENERATED_CONF_PATH));
@@ -2237,9 +2233,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (clusterSecure) {
// if the cluster is secure, make sure that
// the relevant security settings go over
-/*
- addConfOptionToCLI(commandLine, config, KEY_SECURITY);
-*/
addConfOptionToCLI(commandLine,
config,
DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
@@ -2292,7 +2285,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (ArrayUtils.isEmpty(envs)) {
return Collections.emptyMap();
}
- Map<String, String> amLaunchEnv = new HashMap<String, String>();
+ Map<String, String> amLaunchEnv = new HashMap<>();
for (String env : envs) {
if (StringUtils.isNotEmpty(env)) {
// Each env name/value is separated by equals sign (=)
@@ -2328,7 +2321,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (placeholderMatcher.find()) {
String placeholderKey = placeholderMatcher.group();
String systemKey = placeholderKey
- .substring(2, placeholderKey.length() - 1).toUpperCase()
+ .substring(2, placeholderKey.length() - 1).toUpperCase(Locale.ENGLISH)
.replaceAll("\\.", "_");
String placeholderValue = SliderUtils.getSystemEnv(systemKey);
log.debug("Placeholder {}={}", placeholderKey, placeholderValue);
@@ -2478,8 +2471,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws FileNotFoundException if the path does not exist
*/
public Path createPathThatMustExist(String uri) throws
- SliderException,
- IOException {
+ SliderException, IOException {
return sliderFileSystem.createPathThatMustExist(uri);
}
@@ -2675,9 +2667,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throws IOException, YarnException {
Set<String> appInstances = getApplicationList(clustername, args);
// getApplicationList never returns null
- return appInstances.size() > 0 ? EXIT_SUCCESS
- : (appInstances.size() == 0 && isUnset(clustername)) ? EXIT_SUCCESS
- : EXIT_FALSE;
+ return !appInstances.isEmpty() ? EXIT_SUCCESS
+ : ((appInstances.isEmpty() && isUnset(clustername)) ? EXIT_SUCCESS
+ : EXIT_FALSE);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ff77d06/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index cc19052..0349ebd 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1225,11 +1225,11 @@ public final class SliderUtils {
* @param conf configuration to look at
* @return true if the cluster is secure
* @throws IOException cluster is secure
- * @throws BadConfigException the configuration/process is invalid
+ * @throws SliderException the configuration/process is invalid
*/
public static boolean maybeInitSecurity(Configuration conf) throws
IOException,
- BadConfigException {
+ SliderException {
boolean clusterSecure = isHadoopClusterSecure(conf);
if (clusterSecure) {
log.debug("Enabling security");
@@ -1247,7 +1247,7 @@ public final class SliderUtils {
*/
public static boolean initProcessSecurity(Configuration conf) throws
IOException,
- BadConfigException {
+ SliderException {
if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
//security is already inited
@@ -1273,7 +1273,8 @@ public final class SliderUtils {
log.debug("Authenticating as {}", authUser);
log.debug("Login user is {}", UserGroupInformation.getLoginUser());
if (!UserGroupInformation.isSecurityEnabled()) {
- throw new BadConfigException("Although secure mode is enabled," +
+ throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZE,
+ "Although secure mode is enabled," +
"the application has already set up its user as an insecure entity %s",
authUser);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ff77d06/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index e1dc4f9..9bd4dc9 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -174,8 +174,7 @@ public class SliderAMClientProvider extends AbstractClientProvider
Path tempPath, boolean miniClusterTestRun)
throws IOException, SliderException {
- Map<String, LocalResource> providerResources =
- new HashMap<String, LocalResource>();
+ Map<String, LocalResource> providerResources = new HashMap<>();
ProviderUtils.addProviderJar(providerResources,
this,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ff77d06/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 777fa04..5861256 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -492,7 +492,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// create and register monitoring services
addService(metricsAndMonitoring);
metrics = metricsAndMonitoring.getMetrics();
-/*
+/* TODO: turn these one once the metrics testing is more under control
metrics.registerAll(new ThreadStatesGaugeSet());
metrics.registerAll(new MemoryUsageGaugeSet());
metrics.registerAll(new GarbageCollectorMetricSet());
@@ -678,8 +678,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
appMasterContainerID = ConverterUtils.toContainerId(
SliderUtils.mandatoryEnvVariable(
- ApplicationConstants.Environment.CONTAINER_ID.name())
- );
+ ApplicationConstants.Environment.CONTAINER_ID.name()));
appAttemptID = appMasterContainerID.getApplicationAttemptId();
ApplicationId appid = appAttemptID.getApplicationId();
@@ -792,8 +791,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// build the handler for RM request/release operations; this uses
// the max value as part of its lookup
- rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient,
- maxResources);
+ rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maxResources);
// set the RM-defined maximum cluster values
appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores));
@@ -814,8 +812,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// principal. Can do so now since AM registration with RM above required
// tokens associated to principal
String principal = securityConfiguration.getPrincipal();
- File localKeytabFile =
- securityConfiguration.getKeytabFile(instanceDefinition);
+ File localKeytabFile = securityConfiguration.getKeytabFile(instanceDefinition);
// Now log in...
login(principal, localKeytabFile);
// obtain new FS reference that should be kerberos based and different
@@ -832,8 +829,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
Configuration providerConf =
providerService.loadProviderConfigurationInformation(confDir);
- providerService
- .initializeApplicationConfiguration(instanceDefinition, fs);
+ providerService.initializeApplicationConfiguration(instanceDefinition, fs);
providerService.validateApplicationConfiguration(instanceDefinition,
confDir,
@@ -1080,6 +1076,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
protected void login(String principal, File localKeytabFile)
throws IOException, SliderException {
+ log.info("Logging in as {} with keytab {}", principal, localKeytabFile);
UserGroupInformation.loginUserFromKeytab(principal,
localKeytabFile.getAbsolutePath());
validateLoginUser(UserGroupInformation.getLoginUser());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ff77d06/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
index 4ff6916..a01fb18 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
@@ -17,18 +17,11 @@
package org.apache.slider.server.appmaster.security;
import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.slider.common.SliderExitCodes;
+import static org.apache.slider.core.main.LauncherExitCodes.EXIT_UNAUTHORIZE;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.exceptions.SliderException;
@@ -39,7 +32,7 @@ import java.io.File;
import java.io.IOException;
/**
- *
+ * Class keeping code security information
*/
public class SecurityConfiguration {
@@ -71,7 +64,7 @@ public class SecurityConfiguration {
try {
loginUser = getLoginUser();
} catch (IOException e) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, e,
+ throw new SliderException(EXIT_UNAUTHORIZE, e,
"No principal configured for the application and "
+ "exception raised during retrieval of login user. "
+ "Unable to proceed with application "
@@ -81,7 +74,7 @@ public class SecurityConfiguration {
SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
}
if (loginUser == null) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ throw new SliderException(EXIT_UNAUTHORIZE,
"No principal configured for the application "
+ "and no login user found. "
+ "Unable to proceed with application "
@@ -100,7 +93,7 @@ public class SecurityConfiguration {
.getComponent(SliderKeys.COMPONENT_AM)
.get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ throw new SliderException(EXIT_UNAUTHORIZE,
"Both a keytab on the cluster host (%s) and a"
+ " keytab to be retrieved from HDFS (%s) are"
+ " specified. Please configure only one keytab"
@@ -160,8 +153,7 @@ public class SecurityConfiguration {
// download keytab to local, protected directory
localKeytabFile = new File(SliderKeys.KEYTAB_DIR, keytabName);
} else {
- log.info("Leveraging host keytab file {} for login",
- keytabFullPath);
+ log.info("Using host keytab file {} for login", keytabFullPath);
localKeytabFile = new File(keytabFullPath);
}
return localKeytabFile;