You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2016/02/03 20:41:45 UTC
[1/7] incubator-slider git commit: SLIDER-1077: improve
oozie/credential support. This isn't quite right yet
Repository: incubator-slider
Updated Branches:
refs/heads/develop 8d4da2ddb -> b6b3e6276
SLIDER-1077: improve oozie/credential support. This isn't quite right yet
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5dfac853
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5dfac853
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5dfac853
Branch: refs/heads/develop
Commit: 5dfac853256ef4a3cf8f0a5fec003cdc4848f7f9
Parents: 8d4da2d
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jan 29 20:06:00 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Jan 29 20:06:00 2016 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 21 +-
.../slider/client/SliderYarnClientImpl.java | 2 +-
.../slider/core/launch/AbstractLauncher.java | 53 ++--
.../slider/core/launch/AppMasterLauncher.java | 68 +++--
.../slider/core/launch/ContainerLauncher.java | 8 +
.../slider/core/launch/CredentialUtils.java | 251 +++++++++++++++++++
.../server/appmaster/RoleLaunchService.java | 21 +-
.../server/appmaster/SliderAppMaster.java | 101 ++++----
.../core/launch/TestAppMasterLauncher.java | 6 +-
.../TestAppMasterLauncherWithAmReset.java | 4 +-
10 files changed, 415 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index aa19a3b..5f694e2 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.KerberosDiags;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
@@ -131,6 +132,7 @@ import org.apache.slider.core.exceptions.UsageException;
import org.apache.slider.core.exceptions.WaitTimeoutException;
import org.apache.slider.core.launch.AppMasterLauncher;
import org.apache.slider.core.launch.ClasspathConstructor;
+import org.apache.slider.core.launch.CredentialUtils;
import org.apache.slider.core.launch.JavaCommandLineBuilder;
import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
@@ -1909,6 +1911,22 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
getApplicationDefinitionPath(appOperations));
+
+ Credentials credentials = null;
+ if (clusterSecure) {
+ // pick up oozie credentials
+ credentials = CredentialUtils.loadFromEnvironment(
+ System.getenv(), config);
+ if (credentials == null) {
+ // nothing from oozie, so build up directly
+ credentials = new Credentials(
+ UserGroupInformation.getCurrentUser().getCredentials());
+ CredentialUtils.addRMRenewableFSDelegationTokens(config,
+ sliderFileSystem.getFileSystem(),
+ credentials);
+ }
+ }
+
AppMasterLauncher amLauncher = new AppMasterLauncher(clustername,
SliderKeys.APP_TYPE,
config,
@@ -1917,7 +1935,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
clusterSecure,
sliderAMResourceComponent,
resourceGlobalOptions,
- applicationTags);
+ applicationTags,
+ credentials);
ApplicationId appId = amLauncher.getApplicationId();
// set the application name;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 85a582b..d471cdb 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -190,7 +190,7 @@ public class SliderYarnClientImpl extends YarnClientImpl {
}
/**
- * Force kill a yarn application by ID. No niceities here
+ * Force kill a yarn application by ID. No niceties here
* @param applicationId app Id. "all" means "kill all instances of the current user
*
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 22bf328..f92ffb1 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -24,7 +24,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -68,25 +67,39 @@ public abstract class AbstractLauncher extends Configured {
* Env vars; set up at final launch stage
*/
protected final Map<String, String> envVars = new HashMap<>();
+
protected final MapOperations env = new MapOperations("env", envVars);
protected final ContainerLaunchContext containerLaunchContext =
Records.newRecord(ContainerLaunchContext.class);
protected final List<String> commands = new ArrayList<>(20);
protected final Map<String, LocalResource> localResources = new HashMap<>();
private final Map<String, ByteBuffer> serviceData = new HashMap<>();
+
// security
- protected final Credentials credentials = new Credentials();
+ protected final Credentials credentials;
protected LogAggregationContext logAggregationContext;
+ /**
+ * Create instance
+ * @param conf configuration
+ * @param coreFileSystem filesystem
+ * @param credentials initial set of credentials -null is permitted
+ */
+ protected AbstractLauncher(Configuration conf,
+ CoreFileSystem coreFileSystem,
+ Credentials credentials) {
+ super(conf);
+ this.coreFileSystem = coreFileSystem;
+ this.credentials = credentials != null ? credentials: new Credentials();
+ }
protected AbstractLauncher(Configuration conf,
CoreFileSystem fs) {
- super(conf);
- this.coreFileSystem = fs;
+ this(conf, fs, null);
}
protected AbstractLauncher(CoreFileSystem fs) {
- this.coreFileSystem = fs;
+ this(null, fs, null);
}
/**
@@ -133,7 +146,6 @@ public abstract class AbstractLauncher extends Configured {
localResources.putAll(resourceMap);
}
-
public Map<String, ByteBuffer> getServiceData() {
return serviceData;
}
@@ -169,7 +181,7 @@ public abstract class AbstractLauncher extends Configured {
/**
* Get all commands as a string, separated by ";". This is for diagnostics
- * @return a string descriptionof the commands
+ * @return a string description of the commands
*/
public String getCommandsAsString() {
return SliderUtils.join(getCommands(), "; ");
@@ -194,7 +206,7 @@ public abstract class AbstractLauncher extends Configured {
}
}
containerLaunchContext.setEnvironment(env);
-
+
//service data
if (log.isDebugEnabled()) {
log.debug("Service Data size");
@@ -211,21 +223,8 @@ public abstract class AbstractLauncher extends Configured {
//tokens
log.debug("{} tokens", credentials.numberOfTokens());
- DataOutputBuffer dob = new DataOutputBuffer();
- String tokenFileName =
- this.getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY);
- if (tokenFileName != null) {
- // use delegation tokens, i.e. from Oozie
- Credentials creds =
- Credentials.readTokenStorageFile(new File(tokenFileName), getConf());
- creds.writeTokenStorageToStream(dob);
- } else {
- // normal auth
- credentials.writeTokenStorageToStream(dob);
- }
-
- ByteBuffer tokenBuffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- containerLaunchContext.setTokens(tokenBuffer);
+ containerLaunchContext.setTokens(CredentialUtils.marshallCredentials(
+ credentials));
return containerLaunchContext;
}
@@ -282,7 +281,7 @@ public abstract class AbstractLauncher extends Configured {
/**
* Extract the value for option
- * yarn.resourcemanager.am.retry-count-window-ms
+ * {@code yarn.resourcemanager.am.retry-count-window-ms}
* and set it on the ApplicationSubmissionContext. Use the default value
* if option is not set.
*
@@ -433,7 +432,7 @@ public abstract class AbstractLauncher extends Configured {
public String[] dumpEnvToString() {
- List<String> nodeEnv = new ArrayList<String>();
+ List<String> nodeEnv = new ArrayList<>();
for (Map.Entry<String, String> entry : env.entrySet()) {
String envElt = String.format("%s=\"%s\"",
@@ -453,8 +452,8 @@ public abstract class AbstractLauncher extends Configured {
* @param destRelativeDir relative path under destination local dir
* @throws IOException IO problems
*/
- public void submitDirectory(Path srcDir, String destRelativeDir) throws
- IOException {
+ public void submitDirectory(Path srcDir, String destRelativeDir)
+ throws IOException {
//add the configuration resources
Map<String, LocalResource> confResources;
confResources = coreFileSystem.submitDirectory(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index c82affa..091b80e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -20,7 +20,7 @@ package org.apache.slider.core.launch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
@@ -34,12 +34,10 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.Records;
import org.apache.slider.client.SliderYarnClientImpl;
import org.apache.slider.common.tools.CoreFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.net.InetAddress;
import java.text.DateFormat;
import java.util.Date;
import java.util.Map;
@@ -76,20 +74,21 @@ public class AppMasterLauncher extends AbstractLauncher {
* @param options map of options. All values are extracted in this constructor only
* @param resourceGlobalOptions global options
* @param applicationTags any app tags
+ * @param credentials initial set of credentials
* @throws IOException
* @throws YarnException
*/
public AppMasterLauncher(String name,
- String type,
- Configuration conf,
- CoreFileSystem fs,
- SliderYarnClientImpl yarnClient,
- boolean secureCluster,
- Map<String, String> options,
- Map<String, String> resourceGlobalOptions,
- Set<String> applicationTags
- ) throws IOException, YarnException {
- super(conf, fs);
+ String type,
+ Configuration conf,
+ CoreFileSystem fs,
+ SliderYarnClientImpl yarnClient,
+ boolean secureCluster,
+ Map<String, String> options,
+ Map<String, String> resourceGlobalOptions,
+ Set<String> applicationTags,
+ Credentials credentials) throws IOException, YarnException {
+ super(conf, fs, credentials);
this.yarnClient = yarnClient;
this.application = yarnClient.createApplication();
this.name = name;
@@ -165,10 +164,8 @@ public class AppMasterLauncher extends AbstractLauncher {
* Complete the launch context (copy in env vars, etc).
* @return the container to launch
*/
- public ApplicationSubmissionContext completeAppMasterLaunch() throws
- IOException {
-
-
+ public ApplicationSubmissionContext completeAppMasterLaunch()
+ throws IOException {
//queue priority
Priority pri = Records.newRecord(Priority.class);
@@ -196,6 +193,7 @@ public class AppMasterLauncher extends AbstractLauncher {
}
if (secureCluster) {
+ //tokens
addSecurityTokens();
} else {
propagateUsernameInInsecureCluster();
@@ -211,42 +209,40 @@ public class AppMasterLauncher extends AbstractLauncher {
*/
private void addSecurityTokens() throws IOException {
- String tokenRenewer = SecurityUtil.getServerPrincipal(
- getConf().get(YarnConfiguration.RM_PRINCIPAL),
- InetAddress.getLocalHost().getCanonicalHostName());
- if (SliderUtils.isUnset(tokenRenewer)) {
- throw new IOException(
- "Can't get Master Kerberos principal for the RM to use as renewer: "
- + YarnConfiguration.RM_PRINCIPAL
- );
- }
+ CredentialUtils.addRMRenewableFSDelegationTokens(getConf(),
+ coreFileSystem.getFileSystem(), credentials);
+
+ String tokenRenewer = CredentialUtils.getRMPrincipal(getConf());
Token<? extends TokenIdentifier>[] tokens = null;
- boolean tokensProvided = getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null;
+ boolean tokensProvided = getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) !=
+ null;
if (!tokensProvided) {
- // For now, only getting tokens for the default file-system.
- FileSystem fs = coreFileSystem.getFileSystem();
- tokens = fs.addDelegationTokens(tokenRenewer, credentials);
+ // For now, only getting tokens for the default file-system.
+ FileSystem fs = coreFileSystem.getFileSystem();
+ tokens = fs.addDelegationTokens(tokenRenewer, credentials);
}
// obtain the token expiry from the first token - should be the same for all
// HDFS tokens
if (tokens != null && tokens.length > 0) {
AbstractDelegationTokenIdentifier id =
- (AbstractDelegationTokenIdentifier)tokens[0].decodeIdentifier();
+ (AbstractDelegationTokenIdentifier) tokens[0].decodeIdentifier();
Date d = new Date(id.getIssueDate() + 24 * 60 * 60 * 1000);
- log.info("HDFS delegation tokens for AM launch context require renewal by {}",
- DateFormat.getDateTimeInstance().format(d));
+ log.info(
+ "HDFS delegation tokens for AM launch context require renewal by {}",
+ DateFormat.getDateTimeInstance().format(d));
} else {
if (!tokensProvided) {
log.warn("No HDFS delegation tokens obtained for AM launch context");
} else {
- log.info("Tokens provided via "+ MAPREDUCE_JOB_CREDENTIALS_BINARY +" property "
- + "being used for AM launch");
+ log.info("Tokens provided via " + MAPREDUCE_JOB_CREDENTIALS_BINARY +
+ " property "
+ + "being used for AM launch");
}
}
- }
+ }
/**
* Submit the application.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index 69b937d..e586743 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -20,6 +20,7 @@ package org.apache.slider.core.launch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.Container;
@@ -41,6 +42,13 @@ public class ContainerLauncher extends AbstractLauncher {
public final Container container;
public ContainerLauncher(Configuration conf,
+ CoreFileSystem coreFileSystem,
+ Container container, Credentials credentials) {
+ super(conf, coreFileSystem, credentials);
+ this.container = container;
+ }
+
+ public ContainerLauncher(Configuration conf,
CoreFileSystem fs,
Container container) {
super(conf, fs);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
new file mode 100644
index 0000000..32068e2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.text.DateFormat;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+
+/**
+ * Utils to work with credentials and tokens.
+ *
+ * Designed to be movable to Hadoop core
+ */
+public final class CredentialUtils {
+
+ private CredentialUtils() {
+ }
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CredentialUtils.class);
+
+ /**
+ * Save credentials to a byte buffer. Returns null if there were no
+ * credentials to save
+ * @param credentials credential set
+ * @return a byte buffer of serialized tokens
+ * @throws IOException if the credentials could not be written to the stream
+ */
+ public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException {
+ ByteBuffer buffer = null;
+ if (!credentials.getAllTokens().isEmpty()) {
+ DataOutputBuffer dob = new DataOutputBuffer();
+ credentials.writeTokenStorageToStream(dob);
+ dob.close();
+ buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ }
+ return buffer;
+ }
+
+ /**
+ * Load the credentials from the environment. This looks at
+ * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION}
+ * and attempts to read in the value
+ * @param env environment to resolve the variable from
+ * @param conf configuration use when reading the tokens
+ * @return a set of credentials, or null if the environment did not
+ * specify any
+ * @throws IOException if a location for credentials was defined, but
+ * the credentials could not be loaded.
+ */
+ public static Credentials loadFromEnvironment(Map<String, String> env,
+ Configuration conf)
+ throws IOException {
+ String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
+ if (tokenFilename != null) {
+ // use delegation tokens, i.e. from Oozie
+ File file = new File(tokenFilename.trim());
+ String details = String.format("Token File %s from environment variable %s",
+ file,
+ HADOOP_TOKEN_FILE_LOCATION);
+ LOG.debug("Using {}", details);
+ if (!file.exists()) {
+ throw new FileNotFoundException("No " + details);
+ }
+ if (!file.isFile() && !file.canRead()) {
+ throw new IOException("Cannot read " + details);
+ }
+ Credentials creds = Credentials.readTokenStorageFile(file, conf);
+ return creds;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Look up and return the resource manager's principal. This method
+ * automatically does the <code>_HOST</code> replacement in the principal and
+ * correctly handles HA resource manager configurations.
+ *
+ * From: YARN-4629
+ * @param conf the {@link Configuration} file from which to read the
+ * principal
+ * @return the resource manager's principal string
+ * @throws IOException thrown if there's an error replacing the host name
+ */
+ public static String getRMPrincipal(Configuration conf) throws IOException {
+ String principal = conf.get(RM_PRINCIPAL, "");
+ String hostname;
+ Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);
+
+ if (HAUtil.isHAEnabled(conf)) {
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ if (yarnConf.get(RM_HA_ID) == null) {
+ // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
+ // Any valid RM HA ID should work.
+ String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
+ Preconditions.checkState((rmIds != null) && (rmIds.length > 0),
+ "Not set " + RM_HA_IDS);
+ yarnConf.set(RM_HA_ID, rmIds[0]);
+ }
+
+ hostname = yarnConf.getSocketAddr(
+ RM_ADDRESS,
+ DEFAULT_RM_ADDRESS,
+ DEFAULT_RM_PORT).getHostName();
+ } else {
+ hostname = conf.getSocketAddr(
+ RM_ADDRESS,
+ DEFAULT_RM_ADDRESS,
+ DEFAULT_RM_PORT).getHostName();
+ }
+ return SecurityUtil.getServerPrincipal(principal, hostname);
+ }
+
+ /**
+ * Create and add any filesystem delegation tokens with
+ * the RM(s) configured to be able to renew them. Returns null
+ * on an insecure cluster (i.e. harmless)
+ * @param conf configuration
+ * @param fs filesystem
+ * @param credentials credentials to update
+ * @return a list of all added tokens.
+ * @throws IOException
+ */
+ public static Token<?>[] addRMRenewableFSDelegationTokens(Configuration conf,
+ FileSystem fs,
+ Credentials credentials) throws IOException {
+ Preconditions.checkArgument(conf != null);
+ Preconditions.checkArgument(credentials != null);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ String tokenRenewer = CredentialUtils.getRMPrincipal(conf);
+ Token<? extends TokenIdentifier>[] tokens = null;
+ return fs.addDelegationTokens(tokenRenewer, credentials);
+ }
+ return null;
+ }
+
+ /**
+ * Add an FS delegation token which can be renewed by the current user
+ * @param fs filesystem
+ * @param credentials credentials to update
+ * @throws IOException problems.
+ */
+ public static void addSelfRenewableFSDelegationTokens(
+ FileSystem fs,
+ Credentials credentials) throws IOException {
+ Preconditions.checkArgument(fs != null);
+ Preconditions.checkArgument(credentials != null);
+ fs.addDelegationTokens(
+ UserGroupInformation.getLoginUser().getShortUserName(),
+ credentials);
+ }
+
+ /**
+ * Filter a list of tokens from a set of credentials
+ * @param credentials credential source (a new credential set os re
+ * @param filter List of tokens to strip out
+ * @return a new, filtered, set of credentials
+ */
+ public static Credentials filterTokens(Credentials credentials,
+ List<Text> filter) {
+ Credentials result = new Credentials(credentials);
+ Iterator<Token<? extends TokenIdentifier>> iter =
+ result.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<? extends TokenIdentifier> token = iter.next();
+ LOG.debug("Token {}", token.getKind());
+ if (filter.contains(token.getKind())) {
+ LOG.debug("Filtering token {}", token.getKind());
+ iter.remove();
+ }
+ }
+ return result;
+ }
+
+ public static String dumpTokens(Credentials credentials, String separator) {
+ Collection<Token<? extends TokenIdentifier>> allTokens
+ = credentials.getAllTokens();
+ StringBuilder buffer = new StringBuilder(allTokens.size()* 128);
+ DateFormat df = DateFormat.getDateTimeInstance(
+ DateFormat.SHORT, DateFormat.SHORT);
+ for (Token<? extends TokenIdentifier> token : allTokens) {
+ buffer.append(toString(token)).append(separator);
+ }
+ return buffer.toString();
+ }
+
+ public static String toString(Token<? extends TokenIdentifier> token) {
+ DateFormat df = DateFormat.getDateTimeInstance(
+ DateFormat.SHORT, DateFormat.SHORT);
+ StringBuilder buffer = new StringBuilder(128);
+ buffer.append(token.toString());
+ try {
+ TokenIdentifier ti = token.decodeIdentifier();
+ buffer.append("; ").append(ti);
+ if (ti instanceof AbstractDelegationTokenIdentifier) {
+ AbstractDelegationTokenIdentifier dt
+ = (AbstractDelegationTokenIdentifier) ti;
+ buffer.append(" Issued: ")
+ .append(df.format(new Date(dt.getIssueDate())));
+ buffer.append(" Max Date: ")
+ .append(df.format(new Date(dt.getMaxDate())));
+ }
+ } catch (IOException e) {
+ LOG.debug("Failed to decode {}: {}", token, e, e);
+ }
+ return buffer.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index d4f2fd5..7515c1a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -21,6 +21,7 @@ package org.apache.slider.server.appmaster;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderKeys;
@@ -63,7 +64,7 @@ public class RoleLaunchService
private final QueueAccess actionQueue;
/**
- * Provider bulding up the command
+ * Provider building up the command
*/
private final ProviderService provider;
@@ -122,23 +123,26 @@ public class RoleLaunchService
* @param container container target
* @param role role
* @param clusterSpec cluster spec to use for template
+ * @param credentials credentials to use
*/
public void launchRole(ContainerAssignment assignment,
- AggregateConf clusterSpec) {
+ AggregateConf clusterSpec,
+ Credentials credentials) {
RoleStatus role = assignment.role;
String roleName = role.getName();
// prelaunch safety check
Preconditions.checkArgument(provider.isSupportedRole(roleName));
RoleLaunchService.RoleLauncher launcher =
new RoleLaunchService.RoleLauncher(assignment,
- clusterSpec,
+ clusterSpec,
clusterSpec.getResourceOperations().getOrAddComponent(roleName),
- clusterSpec.getAppConfOperations().getOrAddComponent(roleName));
+ clusterSpec.getAppConfOperations().getOrAddComponent(roleName),
+ credentials);
execute(launcher);
}
/**
- * Thread that runs on the AM to launch a region server.
+ * Thread that runs on the AM to launch a container
*/
private class RoleLauncher implements Runnable {
@@ -150,13 +154,16 @@ public class RoleLaunchService
private final MapOperations appComponent;
private final AggregateConf instanceDefinition;
public final ProviderRole role;
+ private final Credentials credentials;
private Exception raisedException;
public RoleLauncher(ContainerAssignment assignment,
AggregateConf instanceDefinition,
MapOperations resourceComponent,
- MapOperations appComponent) {
+ MapOperations appComponent,
+ Credentials credentials) {
this.assignment = assignment;
+ this.credentials = credentials;
this.container = assignment.container;
RoleStatus roleStatus = assignment.role;
@@ -187,7 +194,7 @@ public class RoleLaunchService
public void run() {
try {
ContainerLauncher containerLauncher =
- new ContainerLauncher(getConfig(), fs, container);
+ new ContainerLauncher(getConfig(), fs, container, credentials);
containerLauncher.setupUGI();
containerLauncher.putEnv(envVars);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 729d46e..82c9fb9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -20,9 +20,6 @@ package org.apache.slider.server.appmaster;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.health.HealthCheckRegistry;
-import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
-import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
-import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
@@ -35,7 +32,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
@@ -110,6 +106,7 @@ import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.launch.CredentialUtils;
import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
@@ -257,7 +254,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public NMClientAsync nmClientAsync;
/**
- * token blob
+ * Credentials for propagating down to launched containers
*/
private Credentials containerCredentials;
@@ -698,7 +695,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
/*
* Extract the container ID. This is then
- * turned into an (incompete) container
+ * turned into an (incomplete) container
*/
appMasterContainerID = ConverterUtils.toContainerId(
SliderUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name()));
@@ -1117,35 +1114,29 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
- private void processAMCredentials(SecurityConfiguration securityConfiguration)
+ /**
+ * Process the initial user to obtain the set of user
+ * supplied credentials (tokens were passed in by client).
+ * Removes the AM/RM token.
+ * If a keytab has been provided, also strip the HDFS delegation token.
+ * @param securityConfig slider security config
+ * @throws IOException
+ */
+ private void processAMCredentials(SecurityConfiguration securityConfig)
throws IOException {
- // process the initial user to obtain the set of user
- // supplied credentials (tokens were passed in by client). Remove AMRM
- // token and HDFS delegation token, the latter because we will provide an
- // up to date token for container launches (getContainerCredentials()).
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- Credentials credentials = currentUser.getCredentials();
- List<Text> filteredTokens = new ArrayList<>();
+
+ List<Text> filteredTokens = new ArrayList<>(2);
filteredTokens.add(AMRMTokenIdentifier.KIND_NAME);
- boolean keytabProvided = securityConfiguration.isKeytabProvided();
+ boolean keytabProvided = securityConfig.isKeytabProvided();
log.info("Slider AM Security Mode: {}", keytabProvided ? "KEYTAB" : "TOKEN");
if (keytabProvided) {
filteredTokens.add(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
}
- Iterator<Token<? extends TokenIdentifier>> iter =
- credentials.getAllTokens().iterator();
- while (iter.hasNext()) {
- Token<? extends TokenIdentifier> token = iter.next();
- log.info("Token {}", token.getKind());
- if (filteredTokens.contains(token.getKind())) {
- log.debug("Filtering token {} from AM tokens", token.getKind());
- iter.remove();
- }
- }
- // at this point this credentials map is probably clear, but leaving this
- // code to allow for future tokens...
- containerCredentials = credentials;
+ containerCredentials = CredentialUtils.filterTokens(
+ UserGroupInformation.getCurrentUser().getCredentials(),
+ filteredTokens);
+ log.info(CredentialUtils.dumpTokens(containerCredentials, "\n"));
}
/**
@@ -1698,9 +1689,18 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//for each assignment: instantiate that role
for (ContainerAssignment assignment : assignments) {
- launchService.launchRole(assignment, getInstanceDefinition());
+ try {
+ launchService.launchRole(assignment, getInstanceDefinition(),
+ buildContainerCredentials());
+ } catch (IOException e) {
+ // Can be caused by failure to renew credentials with the remote
+ // service. If so, don't launch the application. Container is retained,
+ // though YARN will take it away after a timeout.
+ log.error("Failed to build credentials to launch container: {}", e, e);
+
+ }
}
-
+
//for all the operations, exec them
execute(operations);
log.info("Diagnostics: {}", getContainerDiagnosticInfo());
@@ -2211,34 +2211,49 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// add current HDFS delegation token with an up to date token
ByteBuffer tokens = getContainerCredentials();
+ /*
+ ByteBuffer tokens = getContainerCredentials();
+
if (tokens != null) {
ctx.setTokens(tokens);
} else {
log.warn("No delegation tokens obtained and set for launch context");
}
+*/
appState.containerStartSubmitted(container, instance);
nmClientAsync.startContainerAsync(container, ctx);
}
+ /**
+ * Build the credentials needed for containers. This will include
+ * getting new delegation tokens for HDFS if the AM is running
+ * with a keytab.
+ * @return a buffer of credentials
+ * @throws IOException
+ */
private ByteBuffer getContainerCredentials() throws IOException {
// a delegation token can be retrieved from filesystem since
// the login is via a keytab (see above)
+ Credentials credentials = buildContainerCredentials();
+ return CredentialUtils.marshallCredentials(credentials);
+ }
+
+ /**
+ * Build the credentials needed for containers. This will include
+ * getting new delegation tokens for HDFS if the AM is running
+ * with a keytab.
+ * @return a buffer of credentials
+ * @throws IOException
+ */
+
+ private Credentials buildContainerCredentials() throws IOException {
Credentials credentials = new Credentials(containerCredentials);
- ByteBuffer tokens = null;
if (securityConfiguration.isKeytabProvided()) {
- Token<? extends TokenIdentifier>[] hdfsTokens =
- getClusterFS().getFileSystem().addDelegationTokens(
- UserGroupInformation.getLoginUser().getShortUserName(),
- credentials);
+ CredentialUtils.addSelfRenewableFSDelegationTokens(
+ getClusterFS().getFileSystem(),
+ credentials);
}
- if (!credentials.getAllTokens().isEmpty()) {
- DataOutputBuffer dob = new DataOutputBuffer();
- credentials.writeTokenStorageToStream(dob);
- dob.close();
- tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- }
-
- return tokens;
+ return credentials;
}
@Override // NMClientAsync.CallbackHandler
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
index 60af770..eae9658 100644
--- a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
+++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
@@ -88,7 +88,7 @@ public class TestAppMasterLauncher {
EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags);
+ null, mockYarnClient, false, null, options, tags, null);
// Verify the include/exclude patterns
String expectedInclude = "slider*.txt|agent.out";
@@ -109,7 +109,7 @@ public class TestAppMasterLauncher {
EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags);
+ null, mockYarnClient, false, null, options, tags, null);
// Verify the include/exclude patterns
String expectedInclude = isOldApi ? "" : ".*";
@@ -128,7 +128,7 @@ public class TestAppMasterLauncher {
EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags);
+ null, mockYarnClient, false, null, options, tags, null);
// Verify the include/exclude patterns
String expectedInclude = isOldApi ? "" : ".*";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5dfac853/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
index cc64cab..a8f6b26 100644
--- a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
+++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
@@ -64,7 +64,7 @@ public class TestAppMasterLauncherWithAmReset {
EasyMock.replay(mockYarnClient, yarnClientApp);
appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags);
+ null, mockYarnClient, false, null, options, tags, null);
ApplicationSubmissionContext ctx = appMasterLauncher.application
.getApplicationSubmissionContext();
@@ -80,7 +80,7 @@ public class TestAppMasterLauncherWithAmReset {
EasyMock.replay(mockYarnClient, yarnClientApp);
appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags);
+ null, mockYarnClient, false, null, options, tags, null);
ApplicationSubmissionContext ctx = appMasterLauncher.application
.getApplicationSubmissionContext();
[6/7] incubator-slider git commit: SLIDER-1077
Posted by st...@apache.org.
SLIDER-1077
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/031d88a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/031d88a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/031d88a7
Branch: refs/heads/develop
Commit: 031d88a740ec8f461863b91c843144a01764dee1
Parents: 297e931
Author: Steve Loughran <st...@apache.org>
Authored: Wed Feb 3 18:10:01 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Feb 3 18:10:01 2016 +0000
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 36 ++------------------
1 file changed, 3 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/031d88a7/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9a284e3..24c32bb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -20,6 +20,9 @@ package org.apache.slider.server.appmaster;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.health.HealthCheckRegistry;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
@@ -2205,25 +2208,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public void startContainer(Container container,
ContainerLaunchContext ctx,
RoleInstance instance) throws IOException {
- // Set up tokens for the container too. Today, for normal shell commands,
- // the container in distribute-shell doesn't need any tokens. We are
- // populating them mainly for NodeManagers to be able to download any
- // files in the distributed file-system. The tokens are otherwise also
- // useful in cases, for e.g., when one is running a "hadoop dfs" command
- // inside the distributed shell.
-
- // add current HDFS delegation token with an up to date token
- ByteBuffer tokens = getContainerCredentials();
-
- /*
- ByteBuffer tokens = getContainerCredentials();
-
- if (tokens != null) {
- ctx.setTokens(tokens);
- } else {
- log.warn("No delegation tokens obtained and set for launch context");
- }
-*/
appState.containerStartSubmitted(container, instance);
nmClientAsync.startContainerAsync(container, ctx);
}
@@ -2235,20 +2219,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* @return a buffer of credentials
* @throws IOException
*/
- private ByteBuffer getContainerCredentials() throws IOException {
- // a delegation token can be retrieved from filesystem since
- // the login is via a keytab (see above)
- Credentials credentials = buildContainerCredentials();
- return CredentialUtils.marshallCredentials(credentials);
- }
-
- /**
- * Build the credentials needed for containers. This will include
- * getting new delegation tokens for HDFS if the AM is running
- * with a keytab.
- * @return a buffer of credentials
- * @throws IOException
- */
private Credentials buildContainerCredentials() throws IOException {
Credentials credentials = new Credentials(containerCredentials);
[7/7] incubator-slider git commit: Merge branch
'feature/SLIDER-1077-oozie' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-1077-oozie' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b6b3e627
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b6b3e627
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b6b3e627
Branch: refs/heads/develop
Commit: b6b3e627603f01b08de8cd6e458df72b987aac72
Parents: 8d4da2d 031d88a
Author: Steve Loughran <st...@apache.org>
Authored: Wed Feb 3 19:41:16 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Feb 3 19:41:16 2016 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 48 ++-
.../slider/client/SliderYarnClientImpl.java | 2 +-
.../apache/slider/client/TokensOperation.java | 109 ++++++
.../org/apache/slider/common/Constants.java | 2 +-
.../apache/slider/common/SliderXmlConfKeys.java | 2 +-
.../slider/common/params/ActionTokensArgs.java | 78 ++++
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/params/ClientArgs.java | 21 +-
.../slider/common/params/SliderActions.java | 16 +-
.../apache/slider/common/tools/SliderUtils.java | 65 ++--
.../slider/core/launch/AbstractLauncher.java | 68 ++--
.../slider/core/launch/AppMasterLauncher.java | 85 +----
.../slider/core/launch/ContainerLauncher.java | 8 +-
.../slider/core/launch/CredentialUtils.java | 361 +++++++++++++++++++
.../server/appmaster/RoleLaunchService.java | 21 +-
.../server/appmaster/SliderAppMaster.java | 111 +++---
.../client/TestSliderTokensCommand.groovy | 129 +++++++
.../apache/slider/test/SliderTestUtils.groovy | 20 +-
.../core/launch/TestAppMasterLauncher.java | 6 +-
.../TestAppMasterLauncherWithAmReset.java | 4 +-
.../funtest/framework/CommandTestBase.groovy | 4 +
.../funtest/commands/KDiagCommandIT.groovy | 2 +-
22 files changed, 928 insertions(+), 235 deletions(-)
----------------------------------------------------------------------
[5/7] incubator-slider git commit: SLIDER-1081 add a command,
"slider tokens" to save/list tokens to a file
Posted by st...@apache.org.
SLIDER-1081 add a command, "slider tokens" to save/list tokens to a file
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/297e9319
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/297e9319
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/297e9319
Branch: refs/heads/develop
Commit: 297e9319f1058a8a5bf0af6582972b0af404a4fa
Parents: 8004bc8
Author: Steve Loughran <st...@apache.org>
Authored: Wed Feb 3 17:35:05 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Feb 3 18:00:37 2016 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 22 ++-
.../apache/slider/client/TokensOperation.java | 109 ++++++++++++
.../slider/common/params/ActionTokensArgs.java | 78 +++++++++
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/params/ClientArgs.java | 10 ++
.../slider/common/params/SliderActions.java | 16 +-
.../slider/core/launch/CredentialUtils.java | 174 +++++++++++++++----
.../client/TestSliderTokensCommand.groovy | 129 ++++++++++++++
.../apache/slider/test/SliderTestUtils.groovy | 20 ++-
9 files changed, 514 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index c141d25..21a1cb6 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -102,6 +102,7 @@ import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.ActionTokensArgs;
import org.apache.slider.common.params.ActionUpgradeArgs;
import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.params.ClientArgs;
@@ -448,6 +449,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
break;
+ case ACTION_TOKENS:
+ exitCode = actionTokens(serviceArgs.getActionTokenArgs());
+ break;
+
case ACTION_UPDATE:
exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
break;
@@ -1916,7 +1921,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Credentials credentials = null;
if (clusterSecure) {
// pick up oozie credentials
- credentials = CredentialUtils.loadFromEnvironment(System.getenv(), config);
+ credentials = CredentialUtils.loadTokensFromEnvironment(System.getenv(),
+ config);
if (credentials == null) {
// nothing from oozie, so build up directly
credentials = new Credentials(
@@ -4373,6 +4379,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throws IOException, YarnException {
return new SliderApplicationIpcClient(createClusterOperations());
}
+
+ /**
+ * Save/list tokens. This is for testing oozie integration
+ * @param args commands
+ * @return status
+ */
+ private int actionTokens(ActionTokensArgs args)
+ throws IOException, YarnException {
+ return new TokensOperation().actionTokens(args,
+ sliderFileSystem.getFileSystem(),
+ getConfig(),
+ yarnClient);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java b/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
new file mode 100644
index 0000000..9b9c141
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.common.params.ActionTokensArgs;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.NotFoundException;
+import static org.apache.slider.core.launch.CredentialUtils.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+public class TokensOperation {
+
+ private static final Logger log = LoggerFactory.getLogger(TokensOperation.class);
+ public static final String E_INSECURE
+ = "Cluster is not secure -tokens cannot be acquired";
+ public static final String E_MISSING_SOURCE_FILE = "Missing source file: ";
+ public static final String E_NO_KEYTAB = "No keytab: ";
+
+ public int actionTokens(ActionTokensArgs args, FileSystem fs,
+ Configuration conf,
+ YarnClientImpl yarnClient)
+ throws IOException, YarnException {
+ Credentials credentials;
+ String footnote = "";
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+ boolean isSecure = UserGroupInformation.isSecurityEnabled();
+ if (args.keytab != null) {
+ File keytab = args.keytab;
+ if (!keytab.isFile()) {
+ throw new NotFoundException(E_NO_KEYTAB + keytab.getAbsolutePath());
+ }
+ String principal = args.principal;
+ log.info("Logging in as {} from keytab {}", principal, keytab);
+ user = UserGroupInformation.loginUserFromKeytabAndReturnUGI(
+ principal, keytab.getCanonicalPath());
+ }
+ Credentials userCredentials = user.getCredentials();
+ File output = args.output;
+ if (output != null) {
+ if (!isSecure) {
+ throw new BadClusterStateException(E_INSECURE);
+ }
+ credentials = new Credentials(userCredentials);
+ // filesystem
+ addRMRenewableFSDelegationTokens(conf, fs, credentials);
+ addRMDelegationToken(yarnClient, credentials);
+ if (maybeAddTimelineToken(conf, credentials) != null) {
+ log.debug("Added timeline token");
+ }
+ saveTokens(output, credentials);
+ String filename = output.getCanonicalPath();
+ footnote = String.format("%d tokens saved to %s\n" +
+ "To use these in the environment:\n" +
+ "export %s=%s",
+ credentials.numberOfTokens(),
+ filename, UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename);
+ } else if (args.source != null) {
+ File source = args.source;
+ log.info("Reading credentials from file {}", source);
+ if (!source.isFile()) {
+ throw new NotFoundException( E_MISSING_SOURCE_FILE + source.getAbsolutePath());
+ }
+ credentials = Credentials.readTokenStorageFile(args.source, conf);
+ } else {
+ StringBuffer origin = new StringBuffer();
+ File file = locateEnvCredentials(System.getenv(), conf,
+ origin);
+ if (file != null) {
+ log.info("Credential Source {}", origin);
+ } else {
+ log.info("Credential source: logged in user");
+ }
+ credentials = userCredentials;
+ }
+ // list the tokens
+ log.info("\n{}", dumpTokens(credentials, "\n"));
+ if (!footnote.isEmpty()) {
+ log.info(footnote);
+ }
+ return 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java
new file mode 100644
index 0000000..9f93c4e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.UsageException;
+
+import java.io.File;
+
+@Parameters(commandNames = {SliderActions.ACTION_TOKENS},
+ commandDescription = "save tokens to a file or list tokens in a file")
+public class ActionTokensArgs extends AbstractActionArgs {
+
+ public static final String DUPLICATE_ARGS = "Only one of " +
+ ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
+
+ public static final String MISSING_KT_PROVIDER =
+ "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
+ + " must be provided";
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_TOKENS;
+ }
+
+ @Parameter(names = {ARG_OUTPUT},
+ description = "File to write")
+ public File output;
+
+ @Parameter(names = {ARG_SOURCE},
+ description = "source file")
+ public File source;
+
+ @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
+ public File keytab;
+
+ @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
+ public String principal="";
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (output != null && source != null) {
+ throw new BadCommandArgumentsException(DUPLICATE_ARGS);
+ }
+
+ // this is actually a !xor
+ if (keytab != null ^ !principal.isEmpty()) {
+ throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index d133f25..bac20d7 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -113,6 +113,7 @@ public interface Arguments {
String ARG_SERVICETYPE = "--servicetype";
String ARG_SERVICES = "--services";
String ARG_SLIDER = "--slider";
+ String ARG_SOURCE = "--source";
String ARG_STATE = "--state";
String ARG_SYSPROP = "-S";
String ARG_TEMPLATE = "--template";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index b441a2a..0a658ea 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -77,6 +77,7 @@ public class ClientArgs extends CommonArgs {
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
+ private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
private final ActionUpgradeArgs actionUpgradeArgs = new ActionUpgradeArgs();
private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
@@ -117,6 +118,7 @@ public class ClientArgs extends CommonArgs {
actionResolveArgs,
actionStatusArgs,
actionThawArgs,
+ actionTokenArgs,
actionUpdateArgs,
actionUpgradeArgs,
actionVersionArgs
@@ -233,6 +235,10 @@ public class ClientArgs extends CommonArgs {
return actionThawArgs;
}
+ public ActionTokensArgs getActionTokenArgs() {
+ return actionTokenArgs;
+ }
+
/**
* Look at the chosen action and bind it as the core action for the operation.
* @throws SliderException bad argument or similar
@@ -344,6 +350,10 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionStatusArgs);
break;
+ case ACTION_TOKENS:
+ bindCoreAction(actionTokenArgs);
+ break;
+
case ACTION_UPDATE:
bindCoreAction(actionUpdateArgs);
break;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 5849e5e..aab7c98 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -26,7 +26,9 @@ package org.apache.slider.common.params;
public interface SliderActions {
String ACTION_AM_SUICIDE = "am-suicide";
String ACTION_BUILD = "build";
+ String ACTION_CLIENT = "client";
String ACTION_CREATE = "create";
+ String ACTION_DIAGNOSTICS = "diagnostics";
String ACTION_DEPENDENCY = "dependency";
String ACTION_UPDATE = "update";
String ACTION_UPGRADE = "upgrade";
@@ -36,26 +38,26 @@ public interface SliderActions {
String ACTION_FLEX = "flex";
String ACTION_FREEZE = "stop";
String ACTION_HELP = "help";
+ String ACTION_INSTALL_KEYTAB = "install-keytab";
+ String ACTION_INSTALL_PACKAGE = "install-package";
String ACTION_KDIAG = "kdiag";
+ String ACTION_KEYTAB = "keytab";
String ACTION_KILL_CONTAINER = "kill-container";
String ACTION_LIST = "list";
String ACTION_LOOKUP = "lookup";
String ACTION_NODES = "nodes";
+ String ACTION_PACKAGE = "package";
String ACTION_PREFLIGHT = "preflight";
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
String ACTION_RESOLVE = "resolve";
String ACTION_STATUS = "status";
String ACTION_THAW = "start";
+ String ACTION_TOKENS = "tokens";
+
String ACTION_VERSION = "version";
- String ACTION_DIAGNOSTICS = "diagnostics";
- String ACTION_INSTALL_PACKAGE = "install-package";
- String ACTION_PACKAGE = "package";
- String ACTION_INSTALL_KEYTAB = "install-keytab";
- String ACTION_CLIENT = "client";
- String ACTION_KEYTAB = "keytab";
String DESCRIBE_ACTION_AM_SUICIDE =
- "Tell the Slider Application Master to simulate a process failure by terminating itself";
+ "Tell the Slider Application Master to simulate a process failure by terminating itself";
String DESCRIBE_ACTION_BUILD =
"Build a Slider cluster specification, but do not start it";
String DESCRIBE_ACTION_CREATE =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
index 3245c13..0f4f534 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
@@ -29,18 +29,30 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.common.SliderXmlConfKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.text.DateFormat;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
@@ -83,47 +95,74 @@ public final class CredentialUtils {
return buffer;
}
- /**
- * Load the credentials from the environment. This looks at
- * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION}
- * and attempts to read in the value
- * @param env environment to resolve the variable from
- * @param conf configuration use when reading the tokens
- * @return a set of credentials, or null if the environment did not
- * specify any
- * @throws IOException if a location for credentials was defined, but
- * the credentials could not be loaded.
- */
- public static Credentials loadFromEnvironment(Map<String, String> env,
- Configuration conf)
- throws IOException {
+ public static File locateEnvCredentials(Map<String, String> env,
+ Configuration conf,
+ StringBuffer sourceTextOut) throws FileNotFoundException {
String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
- String source = HADOOP_TOKEN_FILE_LOCATION;
+ String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION;
if (tokenFilename == null) {
tokenFilename = conf.get(JOB_CREDENTIALS_BINARY);
- source = "Configuration option " + JOB_CREDENTIALS_BINARY;
+ source = "configuration option " + JOB_CREDENTIALS_BINARY;
}
if (tokenFilename != null) {
// use delegation tokens, i.e. from Oozie
File file = new File(tokenFilename.trim());
- String details = String.format("Token File %s from environment variable %s",
+ String details = String.format(
+ "Token File %s from %s",
file,
source);
- LOG.debug("Using {}", details);
if (!file.exists()) {
throw new FileNotFoundException("No " + details);
}
if (!file.isFile() && !file.canRead()) {
- throw new IOException("Cannot read " + details);
+ throw new FileNotFoundException("Cannot read " + details);
}
- Credentials creds = Credentials.readTokenStorageFile(file, conf);
- return creds;
+ sourceTextOut.append(details);
+ return file;
} else {
return null;
}
}
/**
+ * Load the credentials from the environment. This looks at
+ * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION}
+ * and attempts to read in the value
+ * @param env environment to resolve the variable from
+ * @param conf configuration use when reading the tokens
+ * @return a set of credentials, or null if the environment did not
+ * specify any
+ * @throws IOException if a location for credentials was defined, but
+ * the credentials could not be loaded.
+ */
+ public static Credentials loadTokensFromEnvironment(Map<String, String> env,
+ Configuration conf)
+ throws IOException {
+ StringBuffer origin = new StringBuffer();
+ File file = locateEnvCredentials(env, conf, origin);
+ if (file != null) {
+ LOG.debug("Using {}", origin);
+ return Credentials.readTokenStorageFile(file, conf);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Save credentials to a file
+ * @param file file to save to (will be overwritten)
+ * @param credentials credentials to write
+ * @throws IOException
+ */
+ public static void saveTokens(File file,
+ Credentials credentials) throws IOException {
+ try(DataOutputStream daos = new DataOutputStream(
+ new FileOutputStream(file))) {
+ credentials.writeTokenStorageToStream(daos);
+ }
+ }
+
+ /**
* Look up and return the resource manager's principal. This method
* automatically does the <code>_HOST</code> replacement in the principal and
* correctly handles HA resource manager configurations.
@@ -179,8 +218,8 @@ public final class CredentialUtils {
Preconditions.checkArgument(conf != null);
Preconditions.checkArgument(credentials != null);
if (UserGroupInformation.isSecurityEnabled()) {
- String tokenRenewer = CredentialUtils.getRMPrincipal(conf);
- return fs.addDelegationTokens(tokenRenewer, credentials);
+ return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf),
+ credentials);
}
return null;
}
@@ -197,10 +236,58 @@ public final class CredentialUtils {
Preconditions.checkArgument(fs != null);
Preconditions.checkArgument(credentials != null);
fs.addDelegationTokens(
- UserGroupInformation.getLoginUser().getShortUserName(),
+ getSelfRenewer(),
credentials);
}
+ public static String getSelfRenewer() throws IOException {
+ return UserGroupInformation.getLoginUser().getShortUserName();
+ }
+
+ /**
+ * Create and add an RM delegation token to the credentials
+ * @param yarnClient
+ * @param credentials to add token to
+ * @return the token which was added
+ * @throws IOException
+ * @throws YarnException
+ */
+ public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient,
+ Credentials credentials)
+ throws IOException, YarnException {
+ Configuration conf = yarnClient.getConfig();
+ Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf));
+ Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
+ Token<TokenIdentifier> rmDelegationToken =
+ ConverterUtils.convertFromYarn(
+ yarnClient.getRMDelegationToken(rmPrincipal),
+ rmDTService);
+ credentials.addToken(rmDelegationToken.getService(), rmDelegationToken);
+ return rmDelegationToken;
+ }
+
+ public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(
+ Configuration conf,
+ Credentials credentials)
+ throws IOException, YarnException {
+ if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
+ LOG.debug("Timeline service enabled -fetching token");
+
+ try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
+ timelineClient.init(conf);
+ timelineClient.start();
+ Token<TimelineDelegationTokenIdentifier> token =
+ timelineClient.getDelegationToken(
+ CredentialUtils.getRMPrincipal(conf));
+ credentials.addToken(token.getService(), token);
+ return token;
+ }
+ } else {
+ LOG.debug("Timeline service is disabled");
+ return null;
+ }
+ }
+
/**
* Filter a list of tokens from a set of credentials
* @param credentials credential source (a new credential set os re
@@ -224,18 +311,22 @@ public final class CredentialUtils {
}
public static String dumpTokens(Credentials credentials, String separator) {
- Collection<Token<? extends TokenIdentifier>> allTokens
- = credentials.getAllTokens();
- StringBuilder buffer = new StringBuilder(allTokens.size()* 128);
- DateFormat df = DateFormat.getDateTimeInstance(
- DateFormat.SHORT, DateFormat.SHORT);
- for (Token<? extends TokenIdentifier> token : allTokens) {
- buffer.append(toString(token)).append(separator);
+ ArrayList<Token<? extends TokenIdentifier>> sorted =
+ new ArrayList<>(credentials.getAllTokens());
+ Collections.sort(sorted, new TokenComparator());
+ StringBuilder buffer = new StringBuilder(sorted.size()* 128);
+ for (Token<? extends TokenIdentifier> token : sorted) {
+ buffer.append(tokenToString(token)).append(separator);
}
return buffer.toString();
}
- public static String toString(Token<? extends TokenIdentifier> token) {
+ /**
+ * Create a string for people to look at
+ * @param token token to convert to a string form
+ * @return a printable view of the token
+ */
+ public static String tokenToString(Token<? extends TokenIdentifier> token) {
DateFormat df = DateFormat.getDateTimeInstance(
DateFormat.SHORT, DateFormat.SHORT);
StringBuilder buffer = new StringBuilder(128);
@@ -244,16 +335,27 @@ public final class CredentialUtils {
TokenIdentifier ti = token.decodeIdentifier();
buffer.append("; ").append(ti);
if (ti instanceof AbstractDelegationTokenIdentifier) {
- AbstractDelegationTokenIdentifier dt
- = (AbstractDelegationTokenIdentifier) ti;
- buffer.append(" Issued: ")
+ // details in human readable form, and compensate for information HDFS DT omits
+ AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti;
+ buffer.append("; Renewer: ").append(dt.getRenewer());
+ buffer.append("; Issued: ")
.append(df.format(new Date(dt.getIssueDate())));
- buffer.append(" Max Date: ")
+ buffer.append("; Max Date: ")
.append(df.format(new Date(dt.getMaxDate())));
}
} catch (IOException e) {
+ //marshall problem; not ours
LOG.debug("Failed to decode {}: {}", token, e, e);
}
return buffer.toString();
}
+
+ private static class TokenComparator
+ implements Comparator<Token<? extends TokenIdentifier>>, Serializable {
+ @Override
+ public int compare(Token<? extends TokenIdentifier> left,
+ Token<? extends TokenIdentifier> right) {
+ return left.getKind().toString().compareTo(right.getKind().toString());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy
new file mode 100644
index 0000000..ee70979
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.params.ActionTokensArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.exceptions.BadClusterStateException
+import org.apache.slider.core.exceptions.NotFoundException
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.junit.Test
+
+/**
+ * Test the argument parsing/validation logic
+ */
+@CompileStatic
+@Slf4j
+class TestSliderTokensCommand extends ServiceLauncherBaseTest {
+
+ public static YarnConfiguration config = createTestConfig()
+
+ public static YarnConfiguration createTestConfig() {
+ def configuration = new YarnConfiguration()
+ configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032")
+ return configuration
+ }
+
+ @Test
+ public void testBadSourceArgs() throws Throwable {
+ launchExpectingException(SliderClient,
+ config,
+ ActionTokensArgs.DUPLICATE_ARGS,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_SOURCE, "target/tokens.bin",
+ Arguments.ARG_OUTPUT, "target/tokens.bin",
+ ])
+ }
+
+ @Test
+ public void testKTNoPrincipal() throws Throwable {
+ launchExpectingException(SliderClient,
+ config,
+ ActionTokensArgs.MISSING_KT_PROVIDER,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_KEYTAB, "target/keytab",
+ ])
+ }
+
+ @Test
+ public void testPrincipalNoKT() throws Throwable {
+ launchExpectingException(SliderClient,
+ config,
+ ActionTokensArgs.MISSING_KT_PROVIDER,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_PRINCIPAL, "bob@REALM",
+ ])
+ }
+
+ /**
+ * A missing keytab is an error
+ * @throws Throwable
+ */
+ @Test
+ public void testMissingKT() throws Throwable {
+ def ex = launchExpectingException(SliderClient,
+ config,
+ TokensOperation.E_NO_KEYTAB,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_PRINCIPAL, "bob@REALM",
+ Arguments.ARG_KEYTAB, "target/keytab",
+ ])
+ if (!(ex instanceof NotFoundException)) {
+ throw ex
+ }
+ }
+
+ @Test
+ public void testMissingSourceFile() throws Throwable {
+ def ex = launchExpectingException(SliderClient,
+ config,
+ TokensOperation.E_MISSING_SOURCE_FILE,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_SOURCE, "target/tokens.bin",
+ ])
+ if (!(ex instanceof NotFoundException)) {
+ throw ex
+ }
+ }
+
+ @Test
+ public void testListHarmlessWhenInsecure() throws Throwable {
+ execSliderCommand(0, config, [SliderActions.ACTION_TOKENS])
+ }
+
+ @Test
+ public void testCreateFailsWhenInsecure() throws Throwable {
+ def ex = launchExpectingException(SliderClient,
+ config,
+ TokensOperation.E_INSECURE,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_OUTPUT, "target/tokens.bin",
+ ])
+ if (!(ex instanceof BadClusterStateException)) {
+ throw ex
+ }
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index cb6ce0e..0a3b040 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -990,7 +990,7 @@ class SliderTestUtils extends Assert {
* of return code takes place
* @param conf configuration
* @param args arg list
- * @return the return code
+ * @return the launcher
*/
protected static ServiceLauncher<SliderClient> execSliderCommand(
Configuration conf,
@@ -1005,6 +1005,24 @@ class SliderTestUtils extends Assert {
return serviceLauncher
}
+ /**
+ * Launch a slider command to a given exit code.
+ * Most failures will trigger exceptions; this is for the exit code of the runService()
+ * call.
+ * @param exitCode desired exit code
+ * @param conf configuration
+ * @param args arg list
+ * @return the launcher
+ */
+ protected static ServiceLauncher<SliderClient> execSliderCommand(
+ int exitCode,
+ Configuration conf,
+ List args) {
+ ServiceLauncher<SliderClient> serviceLauncher = execSliderCommand(conf, args)
+ assert exitCode == serviceLauncher.serviceExitCode
+ serviceLauncher
+ }
+
public static ServiceLauncher launch(Class serviceClass,
Configuration conf,
List<Object> args) throws
[4/7] incubator-slider git commit: SLIDER-1077 sort variables in order
Posted by st...@apache.org.
SLIDER-1077 sort variables in order
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8004bc88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8004bc88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8004bc88
Branch: refs/heads/develop
Commit: 8004bc880396752847ef84e25df156be0eaad51d
Parents: f56a931
Author: Steve Loughran <st...@apache.org>
Authored: Tue Feb 2 17:48:28 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Feb 2 17:48:28 2016 +0000
----------------------------------------------------------------------
.../java/org/apache/slider/common/params/ClientArgs.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8004bc88/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index e5e6888..b441a2a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -47,8 +47,14 @@ public class ClientArgs extends CommonArgs {
* KEEP IN ALPHABETICAL ORDER
*/
private AbstractClusterBuildingActionArgs buildingActionArgs;
+
+ // =========================================================
+ // Keep all of these in alphabetical order. Thanks.
+ // =========================================================
+
private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs();
private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
+ private final ActionClientArgs actionClientArgs = new ActionClientArgs();
private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
@@ -58,8 +64,6 @@ public class ClientArgs extends CommonArgs {
private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs();
- private final ActionPackageArgs actionPackageArgs = new ActionPackageArgs();
- private final ActionClientArgs actionClientArgs = new ActionClientArgs();
private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs();
private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
@@ -68,13 +72,14 @@ public class ClientArgs extends CommonArgs {
private final ActionListArgs actionListArgs = new ActionListArgs();
private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs();
private final ActionNodesArgs actionNodesArgs = new ActionNodesArgs();
+ private final ActionPackageArgs actionPackageArgs = new ActionPackageArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
- private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
private final ActionUpgradeArgs actionUpgradeArgs = new ActionUpgradeArgs();
+ private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
public ClientArgs(String[] args) {
super(args);
[2/7] incubator-slider git commit: SLIDER-1080 move SliderUtils to
java 7 language features
Posted by st...@apache.org.
SLIDER-1080 move SliderUtils to java 7 language features
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f78465e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f78465e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f78465e0
Branch: refs/heads/develop
Commit: f78465e0c04d0255bf9360b014923aa2262fc7b6
Parents: 5dfac85
Author: Steve Loughran <st...@apache.org>
Authored: Mon Feb 1 14:08:22 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Feb 1 14:08:22 2016 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 65 +++++++-------------
1 file changed, 22 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f78465e0/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index b114d8a..7277c21 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -335,18 +335,14 @@ public final class SliderUtils {
InetSocketAddress address,
int connectTimeout)
throws IOException {
- Socket socket = null;
- try {
- socket = new Socket();
+ try(Socket socket = new Socket()) {
socket.connect(address, connectTimeout);
} catch (Exception e) {
throw new IOException("Failed to connect to " + name
+ " at " + address
- + " after " + connectTimeout + "millisconds"
+ + " after " + connectTimeout + "milliseconds"
+ ": " + e,
e);
- } finally {
- IOUtils.closeSocket(socket);
}
}
@@ -516,7 +512,7 @@ public final class SliderUtils {
* @return a stringified list
*/
public static List<String> collectionToStringList(Collection c) {
- List<String> l = new ArrayList<String>(c.size());
+ List<String> l = new ArrayList<>(c.size());
for (Object o : c) {
l.add(o.toString());
}
@@ -788,9 +784,9 @@ public final class SliderUtils {
return;
}
List<ApplicationReport> nonLiveInstance =
- new ArrayList<ApplicationReport>(instances.size());
+ new ArrayList<>(instances.size());
List<ApplicationReport> liveInstance =
- new ArrayList<ApplicationReport>(instances.size());
+ new ArrayList<>(instances.size());
for (ApplicationReport report : instances) {
if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
@@ -826,7 +822,7 @@ public final class SliderUtils {
public static Map<String, ApplicationReport> buildApplicationReportMap(
List<ApplicationReport> instances,
YarnApplicationState minState, YarnApplicationState maxState) {
- TreeMap<String, ApplicationReport> map = new TreeMap<String, ApplicationReport>();
+ TreeMap<String, ApplicationReport> map = new TreeMap<>();
for (ApplicationReport report : instances) {
YarnApplicationState state = report.getYarnApplicationState();
if (state.ordinal() >= minState.ordinal() &&
@@ -844,7 +840,7 @@ public final class SliderUtils {
* @return a map whose iterator returns the string-sorted ordering of entries
*/
public static Map<String, String> sortedMap(Map<String, String> source) {
- Map<String, String> out = new TreeMap<String, String>(source);
+ Map<String, String> out = new TreeMap<>(source);
return out;
}
@@ -854,7 +850,7 @@ public final class SliderUtils {
* @return a string map
*/
public static Map<String, String> toMap(Properties properties) {
- Map<String, String> out = new HashMap<String, String>(properties.size());
+ Map<String, String> out = new HashMap<>(properties.size());
for (Map.Entry<Object, Object> entry : properties.entrySet()) {
out.put(entry.getKey().toString(), entry.getValue().toString());
}
@@ -1136,7 +1132,7 @@ public final class SliderUtils {
* @return a possibly empty map of environment variables.
*/
public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
- Map<String, String> env = new HashMap<String, String>();
+ Map<String, String> env = new HashMap<>();
if (roleOpts != null) {
for (Map.Entry<String, String> entry : roleOpts.entrySet()) {
String key = entry.getKey();
@@ -1470,7 +1466,7 @@ public final class SliderUtils {
/**
* Convert a char sequence to a string.
- * This ensures that comparisions work
+ * This ensures that comparisons work
* @param charSequence source
* @return the string equivalent
*/
@@ -1870,30 +1866,20 @@ public final class SliderUtils {
List<String> files = new ArrayList<>();
generateFileList(files, srcFolder, srcFolder, true, filter);
- TarArchiveOutputStream taos = null;
- try {
- taos = new TarArchiveOutputStream(new GZIPOutputStream(
- new BufferedOutputStream(new FileOutputStream(tarGzipFile))));
+ try(TarArchiveOutputStream taos =
+ new TarArchiveOutputStream(new GZIPOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
for (String file : files) {
File srcFile = new File(srcFolder, file);
TarArchiveEntry tarEntry = new TarArchiveEntry(
srcFile, file);
taos.putArchiveEntry(tarEntry);
- FileInputStream in = new FileInputStream(srcFile);
- try {
+ try(FileInputStream in = new FileInputStream(srcFile)) {
org.apache.commons.io.IOUtils.copy(in, taos);
- } finally {
- if (in != null) {
- in.close();
- }
}
taos.flush();
taos.closeArchiveEntry();
}
- } finally {
- if (taos != null) {
- taos.close();
- }
}
}
@@ -1912,7 +1898,7 @@ public final class SliderUtils {
* @return true if this is invoked in an HDP cluster or false otherwise
*/
public static boolean isHdp() {
- return StringUtils.isNotEmpty(getHdpVersion()) ? true : false;
+ return StringUtils.isNotEmpty(getHdpVersion());
}
/**
@@ -1977,9 +1963,7 @@ public final class SliderUtils {
String entry)
throws IOException {
InputStream is = null;
- FSDataInputStream appStream = null;
- try {
- appStream = fs.open(appPath);
+ try(FSDataInputStream appStream = fs.open(appPath)) {
ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
ZipArchiveEntry zipEntry;
boolean done = false;
@@ -2010,8 +1994,6 @@ public final class SliderUtils {
done = true;
}
}
- } finally {
- IOUtils.closeStream(appStream);
}
return is;
@@ -2162,11 +2144,8 @@ public final class SliderUtils {
parentDir.mkdirs();
}
SliderUtils.verifyIsDir(parentDir, log);
- FileOutputStream out = new FileOutputStream(outfile);
- try {
+ try(FileOutputStream out = new FileOutputStream(outfile)) {
out.write(data);
- } finally {
- IOUtils.closeStream(out);
}
}
@@ -2372,11 +2351,11 @@ public final class SliderUtils {
public static void validateHDFSFile(SliderFileSystem sliderFileSystem,
String pathStr)
throws IOException, URISyntaxException {
- URI pathURI = new URI(pathStr);
- InputStream inputStream =
- sliderFileSystem.getFileSystem().open(new Path(pathURI));
- if (inputStream == null) {
- throw new IOException("HDFS file " + pathStr + " can't be opened");
+ try(InputStream inputStream =
+ sliderFileSystem.getFileSystem().open(new Path(new URI(pathStr)))) {
+ if (inputStream == null) {
+ throw new IOException("HDFS file " + pathStr + " can't be opened");
+ }
}
}
[3/7] incubator-slider git commit: SLIDER-1080 switch to
$HADOOP_TOKEN_FILE_LOCATION as origin of oozie tokens;
fall back to Conf.get("mapreduce.job.credentials.binary"). This is done in
CredentialUtils and not in Launcher code. Also, HADOOP_JAAS_DEBUG i
Posted by st...@apache.org.
SLIDER-1080 switch to $HADOOP_TOKEN_FILE_LOCATION as origin of oozie tokens; fall back to Conf.get("mapreduce.job.credentials.binary"). This is done in CredentialUtils and not in Launcher code. Also, HADOOP_JAAS_DEBUG is auto-propagated from client to AM, then AM to containers. If you want to debug Hadoop and Kerberos, you want to debug it everywhere.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f56a9311
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f56a9311
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f56a9311
Branch: refs/heads/develop
Commit: f56a9311cebaa088762a6d973582b89b425e7936
Parents: f78465e
Author: Steve Loughran <st...@apache.org>
Authored: Mon Feb 1 16:21:20 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Feb 1 16:21:20 2016 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 11 +++--
.../org/apache/slider/common/Constants.java | 2 +-
.../apache/slider/common/SliderXmlConfKeys.java | 2 +-
.../slider/core/launch/AbstractLauncher.java | 31 +++++++-----
.../slider/core/launch/AppMasterLauncher.java | 51 ++------------------
.../slider/core/launch/ContainerLauncher.java | 10 +---
.../slider/core/launch/CredentialUtils.java | 12 ++++-
.../server/appmaster/SliderAppMaster.java | 10 ++--
.../funtest/framework/CommandTestBase.groovy | 4 ++
.../funtest/commands/KDiagCommandIT.groovy | 2 +-
10 files changed, 55 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 5f694e2..c141d25 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -208,6 +208,7 @@ import static org.apache.hadoop.registry.client.binding.RegistryUtils.*;
import static org.apache.slider.api.InternalKeys.*;
import static org.apache.slider.api.OptionKeys.*;
import static org.apache.slider.api.ResourceKeys.*;
+import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
import static org.apache.slider.common.params.SliderActions.*;
import static org.apache.slider.common.tools.SliderUtils.*;
@@ -1915,8 +1916,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Credentials credentials = null;
if (clusterSecure) {
// pick up oozie credentials
- credentials = CredentialUtils.loadFromEnvironment(
- System.getenv(), config);
+ credentials = CredentialUtils.loadFromEnvironment(System.getenv(), config);
if (credentials == null) {
// nothing from oozie, so build up directly
credentials = new Credentials(
@@ -1924,6 +1924,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
CredentialUtils.addRMRenewableFSDelegationTokens(config,
sliderFileSystem.getFileSystem(),
credentials);
+
+ } else {
+ log.info("Using externally supplied credentials to launch AM");
}
}
@@ -2087,8 +2090,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
amLauncher.setEnv("LANG", "en_US.UTF-8");
amLauncher.setEnv("LC_ALL", "en_US.UTF-8");
amLauncher.setEnv("LANGUAGE", "en_US.UTF-8");
+ amLauncher.maybeSetEnv(HADOOP_JAAS_DEBUG,
+ System.getenv(HADOOP_JAAS_DEBUG));
amLauncher.putEnv(getAmLaunchEnv(config));
-
+
for (Map.Entry<String, String> envs : getSystemEnv().entrySet()) {
log.debug("System env {}={}", envs.getKey(), envs.getValue());
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/common/Constants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/Constants.java b/slider-core/src/main/java/org/apache/slider/common/Constants.java
index fdb3452..0e3559a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/Constants.java
+++ b/slider-core/src/main/java/org/apache/slider/common/Constants.java
@@ -22,7 +22,7 @@ public class Constants {
public static final int CONNECT_TIMEOUT = 10000;
public static final int RPC_TIMEOUT = 15000;
- public static final String ENV_JAAS_DEBUG = "HADOOP_JAAS_DEBUG";
+ public static final String HADOOP_JAAS_DEBUG = "HADOOP_JAAS_DEBUG";
public static final String KRB5_CCNAME = "KRB5CCNAME";
public static final String JAVA_SECURITY_KRB5_CONF
= "java.security.krb5.conf";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 26109a7..72dd44f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -202,5 +202,5 @@ public interface SliderXmlConfKeys {
String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file";
String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";
-
+ String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index f92ffb1..2ec3cda 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -58,7 +58,6 @@ public abstract class AbstractLauncher extends Configured {
private static final Logger log =
LoggerFactory.getLogger(AbstractLauncher.class);
public static final String CLASSPATH = "CLASSPATH";
- public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
/**
* Filesystem to use for the launch
*/
@@ -67,20 +66,18 @@ public abstract class AbstractLauncher extends Configured {
* Env vars; set up at final launch stage
*/
protected final Map<String, String> envVars = new HashMap<>();
-
protected final MapOperations env = new MapOperations("env", envVars);
protected final ContainerLaunchContext containerLaunchContext =
Records.newRecord(ContainerLaunchContext.class);
protected final List<String> commands = new ArrayList<>(20);
protected final Map<String, LocalResource> localResources = new HashMap<>();
private final Map<String, ByteBuffer> serviceData = new HashMap<>();
-
// security
protected final Credentials credentials;
protected LogAggregationContext logAggregationContext;
/**
- * Create instance
+ * Create instance.
* @param conf configuration
* @param coreFileSystem filesystem
* @param credentials initial set of credentials -null is permitted
@@ -93,15 +90,6 @@ public abstract class AbstractLauncher extends Configured {
this.credentials = credentials != null ? credentials: new Credentials();
}
- protected AbstractLauncher(Configuration conf,
- CoreFileSystem fs) {
- this(conf, fs, null);
- }
-
- protected AbstractLauncher(CoreFileSystem fs) {
- this(null, fs, null);
- }
-
/**
* Get the container. Until "completed", this isn't valid to launch.
* @return the container to launch
@@ -401,12 +389,29 @@ public abstract class AbstractLauncher extends Configured {
public void setClasspath(ClasspathConstructor classpath) {
setEnv(CLASSPATH, classpath.buildClasspath());
}
+
+ /**
+ * Set an environment variable in the launch context
+ * @param var variable name
+ * @param value value (must be non null)
+ */
public void setEnv(String var, String value) {
Preconditions.checkArgument(var != null, "null variable name");
Preconditions.checkArgument(value != null, "null value");
env.put(var, value);
}
+ /**
+ * Set an environment variable if its value is non-null.
+ * @param var variable name
+ * @param value value (may be null)
+ */
+ public void maybeSetEnv(String var, String value) {
+ if (value != null) {
+ setEnv(var, value);
+ }
+ }
+
public void putEnv(Map<String, String> map) {
env.putAll(map);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index 091b80e..7190c3a 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -19,11 +19,7 @@
package org.apache.slider.core.launch;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -38,8 +34,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.text.DateFormat;
-import java.util.Date;
import java.util.Map;
import java.util.Set;
@@ -194,7 +188,9 @@ public class AppMasterLauncher extends AbstractLauncher {
if (secureCluster) {
//tokens
- addSecurityTokens();
+ log.debug("Credentials: {}",
+ CredentialUtils.dumpTokens(getCredentials(), "\n"));
+
} else {
propagateUsernameInInsecureCluster();
}
@@ -204,47 +200,6 @@ public class AppMasterLauncher extends AbstractLauncher {
}
/**
- * Add the security tokens if this is a secure cluster
- * @throws IOException
- */
- private void addSecurityTokens() throws IOException {
-
- CredentialUtils.addRMRenewableFSDelegationTokens(getConf(),
- coreFileSystem.getFileSystem(), credentials);
-
- String tokenRenewer = CredentialUtils.getRMPrincipal(getConf());
-
- Token<? extends TokenIdentifier>[] tokens = null;
- boolean tokensProvided = getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) !=
- null;
- if (!tokensProvided) {
- // For now, only getting tokens for the default file-system.
- FileSystem fs = coreFileSystem.getFileSystem();
- tokens = fs.addDelegationTokens(tokenRenewer, credentials);
- }
- // obtain the token expiry from the first token - should be the same for all
- // HDFS tokens
- if (tokens != null && tokens.length > 0) {
- AbstractDelegationTokenIdentifier id =
- (AbstractDelegationTokenIdentifier) tokens[0].decodeIdentifier();
- Date d = new Date(id.getIssueDate() + 24 * 60 * 60 * 1000);
- log.info(
- "HDFS delegation tokens for AM launch context require renewal by {}",
- DateFormat.getDateTimeInstance().format(d));
- } else {
- if (!tokensProvided) {
- log.warn("No HDFS delegation tokens obtained for AM launch context");
- } else {
- log.info("Tokens provided via " + MAPREDUCE_JOB_CREDENTIALS_BINARY +
- " property "
- + "being used for AM launch");
- }
-
- }
-
- }
-
- /**
* Submit the application.
* @return a launched application representing the submitted application
* @throws IOException
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index e586743..d220928 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -43,18 +43,12 @@ public class ContainerLauncher extends AbstractLauncher {
public ContainerLauncher(Configuration conf,
CoreFileSystem coreFileSystem,
- Container container, Credentials credentials) {
+ Container container,
+ Credentials credentials) {
super(conf, coreFileSystem, credentials);
this.container = container;
}
- public ContainerLauncher(Configuration conf,
- CoreFileSystem fs,
- Container container) {
- super(conf, fs);
- this.container = container;
- }
-
/**
* This code is in the dist shell examples -it's been moved here
* so that if it is needed, it's still here
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
index 32068e2..3245c13 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,6 +56,9 @@ import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
*/
public final class CredentialUtils {
+ public static final String JOB_CREDENTIALS_BINARY
+ = SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY;
+
private CredentialUtils() {
}
@@ -94,12 +98,17 @@ public final class CredentialUtils {
Configuration conf)
throws IOException {
String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
+ String source = HADOOP_TOKEN_FILE_LOCATION;
+ if (tokenFilename == null) {
+ tokenFilename = conf.get(JOB_CREDENTIALS_BINARY);
+ source = "Configuration option " + JOB_CREDENTIALS_BINARY;
+ }
if (tokenFilename != null) {
// use delegation tokens, i.e. from Oozie
File file = new File(tokenFilename.trim());
String details = String.format("Token File %s from environment variable %s",
file,
- HADOOP_TOKEN_FILE_LOCATION);
+ source);
LOG.debug("Using {}", details);
if (!file.exists()) {
throw new FileNotFoundException("No " + details);
@@ -171,7 +180,6 @@ public final class CredentialUtils {
Preconditions.checkArgument(credentials != null);
if (UserGroupInformation.isSecurityEnabled()) {
String tokenRenewer = CredentialUtils.getRMPrincipal(conf);
- Token<? extends TokenIdentifier>[] tokens = null;
return fs.addDelegationTokens(tokenRenewer, credentials);
}
return null;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 82c9fb9..9a284e3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
+
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -775,7 +777,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
uploadServerCertForLocalization(clustername, fs);
}
-
webAppPort = getPortToRequest();
if (webAppPort == 0) {
// failure to find a port
@@ -914,6 +915,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
if (hadoop_user_name != null) {
envVars.put(HADOOP_USER_NAME, hadoop_user_name);
}
+ String debug_kerberos = System.getenv(HADOOP_JAAS_DEBUG);
+ if (debug_kerberos != null) {
+ envVars.put(HADOOP_JAAS_DEBUG, debug_kerberos);
+ }
}
String rolesTmpSubdir = appMasterContainerID.toString() + "/roles";
@@ -1208,8 +1213,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.error("User is not holding on a keytab in a secure deployment:" +
" slider will fail as tokens expire");
}
- Credentials credentials =
- user.getCredentials();
+ Credentials credentials = user.getCredentials();
Iterator<Token<? extends TokenIdentifier>> iter =
credentials.getAllTokens().iterator();
while (iter.hasNext()) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index a887271..1a0d894 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.StatusKeys
import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.Constants
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
@@ -760,6 +761,9 @@ abstract class CommandTestBase extends SliderTestUtils {
maybeAddCommandOption(commands,
[ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL],
SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL));
+
+ commands << ARG_COMP_OPT << SliderKeys.COMPONENT_AM <<
+ "env." + Constants.HADOOP_JAAS_DEBUG << "true";
commands.addAll(extraArgs)
SliderShell shell = new SliderShell(commands)
if (0 != shell.execute()) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy
index ebdd00c..5fd46fa 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy
@@ -37,7 +37,7 @@ public class KDiagCommandIT extends CommandTestBase implements Arguments {
SliderActions.ACTION_KDIAG,
ARG_KEYLEN, "128"
],
- [(ENV_JAAS_DEBUG): "true"]
+ [(HADOOP_JAAS_DEBUG): "true"]
)
shell.execute()
assertSuccess(shell)