You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2016/02/03 20:41:34 UTC
[2/3] incubator-slider git commit: SLIDER-1081 add a command,
"slider tokens" to save/list tokens to a file
SLIDER-1081 add a command, "slider tokens" to save/list tokens to a file
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/297e9319
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/297e9319
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/297e9319
Branch: refs/heads/feature/SLIDER-1077-oozie
Commit: 297e9319f1058a8a5bf0af6582972b0af404a4fa
Parents: 8004bc8
Author: Steve Loughran <st...@apache.org>
Authored: Wed Feb 3 17:35:05 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Feb 3 18:00:37 2016 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 22 ++-
.../apache/slider/client/TokensOperation.java | 109 ++++++++++++
.../slider/common/params/ActionTokensArgs.java | 78 +++++++++
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/params/ClientArgs.java | 10 ++
.../slider/common/params/SliderActions.java | 16 +-
.../slider/core/launch/CredentialUtils.java | 174 +++++++++++++++----
.../client/TestSliderTokensCommand.groovy | 129 ++++++++++++++
.../apache/slider/test/SliderTestUtils.groovy | 20 ++-
9 files changed, 514 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index c141d25..21a1cb6 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -102,6 +102,7 @@ import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.ActionTokensArgs;
import org.apache.slider.common.params.ActionUpgradeArgs;
import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.params.ClientArgs;
@@ -448,6 +449,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
break;
+ case ACTION_TOKENS:
+ exitCode = actionTokens(serviceArgs.getActionTokenArgs());
+ break;
+
case ACTION_UPDATE:
exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
break;
@@ -1916,7 +1921,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Credentials credentials = null;
if (clusterSecure) {
// pick up oozie credentials
- credentials = CredentialUtils.loadFromEnvironment(System.getenv(), config);
+ credentials = CredentialUtils.loadTokensFromEnvironment(System.getenv(),
+ config);
if (credentials == null) {
// nothing from oozie, so build up directly
credentials = new Credentials(
@@ -4373,6 +4379,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throws IOException, YarnException {
return new SliderApplicationIpcClient(createClusterOperations());
}
+
+ /**
+ * Save/list tokens. This is for testing oozie integration
+ * @param args commands
+ * @return status
+ */
+ private int actionTokens(ActionTokensArgs args)
+ throws IOException, YarnException {
+ return new TokensOperation().actionTokens(args,
+ sliderFileSystem.getFileSystem(),
+ getConfig(),
+ yarnClient);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java b/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
new file mode 100644
index 0000000..9b9c141
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.common.params.ActionTokensArgs;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.NotFoundException;
+import static org.apache.slider.core.launch.CredentialUtils.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+public class TokensOperation {
+
+ private static final Logger log = LoggerFactory.getLogger(TokensOperation.class);
+ public static final String E_INSECURE
+ = "Cluster is not secure -tokens cannot be acquired";
+ public static final String E_MISSING_SOURCE_FILE = "Missing source file: ";
+ public static final String E_NO_KEYTAB = "No keytab: ";
+
+ public int actionTokens(ActionTokensArgs args, FileSystem fs,
+ Configuration conf,
+ YarnClientImpl yarnClient)
+ throws IOException, YarnException {
+ Credentials credentials;
+ String footnote = "";
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+ boolean isSecure = UserGroupInformation.isSecurityEnabled();
+ if (args.keytab != null) {
+ File keytab = args.keytab;
+ if (!keytab.isFile()) {
+ throw new NotFoundException(E_NO_KEYTAB + keytab.getAbsolutePath());
+ }
+ String principal = args.principal;
+ log.info("Logging in as {} from keytab {}", principal, keytab);
+ user = UserGroupInformation.loginUserFromKeytabAndReturnUGI(
+ principal, keytab.getCanonicalPath());
+ }
+ Credentials userCredentials = user.getCredentials();
+ File output = args.output;
+ if (output != null) {
+ if (!isSecure) {
+ throw new BadClusterStateException(E_INSECURE);
+ }
+ credentials = new Credentials(userCredentials);
+ // filesystem
+ addRMRenewableFSDelegationTokens(conf, fs, credentials);
+ addRMDelegationToken(yarnClient, credentials);
+ if (maybeAddTimelineToken(conf, credentials) != null) {
+ log.debug("Added timeline token");
+ }
+ saveTokens(output, credentials);
+ String filename = output.getCanonicalPath();
+ footnote = String.format("%d tokens saved to %s\n" +
+ "To use these in the environment:\n" +
+ "export %s=%s",
+ credentials.numberOfTokens(),
+ filename, UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename);
+ } else if (args.source != null) {
+ File source = args.source;
+ log.info("Reading credentials from file {}", source);
+ if (!source.isFile()) {
+ throw new NotFoundException( E_MISSING_SOURCE_FILE + source.getAbsolutePath());
+ }
+ credentials = Credentials.readTokenStorageFile(args.source, conf);
+ } else {
+ StringBuffer origin = new StringBuffer();
+ File file = locateEnvCredentials(System.getenv(), conf,
+ origin);
+ if (file != null) {
+ log.info("Credential Source {}", origin);
+ } else {
+ log.info("Credential source: logged in user");
+ }
+ credentials = userCredentials;
+ }
+ // list the tokens
+ log.info("\n{}", dumpTokens(credentials, "\n"));
+ if (!footnote.isEmpty()) {
+ log.info(footnote);
+ }
+ return 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java
new file mode 100644
index 0000000..9f93c4e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.UsageException;
+
+import java.io.File;
+
+@Parameters(commandNames = {SliderActions.ACTION_TOKENS},
+ commandDescription = "save tokens to a file or list tokens in a file")
+public class ActionTokensArgs extends AbstractActionArgs {
+
+ public static final String DUPLICATE_ARGS = "Only one of " +
+ ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
+
+ public static final String MISSING_KT_PROVIDER =
+ "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
+ + " must be provided";
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_TOKENS;
+ }
+
+ @Parameter(names = {ARG_OUTPUT},
+ description = "File to write")
+ public File output;
+
+ @Parameter(names = {ARG_SOURCE},
+ description = "source file")
+ public File source;
+
+ @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
+ public File keytab;
+
+ @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
+ public String principal="";
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (output != null && source != null) {
+ throw new BadCommandArgumentsException(DUPLICATE_ARGS);
+ }
+
+ // this is actually a !xor
+ if (keytab != null ^ !principal.isEmpty()) {
+ throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index d133f25..bac20d7 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -113,6 +113,7 @@ public interface Arguments {
String ARG_SERVICETYPE = "--servicetype";
String ARG_SERVICES = "--services";
String ARG_SLIDER = "--slider";
+ String ARG_SOURCE = "--source";
String ARG_STATE = "--state";
String ARG_SYSPROP = "-S";
String ARG_TEMPLATE = "--template";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index b441a2a..0a658ea 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -77,6 +77,7 @@ public class ClientArgs extends CommonArgs {
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
+ private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
private final ActionUpgradeArgs actionUpgradeArgs = new ActionUpgradeArgs();
private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
@@ -117,6 +118,7 @@ public class ClientArgs extends CommonArgs {
actionResolveArgs,
actionStatusArgs,
actionThawArgs,
+ actionTokenArgs,
actionUpdateArgs,
actionUpgradeArgs,
actionVersionArgs
@@ -233,6 +235,10 @@ public class ClientArgs extends CommonArgs {
return actionThawArgs;
}
+ public ActionTokensArgs getActionTokenArgs() {
+ return actionTokenArgs;
+ }
+
/**
* Look at the chosen action and bind it as the core action for the operation.
* @throws SliderException bad argument or similar
@@ -344,6 +350,10 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionStatusArgs);
break;
+ case ACTION_TOKENS:
+ bindCoreAction(actionTokenArgs);
+ break;
+
case ACTION_UPDATE:
bindCoreAction(actionUpdateArgs);
break;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 5849e5e..aab7c98 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -26,7 +26,9 @@ package org.apache.slider.common.params;
public interface SliderActions {
String ACTION_AM_SUICIDE = "am-suicide";
String ACTION_BUILD = "build";
+ String ACTION_CLIENT = "client";
String ACTION_CREATE = "create";
+ String ACTION_DIAGNOSTICS = "diagnostics";
String ACTION_DEPENDENCY = "dependency";
String ACTION_UPDATE = "update";
String ACTION_UPGRADE = "upgrade";
@@ -36,26 +38,26 @@ public interface SliderActions {
String ACTION_FLEX = "flex";
String ACTION_FREEZE = "stop";
String ACTION_HELP = "help";
+ String ACTION_INSTALL_KEYTAB = "install-keytab";
+ String ACTION_INSTALL_PACKAGE = "install-package";
String ACTION_KDIAG = "kdiag";
+ String ACTION_KEYTAB = "keytab";
String ACTION_KILL_CONTAINER = "kill-container";
String ACTION_LIST = "list";
String ACTION_LOOKUP = "lookup";
String ACTION_NODES = "nodes";
+ String ACTION_PACKAGE = "package";
String ACTION_PREFLIGHT = "preflight";
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
String ACTION_RESOLVE = "resolve";
String ACTION_STATUS = "status";
String ACTION_THAW = "start";
+ String ACTION_TOKENS = "tokens";
+
String ACTION_VERSION = "version";
- String ACTION_DIAGNOSTICS = "diagnostics";
- String ACTION_INSTALL_PACKAGE = "install-package";
- String ACTION_PACKAGE = "package";
- String ACTION_INSTALL_KEYTAB = "install-keytab";
- String ACTION_CLIENT = "client";
- String ACTION_KEYTAB = "keytab";
String DESCRIBE_ACTION_AM_SUICIDE =
- "Tell the Slider Application Master to simulate a process failure by terminating itself";
+ "Tell the Slider Application Master to simulate a process failure by terminating itself";
String DESCRIBE_ACTION_BUILD =
"Build a Slider cluster specification, but do not start it";
String DESCRIBE_ACTION_CREATE =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
index 3245c13..0f4f534 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
@@ -29,18 +29,30 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.common.SliderXmlConfKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.text.DateFormat;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
@@ -83,47 +95,74 @@ public final class CredentialUtils {
return buffer;
}
- /**
- * Load the credentials from the environment. This looks at
- * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION}
- * and attempts to read in the value
- * @param env environment to resolve the variable from
- * @param conf configuration use when reading the tokens
- * @return a set of credentials, or null if the environment did not
- * specify any
- * @throws IOException if a location for credentials was defined, but
- * the credentials could not be loaded.
- */
- public static Credentials loadFromEnvironment(Map<String, String> env,
- Configuration conf)
- throws IOException {
+ public static File locateEnvCredentials(Map<String, String> env,
+ Configuration conf,
+ StringBuffer sourceTextOut) throws FileNotFoundException {
String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
- String source = HADOOP_TOKEN_FILE_LOCATION;
+ String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION;
if (tokenFilename == null) {
tokenFilename = conf.get(JOB_CREDENTIALS_BINARY);
- source = "Configuration option " + JOB_CREDENTIALS_BINARY;
+ source = "configuration option " + JOB_CREDENTIALS_BINARY;
}
if (tokenFilename != null) {
// use delegation tokens, i.e. from Oozie
File file = new File(tokenFilename.trim());
- String details = String.format("Token File %s from environment variable %s",
+ String details = String.format(
+ "Token File %s from %s",
file,
source);
- LOG.debug("Using {}", details);
if (!file.exists()) {
throw new FileNotFoundException("No " + details);
}
if (!file.isFile() && !file.canRead()) {
- throw new IOException("Cannot read " + details);
+ throw new FileNotFoundException("Cannot read " + details);
}
- Credentials creds = Credentials.readTokenStorageFile(file, conf);
- return creds;
+ sourceTextOut.append(details);
+ return file;
} else {
return null;
}
}
/**
+ * Load the credentials from the environment. This looks at
+ * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION}
+ * and attempts to read in the value
+ * @param env environment to resolve the variable from
+ * @param conf configuration use when reading the tokens
+ * @return a set of credentials, or null if the environment did not
+ * specify any
+ * @throws IOException if a location for credentials was defined, but
+ * the credentials could not be loaded.
+ */
+ public static Credentials loadTokensFromEnvironment(Map<String, String> env,
+ Configuration conf)
+ throws IOException {
+ StringBuffer origin = new StringBuffer();
+ File file = locateEnvCredentials(env, conf, origin);
+ if (file != null) {
+ LOG.debug("Using {}", origin);
+ return Credentials.readTokenStorageFile(file, conf);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Save credentials to a file
+ * @param file file to save to (will be overwritten)
+ * @param credentials credentials to write
+ * @throws IOException
+ */
+ public static void saveTokens(File file,
+ Credentials credentials) throws IOException {
+ try(DataOutputStream daos = new DataOutputStream(
+ new FileOutputStream(file))) {
+ credentials.writeTokenStorageToStream(daos);
+ }
+ }
+
+ /**
* Look up and return the resource manager's principal. This method
* automatically does the <code>_HOST</code> replacement in the principal and
* correctly handles HA resource manager configurations.
@@ -179,8 +218,8 @@ public final class CredentialUtils {
Preconditions.checkArgument(conf != null);
Preconditions.checkArgument(credentials != null);
if (UserGroupInformation.isSecurityEnabled()) {
- String tokenRenewer = CredentialUtils.getRMPrincipal(conf);
- return fs.addDelegationTokens(tokenRenewer, credentials);
+ return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf),
+ credentials);
}
return null;
}
@@ -197,10 +236,58 @@ public final class CredentialUtils {
Preconditions.checkArgument(fs != null);
Preconditions.checkArgument(credentials != null);
fs.addDelegationTokens(
- UserGroupInformation.getLoginUser().getShortUserName(),
+ getSelfRenewer(),
credentials);
}
+ public static String getSelfRenewer() throws IOException {
+ return UserGroupInformation.getLoginUser().getShortUserName();
+ }
+
+ /**
+ * Create and add an RM delegation token to the credentials
+ * @param yarnClient
+ * @param credentials to add token to
+ * @return the token which was added
+ * @throws IOException
+ * @throws YarnException
+ */
+ public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient,
+ Credentials credentials)
+ throws IOException, YarnException {
+ Configuration conf = yarnClient.getConfig();
+ Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf));
+ Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
+ Token<TokenIdentifier> rmDelegationToken =
+ ConverterUtils.convertFromYarn(
+ yarnClient.getRMDelegationToken(rmPrincipal),
+ rmDTService);
+ credentials.addToken(rmDelegationToken.getService(), rmDelegationToken);
+ return rmDelegationToken;
+ }
+
+ public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(
+ Configuration conf,
+ Credentials credentials)
+ throws IOException, YarnException {
+ if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
+ LOG.debug("Timeline service enabled -fetching token");
+
+ try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
+ timelineClient.init(conf);
+ timelineClient.start();
+ Token<TimelineDelegationTokenIdentifier> token =
+ timelineClient.getDelegationToken(
+ CredentialUtils.getRMPrincipal(conf));
+ credentials.addToken(token.getService(), token);
+ return token;
+ }
+ } else {
+ LOG.debug("Timeline service is disabled");
+ return null;
+ }
+ }
+
/**
* Filter a list of tokens from a set of credentials
* @param credentials credential source (a new credential set os re
@@ -224,18 +311,22 @@ public final class CredentialUtils {
}
public static String dumpTokens(Credentials credentials, String separator) {
- Collection<Token<? extends TokenIdentifier>> allTokens
- = credentials.getAllTokens();
- StringBuilder buffer = new StringBuilder(allTokens.size()* 128);
- DateFormat df = DateFormat.getDateTimeInstance(
- DateFormat.SHORT, DateFormat.SHORT);
- for (Token<? extends TokenIdentifier> token : allTokens) {
- buffer.append(toString(token)).append(separator);
+ ArrayList<Token<? extends TokenIdentifier>> sorted =
+ new ArrayList<>(credentials.getAllTokens());
+ Collections.sort(sorted, new TokenComparator());
+ StringBuilder buffer = new StringBuilder(sorted.size()* 128);
+ for (Token<? extends TokenIdentifier> token : sorted) {
+ buffer.append(tokenToString(token)).append(separator);
}
return buffer.toString();
}
- public static String toString(Token<? extends TokenIdentifier> token) {
+ /**
+ * Create a string for people to look at
+ * @param token token to convert to a string form
+ * @return a printable view of the token
+ */
+ public static String tokenToString(Token<? extends TokenIdentifier> token) {
DateFormat df = DateFormat.getDateTimeInstance(
DateFormat.SHORT, DateFormat.SHORT);
StringBuilder buffer = new StringBuilder(128);
@@ -244,16 +335,27 @@ public final class CredentialUtils {
TokenIdentifier ti = token.decodeIdentifier();
buffer.append("; ").append(ti);
if (ti instanceof AbstractDelegationTokenIdentifier) {
- AbstractDelegationTokenIdentifier dt
- = (AbstractDelegationTokenIdentifier) ti;
- buffer.append(" Issued: ")
+ // details in human readable form, and compensate for information HDFS DT omits
+ AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti;
+ buffer.append("; Renewer: ").append(dt.getRenewer());
+ buffer.append("; Issued: ")
.append(df.format(new Date(dt.getIssueDate())));
- buffer.append(" Max Date: ")
+ buffer.append("; Max Date: ")
.append(df.format(new Date(dt.getMaxDate())));
}
} catch (IOException e) {
+ //marshall problem; not ours
LOG.debug("Failed to decode {}: {}", token, e, e);
}
return buffer.toString();
}
+
+ private static class TokenComparator
+ implements Comparator<Token<? extends TokenIdentifier>>, Serializable {
+ @Override
+ public int compare(Token<? extends TokenIdentifier> left,
+ Token<? extends TokenIdentifier> right) {
+ return left.getKind().toString().compareTo(right.getKind().toString());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy
new file mode 100644
index 0000000..ee70979
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.params.ActionTokensArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.exceptions.BadClusterStateException
+import org.apache.slider.core.exceptions.NotFoundException
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.junit.Test
+
+/**
+ * Test the argument parsing/validation logic
+ */
+@CompileStatic
+@Slf4j
+class TestSliderTokensCommand extends ServiceLauncherBaseTest {
+
+ public static YarnConfiguration config = createTestConfig()
+
+ public static YarnConfiguration createTestConfig() {
+ def configuration = new YarnConfiguration()
+ configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032")
+ return configuration
+ }
+
+ @Test
+ public void testBadSourceArgs() throws Throwable {
+ launchExpectingException(SliderClient,
+ config,
+ ActionTokensArgs.DUPLICATE_ARGS,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_SOURCE, "target/tokens.bin",
+ Arguments.ARG_OUTPUT, "target/tokens.bin",
+ ])
+ }
+
+ @Test
+ public void testKTNoPrincipal() throws Throwable {
+ launchExpectingException(SliderClient,
+ config,
+ ActionTokensArgs.MISSING_KT_PROVIDER,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_KEYTAB, "target/keytab",
+ ])
+ }
+
+ @Test
+ public void testPrincipalNoKT() throws Throwable {
+ launchExpectingException(SliderClient,
+ config,
+ ActionTokensArgs.MISSING_KT_PROVIDER,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_PRINCIPAL, "bob@REALM",
+ ])
+ }
+
+ /**
+ * A missing keytab is an error
+ * @throws Throwable
+ */
+ @Test
+ public void testMissingKT() throws Throwable {
+ def ex = launchExpectingException(SliderClient,
+ config,
+ TokensOperation.E_NO_KEYTAB,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_PRINCIPAL, "bob@REALM",
+ Arguments.ARG_KEYTAB, "target/keytab",
+ ])
+ if (!(ex instanceof NotFoundException)) {
+ throw ex
+ }
+ }
+
+ @Test
+ public void testMissingSourceFile() throws Throwable {
+ def ex = launchExpectingException(SliderClient,
+ config,
+ TokensOperation.E_MISSING_SOURCE_FILE,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_SOURCE, "target/tokens.bin",
+ ])
+ if (!(ex instanceof NotFoundException)) {
+ throw ex
+ }
+ }
+
+ @Test
+ public void testListHarmlessWhenInsecure() throws Throwable {
+ execSliderCommand(0, config, [SliderActions.ACTION_TOKENS])
+ }
+
+ @Test
+ public void testCreateFailsWhenInsecure() throws Throwable {
+ def ex = launchExpectingException(SliderClient,
+ config,
+ TokensOperation.E_INSECURE,
+ [SliderActions.ACTION_TOKENS,
+ Arguments.ARG_OUTPUT, "target/tokens.bin",
+ ])
+ if (!(ex instanceof BadClusterStateException)) {
+ throw ex
+ }
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index cb6ce0e..0a3b040 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -990,7 +990,7 @@ class SliderTestUtils extends Assert {
* of return code takes place
* @param conf configuration
* @param args arg list
- * @return the return code
+ * @return the launcher
*/
protected static ServiceLauncher<SliderClient> execSliderCommand(
Configuration conf,
@@ -1005,6 +1005,24 @@ class SliderTestUtils extends Assert {
return serviceLauncher
}
+ /**
+ * Launch a slider command to a given exit code.
+ * Most failures will trigger exceptions; this is for the exit code of the runService()
+ * call.
+ * @param exitCode desired exit code
+ * @param conf configuration
+ * @param args arg list
+ * @return the launcher
+ */
+ protected static ServiceLauncher<SliderClient> execSliderCommand(
+ int exitCode,
+ Configuration conf,
+ List args) {
+ ServiceLauncher<SliderClient> serviceLauncher = execSliderCommand(conf, args)
+ assert exitCode == serviceLauncher.serviceExitCode
+ serviceLauncher
+ }
+
public static ServiceLauncher launch(Class serviceClass,
Configuration conf,
List<Object> args) throws