You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2016/02/01 17:21:35 UTC
[1/2] incubator-slider git commit: SLIDER-1080 move SliderUtils to
java 7 language features
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-1077-oozie 5dfac8532 -> f56a9311c
SLIDER-1080 move SliderUtils to java 7 language features
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f78465e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f78465e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f78465e0
Branch: refs/heads/feature/SLIDER-1077-oozie
Commit: f78465e0c04d0255bf9360b014923aa2262fc7b6
Parents: 5dfac85
Author: Steve Loughran <st...@apache.org>
Authored: Mon Feb 1 14:08:22 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Feb 1 14:08:22 2016 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 65 +++++++-------------
1 file changed, 22 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f78465e0/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index b114d8a..7277c21 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -335,18 +335,14 @@ public final class SliderUtils {
InetSocketAddress address,
int connectTimeout)
throws IOException {
- Socket socket = null;
- try {
- socket = new Socket();
+ try(Socket socket = new Socket()) {
socket.connect(address, connectTimeout);
} catch (Exception e) {
throw new IOException("Failed to connect to " + name
+ " at " + address
- + " after " + connectTimeout + "millisconds"
+ + " after " + connectTimeout + "milliseconds"
+ ": " + e,
e);
- } finally {
- IOUtils.closeSocket(socket);
}
}
@@ -516,7 +512,7 @@ public final class SliderUtils {
* @return a stringified list
*/
public static List<String> collectionToStringList(Collection c) {
- List<String> l = new ArrayList<String>(c.size());
+ List<String> l = new ArrayList<>(c.size());
for (Object o : c) {
l.add(o.toString());
}
@@ -788,9 +784,9 @@ public final class SliderUtils {
return;
}
List<ApplicationReport> nonLiveInstance =
- new ArrayList<ApplicationReport>(instances.size());
+ new ArrayList<>(instances.size());
List<ApplicationReport> liveInstance =
- new ArrayList<ApplicationReport>(instances.size());
+ new ArrayList<>(instances.size());
for (ApplicationReport report : instances) {
if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
@@ -826,7 +822,7 @@ public final class SliderUtils {
public static Map<String, ApplicationReport> buildApplicationReportMap(
List<ApplicationReport> instances,
YarnApplicationState minState, YarnApplicationState maxState) {
- TreeMap<String, ApplicationReport> map = new TreeMap<String, ApplicationReport>();
+ TreeMap<String, ApplicationReport> map = new TreeMap<>();
for (ApplicationReport report : instances) {
YarnApplicationState state = report.getYarnApplicationState();
if (state.ordinal() >= minState.ordinal() &&
@@ -844,7 +840,7 @@ public final class SliderUtils {
* @return a map whose iterator returns the string-sorted ordering of entries
*/
public static Map<String, String> sortedMap(Map<String, String> source) {
- Map<String, String> out = new TreeMap<String, String>(source);
+ Map<String, String> out = new TreeMap<>(source);
return out;
}
@@ -854,7 +850,7 @@ public final class SliderUtils {
* @return a string map
*/
public static Map<String, String> toMap(Properties properties) {
- Map<String, String> out = new HashMap<String, String>(properties.size());
+ Map<String, String> out = new HashMap<>(properties.size());
for (Map.Entry<Object, Object> entry : properties.entrySet()) {
out.put(entry.getKey().toString(), entry.getValue().toString());
}
@@ -1136,7 +1132,7 @@ public final class SliderUtils {
* @return a possibly empty map of environment variables.
*/
public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
- Map<String, String> env = new HashMap<String, String>();
+ Map<String, String> env = new HashMap<>();
if (roleOpts != null) {
for (Map.Entry<String, String> entry : roleOpts.entrySet()) {
String key = entry.getKey();
@@ -1470,7 +1466,7 @@ public final class SliderUtils {
/**
* Convert a char sequence to a string.
- * This ensures that comparisions work
+ * This ensures that comparisons work
* @param charSequence source
* @return the string equivalent
*/
@@ -1870,30 +1866,20 @@ public final class SliderUtils {
List<String> files = new ArrayList<>();
generateFileList(files, srcFolder, srcFolder, true, filter);
- TarArchiveOutputStream taos = null;
- try {
- taos = new TarArchiveOutputStream(new GZIPOutputStream(
- new BufferedOutputStream(new FileOutputStream(tarGzipFile))));
+ try(TarArchiveOutputStream taos =
+ new TarArchiveOutputStream(new GZIPOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
for (String file : files) {
File srcFile = new File(srcFolder, file);
TarArchiveEntry tarEntry = new TarArchiveEntry(
srcFile, file);
taos.putArchiveEntry(tarEntry);
- FileInputStream in = new FileInputStream(srcFile);
- try {
+ try(FileInputStream in = new FileInputStream(srcFile)) {
org.apache.commons.io.IOUtils.copy(in, taos);
- } finally {
- if (in != null) {
- in.close();
- }
}
taos.flush();
taos.closeArchiveEntry();
}
- } finally {
- if (taos != null) {
- taos.close();
- }
}
}
@@ -1912,7 +1898,7 @@ public final class SliderUtils {
* @return true if this is invoked in an HDP cluster or false otherwise
*/
public static boolean isHdp() {
- return StringUtils.isNotEmpty(getHdpVersion()) ? true : false;
+ return StringUtils.isNotEmpty(getHdpVersion());
}
/**
@@ -1977,9 +1963,7 @@ public final class SliderUtils {
String entry)
throws IOException {
InputStream is = null;
- FSDataInputStream appStream = null;
- try {
- appStream = fs.open(appPath);
+ try(FSDataInputStream appStream = fs.open(appPath)) {
ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
ZipArchiveEntry zipEntry;
boolean done = false;
@@ -2010,8 +1994,6 @@ public final class SliderUtils {
done = true;
}
}
- } finally {
- IOUtils.closeStream(appStream);
}
return is;
@@ -2162,11 +2144,8 @@ public final class SliderUtils {
parentDir.mkdirs();
}
SliderUtils.verifyIsDir(parentDir, log);
- FileOutputStream out = new FileOutputStream(outfile);
- try {
+ try(FileOutputStream out = new FileOutputStream(outfile)) {
out.write(data);
- } finally {
- IOUtils.closeStream(out);
}
}
@@ -2372,11 +2351,11 @@ public final class SliderUtils {
public static void validateHDFSFile(SliderFileSystem sliderFileSystem,
String pathStr)
throws IOException, URISyntaxException {
- URI pathURI = new URI(pathStr);
- InputStream inputStream =
- sliderFileSystem.getFileSystem().open(new Path(pathURI));
- if (inputStream == null) {
- throw new IOException("HDFS file " + pathStr + " can't be opened");
+ try(InputStream inputStream =
+ sliderFileSystem.getFileSystem().open(new Path(new URI(pathStr)))) {
+ if (inputStream == null) {
+ throw new IOException("HDFS file " + pathStr + " can't be opened");
+ }
}
}
[2/2] incubator-slider git commit: SLIDER-1080 switch to
$HADOOP_TOKEN_FILE_LOCATION as origin of oozie tokens;
fall back to Conf.get("mapreduce.job.credentials.binary"). This is done in
CredentialUtils and not in Launcher code. Also, HADOOP_JAAS_DEBUG i
Posted by st...@apache.org.
SLIDER-1080 switch to $HADOOP_TOKEN_FILE_LOCATION as origin of oozie tokens; fall back to Conf.get("mapreduce.job.credentials.binary"). This is done in CredentialUtils and not in Launcher code. Also, HADOOP_JAAS_DEBUG is auto-propagated from client to AM, then AM to containers. If you want to debug Hadoop and Kerberos, you want to debug it everywhere.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f56a9311
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f56a9311
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f56a9311
Branch: refs/heads/feature/SLIDER-1077-oozie
Commit: f56a9311cebaa088762a6d973582b89b425e7936
Parents: f78465e
Author: Steve Loughran <st...@apache.org>
Authored: Mon Feb 1 16:21:20 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Feb 1 16:21:20 2016 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 11 +++--
.../org/apache/slider/common/Constants.java | 2 +-
.../apache/slider/common/SliderXmlConfKeys.java | 2 +-
.../slider/core/launch/AbstractLauncher.java | 31 +++++++-----
.../slider/core/launch/AppMasterLauncher.java | 51 ++------------------
.../slider/core/launch/ContainerLauncher.java | 10 +---
.../slider/core/launch/CredentialUtils.java | 12 ++++-
.../server/appmaster/SliderAppMaster.java | 10 ++--
.../funtest/framework/CommandTestBase.groovy | 4 ++
.../funtest/commands/KDiagCommandIT.groovy | 2 +-
10 files changed, 55 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 5f694e2..c141d25 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -208,6 +208,7 @@ import static org.apache.hadoop.registry.client.binding.RegistryUtils.*;
import static org.apache.slider.api.InternalKeys.*;
import static org.apache.slider.api.OptionKeys.*;
import static org.apache.slider.api.ResourceKeys.*;
+import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
import static org.apache.slider.common.params.SliderActions.*;
import static org.apache.slider.common.tools.SliderUtils.*;
@@ -1915,8 +1916,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Credentials credentials = null;
if (clusterSecure) {
// pick up oozie credentials
- credentials = CredentialUtils.loadFromEnvironment(
- System.getenv(), config);
+ credentials = CredentialUtils.loadFromEnvironment(System.getenv(), config);
if (credentials == null) {
// nothing from oozie, so build up directly
credentials = new Credentials(
@@ -1924,6 +1924,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
CredentialUtils.addRMRenewableFSDelegationTokens(config,
sliderFileSystem.getFileSystem(),
credentials);
+
+ } else {
+ log.info("Using externally supplied credentials to launch AM");
}
}
@@ -2087,8 +2090,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
amLauncher.setEnv("LANG", "en_US.UTF-8");
amLauncher.setEnv("LC_ALL", "en_US.UTF-8");
amLauncher.setEnv("LANGUAGE", "en_US.UTF-8");
+ amLauncher.maybeSetEnv(HADOOP_JAAS_DEBUG,
+ System.getenv(HADOOP_JAAS_DEBUG));
amLauncher.putEnv(getAmLaunchEnv(config));
-
+
for (Map.Entry<String, String> envs : getSystemEnv().entrySet()) {
log.debug("System env {}={}", envs.getKey(), envs.getValue());
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/common/Constants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/Constants.java b/slider-core/src/main/java/org/apache/slider/common/Constants.java
index fdb3452..0e3559a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/Constants.java
+++ b/slider-core/src/main/java/org/apache/slider/common/Constants.java
@@ -22,7 +22,7 @@ public class Constants {
public static final int CONNECT_TIMEOUT = 10000;
public static final int RPC_TIMEOUT = 15000;
- public static final String ENV_JAAS_DEBUG = "HADOOP_JAAS_DEBUG";
+ public static final String HADOOP_JAAS_DEBUG = "HADOOP_JAAS_DEBUG";
public static final String KRB5_CCNAME = "KRB5CCNAME";
public static final String JAVA_SECURITY_KRB5_CONF
= "java.security.krb5.conf";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 26109a7..72dd44f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -202,5 +202,5 @@ public interface SliderXmlConfKeys {
String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file";
String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";
-
+ String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index f92ffb1..2ec3cda 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -58,7 +58,6 @@ public abstract class AbstractLauncher extends Configured {
private static final Logger log =
LoggerFactory.getLogger(AbstractLauncher.class);
public static final String CLASSPATH = "CLASSPATH";
- public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
/**
* Filesystem to use for the launch
*/
@@ -67,20 +66,18 @@ public abstract class AbstractLauncher extends Configured {
* Env vars; set up at final launch stage
*/
protected final Map<String, String> envVars = new HashMap<>();
-
protected final MapOperations env = new MapOperations("env", envVars);
protected final ContainerLaunchContext containerLaunchContext =
Records.newRecord(ContainerLaunchContext.class);
protected final List<String> commands = new ArrayList<>(20);
protected final Map<String, LocalResource> localResources = new HashMap<>();
private final Map<String, ByteBuffer> serviceData = new HashMap<>();
-
// security
protected final Credentials credentials;
protected LogAggregationContext logAggregationContext;
/**
- * Create instance
+ * Create instance.
* @param conf configuration
* @param coreFileSystem filesystem
* @param credentials initial set of credentials -null is permitted
@@ -93,15 +90,6 @@ public abstract class AbstractLauncher extends Configured {
this.credentials = credentials != null ? credentials: new Credentials();
}
- protected AbstractLauncher(Configuration conf,
- CoreFileSystem fs) {
- this(conf, fs, null);
- }
-
- protected AbstractLauncher(CoreFileSystem fs) {
- this(null, fs, null);
- }
-
/**
* Get the container. Until "completed", this isn't valid to launch.
* @return the container to launch
@@ -401,12 +389,29 @@ public abstract class AbstractLauncher extends Configured {
public void setClasspath(ClasspathConstructor classpath) {
setEnv(CLASSPATH, classpath.buildClasspath());
}
+
+ /**
+ * Set an environment variable in the launch context
+ * @param var variable name
+ * @param value value (must be non null)
+ */
public void setEnv(String var, String value) {
Preconditions.checkArgument(var != null, "null variable name");
Preconditions.checkArgument(value != null, "null value");
env.put(var, value);
}
+ /**
+ * Set an environment variable if its value is non-null.
+ * @param var variable name
+ * @param value value (may be null)
+ */
+ public void maybeSetEnv(String var, String value) {
+ if (value != null) {
+ setEnv(var, value);
+ }
+ }
+
public void putEnv(Map<String, String> map) {
env.putAll(map);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index 091b80e..7190c3a 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -19,11 +19,7 @@
package org.apache.slider.core.launch;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -38,8 +34,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.text.DateFormat;
-import java.util.Date;
import java.util.Map;
import java.util.Set;
@@ -194,7 +188,9 @@ public class AppMasterLauncher extends AbstractLauncher {
if (secureCluster) {
//tokens
- addSecurityTokens();
+ log.debug("Credentials: {}",
+ CredentialUtils.dumpTokens(getCredentials(), "\n"));
+
} else {
propagateUsernameInInsecureCluster();
}
@@ -204,47 +200,6 @@ public class AppMasterLauncher extends AbstractLauncher {
}
/**
- * Add the security tokens if this is a secure cluster
- * @throws IOException
- */
- private void addSecurityTokens() throws IOException {
-
- CredentialUtils.addRMRenewableFSDelegationTokens(getConf(),
- coreFileSystem.getFileSystem(), credentials);
-
- String tokenRenewer = CredentialUtils.getRMPrincipal(getConf());
-
- Token<? extends TokenIdentifier>[] tokens = null;
- boolean tokensProvided = getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) !=
- null;
- if (!tokensProvided) {
- // For now, only getting tokens for the default file-system.
- FileSystem fs = coreFileSystem.getFileSystem();
- tokens = fs.addDelegationTokens(tokenRenewer, credentials);
- }
- // obtain the token expiry from the first token - should be the same for all
- // HDFS tokens
- if (tokens != null && tokens.length > 0) {
- AbstractDelegationTokenIdentifier id =
- (AbstractDelegationTokenIdentifier) tokens[0].decodeIdentifier();
- Date d = new Date(id.getIssueDate() + 24 * 60 * 60 * 1000);
- log.info(
- "HDFS delegation tokens for AM launch context require renewal by {}",
- DateFormat.getDateTimeInstance().format(d));
- } else {
- if (!tokensProvided) {
- log.warn("No HDFS delegation tokens obtained for AM launch context");
- } else {
- log.info("Tokens provided via " + MAPREDUCE_JOB_CREDENTIALS_BINARY +
- " property "
- + "being used for AM launch");
- }
-
- }
-
- }
-
- /**
* Submit the application.
* @return a launched application representing the submitted application
* @throws IOException
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index e586743..d220928 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -43,18 +43,12 @@ public class ContainerLauncher extends AbstractLauncher {
public ContainerLauncher(Configuration conf,
CoreFileSystem coreFileSystem,
- Container container, Credentials credentials) {
+ Container container,
+ Credentials credentials) {
super(conf, coreFileSystem, credentials);
this.container = container;
}
- public ContainerLauncher(Configuration conf,
- CoreFileSystem fs,
- Container container) {
- super(conf, fs);
- this.container = container;
- }
-
/**
* This code is in the dist shell examples -it's been moved here
* so that if it is needed, it's still here
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
index 32068e2..3245c13 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,6 +56,9 @@ import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
*/
public final class CredentialUtils {
+ public static final String JOB_CREDENTIALS_BINARY
+ = SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY;
+
private CredentialUtils() {
}
@@ -94,12 +98,17 @@ public final class CredentialUtils {
Configuration conf)
throws IOException {
String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
+ String source = HADOOP_TOKEN_FILE_LOCATION;
+ if (tokenFilename == null) {
+ tokenFilename = conf.get(JOB_CREDENTIALS_BINARY);
+ source = "Configuration option " + JOB_CREDENTIALS_BINARY;
+ }
if (tokenFilename != null) {
// use delegation tokens, i.e. from Oozie
File file = new File(tokenFilename.trim());
String details = String.format("Token File %s from environment variable %s",
file,
- HADOOP_TOKEN_FILE_LOCATION);
+ source);
LOG.debug("Using {}", details);
if (!file.exists()) {
throw new FileNotFoundException("No " + details);
@@ -171,7 +180,6 @@ public final class CredentialUtils {
Preconditions.checkArgument(credentials != null);
if (UserGroupInformation.isSecurityEnabled()) {
String tokenRenewer = CredentialUtils.getRMPrincipal(conf);
- Token<? extends TokenIdentifier>[] tokens = null;
return fs.addDelegationTokens(tokenRenewer, credentials);
}
return null;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 82c9fb9..9a284e3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
+
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -775,7 +777,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
uploadServerCertForLocalization(clustername, fs);
}
-
webAppPort = getPortToRequest();
if (webAppPort == 0) {
// failure to find a port
@@ -914,6 +915,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
if (hadoop_user_name != null) {
envVars.put(HADOOP_USER_NAME, hadoop_user_name);
}
+ String debug_kerberos = System.getenv(HADOOP_JAAS_DEBUG);
+ if (debug_kerberos != null) {
+ envVars.put(HADOOP_JAAS_DEBUG, debug_kerberos);
+ }
}
String rolesTmpSubdir = appMasterContainerID.toString() + "/roles";
@@ -1208,8 +1213,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.error("User is not holding on a keytab in a secure deployment:" +
" slider will fail as tokens expire");
}
- Credentials credentials =
- user.getCredentials();
+ Credentials credentials = user.getCredentials();
Iterator<Token<? extends TokenIdentifier>> iter =
credentials.getAllTokens().iterator();
while (iter.hasNext()) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index a887271..1a0d894 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.StatusKeys
import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.Constants
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
@@ -760,6 +761,9 @@ abstract class CommandTestBase extends SliderTestUtils {
maybeAddCommandOption(commands,
[ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL],
SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL));
+
+ commands << ARG_COMP_OPT << SliderKeys.COMPONENT_AM <<
+ "env." + Constants.HADOOP_JAAS_DEBUG << "true";
commands.addAll(extraArgs)
SliderShell shell = new SliderShell(commands)
if (0 != shell.execute()) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy
index ebdd00c..5fd46fa 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy
@@ -37,7 +37,7 @@ public class KDiagCommandIT extends CommandTestBase implements Arguments {
SliderActions.ACTION_KDIAG,
ARG_KEYLEN, "128"
],
- [(ENV_JAAS_DEBUG): "true"]
+ [(HADOOP_JAAS_DEBUG): "true"]
)
shell.execute()
assertSuccess(shell)