You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2015/03/14 01:36:36 UTC
incubator-reef git commit: [REEF-204] Fixed failure to react to
misconfigured Hadoop 2.6 clusters.
Repository: incubator-reef
Updated Branches:
refs/heads/master 871071203 -> f47abcf59
[REEF-204] Fixed failure to react to misconfigured Hadoop 2.6 clusters.
YARN 2.6 added `YarnConfiguration.YARN_APPLICATION_CLASSPATH` and
`DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH`. We assumed that
their presence also indicates that they are configured correctly.
However, HDInsight 3.2 uses Hadoop 2.6 but does NOT configure
`YarnConfiguration.YARN_APPLICATION_CLASSPATH` correctly.
This change checks whether this option is empty. If it it is, we now
revert to the legacy behavior of Hadoop 2.4 and earlier: The classpath
is constructed from environment variables (%HADOOP_HOME% and such).
JIRA:
[REEF-204](https://issues.apache.org/jira/browse/REEF-204)
Pull Request:
This closes #112
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/f47abcf5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/f47abcf5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/f47abcf5
Branch: refs/heads/master
Commit: f47abcf59a748d04f239cbbaea448a794504ae11
Parents: 8710712
Author: Beysim Sezgin <be...@microsoft.com>
Authored: Fri Mar 13 16:49:40 2015 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Fri Mar 13 17:28:58 2015 -0700
----------------------------------------------------------------------
...safeHDInsightRuntimeConfigurationStatic.java | 3 ++
.../reef/runtime/yarn/ClassPathBuilder.java | 16 +++++++-
.../runtime/yarn/YarnClasspathProvider.java | 39 +++++++++++++++-----
3 files changed, 46 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/f47abcf5/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
index 37a46bf..cf7686d 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
@@ -24,8 +24,10 @@ import org.apache.reef.client.RunningJob;
import org.apache.reef.runtime.common.client.REEFImplementation;
import org.apache.reef.runtime.common.client.RunningJobImpl;
import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
+import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
import org.apache.reef.runtime.common.launch.REEFMessageCodec;
import org.apache.reef.runtime.hdinsight.client.sslhacks.UnsafeClientConstructor;
+import org.apache.reef.runtime.yarn.YarnClasspathProvider;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
import org.apache.reef.util.logging.LoggingSetup;
@@ -41,6 +43,7 @@ public final class UnsafeHDInsightRuntimeConfigurationStatic extends Configurati
public static final ConfigurationModule CONF = new UnsafeHDInsightRuntimeConfigurationStatic()
.bindImplementation(REEF.class, REEFImplementation.class)
+ .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
.bindImplementation(RunningJob.class, RunningJobImpl.class)
.bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
.bindImplementation(JobSubmissionHandler.class, HDInsightJobSubmissionHandler.class)
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/f47abcf5/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java
index fa7780b..5a96b87 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java
@@ -45,8 +45,8 @@ final class ClassPathBuilder {
*/
private static boolean couldBeYarnConfigurationPath(final String path) {
return path.contains("conf") ||
- path.contains("etc") ||
- path.contains(HadoopEnvironment.HADOOP_CONF_DIR);
+ path.contains("etc") ||
+ path.contains(HadoopEnvironment.HADOOP_CONF_DIR);
}
/**
@@ -94,6 +94,18 @@ final class ClassPathBuilder {
}
/**
+ * Adds all the given entries to the classpath suffix.
+ *
+ * @param entries
+ */
+ void addAllToSuffix(final String... entries) {
+ for (final String classPathEntry : entries) {
+ this.addToPrefix(classPathEntry);
+ }
+ }
+
+
+ /**
* @return the suffix in an immutable list.
*/
List<String> getSuffixAsImmutableList() {
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/f47abcf5/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java
index c5a83a3..0a732ad 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java
@@ -23,6 +23,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
import org.apache.reef.util.OSUtils;
+import org.apache.reef.util.Optional;
import javax.inject.Inject;
import java.util.List;
@@ -47,7 +48,6 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider {
// Used when we can't get a classpath from YARN
private static final String[] LEGACY_CLASSPATH_LIST = new String[]{
- HADOOP_CONF_DIR,
HADOOP_HOME + "/*",
HADOOP_HOME + "/lib/*",
HADOOP_COMMON_HOME + "/*",
@@ -76,19 +76,21 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider {
boolean needsLegacyClasspath = false; // will be set to true below whenever we encounter issues with the YARN Configuration
final ClassPathBuilder builder = new ClassPathBuilder();
+
try {
// Add the classpath actually configured on this cluster
- final String[] yarnClassPath = yarnConfiguration.getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
- if (null == yarnClassPath || yarnClassPath.length == 0) {
- needsLegacyClasspath = true;
- } else {
- builder.addAll(yarnClassPath);
- }
+ final Optional<String[]> yarnClassPath = getTrimmedStrings(yarnConfiguration, YarnConfiguration.YARN_APPLICATION_CLASSPATH);
final String[] yarnDefaultClassPath = YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH;
- if (null == yarnDefaultClassPath || yarnDefaultClassPath.length == 0) {
- LOG.log(Level.SEVERE, "YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH is empty. This indicates a broken cluster configuration");
+
+ if (!yarnClassPath.isPresent()) {
+ LOG.log(Level.SEVERE, "YarnConfiguration.YARN_APPLICATION_CLASSPATH is empty. This indicates a broken cluster configuration.");
needsLegacyClasspath = true;
} else {
+ if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) {
+ LOG.log(CLASSPATH_LOG_LEVEL, "YarnConfiguration.YARN_APPLICATION_CLASSPATH is [" + StringUtils.join(yarnClassPath.get(), '|') + "]");
+ LOG.log(CLASSPATH_LOG_LEVEL, "YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH is [" + StringUtils.join(yarnDefaultClassPath, '|') + "]");
+ }
+ builder.addAll(yarnClassPath.get());
builder.addAll(yarnDefaultClassPath);
}
} catch (final NoSuchFieldError e) {
@@ -99,7 +101,8 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider {
}
if (needsLegacyClasspath) {
- builder.addAll(LEGACY_CLASSPATH_LIST);
+ builder.addToPrefix(HADOOP_CONF_DIR);
+ builder.addAllToSuffix(LEGACY_CLASSPATH_LIST);
}
this.classPathPrefix = builder.getPrefixAsImmutableList();
@@ -107,6 +110,22 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider {
this.logClasspath();
}
+ /**
+ * Fetches the string[] under the given key, if it exists and contains entries (.length >0)
+ *
+ * @param configuration
+ * @param key
+ * @return
+ */
+ private static Optional<String[]> getTrimmedStrings(final YarnConfiguration configuration, final String key) {
+ final String[] result = configuration.getTrimmedStrings(key);
+ if (null == result || result.length == 0) {
+ return Optional.empty();
+ } else {
+ return Optional.of(result);
+ }
+ }
+
@Override
public List<String> getDriverClasspathPrefix() {
return this.classPathPrefix;