You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by mo...@apache.org on 2016/11/18 02:00:37 UTC
reef git commit: [REEF-1669]Removing YarnClient from
YarnContainerManager
Repository: reef
Updated Branches:
refs/heads/master 4392ab83b -> 4e4e568fa
[REEF-1669]Removing YarnClient from YarnContainerManager
Currently we have both YarnClient and resourceManager (AMRMClientAsync) in YarnContainerManager. We call init() and start() for both of them. But inside driver, we are not able to get correct YarnConfiguration for YarnClient, that makes yarnClient.getNodeReports() fail to connect to RM.
In YarnContainerManager, we already have onNodesUpdated() which gives the list of the NodeReports on the fly. yarnClient.getNodeReports() was called at beginning and it only gives a static list. So looks like we don't need to call it anyway.
Based on RM folks, REEF should not use YarnClient at driver side. This PR is to remove YarnClient from YarnContainerManager. It also added some logs for environment variables with a lower log level or debug.
JIRA: [REEF-1669](https://issues.apache.org/jira/browse/REEF-1669)
This closes #1182
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/4e4e568f
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/4e4e568f
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/4e4e568f
Branch: refs/heads/master
Commit: 4e4e568faa3c3d625433b17bb4d15e0cd7897d7a
Parents: 4392ab8
Author: Julia Wang <jw...@yahoo.com>
Authored: Wed Nov 16 11:29:05 2016 -0800
Committer: Sergiy Matusevych <mo...@apache.org>
Committed: Thu Nov 17 17:59:21 2016 -0800
----------------------------------------------------------------------
.../reef/runtime/yarn/YarnClasspathProvider.java | 11 +++++++++++
.../runtime/yarn/driver/YarnContainerManager.java | 18 ------------------
2 files changed, 11 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/4e4e568f/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java
index 8b400de..ad3685e 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java
@@ -27,6 +27,7 @@ import org.apache.reef.util.Optional;
import javax.inject.Inject;
import java.util.List;
+import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -59,6 +60,7 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider {
HADOOP_HDFS_HOME + "/lib/*",
HADOOP_MAPRED_HOME + "/*",
HADOOP_MAPRED_HOME + "/lib/*",
+ HADOOP_COMMON_HOME + "/etc/hadoop/client/*",
HADOOP_HOME + "/etc/hadoop",
HADOOP_HOME + "/share/hadoop/common/*",
HADOOP_HOME + "/share/hadoop/common/lib/*",
@@ -74,6 +76,7 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider {
@Inject
YarnClasspathProvider(final YarnConfiguration yarnConfiguration) {
+ logEnvVariable();
boolean needsLegacyClasspath = false;
// will be set to true below whenever we encounter issues with the YARN Configuration
final ClassPathBuilder builder = new ClassPathBuilder();
@@ -153,6 +156,14 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider {
return this.classPathSuffix;
}
+ private static void logEnvVariable() {
+ if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) {
+ for (final Map.Entry<String, String> entry : System.getenv().entrySet()) {
+ LOG.log(CLASSPATH_LOG_LEVEL, "Environment variable: Key: {0}, Value: {1}.",
+ new Object[]{entry.getKey(), entry.getValue()});
+ }
+ }
+ }
private void logClasspath() {
if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) {
http://git-wip-us.apache.org/repos/asf/reef/blob/4e4e568f/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
index 58fcca1..41be1e8 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.fs.*;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
@@ -63,7 +62,6 @@ final class YarnContainerManager
private static final String RUNTIME_NAME = "YARN";
- private final YarnClient yarnClient = YarnClient.createYarnClient();
private final Queue<AMRMClient.ContainerRequest> requestsBeforeSentToRM = new ConcurrentLinkedQueue<>();
private final Queue<AMRMClient.ContainerRequest> requestsAfterSentToRM = new ConcurrentLinkedQueue<>();
private final Map<String, String> nodeIdToRackName = new ConcurrentHashMap<>();
@@ -107,8 +105,6 @@ final class YarnContainerManager
this.trackingURLProvider = trackingURLProvider;
this.rackNameFormatter = rackNameFormatter;
- this.yarnClient.init(this.yarnConf);
-
this.resourceManager = AMRMClientAsync.createAMRMClientAsync(yarnRMHeartbeatPeriod, this);
this.nodeManager = new NMClientAsyncImpl(this);
@@ -303,9 +299,6 @@ final class YarnContainerManager
* This method is called from DriverRuntimeStartHandler via YARNRuntimeStartHandler.
*/
void onStart() {
-
- this.yarnClient.start();
-
this.resourceManager.init(this.yarnConf);
this.resourceManager.start();
@@ -313,16 +306,6 @@ final class YarnContainerManager
this.nodeManager.start();
try {
- for (final NodeReport nodeReport : this.yarnClient.getNodeReports(NodeState.RUNNING)) {
- onNodeReport(nodeReport);
- }
- } catch (IOException | YarnException e) {
- LOG.log(Level.WARNING, "Unable to fetch node reports from YARN.", e);
- onRuntimeError(e);
- }
-
- try {
-
this.registration.setRegistration(
this.resourceManager.registerApplicationMaster("", 0, this.trackingURLProvider.getTrackingUrl()));
@@ -334,7 +317,6 @@ final class YarnContainerManager
try (final FSDataOutputStream out = fs.create(outputFileName)) {
out.writeBytes(this.trackingURLProvider.getTrackingUrl() + '\n');
}
-
} catch (final YarnException | IOException e) {
LOG.log(Level.WARNING, "Unable to register application master.", e);
onRuntimeError(e);