You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by mo...@apache.org on 2017/07/21 17:01:59 UTC
reef git commit: [REEF-1831] Setting FileSytemUrl for
YarnConfiguration used in FileSystem
Repository: reef
Updated Branches:
refs/heads/master 564b88066 -> 5abb8b13a
[REEF-1831] Setting FileSytemUrl for YarnConfiguration used in FileSystem
* Inject FileSystemUrl in YarnConfigurationConstructor
* Set FileSystemUrl for FileSystem in YarnConfigurationCOnstructor
* Bind FileSystemUrl to driver configuration
JIRA: [REEF-1831](https://issues.apache.org/jira/browse/REEF-1831)
This closes #1332
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/5abb8b13
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/5abb8b13
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/5abb8b13
Branch: refs/heads/master
Commit: 5abb8b13a462adff9b8fb1d93d669610857b03b8
Parents: 564b880
Author: Julia Wang <ju...@apache.org>
Authored: Thu Jul 13 18:51:09 2017 -0700
Committer: Sergiy Matusevych <mo...@apache.org>
Committed: Fri Jul 21 10:01:32 2017 -0700
----------------------------------------------------------------------
.../bridge/client/YarnJobSubmissionClient.java | 3 ++-
.../yarn/driver/YarnContainerManager.java | 11 ++---------
.../yarn/util/YarnConfigurationConstructor.java | 19 +++++++++++++++++--
3 files changed, 21 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/5abb8b13/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java
index f536bd3..fcf03ba 100644
--- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java
+++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java
@@ -277,7 +277,7 @@ public final class YarnJobSubmissionClient {
if (!yarnSubmission.getFileSystemUrl().equalsIgnoreCase(FileSystemUrl.DEFAULT_VALUE)) {
LOG.log(Level.INFO, "getFileSystemUrl: {0}", yarnSubmission.getFileSystemUrl());
} else {
- LOG.log(Level.INFO, "FileSystemUrl is not set");
+ LOG.log(Level.INFO, "FileSystemUrl is not set, use default from the environment.");
}
final List<String> launchCommandPrefix = new ArrayList<String>() {{
@@ -288,6 +288,7 @@ public final class YarnJobSubmissionClient {
.bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
.bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
.bindNamedParameter(JobSubmissionDirectoryPrefix.class, yarnSubmission.getJobSubmissionDirectoryPrefix())
+ .bindNamedParameter(FileSystemUrl.class, yarnSubmission.getFileSystemUrl())
.bindList(DriverLaunchCommandPrefix.class, launchCommandPrefix)
.build();
final YarnJobSubmissionClient client = Tang.Factory.getTang().newInjector(yarnJobSubmissionClientConfig)
http://git-wip-us.apache.org/repos/asf/reef/blob/5abb8b13/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
index f89afa0..de8d0b1 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
@@ -43,7 +43,6 @@ import org.apache.reef.runtime.common.driver.resourcemanager.ResourceStatusEvent
import org.apache.reef.runtime.common.driver.resourcemanager.RuntimeStatusEventImpl;
import org.apache.reef.runtime.common.files.REEFFileNames;
import org.apache.reef.runtime.yarn.client.unmanaged.YarnProxyUser;
-import org.apache.reef.runtime.yarn.driver.parameters.FileSystemUrl;
import org.apache.reef.runtime.yarn.driver.parameters.JobSubmissionDirectory;
import org.apache.reef.runtime.yarn.driver.parameters.YarnHeartbeatPeriod;
import org.apache.reef.tang.InjectionFuture;
@@ -88,9 +87,7 @@ final class YarnContainerManager implements AMRMClientAsync.CallbackHandler, NMC
private final DriverStatusManager driverStatusManager;
private final String trackingUrl;
private final String amRegistrationHost;
-
private final String jobSubmissionDirectory;
- private final String fileSystemUrl;
private final REEFFileNames reefFileNames;
private final RackNameFormatter rackNameFormatter;
private final InjectionFuture<ProgressProvider> progressProvider;
@@ -99,7 +96,6 @@ final class YarnContainerManager implements AMRMClientAsync.CallbackHandler, NMC
private YarnContainerManager(
@Parameter(YarnHeartbeatPeriod.class) final int yarnRMHeartbeatPeriod,
@Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory,
- @Parameter(FileSystemUrl.class) final String fileSystemUrl,
final YarnConfiguration yarnConf,
final YarnProxyUser yarnProxyUser,
final REEFEventHandlers reefEventHandlers,
@@ -130,14 +126,11 @@ final class YarnContainerManager implements AMRMClientAsync.CallbackHandler, NMC
this.nodeManager = new NMClientAsyncImpl(this);
this.jobSubmissionDirectory = jobSubmissionDirectory;
- this.fileSystemUrl = fileSystemUrl;
this.reefFileNames = reefFileNames;
this.progressProvider = progressProvider;
- LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {2}, fileSystemUrl: {3}, " +
- "jobSubmissionDirectory: {4}.",
- new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.fileSystemUrl,
- this.jobSubmissionDirectory});
+ LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {3}, jobSubmissionDirectory: {4}.",
+ new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.jobSubmissionDirectory});
}
/**
http://git-wip-us.apache.org/repos/asf/reef/blob/5abb8b13/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java
index 1f318a7..a54b8e3 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java
@@ -19,20 +19,35 @@
package org.apache.reef.runtime.yarn.util;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.reef.runtime.yarn.driver.parameters.FileSystemUrl;
import org.apache.reef.tang.ExternalConstructor;
+import org.apache.reef.tang.annotations.Parameter;
import javax.inject.Inject;
+import java.util.logging.Level;
+import java.util.logging.Logger;
/**
* An external constructor that creates YarnConfiguration instances.
*/
public final class YarnConfigurationConstructor implements ExternalConstructor<YarnConfiguration> {
+
+ private static final Logger LOG = Logger.getLogger(YarnConfigurationConstructor.class.getName());
+
+ private final String fileSystemUrl;
+
@Inject
- YarnConfigurationConstructor() {
+ YarnConfigurationConstructor(@Parameter(FileSystemUrl.class) final String fileSystemUrl) {
+ this.fileSystemUrl = fileSystemUrl;
}
@Override
public YarnConfiguration newInstance() {
- return new YarnConfiguration();
+ YarnConfiguration yarnConfiguration = new YarnConfiguration();
+ if (!fileSystemUrl.equals(FileSystemUrl.DEFAULT_VALUE)) {
+ yarnConfiguration.set("fs.defaultFS", fileSystemUrl);
+ LOG.log(Level.INFO, "Set fileSystemUrl in YarnConfigurationConstructor: {0}", fileSystemUrl);
+ }
+ return yarnConfiguration;
}
}