You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by ch...@apache.org on 2014/12/18 05:30:19 UTC
incubator-reef git commit: [REEF-72]: Make DriverIdentifier optional
Repository: incubator-reef
Updated Branches:
refs/heads/master ec477fd41 -> 19019dc1a
[REEF-72]: Make DriverIdentifier optional
This makes the DriverIdentifier optional. If it isn't set, we generate a unique one in JobSubmissionHelper.
JIRA:
[REEF-72] https://issues.apache.org/jira/browse/REEF-72
Pull Request:
Closes #34
Author:
Markus Weimer weimer@apache.org
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/19019dc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/19019dc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/19019dc1
Branch: refs/heads/master
Commit: 19019dc1a2d5f0358cd2d8499e06d89fd6b1673d
Parents: ec477fd
Author: Markus Weimer <we...@apache.org>
Authored: Thu Dec 18 13:24:12 2014 +0900
Committer: chobrian <ch...@apache.org>
Committed: Thu Dec 18 13:28:13 2014 +0900
----------------------------------------------------------------------
.../apache/reef/client/DriverConfiguration.java | 2 +-
.../driver/parameters/DriverIdentifier.java | 6 ++++-
.../common/client/JobSubmissionHelper.java | 25 ++++++++++++++++----
3 files changed, 27 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/19019dc1/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java
----------------------------------------------------------------------
diff --git a/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java b/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java
index 5ed84a3..5573fed 100644
--- a/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java
+++ b/reef-common/src/main/java/org/apache/reef/client/DriverConfiguration.java
@@ -49,7 +49,7 @@ public final class DriverConfiguration extends ConfigurationModuleBuilder {
/**
* Identifies the driver and therefore the JOB. Expect to see this e.g. on YARN's dashboard.
*/
- public static final RequiredParameter<String> DRIVER_IDENTIFIER = new RequiredParameter<>();
+ public static final OptionalParameter<String> DRIVER_IDENTIFIER = new OptionalParameter<>();
/**
* The amount of memory to be allocated for the Driver. This is the size of the AM container in YARN.
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/19019dc1/reef-common/src/main/java/org/apache/reef/driver/parameters/DriverIdentifier.java
----------------------------------------------------------------------
diff --git a/reef-common/src/main/java/org/apache/reef/driver/parameters/DriverIdentifier.java b/reef-common/src/main/java/org/apache/reef/driver/parameters/DriverIdentifier.java
index 484aa67..1a0002b 100644
--- a/reef-common/src/main/java/org/apache/reef/driver/parameters/DriverIdentifier.java
+++ b/reef-common/src/main/java/org/apache/reef/driver/parameters/DriverIdentifier.java
@@ -24,8 +24,12 @@ import org.apache.reef.tang.annotations.NamedParameter;
/**
* Driver Identifier
*/
-@NamedParameter(doc = "Driver Identifier", default_value = "Unnamed REEF Job")
+@NamedParameter(doc = "Driver Identifier", default_value = DriverIdentifier.DEFAULT_VALUE)
public final class DriverIdentifier implements Name<String> {
+
private DriverIdentifier() {
}
+
+ public static final String DEFAULT_VALUE = "##NONE##DEFAULT##NEVERUSE##";
+
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/19019dc1/reef-common/src/main/java/org/apache/reef/runtime/common/client/JobSubmissionHelper.java
----------------------------------------------------------------------
diff --git a/reef-common/src/main/java/org/apache/reef/runtime/common/client/JobSubmissionHelper.java b/reef-common/src/main/java/org/apache/reef/runtime/common/client/JobSubmissionHelper.java
index ee9a105..6b89c09 100644
--- a/reef-common/src/main/java/org/apache/reef/runtime/common/client/JobSubmissionHelper.java
+++ b/reef-common/src/main/java/org/apache/reef/runtime/common/client/JobSubmissionHelper.java
@@ -27,7 +27,6 @@ import org.apache.reef.tang.Tang;
import org.apache.reef.tang.exceptions.InjectionException;
import org.apache.reef.tang.formats.ConfigurationSerializer;
import org.apache.reef.util.JARFileMaker;
-import org.apache.reef.util.REEFVersion;
import javax.inject.Inject;
import java.io.File;
@@ -76,7 +75,7 @@ final class JobSubmissionHelper {
final Injector injector = Tang.Factory.getTang().newInjector(driverConfiguration);
final ClientRuntimeProtocol.JobSubmissionProto.Builder jbuilder = ClientRuntimeProtocol.JobSubmissionProto.newBuilder()
- .setIdentifier(injector.getNamedInstance(DriverIdentifier.class))
+ .setIdentifier(returnOrGenerateDriverId(injector.getNamedInstance(DriverIdentifier.class)))
.setDriverMemory(injector.getNamedInstance(DriverMemory.class))
.setUserName(System.getProperty("user.name"))
.setConfiguration(configurationSerializer.toString(driverConfiguration));
@@ -105,6 +104,24 @@ final class JobSubmissionHelper {
return jbuilder;
}
+
+ /**
+ * @param configuredId
+ * @return the given driver ID (if it is not the default) or generates a new unique one if it is.
+ */
+ private static String returnOrGenerateDriverId(final String configuredId) {
+ final String result;
+ if (configuredId.equals(DriverIdentifier.DEFAULT_VALUE)) {
+ // Generate a uniqe driver ID
+ LOG.log(Level.FINE, "No Job Identifier given. Generating a unique one.");
+ result = "REEF-" + System.getProperty("user.name", "UNKNOWN_USER") + "-" + System.currentTimeMillis();
+ } else {
+ result = configuredId;
+ }
+ return result;
+ }
+
+
/**
* Turns a pathname into the right protocol for job submission.
*
@@ -113,7 +130,7 @@ final class JobSubmissionHelper {
* @return
* @throws IOException
*/
- private ReefServiceProtos.FileResourceProto getFileResourceProto(final String fileName, final ReefServiceProtos.FileType type) throws IOException {
+ private static ReefServiceProtos.FileResourceProto getFileResourceProto(final String fileName, final ReefServiceProtos.FileType type) throws IOException {
File file = new File(fileName);
if (file.exists()) {
// It is a local file and can be added.
@@ -151,7 +168,7 @@ final class JobSubmissionHelper {
* @return
* @throws IOException
*/
- private File toJar(final File file) throws IOException {
+ private static File toJar(final File file) throws IOException {
final File tempFolder = Files.createTempDirectory("reef-tmp-tempFolder").toFile();
final File jarFile = File.createTempFile(file.getCanonicalFile().getName(), ".jar", tempFolder);
LOG.log(Level.FINEST, "Adding contents of folder {0} to {1}", new Object[]{file, jarFile});