You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by jo...@apache.org on 2015/11/13 08:27:39 UTC
incubator-reef git commit: [REEF-939] Mesos runtime should have
JobSubmissionDirectoryPrefix
Repository: incubator-reef
Updated Branches:
refs/heads/master 53584db94 -> 835bf945c
[REEF-939] Mesos runtime should have JobSubmissionDirectoryPrefix
This PR adds JobSubmissionDirectoryPrefix for Mesos runtime
to prevent `runmesostests.sh` make many folders in the root of HDFS.
JIRA:
[REEF-939](https://issues.apache.org/jira/browse/REEF-939)
Pull Request:
Closes #634
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/835bf945
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/835bf945
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/835bf945
Branch: refs/heads/master
Commit: 835bf945c167d5db4439a19baaa469ac90b6a9d0
Parents: 53584db
Author: Dongjoon Hyun <do...@apache.org>
Authored: Fri Nov 13 13:53:08 2015 +0900
Committer: John Yang <jo...@apache.org>
Committed: Fri Nov 13 16:25:18 2015 +0900
----------------------------------------------------------------------
.../mesos/driver/MesosDriverConfiguration.java | 7 +++++
.../runtime/mesos/driver/REEFScheduler.java | 9 ++++--
.../JobSubmissionDirectoryPrefix.java | 29 ++++++++++++++++++++
3 files changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/835bf945/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/MesosDriverConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/MesosDriverConfiguration.java b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/MesosDriverConfiguration.java
index 6034b45..fed03a4 100644
--- a/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/MesosDriverConfiguration.java
+++ b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/MesosDriverConfiguration.java
@@ -31,6 +31,7 @@ import org.apache.reef.runtime.common.launch.parameters.LaunchID;
import org.apache.reef.runtime.common.parameters.JVMHeapSlack;
import org.apache.reef.runtime.mesos.MesosClasspathProvider;
import org.apache.reef.runtime.mesos.driver.parameters.MesosMasterIp;
+import org.apache.reef.runtime.mesos.driver.parameters.JobSubmissionDirectoryPrefix;
import org.apache.reef.runtime.mesos.util.HDFSConfigurationConstructor;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
@@ -74,6 +75,11 @@ public final class MesosDriverConfiguration extends ConfigurationModuleBuilder {
*/
public static final RequiredParameter<Integer> SCHEDULER_DRIVER_CAPACITY = new RequiredParameter<>();
+ /**
+ * The client remote identifier.
+ */
+ public static final OptionalParameter<String> JOB_SUBMISSION_DIRECTORY_PREFIX = new OptionalParameter<>();
+
public static final ConfigurationModule CONF = new MesosDriverConfiguration()
.bindImplementation(ResourceLaunchHandler.class, MesosResourceLaunchHandler.class)
.bindImplementation(ResourceReleaseHandler.class, MesosResourceReleaseHandler.class)
@@ -87,6 +93,7 @@ public final class MesosDriverConfiguration extends ConfigurationModuleBuilder {
.bindImplementation(RuntimeClasspathProvider.class, MesosClasspathProvider.class)
.bindNamedParameter(StageConfiguration.Capacity.class, SCHEDULER_DRIVER_CAPACITY)
+ .bindNamedParameter(JobSubmissionDirectoryPrefix.class, JOB_SUBMISSION_DIRECTORY_PREFIX)
.bindNamedParameter(StageConfiguration.StageHandler.class, MesosSchedulerDriverExecutor.class)
.bindImplementation(EStage.class, SingleThreadStage.class)
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/835bf945/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/REEFScheduler.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/REEFScheduler.java b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/REEFScheduler.java
index 68476a0..045bb49 100644
--- a/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/REEFScheduler.java
+++ b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/REEFScheduler.java
@@ -35,6 +35,7 @@ import org.apache.reef.runtime.common.driver.resourcemanager.RuntimeStatusEventI
import org.apache.reef.runtime.common.files.ClasspathProvider;
import org.apache.reef.runtime.common.files.REEFFileNames;
import org.apache.reef.runtime.mesos.driver.parameters.MesosMasterIp;
+import org.apache.reef.runtime.mesos.driver.parameters.JobSubmissionDirectoryPrefix;
import org.apache.reef.runtime.mesos.evaluator.REEFExecutor;
import org.apache.reef.runtime.mesos.util.EvaluatorControl;
import org.apache.reef.runtime.mesos.util.EvaluatorRelease;
@@ -104,6 +105,7 @@ final class REEFScheduler implements Scheduler {
private final MesosRemoteManager mesosRemoteManager;
private final SchedulerDriver mesosMaster;
+ private final String jobSubmissionDirectoryPrefix;
private final EStage<SchedulerDriver> schedulerDriverEStage;
private final Map<String, Offer> offers = new ConcurrentHashMap<>();
@@ -120,11 +122,13 @@ final class REEFScheduler implements Scheduler {
final EStage<SchedulerDriver> schedulerDriverEStage,
final ClasspathProvider classpath,
@Parameter(JobIdentifier.class) final String jobIdentifier,
- @Parameter(MesosMasterIp.class) final String masterIp) {
+ @Parameter(MesosMasterIp.class) final String masterIp,
+ @Parameter(JobSubmissionDirectoryPrefix.class) final String jobSubmissionDirectoryPrefix) {
this.mesosRemoteManager = mesosRemoteManager;
this.reefEventHandlers = reefEventHandlers;
this.executors = executors;
this.fileNames = fileNames;
+ this.jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix;
this.reefTarUri = getReefTarUri(jobIdentifier);
this.classpath = classpath;
this.schedulerDriverEStage = schedulerDriverEStage;
@@ -498,7 +502,8 @@ final class REEFScheduler implements Scheduler {
// Upload REEF_TAR to HDFS
final FileSystem fileSystem = FileSystem.get(new Configuration());
final org.apache.hadoop.fs.Path src = new org.apache.hadoop.fs.Path(REEF_TAR);
- final String reefTarUriValue = fileSystem.getUri().toString() + "/" + jobIdentifier + "/" + REEF_TAR;
+ final String reefTarUriValue = fileSystem.getUri().toString() + this.jobSubmissionDirectoryPrefix + "/" +
+ jobIdentifier + "/" + REEF_TAR;
final org.apache.hadoop.fs.Path dst = new org.apache.hadoop.fs.Path(reefTarUriValue);
fileSystem.copyFromLocalFile(src, dst);
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/835bf945/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/parameters/JobSubmissionDirectoryPrefix.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/parameters/JobSubmissionDirectoryPrefix.java b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/parameters/JobSubmissionDirectoryPrefix.java
new file mode 100644
index 0000000..1ada5a4
--- /dev/null
+++ b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/parameters/JobSubmissionDirectoryPrefix.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.mesos.driver.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * The job submission directory.
+ */
+@NamedParameter(doc = "The job submission directory prefix.", default_value = "/vol1/tmp")
+public final class JobSubmissionDirectoryPrefix implements Name<String> {
+}