You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@myriad.apache.org by da...@apache.org on 2016/05/09 18:55:18 UTC
incubator-myriad git commit: Made JVM URI,
Path and JAVA_LIBRARY_PATH configurable for executors JIRA:
[Myriad-196] https://issues.apache.org/jira/browse/MYRIAD-196 Pull Request:
Closes #65 Author: Mohit Soni
Repository: incubator-myriad
Updated Branches:
refs/heads/master 1daad6324 -> f1bc1d5ee
Made JVM URI, Path and JAVA_LIBRARY_PATH configurable for executors
JIRA:
[Myriad-196] https://issues.apache.org/jira/browse/MYRIAD-196
Pull Request:
Closes #65
Author:
Mohit Soni <mo...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/f1bc1d5e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/f1bc1d5e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/f1bc1d5e
Branch: refs/heads/master
Commit: f1bc1d5ee5cbf0a7e93c7922760fb4b2a973e519
Parents: 1daad63
Author: Mohit Soni <mo...@gmail.com>
Authored: Sun May 1 00:52:54 2016 -0700
Committer: darinj <da...@apache.org>
Committed: Mon May 9 14:46:19 2016 -0400
----------------------------------------------------------------------
.../MyriadExecutorConfiguration.java | 14 ++++++++++
.../apache/myriad/scheduler/TaskFactory.java | 27 +++++++++-----------
.../main/resources/myriad-config-default.yml | 3 +++
3 files changed, 29 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/f1bc1d5e/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
index 4479725..6afc086 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
@@ -44,6 +44,16 @@ public class MyriadExecutorConfiguration {
@JsonSerialize(using = OptionalSerializerString.class)
private String nodeManagerUri;
+ /**
+ * Download URL for JRE.
+ * Ex: jvmUri: http://www.apache.org/myriad/jre-1.8.99.tar.gz
+ * If provided, Mesos fetcher will be used to download and extract it
+ * inside sandbox.
+ */
+ @JsonProperty
+ @JsonSerialize(using = OptionalSerializerString.class)
+ private String jvmUri;
+
public Optional<Double> getJvmMaxMemoryMB() {
return Optional.fromNullable(jvmMaxMemoryMB);
}
@@ -55,4 +65,8 @@ public class MyriadExecutorConfiguration {
public Optional<String> getNodeManagerUri() {
return Optional.fromNullable(nodeManagerUri);
}
+
+ public Optional<String> getJvmUri() {
+ return Optional.fromNullable(jvmUri);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/f1bc1d5e/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
index 8412eba..ecf0276 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
@@ -20,22 +20,8 @@ package org.apache.myriad.scheduler;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Random;
-import javax.inject.Inject;
-import org.apache.mesos.Protos.CommandInfo;
+import org.apache.mesos.Protos.*;
import org.apache.mesos.Protos.CommandInfo.URI;
-import org.apache.mesos.Protos.ExecutorID;
-import org.apache.mesos.Protos.ExecutorInfo;
-import org.apache.mesos.Protos.FrameworkID;
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.Resource;
-import org.apache.mesos.Protos.TaskID;
-import org.apache.mesos.Protos.TaskInfo;
-import org.apache.mesos.Protos.Value;
import org.apache.mesos.Protos.Value.Range;
import org.apache.myriad.configuration.MyriadConfiguration;
import org.apache.myriad.configuration.MyriadExecutorConfiguration;
@@ -43,6 +29,9 @@ import org.apache.myriad.state.NodeTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.inject.Inject;
+import java.util.*;
+
/**
* Creates Tasks based on mesos offers
@@ -137,6 +126,13 @@ public interface TaskFactory {
CommandInfo.Builder commandInfo = CommandInfo.newBuilder();
String cmd;
+ if (myriadExecutorConfiguration.getJvmUri().isPresent()) {
+ final String jvmRemoteUri = myriadExecutorConfiguration.getJvmUri().get();
+ LOGGER.info("Getting JRE distribution from:" + jvmRemoteUri);
+ URI jvmUri = URI.newBuilder().setValue(jvmRemoteUri).setExtract(true).build();
+ commandInfo.addUris(jvmUri);
+ }
+
if (myriadExecutorConfiguration.getNodeManagerUri().isPresent()) {
//Both FrameworkUser and FrameworkSuperuser to get all of the directory permissions correct.
if (!(cfg.getFrameworkUser().isPresent() && cfg.getFrameworkSuperUser().isPresent())) {
@@ -166,6 +162,7 @@ public interface TaskFactory {
commandInfo.setUser(cfg.getFrameworkUser().get());
}
}
+
return commandInfo.build();
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/f1bc1d5e/myriad-scheduler/src/main/resources/myriad-config-default.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/resources/myriad-config-default.yml b/myriad-scheduler/src/main/resources/myriad-config-default.yml
index ed4bef9..a90de84 100644
--- a/myriad-scheduler/src/main/resources/myriad-config-default.yml
+++ b/myriad-scheduler/src/main/resources/myriad-config-default.yml
@@ -56,9 +56,12 @@ executor:
#The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid.
#nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz
#path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar
+ #jvmUri: https://downloads.mycompany.com/java/jre-7u76-linux-x64.tar.gz
yarnEnvironment:
YARN_HOME: /usr/local/hadoop
#YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set
#JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary
+ #JAVA_HOME: jre1.7.0_76 # Path to JRE distribution, relative to sandbox directory
+ #JAVA_LIBRARY_PATH: /opt/mycompany/lib
mesosAuthenticationPrincipal:
mesosAuthenticationSecretFilename: