You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by il...@apache.org on 2019/03/19 16:59:59 UTC
[ignite] branch master updated: IGNITE-11508 Add IGNITE_YARN_QUEUE
property to override YARN queue name - Fixes #6302.
This is an automated email from the ASF dual-hosted git repository.
ilyak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 2efa886 IGNITE-11508 Add IGNITE_YARN_QUEUE property to override YARN queue name - Fixes #6302.
2efa886 is described below
commit 2efa886b9a1f8d138a1aab48315bb111695a9667
Author: Aravinda Reddy <ar...@gmail.com>
AuthorDate: Tue Mar 19 19:59:35 2019 +0300
IGNITE-11508 Add IGNITE_YARN_QUEUE property to override YARN queue name - Fixes #6302.
Signed-off-by: Ilya Kasnacheev <il...@gmail.com>
---
.../org/apache/ignite/yarn/ClusterProperties.java | 65 +++++++++++++++-------
.../org/apache/ignite/yarn/IgniteYarnClient.java | 4 +-
.../yarn/IgniteApplicationMasterSelfTest.java | 14 +++++
3 files changed, 61 insertions(+), 22 deletions(-)
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
index 75ad27a..9af3d1e 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
@@ -48,7 +48,7 @@ public class ClusterProperties {
/** */
public static final double DEFAULT_MEM_PER_NODE = 2048;
-
+
/**
* The minimum memory overhead: overhead is by default 0.1* MEMORY_PER_NODE,
* with a minimum of DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE.
@@ -69,7 +69,7 @@ public class ClusterProperties {
/** Memory limit. */
private double memPerNode = DEFAULT_MEM_PER_NODE;
-
+
/** */
public static final String IGNITE_MEMORY_OVERHEAD_PER_NODE = "IGNITE_MEMORY_OVERHEAD_PER_NODE";
@@ -154,6 +154,15 @@ public class ClusterProperties {
/** Url to ignite config. */
private Pattern hostnameConstraint = null;
+ /** Ignite Yarn Queue */
+ public static final String IGNITE_YARN_QUEUE = "IGNITE_YARN_QUEUE";
+
+ /** Ignite Yarn default Queue */
+ public static final String DEFAULT_IGNITE_YARN_QUEUE = "default";
+
+ /** Path to users libs. */
+ private String yarnQueue = DEFAULT_IGNITE_YARN_QUEUE;
+
/** */
public ClusterProperties() {
// No-op.
@@ -193,33 +202,33 @@ public class ClusterProperties {
* @param mem Memory.
*/
public void memoryPerNode(double mem) {
- this.memPerNode = mem;
+ this.memPerNode = mem;
}
/**
* @return Memory overhead for requested memory.
*/
public double memoryOverHeadPerNode() {
- return memOverHeadPerNode;
- }
+ return memOverHeadPerNode;
+ }
/**
* Sets memory overhead requested to YARN.
*
* @param memOverHeadPerNode Memory over head per node.
*/
- public void memoryOverHeadPerNode(double memOverHeadPerNode) {
- this.memOverHeadPerNode = memOverHeadPerNode;
- }
-
- /**
- * @return Provide the total memory requested to ResourceManagers (memoryPerNode + memoryOverheadPerNode).
- */
- public double totalMemoryPerNode(){
- return memoryPerNode() + memoryOverHeadPerNode();
- }
-
- /**
+ public void memoryOverHeadPerNode(double memOverHeadPerNode) {
+ this.memOverHeadPerNode = memOverHeadPerNode;
+ }
+
+ /**
+ * @return Provide the total memory requested to ResourceManagers (memoryPerNode + memoryOverheadPerNode).
+ */
+ public double totalMemoryPerNode(){
+ return memoryPerNode() + memoryOverHeadPerNode();
+ }
+
+ /**
* @return Instance count limit.
*/
public double instances() {
@@ -243,6 +252,15 @@ public class ClusterProperties {
}
/**
+ * Sets Yarn Queue
+ *
+ * @param queue queue name.
+ */
+ public void yarnQueue(String queue) {
+ this.yarnQueue = queue;
+ }
+
+ /**
* @return Ignite version.
*/
public String igniteUrl() {
@@ -313,13 +331,18 @@ public class ClusterProperties {
}
/**
+ * @return Yarn Queue
+ */
+ public String yarnQueue() { return yarnQueue; }
+
+ /**
* Instantiate a ClusterProperties from a set of properties.
*
* @param props If {@code null} will be used system properties.
* @return Cluster properties.
*/
private static ClusterProperties fromProperties(Properties props) {
- ClusterProperties prop = new ClusterProperties();
+ ClusterProperties prop = new ClusterProperties();
prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, props, DEFAULT_CLUSTER_NAME);
@@ -340,6 +363,7 @@ public class ClusterProperties {
prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, props, DEFAULT_IGNITE_RELEASES_DIR);
prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, props, null);
prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, props, null);
+ prop.yarnQueue = getStringProperty(IGNITE_YARN_QUEUE, props, DEFAULT_IGNITE_YARN_QUEUE);
String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, props, null);
@@ -354,7 +378,7 @@ public class ClusterProperties {
return prop;
}
-
+
/**
* @param config Path to config file.
* @return Cluster configuration.
@@ -407,6 +431,7 @@ public class ClusterProperties {
envs.put(IGNITE_RELEASES_DIR, toEnvVal(igniteReleasesDir));
envs.put(IGNITE_CONFIG_XML, toEnvVal(igniteCfg));
envs.put(IGNITE_USERS_LIBS, toEnvVal(userLibs));
+ envs.put(IGNITE_YARN_QUEUE, toEnvVal(yarnQueue));
if (hostnameConstraint != null)
envs.put(IGNITE_HOSTNAME_CONSTRAINT, toEnvVal(hostnameConstraint.pattern()));
@@ -455,4 +480,4 @@ public class ClusterProperties {
private String toEnvVal(Object val) {
return val == null ? EMPTY_STRING : val.toString();
}
-}
\ No newline at end of file
+}
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
index 2a9a53e..869a0d0 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
@@ -141,7 +141,7 @@ public class IgniteYarnClient {
appContext.setApplicationName("ignition"); // application name
appContext.setAMContainerSpec(amContainer);
appContext.setResource(capability);
- appContext.setQueue("default"); // queue
+ appContext.setQueue(props.yarnQueue()); // queue
// Submit application
ApplicationId appId = appContext.getApplicationId();
@@ -211,4 +211,4 @@ public class IgniteYarnClient {
Environment.PWD.$() + File.separator + "*",
File.pathSeparator);
}
-}
\ No newline at end of file
+}
diff --git a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java b/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
index 345baf7..c29dde3 100644
--- a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
+++ b/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
@@ -295,6 +295,20 @@ public class IgniteApplicationMasterSelfTest {
}
/**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testQueue() throws Exception {
+ // Default Queue check
+ Map<String, String> result = props.toEnvs();
+ assertEquals(ClusterProperties.DEFAULT_IGNITE_YARN_QUEUE, result.get(ClusterProperties.IGNITE_YARN_QUEUE));
+
+ props.yarnQueue("ignite");
+ result = props.toEnvs();
+ assertEquals("ignite", result.get(ClusterProperties.IGNITE_YARN_QUEUE));
+ }
+
+ /**
* @param host Host.
* @param cpu Cpu count.
* @param mem Memory.