You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by nt...@apache.org on 2016/02/01 15:21:49 UTC
[41/49] ignite git commit: Fixed IGNITE-2419 Ignite on YARN do not
handle memory overhead. This closes #414.
Fixed IGNITE-2419 Ignite on YARN do not handle memory overhead. This closes #414.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1945b988
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1945b988
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1945b988
Branch: refs/heads/ignite-2435
Commit: 1945b98849cb52fc297c726fa79a270706135581
Parents: 5969129
Author: Edouard Chevalier <ed...@techmydata.net>
Authored: Mon Feb 1 07:34:59 2016 +0300
Committer: Tikhonov Nikolay <ti...@gmail.com>
Committed: Mon Feb 1 07:34:59 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/yarn/ApplicationMaster.java | 12 +-
.../apache/ignite/yarn/ClusterProperties.java | 144 +++++++++++--------
.../yarn/IgniteApplicationMasterSelfTest.java | 52 +++++++
3 files changed, 140 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1945b988/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
index 755e4e4..b9ab02d 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
@@ -137,8 +137,8 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
+ "cp -r ./libs/* ./ignite/*/libs/ || true && "
+ "./ignite/*/bin/ignite.sh "
+ "./ignite-config.xml"
- + " -J-Xmx" + c.getResource().getMemory() + "m"
- + " -J-Xms" + c.getResource().getMemory() + "m"
+ + " -J-Xmx" + ((int)props.memoryPerNode()) + "m"
+ + " -J-Xms" + ((int)props.memoryPerNode()) + "m"
+ IgniteYarnUtils.YARN_LOG_OUT
));
@@ -178,7 +178,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
// Check that slave satisfies min requirements.
if (cont.getResource().getVirtualCores() < props.cpusPerNode()
- || cont.getResource().getMemory() < props.memoryPerNode()) {
+ || cont.getResource().getMemory() < props.totalMemoryPerNode()) {
log.log(Level.FINE, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}",
new Object[]{cont.getNodeId().getHost(), cont.getResource().getVirtualCores(),
cont.getResource().getMemory()});
@@ -291,7 +291,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
// Resource requirements for worker containers.
Resource capability = Records.newRecord(Resource.class);
- capability.setMemory((int)props.memoryPerNode());
+ capability.setMemory((int)props.totalMemoryPerNode());
capability.setVirtualCores((int)props.cpusPerNode());
for (int i = 0; i < props.instances() - runningCnt; ++i) {
@@ -302,7 +302,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
rmClient.addContainerRequest(containerAsk);
log.log(Level.INFO, "Making request. Memory: {0}, cpu {1}.",
- new Object[]{props.memoryPerNode(), props.cpusPerNode()});
+ new Object[]{props.totalMemoryPerNode(), props.cpusPerNode()});
}
}
@@ -329,7 +329,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
private boolean checkAvailableResource() {
Resource availableRes = rmClient.getAvailableResources();
- return availableRes == null || availableRes.getMemory() >= props.memoryPerNode()
+ return availableRes == null || availableRes.getMemory() >= props.totalMemoryPerNode()
&& availableRes.getVirtualCores() >= props.cpusPerNode();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1945b988/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
index d040e9f..647aef2 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
@@ -48,6 +48,12 @@ public class ClusterProperties {
/** */
public static final double DEFAULT_MEM_PER_NODE = 2048;
+
+ /**
+ * The minimum memory overhead: overhead is by default 0.1* MEMORY_PER_NODE,
+ * with a minimum of DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE.
+ */
+ public static final double DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE = 384;
/** Cluster name. */
private String clusterName = DEFAULT_CLUSTER_NAME;
@@ -63,6 +69,12 @@ public class ClusterProperties {
/** Memory limit. */
private double memPerNode = DEFAULT_MEM_PER_NODE;
+
+ /** */
+ public static final String IGNITE_MEMORY_OVERHEAD_PER_NODE = "IGNITE_MEMORY_OVERHEAD_PER_NODE";
+
+ /** Memory over head to request yarn. */
+ private double memOverHeadPerNode = 0;
/** */
public static final String IGNITE_NODE_COUNT = "IGNITE_NODE_COUNT";
@@ -185,7 +197,30 @@ public class ClusterProperties {
}
/**
- * @return instance count limit.
+ * @return Memory overhead for requested memory.
+ */
+ public double memoryOverHeadPerNode() {
+ return memOverHeadPerNode;
+ }
+
+ /**
+ * Sets memory overhead requested to YARN.
+ *
+ * @param memOverHeadPerNode Memory over head per node.
+ */
+ public void memoryOverHeadPerNode(double memOverHeadPerNode) {
+ this.memOverHeadPerNode = memOverHeadPerNode;
+ }
+
+ /**
+ * @return Provide the total memory requested to ResourceManagers (memoryPerNode + memoryOverheadPerNode).
+ */
+ public double totalMemoryPerNode(){
+ return memoryPerNode() + memoryOverHeadPerNode();
+ }
+
+ /**
+ * @return Instance count limit.
*/
public double instances() {
return nodeCnt;
@@ -278,7 +313,50 @@ public class ClusterProperties {
}
/**
- * @param config path to config file.
+ * Instantiate a ClusterProperties from a set of properties.
+ *
+ * @param props If {@code null} will be used system properties.
+ * @return Cluster properties.
+ */
+ private static ClusterProperties fromProperties(Properties props) {
+ ClusterProperties prop = new ClusterProperties();
+
+ prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, props, DEFAULT_CLUSTER_NAME);
+
+ prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, props, DEFAULT_CPU_PER_NODE);
+ prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, props, DEFAULT_MEM_PER_NODE);
+ // The minimum memory overhead: overhead is by default 0.1* MEMORY_PER_NODE,
+ // with a minimum of DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE
+ prop.memOverHeadPerNode = getDoubleProperty(IGNITE_MEMORY_OVERHEAD_PER_NODE, props,
+ Math.max( 0.1 * prop.memPerNode, DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE));
+ prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, props, DEFAULT_IGNITE_NODE_COUNT);
+
+ prop.igniteUrl = getStringProperty(IGNITE_URL, props, null);
+ prop.ignitePath = getStringProperty(IGNITE_PATH, props, null);
+ prop.licencePath = getStringProperty(LICENCE_PATH, props, null);
+ prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, props, null);
+ prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, props, DEFAULT_IGNITE_WORK_DIR);
+ prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, props, DEFAULT_IGNITE_LOCAL_WORK_DIR);
+ prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, props, DEFAULT_IGNITE_RELEASES_DIR);
+ prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, props, null);
+ prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, props, null);
+
+ String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, props, null);
+
+ if (pattern != null) {
+ try {
+ prop.hostnameConstraint = Pattern.compile(pattern);
+ }
+ catch (PatternSyntaxException e) {
+ log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern. It will be ignore.", e);
+ }
+ }
+
+ return prop;
+ }
+
+ /**
+ * @param config Path to config file.
* @return Cluster configuration.
*/
public static ClusterProperties from(String config) {
@@ -291,36 +369,7 @@ public class ClusterProperties {
props.load(new FileInputStream(config));
}
- ClusterProperties prop = new ClusterProperties();
-
- prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, props, DEFAULT_CLUSTER_NAME);
-
- prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, props, DEFAULT_CPU_PER_NODE);
- prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, props, DEFAULT_MEM_PER_NODE);
- prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, props, DEFAULT_IGNITE_NODE_COUNT);
-
- prop.igniteUrl = getStringProperty(IGNITE_URL, props, null);
- prop.ignitePath = getStringProperty(IGNITE_PATH, props, null);
- prop.licencePath = getStringProperty(LICENCE_PATH, props, null);
- prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, props, null);
- prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, props, DEFAULT_IGNITE_WORK_DIR);
- prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, props, DEFAULT_IGNITE_LOCAL_WORK_DIR);
- prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, props, DEFAULT_IGNITE_RELEASES_DIR);
- prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, props, null);
- prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, props, null);
-
- String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, props, null);
-
- if (pattern != null) {
- try {
- prop.hostnameConstraint = Pattern.compile(pattern);
- }
- catch (PatternSyntaxException e) {
- log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern. It will be ignore.", e);
- }
- }
-
- return prop;
+ return fromProperties(props);
}
catch (IOException e) {
throw new RuntimeException(e);
@@ -331,36 +380,7 @@ public class ClusterProperties {
* @return Cluster configuration.
*/
public static ClusterProperties from() {
- ClusterProperties prop = new ClusterProperties();
-
- prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, null, DEFAULT_CLUSTER_NAME);
-
- prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, null, DEFAULT_CPU_PER_NODE);
- prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, null, DEFAULT_MEM_PER_NODE);
- prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, null, DEFAULT_IGNITE_NODE_COUNT);
-
- prop.igniteUrl = getStringProperty(IGNITE_URL, null, null);
- prop.ignitePath = getStringProperty(IGNITE_PATH, null, null);
- prop.licencePath = getStringProperty(LICENCE_PATH, null, null);
- prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, null, null);
- prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, null, DEFAULT_IGNITE_WORK_DIR);
- prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, null, DEFAULT_IGNITE_LOCAL_WORK_DIR);
- prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, null, DEFAULT_IGNITE_RELEASES_DIR);
- prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, null, null);
- prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, null, null);
-
- String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, null, null);
-
- if (pattern != null) {
- try {
- prop.hostnameConstraint = Pattern.compile(pattern);
- }
- catch (PatternSyntaxException e) {
- log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern. It will be ignore.", e);
- }
- }
-
- return prop;
+ return fromProperties(null);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/1945b988/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java b/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
index 97f6a12..1190313 100644
--- a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
+++ b/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
@@ -103,6 +103,58 @@ public class IgniteApplicationMasterSelfTest extends TestCase {
assertEquals(1024, req.getCapability().getMemory());
}
}
+
+ /**
+ * Tests whether memory overhead is allocated within container memory.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMemoryOverHeadAllocation() throws Exception {
+ appMaster.setRmClient(rmMock);
+ appMaster.setNmClient(new NMMock());
+
+ props.cpusPerNode(2);
+ props.memoryPerNode(1024);
+ props.memoryOverHeadPerNode(512);
+ props.instances(3);
+
+ Thread thread = runAppMaster(appMaster);
+
+ List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000);
+
+ interruptedThread(thread);
+
+ assertEquals(3, contRequests.size());
+
+ for (AMRMClient.ContainerRequest req : contRequests) {
+ assertEquals(2, req.getCapability().getVirtualCores());
+ assertEquals(1024 + 512, req.getCapability().getMemory());
+ }
+ }
+
+ /**
+ * Tests whether memory overhead prevents from allocating container.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMemoryOverHeadPreventAllocation() throws Exception {
+ rmMock.availableRes(new MockResource(1024, 2));
+ appMaster.setRmClient(rmMock);
+ appMaster.setNmClient(new NMMock());
+
+ props.cpusPerNode(2);
+ props.memoryPerNode(1024);
+ props.memoryOverHeadPerNode(512);
+ props.instances(3);
+
+ Thread thread = runAppMaster(appMaster);
+
+ List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000);
+
+ interruptedThread(thread);
+
+ assertEquals(0, contRequests.size());
+ }
/**
* @throws Exception If failed.