You are viewing a plain text version of this content. The canonical link for it is here.
Posted to s4-commits@incubator.apache.org by mm...@apache.org on 2012/11/26 16:40:48 UTC
git commit: Improve resource specification - as memory for container
parameter: Xmx parameter = 0.8 * the arg - this can be overriden by the user
in container vm parameters (for app master and S4 nodes)
Updated Branches:
refs/heads/S4-25 ba7b94b24 -> 830f5dfa8
Improve resource specification
- as memory for container parameter: Xmx parameter = 0.8 * the arg
- this can be overriden by the user in container vm parameters (for app master and S4 nodes)
Project: http://git-wip-us.apache.org/repos/asf/incubator-s4/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s4/commit/830f5dfa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s4/tree/830f5dfa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s4/diff/830f5dfa
Branch: refs/heads/S4-25
Commit: 830f5dfa896a92f0362a8405fcbe898084fb2d0b
Parents: ba7b94b
Author: Matthieu Morel <mm...@apache.org>
Authored: Mon Nov 26 17:07:35 2012 +0100
Committer: Matthieu Morel <mm...@apache.org>
Committed: Mon Nov 26 17:14:46 2012 +0100
----------------------------------------------------------------------
.../org/apache/s4/tools/yarn/CommonS4YarnArgs.java | 22 ++-----
.../apache/s4/tools/yarn/S4ApplicationMaster.java | 2 +
.../org/apache/s4/tools/yarn/S4CLIYarnArgs.java | 13 +++-
.../org/apache/s4/tools/yarn/S4YarnClient.java | 5 +-
.../main/java/org/apache/s4/tools/yarn/Utils.java | 49 +++++++++++++++
.../java/org/apache/s4/tools/yarn/TestUtils.java | 21 ++++++
.../apache/s4/tools/yarn/TestYarnDeployment.java | 1 +
7 files changed, 94 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/830f5dfa/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
index 9789de5..926b364 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
@@ -5,19 +5,19 @@ import java.util.List;
import org.apache.s4.core.Main.InlineConfigParameterConverter;
-import com.beust.jcommander.IParameterValidator;
import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
public class CommonS4YarnArgs {
+ public static final String JAVA_MEMORY_ALLOCATION_NOTICE = "For a Java process, like an S4 node, the total amount of memory is : java heap size + rest of process memory, including thread stack size, direct memory etc... By default, the S4 node, as a Java process, will be started with a java heap with a size "
+ + Utils.CONTAINER_MEMORY_REDUCTION_FACTOR + " times the container size. Can be overriden in ";
public static final String NAMED_STRING_PARAMETERS = "-namedStringParameters";
public static final String EXTRA_MODULES_CLASSES = "-extraModulesClasses";
public static final String PRIORITY = "-priority";
- public static final String S4_NODE_MEMORY = "-s4NodeMemory";
+ public static final String S4_NODE_CONTAINER_MEMORY = "-s4NodeContainerMemory";
public static final String S4_NODE_JVM_PARAMETERS = "-s4NodeJVMParameters";
@@ -30,7 +30,8 @@ public class CommonS4YarnArgs {
@Parameter(names = "-zk", description = "S4 Zookeeper cluster manager connection string", required = true)
String zkString;
- @Parameter(names = { S4_NODE_MEMORY, "-container_memory" }, description = "YARN parameter: Amount of memory in MB to be requested to host the S4 node", required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
+ @Parameter(names = { S4_NODE_CONTAINER_MEMORY, "-container_memory" }, description = "YARN parameter: Amount of memory in MB to be requested to host the S4 node. "
+ + JAVA_MEMORY_ALLOCATION_NOTICE + S4_NODE_JVM_PARAMETERS, required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
int containerMemory = 256;
@Parameter(names = PRIORITY, description = "YARN parameter: Application priority", required = false)
@@ -55,18 +56,7 @@ public class CommonS4YarnArgs {
List<String> extraNamedParameters = new ArrayList<String>();
// TODO parse JVM parameters that include commas
- @Parameter(names = S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list. The memory parameter -Xmx must be configured through "
- + S4_NODE_MEMORY, required = false, validateWith = NodeJVMParametersValidator.class)
+ @Parameter(names = S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list.", required = false)
List<String> extraS4NodeJVMParams = new ArrayList<String>();
- public static class NodeJVMParametersValidator implements IParameterValidator {
-
- @Override
- public void validate(String name, String value) throws ParameterException {
- if (value.matches(".*-Xmx\\d+.*")) {
- throw new ParameterException("-Xmx JVM parameter cannot be specified here. You must use the "
- + S4_NODE_MEMORY + " parameter instead.");
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/830f5dfa/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
index f1efb35..59de9b7 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
@@ -178,6 +178,8 @@ public class S4ApplicationMaster {
*/
public void init() throws IOException {
+ containerMemory = Utils.extractMemoryParam(yarnArgs.containerMemory, yarnArgs.extraS4NodeJVMParams);
+
Map<String, String> envs = System.getenv();
appAttemptID = Records.newRecord(ApplicationAttemptId.class);
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/830f5dfa/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4CLIYarnArgs.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4CLIYarnArgs.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4CLIYarnArgs.java
index 2203fc0..3fabf31 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4CLIYarnArgs.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4CLIYarnArgs.java
@@ -1,5 +1,8 @@
package org.apache.s4.tools.yarn;
+import java.util.ArrayList;
+import java.util.List;
+
import com.beust.jcommander.IParameterValidator;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
@@ -8,7 +11,9 @@ import com.beust.jcommander.Parameters;
@Parameters(separators = "=")
class S4CLIYarnArgs extends CommonS4YarnArgs {
- public static final String S4_YARN_MASTER_MEMORY = "-s4YarnMasterMemory";
+ private static final String S4_YARN_APP_MASTER_JVM_PARAMETERS = "-s4YarnAppMasterJVMParameters";
+
+ public static final String S4_YARN_MASTER_MEMORY = "-s4YarnMasterContainerMemory";
public static final String QUEUE = "-queue";
@@ -35,9 +40,13 @@ class S4CLIYarnArgs extends CommonS4YarnArgs {
@Parameter(names = "-timeout", description = "YARN parameter: Application timeout in milliseconds (default is: -1 = no timeout)", required = false)
int timeout = -1;
- @Parameter(names = { "-master_memory", S4_YARN_MASTER_MEMORY }, description = "YARN parameter: Amount of memory in MB to be requested to run the application master", required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
+ @Parameter(names = { "-master_memory", S4_YARN_MASTER_MEMORY }, description = "YARN parameter: Amount of memory in MB to be requested to run the application master."
+ + JAVA_MEMORY_ALLOCATION_NOTICE + S4_YARN_APP_MASTER_JVM_PARAMETERS, required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
int masterMemory = 256;
+ @Parameter(names = { S4_YARN_APP_MASTER_JVM_PARAMETERS }, description = "JVM parameters for the S4 YARN application master process")
+ List<String> appMasterJVMParameters = new ArrayList<String>();
+
@Parameter(names = "-log_properties", description = "YARN parameter: log4j.properties file", required = false)
String logProperties = "";
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/830f5dfa/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
index 77c120c..254b7e8 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
@@ -138,6 +138,7 @@ public class S4YarnClient extends YarnClientImpl {
logger.info("S4YarnClient args = " + Arrays.toString(args));
Tools.parseArgs(yarnArgs, args);
+
boolean result = false;
try {
@@ -181,6 +182,8 @@ public class S4YarnClient extends YarnClientImpl {
public S4YarnClient(S4CLIYarnArgs yarnArgs, Configuration conf) throws Exception {
this.yarnArgs = yarnArgs;
this.conf = conf;
+ amMemory = Utils.extractMemoryParam(yarnArgs.containerMemory, yarnArgs.appMasterJVMParameters);
+
init(this.conf);
}
@@ -321,7 +324,7 @@ public class S4YarnClient extends YarnClientImpl {
// Set Application Master class name
vargs.add(S4ApplicationMaster.class.getName());
// Set params for Application Master
- vargs.add(CommonS4YarnArgs.S4_NODE_MEMORY + " " + String.valueOf(yarnArgs.containerMemory));
+ vargs.add(CommonS4YarnArgs.S4_NODE_CONTAINER_MEMORY + " " + String.valueOf(yarnArgs.containerMemory));
vargs.add(CommonS4YarnArgs.NB_S4_NODES + " " + String.valueOf(yarnArgs.numContainers));
vargs.add(CommonS4YarnArgs.PRIORITY + " " + String.valueOf(yarnArgs.priority));
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/830f5dfa/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/Utils.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/Utils.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/Utils.java
new file mode 100644
index 0000000..7dccb16
--- /dev/null
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/Utils.java
@@ -0,0 +1,49 @@
+package org.apache.s4.tools.yarn;
+
+import java.util.List;
+import java.util.regex.Pattern;
+
+public class Utils {
+
+ /**
+ * The container memory reduction factor as used in: {@link CommonS4YarnArgs#JAVA_MEMORY_ALLOCATION_NOTICE}
+ *
+ */
+ public static final double CONTAINER_MEMORY_REDUCTION_FACTOR = 0.8;
+
+ /**
+ * Extracts Xmx JVM arg by evaluating container memory arg and JVM overriding args
+ *
+ * @param containerMemoryArg
+ * The container memory parameter, in MB
+ * @param jvmParamsArg
+ * The list of JVM parameters, which may include Xmx parameter
+ * @return Memory to reserve as Xmx java parameter, in MB
+ */
+ static int extractMemoryParam(int containerMemoryArg, List<String> jvmParamsArg) {
+ int memory = (int) (containerMemoryArg * CONTAINER_MEMORY_REDUCTION_FACTOR);
+ // override maximum heap size
+ for (String appVmArg : jvmParamsArg) {
+ if (appVmArg.trim().matches("-Xmx\\d+[gGmMkK]")) {
+ java.util.regex.Matcher m = Pattern.compile("-Xmx(\\d+)([gGmMkK])").matcher(appVmArg);
+ m.matches();
+ int memoryInKBytes = Integer.valueOf(m.group(1)) * HeapSizeMultiplier.valueOf(m.group(2)).multiplier
+ / 1000000;
+ memory = memoryInKBytes;
+ break;
+ }
+ }
+ return memory;
+ }
+
+}
+
+enum HeapSizeMultiplier {
+ g((int) Math.pow(10, 9)), G((int) Math.pow(10, 9)), m((int) Math.pow(10, 6)), M((int) Math.pow(10, 6)), k(
+ (int) Math.pow(10, 3)), K((int) Math.pow(10, 3));
+ int multiplier;
+
+ HeapSizeMultiplier(int multiplier) {
+ this.multiplier = multiplier;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/830f5dfa/subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestUtils.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestUtils.java b/subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestUtils.java
new file mode 100644
index 0000000..d171525
--- /dev/null
+++ b/subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestUtils.java
@@ -0,0 +1,21 @@
+package org.apache.s4.tools.yarn;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+
+public class TestUtils {
+
+ @Test
+ public void testUtils() {
+ Assert.assertEquals(200,
+ Utils.extractMemoryParam(10, ImmutableList.of(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, "-Xmx200M")));
+ Assert.assertEquals(200,
+ Utils.extractMemoryParam(500, ImmutableList.of(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, "-Xmx200M")));
+ Assert.assertEquals(((int) (500 * Utils.CONTAINER_MEMORY_REDUCTION_FACTOR)),
+ Utils.extractMemoryParam(500, ImmutableList.of(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, "-Xms200M")));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/830f5dfa/subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestYarnDeployment.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestYarnDeployment.java b/subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestYarnDeployment.java
index 771c929..76eaa25 100644
--- a/subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestYarnDeployment.java
+++ b/subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestYarnDeployment.java
@@ -180,4 +180,5 @@ public class TestYarnDeployment extends ZkBasedTest {
client.killApplication(client.getApplicationList().get(0).getApplicationId());
}
+
}