You are viewing a plain text version of this content. The canonical link for it is here.
Posted to s4-commits@incubator.apache.org by mm...@apache.org on 2012/11/22 16:56:53 UTC
git commit: Prevent users from setting -Xmx parameter directly for
spawned yarn processes
Updated Branches:
refs/heads/S4-25 a0214813e -> ba7b94b24
Prevent users from setting -Xmx parameter directly for spawned yarn processes
Project: http://git-wip-us.apache.org/repos/asf/incubator-s4/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s4/commit/ba7b94b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s4/tree/ba7b94b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s4/diff/ba7b94b2
Branch: refs/heads/S4-25
Commit: ba7b94b24fdd9d2ecd5ce12ff130debc4ba6a26e
Parents: a021481
Author: Matthieu Morel <mm...@apache.org>
Authored: Thu Nov 22 16:42:08 2012 +0100
Committer: Matthieu Morel <mm...@apache.org>
Committed: Thu Nov 22 16:42:16 2012 +0100
----------------------------------------------------------------------
.../org/apache/s4/tools/yarn/CommonS4YarnArgs.java | 16 ++++++++-
.../apache/s4/tools/yarn/S4ApplicationMaster.java | 25 ++++++---------
.../org/apache/s4/tools/yarn/S4YarnClient.java | 8 +++--
3 files changed, 29 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ba7b94b2/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
index 222ac5a..9789de5 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
@@ -5,7 +5,9 @@ import java.util.List;
import org.apache.s4.core.Main.InlineConfigParameterConverter;
+import com.beust.jcommander.IParameterValidator;
import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
public class CommonS4YarnArgs {
@@ -53,8 +55,18 @@ public class CommonS4YarnArgs {
List<String> extraNamedParameters = new ArrayList<String>();
// TODO parse JVM parameters that include commas
- @Parameter(names = S4CLIYarnArgs.S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list. The memory is usually configured through "
- + S4_NODE_MEMORY, required = false)
+ @Parameter(names = S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list. The memory parameter -Xmx must be configured through "
+ + S4_NODE_MEMORY, required = false, validateWith = NodeJVMParametersValidator.class)
List<String> extraS4NodeJVMParams = new ArrayList<String>();
+ public static class NodeJVMParametersValidator implements IParameterValidator {
+
+ @Override
+ public void validate(String name, String value) throws ParameterException {
+ if (value.matches(".*-Xmx\\d+.*")) {
+ throw new ParameterException("-Xmx JVM parameter cannot be specified here. You must use the "
+ + S4_NODE_MEMORY + " parameter instead.");
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ba7b94b2/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
index d7dc788..f1efb35 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
@@ -99,34 +99,34 @@ public class S4ApplicationMaster {
// For status update for clients - yet to be implemented
// Hostname of the container
- private String appMasterHostname = "";
+ private final String appMasterHostname = "";
// Port on which the app master listens for status update requests from clients
- private int appMasterRpcPort = 0;
+ private final int appMasterRpcPort = 0;
// Tracking url to which app master publishes info for clients to monitor
- private String appMasterTrackingUrl = "";
+ private final String appMasterTrackingUrl = "";
// Incremental counter for rpc calls to the RM
- private AtomicInteger rmRequestID = new AtomicInteger();
+ private final AtomicInteger rmRequestID = new AtomicInteger();
// Simple flag to denote whether all works is done
private boolean appDone = false;
// Counter for completed containers ( complete denotes successful or failed )
- private AtomicInteger numCompletedContainers = new AtomicInteger();
+ private final AtomicInteger numCompletedContainers = new AtomicInteger();
// Allocated container count so that we know how many containers has the RM
// allocated to us
- private AtomicInteger numAllocatedContainers = new AtomicInteger();
+ private final AtomicInteger numAllocatedContainers = new AtomicInteger();
// Count of failed containers
- private AtomicInteger numFailedContainers = new AtomicInteger();
+ private final AtomicInteger numFailedContainers = new AtomicInteger();
// Count of containers already requested from the RM
// Needed as once requested, we should not request for containers again and again.
// Only request for more if the original requirement changes.
- private AtomicInteger numRequestedContainers = new AtomicInteger();
+ private final AtomicInteger numRequestedContainers = new AtomicInteger();
// Containers to be released
- private CopyOnWriteArrayList<ContainerId> releasedContainers = new CopyOnWriteArrayList<ContainerId>();
+ private final CopyOnWriteArrayList<ContainerId> releasedContainers = new CopyOnWriteArrayList<ContainerId>();
// Launch threads
- private List<Thread> launchThreads = new ArrayList<Thread>();
+ private final List<Thread> launchThreads = new ArrayList<Thread>();
private int containerMemory;
@@ -144,8 +144,6 @@ public class S4ApplicationMaster {
Tools.parseArgs(yarnArgs, args);
- Thread.sleep(10000);
-
S4ApplicationMaster appMaster = new S4ApplicationMaster();
logger.info("Initializing ApplicationMaster with args " + Arrays.toString(args));
appMaster.init();
@@ -499,9 +497,6 @@ public class S4ApplicationMaster {
S4YarnClient.addListElementsToCommandLineBuffer(vargs, null, " ", yarnArgs.extraS4NodeJVMParams);
- if (!yarnArgs.extraS4NodeJVMParams.isEmpty()) {
- vargs.add(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS);
- }
// TODO add memory parameter
// vargs.add("-Xdebug");
// vargs.add("-Xrunjdwp:transport=dt_socket,address=8889,server=y");
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ba7b94b2/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
index a4b5a12..77c120c 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
@@ -120,7 +120,7 @@ public class S4YarnClient extends YarnClientImpl {
private static Logger logger = LoggerFactory.getLogger(S4YarnClient.class);
// Configuration
- private Configuration conf;
+ private final Configuration conf;
S4CLIYarnArgs yarnArgs;
@@ -325,8 +325,10 @@ public class S4YarnClient extends YarnClientImpl {
vargs.add(CommonS4YarnArgs.NB_S4_NODES + " " + String.valueOf(yarnArgs.numContainers));
vargs.add(CommonS4YarnArgs.PRIORITY + " " + String.valueOf(yarnArgs.priority));
- addListElementsToCommandLineBuffer(vargs, CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, ",", yarnArgs.extraS4NodeJVMParams);
- addListElementsToCommandLineBuffer(vargs, CommonS4YarnArgs.EXTRA_MODULES_CLASSES, ",", yarnArgs.extraModulesClasses);
+ addListElementsToCommandLineBuffer(vargs, CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, ",",
+ yarnArgs.extraS4NodeJVMParams);
+ addListElementsToCommandLineBuffer(vargs, CommonS4YarnArgs.EXTRA_MODULES_CLASSES, ",",
+ yarnArgs.extraModulesClasses);
addListElementsToCommandLineBuffer(vargs, "-namedStringParameters", ",", yarnArgs.extraNamedParameters);
vargs.add("-c " + String.valueOf(yarnArgs.cluster));