You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2018/03/16 20:21:21 UTC
incubator-gobblin git commit: [GOBBLIN-429] Pass jvm options as a
list to child process
Repository: incubator-gobblin
Updated Branches:
refs/heads/master 7e1d8433a -> 0795fa7a0
[GOBBLIN-429] Pass jvm options as a list to child process
Closes #2306 from yukuai518/jvm
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/0795fa7a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/0795fa7a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/0795fa7a
Branch: refs/heads/master
Commit: 0795fa7a07d17957288a3c9cde75dde91e7649be
Parents: 7e1d843
Author: Kuai Yu <ku...@linkedin.com>
Authored: Fri Mar 16 13:21:13 2018 -0700
Committer: Hung Tran <hu...@linkedin.com>
Committed: Fri Mar 16 13:21:13 2018 -0700
----------------------------------------------------------------------
.../cluster/GobblinClusterConfigurationKeys.java | 4 ++--
.../org/apache/gobblin/cluster/SingleTaskLauncher.java | 12 ++++++++----
gobblin-cluster/src/test/resources/BasicCluster.conf | 2 ++
3 files changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/0795fa7a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
index 5719b23..76382e7 100644
--- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
+++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
@@ -37,8 +37,8 @@ public class GobblinClusterConfigurationKeys {
GOBBLIN_CLUSTER_PREFIX + "task.classpath";
public static final String TASK_LOG_CONFIG =
GOBBLIN_CLUSTER_PREFIX + "task.log.config";
- public static final String TASK_JAVA_OPTIONS =
- GOBBLIN_CLUSTER_PREFIX + "task.java.options";
+ public static final String TASK_JVM_OPTIONS =
+ GOBBLIN_CLUSTER_PREFIX + "task.jvm.options";
// General Gobblin Cluster application configuration properties.
public static final String APPLICATION_NAME_OPTION_NAME = "app_name";
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/0795fa7a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java
----------------------------------------------------------------------
diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java
index 078f122..8cbbc00 100644
--- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java
+++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java
@@ -23,6 +23,7 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.lang3.text.StrTokenizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,7 +61,7 @@ class SingleTaskLauncher {
// The -cp parameter list can be very long.
final String completeCmdLine = String.join(" ", command);
- logger.debug("cmd line:\n{}", completeCmdLine);
+ logger.info("cmd line:\n{}", completeCmdLine);
final Process taskProcess = this.processBuilder.start(command);
@@ -88,9 +89,12 @@ class SingleTaskLauncher {
}
private void addJavaOptions() {
- if (sysConfig.hasPath(GobblinClusterConfigurationKeys.TASK_JAVA_OPTIONS)) {
- final String javaOptions = sysConfig.getString(GobblinClusterConfigurationKeys.TASK_JAVA_OPTIONS);
- this.cmd.add(javaOptions);
+ if (sysConfig.hasPath(GobblinClusterConfigurationKeys.TASK_JVM_OPTIONS)) {
+ final String taskJvmOptions = sysConfig.getString(GobblinClusterConfigurationKeys.TASK_JVM_OPTIONS);
+ StrTokenizer tokenizer = new StrTokenizer(taskJvmOptions, ' ', '"');
+ while(tokenizer.hasNext()) {
+ this.cmd.add(tokenizer.next());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/0795fa7a/gobblin-cluster/src/test/resources/BasicCluster.conf
----------------------------------------------------------------------
diff --git a/gobblin-cluster/src/test/resources/BasicCluster.conf b/gobblin-cluster/src/test/resources/BasicCluster.conf
index a42b41d..92016f8 100644
--- a/gobblin-cluster/src/test/resources/BasicCluster.conf
+++ b/gobblin-cluster/src/test/resources/BasicCluster.conf
@@ -21,4 +21,6 @@ gobblin.cluster.workDir=/tmp/gobblinClusterBasicTest/
gobblin.cluster.job.conf.path=${gobblin.cluster.workDir}/jobs
gobblin.cluster.standaloneMode=true
gobblin.cluster.job.executeInSchedulingThread=false
+gobblin.cluster.enableTaskInSeparateProcess=false
+gobblin.cluster.task.jvm.options="-Xms10m -Xmx1g -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20"
jobexecutor.threadpool.size=20