You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by cr...@apache.org on 2013/11/26 17:16:33 UTC
git commit: SAMZA-76;
switch SAMZA_OPTS to JAVA_OPTS, and make default container size 1G
with an Xmx of 768.
Updated Branches:
refs/heads/master 8028d8deb -> 65fbc6866
SAMZA-76; switch SAMZA_OPTS to JAVA_OPTS, and make default container size 1G with an Xmx of 768.
Project: http://git-wip-us.apache.org/repos/asf/incubator-samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-samza/commit/65fbc686
Tree: http://git-wip-us.apache.org/repos/asf/incubator-samza/tree/65fbc686
Diff: http://git-wip-us.apache.org/repos/asf/incubator-samza/diff/65fbc686
Branch: refs/heads/master
Commit: 65fbc6866e7fd1403a64d3048ac2f0e628f4d0b7
Parents: 8028d8d
Author: Chris Riccomini <cr...@gmail.com>
Authored: Tue Nov 26 08:16:29 2013 -0800
Committer: Chris Riccomini <cr...@gmail.com>
Committed: Tue Nov 26 08:16:29 2013 -0800
----------------------------------------------------------------------
.../documentation/0.7.0/jobs/configuration-table.html | 7 ++++++-
.../org/apache/samza/config/ShellCommandConfig.scala | 2 +-
.../org/apache/samza/job/ShellCommandBuilder.scala | 2 +-
samza-shell/src/main/bash/run-class.sh | 10 +++++-----
.../samza/job/yarn/SamzaAppMasterTaskManager.scala | 2 +-
.../main/scala/org/apache/samza/job/yarn/YarnJob.scala | 13 ++++++++++---
6 files changed, 24 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/docs/learn/documentation/0.7.0/jobs/configuration-table.html
----------------------------------------------------------------------
diff --git a/docs/learn/documentation/0.7.0/jobs/configuration-table.html b/docs/learn/documentation/0.7.0/jobs/configuration-table.html
index 41353b2..7232e02 100644
--- a/docs/learn/documentation/0.7.0/jobs/configuration-table.html
+++ b/docs/learn/documentation/0.7.0/jobs/configuration-table.html
@@ -185,10 +185,15 @@
</tr>
<tr>
<td>yarn.container.memory.mb</td>
- <td>768</td>
+ <td>1024</td>
<td>How much memory to ask for (per-container), when Samza is starting a YARN container.</td>
</tr>
<tr>
+ <td>yarn.am.container.memory.mb</td>
+ <td>1024</td>
+ <td>How much memory to ask for (per-application-master), when Samza is starting a YARN container.</td>
+ </tr>
+ <tr>
<td>yarn.container.count</td>
<td>1</td>
<td>How many containers to start when a Samza job is started in YARN. Partitions are divided evenly among the containers.</td>
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala b/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala
index 27ba5e4..b6efe46 100644
--- a/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala
+++ b/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala
@@ -38,7 +38,7 @@ object ShellCommandConfig {
/**
* Arguments to be passed to the processing running the TaskRunner (or equivalent, for non JVM languages).
*/
- val ENV_SAMZA_OPTS = "SAMZA_OPTS"
+ val ENV_JAVA_OPTS = "JAVA_OPTS"
val COMMAND_SHELL_EXECUTE = "task.execute"
val TASK_JVM_OPTS = "task.opts"
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala b/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala
index 09b2d22..c67e46d 100644
--- a/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala
+++ b/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala
@@ -34,6 +34,6 @@ class ShellCommandBuilder extends CommandBuilder {
ShellCommandConfig.ENV_CONTAINER_NAME -> name,
ShellCommandConfig.ENV_PARTITION_IDS -> parts,
ShellCommandConfig.ENV_CONFIG -> JsonConfigSerializer.toJson(config),
- ShellCommandConfig.ENV_SAMZA_OPTS -> config.getTaskOpts.getOrElse(""))
+ ShellCommandConfig.ENV_JAVA_OPTS -> config.getTaskOpts.getOrElse(""))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-shell/src/main/bash/run-class.sh
----------------------------------------------------------------------
diff --git a/samza-shell/src/main/bash/run-class.sh b/samza-shell/src/main/bash/run-class.sh
index b593765..f4124e5 100755
--- a/samza-shell/src/main/bash/run-class.sh
+++ b/samza-shell/src/main/bash/run-class.sh
@@ -54,12 +54,12 @@ if [ -z "$SAMZA_CONTAINER_NAME" ]; then
SAMZA_CONTAINER_NAME="undefined-samza-container-name"
fi
-if [ -z "$SAMZA_OPTS" ]; then
- SAMZA_OPTS="-Xmx160M -XX:+PrintGCDateStamps -Xloggc:$SAMZA_LOG_DIR/gc.log -Dsamza.log.dir=$SAMZA_LOG_DIR -Dsamza.container.name=$SAMZA_CONTAINER_NAME"
+if [ -z "$JAVA_OPTS" ]; then
+ JAVA_OPTS="-Xmx768M -XX:+PrintGCDateStamps -Xloggc:$SAMZA_LOG_DIR/gc.log -Dsamza.log.dir=$SAMZA_LOG_DIR -Dsamza.container.name=$SAMZA_CONTAINER_NAME"
if [ -f $base_dir/lib/log4j.xml ]; then
- SAMZA_OPTS="$SAMZA_OPTS -Dlog4j.configuration=file:$base_dir/lib/log4j.xml"
+ JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$base_dir/lib/log4j.xml"
fi
fi
-echo $JAVA $SAMZA_OPTS -cp $CLASSPATH $@
-exec $JAVA $SAMZA_OPTS -cp $CLASSPATH $@
+echo $JAVA $JAVA_OPTS -cp $CLASSPATH $@
+exec $JAVA $JAVA_OPTS -cp $CLASSPATH $@
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala
----------------------------------------------------------------------
diff --git a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala
index 80d2972..9f4db17 100644
--- a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala
+++ b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala
@@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.client.api.NMClient
import org.apache.hadoop.yarn.client.api.impl.NMClientImpl
object SamzaAppMasterTaskManager {
- val DEFAULT_CONTAINER_MEM = 256
+ val DEFAULT_CONTAINER_MEM = 1024
val DEFAULT_CPU_CORES = 1
val DEFAULT_CONTAINER_RETRY_COUNT = 8
val DEFAULT_CONTAINER_RETRY_WINDOW_MS = 300000
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala
----------------------------------------------------------------------
diff --git a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala
index b79c0aa..34af73b 100644
--- a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala
+++ b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala
@@ -37,17 +37,24 @@ import org.apache.samza.config.YarnConfig
import org.apache.samza.config.ShellCommandConfig
import org.apache.samza.SamzaException
+
+object YarnJob {
+ val DEFAULT_AM_CONTAINER_MEM = 1024
+}
+
/**
* Starts the application manager
*/
class YarnJob(config: Config, hadoopConfig: Configuration) extends StreamJob {
+ import YarnJob._
+
val client = new ClientHelper(hadoopConfig)
var appId: Option[ApplicationId] = None
-
+
def submit: YarnJob = {
appId = client.submitApplication(
new Path(config.getPackagePath.getOrElse(throw new SamzaException("No YARN package path defined in config."))),
- config.getAMContainerMaxMemoryMb.getOrElse(512),
+ config.getAMContainerMaxMemoryMb.getOrElse(DEFAULT_AM_CONTAINER_MEM),
1,
List(
"export SAMZA_LOG_DIR=%s && ln -sfn %s logs && exec ./__package/bin/run-am.sh 1>logs/%s 2>logs/%s"
@@ -55,7 +62,7 @@ class YarnJob(config: Config, hadoopConfig: Configuration) extends StreamJob {
Some(Map(
ShellCommandConfig.ENV_CONFIG -> Util.envVarEscape(JsonConfigSerializer.toJson(config)),
ShellCommandConfig.ENV_CONTAINER_NAME -> Util.envVarEscape("application-master"),
- ShellCommandConfig.ENV_SAMZA_OPTS -> Util.envVarEscape(config.getAmOpts.getOrElse("")))),
+ ShellCommandConfig.ENV_JAVA_OPTS -> Util.envVarEscape(config.getAmOpts.getOrElse("")))),
Some("%s_%s" format (config.getName.get, config.getJobId.getOrElse(1))))
this