You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by cr...@apache.org on 2014/10/30 20:16:21 UTC
git commit: SAMZA-450; allow log4j.xml to be overridden
Repository: incubator-samza
Updated Branches:
refs/heads/0.8.0 263d92178 -> 8f65fa428
SAMZA-450; allow log4j.xml to be overridden
Project: http://git-wip-us.apache.org/repos/asf/incubator-samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-samza/commit/8f65fa42
Tree: http://git-wip-us.apache.org/repos/asf/incubator-samza/tree/8f65fa42
Diff: http://git-wip-us.apache.org/repos/asf/incubator-samza/diff/8f65fa42
Branch: refs/heads/0.8.0
Commit: 8f65fa428a85e0a5a23054012ba1ecd785c7de03
Parents: 263d921
Author: Chris Riccomini <cr...@criccomi-mn.linkedin.biz>
Authored: Thu Oct 30 12:16:08 2014 -0700
Committer: Chris Riccomini <cr...@criccomi-mn.linkedin.biz>
Committed: Thu Oct 30 12:16:08 2014 -0700
----------------------------------------------------------------------
.../samza/job/local/ThreadJobFactory.scala | 2 +-
samza-shell/src/main/bash/checkpoint-tool.sh | 5 ++---
samza-shell/src/main/bash/kill-yarn-job.sh | 5 ++---
samza-shell/src/main/bash/run-am.sh | 2 +-
samza-shell/src/main/bash/run-class.sh | 20 +++++++++++++-------
samza-shell/src/main/bash/run-container.sh | 2 +-
samza-shell/src/main/bash/run-job.sh | 5 ++---
samza-shell/src/main/bash/run-migration-tool.sh | 5 ++---
8 files changed, 24 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala b/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala
index 766df76..cc46d38 100644
--- a/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala
+++ b/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala
@@ -54,7 +54,7 @@ class ThreadJobFactory extends StreamJobFactory with Logging {
// Give developers a nice friendly warning if they've specified task.opts and are using a threaded job.
config.getTaskOpts match {
- case Some(taskOpts) => warn("%s was specified in config, but is not being used because job is being executed with ThreadJob. You probably want to run %s=%s." format (TASK_JVM_OPTS, STREAM_JOB_FACTORY_CLASS, classOf[ThreadJobFactory].getName))
+ case Some(taskOpts) => warn("%s was specified in config, but is not being used because job is being executed with ThreadJob. You probably want to run %s=%s." format (TASK_JVM_OPTS, STREAM_JOB_FACTORY_CLASS, classOf[ProcessJobFactory].getName))
case _ => None
}
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/checkpoint-tool.sh
----------------------------------------------------------------------
diff --git a/samza-shell/src/main/bash/checkpoint-tool.sh b/samza-shell/src/main/bash/checkpoint-tool.sh
index 5688b59..ccf7db2 100755
--- a/samza-shell/src/main/bash/checkpoint-tool.sh
+++ b/samza-shell/src/main/bash/checkpoint-tool.sh
@@ -16,8 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-if [ -z "$SAMZA_LOG4J_CONFIG" ]; then
- export SAMZA_LOG4J_CONFIG=file:$(dirname $0)/log4j-console.xml
-fi
+# Override log4j.xml to point to log4j-console.xml
+[[ $JAVA_OPTS != *-Dlog4j.configuration* ]] && export JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$(dirname $0)/log4j-console.xml"
exec $(dirname $0)/run-class.sh org.apache.samza.checkpoint.CheckpointTool $@
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/kill-yarn-job.sh
----------------------------------------------------------------------
diff --git a/samza-shell/src/main/bash/kill-yarn-job.sh b/samza-shell/src/main/bash/kill-yarn-job.sh
index d42e086..d2bc37f 100755
--- a/samza-shell/src/main/bash/kill-yarn-job.sh
+++ b/samza-shell/src/main/bash/kill-yarn-job.sh
@@ -16,8 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-if [ -z "$SAMZA_LOG4J_CONFIG" ]; then
- export SAMZA_LOG4J_CONFIG=file:$(dirname $0)/log4j-console.xml
-fi
+# Override log4j.xml to point to log4j-console.xml
+[[ $JAVA_OPTS != *-Dlog4j.configuration* ]] && export JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$(dirname $0)/log4j-console.xml"
exec $(dirname $0)/run-class.sh org.apache.hadoop.yarn.client.cli.ApplicationCLI application -kill $@
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-am.sh
----------------------------------------------------------------------
diff --git a/samza-shell/src/main/bash/run-am.sh b/samza-shell/src/main/bash/run-am.sh
index 3a6d40d..22b0e73 100755
--- a/samza-shell/src/main/bash/run-am.sh
+++ b/samza-shell/src/main/bash/run-am.sh
@@ -17,6 +17,6 @@
# under the License.
# Check if server is set. If not - set server optimization
-[[ $JAVA_OPTS != *-server* ]] && JAVA_OPTS="$JAVA_OPTS -server"
+[[ $JAVA_OPTS != *-server* ]] && export JAVA_OPTS="$JAVA_OPTS -server"
exec $(dirname $0)/run-class.sh org.apache.samza.job.yarn.SamzaAppMaster $@
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-class.sh
----------------------------------------------------------------------
diff --git a/samza-shell/src/main/bash/run-class.sh b/samza-shell/src/main/bash/run-class.sh
index 2395b59..3f7c49a 100755
--- a/samza-shell/src/main/bash/run-class.sh
+++ b/samza-shell/src/main/bash/run-class.sh
@@ -37,6 +37,7 @@ HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HOME/.samza}"
HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
CLASSPATH=$HADOOP_CONF_DIR
GC_LOG_ROTATION_OPTS="-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10241024"
+DEFAULT_LOG4J_FILE=$base_dir/lib/log4j.xml
for file in $base_dir/lib/*.[jw]ar;
do
@@ -77,13 +78,20 @@ function check_and_enable_64_bit_mode {
fi
}
-# Initialize SAMZA_LOG4J_CONFIG if currently undefined
-if [ -z "$SAMZA_LOG4J_CONFIG" ]; then
- export SAMZA_LOG4J_CONFIG=file:$base_dir/lib/log4j.xml
-fi
-
### Inherit JVM_OPTS from task.opts configuration, and initialize defaults ###
+# Check if log4j configuration is specified. If not - set to lib/log4j.xml
+[[ $JAVA_OPTS != *-Dlog4j.configuration* && -f $DEFAULT_LOG4J_FILE ]] && JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$DEFAULT_LOG4J_FILE"
+
+# Check if samza.log.dir is specified. If not - set to environment variable if it is set
+[[ $JAVA_OPTS != *-Dsamza.log.dir* && ! -z "$SAMZA_LOG_DIR" ]] && JAVA_OPTS="$JAVA_OPTS -Dsamza.log.dir=$SAMZA_LOG_DIR"
+
+# Check if samza.container.name is specified. If not - set to environment variable if it is set
+[[ $JAVA_OPTS != *-Dsamza.container.name* && ! -z "$SAMZA_CONTAINER_NAME" ]] && JAVA_OPTS="$JAVA_OPTS -Dsamza.container.name=$SAMZA_CONTAINER_NAME"
+
+# Check if java.io.tmpdir is specified. If not - set to tmp in the base_dir
+[[ $JAVA_OPTS != *-Djava.io.tmpdir* ]] && JAVA_OPTS="$JAVA_OPTS -Djava.io.tmpdir=$JAVA_TEMP_DIR"
+
# Check if a max-heap size is specified. If not - set a 768M heap
[[ $JAVA_OPTS != *-Xmx* ]] && JAVA_OPTS="$JAVA_OPTS -Xmx768M"
@@ -96,7 +104,5 @@ fi
# Check if 64 bit is set. If not - try and set it if it's supported
[[ $JAVA_OPTS != *-d64* ]] && check_and_enable_64_bit_mode
-JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=$SAMZA_LOG4J_CONFIG -Dsamza.log.dir=$SAMZA_LOG_DIR -Dsamza.container.name=$SAMZA_CONTAINER_NAME -Djava.io.tmpdir=$JAVA_TEMP_DIR"
-
echo $JAVA $JAVA_OPTS -cp $CLASSPATH $@
exec $JAVA $JAVA_OPTS -cp $CLASSPATH $@
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-container.sh
----------------------------------------------------------------------
diff --git a/samza-shell/src/main/bash/run-container.sh b/samza-shell/src/main/bash/run-container.sh
index af00eba..366bd79 100755
--- a/samza-shell/src/main/bash/run-container.sh
+++ b/samza-shell/src/main/bash/run-container.sh
@@ -17,6 +17,6 @@
# under the License.
# Check if server is set. If not - set server optimization
-[[ $JAVA_OPTS != *-server* ]] && JAVA_OPTS="$JAVA_OPTS -server"
+[[ $JAVA_OPTS != *-server* ]] && export JAVA_OPTS="$JAVA_OPTS -server"
exec $(dirname $0)/run-class.sh org.apache.samza.container.SamzaContainer $@
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-job.sh
----------------------------------------------------------------------
diff --git a/samza-shell/src/main/bash/run-job.sh b/samza-shell/src/main/bash/run-job.sh
index d43b556..346731e 100755
--- a/samza-shell/src/main/bash/run-job.sh
+++ b/samza-shell/src/main/bash/run-job.sh
@@ -16,8 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-if [ -z "$SAMZA_LOG4J_CONFIG" ]; then
- export SAMZA_LOG4J_CONFIG=file:$(dirname $0)/log4j-console.xml
-fi
+# Override log4j.xml to point to log4j-console.xml
+[[ $JAVA_OPTS != *-Dlog4j.configuration* ]] && export JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$(dirname $0)/log4j-console.xml"
exec $(dirname $0)/run-class.sh org.apache.samza.job.JobRunner $@
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-migration-tool.sh
----------------------------------------------------------------------
diff --git a/samza-shell/src/main/bash/run-migration-tool.sh b/samza-shell/src/main/bash/run-migration-tool.sh
index 7ac7f27..ae7042b 100644
--- a/samza-shell/src/main/bash/run-migration-tool.sh
+++ b/samza-shell/src/main/bash/run-migration-tool.sh
@@ -16,8 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-if [ -z "$SAMZA_LOG4J_CONFIG" ]; then
- export SAMZA_LOG4J_CONFIG=file:$(dirname $0)/log4j-console.xml
-fi
+# Override log4j.xml to point to log4j-console.xml
+[[ $JAVA_OPTS != *-Dlog4j.configuration* ]] && export JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$(dirname $0)/log4j-console.xml"
exec $(dirname $0)/run-class.sh org.apache.samza.util.CheckpointMigrationTool $@