You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2008/10/31 19:57:06 UTC
svn commit: r709533 [1/2] - in /hadoop/core/trunk: ./ src/contrib/chukwa/
src/contrib/chukwa/bin/ src/contrib/chukwa/conf/
src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/
src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/...
Author: omalley
Date: Fri Oct 31 11:57:04 2008
New Revision: 709533
URL: http://svn.apache.org/viewvc?rev=709533&view=rev
Log:
HADOOP-4433. Improve data loader for collecting metrics and log files.
(Eric Yang via omalley)
Added:
hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt
hadoop/core/trunk/src/contrib/chukwa/bin/chukwa
hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh
hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh
hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh
hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh
hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh
hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh
hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh
hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh
hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh
hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh
hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh
hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh
hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh
hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template
hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties
hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/Log4JMetricsContext.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/ErStreamHandler.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/TorqueDataLoader.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/TorqueInfoProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/TorqueTimerTask.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/ExecHelper.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ClusterConfig.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/PidFile.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/chukwa/bin/VERSION
hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh
hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh
hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh
hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh
hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh
hadoop/core/trunk/src/contrib/chukwa/build.xml
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template
hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties
hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties
hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties
hadoop/core/trunk/src/contrib/chukwa/conf/util.properties
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Exec.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Log4JMetricsContext.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Oct 31 11:57:04 2008
@@ -46,9 +46,6 @@
HADOOP-4284. Support filters that apply to all requests, or global filters,
to HttpServer. (Kan Zhang via cdouglas)
- HADOOP-4431. Add versionning/tags to Chukwa Chunk.
- (Jerome Boulon via Johan)
-
HADOOP-4276. Improve the hashing functions and deserialization of the
mapred ID classes. (omalley)
Added: hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt (added)
+++ hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt Fri Oct 31 11:57:04 2008
@@ -0,0 +1,25 @@
+Trunk (unreleased changes)
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ HADOOP-4431. Add versionning/tags to Chukwa Chunk.
+ (Jerome Boulon via Johan)
+
+ HADOOP-4433. Improve data loader for collecting metrics and log files.
+ (Eric Yang via omalley)
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+Release 0.19.0 - Unreleased
+
+ NEW FEATURES
+
+ HADOOP-3719. Initial checkin of Chukwa, which is a data collection and
+ analysis framework. (Jerome Boulon, Andy Konwinski, Ari Rabkin,
+ and Eric Yang)
Modified: hadoop/core/trunk/src/contrib/chukwa/bin/VERSION
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/VERSION?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/VERSION (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/VERSION Fri Oct 31 11:57:04 2008
@@ -1 +1 @@
-0.0.1
+0.1.0
Added: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa Fri Oct 31 11:57:04 2008
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# The Chukwa command script
+#
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# get arguments
+COMMAND=$1
+shift
+
+if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
+ . "${CHUKWA_CONF_DIR}/chukwa-env.sh"
+fi
+
+exec "${CHUKWA_HOME}/bin/$@"
Modified: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh Fri Oct 31 11:57:04 2008
@@ -36,69 +36,70 @@
bin=`cd "$bin"; pwd`
this="$bin/$script"
-. ${bin}/../conf/chukwa-env.sh
-
-export HADOOP_HOME="${HADOOP_HOME:-${bin}/../../../..}"
# the root of the Chukwa installation
-if [ -z $CHUKWA_HOME ] ; then
-CHUKWA_HOME=`dirname "$this"`/..
-export CHUKWA_HOME=`cd $CHUKWA_HOME; pwd`
+export CHUKWA_HOME=`dirname "$this"`/..
+
+#check to see if the conf dir is given as an optional argument
+if [ $# -gt 1 ]
+then
+ if [ "--config" = "$1" ]
+ then
+ shift
+ confdir=$1
+ shift
+ CHUKWA_CONF_DIR=$confdir
+ fi
fi
-chukwaVersion=`cat ${CHUKWA_HOME}/bin/VERSION`
-DEFAULT_CHUKWA_HOME=${CHUKWA_HOME}/logs/
-export CHUKWA_LOG_DIR="${CHUKWA_LOG_DIR:-$DEFAULT_CHUKWA_HOME}"
-if [ ! -d $CHUKWA_LOG_DIR ]; then
- mkdir -p $CHUKWA_LOG_DIR
+#check to see it is specified whether to use the slaves or the
+# masters file
+if [ $# -gt 1 ]
+then
+ if [ "--hosts" = "$1" ]
+ then
+ shift
+ slavesfile=$1
+ shift
+ export CHUKWA_SLAVES="${CHUKWA_CONF_DIR}/$slavesfile"
+ fi
fi
-export chuwaRecordsRepository="/chukwa/repos/demo"
+#check to see if the conf dir is given as an optional argument
+if [ $# -gt 1 ]
+then
+ if [ "--watchdog" = "$1" ]
+ then
+ shift
+ WATCHDOG="true"
+ fi
+fi
-export DATACONFIG=${CHUKWA_HOME}/conf/mdl.xml
-common=`ls ${CHUKWA_HOME}/lib/*.jar`
-export common=`echo ${common} | sed 'y/ /:/'`
+export CHUKWA_LOG_DIR="$CHUKWA_HOME/logs"
-#chukwaCore=${HADOOP_HOME}/build/contrib/chukwa/chukwa-core-${chukwaVersion}.jar
-chukwaCore=${HADOOP_HOME}/build/contrib/chukwa
-if [ -a $chukwaCore ] ; then
- export chukwaCore
-else
- echo ${chukwaCore} does not exist
- export chukwaCore=${CHUKWA_HOME}/chukwa-core-${chukwaVersion}.jar
-fi
+CHUKWA_VERSION=`cat ${CHUKWA_HOME}/bin/VERSION`
-#chukwaAgent=${HADOOP_HOME}/build/contrib/chukwa/chukwa-agent-${chukwaVersion}.jar
-chukwaAgent=${HADOOP_HOME}/build/contrib/chukwa
-if [ -a $chukwaAgent ] ; then
- export chukwaAgent
-else
- echo ${chukwaAgent} does not exist
- export chukwaAgent=${CHUKWA_HOME}/chukwa-agent-${chukwaVersion}.jar
+# Allow alternate conf dir location.
+if [ -z "$CHUKWA_CONF_DIR" ]; then
+ CHUKWA_CONF_DIR="${CHUKWA_CONF_DIR:-$CHUKWA_HOME/conf}"
+ export CHUKWA_CONF_DIR=${CHUKWA_HOME}/conf
fi
-echo chukwaCore is ${chukwaCore} and chukwaAgent is ${chukwaAgent}
+if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
+ . "${CHUKWA_CONF_DIR}/chukwa-env.sh"
+fi
+export DATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml
+COMMON=`ls ${CHUKWA_HOME}/lib/*.jar`
+export COMMON=`echo ${COMMON} | sed 'y/ /:/'`
+export CHUKWA_CORE=${CHUKWA_HOME}/chukwa-core-${CHUKWA_VERSION}.jar
+export CHUKWA_AGENT=${CHUKWA_HOME}/chukwa-agent-${CHUKWA_VERSION}.jar
+export HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-*-core.jar`
export CURRENT_DATE=`date +%Y%m%d%H%M`
-export TS_CONFIG=${CHUKWA_HOME}/conf/ts
-export tomcat=${CHUKWA_HOME}/opt/apache-tomcat-6.0.16
-if [ -d ${HADOOP_HOME}/build/classes ]; then
- DEFAULT_HADOOP_JAR=${HADOOP_HOME}/build/classes
-# this doesn't work, but needs to be replaced with something that does
-#elif [ls ${HADOOP_HOME}/build/hadoop-*-core.jar` ]; then
-# echo setting DEFAULT_HADOOP_JAR to `ls ${HADOOP_HOME}/build/hadoop-*-core.jar`
-# DEFAULT_HADOOP_JAR=`ls ${HADOOP_HOME}/build/hadoop-*-core.jar`
-else
- DEFAULT_HADOOP_JAR=${CHUKWA_HOME}/hadoopjars/hadoop-0.18.0-core.jar
-fi
-export HADOOP_JAR=${HADOOP_JAR:-$DEFAULT_HADOOP_JAR}
-
-echo
-echo HADOOP_JAR is $HADOOP_JAR
-echo
-
-export CHUKWA_LOG_DIR="${CHUKWA_HOME}/logs/"
-DEFAULT_PID_DIR=${CHUKWA_HOME}/var/run
-export CHUKWA_PID_DIR="${CHUKWA_PID_DIR:-$DEFAULT_PID_DIR}"
-export chuwaRecordsRepository="/chukwa/repos/demo"
+
+if [ -z "$JAVA_HOME" ] ; then
+ export JAVA_HOME=/usr/lib/j2sdk1.5-sun
+fi
+
+export JPS=${JAVA_HOME}/bin/jps
Added: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Runs a Chukwa command as a daemon.
+#
+# Environment Variables
+#
+# CHUKWA_CONF_DIR Alternate conf dir. Default is ${CHUKWA_HOME}/conf.
+# CHUKWA_LOG_DIR Where log files are stored. PWD by default.
+# CHUKWA_MASTER host:path where chukwa code should be rsync'd from
+# CHUKWA_PID_DIR The pid files are stored. ${CHUKWA_HOME}/var/tmp by default.
+# CHUKWA_IDENT_STRING A string representing this instance of chukwa. $USER by default
+# CHUKWA_NICENESS The scheduling priority for daemons. Defaults to 0.
+##
+
+usage="Usage: chukwa-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <chukwa-command> <args...>"
+
+# if no args specified, show usage
+if [ $# -le 1 ]; then
+ echo $usage
+ exit 1
+fi
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# get arguments
+startStop=$1
+shift
+command=$1
+shift
+
+chukwa_rotate_log ()
+{
+ log=$1;
+ num=5;
+ if [ -n "$2" ]; then
+ num=$2
+ fi
+ if [ -f "$log" ]; then # rotate logs
+ while [ $num -gt 1 ]; do
+ prev=`expr $num - 1`
+ [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
+ num=$prev
+ done
+ mv "$log" "$log.$num";
+ fi
+}
+
+if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
+ . "${CHUKWA_CONF_DIR}/chukwa-env.sh"
+fi
+
+# get log directory
+if [ "$CHUKWA_LOG_DIR" = "" ]; then
+ export CHUKWA_LOG_DIR="$CHUKWA_HOME/logs"
+fi
+mkdir -p "$CHUKWA_LOG_DIR"
+
+if [ "$CHUKWA_PID_DIR" = "" ]; then
+ CHUKWA_PID_DIR=$CHUKWA_HOME/var/run
+fi
+
+if [ "$CHUKWA_IDENT_STRING" = "" ]; then
+ export CHUKWA_IDENT_STRING="$USER"
+fi
+
+# some variables
+export CHUKWA_LOGFILE=chukwa-$CHUKWA_IDENT_STRING-$command-$HOSTNAME.log
+export CHUKWA_ROOT_LOGGER="INFO,DRFA"
+log=$CHUKWA_LOG_DIR/chukwa-$CHUKWA_IDENT_STRING-$command-$HOSTNAME.out
+pid=$CHUKWA_PID_DIR/chukwa-$CHUKWA_IDENT_STRING-$command.pid
+
+# Set default scheduling priority
+if [ "$CHUKWA_NICENESS" = "" ]; then
+ export CHUKWA_NICENESS=0
+fi
+
+case $startStop in
+
+ (start)
+ MAIL=`cat ${CHUKWA_HOME}/conf/alert.conf`
+
+ if [ "${WATCHDOG}" != "" ]; then
+ mkdir -p ${CHUKWA_HOME}/var/tmp >&/dev/null
+ crontab -l > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+ crontest=$?
+
+ if [ "X${crontest}" != "X0" ]; then
+ cat > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
+MAILTO=${MAIL}
+1,30 * * * * ${CHUKWA_HOME}/bin/watchdog.sh
+CRON
+ else
+ grep -v "${CHUKWA_HOME}/bin/watchdog.sh" ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} | grep -v MAILTO > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
+ echo "MAILTO=${MAIL}" > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+ cat ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2 >> ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+ rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
+ cat >> ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
+1,30 * * * * ${CHUKWA_HOME}/bin/watchdog.sh
+CRON
+ fi
+
+ # save crontab
+ echo -n "Registering watchdog.."
+ mkdir -p ${CHUKWA_HOME}/var/tmp >&/dev/null
+ crontab ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} > /dev/null 2>&1
+ rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+ echo "done"
+ fi
+
+ mkdir -p "$CHUKWA_PID_DIR"
+
+ if [ -f $pid ]; then
+ if kill -0 `cat $pid` > /dev/null 2>&1; then
+ echo $command running as process `cat $pid`. Stop it first.
+ exit 1
+ fi
+ fi
+
+ if [ "$CHUKWA_MASTER" != "" ]; then
+ echo rsync from $CHUKWA_MASTER
+ rsync -a -e ssh --delete --exclude=.svn $CHUKWA_MASTER/ "$CHUKWA_HOME"
+ fi
+
+ chukwa_rotate_log $log
+ echo starting $command, logging to $log
+ cd "$CHUKWA_HOME"
+ nohup nice -n $CHUKWA_NICENESS "$CHUKWA_HOME"/bin/chukwa -config $command "$@" > "$log" 2>&1 < /dev/null &
+ echo $! > $pid
+ sleep 1; head "$log"
+ ;;
+
+ (stop)
+
+ if [ "${WATCHDOG}" != "" ]; then
+ # remove watchdog
+ crontab -l | grep -v ${CHUKWA_HOME}/bin/watchdog.sh > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+ crontab ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+ rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+ fi
+
+ if [ -f $CHUKWA_HOME/bin/$command ]; then
+ $CHUKWA_HOME/bin/$command stop
+ rm -f $pid
+ else
+ if [ -f $pid ]; then
+ if kill -0 `cat $pid` > /dev/null 2>&1; then
+ echo stopping $command
+ kill `cat $pid`
+ rm -f $pid
+ else
+ echo no $command to stop
+ fi
+ else
+ echo no $command to stop
+ fi
+ fi
+ ;;
+
+ (*)
+ echo $usage
+ exit 1
+ ;;
+
+esac
+
+
Added: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Run a Hadoop command on all slave hosts.
+
+usage="Usage: chukwa-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command args..."
+
+# if no args specified, show usage
+if [ $# -le 1 ]; then
+ echo $usage
+ exit 1
+fi
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. $bin/chukwa-config.sh
+
+if [ ${WATCHDOG}!="" ]; then
+ exec "$bin/slaves.sh" --config $CHUKWA_CONF_DIR cd "$CHUKWA_HOME" \; "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR --watchdog "$@"
+else
+ exec "$bin/slaves.sh" --config $CHUKWA_CONF_DIR cd "$CHUKWA_HOME" \; "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR "$@"
+fi
Added: hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+if [ "X$1" = "Xstop" ]; then
+ echo -n "Shutting down Node Activity Data Loader..."
+ if [ -f ${CHUKWA_HOME}/var/run/PbsNodes-data-loader.pid ]; then
+ kill -TERM `cat ${CHUKWA_HOME}/var/run/PbsNodes-data-loader.pid`
+ fi
+ echo "done"
+ exit 0
+fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/PbsNodes-data-loader.pid"
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -ge 1 ]; then
+ EXISTS=1
+ fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+ ${JAVA_HOME}/bin/java -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=PbsNodes -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec "${nodeActivityCmde}" &
+fi
+
+
Modified: hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh Fri Oct 31 11:57:04 2008
@@ -1,81 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-java=$JAVA_HOME/bin/java
-
-. "$bin"/chukwa-config.sh
-
-# remove watchdog
-crontab -l | grep -v ${CHUKWA_HOME}/bin/watchdog.sh > /tmp/cron.${CURRENT_DATE}
-crontab /tmp/cron.${CURRENT_DATE}
-rm -f /tmp/cron.${CURRENT_DATE}
-
-# stop torque data loader
-pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
-if [ -f $pidFile ]; then
- echo -n "Shutting down Torque Data Loader.."
- torquepid=`head ${pidFile}`
- kill -HUP ${torquepid}
- # kill -HUP `ps eww |grep TorqueDataLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
- rm ${pidFile}
- echo "done"
-else
- echo " no $pidFile"
-fi
-
-# stop util data loader
-pidFile=$CHUKWA_HOME/var/run/UtilDataLoader.pid
-if [ -f $pidFile ]; then
- echo -n "Shutting down Util Data Loader.."
- utilpid=`head ${pidFile}`
- #kill -HUP `ps eww |grep UtilDataLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
- kill -HUP ${utilpid}
- rm ${pidFile}
- echo "done"
-else
- echo " no $pidFile"
-fi
-
-# stop queue info data loader
-pidFile=$CHUKWA_HOME/var/run/QueueInfoDataLoader.pid
-if [ -f $pidFile ]; then
- echo -n "Shutting down Queue Info Data Loader.."
- queuepid=`head ${pidFile}`
- #kill -HUP `ps eww |grep QueueInfoDataLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
- kill -HUP ${queuepid}
- rm ${pidFile}
- echo "done"
-else
- echo " no $pidFile"
-fi
-
-
-# stop queue info data loader
-pidFile=$CHUKWA_HOME/var/run/MapReduceLogLoader.pid
-if [ -f $pidFile ]; then
- echo -n "Shutting down Map Reduce Log Loader.."
- logpid=`head ${pidFile}`
- #kill -HUP `ps eww |grep MapReduceLogLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
- kill -HUP ${logpid}
- rm ${pidFile}
- echo "done"
-else
- echo " no $pidFile"
-fi
-
Added: hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Run a shell command on all slave hosts.
+#
+# Environment Variables
+#
+# CHUKWA_SLAVES File naming remote hosts.
+# Default is ${CHUKWA_CONF_DIR}/chukwa-slaves.
+# CHUKWA_CONF_DIR Alternate conf dir. Default is ${CHUKWA_HOME}/conf.
+# CHUKWA_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
+# CHUKWA_SSH_OPTS Options passed to ssh when running remote commands.
+##
+
+usage="Usage: slaves.sh [--config confdir] command..."
+
+# if no args specified, show usage
+if [ $# -le 0 ]; then
+ echo $usage
+ exit 1
+fi
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# If the slaves file is specified in the command line,
+# then it takes precedence over the definition in
+# hadoop-env.sh. Save it here.
+HOSTLIST=$CHUKWA_SLAVES
+
+if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
+ . "${CHUKWA_CONF_DIR}/chukwa-env.sh"
+fi
+
+if [ "$HOSTLIST" = "" ]; then
+ if [ "$CHUKWA_SLAVES" = "" ]; then
+ export HOSTLIST="${CHUKWA_CONF_DIR}/chukwa-slaves"
+ else
+ export HOSTLIST="${CHUKWA_SLAVES}"
+ fi
+fi
+
+for slave in `cat "$HOSTLIST"`; do
+ ssh $CHUKWA_SSH_OPTS $slave $"${@// /\\ }" \
+ 2>&1 | sed "s/^/$slave: /" &
+ if [ "$CHUKWA_SLAVE_SLEEP" != "" ]; then
+ sleep $CHUKWA_SLAVE_SLEEP
+ fi
+done
+
+wait
Added: hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Start all chukwa daemons. Run this on master node.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# start probes
+"$bin"/start-probes.sh --config $CHUKWA_CONF_DIR
+
+# start data processors
+"$bin"/start-data-processors.sh --config $CHUKWA_CONF_DIR
+
+# start collectors
+"$bin"/start-collectors.sh --config $CHUKWA_CONF_DIR
Added: hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Start hadoop dfs daemons.
+# Optinally upgrade or rollback dfs state.
+# Run this on master node.
+
+usage="Usage: start-collectors.sh"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# start jetty collectors
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts collectors --watchdog start jettyCollector.sh
Added: hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+java=$JAVA_HOME/bin/java
+
+. "$bin"/chukwa-config.sh
+if [ ! -d ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16 ]; then
+ if [ -f ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16.tar.gz ]; then
+ tar fxz ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16.tar.gz -C ${CHUKWA_HOME}/opt
+ fi
+fi
+
+if [ ! -f ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps/hicc-${CHUKWA_VERSION}.war ]; then
+ if [ -f ${CHUKWA_HOME}/hicc-${CHUKWA_VERSION}.war ]; then
+ cp ${CHUKWA_HOME}/hicc-${CHUKWA_VERSION}.war ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps
+ fi
+fi
+
+# start data processors
+"$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR --watchdog start processSinkFiles.sh watchdog
+
+# start database admin script
+"$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR start dbAdmin.sh
Added: hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Start hadoop dfs daemons.
+# Optinally upgrade or rollback dfs state.
+# Run this on master node.
+
+usage="Usage: start-probes.sh"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# start agents
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --watchdog start agent.sh
+# start system data loader daemons
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR start systemDataLoader.sh
+
+# start torque data loader daemons
+if [ ${TORQUE_HOME} != "" ]; then
+ "$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR start torqueDataLoader.sh
+fi
+if [ ${nodeActivityCmde} != "" ]; then
+ "$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR start nodeActivityDataLoader.sh
+fi
Modified: hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh Fri Oct 31 11:57:04 2008
@@ -1,54 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-java=$JAVA_HOME/bin/java
-
-. "$bin"/chukwa-config.sh
-crontab -l > /tmp/cron.${CURRENT_DATE}
-crontest=$?
-
-if [ ! -d ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16 ]; then
- tar fxz ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16.tar.gz -C ${CHUKWA_HOME}/opt
-fi
-
-if [ ! -f ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps/ts.war ]; then
- cp ${CHUKWA_HOME}/ts.war ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps
-fi
-
-if [ "X$?" != "X0" ]; then
- cat > /tmp/cron.${CURRENT_DATE} << CRON
-* * * * * ${CHUKWA_HOME}/bin/watchdog.sh
-10 * * * * ${CHUKWA_HOME}/tools/expire.sh 3 /grid/0/tmp/mdl
-CRON
-else
- grep -v "${CHUKWA_HOME}/bin/watchdog.sh" /tmp/cron.${CURRENT_DATE} | \
- grep -v "${CHUKWA_HOME}/tools/expire.sh 3 /grid/0/tmp/mdl" > /tmp/cron.${CURRENT_DATE}.2
- mv /tmp/cron.${CURRENT_DATE}.2 /tmp/cron.${CURRENT_DATE}
- cat >> /tmp/cron.${CURRENT_DATE} << CRON
-* * * * * ${CHUKWA_HOME}/bin/watchdog.sh
-10 * * * * ${CHUKWA_HOME}/tools/expire.sh 3 /grid/0/tmp/mdl
-CRON
-fi
-
-# save crontab
-echo -n "Registering data loader cron jobs.."
-crontab /tmp/cron.${CURRENT_DATE} > /dev/null 2>&1
-rm -f /tmp/cron.${CURRENT_DATE}
-echo "done"
-
Added: hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Stop all chukwa daemons. Run this on master node.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+"$bin"/stop-probes.sh --config $CHUKWA_CONF_DIR
+"$bin"/stop-data-processors.sh --config $CHUKWA_CONF_DIR
+"$bin"/stop-collectors.sh --config $CHUKWA_CONF_DIR
Added: hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Stop collectors. Run this on master node.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts collectors --watchdog stop jettyCollector.sh
Added: hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+java=$JAVA_HOME/bin/java
+
+. "$bin"/chukwa-config.sh
+
+# stop processSinkFiles.sh
+pidFile=$CHUKWA_HOME/var/run/ProcessSinkFiles.pid
+if [ -f $pidFile ]; then
+ echo -n "Shutting down Data Processors.."
+ DP_PID=`head ${pidFile}`
+ kill -TERM ${DP_PID}
+ rm ${pidFile}
+ echo "done"
+else
+ echo " no $pidFile"
+fi
+
+# stop dbAdmin.sh
+pidFile=$CHUKWA_HOME/var/run/dbAdmin.pid
+if [ -f $pidFile ]; then
+ echo -n "Shutting down Database Admin.."
+ DBADMIN_PID=`head ${pidFile}`
+ kill -TERM ${DBADMIN_PID}
+ rm ${pidFile}
+ echo "done"
+else
+ echo " no $pidFile"
+fi
+
Added: hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Stop probes. Run this on master node.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR stop systemDataLoader.sh
+"$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR stop torqueDataLoader.sh
+"$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR stop nodeActivityDataLoader.sh
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --watchdog stop agent.sh
Modified: hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh Fri Oct 31 11:57:04 2008
@@ -1,3 +1,6 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
@@ -12,17 +15,92 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-pid=$$
-
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/chukwa-config.sh
-echo "${pid}" > "$CHUKWA_HOME/var/run/systemDataLoader.pid"
+JVM_OPTS="-Xms4M -Xmx4M"
-${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DRECORD_TYPE=Sar -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaCore}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec sar -q -r -n FULL 55 &
-${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DRECORD_TYPE=Iostat -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaCore}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec iostat -x 55 2 &
-${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DRECORD_TYPE=Top -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaCore}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec top -b -n 1 -c &
-#${JAVA_HOME}/bin/java -DRECORD_TYPE=Df -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaAgent}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec df -x nfs -x none &
+if [ "X$1" = "Xstop" ]; then
+ echo -n "Shutting down System Data Loader..."
+ if [ -f ${CHUKWA_HOME}/var/run/Sar-data-loader.pid ]; then
+ kill -TERM `cat ${CHUKWA_HOME}/var/run/Sar-data-loader.pid`
+ fi
+ if [ -f ${CHUKWA_HOME}/var/run/Iostat-data-loader.pid ]; then
+ kill -TERM `cat ${CHUKWA_HOME}/var/run/Iostat-data-loader.pid`
+ fi
+ if [ -f ${CHUKWA_HOME}/var/run/Top-data-loader.pid ]; then
+ kill -TERM `cat ${CHUKWA_HOME}/var/run/Top-data-loader.pid`
+ fi
+ if [ -f ${CHUKWA_HOME}/var/run/Df-data-loader.pid ]; then
+ kill -TERM `cat ${CHUKWA_HOME}/var/run/Df-data-loader.pid`
+ fi
+ echo "done"
+ exit 0
+fi
+
+echo -n "Starting System Data Loader..."
+
+#test=`grep -q SysLog ${CHUKWA_HOME}/var/chukwa_checkpoint*`
+#if [ "X${test}"="X1" ]; then
+# echo "add org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped SysLog 0 /var/log/messages 0" | nc localhost 9093 >&/dev/null & disown -h
+#fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/Sar-data-loader.pid"
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -ge 1 ]; then
+ EXISTS=1
+ fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+ ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Sar -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec sar -q -r -n FULL 55 &
+fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/Iostat-data-loader.pid"
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -ge 1 ]; then
+ EXISTS=1
+ fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+ ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Iostat -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec iostat -x 55 2 &
+fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/Top-data-loader.pid"
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -ge 1 ]; then
+ EXISTS=1
+ fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+ ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Top -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec top -b -n 1 -c &
+fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/Df-data-loader.pid"
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -ge 1 ]; then
+ EXISTS=1
+ fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+ ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Df -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec df -l &
+fi
+echo "done"
Added: hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+java=$JAVA_HOME/bin/java
+
+if [ "X$1" = "Xstop" ]; then
+ echo -n "Shutting down Torque Data Loader..."
+ if [ -f ${CHUKWA_HOME}/var/run/TorqueDataLoader.pid ]; then
+ kill -TERM `cat ${CHUKWA_HOME}/var/run/TorqueDataLoader.pid`
+ rm -f ${CHUKWA_HOME}/var/run/TorqueDataLoader.pid
+ fi
+ echo "done"
+ exit 0
+fi
+
+min=`date +%M`
+
+
+# start torque data loader
+pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep TorqueDataLoader | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ ${java} -DDOMAIN=${DOMAIN} -DTORQUE_SERVER=${TORQUE_SERVER} -DTORQUE_HOME=${TORQUE_HOME} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Torque -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.mdl.TorqueDataLoader&
+ fi
+else
+ ${java} -DDOMAIN=${DOMAIN} -DTORQUE_SERVER=${TORQUE_SERVER} -DTORQUE_HOME=${TORQUE_HOME} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Torque -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.mdl.TorqueDataLoader&
+fi
Modified: hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh Fri Oct 31 11:57:04 2008
@@ -21,92 +21,127 @@
. "$bin"/chukwa-config.sh
java=$JAVA_HOME/bin/java
-jps=$JAVA_HOME/bin/jps
min=`date +%M`
+if [ "$CHUKWA_IDENT_STRING" = "" ]; then
+ export CHUKWA_IDENT_STRING="$USER"
+fi
-# start torque data loader
-pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
+# monitor agent
+pidFile=$CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-agent.sh.pid
if [ -f $pidFile ]; then
pid=`head ${pidFile}`
- ChildPIDRunningStatus=`${jps} | grep ${pid} | grep TorqueDataLoader | grep -v grep | wc -l`
- #ChildPIDRunningStatus=`ps -ef | grep TorqueDataLoader | grep -v grep | wc -l`
+ ChildPIDRunningStatus=`ps ax | grep ${pid} | grep agent.sh | grep -v grep | wc -l`
if [ $ChildPIDRunningStatus -lt 1 ]; then
- ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=torque.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.TorqueDataLoader&
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: agent pid file exists, but process missing. Restarting agent.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start agent.sh &
fi
-else
- ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=torque.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.TorqueDataLoader&
fi
-# start util data loader
-pidFile=$CHUKWA_HOME/var/run/UtilDataLoader.pid
+
+# monitor collector
+pidFile=$CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-jettyCollector.sh.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`ps ax | grep ${pid} | grep jettyCollector.sh | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: collector pid file exists, but process missing. Restarting jettyCollector.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start jettyCollector.sh &
+ fi
+fi
+
+# monitor node activity data loader
+pidFile=$CHUKWA_HOME/var/run/PbsNodes-data-loader.pid
if [ -f $pidFile ]; then
pid=`head ${pidFile}`
- ChildPIDRunningStatus=`${jps} | grep ${pid} | grep UtilDataLoader | grep -v grep | wc -l`
- #ChildPIDRunningStatus=`ps -ef | grep UtilDataLoader | grep -v grep | wc -l`
- if [ $ChildPIDRunningStatus -lt 1 ]; then
- ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=util.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.UtilDataLoader&
- fi
-else
- ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=util.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.UtilDataLoader&
-fi
-
-# start queue info data loader
-pidFile=$CHUKWA_HOME/var/run/QueueInfoDataLoader.pid
-if [ -f $pidFile ]; then
- pid=`head ${pidFile}`
- ChildPIDRunningStatus=`${jps} | grep ${pid} | grep QueueInfoDataLoader | grep -v grep | wc -l`
- #ChildPIDRunningStatus=`ps -ef | grep QueueInfoDataLoader | grep -v grep | wc -l`
- if [ $ChildPIDRunningStatus -lt 1 ]; then
- ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=queueinfo.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.QueueInfoDataLoader&
- fi
-else
- ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=queueinfo.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.QueueInfoDataLoader&
-fi
-
-# start map reduce log data loader
-tenmin=`echo ${min} | cut -b 2-`
-if [ "X${tenmin}" == "X0" ]; then
- pidFile=$CHUKWA_HOME/var/run/JobLogDataLoader.pid
- if [ -f $pidFile ]; then
- pid=`head ${pidFile}`
- ChildPIDRunningStatus=`${jps} | grep ${pid} | grep JobLogDataLoader | wc -l`
- if [ $ChildPIDRunningStatus -lt 1 ]; then
- ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=joblog.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.JobLogDataLoader &
- fi
- else
- ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=joblog.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.JobLogDataLoader &
- fi
-fi
-
-# start node activity plugin
-tenmin=`echo ${min} | cut -b 2-`
-if [ "X${tenmin}" == "X0" ]; then
- pidFile=$CHUKWA_HOME/var/run/NodeActivityPlugin.pid
- if [ -f $pidFile ]; then
- pid=`head ${pidFile}`
- ChildPIDRunningStatus=`${jps} | grep ${pid} | grep NodeActivityMDL | wc -l`
- if [ $ChildPIDRunningStatus -lt 1 ]; then
- ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=nodeActivity.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.plugin.nodeactivity.NodeActivityMDL&
- fi
- else
- ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=nodeActivity.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.plugin.nodeactivity.NodeActivityMDL&
- fi
-fi
-
-# start database summary loader
-tenmin=`echo ${min} | cut -b 2-`
-if [ "X${tenmin}" == "X0" ]; then
- pidFile=$CHUKWA_HOME/var/run/DBSummaryLoader.pid
- if [ -f $pidFile ]; then
- pid=`head ${pidFile}`
- ChildPIDRunningStatus=`${jps} | grep ${pid} | grep DBSummaryLoader | wc -l`
- if [ $ChildPIDRunningStatus -lt 1 ]; then
- ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=log4j.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${ckit}:${common} org.apache.hadoop.chukwa.extraction.DBSummaryLoader &
- fi
- else
- ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=log4j.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${ckit}:${common} org.apache.hadoop.chukwa.extraction.DBSummaryLoader &
- fi
+ ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: PbsNodes-data-loader pid file exists, but process missing. Restarting nodeActivityDataLoader.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start nodeActivityDataLoader.sh &
+ fi
fi
+# monitor system data loader
+pidFile=$CHUKWA_HOME/var/run/Df-data-loader.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: Df-data-loader pid file exists, but process missing. Restarting systemDataLoader.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start systemDataLoader.sh &
+ fi
+fi
+
+pidFile=$CHUKWA_HOME/var/run/Iostat-data-loader.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: Iostat-data-loader pid file exists, but process missing. Restarting systemDataLoader.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start systemDataLoader.sh &
+ fi
+fi
+
+pidFile=$CHUKWA_HOME/var/run/Sar-data-loader.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: Sar-data-loader pid file exists, but process missing. Restarting systemDataLoader.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start systemDataLoader.sh &
+ fi
+fi
+
+pidFile=$CHUKWA_HOME/var/run/Top-data-loader.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: Top-data-loader pid file exists, but process missing. Restarting systemDataLoader.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start systemDataLoader.sh &
+ fi
+fi
+
+# monitor torque data loader
+pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: pid file exists, but process missing. Restarting torqueDataLoader.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start torqueDataLoader.sh &
+ fi
+fi
+
+# monitor dataSinkFiles.sh
+pidFile=$CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-processSinkFiles.sh.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`ps ax | grep ${pid} | grep processSinkFiles.sh | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: pid file exists, but process missing. Restarting processSinkFiles.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start processSinkFiles.sh &
+ fi
+fi
+
+# monitor dbAdmin.sh
+pidFile=$CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-dbAdmin.sh.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`ps ax | grep ${pid} | grep dbAdmin.sh | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ HOSTNAME=`hostname`
+ echo "${HOSTNAME}: pid file exists, but process missing. Restarting dbAdmin.sh."
+ "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start dbAdmin.sh &
+ fi
+fi
Modified: hadoop/core/trunk/src/contrib/chukwa/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/build.xml?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/build.xml (original)
+++ hadoop/core/trunk/src/contrib/chukwa/build.xml Fri Oct 31 11:57:04 2008
@@ -20,9 +20,7 @@
<project name="chukwa" default="main">
<import file="../build-contrib.xml"/>
-
-
- <property name="chukwaVersion" value="0.0.1"/>
+ <property name="chukwaVersion" value="0.1.0"/>
<property name="lib.dir" value="${basedir}/lib"/>
<property name="src.dir" value="${basedir}/src"/>
@@ -60,15 +58,6 @@
<mkdir dir="${build.dir}"/>
<mkdir dir="${build.classes}"/>
<mkdir dir="${build.dir}/test"/>
-
- <!--
- <copy todir="${build.dir}">
- <fileset dir="${basedir}">
- <exclude name="**/VERSION"/>
- </fileset>
- </copy>
-
--->
<exec executable="echo" output="${basedir}/bin/VERSION">
<arg line="${chukwaVersion}" />
</exec>
@@ -106,9 +95,6 @@
- <target name="test" depends="compile,compile-test,test-chunk" description="Chukwa Tests">
- </target>
-
<target name="test-chunk" depends="compile,compile-test" description="Test chunk">
<junit showoutput="yes"
@@ -136,16 +122,16 @@
<mkdir dir="${build.dir}/collector/WEB-INF/lib"/>
<mkdir dir="${build.dir}/collector/META-INF"/>
<copy todir="${build.dir}/collector/WEB-INF/classes">
- <fileset dir="${build.dir}">
+ <fileset dir="${build.classes}">
<include name="org/apache/hadoop/chukwa/conf/**/*.class" />
</fileset>
- <fileset dir="${build.dir}">
+ <fileset dir="${build.classes}">
<include name="org/apache/hadoop/chukwa/datacollection/**/*.class" />
</fileset>
- <fileset dir="${build.dir}">
+ <fileset dir="${build.classes}">
<include name="org/apache/hadoop/chukwa/extraction/**/*.class" />
</fileset>
- <fileset dir="${build.dir}">
+ <fileset dir="${build.classes}">
<include name="org/apache/hadoop/chukwa/util/**/*.class" />
</fileset>
<fileset dir="${basedir}/src/java">
@@ -182,7 +168,7 @@
</jar>
</target>
<target name="tools_jar" depends="compile, collector" description="Create tools jar">
- <jar jarfile="${build.dir}/tools-${chukwaVersion}.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/inputtools/**/*.class">
+ <jar jarfile="${build.dir}/tools-${chukwaVersion}.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/inputtools/**/*.class">
<fileset dir="${basedir}/src/java">
<include name="org/apache/hadoop/chukwa/inputtools/**/*.java"/>
</fileset>
@@ -190,7 +176,7 @@
</target>
<target name="agent_jar" depends="compile, collector" description="Create agent jar">
- <jar jarfile="${build.dir}/chukwa-agent-${chukwaVersion}.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/client/**/*.class" >
+ <jar jarfile="${build.dir}/chukwa-agent-${chukwaVersion}.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/client/**/*.class" >
<fileset dir="${build.dir}">
<include name="org/apache/hadoop/chukwa/conf/**/*.class"/>
</fileset>
@@ -226,7 +212,7 @@
</target>
<target name="chukwa_jar" depends="compile, collector" description="Create chukwa-core jar">
- <jar jarfile="${build.dir}/chukwa-core-${chukwaVersion}.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/datacollection/**/*.class" >
+ <jar jarfile="${build.dir}/chukwa-core-${chukwaVersion}.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/datacollection/**/*.class" >
<manifest>
<section name="org/apache/hadoop/chukwa">
<attribute name="Implementation-Title" value="Chukwa"/>
@@ -246,7 +232,7 @@
<target name="chukwa-hadoop_jar" depends="compile" description="Create chukwa_hadoop jar for use with getting hadoop to use chukwa">
- <jar jarfile="${build.dir}/chukwa-hadoop-${chukwaVersion}-client.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/inputtools/log4j/**/*.class">
+ <jar jarfile="${build.dir}/chukwa-hadoop-${chukwaVersion}-client.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/inputtools/log4j/**/*.class">
<fileset dir="${basedir}/src/java">
<include name="org/apache/hadoop/chukwa/inputtools/log4j/**/*.java"/>
<include name="org/apache/hadoop/chukwa/datacollection/client/**/*.java"/>
@@ -261,7 +247,6 @@
</target>
<target name="compress" depends="compile,collector,collector_jar,tools_jar,agent_jar,chukwa_jar,chukwa-hadoop_jar" description="Compression target">
- <!-- <jar jarfile="${build.dir}/chukwa.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/**/*.class" /> -->
<copy todir="." includeEmptyDirs="false">
<fileset dir="${build.dir}">
@@ -271,6 +256,32 @@
</copy>
</target>
+ <target name="test" depends="compile,compile-test,test-chunk,test-input-tools" description="Automated Test Framework">
+ </target>
+
+ <target name="test-input-tools" depends="compile,compile-test" description="Test Input Tools">
+ <junit showoutput="${test.output}"
+ printsummary="${test.junit.printsummary}"
+ haltonfailure="${test.junit.haltonfailure}"
+ fork="yes"
+ forkmode="${test.junit.fork.mode}"
+ maxmemory="${test.junit.maxmemory}"
+ dir="${basedir}" timeout="${test.timeout}"
+ errorProperty="tests.failed" failureProperty="tests.failed">
+ <sysproperty key="CHUKWA_HOME" value="${basedir}"/>
+ <sysproperty key="CHUKWA_CONF_DIR" value="${basedir}/conf"/>
+ <classpath refid="chukwaTestClasspath"/>
+ <formatter type="${test.junit.output.format}" />
+ <batchtest todir="${build.dir}/test" >
+ <fileset dir="${test.src.dir}" includes="org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/*.java"/>
+ </batchtest>
+ </junit>
+ <delete>
+ <fileset dir="${basedir}/var" includes="*"/>
+ </delete>
+ <fail if="tests.failed">Tests failed!</fail>
+ </target>
+
<!-- ====================================================== -->
<!-- Macro definitions -->
<!-- ====================================================== -->
@@ -323,7 +334,7 @@
</copy>
<copy todir="${build.dir}/${final.name}/conf">
- <fileset dir="${conf.dir}" excludes="**/*.template"/>
+ <fileset dir="${conf.dir}"/>
</copy>
<copy todir="${build.dir}/${final.name}/docs">
@@ -372,6 +383,8 @@
<param.listofitems>
<tarfileset dir="${build.dir}" mode="664">
<exclude name="${final.name}/org/*" />
+ <exclude name="${final.name}/collector/**" />
+ <exclude name="${final.name}/${final.name}/**" />
<include name="${final.name}/**" />
</tarfileset>
<tarfileset dir="${build.dir}" mode="755">
@@ -420,8 +433,8 @@
</path>
<path id="chukwaTestClasspath">
<pathelement location="${hadoop.root}/build/classes"/>
- <pathelement location="${hadoop.root}/build/contrib/chukwa/classes/"/>
- <pathelement location="${hadoop.root}/build/contrib/chukwa/test/classes/"/>
+ <pathelement location="${build.classes}"/>
+ <pathelement location="${test.build.classes}"/>
<fileset dir="${lib.dir}">
<include name="**/*.jar" />
<exclude name="**/excluded/" />
Added: hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template Fri Oct 31 11:57:04 2008
@@ -0,0 +1 @@
+user@example.com
Modified: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh Fri Oct 31 11:57:04 2008
@@ -6,20 +6,35 @@
# remote nodes.
# The java implementation to use. Required.
-export JAVA_HOME=/Library/Java/Home/
+export JAVA_HOME=/usr/lib/j2sdk1.5-sun
# The location of the Hadoop the collector should use. Default
# assumes that this chukwa is living in hadoop's src/contrib directory
-#export HADOOP_HOME=
+export HADOOP_HOME="/usr/lib/hadoop/current"
# The directory where pid files are stored. CHUKWA_HOME/var/run by default.
#export CHUKWA_PID_DIR=
# The location of chukwa logs, defaults to CHUKWA_HOME/logs
-#export CHUKWA_LOG_DIR=
+export CHUKWA_LOG_DIR=${CHUKWA_HOME}/logs
# The location of a hadoop jars. use this if you are running a collector
# without a running HDFS (i.e. which writes sequence files to local disk)
# if this is not set, the default is to check HADOOP_HOME for jars or
# classes, if those are not found, uses hadoop jars which come with chukwa
-#export HADOOP_JAR=
+export HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-*-core.jar`
+
+# The location of chukwa data repository
+export chuwaRecordsRepository="/chukwa/repos/"
+
+# The location of torque pbsnodes command
+export nodeActivityCmde="/usr/lib/torque/current/bin/pbsnodes "
+
+# The server which contain pbsnodes, qstat and tracejob.
+export TORQUE_SERVER=localhost
+
+# The location contain torque binaries.
+export TORQUE_HOME=/usr/lib/torque
+
+# The domain of the cluster
+#export DOMAIN=
Modified: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template Fri Oct 31 11:57:04 2008
@@ -6,20 +6,35 @@
# remote nodes.
# The java implementation to use. Required.
-export JAVA_HOME=/Library/Java/Home/
+export JAVA_HOME=/usr/lib/j2sdk1.5-sun
# The location of the Hadoop the collector should use. Default
# assumes that this chukwa is living in hadoop's src/contrib directory
-#export HADOOP_HOME=
+export HADOOP_HOME="/usr/lib/hadoop/current"
# The directory where pid files are stored. CHUKWA_HOME/var/run by default.
#export CHUKWA_PID_DIR=
# The location of chukwa logs, defaults to CHUKWA_HOME/logs
-#export CHUKWA_LOG_DIR=
+export CHUKWA_LOG_DIR=${CHUKWA_HOME}/logs
# The location of a hadoop jars. use this if you are running a collector
# without a running HDFS (i.e. which writes sequence files to local disk)
# if this is not set, the default is to check HADOOP_HOME for jars or
# classes, if those are not found, uses hadoop jars which come with chukwa
-#export HADOOP_JAR=
+export HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-*-core.jar`
+
+# The location of chukwa data repository
+export chuwaRecordsRepository="/chukwa/repos/"
+
+# The location of torque pbsnodes command
+export nodeActivityCmde="/usr/lib/torque/current/bin/pbsnodes "
+
+# The server which contain pbsnodes, qstat and tracejob.
+export TORQUE_SERVER=localhost
+
+# The location contain torque binaries.
+export TORQUE_HOME=/usr/lib/torque
+
+# The domain of the cluster
+#export DOMAIN=
Added: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template Fri Oct 31 11:57:04 2008
@@ -0,0 +1 @@
+localhost
Added: hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties Fri Oct 31 11:57:04 2008
@@ -0,0 +1,7 @@
+dfs.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+dfs.period=60
+
+jvm.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+jvm.period=60
+
+
Added: hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template Fri Oct 31 11:57:04 2008
@@ -0,0 +1 @@
+add org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped SysLog 0 /var/log/messages 0
Modified: hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties Fri Oct 31 11:57:04 2008
@@ -1,8 +0,0 @@
-log4j.rootLogger=INFO, R
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${CHUKWA_HOME}/logs/NodeActivityPlugin.log
-log4j.appender.R.MaxFileSize=10MB
-log4j.appender.R.MaxBackupIndex=10
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
-log4j.logger.org.apache.hadoop.chukwa.ikit.DataConfig=INFO, R
Modified: hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties Fri Oct 31 11:57:04 2008
@@ -1,8 +0,0 @@
-log4j.rootLogger=INFO, R
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${CHUKWA_HOME}/logs/QueueInfoDataLoader.log
-log4j.appender.R.MaxFileSize=10MB
-log4j.appender.R.MaxBackupIndex=10
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
-log4j.logger.org.apache.hadoop.chukwa.ikit.DataConfig=INFO, R
Modified: hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties Fri Oct 31 11:57:04 2008
@@ -1,8 +0,0 @@
-log4j.rootLogger=INFO, R
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${CHUKWA_HOME}/logs/TorqueDataLoader.log
-log4j.appender.R.MaxFileSize=10MB
-log4j.appender.R.MaxBackupIndex=10
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
-log4j.logger.org.apache.hadoop.chukwa.ikit.DataConfig=INFO, R
Modified: hadoop/core/trunk/src/contrib/chukwa/conf/util.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/util.properties?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/util.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/util.properties Fri Oct 31 11:57:04 2008
@@ -1,8 +0,0 @@
-log4j.rootLogger=INFO, R
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${CHUKWA_HOME}/logs/UtilDataLoader.log
-log4j.appender.R.MaxFileSize=10MB
-log4j.appender.R.MaxBackupIndex=10
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
-log4j.logger.org.apache.hadoop.chukwa.ikit.DataConfig=INFO, R
Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java Fri Oct 31 11:57:04 2008
@@ -27,27 +27,28 @@
/**
* An instrumentation plugin for Hadoop, to trigger Chukwa-based task logfile collection.
*
- * WARNING: This code depends on hadoop features that have not yet been committed.
- * To allow it to compile, the key lines have been commented out, and marked with
- * 'PENDING'.
+ * WARNING: This code depends on hadoop features only available in 0.19.
+ * It won't do any good if you try to use it with an earlier Hadoop.
*
*/
-public class ChukwaTTInstru
-extends TaskTrackerMetricsInst //PENDING on getting new metrics code into Hadoop
+public class ChukwaTTInstru extends TaskTrackerMetricsInst
{
private Map<TaskAttemptID, Long> stdOutAdaptors;
private Map<TaskAttemptID, Long> stdErrAdaptors;
private ChukwaAgentController chukwa;
+// private TaskTrackerMetricsInst parent; //for chaining together multiple
+ //instrumentation subsystems
public ChukwaTTInstru(TaskTracker t) {
- super(t); //PENDING
+ super(t);
stdOutAdaptors = new HashMap<TaskAttemptID, Long>();
stdErrAdaptors = new HashMap<TaskAttemptID, Long>();
chukwa = new ChukwaAgentController();
}
public void reportTaskLaunch(TaskAttemptID taskid, File stdout, File stderr) {
+// parent.reportTaskLaunch(taskid, stdout, stderr);
long stdoutID = chukwa.addFile("unknown-userdata", stdout.getAbsolutePath());
long stderrID = chukwa.addFile("unknown-userdata", stderr.getAbsolutePath());
stdOutAdaptors.put(taskid, stdoutID);