You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by hd...@apache.org on 2016/02/19 02:03:53 UTC
incubator-eagle git commit: EAGLE-154 hadoop-metric monitoring
sandbox starter script. - hadoop-metric monitoring sandbox starter script and
policy creation. - update file access permission 755 to hadoop-metric-init.sh
file. Added Eagle home path
Repository: incubator-eagle
Updated Branches:
refs/heads/master 88d0126cb -> fba3aebae
EAGLE-154 hadoop-metric monitoring sandbox starter script.
- hadoop-metric monitoring sandbox starter script and policy creation.
- update file access permission 755 to hadoop-metric-init.sh file. Added Eagle home path ${EAGLE_HOME} to locate sandbox-hadoopjmx-topology.conf in hadoop-metric-monitoring.sh.
https://issues.apache.org/jira/browse/EAGLE-154
Author: @hdendukuri <hd...@ebay.com>
Reviewer: @yonzhang <yo...@ebay.com>
Closes https://github.com/apache/incubator-eagle/pull/87
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/fba3aeba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/fba3aeba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/fba3aeba
Branch: refs/heads/master
Commit: fba3aebae232ba4090dc3fdc64dbdd2946b197ce
Parents: 88d0126
Author: hdendukuri <hd...@ebay.com>
Authored: Wed Feb 17 11:32:26 2016 -0800
Committer: hdendukuri <hd...@ebay.com>
Committed: Wed Feb 17 11:32:26 2016 -0800
----------------------------------------------------------------------
.../src/main/bin/hadoop-metric-monitor.sh | 6 +-
.../examples/hadoop-metric-policy-create.sh | 45 +++++++
.../examples/hadoop-metric-sandbox-starter.sh | 125 +++++++++++++++++++
.../src/main/resources/hadoop-metric-init.sh | 0
4 files changed, 173 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fba3aeba/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh b/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
index 7205ae3..327bec1 100644
--- a/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
+++ b/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
@@ -34,7 +34,7 @@ if [ "$active" ]; then
${EAGLE_HOME}/bin/eagle-topology.sh --topology sandbox-hadoopjmx-topology stop
fi
echo "start Eagle Hadoop Metric Monitoring topology"
-${EAGLE_HOME}/bin/eagle-topology.sh --main org.apache.eagle.hadoop.metric.HadoopJmxMetricMonitor --topology sandbox-hadoopjmx-topology --config ../conf/sandbox-hadoopjmx-topology.conf start
+${EAGLE_HOME}/bin/eagle-topology.sh --main org.apache.eagle.hadoop.metric.HadoopJmxMetricMonitor --topology sandbox-hadoopjmx-topology --config ${EAGLE_HOME}/conf/sandbox-hadoopjmx-topology.conf start
######################################################################
## Setup minutely crontab job for HADOOP METRIC
@@ -43,8 +43,8 @@ echo "set up crontab script"
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cp $DIR/../tools/hadoop_jmx_collector/config-sample.json $DIR/../tools/hadoop_jmx_collector/config.json
command="python $DIR/../tools/hadoop_jmx_collector/hadoop_jmx_kafka.py"
-echo $job
job="* * * * * $command >> $DIR/../logs/hadoop_metric.log"
+echo "$job"
cat <(fgrep -i -v "$command" <(crontab -l)) <(echo "$job") | crontab -
-echo "$(crontab -l)"
\ No newline at end of file
+echo "$(crontab -l)"
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fba3aeba/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh b/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh
new file mode 100644
index 0000000..fc0b639
--- /dev/null
+++ b/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with`
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source $(dirname $0)/../bin/eagle-env.sh
+
+##### add policies ##########
+echo ""
+echo "Importing policy: safeModePolicy "
+curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
+ "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService" \
+ -d '
+ [
+ {
+ "prefix": "alertdef",
+ "tags": {
+ "site": "sandbox",
+ "dataSource": "hadoopJmxMetricDataSource",
+ "policyId": "safeModePolicy",
+ "alertExecutorId": "hadoopJmxMetricAlertExecutor",
+ "policyType": "siddhiCEPEngine"
+ },
+ "description": "jmx metric ",
+ "policyDef": "{\"expression\":\"from hadoopJmxMetricEventStream[component==\\\"namenode\\\" and metric == \\\"hadoop.namenode.fsnamesystemstate.fsstate\\\" and convert(value, \\\"long\\\") > 0]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component, site insert into tmp; \",\"type\":\"siddhiCEPEngine\"}",
+ "enabled": true,
+ "dedupeDef": "{\"alertDedupIntervalMin\":10,\"emailDedupIntervalMin\":10}",
+ "notificationDef": "[{\"sender\":\"eagle@apache.org\",\"recipients\":\"eagle@apache.org\",\"subject\":\"missing block found.\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]"
+ }
+ ]
+ '
+
+exit 0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fba3aeba/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh b/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh
new file mode 100644
index 0000000..8f227b0
--- /dev/null
+++ b/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source $(dirname $0)/../bin/eagle-env.sh
+eagle_bin=$EAGLE_HOME/bin
+
+
+###################################################################
+# STEP 1: Check Installation
+###################################################################
+
+echo "STEP [1/3]: checking environment"
+$eagle_bin/eagle-check-env.sh
+[ $? != 0 ] && exit 1
+
+pid_dir=/var/run
+
+# Check HBase if it has been started
+hbase_master_pid=${pid_dir}/hbase/hbase-hbase-master.pid
+hbase_regionserver_pid=${pid_dir}/hbase/hbase-hbase-regionserver.pid
+echo "Checking if hbase is running ..."
+
+if [ -f $hbase_master_pid ] && \
+ ps aux | grep -v grep | grep $(cat $hbase_master_pid) > /dev/null
+then
+ echo "HBase Master is running as process `cat $hbase_master_pid`."
+else
+ echo "Error: HBase Master is not running. Please start it via Ambari."
+ exit 1
+fi
+
+if [ -f $hbase_regionserver_pid ] && \
+ ps aux | grep -v grep | grep $(cat $hbase_regionserver_pid) > /dev/null
+then
+ echo "HBase RegionServer is running as process `cat $hbase_regionserver_pid`."
+else
+ echo "Error: HBase RegionServer is not running. Please start it via Ambari."
+ exit 1
+fi
+
+# Check kafka if it has been started
+kafka_pid=$pid_dir/kafka/kafka.pid
+echo "Checking if kafka is running ..."
+
+if [ -f $kafka_pid ] && ps aux | grep -v grep | grep $(cat $kafka_pid) > /dev/null
+then
+ echo "Kafka is running as process `cat $kafka_pid`."
+else
+ echo "Error: Kafka is not running. Please start it via Ambari."
+ exit 1
+fi
+
+# Check storm if it has been started
+nimbus_pid=$pid_dir/storm/nimbus.pid
+supervisor_pid=$pid_dir/storm/supervisor.pid
+ui_pid=$pid_dir/storm/ui.pid
+echo "Checking if storm is running ..."
+
+if ! ([ -f $nimbus_pid ] && ps aux | grep -v grep | grep $(cat $nimbus_pid) > /dev/null)
+then
+ echo "Error: Storm Nimbus is not running"
+ exit 1
+fi
+
+if ! ([ -f $supervisor_pid ] && ps aux | grep -v grep | grep $(cat $supervisor_pid) > /dev/null)
+then
+ echo "Error: Storm Supervisor is not running"
+ exit 1
+fi
+
+if ! ([ -f $ui_pid ] && ps aux | grep -v grep | grep $(cat $ui_pid) > /dev/null)
+then
+ echo "Error: Storm UI is not running"
+ exit 1
+fi
+
+echo "Storm is running"
+
+
+###################################################################
+# STEP 2: Starting Eagle Service
+###################################################################
+
+echo "STEP [2/3]: Start Eagle Service"
+$eagle_bin/eagle-service.sh start
+
+
+###################################################################
+# STEP 3: Starting Eagle Topology
+###################################################################
+
+echo "STEP [3/3]: Init Eagle Service"
+$eagle_bin/eagle-service-init.sh
+[ $? != 0 ] && exit 1
+
+echo "Creating kafka topics for eagle ... "
+KAFKA_HOME=/usr/hdp/current/kafka-broker
+EAGLE_ZOOKEEPER_QUORUM=localhost:2181
+topic=`${KAFKA_HOME}/bin/kafka-topics.sh --list --zookeeper $EAGLE_ZOOKEEPER_QUORUM --topic nn_jmx_metric_sandbox`
+if [ -z $topic ]; then
+ $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $EAGLE_ZOOKEEPER_QUORUM --replication-factor 1 --partitions 1 --topic nn_jmx_metric_sandbox
+fi
+
+if [ $? = 0 ]; then
+echo "==> Create kafka topic successfully for Hadoop Metric Monitoring"
+else
+echo "==> Failed, exiting"
+exit 1
+fi
+$eagle_bin/hadoop-metric-monitor.sh
+[ $? != 0 ] && exit 1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fba3aeba/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh b/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh
old mode 100644
new mode 100755