You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2015/12/10 02:21:16 UTC
ambari git commit: AMBARI-14309 : Combine AMS HBase normalizer and
FIFO compaction policy commands into one HBase shell call. (Aravindan Vijayan
via swagle)
Repository: ambari
Updated Branches:
refs/heads/branch-2.2 d4e64b2f8 -> 004bdc898
AMBARI-14309 : Combine AMS HBase normalizer and FIFO compaction policy commands into one HBase shell call. (Aravindan Vijayan via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/004bdc89
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/004bdc89
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/004bdc89
Branch: refs/heads/branch-2.2
Commit: 004bdc8987c604e6616f05947cdea4e691df968e
Parents: d4e64b2
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Wed Dec 9 17:21:09 2015 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Wed Dec 9 17:21:09 2015 -0800
----------------------------------------------------------------------
.../conf/unix/ambari-metrics-collector | 190 +++++++++----------
1 file changed, 89 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/004bdc89/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
index 71c5ce6..bf2fc47 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
@@ -17,6 +17,7 @@
#JAVA_HOME=/usr/jdk64/jdk1.7.0_45
PIDFILE=/var/run/ambari-metrics-collector/ambari-metrics-collector.pid
OUTFILE=/var/log/ambari-metrics-collector/ambari-metrics-collector.out
+STARTUPFILE=/var/log/ambari-metrics-collector/ambari-metrics-collector-startup.out
HBASE_ZK_PID=/var/run/ams-hbase/hbase-${USER}-zookeeper.pid
HBASE_MASTER_PID=/var/run/ams-hbase/hbase-${USER}-master.pid
@@ -31,7 +32,7 @@ HBASE_CONF_DIR=/etc/ams-hbase/conf
HBASE_CMD=${HBASE_DIR}/bin/hbase
-METRIC_TABLES=(METRIC_AGGREGATE_DAILY METRIC_AGGREGATE_HOURLY METRIC_AGGREGATE METRIC_RECORD METRIC_RECORD_DAILY METRIC_RECORD_HOURLY METRIC_RECORD_MINUTE)
+METRIC_TABLES=(METRIC_AGGREGATE_DAILY METRIC_AGGREGATE_HOURLY METRIC_AGGREGATE_MINUTE METRIC_AGGREGATE METRIC_RECORD METRIC_RECORD_DAILY METRIC_RECORD_HOURLY METRIC_RECORD_MINUTE)
METRIC_FIFO_COMPACTION_TABLES=(METRIC_AGGREGATE METRIC_RECORD METRIC_RECORD_MINUTE)
METRIC_COLLECTOR=ambari-metrics-collector
@@ -67,7 +68,7 @@ function hbase_daemon
daemon_status "${pid}"
if [[ $? == 0 ]]; then
- echo "${daemon} is running as process $(cat "${pid}"). Continuing"
+ echo "${daemon} is running as process $(cat "${pid}"). Continuing" | tee -a $STARTUPFILE
else
# stale pid file, so just remove it and continue on
rm -f "${pid}" >/dev/null 2>&1
@@ -84,64 +85,82 @@ function write_pidfile
local pidfile="$1"
echo $! > "${pidfile}" 2>/dev/null
if [[ $? -gt 0 ]]; then
- echo "ERROR: Cannot write pid ${pidfile}."
+ echo "ERROR: Cannot write pid ${pidfile}." | tee -a $STARTUPFILE
exit 1;
fi
}
# TODO replace this with Phoenix DDL, when normalization support added to Phoenix
-function enable_normalization
+function enable_normalization_fifo
{
- # Enable normalization for all the tables
+ echo "$(date) Handling HBase normalization/fifo requests" | tee -a $STARTUPFILE
command=""
- for table in "${METRIC_TABLES[@]}"
- do
- command="$command \n alter_async '$table', {NORMALIZATION_ENABLED => 'true'}"
- done
- echo -e ${command} | ${HBASE_CMD} --config ${HBASE_CONF_DIR} shell > /dev/null 2>&1
- if [ $? -ne 0 ]; then
- echo "WARNING: Failed to enable Ambari Metrics data model normalization."
- >&2 echo "WARNING: Failed to enable Ambari Metrics data model normalization."
+
+ # Enable normalization for all the tables
+ echo "$(date) Normalized enabled : ${AMS_HBASE_NORMALIZER_ENABLED}" | tee -a $STARTUPFILE
+ if [[ "${AMS_HBASE_NORMALIZER_ENABLED}" == "true" || "${AMS_HBASE_NORMALIZER_ENABLED}" == "True" ]]
+ then
+ if [ ! -f "$NORMALIZER_ENABLED_STUB_FILE" ] #If stub file not found
+ then
+ echo "$(date) Normalizer stub file not found" | tee -a $STARTUPFILE
+ for table in "${METRIC_TABLES[@]}"
+ do
+ command="$command \n alter '$table', {NORMALIZATION_ENABLED => 'true'}"
+ done
+ touch $NORMALIZER_ENABLED_STUB_FILE
+ fi
else
- touch $NORMALIZER_ENABLED_STUB_FILE
+ if [ -f "$NORMALIZER_ENABLED_STUB_FILE" ] #If stub file found
+ then
+ echo "$(date) Normalizer stub file found" | tee -a $STARTUPFILE
+ rm -f $NORMALIZER_ENABLED_STUB_FILE
+ fi
fi
-}
-function enable_fifo_compaction
-{
- # Enable FIFO compaction for HIGH load tables
- command=""
- for table in "${METRIC_FIFO_COMPACTION_TABLES[@]}"
- do
- command="$command \n alter_async '$table', CONFIGURATION => {'hbase.hstore.blockingStoreFiles' => '1000',
- 'hbase.hstore.defaultengine.compactionpolicy.class' =>
- 'org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy'}"
- done
- echo -e ${command} | ${HBASE_CMD} --config ${HBASE_CONF_DIR} shell > /dev/null 2>&1
- if [ $? -ne 0 ]; then
- echo "WARNING: Failed to enable FIFO compaction policy."
- >&2 echo "WARNING: Failed to enable FIFO compaction policy."
+ #Similarly for HBase FIFO Compaction policy
+ echo "$(date) Fifo enabled : ${AMS_HBASE_FIFO_COMPACTION_ENABLED}" | tee -a $STARTUPFILE
+ if [[ "${AMS_HBASE_FIFO_COMPACTION_ENABLED}" == "true" || "${AMS_HBASE_FIFO_COMPACTION_ENABLED}" == "True" ]]
+ then
+ if [ ! -f "$FIFO_ENABLED_STUB_FILE" ] #If stub file not found
+ then
+ echo "$(date) Fifo stub file not found" | tee -a $STARTUPFILE
+ for table in "${METRIC_FIFO_COMPACTION_TABLES[@]}"
+ do
+ command="$command \n alter '$table', CONFIGURATION => {'hbase.hstore.blockingStoreFiles' => '1000',
+ 'hbase.hstore.defaultengine.compactionpolicy.class' =>
+ 'org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy'}"
+ done
+ touch $FIFO_ENABLED_STUB_FILE
+ fi
else
- touch $FIFO_ENABLED_STUB_FILE
+ if [ -f "$FIFO_ENABLED_STUB_FILE" ] #If stub file found
+ then
+ echo "$(date) Fifo stub file found" | tee -a $STARTUPFILE
+ for table in "${METRIC_FIFO_COMPACTION_TABLES[@]}"
+ do
+ command="$command \n alter '$table', CONFIGURATION => {'hbase.hstore.defaultengine.compactionpolicy.class' =>
+ 'org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy',
+ 'hbase.hstore.blockingStoreFiles' => '300'}"
+ done
+ rm -f $FIFO_ENABLED_STUB_FILE
+ fi
fi
-}
-function disable_fifo_compaction
-{
- # Disable FIFO compaction for HIGH load tables
- command=""
- for table in "${METRIC_FIFO_COMPACTION_TABLES[@]}"
- do
- command="$command \n alter_async '$table', CONFIGURATION => {'hbase.hstore.defaultengine.compactionpolicy.class' =>
- 'org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy',
- 'hbase.hstore.blockingStoreFiles' => '300'}"
- done
- echo -e ${command} | ${HBASE_CMD} --config ${HBASE_CONF_DIR} shell > /dev/null 2>&1
- if [ $? -ne 0 ]; then
- echo "WARNING: Failed to disable FIFO compaction policy."
- >&2 echo "WARNING: Failed to disable FIFO compaction policy."
+ if [[ ! -z "$command" ]]
+ then
+ echo "$(date) Executing HBase shell command..." | tee -a $STARTUPFILE
+ echo -e ${command} | tee -a $STARTUPFILE
+ echo -e ${command} | ${HBASE_CMD} --config ${HBASE_CONF_DIR} shell > /dev/null 2>&1
+ if [ $? -ne 0 ]; then
+ echo "WARNING: Failed to enable Ambari Metrics data model normalization."
+ >&2 echo "WARNING: Failed to enable Ambari Metrics data model normalization."
+ rm -f $NORMALIZER_ENABLED_STUB_FILE
+ rm -f $FIFO_ENABLED_STUB_FILE
+ else
+ echo "$(date) HBase shell command completed" | tee -a $STARTUPFILE
+ fi
else
- rm -f $FIFO_ENABLED_STUB_FILE
+ echo "$(date) Nothing to execute against HBase shell" | tee -a $STARTUPFILE
fi
}
@@ -149,19 +168,19 @@ function hadoop_java_setup
{
# Bail if we did not detect it
if [[ -z "${JAVA_HOME}" ]]; then
- echo "ERROR: JAVA_HOME is not set and could not be found."
+ echo "ERROR: JAVA_HOME is not set and could not be found." | tee -a $STARTUPFILE
exit 1
fi
if [[ ! -d "${JAVA_HOME}" ]]; then
- echo "ERROR: JAVA_HOME ${JAVA_HOME} does not exist."
+ echo "ERROR: JAVA_HOME ${JAVA_HOME} does not exist." | tee -a $STARTUPFILE
exit 1
fi
JAVA="${JAVA_HOME}/bin/java"
if [[ ! -x "$JAVA" ]]; then
- echo "ERROR: $JAVA is not executable."
+ echo "ERROR: $JAVA is not executable." | tee -a $STARTUPFILE
exit 1
fi
# shellcheck disable=SC2034
@@ -217,10 +236,10 @@ function start()
# hbase_daemon "master" "start"
# hbase_daemon "regionserver" "start"
if [ !"${DISTRIBUTED_HBASE}" ]; then
- echo "Starting HBase."
+ echo "$(date) Starting HBase." | tee -a $STARTUPFILE
hbase_daemon "master" "start"
else
- echo "Launching in distributed mode. Assuming Hbase daemons up and running."
+ echo "$(date) Launching in distributed mode. Assuming Hbase daemons up and running." | tee -a $STARTUPFILE
fi
CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
@@ -235,48 +254,49 @@ function start()
daemon_status "${PIDFILE}"
if [[ $? == 0 ]]; then
- echo "AMS is running as process $(cat "${PIDFILE}"). Exiting"
+ echo "AMS is running as process $(cat "${PIDFILE}"). Exiting" | tee -a $STARTUPFILE
exit 0
else
# stale pid file, so just remove it and continue on
rm -f "${PIDFILE}" >/dev/null 2>&1
fi
- nohup "${JAVA}" "-Xms$AMS_COLLECTOR_HEAPSIZE" "-Xmx$AMS_COLLECTOR_HEAPSIZE" "${AMS_COLLECTOR_OPTS}" "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}" "-Djava.net.preferIPv4Stack=true" "-Dams.log.dir=${AMS_COLLECTOR_LOG_DIR}" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" >> $OUTFILE 2>&1 &
+ nohup "${JAVA}" "-Xms$AMS_COLLECTOR_HEAPSIZE" "-Xmx$AMS_COLLECTOR_HEAPSIZE" "${AMS_COLLECTOR_OPTS}" "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}" "-Djava.net.preferIPv4Stack=true" "-Dams.log.dir=${AMS_COLLECTOR_LOG_DIR}" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" > $OUTFILE 2>&1 &
PID=$!
write_pidfile "${PIDFILE}"
sleep 2
- echo "Verifying ${METRIC_COLLECTOR} process status..."
+ echo "Verifying ${METRIC_COLLECTOR} process status..." | tee -a $STARTUPFILE
if [ -z "`ps ax -o pid | grep ${PID}`" ]; then
if [ -s ${OUTFILE} ]; then
- echo "ERROR: ${METRIC_COLLECTOR} start failed. For more details, see ${OUTFILE}:"
+ echo "ERROR: ${METRIC_COLLECTOR} start failed. For more details, see ${OUTFILE}:" | tee -a $STARTUPFILE
echo "===================="
tail -n 10 ${OUTFILE}
echo "===================="
else
- echo "ERROR: ${METRIC_COLLECTOR} start failed"
+ echo "ERROR: ${METRIC_COLLECTOR} start failed" | tee -a $STARTUPFILE
rm -f ${PIDFILE}
fi
- echo "Collector out at: ${OUTFILE}"
+ echo "Collector out at: ${OUTFILE}" | tee -a $STARTUPFILE
exit -1
fi
- echo "$(date) Collector successfully started." | tee -a $OUTFILE
- echo "$(date) Initializing Ambari Metrics data model" | tee -a $OUTFILE
+ rm -f $STARTUPFILE #Deleting startup file
+ echo "$(date) Collector successfully started." | tee -a $STARTUPFILE
+ echo "$(date) Initializing Ambari Metrics data model" | tee -a $STARTUPFILE
start=$SECONDS
# Wait until METRIC_* tables created
for retry in {1..5}
do
echo 'list' | ${HBASE_CMD} --config ${HBASE_CONF_DIR} shell | grep ^${METRIC_TABLES[0]} > /dev/null 2>&1
if [ $? -eq 0 ]; then
- echo "$(date) Ambari Metrics data model initialization completed." | tee -a $OUTFILE
+ echo "$(date) Ambari Metrics data model initialization completed." | tee -a $STARTUPFILE
break
fi
- echo "$(date) Ambari Metrics data model initialization check $retry" | tee -a $OUTFILE
+ echo "$(date) Ambari Metrics data model initialization check $retry" | tee -a $STARTUPFILE
duration=$(( SECONDS - start ))
if [ $duration -gt 300 ]; then
- echo "$(date) Ambari Metrics data model initialization timed out" | tee -a $OUTFILE
+ echo "$(date) Ambari Metrics data model initialization timed out" | tee -a $STARTUPFILE
break
fi
sleep 5
@@ -285,40 +305,7 @@ function start()
echo "WARNING: Ambari Metrics data model initialization failed."
>&2 echo "WARNING: Ambari Metrics data model initialization failed."
else
-
- #
- # if hbase.normalizer.enabled = true
- # if stub file DOES NOT exist
- # enable_normalizer + create stub file
- # else
- # if stub file exists
- # delete Stub file.
- if [[ "${AMS_HBASE_NORMALIZER_ENABLED}" == "true" || "${AMS_HBASE_NORMALIZER_ENABLED}" == "True" ]]
- then
- if [ ! -f "$NORMALIZER_ENABLED_STUB_FILE" ]
- then
- enable_normalization
- fi
- else
- if [ -f "$NORMALIZER_ENABLED_STUB_FILE" ]
- then
- rm -f $NORMALIZER_ENABLED_STUB_FILE
- fi
- fi
-
- #Similarly for HBase FIFO Compaction policy
- if [[ "${AMS_HBASE_FIFO_COMPACTION_ENABLED}" == "true" || "${AMS_HBASE_FIFO_COMPACTION_ENABLED}" == "True" ]]
- then
- if [ ! -f "$FIFO_ENABLED_STUB_FILE" ]
- then
- enable_fifo_compaction
- fi
- else
- if [ -f "$FIFO_ENABLED_STUB_FILE" ]
- then
- disable_fifo_compaction
- fi
- fi
+ enable_normalization_fifo
fi
}
@@ -333,12 +320,12 @@ function stop()
sleep "${STOP_TIMEOUT}"
if kill -0 "${pid}" > /dev/null 2>&1; then
- echo "WARNING: ${METRIC_COLLECTOR} did not stop gracefully after ${STOP_TIMEOUT} seconds: Trying to kill with kill -9"
+ echo "WARNING: ${METRIC_COLLECTOR} did not stop gracefully after ${STOP_TIMEOUT} seconds: Trying to kill with kill -9" | tee -a $STARTUPFILE
kill -9 "${pid}" >/dev/null 2>&1
fi
if ps -p "${pid}" > /dev/null 2>&1; then
- echo "ERROR: Unable to kill ${pid}"
+ echo "ERROR: Unable to kill ${pid}" | tee -a $STARTUPFILE
else
rm -f "${pidfile}" >/dev/null 2>&1
fi
@@ -346,7 +333,7 @@ function stop()
#stop hbase daemons
if [ !"${DISTRIBUTED_HBASE}" ]; then
- echo "Stopping HBase master"
+ echo "Stopping HBase master" | tee -a $STARTUPFILE
hbase_daemon "master" "stop"
fi
}
@@ -360,10 +347,10 @@ while [[ -z "${_ams_configs_done}" ]]; do
if [[ -d "${confdir}" ]]; then
COLLECTOR_CONF_DIR="${confdir}"
elif [[ -z "${confdir}" ]]; then
- echo "ERROR: No parameter provided for --config "
+ echo "ERROR: No parameter provided for --config " | tee -a $STARTUPFILE
exit 1
else
- echo "ERROR: Cannot find configuration directory \"${confdir}\""
+ echo "ERROR: Cannot find configuration directory \"${confdir}\"" | tee -a $STARTUPFILE
exit 1
fi
;;
@@ -401,6 +388,7 @@ fi
# set out file path
if [[ -n "${AMS_COLLECTOR_LOG_DIR}" ]]; then
OUTFILE=${AMS_COLLECTOR_LOG_DIR}/ambari-metrics-collector.out
+ STARTUPFILE=${AMS_COLLECTOR_LOG_DIR}/ambari-metrics-collector-startup.out
fi
#TODO manage 3 hbase daemons for start/stop/status