You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/11/13 18:29:33 UTC
ambari git commit: AMBARI-13862 Check status of RS process for AMS in
embedded mode (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 6929008bd -> 9319c67f9
AMBARI-13862 Check status of RS process for AMS in embedded mode (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9319c67f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9319c67f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9319c67f
Branch: refs/heads/trunk
Commit: 9319c67f94a7e61ba4da0cf52a92a3b72ea0cb2a
Parents: 6929008
Author: Dmytro Sen <ds...@apache.org>
Authored: Fri Nov 13 19:28:32 2015 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Fri Nov 13 19:28:32 2015 +0200
----------------------------------------------------------------------
.../conf/unix/ambari-metrics-collector | 1 +
.../ApplicationHistoryServer.java | 2 +-
.../timeline/HBaseTimelineMetricStore.java | 17 +++
.../metrics/timeline/PhoenixHBaseAccessor.java | 1 +
.../timeline/TimelineMetricConfiguration.java | 40 ++++++
.../timeline/TimelineMetricStoreWatcher.java | 130 +++++++++++++++++++
.../TimelineMetricStoreWatcherTest.java | 107 +++++++++++++++
.../0.1.0/configuration/ams-site.xml | 26 ++++
8 files changed, 323 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9319c67f/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
index 0018bc2..94594d0 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
@@ -325,6 +325,7 @@ case "$1" in
DAEMON_STATUS=$?
if [[ !"${DISTRIBUTED_HBASE}" && ( ${DAEMON_STATUS} != 0 || ${HBASE_DAEMON_STATUS} != 0 ) ]]; then
+ echo "Stopping already running AMS daemons, if any."
stop
fi
start
http://git-wip-us.apache.org/repos/asf/ambari/blob/9319c67f/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index d79ca68..11c162a 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -45,7 +45,7 @@ import org.apache.hadoop.yarn.webapp.WebApps;
import com.google.common.annotations.VisibleForTesting;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.*;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.DISABLE_APPLICATION_TIMELINE_STORE;
/**
* History server that keeps track of all types of history in the cluster.
http://git-wip-us.apache.org/repos/asf/ambari/blob/9319c67f/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index aed5fed..1d654fd 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -40,6 +40,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
@@ -50,6 +58,8 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
private final TimelineMetricConfiguration configuration;
private PhoenixHBaseAccessor hBaseAccessor;
private static volatile boolean isInitialized = false;
+ private final ScheduledExecutorService executorService =
+ Executors.newSingleThreadScheduledExecutor();
/**
* Construct the service.
@@ -132,6 +142,13 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
aggregatorDailyThread.start();
}
+ int initDelay = configuration.getTimelineMetricsServiceWatcherInitDelay();
+ int delay = configuration.getTimelineMetricsServiceWatcherDelay();
+ // Start the watchdog
+ executorService.scheduleWithFixedDelay(
+ new TimelineMetricStoreWatcher(this, configuration), initDelay, delay,
+ TimeUnit.SECONDS);
+
isInitialized = true;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9319c67f/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index be06650..b5998ee 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -423,6 +423,7 @@ public class PhoenixHBaseAccessor {
}
}
+ // commit() blocked if HBase unavailable
conn.commit();
} finally {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9319c67f/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index fd51f3d..ea05c25 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -181,6 +181,18 @@ public class TimelineMetricConfiguration {
public static final String HANDLER_THREAD_COUNT =
"timeline.metrics.service.handler.thread.count";
+ public static final String WATCHER_INITIAL_DELAY =
+ "timeline.metrics.service.watcher.initial.delay";
+
+ public static final String WATCHER_DELAY =
+ "timeline.metrics.service.watcher.delay";
+
+ public static final String WATCHER_TIMEOUT =
+ "timeline.metrics.service.watcher.timeout";
+
+ public static final String WATCHER_MAX_FAILURES =
+ "timeline.metrics.service.watcher.max.failures";
+
public static final String PRECISION_TABLE_SPLIT_POINTS =
"timeline.metrics.host.aggregate.splitpoints";
@@ -249,6 +261,34 @@ public class TimelineMetricConfiguration {
return 20;
}
+ public int getTimelineMetricsServiceWatcherInitDelay() {
+ if (metricsConf != null) {
+ return Integer.parseInt(metricsConf.get(WATCHER_INITIAL_DELAY, "120"));
+ }
+ return 120;
+ }
+
+ public int getTimelineMetricsServiceWatcherDelay() {
+ if (metricsConf != null) {
+ return Integer.parseInt(metricsConf.get(WATCHER_DELAY, "30"));
+ }
+ return 30;
+ }
+
+ public int getTimelineMetricsServiceWatcherTimeout() {
+ if (metricsConf != null) {
+ return Integer.parseInt(metricsConf.get(WATCHER_TIMEOUT, "30"));
+ }
+ return 30;
+ }
+
+ public int getTimelineMetricsServiceWatcherMaxFailures() {
+ if (metricsConf != null) {
+ return Integer.parseInt(metricsConf.get(WATCHER_MAX_FAILURES, "3"));
+ }
+ return 3;
+ }
+
public String getTimelineServiceRpcAddress() {
String defaultRpcAddress = "0.0.0.0:60200";
if (metricsConf != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9319c67f/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java
new file mode 100644
index 0000000..363b43a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.hadoop.util.ExitUtil;
+
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Acts as the single TimetineMetricStore Watcher.
+ */
+public class TimelineMetricStoreWatcher implements Runnable {
+
+ private static final Log LOG = LogFactory
+ .getLog(TimelineMetricStoreWatcher.class);
+ private static final String FAKE_METRIC_NAME = "TimelineMetricStoreWatcher.FakeMetric";
+ private static final String FAKE_HOSTNAME = "fakehostname";
+ private static final String FAKE_APP_ID = "timeline_metric_store_watcher";
+
+ private static int failures = 0;
+ private final TimelineMetricConfiguration configuration;
+
+ private TimelineMetricStore timelineMetricStore;
+
+ //used to call timelineMetricStore blocking methods with timeout
+ private ExecutorService executor = Executors.newSingleThreadExecutor();
+
+
+ public TimelineMetricStoreWatcher(TimelineMetricStore timelineMetricStore,
+ TimelineMetricConfiguration configuration) {
+ this.timelineMetricStore = timelineMetricStore;
+ this.configuration = configuration;
+ }
+
+ @Override
+ public void run() {
+
+ if (checkMetricStore()) {
+ failures = 0;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Successfully got metrics from TimelineMetricStore");
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to get metrics from TimelineMetricStore");
+ }
+ failures++;
+ }
+
+ if (failures >= configuration.getTimelineMetricsServiceWatcherMaxFailures()) {
+ String msg = "Error getting metrics from TimelineMetricStore. " +
+ "Shutting down by TimelineMetricStoreWatcher.";
+ LOG.fatal(msg);
+ ExitUtil.terminate(-1, msg);
+ }
+
+ }
+
+ /**
+ * Checks TimelineMetricStore functionality by adding and getting
+ * a fake metric to/from HBase
+ * @return if check was successful
+ */
+ private boolean checkMetricStore() {
+ final long startTime = System.currentTimeMillis();
+ final int delay = configuration.getTimelineMetricsServiceWatcherDelay();
+ final int timeout = configuration.getTimelineMetricsServiceWatcherTimeout();
+
+ TimelineMetric fakeMetric = new TimelineMetric();
+ fakeMetric.setMetricName(FAKE_METRIC_NAME);
+ fakeMetric.setHostName(FAKE_HOSTNAME);
+ fakeMetric.setAppId(FAKE_APP_ID);
+ fakeMetric.setStartTime(startTime);
+ fakeMetric.setTimestamp(startTime);
+ fakeMetric.getMetricValues().put(startTime, 0.0);
+
+ final TimelineMetrics metrics = new TimelineMetrics();
+ metrics.setMetrics(Collections.singletonList(fakeMetric));
+
+ Callable<TimelineMetric> task = new Callable<TimelineMetric>() {
+ public TimelineMetric call() throws Exception {
+ timelineMetricStore.putMetrics(metrics);
+ return timelineMetricStore.getTimelineMetric(
+ FAKE_METRIC_NAME, Collections.singletonList(FAKE_HOSTNAME),
+ FAKE_APP_ID, null, startTime - delay * 2 * 1000,
+ startTime + delay * 2 * 1000, Precision.SECONDS, 1);
+ }
+ };
+
+ Future<TimelineMetric> future = executor.submit(task);
+ TimelineMetric timelineMetric = null;
+ try {
+ timelineMetric = future.get(timeout, TimeUnit.SECONDS);
+ // Phoenix might throw RuntimeExeption's
+ } catch (Exception e) {
+ return false;
+ } finally {
+ future.cancel(true);
+ }
+
+ return timelineMetric != null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9319c67f/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java
new file mode 100644
index 0000000..277a98c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
+
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.List;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replayAll;
+import static org.powermock.api.easymock.PowerMock.verifyAll;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(ExitUtil.class)
+public class TimelineMetricStoreWatcherTest {
+
+ @Test
+ public void testRunPositive() throws Exception {
+ TimelineMetricStore metricStore = createNiceMock(TimelineMetricStore.class);
+
+ expect(metricStore.putMetrics(anyObject(TimelineMetrics.class)))
+ .andReturn(new TimelinePutResponse());
+
+ // metric found
+ expect(metricStore.getTimelineMetric(anyObject(String.class),
+ EasyMock.<List<String>> anyObject(), anyObject(String.class),
+ anyObject(String.class), anyObject(Long.class), anyObject(Long.class),
+ eq(Precision.SECONDS), eq(1)))
+ .andReturn(new TimelineMetric()).anyTimes();
+
+ mockStatic(ExitUtil.class);
+
+ replay(metricStore);
+
+ TimelineMetricStoreWatcher timelineMetricStoreWatcher =
+ new TimelineMetricStoreWatcher(metricStore, new TimelineMetricConfiguration());
+ timelineMetricStoreWatcher.run();
+ timelineMetricStoreWatcher.run();
+ timelineMetricStoreWatcher.run();
+ verify(metricStore);
+
+ }
+
+ @Test
+ public void testRunNegative() throws Exception {
+ TimelineMetricStore metricStore = createNiceMock(TimelineMetricStore.class);
+
+ expect(metricStore.putMetrics(anyObject(TimelineMetrics.class)))
+ .andReturn(new TimelinePutResponse());
+
+ // no metrics found
+ expect(metricStore.getTimelineMetric(anyObject(String.class),
+ EasyMock.<List<String>> anyObject(), anyObject(String.class),
+ anyObject(String.class), anyObject(Long.class), anyObject(Long.class),
+ eq(Precision.SECONDS), eq(1)))
+ .andReturn(null).anyTimes();
+
+ String msg = "Error getting metrics from TimelineMetricStore. " +
+ "Shutting down by TimelineMetricStoreWatcher.";
+ mockStatic(ExitUtil.class);
+ ExitUtil.terminate(-1, msg);
+ expectLastCall().anyTimes();
+
+ replayAll();
+
+ TimelineMetricStoreWatcher timelineMetricStoreWatcher =
+ new TimelineMetricStoreWatcher(metricStore, new TimelineMetricConfiguration());
+ timelineMetricStoreWatcher.run();
+ timelineMetricStoreWatcher.run();
+ timelineMetricStoreWatcher.run();
+
+ verifyAll();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9319c67f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index c73a401..be698fc 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -491,4 +491,30 @@
The interval between two service metrics data exports.
</description>
</property>
+ <property>
+ <name>timeline.metrics.service.watcher.initial.delay</name>
+ <value>120</value>
+ <description>
+ The time to delay first watcher check execution
+ Default resolution is 2 minutes.
+ </description>
+ </property>
+ <property>
+ <name>timeline.metrics.service.watcher.delay</name>
+ <value>30</value>
+ <description>
+ The delay between the termination of one
+ watcher check execution and the commencement of the next
+ Default resolution is 30 seconds.
+ </description>
+ </property>
+ <property>
+ <name>timeline.metrics.service.watcher.timeout</name>
+ <value>30</value>
+ <description>
+ The maximum time to wait for a single watcher check execution
+ Default resolution is 30 seconds.
+ </description>
+ </property>
+
</configuration>