You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2017/06/23 21:27:14 UTC
ambari git commit: AMBARI-21329 : No data on templated Grafana
dashboards on HDF cluster. (avijayan)
Repository: ambari
Updated Branches:
refs/heads/trunk 8f80fe733 -> 8b90145ca
AMBARI-21329 : No data on templated Grafana dashboards on HDF cluster. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8b90145c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8b90145c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8b90145c
Branch: refs/heads/trunk
Commit: 8b90145caadf73c550c2d209eec4044ba924de9a
Parents: 8f80fe7
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Fri Jun 23 14:27:08 2017 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Fri Jun 23 14:27:08 2017 -0700
----------------------------------------------------------------------
.../timeline/AbstractTimelineMetricsSink.java | 25 ++++--
.../cache/HandleConnectExceptionTest.java | 86 +++++++++++++++++++-
.../timeline/HBaseTimelineMetricStore.java | 6 +-
.../timeline/TimelineMetricConfiguration.java | 10 +++
.../system/impl/AmbariMetricSinkImpl.java | 3 +
.../system/impl/DatabaseMetricsSource.java | 4 +-
.../metrics/system/impl/JvmMetricsSource.java | 12 ++-
.../metrics/system/impl/MetricsServiceImpl.java | 5 +-
8 files changed, 133 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index 7a84627..337f640 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -290,22 +290,29 @@ public abstract class AbstractTimelineMetricsSink {
protected boolean emitMetrics(TimelineMetrics metrics) {
String connectUrl;
+ boolean validCollectorHost = true;
+
if (isHostInMemoryAggregationEnabled()) {
connectUrl = constructTimelineMetricUri("http", "localhost", String.valueOf(getHostInMemoryAggregationPort()));
} else {
String collectorHost = getCurrentCollectorHost();
+ if (collectorHost == null) {
+ validCollectorHost = false;
+ }
connectUrl = getCollectorUri(collectorHost);
}
- String jsonData = null;
- LOG.debug("EmitMetrics connectUrl = " + connectUrl);
- try {
- jsonData = mapper.writeValueAsString(metrics);
- } catch (IOException e) {
- LOG.error("Unable to parse metrics", e);
- }
- if (jsonData != null) {
- return emitMetricsJson(connectUrl, jsonData);
+ if (validCollectorHost) {
+ String jsonData = null;
+ LOG.debug("EmitMetrics connectUrl = " + connectUrl);
+ try {
+ jsonData = mapper.writeValueAsString(metrics);
+ } catch (IOException e) {
+ LOG.error("Unable to parse metrics", e);
+ }
+ if (jsonData != null) {
+ return emitMetricsJson(connectUrl, jsonData);
+ }
}
return false;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
index 4eb75eb..3be2162 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
@@ -45,7 +45,7 @@ import static org.powermock.api.easymock.PowerMock.replayAll;
public class HandleConnectExceptionTest {
private static final String COLLECTOR_URL = "collector";
private TestTimelineMetricsSink sink;
-
+
@Before
public void init(){
sink = new TestTimelineMetricsSink();
@@ -88,6 +88,17 @@ public class HandleConnectExceptionTest {
}
}
+ @Test
+ public void testEmitMetricsWithNullHost() {
+ TestTimelineMetricsSinkWithNullHost sinkWithNullHost = new TestTimelineMetricsSinkWithNullHost();
+
+ boolean success = sinkWithNullHost.emitMetrics(new TimelineMetrics());
+ Assert.assertFalse(success);
+
+ success = sinkWithNullHost.emitMetrics(new TimelineMetrics());
+ Assert.assertTrue(success);
+ }
+
private class TestTimelineMetricsSink extends AbstractTimelineMetricsSink{
@Override
protected String getCollectorUri(String host) {
@@ -146,4 +157,77 @@ public class HandleConnectExceptionTest {
}
}
+
+ private class TestTimelineMetricsSinkWithNullHost extends AbstractTimelineMetricsSink {
+
+ int ctr = 0;
+
+ @Override
+ protected String getCollectorUri(String host) {
+ return COLLECTOR_URL;
+ }
+
+ @Override
+ protected String getCollectorProtocol() {
+ return "http";
+ }
+
+ @Override
+ protected String getCollectorPort() {
+ return "2181";
+ }
+
+ @Override
+ protected int getTimeoutSeconds() {
+ return 10;
+ }
+
+ @Override
+ protected String getZookeeperQuorum() {
+ return "localhost:2181";
+ }
+
+ @Override
+ protected Collection<String> getConfiguredCollectorHosts() {
+ return Arrays.asList("localhost");
+ }
+
+ @Override
+ protected String getHostname() {
+ return "h1";
+ }
+
+ @Override
+ protected boolean isHostInMemoryAggregationEnabled() {
+ return false;
+ }
+
+ @Override
+ protected int getHostInMemoryAggregationPort() {
+ return 0;
+ }
+
+ @Override
+ public boolean emitMetrics(TimelineMetrics metrics) {
+ super.init();
+ return super.emitMetrics(metrics);
+ }
+
+ @Override
+ protected synchronized String findPreferredCollectHost() {
+ if (ctr == 0) {
+ ctr++;
+ return null;
+ } else {
+ return "localhost";
+ }
+ }
+
+ @Override
+ protected boolean emitMetricsJson(String connectUrl, String jsonData) {
+ return true;
+ }
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index f984253..12c27a4 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -415,7 +415,11 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
throws SQLException, IOException {
Map<String, Set<String>> hostedApps = metricMetadataManager.getHostedAppsCache();
- Map<String, Set<String>> instanceHosts = metricMetadataManager.getHostedInstanceCache();
+ Map<String, Set<String>> instanceHosts = new HashMap<>();
+ if (configuration.getTimelineMetricsMultipleClusterSupport()) {
+ instanceHosts = metricMetadataManager.getHostedInstanceCache();
+ }
+
Map<String, Map<String, Set<String>>> instanceAppHosts = new HashMap<>();
if (MapUtils.isEmpty(instanceHosts)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 023465b..006a403 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -290,6 +290,9 @@ public class TimelineMetricConfiguration {
public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES =
"timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles";
+ public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
+ "timeline.metrics.support.multiple.clusters";
+
public static final String HOST_APP_ID = "HOST";
public static final String DEFAULT_INSTANCE_PORT = "12001";
@@ -440,6 +443,13 @@ public class TimelineMetricConfiguration {
return 3;
}
+ public boolean getTimelineMetricsMultipleClusterSupport() {
+ if (metricsConf != null) {
+ return Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false"));
+ }
+ return false;
+ }
+
public String getTimelineServiceRpcAddress() {
String defaultRpcAddress = "0.0.0.0:60200";
if (metricsConf != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
index 7dec18f..5c5801d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
@@ -206,6 +206,7 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements
timelineMetricsCache = new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval);
if (CollectionUtils.isNotEmpty(collectorHosts)) {
+ LOG.info("Metric Sink initialized with collectorHosts : " + collectorHosts.toString());
isInitialized = true;
}
}
@@ -234,6 +235,8 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements
timelineMetrics.setMetrics(metricList);
emitMetrics(timelineMetrics);
}
+ } else {
+ LOG.debug("Metric Sink not yet initialized. Discarding metrics.");
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
index 423b19f..a93b96c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
@@ -59,7 +59,7 @@ public class DatabaseMetricsSource extends AbstractMetricsSource {
super.init(metricsConfig, sink);
configuration = metricsConfig;
initializeFilterSets();
- LOG.info("DatabaseMetricsSource initialized.");
+ LOG.info("Initialized Ambari DB Metrics Source...");
}
/**
@@ -83,11 +83,11 @@ public class DatabaseMetricsSource extends AbstractMetricsSource {
@Override
public void start() {
- LOG.info("Starting Database Metrics source...");
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setNameFormat("DatabaseMetricsSource-%d")
.build();
executor = Executors.newSingleThreadExecutor(threadFactory);
+ LOG.info("Started Ambari DB Metrics source...");
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
index 348cc4f..ad05aa2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
@@ -60,20 +60,24 @@ public class JvmMetricsSource extends AbstractMetricsSource {
registerAll(JVM_PREFIX + ".threads", new ThreadStatesGaugeSet(), registry);
registry.register(JVM_PREFIX + ".file.open.descriptor.ratio", new FileDescriptorRatioGauge());
interval = Integer.parseInt(configuration.getProperty("interval", "10"));
- LOG.info("JVM Metrics source initialized.");
+ LOG.info("Initialized JVM Metrics source...");
}
@Override
public void start() {
- LOG.info("Starting JVM Metrics source...");
try {
executor.scheduleWithFixedDelay(new Runnable() {
@Override
public void run() {
- sink.publish(getMetrics());
- LOG.debug("********* Published JVM metrics to sink **********");
+ try {
+ LOG.debug("Publishing JVM metrics to sink");
+ sink.publish(getMetrics());
+ } catch (Exception e) {
+ LOG.debug("Error in publishing JVM metrics to sink.");
+ }
}
}, interval, interval, TimeUnit.SECONDS);
+ LOG.info("Started JVM Metrics source...");
} catch (Exception e) {
LOG.info("Throwing exception when starting metric source", e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
index 0172b1d..ab8d779 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
@@ -61,9 +61,12 @@ public class MetricsServiceImpl implements MetricsService {
Executors.newScheduledThreadPool(1).scheduleWithFixedDelay(new Runnable() {
@Override
public void run() {
- LOG.info("Checking for metrics sink initialization");
if (!sink.isInitialized()) {
+ LOG.info("Attempting to initialize metrics sink");
initializeMetricsSink();
+ if (sink.isInitialized()) {
+ LOG.info("Metric sink initialization successful");
+ }
}
}
}, 5, 5, TimeUnit.MINUTES);