You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2017/06/23 21:18:03 UTC
[1/2] ambari git commit: AMBARI-21328 : NameNode alerts in Unknown
state after Ambari upgrade. (avijayan)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 75cb56ef7 -> 802e8c6dd
AMBARI-21328 : NameNode alerts in Unknown state after Ambari upgrade. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/802e8c6d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/802e8c6d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/802e8c6d
Branch: refs/heads/branch-2.5
Commit: 802e8c6dde449f35d145ff9db0608767b2aa8e10
Parents: 57a1fd1
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Fri Jun 23 14:05:41 2017 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Fri Jun 23 14:05:52 2017 -0700
----------------------------------------------------------------------
.../templates/hadoop-metrics2.properties.j2 | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/802e8c6d/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
index 07cb93e..2f3aab6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
@@ -81,13 +81,13 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
*.sink.timeline.truststore.type = {{metric_truststore_type}}
*.sink.timeline.truststore.password = {{metric_truststore_password}}
-datanode.sink.timeline.collector={{ams_collector_hosts}}
-namenode.sink.timeline.collector={{ams_collector_hosts}}
-resourcemanager.sink.timeline.collector={{ams_collector_hosts}}
-nodemanager.sink.timeline.collector={{ams_collector_hosts}}
-jobhistoryserver.sink.timeline.collector={{ams_collector_hosts}}
-journalnode.sink.timeline.collector={{ams_collector_hosts}}
-applicationhistoryserver.sink.timeline.collector={{ams_collector_hosts}}
+datanode.sink.timeline.collector.hosts={{ams_collector_hosts}}
+namenode.sink.timeline.collector.hosts={{ams_collector_hosts}}
+resourcemanager.sink.timeline.collector.hosts={{ams_collector_hosts}}
+nodemanager.sink.timeline.collector.hosts={{ams_collector_hosts}}
+jobhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}}
+journalnode.sink.timeline.collector.hosts={{ams_collector_hosts}}
+applicationhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}}
resourcemanager.sink.timeline.tagsForPrefix.yarn=Queue
[2/2] ambari git commit: AMBARI-21329 : No data on templated Grafana
dashboards on HDF cluster. (avijayan)
Posted by av...@apache.org.
AMBARI-21329 : No data on templated Grafana dashboards on HDF cluster. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/57a1fd11
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/57a1fd11
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/57a1fd11
Branch: refs/heads/branch-2.5
Commit: 57a1fd11c36ec9e5fd6ccd3952bce4809d8ee16d
Parents: 75cb56e
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Fri Jun 23 14:04:57 2017 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Fri Jun 23 14:05:52 2017 -0700
----------------------------------------------------------------------
.../timeline/AbstractTimelineMetricsSink.java | 22 +++---
.../cache/HandleConnectExceptionTest.java | 76 +++++++++++++++++++-
.../timeline/HBaseTimelineMetricStore.java | 6 +-
.../timeline/TimelineMetricConfiguration.java | 10 +++
.../system/impl/AmbariMetricSinkImpl.java | 3 +
.../system/impl/DatabaseMetricsSource.java | 4 +-
.../metrics/system/impl/JvmMetricsSource.java | 12 ++--
.../metrics/system/impl/MetricsServiceImpl.java | 5 +-
8 files changed, 119 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/57a1fd11/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index b8cba25..229ec9a 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -288,16 +288,18 @@ public abstract class AbstractTimelineMetricsSink {
protected boolean emitMetrics(TimelineMetrics metrics) {
String collectorHost = getCurrentCollectorHost();
- String connectUrl = getCollectorUri(collectorHost);
- String jsonData = null;
- LOG.debug("EmitMetrics connectUrl = " + connectUrl);
- try {
- jsonData = mapper.writeValueAsString(metrics);
- } catch (IOException e) {
- LOG.error("Unable to parse metrics", e);
- }
- if (jsonData != null) {
- return emitMetricsJson(connectUrl, jsonData);
+ if (collectorHost != null) {
+ String connectUrl = getCollectorUri(collectorHost);
+ String jsonData = null;
+ LOG.debug("EmitMetrics connectUrl = " + connectUrl);
+ try {
+ jsonData = mapper.writeValueAsString(metrics);
+ } catch (IOException e) {
+ LOG.error("Unable to parse metrics", e);
+ }
+ if (jsonData != null) {
+ return emitMetricsJson(connectUrl, jsonData);
+ }
}
return false;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/57a1fd11/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
index 32fe32e..1e95271 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
@@ -45,7 +45,7 @@ import static org.powermock.api.easymock.PowerMock.replayAll;
public class HandleConnectExceptionTest {
private static final String COLLECTOR_URL = "collector";
private TestTimelineMetricsSink sink;
-
+
@Before
public void init(){
sink = new TestTimelineMetricsSink();
@@ -88,6 +88,17 @@ public class HandleConnectExceptionTest {
}
}
+ @Test
+ public void testEmitMetricsWithNullHost() {
+ TestTimelineMetricsSinkWithNullHost sinkWithNullHost = new TestTimelineMetricsSinkWithNullHost();
+
+ boolean success = sinkWithNullHost.emitMetrics(new TimelineMetrics());
+ Assert.assertFalse(success);
+
+ success = sinkWithNullHost.emitMetrics(new TimelineMetrics());
+ Assert.assertTrue(success);
+ }
+
private class TestTimelineMetricsSink extends AbstractTimelineMetricsSink{
@Override
protected String getCollectorUri(String host) {
@@ -136,4 +147,67 @@ public class HandleConnectExceptionTest {
}
}
+
+ private class TestTimelineMetricsSinkWithNullHost extends AbstractTimelineMetricsSink {
+
+ int ctr = 0;
+
+ @Override
+ protected String getCollectorUri(String host) {
+ return COLLECTOR_URL;
+ }
+
+ @Override
+ protected String getCollectorProtocol() {
+ return "http";
+ }
+
+ @Override
+ protected String getCollectorPort() {
+ return "2181";
+ }
+
+ @Override
+ protected int getTimeoutSeconds() {
+ return 10;
+ }
+
+ @Override
+ protected String getZookeeperQuorum() {
+ return "localhost:2181";
+ }
+
+ @Override
+ protected Collection<String> getConfiguredCollectorHosts() {
+ return Arrays.asList("localhost");
+ }
+
+ @Override
+ protected String getHostname() {
+ return "h1";
+ }
+
+ @Override
+ public boolean emitMetrics(TimelineMetrics metrics) {
+ super.init();
+ return super.emitMetrics(metrics);
+ }
+
+ @Override
+ protected synchronized String findPreferredCollectHost() {
+ if (ctr == 0) {
+ ctr++;
+ return null;
+ } else {
+ return "localhost";
+ }
+ }
+
+ @Override
+ protected boolean emitMetricsJson(String connectUrl, String jsonData) {
+ return true;
+ }
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/57a1fd11/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index 18a940d..3a26cf4 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -396,7 +396,11 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
throws SQLException, IOException {
Map<String, Set<String>> hostedApps = metricMetadataManager.getHostedAppsCache();
- Map<String, Set<String>> instanceHosts = metricMetadataManager.getHostedInstanceCache();
+ Map<String, Set<String>> instanceHosts = new HashMap<>();
+ if (configuration.getTimelineMetricsMultipleClusterSupport()) {
+ instanceHosts = metricMetadataManager.getHostedInstanceCache();
+ }
+
Map<String, Map<String, Set<String>>> instanceAppHosts = new HashMap<>();
if (MapUtils.isEmpty(instanceHosts)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/57a1fd11/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index beb0904..83df38a 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -287,6 +287,9 @@ public class TimelineMetricConfiguration {
public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES =
"timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles";
+ public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
+ "timeline.metrics.support.multiple.clusters";
+
public static final String HOST_APP_ID = "HOST";
public static final String DEFAULT_INSTANCE_PORT = "12001";
@@ -435,6 +438,13 @@ public class TimelineMetricConfiguration {
return 3;
}
+ public boolean getTimelineMetricsMultipleClusterSupport() {
+ if (metricsConf != null) {
+ return Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false"));
+ }
+ return false;
+ }
+
public String getTimelineServiceRpcAddress() {
String defaultRpcAddress = "0.0.0.0:60200";
if (metricsConf != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/57a1fd11/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
index 767398b..a31fb04 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
@@ -207,6 +207,7 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements
timelineMetricsCache = new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval);
if (CollectionUtils.isNotEmpty(collectorHosts)) {
+ LOG.info("Metric Sink initialized with collectorHosts : " + collectorHosts.toString());
isInitialized = true;
}
}
@@ -235,6 +236,8 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements
timelineMetrics.setMetrics(metricList);
emitMetrics(timelineMetrics);
}
+ } else {
+ LOG.debug("Metric Sink not yet initialized. Discarding metrics.");
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/57a1fd11/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
index fa04f74..5ac3f2a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
@@ -60,7 +60,7 @@ public class DatabaseMetricsSource extends AbstractMetricsSource {
super.init(metricsConfig, sink);
configuration = metricsConfig;
initializeFilterSets();
- LOG.info("DatabaseMetricsSource initialized.");
+ LOG.info("Initialized Ambari DB Metrics Source...");
}
/**
@@ -84,11 +84,11 @@ public class DatabaseMetricsSource extends AbstractMetricsSource {
@Override
public void start() {
- LOG.info("Starting Database Metrics source...");
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setNameFormat("DatabaseMetricsSource-%d")
.build();
executor = Executors.newSingleThreadExecutor(threadFactory);
+ LOG.info("Started Ambari DB Metrics source...");
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/57a1fd11/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
index dbb51cf..be37ff3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
@@ -60,20 +60,24 @@ public class JvmMetricsSource extends AbstractMetricsSource {
registerAll(JVM_PREFIX + ".threads", new ThreadStatesGaugeSet(), registry);
registry.register(JVM_PREFIX + ".file.open.descriptor.ratio", new FileDescriptorRatioGauge());
interval = Integer.parseInt(configuration.getProperty("interval", "10"));
- LOG.info("JVM Metrics source initialized.");
+ LOG.info("Initialized JVM Metrics source...");
}
@Override
public void start() {
- LOG.info("Starting JVM Metrics source...");
try {
executor.scheduleWithFixedDelay(new Runnable() {
@Override
public void run() {
- sink.publish(getMetrics());
- LOG.debug("********* Published JVM metrics to sink **********");
+ try {
+ LOG.debug("Publishing JVM metrics to sink");
+ sink.publish(getMetrics());
+ } catch (Exception e) {
+ LOG.debug("Error in publishing JVM metrics to sink.");
+ }
}
}, interval, interval, TimeUnit.SECONDS);
+ LOG.info("Started JVM Metrics source...");
} catch (Exception e) {
LOG.info("Throwing exception when starting metric source", e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/57a1fd11/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
index d7d905c..9cf5d49 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
@@ -61,9 +61,12 @@ public class MetricsServiceImpl implements MetricsService {
Executors.newScheduledThreadPool(1).scheduleWithFixedDelay(new Runnable() {
@Override
public void run() {
- LOG.info("Checking for metrics sink initialization");
if (!sink.isInitialized()) {
+ LOG.info("Attempting to initialize metrics sink");
initializeMetricsSink();
+ if (sink.isInitialized()) {
+ LOG.info("Metric sink initialization successful");
+ }
}
}
}, 5, 5, TimeUnit.MINUTES);