You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2016/09/14 22:03:51 UTC
[5/5] ambari git commit: AMBARI-18154 : Ambari Dashboard,
Cluster load widget - Incorrect value in Nodes._avg metric. (avijayan)
AMBARI-18154 : Ambari Dashboard, Cluster load widget - Incorrect value in Nodes._avg metric. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aa0528ec
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aa0528ec
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aa0528ec
Branch: refs/heads/trunk
Commit: aa0528ecab47850f44dd5575ab7861083e67c6ea
Parents: 4a09934
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Tue Sep 13 20:17:40 2016 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Wed Sep 14 15:03:34 2016 -0700
----------------------------------------------------------------------
.../TimelineMetricClusterAggregatorSecond.java | 34 ++++++++++++++++++--
...melineMetricClusterAggregatorSecondTest.java | 6 ++--
.../src/main/resources/ganglia_properties.json | 2 +-
3 files changed, 36 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa0528ec/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
index 6731eb3..5f40d21 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
@@ -34,11 +34,15 @@ import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_APP_ID;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.SERVER_SIDE_TIMESIFT_ADJUSTMENT;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_SQL;
@@ -130,6 +134,7 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
throws SQLException, IOException {
Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics =
new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
+ int numLiveHosts = 0;
TimelineMetric metric = null;
if (rs.next()) {
@@ -145,18 +150,25 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
metric.addMetricValues(nextMetric.getMetricValues());
} else {
// Process the current metric
- processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
+ int numHosts = processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
+ numLiveHosts = Math.max(numHosts, numLiveHosts);
metric = nextMetric;
}
}
}
// Process last metric
if (metric != null) {
- processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
+ int numHosts = processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
+ numLiveHosts = Math.max(numHosts, numLiveHosts);
}
// Add app level aggregates to save
aggregateClusterMetrics.putAll(appAggregator.getAggregateClusterMetrics());
+
+ // Add liveHosts metric.
+ long timestamp = timeSlices.get(timeSlices.size() - 1)[1];
+ processLiveHostsMetric(aggregateClusterMetrics, numLiveHosts, timestamp);
+
return aggregateClusterMetrics;
}
@@ -165,10 +177,11 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
* timeline.metrics.cluster.aggregator.minute.timeslice.interval
* Normalize value by averaging them within the interval
*/
- protected void processAggregateClusterMetrics(Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
+ protected int processAggregateClusterMetrics(Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
TimelineMetric metric, List<Long[]> timeSlices) {
// Create time slices
Map<TimelineClusterMetric, Double> clusterMetrics = sliceFromTimelineMetric(metric, timeSlices);
+ int numHosts = 0;
if (clusterMetrics != null && !clusterMetrics.isEmpty()) {
for (Map.Entry<TimelineClusterMetric, Double> clusterMetricEntry :
@@ -188,10 +201,13 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
aggregate.updateMax(avgValue);
aggregate.updateMin(avgValue);
}
+
+ numHosts = aggregate.getNumberOfHosts();
// Update app level aggregates
appAggregator.processTimelineClusterMetric(clusterMetric, metric.getHostName(), avgValue);
}
}
+ return numHosts;
}
protected Map<TimelineClusterMetric, Double> sliceFromTimelineMetric(
@@ -374,4 +390,16 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
return -1l;
}
+ private void processLiveHostsMetric(Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
+ int numLiveHosts, long timestamp) {
+
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(
+ "live_hosts", HOST_APP_ID, null, timestamp, null);
+
+ MetricClusterAggregate metricClusterAggregate = new MetricClusterAggregate((double) numLiveHosts,
+ 1, null, (double) numLiveHosts, (double) numLiveHosts);
+
+ aggregateClusterMetrics.put(timelineClusterMetric, metricClusterAggregate);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa0528ec/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
index 0f93bab..014772f 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
@@ -259,8 +259,9 @@ public class TimelineMetricClusterAggregatorSecondTest {
aggregateClusterMetrics.clear();
timelineMetric.setType("COUNTER");
- secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+ int liveHosts = secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+ Assert.assertEquals(liveHosts, 1);
Assert.assertEquals(aggregateClusterMetrics.size(), 4);
timelineClusterMetric.setTimestamp(startTime + 30*seconds);
Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
@@ -294,8 +295,9 @@ public class TimelineMetricClusterAggregatorSecondTest {
aggregateClusterMetrics.clear();
timelineMetric.setType("COUNTER");
- secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+ liveHosts = secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+ Assert.assertEquals(liveHosts, 1);
Assert.assertEquals(aggregateClusterMetrics.size(), 4);
timelineClusterMetric.setTimestamp(startTime + 60*seconds);
Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa0528ec/ambari-server/src/main/resources/ganglia_properties.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/ganglia_properties.json b/ambari-server/src/main/resources/ganglia_properties.json
index 5632bc0..bd2a547 100644
--- a/ambari-server/src/main/resources/ganglia_properties.json
+++ b/ambari-server/src/main/resources/ganglia_properties.json
@@ -47,7 +47,7 @@
"metric":"load_report.Nodes",
"pointInTime":false,
"temporal":true,
- "amsId":"cpu_num"
+ "amsId":"live_hosts"
},
"metrics/load/Procs":{
"metric":"load_report.Procs",