You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2016/07/22 21:02:29 UTC
[3/3] ambari git commit: AMBARI-17843 : App data aggregated for
hosted apps is being calcualted for all apps,
not just configured ones (avijayan)
AMBARI-17843 : App data aggregated for hosted apps is being calcualted for all apps, not just configured ones (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e8751bba
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e8751bba
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e8751bba
Branch: refs/heads/branch-2.4
Commit: e8751bba7043c954bd21ccacfcf470e61c7a0add
Parents: 30b8ddf
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Fri Jul 22 13:38:46 2016 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Fri Jul 22 14:02:19 2016 -0700
----------------------------------------------------------------------
.../TimelineMetricAppAggregator.java | 61 ++++++++++++++------
.../discovery/TimelineMetricMetadataKey.java | 4 ++
.../TimelineMetricMetadataManager.java | 6 ++
3 files changed, 52 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8751bba/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java
index 05beb76..0d6a09e 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java
@@ -21,6 +21,9 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
import java.util.ArrayList;
@@ -47,11 +50,13 @@ public class TimelineMetricAppAggregator {
private final List<String> appIdsToAggregate;
private final Map<String, Set<String>> hostedAppsMap;
Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics;
+ TimelineMetricMetadataManager metadataManagerInstance;
public TimelineMetricAppAggregator(TimelineMetricMetadataManager metadataManager,
Configuration metricsConf) {
appIdsToAggregate = getAppIdsForHostAggregation(metricsConf);
hostedAppsMap = metadataManager.getHostedAppsCache();
+ metadataManagerInstance = metadataManager;
LOG.info("AppIds configured for aggregation: " + appIdsToAggregate);
}
@@ -122,27 +127,45 @@ public class TimelineMetricAppAggregator {
return;
}
+ TimelineMetricMetadataKey appKey = new TimelineMetricMetadataKey(clusterMetric.getMetricName(), HOST_APP_ID);
Set<String> apps = hostedAppsMap.get(hostname);
for (String appId : apps) {
- // Add a new cluster aggregate metric if none exists
- TimelineClusterMetric appTimelineClusterMetric =
- new TimelineClusterMetric(clusterMetric.getMetricName(),
- appId,
- clusterMetric.getInstanceId(),
- clusterMetric.getTimestamp(),
- clusterMetric.getType()
- );
-
- MetricClusterAggregate clusterAggregate = aggregateClusterMetrics.get(appTimelineClusterMetric);
-
- if (clusterAggregate == null) {
- clusterAggregate = new MetricClusterAggregate(metricValue, 1, null, metricValue, metricValue);
- aggregateClusterMetrics.put(appTimelineClusterMetric, clusterAggregate);
- } else {
- clusterAggregate.updateSum(metricValue);
- clusterAggregate.updateNumberOfHosts(1);
- clusterAggregate.updateMax(metricValue);
- clusterAggregate.updateMin(metricValue);
+ if (appIdsToAggregate.contains(appId)) {
+
+ appKey.setAppId(appId);
+ TimelineMetricMetadata appMetadata = metadataManagerInstance.getMetadataCacheValue(appKey);
+ if (appMetadata == null) {
+ TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(clusterMetric.getMetricName(), HOST_APP_ID);
+ TimelineMetricMetadata hostMetricMetadata = metadataManagerInstance.getMetadataCacheValue(key);
+
+ if (hostMetricMetadata != null) {
+ TimelineMetricMetadata timelineMetricMetadata = new TimelineMetricMetadata(clusterMetric.getMetricName(),
+ appId, hostMetricMetadata.getUnits(), hostMetricMetadata.getType(), hostMetricMetadata.getSeriesStartTime(),
+ hostMetricMetadata.isSupportsAggregates());
+ metadataManagerInstance.putIfModifiedTimelineMetricMetadata(timelineMetricMetadata);
+ }
+ }
+
+ // Add a new cluster aggregate metric if none exists
+ TimelineClusterMetric appTimelineClusterMetric =
+ new TimelineClusterMetric(clusterMetric.getMetricName(),
+ appId,
+ clusterMetric.getInstanceId(),
+ clusterMetric.getTimestamp(),
+ clusterMetric.getType()
+ );
+
+ MetricClusterAggregate clusterAggregate = aggregateClusterMetrics.get(appTimelineClusterMetric);
+
+ if (clusterAggregate == null) {
+ clusterAggregate = new MetricClusterAggregate(metricValue, 1, null, metricValue, metricValue);
+ aggregateClusterMetrics.put(appTimelineClusterMetric, clusterAggregate);
+ } else {
+ clusterAggregate.updateSum(metricValue);
+ clusterAggregate.updateNumberOfHosts(1);
+ clusterAggregate.updateMax(metricValue);
+ clusterAggregate.updateMin(metricValue);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8751bba/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java
index ec97ee5..504b502 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java
@@ -34,6 +34,10 @@ public class TimelineMetricMetadataKey {
return appId;
}
+ public void setAppId(String appId) {
+ this.appId = appId;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8751bba/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
index 9fa2066..a79f9f6 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.metrics2.sink.timeline.MetadataException;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
+
import java.sql.SQLException;
import java.util.Collection;
import java.util.HashSet;
@@ -96,6 +98,10 @@ public class TimelineMetricMetadataManager {
return METADATA_CACHE;
}
+ public TimelineMetricMetadata getMetadataCacheValue(TimelineMetricMetadataKey key) {
+ return METADATA_CACHE.get(key);
+ }
+
public Map<String, Set<String>> getHostedAppsCache() {
return HOSTED_APPS_MAP;
}