You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2015/05/21 19:08:30 UTC
ambari git commit: AMBARI-11260. Cluster aggregate metric graphs are
showing averages, not sum. (swagle)
Repository: ambari
Updated Branches:
refs/heads/trunk 97eec61d1 -> 9371bb760
AMBARI-11260. Cluster aggregate metric graphs are showing averages, not sum. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9371bb76
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9371bb76
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9371bb76
Branch: refs/heads/trunk
Commit: 9371bb7601c42b250ec258b85a95a5cc1f55ba61
Parents: 97eec61
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu May 21 10:08:26 2015 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu May 21 10:08:26 2015 -0700
----------------------------------------------------------------------
.../server/api/services/AmbariMetaInfo.java | 92 ++++++++++++-----
.../internal/AbstractPropertyProvider.java | 4 +-
.../metrics/MetricsReportPropertyProvider.java | 2 +
.../MetricsReportPropertyProviderProxy.java | 15 +++
.../timeline/AMSComponentPropertyProvider.java | 58 +----------
.../metrics/timeline/AMSPropertyProvider.java | 34 ++-----
.../timeline/AMSReportPropertyProvider.java | 42 +++++++-
.../controller/utilities/PropertyHelper.java | 76 ++++++++++++++
.../timeline/AMSPropertyProviderTest.java | 2 +
.../timeline/AMSReportPropertyProviderTest.java | 50 +++++++++-
.../ams/aggregate_cluster_metrics.json | 100 +++++++++++++++++++
11 files changed, 361 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index 0761614..f411aaa 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@ -18,24 +18,10 @@
package org.apache.ambari.server.api.services;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Scanner;
-import java.util.Set;
-
-import javax.xml.bind.JAXBException;
-
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ParentObjectNotFoundException;
import org.apache.ambari.server.StackAccessException;
@@ -71,16 +57,32 @@ import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptorFactory;
+import org.apache.ambari.server.state.stack.Metric;
import org.apache.ambari.server.state.stack.MetricDefinition;
import org.apache.ambari.server.state.stack.OsFamily;
import org.apache.ambari.server.state.stack.UpgradePack;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
+import javax.xml.bind.JAXBException;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.Set;
+
+import static org.apache.ambari.server.controller.spi.Resource.InternalType.Component;
+import static org.apache.ambari.server.controller.utilities.PropertyHelper.AGGREGATE_FUNCTION_IDENTIFIERS;
/**
@@ -867,7 +869,7 @@ public class AmbariMetaInfo {
try {
map = gson.fromJson(new FileReader(svc.getMetricsFile()), type);
- svc.setMetrics(map);
+ svc.setMetrics(updateComponentMetricMapWithAggregateFunctionIds(map));
} catch (Exception e) {
LOG.error ("Could not read the metrics file", e);
@@ -879,6 +881,50 @@ public class AmbariMetaInfo {
}
/**
+ * Add aggregate function support for all stack defined metrics.
+ */
+ private Map<String, Map<String, List<MetricDefinition>>> updateComponentMetricMapWithAggregateFunctionIds(
+ Map<String, Map<String, List<MetricDefinition>>> metricMap) {
+
+ if (!metricMap.isEmpty()) {
+ // For every Component
+ for (Map<String, List<MetricDefinition>> componentMetricDef : metricMap.values()) {
+ // For every Component / HostComponent category
+ for (Map.Entry<String, List<MetricDefinition>> metricDefEntry : componentMetricDef.entrySet()) {
+ // NOTE: Only Component aggregates supported for now.
+ if (metricDefEntry.getKey().equals(Component.name())) {
+ //For every metric definition
+ for (MetricDefinition metricDefinition : metricDefEntry.getValue()) {
+ Map<String, Metric> newMetrics = new HashMap<String, Metric>();
+ // Metrics System metrics only
+ if (metricDefinition.getType().equals("ganglia")) {
+ // For every function id
+ for (String identifierToAdd : AGGREGATE_FUNCTION_IDENTIFIERS) {
+ for (Map.Entry<String, Metric> metricEntry : metricDefinition.getMetrics().entrySet()) {
+ String newMetricKey = metricEntry.getKey() + identifierToAdd;
+ Metric currentMetric = metricEntry.getValue();
+ Metric newMetric = new Metric(
+ currentMetric.getName() + identifierToAdd,
+ currentMetric.isPointInTime(),
+ currentMetric.isTemporal(),
+ currentMetric.isAmsHostMetric(),
+ currentMetric.getUnit()
+ );
+ newMetrics.put(newMetricKey, newMetric);
+ }
+ }
+ }
+ metricDefinition.getMetrics().putAll(newMetrics);
+ }
+ }
+ }
+ }
+ }
+
+ return metricMap;
+ }
+
+ /**
* Gets the metrics for a Role (component).
* @return the list of defined metrics.
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
index 98b42e0..96fa24b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
@@ -21,6 +21,8 @@ package org.apache.ambari.server.controller.internal;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.text.DecimalFormat;
+import java.util.Date;
+import java.util.EnumMap;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -32,6 +34,7 @@ import java.util.regex.Pattern;
import org.apache.ambari.server.controller.metrics.MetricReportingAdapter;
import org.apache.ambari.server.controller.spi.PropertyProvider;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
/**
@@ -384,5 +387,4 @@ public abstract class AbstractPropertyProvider extends BaseProvider implements P
return length > 0 ? dataPoints[length - 1][0] : 0;
}
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java
index 20e11ad..5399436 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java
@@ -35,6 +35,8 @@ public abstract class MetricsReportPropertyProvider extends AbstractPropertyProv
protected final ComponentSSLConfiguration configuration;
+ protected static final MetricsPaddingMethod DEFAULT_PADDING_METHOD =
+ new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS);
// ----- Constants --------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java
index 1bcf01a..a92cb37 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java
@@ -80,6 +80,21 @@ public class MetricsReportPropertyProviderProxy extends AbstractPropertyProvider
clusterNamePropertyId);
}
+ /**
+ * Allow delegates to support special properties not stack defined.
+ */
+ @Override
+ public Set<String> checkPropertyIds(Set<String> propertyIds) {
+ MetricsService metricsService = metricsServiceProvider.getMetricsServiceType();
+ Set<String> checkedPropertyIds = super.checkPropertyIds(propertyIds);
+
+ if (metricsService != null && metricsService.equals(TIMELINE_METRICS)) {
+ return amsMetricsReportProvider.checkPropertyIds(checkedPropertyIds);
+ } else {
+ return checkedPropertyIds;
+ }
+ }
+
@Override
public Set<Resource> populateResources(Set<Resource> resources, Request request,
Predicate predicate) throws SystemException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
index ded96cf..d5f415a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
@@ -20,17 +20,10 @@ package org.apache.ambari.server.controller.metrics.timeline;
import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
import org.apache.ambari.server.controller.internal.PropertyInfo;
import org.apache.ambari.server.controller.metrics.MetricHostProvider;
-import org.apache.ambari.server.controller.spi.Predicate;
-import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.controller.utilities.PredicateHelper;
import org.apache.ambari.server.controller.utilities.StreamProvider;
-import org.apache.commons.lang.StringUtils;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
-import java.util.Set;
public class AMSComponentPropertyProvider extends AMSPropertyProvider {
@@ -41,59 +34,10 @@ public class AMSComponentPropertyProvider extends AMSPropertyProvider {
String clusterNamePropertyId,
String componentNamePropertyId) {
- super(updateComponentMetricsWithAggregateFunctionSupport(componentPropertyInfoMap),
- streamProvider, configuration, hostProvider,
+ super(componentPropertyInfoMap, streamProvider, configuration, hostProvider,
clusterNamePropertyId, null, componentNamePropertyId);
}
- /**
- * This method adds supported propertyInfo for component metrics with
- * aggregate function ids. API calls with multiple aggregate functions
- * applied to a single metric need this support.
- *
- * Currently this support is added only for Component metrics,
- * this can be easily extended to all levels by moving this method to the
- * base class: @AMSPropertyProvider.
- */
- private static Map<String, Map<String, PropertyInfo>> updateComponentMetricsWithAggregateFunctionSupport(
- Map<String, Map<String, PropertyInfo>> componentMetrics) {
-
- if (componentMetrics == null || componentMetrics.isEmpty()) {
- return componentMetrics;
- }
-
- // For every component
- for (Map<String, PropertyInfo> componentMetricInfo : componentMetrics.values()) {
- Map<String, PropertyInfo> aggregateMetrics = new HashMap<String, PropertyInfo>();
- // For every metric
- for (Map.Entry<String, PropertyInfo> metricEntry : componentMetricInfo.entrySet()) {
- // For each aggregate function id
- for (String identifierToAdd : aggregateFunctionIdentifierMap.values()) {
- String metricInfoKey = metricEntry.getKey() + identifierToAdd;
- // This disallows metric key suffix of the form "._sum._sum" for
- // the sake of avoiding duplicates
- if (componentMetricInfo.containsKey(metricInfoKey)) {
- continue;
- }
-
- PropertyInfo propertyInfo = metricEntry.getValue();
- PropertyInfo metricInfoValue = new PropertyInfo(
- propertyInfo.getPropertyId() + identifierToAdd,
- propertyInfo.isTemporal(),
- propertyInfo.isPointInTime());
- metricInfoValue.setAmsHostMetric(propertyInfo.isAmsHostMetric());
- metricInfoValue.setAmsId(propertyInfo.getAmsId());
- metricInfoValue.setUnit(propertyInfo.getUnit());
-
- aggregateMetrics.put(metricInfoKey, metricInfoValue);
- }
- }
- componentMetricInfo.putAll(aggregateMetrics);
- }
-
- return componentMetrics;
- }
-
@Override
protected String getHostName(Resource resource) {
return null;
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
index d5af2b7..1011ded 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
@@ -39,13 +39,13 @@ import org.codehaus.jackson.map.AnnotationIntrospector;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.ObjectReader;
import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.SocketTimeoutException;
import java.util.Collection;
import java.util.Collections;
-import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -54,6 +54,7 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+
import static org.apache.ambari.server.Role.HBASE_MASTER;
import static org.apache.ambari.server.Role.HBASE_REGIONSERVER;
import static org.apache.ambari.server.Role.METRICS_COLLECTOR;
@@ -67,14 +68,6 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider {
private final static ObjectReader timelineObjectReader;
private static final String METRIC_REGEXP_PATTERN = "\\([^)]*\\)";
private static final int COLLECTOR_DEFAULT_PORT = 6188;
- protected static enum AGGREGATE_FUNCTION_IDENTIFIER {
- SUM,
- MIN,
- MAX,
- AVG
- }
- protected static final EnumMap<AGGREGATE_FUNCTION_IDENTIFIER, String> aggregateFunctionIdentifierMap =
- new EnumMap<AGGREGATE_FUNCTION_IDENTIFIER, String>(AGGREGATE_FUNCTION_IDENTIFIER.class);
static {
TIMELINE_APPID_MAP.put(HBASE_MASTER.name(), "HBASE");
@@ -87,11 +80,6 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider {
//noinspection deprecation
mapper.getSerializationConfig().setSerializationInclusion(Inclusion.NON_NULL);
timelineObjectReader = mapper.reader(TimelineMetrics.class);
-
- aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.SUM, "._sum");
- aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.MIN, "._min");
- aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.MAX, "._max");
- aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.AVG, "._avg");
}
public AMSPropertyProvider(Map<String, Map<String, PropertyInfo>> componentPropertyInfoMap,
@@ -121,28 +109,18 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider {
*/
@Override
public Set<String> checkPropertyIds(Set<String> propertyIds) {
+ Set<String> supportedIds = new HashSet<String>();
for (String propertyId : propertyIds) {
if (propertyId.startsWith(ZERO_PADDING_PARAM)
- || hasAggregateFunctionSuffix(propertyId)) {
- propertyIds.remove(propertyId);
+ || PropertyHelper.hasAggregateFunctionSuffix(propertyId)) {
+ supportedIds.add(propertyId);
}
}
+ propertyIds.removeAll(supportedIds);
return propertyIds;
}
/**
- * Check if property ends with a trailing suffix
- */
- protected boolean hasAggregateFunctionSuffix(String propertyId) {
- for (String aggregateFunctionId : aggregateFunctionIdentifierMap.values()) {
- if (propertyId.endsWith(aggregateFunctionId)) {
- return true;
- }
- }
- return false;
- }
-
- /**
* The information required to make a single call to the Metrics service.
*/
class MetricsRequest {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
index febcf75..2dbff68 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.controller.metrics.timeline;
import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
import org.apache.ambari.server.controller.internal.PropertyInfo;
import org.apache.ambari.server.controller.metrics.MetricHostProvider;
+import org.apache.ambari.server.controller.metrics.MetricsPaddingMethod;
import org.apache.ambari.server.controller.metrics.MetricsPropertyProvider;
import org.apache.ambari.server.controller.metrics.MetricsReportPropertyProvider;
import org.apache.ambari.server.controller.spi.Predicate;
@@ -27,6 +28,7 @@ import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.controller.spi.TemporalInfo;
+import org.apache.ambari.server.controller.utilities.PropertyHelper;
import org.apache.ambari.server.controller.utilities.StreamProvider;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
@@ -41,16 +43,21 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.SocketTimeoutException;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import static org.apache.ambari.server.controller.metrics.MetricsPaddingMethod.ZERO_PADDING_PARAM;
import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService.TIMELINE_METRICS;
+import static org.apache.ambari.server.controller.utilities.PropertyHelper.updateMetricsWithAggregateFunctionSupport;
public class AMSReportPropertyProvider extends MetricsReportPropertyProvider {
private static ObjectMapper mapper;
private final static ObjectReader timelineObjectReader;
+ private MetricsPaddingMethod metricsPaddingMethod;
static {
mapper = new ObjectMapper();
@@ -71,6 +78,22 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider {
hostProvider, clusterNamePropertyId);
}
+ /**
+ * Support properties with aggregate functions and metrics padding method.
+ */
+ @Override
+ public Set<String> checkPropertyIds(Set<String> propertyIds) {
+ Set<String> supportedIds = new HashSet<String>();
+ for (String propertyId : propertyIds) {
+ if (propertyId.startsWith(ZERO_PADDING_PARAM)
+ || PropertyHelper.hasAggregateFunctionSuffix(propertyId)) {
+ supportedIds.add(propertyId);
+ }
+ }
+ propertyIds.removeAll(supportedIds);
+ return propertyIds;
+ }
+
@Override
public Set<Resource> populateResources(Set<Resource> resources,
Request request, Predicate predicate) throws SystemException {
@@ -96,13 +119,27 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider {
* @throws SystemException if unable to populate the resource
*/
private boolean populateResource(Resource resource, Request request, Predicate predicate)
- throws SystemException {
+ throws SystemException {
Set<String> propertyIds = getPropertyIds();
if (propertyIds.isEmpty()) {
return true;
}
+
+ metricsPaddingMethod = DEFAULT_PADDING_METHOD;
+
+ Set<String> requestPropertyIds = request.getPropertyIds();
+ if (requestPropertyIds != null && !requestPropertyIds.isEmpty()) {
+ for (String propertyId : requestPropertyIds) {
+ if (propertyId.startsWith(ZERO_PADDING_PARAM)) {
+ String paddingStrategyStr = propertyId.substring(ZERO_PADDING_PARAM.length() + 1);
+ metricsPaddingMethod = new MetricsPaddingMethod(
+ MetricsPaddingMethod.PADDING_STRATEGY.valueOf(paddingStrategyStr));
+ }
+ }
+ }
+
String clusterName = (String) resource.getPropertyValue(clusterNamePropertyId);
// Check liveliness of host
@@ -165,6 +202,9 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider {
for (TimelineMetric metric : timelineMetrics.getMetrics()) {
if (metric.getMetricName() != null && metric.getMetricValues() != null) {
+ // Pad zeros or nulls if needed
+ metricsPaddingMethod.applyPaddingStrategy(metric, temporalInfo);
+
String propertyId = propertyIdMap.get(metric.getMetricName());
if (propertyId != null) {
resource.setProperty(propertyId, getValue(metric, true));
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java
index cde8581..cefe953 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java
@@ -20,8 +20,10 @@ package org.apache.ambari.server.controller.utilities;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
@@ -34,6 +36,7 @@ import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.SortRequest;
import org.apache.ambari.server.controller.spi.TemporalInfo;
+import org.apache.commons.lang.StringUtils;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
@@ -49,6 +52,15 @@ public class PropertyHelper {
private static final String KEY_PROPERTIES_FILE = "key_properties.json";
private static final char EXTERNAL_PATH_SEP = '/';
+ /**
+ * Aggregate functions implicitly supported by the Metrics Service
+ */
+ public static final List<String> AGGREGATE_FUNCTION_IDENTIFIERS =
+ Arrays.asList("._sum", "._max", "._min", "._avg");
+
+ private static final List<Resource.InternalType> REPORT_METRIC_RESOURCES =
+ Arrays.asList(Resource.InternalType.Cluster, Resource.InternalType.Host);
+
private static final Map<Resource.InternalType, Set<String>> PROPERTY_IDS = readPropertyIds(PROPERTIES_FILE);
private static final Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>> JMX_PROPERTY_IDS = readPropertyProviderIds(JMX_PROPERTIES_FILE);
private static final Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>> GANGLIA_PROPERTY_IDS = readPropertyProviderIds(GANGLIA_PROPERTIES_FILE);
@@ -439,6 +451,9 @@ public class PropertyHelper {
}
componentMetrics.put(componentEntry.getKey(), metrics);
}
+ if (REPORT_METRIC_RESOURCES.contains(resourceEntry.getKey())) {
+ updateMetricsWithAggregateFunctionSupport(componentMetrics);
+ }
resourceMetrics.put(resourceEntry.getKey(), componentMetrics);
}
return resourceMetrics;
@@ -550,4 +565,65 @@ public class PropertyHelper {
return unit;
}
}
+
+ /**
+ * This method adds supported propertyInfo for component metrics with
+ * aggregate function ids. API calls with multiple aggregate functions
+ * applied to a single metric need this support.
+ *
+ * Currently this support is added only for Component & Report metrics.
+ * This can be easily extended by making the call from the appropriate
+ * sub class of: @AMSPropertyProvider.
+ *
+ */
+ public static void updateMetricsWithAggregateFunctionSupport(Map<String, Map<String, PropertyInfo>> metrics) {
+
+ if (metrics == null || metrics.isEmpty()) {
+ return;
+ }
+
+ // For every component
+ for (Map.Entry<String, Map<String, PropertyInfo>> metricInfoEntry : metrics.entrySet()) {
+ Map<String, PropertyInfo> metricInfo = metricInfoEntry.getValue();
+
+ Map<String, PropertyInfo> aggregateMetrics = new HashMap<String, PropertyInfo>();
+ // For every metric
+ for (Map.Entry<String, PropertyInfo> metricEntry : metricInfo.entrySet()) {
+ // For each aggregate function id
+ for (String identifierToAdd : AGGREGATE_FUNCTION_IDENTIFIERS) {
+ String metricInfoKey = metricEntry.getKey() + identifierToAdd;
+ // This disallows metric key suffix of the form "._sum._sum" for
+ // the sake of avoiding duplicates
+ if (metricInfo.containsKey(metricInfoKey)) {
+ continue;
+ }
+
+ PropertyInfo propertyInfo = metricEntry.getValue();
+ PropertyInfo metricInfoValue = new PropertyInfo(
+ propertyInfo.getPropertyId() + identifierToAdd,
+ propertyInfo.isTemporal(),
+ propertyInfo.isPointInTime());
+ metricInfoValue.setAmsHostMetric(propertyInfo.isAmsHostMetric());
+ metricInfoValue.setAmsId(!StringUtils.isEmpty(propertyInfo.getAmsId()) ?
+ propertyInfo.getAmsId() + identifierToAdd : null);
+ metricInfoValue.setUnit(propertyInfo.getUnit());
+
+ aggregateMetrics.put(metricInfoKey, metricInfoValue);
+ }
+ }
+ metricInfo.putAll(aggregateMetrics);
+ }
+ }
+
+ /**
+ * Check if property ends with a trailing suffix of function id.
+ */
+ public static boolean hasAggregateFunctionSuffix(String propertyId) {
+ for (String aggregateFunctionId : AGGREGATE_FUNCTION_IDENTIFIERS) {
+ if (propertyId.endsWith(aggregateFunctionId)) {
+ return true;
+ }
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
index 81c53cc..923f2f7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
@@ -47,6 +47,7 @@ import org.powermock.modules.junit4.PowerMockRunner;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -444,6 +445,7 @@ public class AMSPropertyProviderTest {
Map<String, Map<String, PropertyInfo>> propertyIds =
PropertyHelper.getMetricPropertyIds(Resource.Type.Component);
+ PropertyHelper.updateMetricsWithAggregateFunctionSupport(propertyIds);
AMSComponentPropertyProvider propertyProvider = new AMSComponentPropertyProvider(
propertyIds,
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java
index 8fba0e7..c0ce419 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java
@@ -42,6 +42,7 @@ public class AMSReportPropertyProviderTest {
private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("Clusters", "cluster_name");
private static final String FILE_PATH_PREFIX = "ams" + File.separator;
private static final String SINGLE_HOST_METRICS_FILE_PATH = FILE_PATH_PREFIX + "single_host_metric.json";
+ private static final String AGGREGATE_CLUSTER_METRICS_FILE_PATH = FILE_PATH_PREFIX + "aggregate_cluster_metrics.json";
@Test
public void testPopulateResources() throws Exception {
@@ -59,13 +60,13 @@ public class AMSReportPropertyProviderTest {
sslConfiguration,
metricHostProvider,
CLUSTER_NAME_PROPERTY_ID
- );
+ );
String propertyId = PropertyHelper.getPropertyId("metrics/cpu", "User");
Resource resource = new ResourceImpl(Resource.Type.Cluster);
resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1");
Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
- temporalInfoMap.put(propertyId, new TemporalInfoImpl(1416528819369L, 1416528819569L, 1L));
+ temporalInfoMap.put(propertyId, new TemporalInfoImpl(1416445244701L, 1416445244901L, 1L));
Request request = PropertyHelper.getReadRequest(
Collections.singleton(propertyId), temporalInfoMap);
Set<Resource> resources =
@@ -77,10 +78,51 @@ public class AMSReportPropertyProviderTest {
URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 8188);
uriBuilder.addParameter("metricNames", "cpu_user");
uriBuilder.addParameter("appId", "HOST");
- uriBuilder.addParameter("startTime", "1416528819369");
- uriBuilder.addParameter("endTime", "1416528819569");
+ uriBuilder.addParameter("startTime", "1416445244701");
+ uriBuilder.addParameter("endTime", "1416445244901");
Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec());
Number[][] val = (Number[][]) res.getPropertyValue("metrics/cpu/User");
Assert.assertEquals(111, val.length);
}
+
+ @Test
+ public void testPopulateResourceWithAggregateFunction() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider(AGGREGATE_CLUSTER_METRICS_FILE_PATH);
+ TestMetricHostProvider metricHostProvider = new TestMetricHostProvider();
+ ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class);
+
+ Map<String, Map<String, PropertyInfo>> propertyIds =
+ PropertyHelper.getMetricPropertyIds(Resource.Type.Cluster);
+
+ AMSReportPropertyProvider propertyProvider =
+ new AMSReportPropertyProvider(
+ propertyIds,
+ streamProvider,
+ sslConfiguration,
+ metricHostProvider,
+ CLUSTER_NAME_PROPERTY_ID
+ );
+
+ String propertyId = PropertyHelper.getPropertyId("metrics/cpu", "User._sum");
+ Resource resource = new ResourceImpl(Resource.Type.Cluster);
+ resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1");
+ Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+ temporalInfoMap.put(propertyId, new TemporalInfoImpl(1432033256912L, 1432033257912L, 1L));
+ Request request = PropertyHelper.getReadRequest(
+ Collections.singleton(propertyId), temporalInfoMap);
+ Set<Resource> resources =
+ propertyProvider.populateResources(Collections.singleton(resource), request, null);
+ Assert.assertEquals(1, resources.size());
+ Resource res = resources.iterator().next();
+ Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next());
+ Assert.assertNotNull(properties);
+ URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 8188);
+ uriBuilder.addParameter("metricNames", "cpu_user._sum");
+ uriBuilder.addParameter("appId", "HOST");
+ uriBuilder.addParameter("startTime", "1432033256912");
+ uriBuilder.addParameter("endTime", "1432033257912");
+ Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec());
+ Number[][] val = (Number[][]) res.getPropertyValue("metrics/cpu/User._sum");
+ Assert.assertEquals(91, val.length);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json b/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json
new file mode 100644
index 0000000..d3987cf
--- /dev/null
+++ b/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json
@@ -0,0 +1,100 @@
+{"metrics": [
+ {
+ "timestamp": 1432033257912,
+ "metricname": "cpu_user._sum",
+ "appid": "HOST",
+ "starttime": 1432033257912,
+ "metrics": {
+ "1432033257912": 10347.5,
+ "1432033287912": 11821.25,
+ "1432033317912": 10200.0,
+ "1432033347912": 12147.5,
+ "1432033377912": 10530.3125,
+ "1432033407912": 12243.75,
+ "1432033437912": 10220.0,
+ "1432033467912": 12512.5,
+ "1432033497912": 10302.5,
+ "1432033527912": 12165.625,
+ "1432033557912": 10200.0,
+ "1432033587912": 12478.75,
+ "1432033617912": 10299.6875,
+ "1432033647912": 12532.5,
+ "1432033677912": 10180.0,
+ "1432033707912": 12047.5,
+ "1432033737912": 10340.9375,
+ "1432033767912": 11628.75,
+ "1432033797912": 10140.0,
+ "1432033827912": 12151.25,
+ "1432033857912": 10375.3125,
+ "1432033887912": 11683.125,
+ "1432033917912": 10200.0,
+ "1432033947912": 12177.5,
+ "1432033977912": 10372.8125,
+ "1432034007912": 11806.25,
+ "1432034037912": 10180.0,
+ "1432034067912": 11233.75,
+ "1432034097912": 10389.0625,
+ "1432034127912": 11730.0,
+ "1432034157912": 10240.0,
+ "1432034187912": 11493.125,
+ "1432034217912": 10360.0,
+ "1432034247912": 11346.25,
+ "1432034277912": 10240.0,
+ "1432034307912": 11329.375,
+ "1432034337912": 10391.5625,
+ "1432034367912": 11148.125,
+ "1432034397912": 10240.0,
+ "1432034427912": 11193.125,
+ "1432034457912": 10399.6875,
+ "1432034487912": 11188.125,
+ "1432034517912": 10280.0,
+ "1432034547912": 11183.125,
+ "1432034577912": 10455.625,
+ "1432034607912": 12975.625,
+ "1432034637912": 10280.0,
+ "1432034667912": 13041.25,
+ "1432034697912": 10297.5,
+ "1432034727912": 13141.875,
+ "1432034757912": 10260.0,
+ "1432034787912": 12880.0,
+ "1432034817912": 10288.75,
+ "1432034847912": 13072.5,
+ "1432034877912": 10240.0,
+ "1432034907912": 13304.375,
+ "1432034937912": 10306.25,
+ "1432034967912": 13011.25,
+ "1432034997912": 10270.0,
+ "1432035027912": 12463.75,
+ "1432035057912": 10339.6875,
+ "1432035087912": 12656.875,
+ "1432035117912": 10270.0,
+ "1432035147912": 12428.125,
+ "1432035177912": 10347.8125,
+ "1432035207912": 12195.0,
+ "1432035237912": 10320.0,
+ "1432035267912": 11796.25,
+ "1432035297912": 10418.4375,
+ "1432035327912": 11716.25,
+ "1432035357912": 10390.0,
+ "1432035387912": 11400.0,
+ "1432035417912": 10463.125,
+ "1432035447912": 11177.5,
+ "1432035477912": 10390.0,
+ "1432035507912": 11073.125,
+ "1432035537912": 10460.9375,
+ "1432035567912": 10285.0,
+ "1432035597912": 10350.0,
+ "1432035627912": 10968.75,
+ "1432035657912": 10440.3125,
+ "1432035687912": 10471.25,
+ "1432035717912": 10380.0,
+ "1432035747912": 10379.375,
+ "1432035777912": 10483.4375,
+ "1432035807912": 10336.875,
+ "1432035837912": 10290.0,
+ "1432035867912": 10173.75,
+ "1432035897912": 10477.8125,
+ "1432035927912": 10346.875
+ }
+ }
+]}
\ No newline at end of file