You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2014/09/19 03:59:36 UTC
git commit: AMBARI-7398. Add metrics support to Slider View
Repository: ambari
Updated Branches:
refs/heads/trunk 01ff6a79b -> 7270881fb
AMBARI-7398. Add metrics support to Slider View
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7270881f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7270881f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7270881f
Branch: refs/heads/trunk
Commit: 7270881fbe18f1b115c2551ff86db290ddd406c2
Parents: 01ff6a7
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 18 18:59:09 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 18 18:59:09 2014 -0700
----------------------------------------------------------------------
contrib/views/slider/pom.xml | 9 +-
.../ambari/view/slider/GangliaMetric.java | 257 +++++++++++++++++++
.../ambari/view/slider/MetricsHolder.java | 44 ++++
.../apache/ambari/view/slider/SliderApp.java | 16 +-
.../ambari/view/slider/SliderAppType.java | 32 +--
.../slider/SliderAppsViewControllerImpl.java | 87 +++++--
.../apache/ambari/view/slider/TemporalInfo.java | 48 ++++
.../rest/client/SliderAppGangliaHelper.java | 196 ++++++++++++++
.../rest/client/SliderAppMasterClient.java | 127 +++++++--
9 files changed, 740 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/views/slider/pom.xml b/contrib/views/slider/pom.xml
index 8c33c93..49348df 100644
--- a/contrib/views/slider/pom.xml
+++ b/contrib/views/slider/pom.xml
@@ -331,6 +331,13 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ <exclusions>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
<version>${jcommander.version}</version>
@@ -397,7 +404,7 @@
<bigtop.version>0.7.0</bigtop.version>
<commons-configuration.version>1.6</commons-configuration.version>
<commons-lang.version>2.6</commons-lang.version>
- <httpclient.version>3.1</httpclient.version>
+ <httpclient.version>4.2.5</httpclient.version>
<jcommander.version>1.30</jcommander.version>
<curator.version>2.4.1</curator.version>
<zookeeper.version>3.4.5</zookeeper.version>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/GangliaMetric.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/GangliaMetric.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/GangliaMetric.java
new file mode 100644
index 0000000..cb51768
--- /dev/null
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/GangliaMetric.java
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.slider;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+@JsonIgnoreProperties(ignoreUnknown = true)
+
+
+/**
+ * Data structure for temporal data returned from Ganglia Web.
+ */
+public class GangliaMetric {
+
+ // Note that the member names correspond to the names in the JSON returned from Ganglia Web.
+
+ /**
+ * The name.
+ */
+ private String ds_name;
+
+ /**
+ * The ganglia cluster name.
+ */
+ private String cluster_name;
+
+ /**
+ * The graph type.
+ */
+ private String graph_type;
+
+ /**
+ * The host name.
+ */
+ private String host_name;
+
+ /**
+ * The metric name.
+ */
+ private String metric_name;
+
+ /**
+ * The temporal data points.
+ */
+ private Number[][] datapoints;
+
+
+ private static final Set<String> PERCENTAGE_METRIC;
+
+ //BUG-3386 Cluster CPU Chart is off the charts
+ // Here can be added other percentage metrics
+ static {
+ Set<String> temp = new HashSet<String>();
+ temp.add("cpu_wio");
+ temp.add("cpu_idle");
+ temp.add("cpu_nice");
+ temp.add("cpu_aidle");
+ temp.add("cpu_system");
+ temp.add("cpu_user");
+ PERCENTAGE_METRIC = Collections.unmodifiableSet(temp);
+ }
+
+
+ // ----- GangliaMetric -----------------------------------------------------
+
+ public String getDs_name() {
+ return ds_name;
+ }
+
+ public void setDs_name(String ds_name) {
+ this.ds_name = ds_name;
+ }
+
+ public String getCluster_name() {
+ return cluster_name;
+ }
+
+ public void setCluster_name(String cluster_name) {
+ this.cluster_name = cluster_name;
+ }
+
+ public String getGraph_type() {
+ return graph_type;
+ }
+
+ public void setGraph_type(String graph_type) {
+ this.graph_type = graph_type;
+ }
+
+ public String getHost_name() {
+ return host_name;
+ }
+
+ public void setHost_name(String host_name) {
+ this.host_name = host_name;
+ }
+
+ public String getMetric_name() {
+ return metric_name;
+ }
+
+ public void setMetric_name(String metric_name) {
+ this.metric_name = metric_name;
+ }
+
+ public Number[][] getDatapoints() {
+ return datapoints;
+ }
+
+
+ public void setDatapoints(Number[][] datapoints) {
+ this.datapoints = datapoints;
+ }
+
+ public void setDatapointsFromList(List<GangliaMetric.TemporalMetric> listTemporalMetrics) {
+ //this.datapoints = datapoints;
+ Number[][] datapointsArray = new Number[listTemporalMetrics.size()][2];
+ int cnt = 0;
+ if (PERCENTAGE_METRIC.contains(metric_name)) {
+ int firstIndex = 0;
+ int lastIndex = listTemporalMetrics.size() - 1;
+ for (int i = firstIndex; i <= lastIndex; ++i) {
+ GangliaMetric.TemporalMetric m = listTemporalMetrics.get(i);
+ Number val = m.getValue();
+ if (100.0 >= val.doubleValue()) {
+ datapointsArray[cnt][0] = val;
+ datapointsArray[cnt][1] = m.getTime();
+ cnt++;
+ }
+ }
+ } else {
+ int firstIndex = 0;
+ int lastIndex = listTemporalMetrics.size() - 1;
+ for (int i = firstIndex; i <= lastIndex; ++i) {
+ GangliaMetric.TemporalMetric m = listTemporalMetrics.get(i);
+ datapointsArray[i][0] = m.getValue();
+ datapointsArray[i][1] = m.getTime();
+ cnt++;
+ }
+ }
+
+ this.datapoints = new Number[cnt][2];
+ for (int i = 0; i < this.datapoints.length; i++) {
+ this.datapoints[i][0] = datapointsArray[i][0];
+ this.datapoints[i][1] = datapointsArray[i][1];
+ }
+
+ }
+
+ // ----- Object overrides --------------------------------------------------
+
+ @Override
+ public String toString() {
+ StringBuilder stringBuilder = new StringBuilder();
+
+ stringBuilder.append("\n");
+ stringBuilder.append("name=");
+ stringBuilder.append(ds_name);
+ stringBuilder.append("\n");
+ stringBuilder.append("cluster name=");
+ stringBuilder.append(cluster_name);
+ stringBuilder.append("\n");
+ stringBuilder.append("graph type=");
+ stringBuilder.append(graph_type);
+ stringBuilder.append("\n");
+ stringBuilder.append("host name=");
+ stringBuilder.append(host_name);
+ stringBuilder.append("\n");
+ stringBuilder.append("api name=");
+ stringBuilder.append(metric_name);
+ stringBuilder.append("\n");
+
+ stringBuilder.append("datapoints (value/timestamp):");
+ stringBuilder.append("\n");
+
+
+ boolean first = true;
+ stringBuilder.append("[");
+ for (Number[] m : datapoints) {
+ if (!first) {
+ stringBuilder.append(",");
+ }
+ stringBuilder.append("[");
+ stringBuilder.append(m[0]);
+ stringBuilder.append(",");
+ stringBuilder.append(m[1].longValue());
+ stringBuilder.append("]");
+ first = false;
+ }
+ stringBuilder.append("]");
+
+ return stringBuilder.toString();
+ }
+
+ public static class TemporalMetric {
+ private Number m_value;
+ private Number m_time;
+ private boolean valid;
+
+ public boolean isValid() {
+ return valid;
+ }
+
+ public TemporalMetric(String value, Number time) {
+ valid = true;
+ try{
+ m_value = convertToNumber(value);
+ } catch (NumberFormatException e) {
+ valid = false;
+ }
+ m_time = time;
+ }
+
+ public Number getValue() {
+ return m_value;
+ }
+
+ public Number getTime() {
+ return m_time;
+ }
+
+ private Number convertToNumber(String s) throws NumberFormatException {
+ Number res;
+ if(s.contains(".")){
+ Double d = Double.parseDouble(s);
+ if(d.isNaN() || d.isInfinite()){
+ throw new NumberFormatException(s);
+ } else {
+ res = d;
+ }
+ } else {
+ res = Long.parseLong(s);
+ }
+ return res;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/MetricsHolder.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/MetricsHolder.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/MetricsHolder.java
new file mode 100644
index 0000000..29f35b0
--- /dev/null
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/MetricsHolder.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.slider;
+
+import org.apache.ambari.view.slider.rest.client.Metric;
+
+import java.util.Map;
+
+public class MetricsHolder {
+ private Map<String, Map<String, Map<String, Metric>>> jmxMetrics;
+ private Map<String, Map<String, Map<String, Metric>>> gangliaMetrics;
+
+ public Map<String, Map<String, Map<String, Metric>>> getJmxMetrics() {
+ return jmxMetrics;
+ }
+
+ public void setJmxMetrics(Map<String, Map<String, Map<String, Metric>>> jmxMetrics) {
+ this.jmxMetrics = jmxMetrics;
+ }
+
+ public Map<String, Map<String, Map<String, Metric>>> getGangliaMetrics() {
+ return gangliaMetrics;
+ }
+
+ public void setGangliaMetrics(Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) {
+ this.gangliaMetrics = gangliaMetrics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java
index 6276299..d75762e 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java
@@ -18,6 +18,7 @@
package org.apache.ambari.view.slider;
+import java.util.List;
import java.util.Map;
public class SliderApp {
@@ -38,8 +39,9 @@ public class SliderApp {
private Map<String, String> urls;
private Map<String, Map<String, String>> configs;
private Map<String, SliderAppComponent> components;
- private Map<String, Object> metrics;
+ private Map<String, Number[][]> metrics;
private Map<String, Object> alerts;
+ private List<String> supportedMetrics;
public String getName() {
return name;
@@ -129,14 +131,22 @@ public class SliderApp {
this.components = components;
}
- public Map<String, Object> getMetrics() {
+ public Map<String, Number[][]> getMetrics() {
return metrics;
}
- public void setMetrics(Map<String, Object> metrics) {
+ public void setMetrics(Map<String, Number[][]> metrics) {
this.metrics = metrics;
}
+ public List<String> getSupportedMetrics() {
+ return supportedMetrics;
+ }
+
+ public void setSupportedMetrics(List<String> supportedMetrics) {
+ this.supportedMetrics = supportedMetrics;
+ }
+
public long getStartTime() {
return startTime;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java
index 528c1fe..82386af 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java
@@ -18,14 +18,9 @@
package org.apache.ambari.view.slider;
-import org.apache.ambari.view.slider.rest.client.Metric;
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-
import java.util.List;
import java.util.Map;
-@JsonIgnoreProperties({ "jmxMetrics", "gangliaMetrics" })
public class SliderAppType {
private String id;
private String typeName;
@@ -34,27 +29,14 @@ public class SliderAppType {
private Map<String, String> typeConfigs;
private List<SliderAppTypeComponent> typeComponents;
private String typePackageFileName;
- @JsonIgnore
- private Map<String, Map<String, Map<String, Metric>>> jmxMetrics;
- @JsonIgnore
- private Map<String, Map<String, Map<String, Metric>>> gangliaMetrics;
-
- @JsonIgnore
- public Map<String, Map<String, Map<String, Metric>>> getJmxMetrics() {
- return jmxMetrics;
- }
+ private List<String> supportedMetrics;
- public void setJmxMetrics(Map<String, Map<String, Map<String, Metric>>> jmxMetrics) {
- this.jmxMetrics = jmxMetrics;
+ public List<String> getSupportedMetrics() {
+ return supportedMetrics;
}
- @JsonIgnore
- public Map<String, Map<String, Map<String, Metric>>> getGangliaMetrics() {
- return gangliaMetrics;
- }
-
- public void setGangliaMetrics(Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) {
- this.gangliaMetrics = gangliaMetrics;
+ public void setSupportedMetrics(List<String> supportedMetrics) {
+ this.supportedMetrics = supportedMetrics;
}
public String getId() {
@@ -113,4 +95,8 @@ public class SliderAppType {
this.typePackageFileName = typePackageFileName;
}
+ public String uniqueName() {
+ return getTypeName() + "-" + getTypeVersion();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
index ef11cfa..6580d90 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
@@ -85,12 +85,14 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
private static final Logger logger = Logger
.getLogger(SliderAppsViewControllerImpl.class);
+ private static String METRICS_PREFIX = "metrics/";
@Inject
private ViewContext viewContext;
private List<SliderAppType> appTypes;
private Integer createAppCounter = -1;
@Inject
private SliderAppsAlerts sliderAlerts;
+ private Map<String, MetricsHolder> appMetrics = new HashMap<String, MetricsHolder>();
private String getAppsFolderPath() {
return viewContext.getAmbariProperty("resources.dir") + "/apps";
@@ -221,10 +223,25 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
}
if (properties != null && !properties.isEmpty()) {
+ SliderAppType matchedAppType = null;
+ List<SliderAppType> matchingAppTypes = getSliderAppTypes(null);
+ if (matchingAppTypes != null && matchingAppTypes.size() > 0) {
+ for (SliderAppType appType : matchingAppTypes) {
+ if ((appType.getTypeName() != null && appType.getTypeName()
+ .equalsIgnoreCase(app.getType()))
+ && (appType.getTypeVersion() != null && appType.getTypeVersion()
+ .equalsIgnoreCase(app.getAppVersion()))) {
+ matchedAppType = appType;
+ break;
+ }
+ }
+ }
+
SliderAppMasterClient sliderAppClient = yarnApp.getTrackingUrl() == null ? null
: new SliderAppMasterClient(yarnApp.getTrackingUrl());
SliderAppMasterData appMasterData = null;
Map<String, String> quickLinks = new HashMap<String, String>();
+ Set<String> gangliaMetrics = new HashSet<String>();
for (String property : properties) {
if ("RUNNING".equals(app.getState())) {
if (sliderAppClient != null) {
@@ -248,25 +265,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
if (quickLinks != null && quickLinks.containsKey("JMX")) {
String jmxUrl = quickLinks.get("JMX");
- List<SliderAppType> appTypes = getSliderAppTypes(null);
- if (appTypes != null && appTypes.size() > 0) {
- for (SliderAppType appType : appTypes) {
- if (logger.isDebugEnabled()) {
- logger.debug("TYPE: " + appType.getTypeName() + " "
- + app.getType());
- logger.debug("VERSION: " + appType.getTypeVersion() + " "
- + app.getAppVersion());
- }
- if ((appType.getTypeName() != null && appType.getTypeName()
- .equalsIgnoreCase(app.getType()))
- && (appType.getTypeVersion() != null && appType
- .getTypeVersion().equalsIgnoreCase(
- app.getAppVersion()))) {
- app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext,
- appType));
- break;
- }
- }
+ if (matchedAppType != null) {
+ MetricsHolder metricsHolder = appMetrics.get(matchedAppType
+ .uniqueName());
+ app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext,
+ matchedAppType, metricsHolder));
}
}
Map<String, Map<String, String>> configs = sliderAppClient
@@ -350,10 +353,29 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
+ yarnApp.getName(), e);
throw new RuntimeException(e.getMessage(), e);
}
+ } else if (property.startsWith(METRICS_PREFIX)) {
+ gangliaMetrics.add(property.substring(METRICS_PREFIX.length()));
+ } else if ("supportedMetrics".equals(property)) {
+ if (matchedAppType != null) {
+ app.setSupportedMetrics(matchedAppType.getSupportedMetrics());
+ }
}
}
}
}
+ if (gangliaMetrics.size() > 0) {
+ if (quickLinks.isEmpty()) {
+ quickLinks = sliderAppClient
+ .getQuickLinks(appMasterData.publisherUrl);
+ }
+ if (quickLinks != null && quickLinks.containsKey("Metrics")) {
+ String metricsUrl = quickLinks.get("Metrics");
+ MetricsHolder metricsHolder = appMetrics.get(matchedAppType
+ .uniqueName());
+ app.setMetrics(sliderAppClient.getGangliaMetrics(metricsUrl,
+ gangliaMetrics, null, viewContext, matchedAppType, metricsHolder));
+ }
+ }
}
return app;
}
@@ -626,9 +648,14 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
appTypeComponentList.add(appTypeComponent);
}
- appType.setJmxMetrics(readMetrics(zipFile, "jmx_metrics.json"));
- appType.setGangliaMetrics(readMetrics(zipFile,
+ MetricsHolder metricsHolder = new MetricsHolder();
+ metricsHolder.setJmxMetrics(readMetrics(zipFile,
+ "jmx_metrics.json"));
+ metricsHolder.setGangliaMetrics(readMetrics(zipFile,
"ganglia_metrics.json"));
+ appType.setSupportedMetrics(getSupportedMetrics(metricsHolder
+ .getGangliaMetrics()));
+ appMetrics.put(appType.uniqueName(), metricsHolder);
appType.setTypeComponents(appTypeComponentList);
appTypes.add(appType);
@@ -644,19 +671,33 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
return appTypes;
}
+ private List<String> getSupportedMetrics(
+ Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) {
+ Set<String> supportedMetrics = new HashSet<String>();
+ if (gangliaMetrics != null && gangliaMetrics.size() > 0) {
+ for (Map<String, Map<String, Metric>> compMetrics : gangliaMetrics
+ .values()) {
+ for (Map<String, Metric> metrics : compMetrics.values()) {
+ supportedMetrics.addAll(metrics.keySet());
+ }
+ }
+ }
+ return new ArrayList<String>(supportedMetrics);
+ }
+
Map<String, Map<String, Map<String, Metric>>> readMetrics(ZipFile zipFile,
String fileName) {
Map<String, Map<String, Map<String, Metric>>> metrics = null;
try {
InputStream inputStream = zipFile.getInputStream(zipFile
- .getEntry("jmx_metrics.json"));
+ .getEntry(fileName));
ObjectMapper mapper = new ObjectMapper();
metrics = mapper.readValue(inputStream,
new TypeReference<Map<String, Map<String, Map<String, Metric>>>>() {
});
} catch (IOException e) {
- logger.info("Error reading metrics. " + e.getMessage());
+ logger.info("Error reading metrics for file " + fileName + ". " + e.getMessage());
}
return metrics;
http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/TemporalInfo.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/TemporalInfo.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/TemporalInfo.java
new file mode 100644
index 0000000..a501615
--- /dev/null
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/TemporalInfo.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.slider;
+
+/**
+ * Temporal query data.
+ */
+public interface TemporalInfo {
+ /**
+ * Get the start of the requested time range. The time is given in
+ * seconds since the Unix epoch.
+ *
+ * @return the start time in seconds
+ */
+ Long getStartTime();
+
+ /**
+ * Get the end of the requested time range. The time is given in
+ * seconds since the Unix epoch.
+ *
+ * @return the end time in seconds
+ */
+ Long getEndTime();
+
+ /**
+ * Get the requested time between each data point of the temporal
+ * data. The time is given in seconds.
+ *
+ * @return the step time in seconds
+ */
+ Long getStep();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java
new file mode 100644
index 0000000..fba7d23
--- /dev/null
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.slider.rest.client;
+
+import org.apache.ambari.view.SystemException;
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.slider.GangliaMetric;
+import org.apache.ambari.view.slider.TemporalInfo;
+import org.apache.log4j.Logger;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class SliderAppGangliaHelper {
+
+ private static final Logger logger = Logger
+ .getLogger(SliderAppGangliaHelper.class);
+
+ private static String getSetString(Set<String> set, int limit) {
+ StringBuilder sb = new StringBuilder();
+
+ if (limit == -1 || set.size() <= limit) {
+ for (String cluster : set) {
+ if (sb.length() > 0) {
+ sb.append(",");
+ }
+ sb.append(cluster);
+ }
+ }
+ return sb.toString();
+ }
+
+ private static Number convertToNumber(String s) {
+ return s.contains(".") ? Double.parseDouble(s) : Long.parseLong(s);
+ }
+
+ public static Map<String, GangliaMetric> getGangliaMetrics(ViewContext context,
+ String spec,
+ String params) throws IOException {
+ Map<String, GangliaMetric> receivedMetrics = new HashMap<String, GangliaMetric>();
+ Map<String, String> headers = new HashMap<String, String>();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(
+ context.getURLStreamProvider().readFrom(spec, "POST", params, headers)));
+
+ String feedStart = reader.readLine();
+ if (feedStart == null || feedStart.isEmpty()) {
+ logger.info("Empty feed while getting ganglia metrics for spec => " +
+ spec);
+ return null;
+ }
+ int startTime = convertToNumber(feedStart).intValue();
+
+ String dsName = reader.readLine();
+ if (dsName == null || dsName.isEmpty()) {
+ logger.info("Feed without body while reading ganglia metrics for spec " +
+ "=> " + spec);
+ return null;
+ }
+
+ while (!"[~EOF]".equals(dsName)) {
+ GangliaMetric metric = new GangliaMetric();
+ List<GangliaMetric.TemporalMetric> listTemporalMetrics =
+ new ArrayList<GangliaMetric.TemporalMetric>();
+
+ metric.setDs_name(dsName);
+ metric.setCluster_name(reader.readLine());
+ metric.setHost_name(reader.readLine());
+ metric.setMetric_name(reader.readLine());
+
+ String timeStr = reader.readLine();
+ String stepStr = reader.readLine();
+ if (timeStr == null || timeStr.isEmpty() || stepStr == null
+ || stepStr.isEmpty()) {
+ logger.info("Unexpected end of stream reached while getting ganglia " +
+ "metrics for spec => " + spec);
+ return null;
+ }
+ int time = convertToNumber(timeStr).intValue();
+ int step = convertToNumber(stepStr).intValue();
+
+ String val = reader.readLine();
+ String lastVal = null;
+
+ while (val != null && !"[~EOM]".equals(val)) {
+ if (val.startsWith("[~r]")) {
+ Integer repeat = Integer.valueOf(val.substring(4)) - 1;
+ for (int i = 0; i < repeat; ++i) {
+ if (!"[~n]".equals(lastVal)) {
+ GangliaMetric.TemporalMetric tm = new GangliaMetric.TemporalMetric(lastVal, time);
+ if (tm.isValid()) listTemporalMetrics.add(tm);
+ }
+ time += step;
+ }
+ } else {
+ if (!"[~n]".equals(val)) {
+ GangliaMetric.TemporalMetric tm = new GangliaMetric.TemporalMetric(val, time);
+ if (tm.isValid()) listTemporalMetrics.add(tm);
+ }
+ time += step;
+ }
+ lastVal = val;
+ val = reader.readLine();
+ }
+
+ metric.setDatapointsFromList(listTemporalMetrics);
+ receivedMetrics.put(metric.getMetric_name(), metric);
+
+ dsName = reader.readLine();
+ if (dsName == null || dsName.isEmpty()) {
+ logger.info("Unexpected end of stream reached while getting ganglia " +
+ "metrics for spec => " + spec);
+ return null;
+ }
+ }
+ String feedEnd = reader.readLine();
+ if (feedEnd == null || feedEnd.isEmpty()) {
+ logger.info("Error reading end of feed while getting ganglia metrics " +
+ "for spec => " + spec);
+ } else {
+
+ int endTime = convertToNumber(feedEnd).intValue();
+ int totalTime = endTime - startTime;
+ if (logger.isInfoEnabled() && totalTime > 3) {
+ logger.info("Ganglia resource population time: " + totalTime);
+ }
+ }
+ return receivedMetrics;
+ }
+
+ public static String getSpec(String gangliaUrl,
+ Set<String> metricSet,
+ TemporalInfo temporalInfo) throws SystemException, URISyntaxException {
+
+ String metrics = getSetString(metricSet, -1);
+
+ URIBuilder uriBuilder = new URIBuilder(gangliaUrl);
+
+ uriBuilder.setParameter("h", "__SummaryInfo__");
+
+ if (metrics.length() > 0) {
+ uriBuilder.setParameter("m", metrics);
+ } else {
+ // get all metrics
+ uriBuilder.setParameter("m", ".*");
+ }
+
+ if (temporalInfo != null) {
+ long startTime = temporalInfo.getStartTime();
+ if (startTime != -1) {
+ uriBuilder.setParameter("s", String.valueOf(startTime));
+ }
+
+ long endTime = temporalInfo.getEndTime();
+ if (endTime != -1) {
+ uriBuilder.setParameter("e", String.valueOf(endTime));
+ }
+
+ long step = temporalInfo.getStep();
+ if (step != -1) {
+ uriBuilder.setParameter("r", String.valueOf(step));
+ }
+ } else {
+ long endTime = System.currentTimeMillis() / 1000;
+ long startTime = System.currentTimeMillis() / 1000 - 60 * 60;
+ uriBuilder.setParameter("e", String.valueOf(endTime));
+ uriBuilder.setParameter("s", String.valueOf(startTime));
+ uriBuilder.setParameter("r", "15");
+ }
+
+ return uriBuilder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java
index ab6289c..8bc7400 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java
@@ -22,14 +22,19 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.ambari.view.URLStreamProvider;
import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.slider.GangliaMetric;
+import org.apache.ambari.view.slider.MetricsHolder;
import org.apache.ambari.view.slider.SliderAppType;
import org.apache.ambari.view.slider.SliderAppTypeComponent;
+import org.apache.ambari.view.slider.TemporalInfo;
import org.apache.commons.httpclient.HttpException;
import org.apache.log4j.Logger;
@@ -94,7 +99,7 @@ public class SliderAppMasterClient extends BaseHttpClient {
quickLinks.put("JMX", entry.getValue().getAsString());
} else if ("org.apache.slider.monitor".equals(entry.getKey())) {
quickLinks.put("UI", entry.getValue().getAsString());
- } else if ("org.apache.slider.metrics".equals(entry.getKey())) {
+ } else if ("app.metrics".equals(entry.getKey())) {
quickLinks.put("Metrics", entry.getValue().getAsString());
} else {
quickLinks.put(entry.getKey(), entry.getValue().getAsString());
@@ -148,17 +153,85 @@ public class SliderAppMasterClient extends BaseHttpClient {
}
}
+ public Map<String, Number[][]> getGangliaMetrics(String gangliaUrl,
+ Set<String> metricsRequested,
+ TemporalInfo temporalInfo,
+ ViewContext context,
+ SliderAppType appType,
+ MetricsHolder metricsHolder) {
+ Map<String, Number[][]> retVal = new HashMap<String, Number[][]>();
+
+ if (appType == null || metricsHolder == null || metricsHolder.getGangliaMetrics() == null) {
+ logger.info("AppType must be provided and it must contain ganglia_metrics.json to extract jmx properties");
+ return retVal;
+ }
+
+ Map<String, GangliaMetric> receivedMetrics = null;
+ List<String> components = new ArrayList<String>();
+ for (SliderAppTypeComponent appTypeComponent : appType.getTypeComponents()) {
+ components.add(appTypeComponent.getName());
+ }
+
+ Map<String, Map<String, Map<String, Metric>>> metrics = metricsHolder.getGangliaMetrics();
+ Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, components);
+ Set<String> metricsToRead = new HashSet<String>();
+ Map<String, String> reverseNameLookup = new HashMap<String, String>();
+ for (String key : relevantMetrics.keySet()) {
+ if (metricsRequested.contains(key)) {
+ String metricName = relevantMetrics.get(key).getMetric();
+ metricsToRead.add(metricName);
+ reverseNameLookup.put(metricName, key);
+ }
+ }
+
+ if (metricsToRead.size() != 0) {
+ try {
+ String specWithParams = SliderAppGangliaHelper.getSpec(gangliaUrl, metricsToRead, temporalInfo);
+ logger.info("Using spec: " + specWithParams);
+ if (specWithParams != null) {
+
+ String spec = null;
+ String params = null;
+ String[] tokens = specWithParams.split("\\?", 2);
+
+ try {
+ spec = tokens[0];
+ params = tokens[1];
+ } catch (ArrayIndexOutOfBoundsException e) {
+ logger.info(e.toString());
+ }
+
+ receivedMetrics = SliderAppGangliaHelper.getGangliaMetrics(context, spec, params);
+ }
+ } catch (Exception e) {
+ logger.warn("Unable to retrieve ganglia metrics. " + e.getMessage());
+ }
+ }
+
+ if (receivedMetrics != null) {
+ for (GangliaMetric metric : receivedMetrics.values()) {
+ if (reverseNameLookup.containsKey(metric.getMetric_name())) {
+ retVal.put(reverseNameLookup.get(metric.getMetric_name()), metric.getDatapoints());
+ }
+ }
+ }
+
+ return retVal;
+ }
+
/**
* Provides only the interesting JMX metric names and values.
- *
+ *
* @param jmxUrl
- *
+ *
* @return
*/
- public Map<String, String> getJmx(String jmxUrl, ViewContext context,
- SliderAppType appType) {
+ public Map<String, String> getJmx(String jmxUrl,
+ ViewContext context,
+ SliderAppType appType,
+ MetricsHolder metricsHolder) {
Map<String, String> jmxProperties = new HashMap<String, String>();
- if (appType == null || appType.getJmxMetrics() == null) {
+ if (appType == null || metricsHolder == null || metricsHolder.getJmxMetrics() == null) {
logger
.info("AppType must be provided and it must contain jmx_metrics.json to extract jmx properties");
return jmxProperties;
@@ -169,12 +242,14 @@ public class SliderAppMasterClient extends BaseHttpClient {
components.add(appTypeComponent.getName());
}
- Map<String, Map<String, Map<String, Metric>>> metrics = appType
- .getJmxMetrics();
- Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics,
- components);
- SliderAppJmxHelper.JMXTypes jmxType = SliderAppJmxHelper
- .jmxTypeExpected(relevantMetrics);
+ Map<String, Map<String, Map<String, Metric>>> metrics = metricsHolder.getJmxMetrics();
+ Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, components);
+ if (relevantMetrics.size() == 0) {
+ logger.info("No metrics found for components defined in the app.");
+ return jmxProperties;
+ }
+
+ SliderAppJmxHelper.JMXTypes jmxType = SliderAppJmxHelper.jmxTypeExpected(relevantMetrics);
if (jmxType == null) {
logger
.info("jmx_metrics.json is malformed. It may have mixed metric key types of unsupported metric key types.");
@@ -195,20 +270,20 @@ public class SliderAppMasterClient extends BaseHttpClient {
if (jmxStream != null) {
switch (jmxType) {
- case JMX_BEAN:
- SliderAppJmxHelper.extractMetricsFromJmxBean(jmxStream, jmxUrl,
- jmxProperties, relevantMetrics);
- break;
- case JSON:
- SliderAppJmxHelper.extractMetricsFromJmxJson(jmxStream, jmxUrl,
- jmxProperties, relevantMetrics);
- break;
- case XML:
- SliderAppJmxHelper.extractMetricsFromJmxXML(jmxStream, jmxUrl,
- jmxProperties, relevantMetrics);
- break;
- default:
- logger.info("Unsupported jmx type.");
+ case JMX_BEAN:
+ SliderAppJmxHelper.extractMetricsFromJmxBean(jmxStream, jmxUrl,
+ jmxProperties, relevantMetrics);
+ break;
+ case JSON:
+ SliderAppJmxHelper.extractMetricsFromJmxJson(jmxStream, jmxUrl,
+ jmxProperties, relevantMetrics);
+ break;
+ case XML:
+ SliderAppJmxHelper.extractMetricsFromJmxXML(jmxStream, jmxUrl,
+ jmxProperties, relevantMetrics);
+ break;
+ default:
+ logger.info("Unsupported jmx type.");
}
}
} catch (Exception e) {