You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/05/23 03:46:19 UTC
git commit: AMBARI-5863. Slider Apps view API for app does not
populate subresources. (srimanth)
Repository: ambari
Updated Branches:
refs/heads/trunk f715c3a4f -> f58782c74
AMBARI-5863. Slider Apps view API for app does not populate subresources. (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f58782c7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f58782c7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f58782c7
Branch: refs/heads/trunk
Commit: f58782c7405930d6629f6f819d84d64d32d92a97
Parents: f715c3a
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Thu May 22 18:36:36 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Thu May 22 18:36:52 2014 -0700
----------------------------------------------------------------------
.../slider/slider/0.30.0/slider-0.30.0.jar | Bin 1024630 -> 1029968 bytes
.../view/slider/SliderAppsResourceProvider.java | 4 +-
.../view/slider/SliderAppsViewController.java | 30 +++-
.../slider/SliderAppsViewControllerImpl.java | 140 ++++++++++++++--
.../view/slider/rest/client/BaseHttpClient.java | 20 +++
.../rest/client/SliderAppMasterClient.java | 161 +++++++++++++++++++
6 files changed, 340 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar
----------------------------------------------------------------------
diff --git a/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar b/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar
index 7c0ac2d..6e7b8d1 100644
Binary files a/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar and b/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar differ
http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java
index c5237fc..138a1ab 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java
@@ -62,7 +62,7 @@ public class SliderAppsResourceProvider implements ResourceProvider<SliderApp> {
throws SystemException, NoSuchResourceException,
UnsupportedPropertyException {
try {
- SliderApp sliderApp = sliderController.getSliderApp(resourceId);
+ SliderApp sliderApp = sliderController.getSliderApp(resourceId, properties);
if (sliderApp == null)
throw new NoSuchResourceException(resourceId);
return sliderApp;
@@ -81,7 +81,7 @@ public class SliderAppsResourceProvider implements ResourceProvider<SliderApp> {
UnsupportedPropertyException {
Set<SliderApp> appSet = new HashSet<SliderApp>();
try {
- List<SliderApp> sliderApps = sliderController.getSliderApps();
+ List<SliderApp> sliderApps = sliderController.getSliderApps(request.getPropertyIds());
for (SliderApp app : sliderApps)
appSet.add(app);
} catch (YarnException e) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java
index 3cc7ab8..90ff76c 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java
@@ -20,6 +20,7 @@ package org.apache.ambari.view.slider;
import java.io.IOException;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -30,7 +31,30 @@ public interface SliderAppsViewController {
public ViewStatus getViewStatus();
- public SliderApp getSliderApp(String applicationId) throws YarnException, IOException;
-
- public List<SliderApp> getSliderApps() throws YarnException, IOException;
+ /**
+ * Provides information about requested Slider App.
+ *
+ * @param applicationId
+ * @param properties
+ * Identifies specific properties to show up. Provide
+ * <code>null</code> for default properties.
+ * @return
+ * @throws YarnException
+ * @throws IOException
+ */
+ public SliderApp getSliderApp(String applicationId, Set<String> properties)
+ throws YarnException, IOException;
+
+ /**
+ * Provides list of Slider apps with requested properties populated.
+ *
+ * @param properties
+ * Identifies specific properties to show up. Provide
+ * <code>null</code> for default properties.
+ * @return
+ * @throws YarnException
+ * @throws IOException
+ */
+ public List<SliderApp> getSliderApps(Set<String> properties)
+ throws YarnException, IOException;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
index 8c1da9c..6a884ce 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
@@ -20,8 +20,11 @@ package org.apache.ambari.view.slider;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.slider.clients.AmbariClient;
@@ -30,6 +33,8 @@ import org.apache.ambari.view.slider.clients.AmbariClusterInfo;
import org.apache.ambari.view.slider.clients.AmbariHostComponent;
import org.apache.ambari.view.slider.clients.AmbariService;
import org.apache.ambari.view.slider.clients.AmbariServiceInfo;
+import org.apache.ambari.view.slider.rest.client.SliderAppMasterClient;
+import org.apache.ambari.view.slider.rest.client.SliderAppMasterClient.SliderAppMasterData;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -37,7 +42,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.log4j.Logger;
+import org.apache.slider.api.ClusterDescription;
import org.apache.slider.client.SliderClient;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -137,8 +145,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
@Override
- public SliderApp getSliderApp(String applicationId) throws YarnException,
- IOException {
+ public SliderApp getSliderApp(String applicationId, Set<String> properties)
+ throws YarnException, IOException {
if (applicationId != null) {
int index = applicationId.indexOf('_');
if (index > -1 && index < applicationId.length() - 1) {
@@ -150,9 +158,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
Thread.currentThread().setContextClassLoader(
getClass().getClassLoader());
try {
- ApplicationReport yarnApp = getSliderClient().getApplicationReport(
- appId);
- return mapToSliderApp(yarnApp);
+ SliderClient sliderClient = getSliderClient();
+ ApplicationReport yarnApp = sliderClient.getApplicationReport(appId);
+ return populateSliderApp(yarnApp, properties, sliderClient);
} finally {
Thread.currentThread().setContextClassLoader(currentClassLoader);
}
@@ -161,7 +169,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
return null;
}
- private SliderApp mapToSliderApp(ApplicationReport yarnApp) {
+ private SliderApp populateSliderApp(ApplicationReport yarnApp,
+ Set<String> properties, SliderClient sliderClient) {
if (yarnApp == null)
return null;
SliderApp app = new SliderApp();
@@ -175,6 +184,110 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
app.setType(yarnApp.getApplicationType());
app.setStartTime(yarnApp.getStartTime());
app.setEndTime(yarnApp.getFinishTime());
+ if (properties != null && !properties.isEmpty()) {
+ SliderAppMasterClient sliderAppClient = yarnApp.getTrackingUrl() == null ? null
+ : new SliderAppMasterClient(yarnApp.getTrackingUrl());
+ SliderAppMasterData appMasterData = null;
+ Map<String, String> quickLinks = new HashMap<String, String>();
+ for (String property : properties) {
+ if ("RUNNING".equals(app.getState())) {
+ if (sliderAppClient != null) {
+ if (appMasterData == null)
+ appMasterData = sliderAppClient.getAppMasterData();
+ if ("urls".equals(property.toLowerCase())) {
+ if (quickLinks.isEmpty())
+ quickLinks = sliderAppClient
+ .getQuickLinks(appMasterData.publisherUrl);
+ app.setUrls(quickLinks);
+ } else if ("configs".equals(property.toLowerCase())) {
+ Map<String, Map<String, String>> configs = sliderAppClient
+ .getConfigs(appMasterData.publisherUrl);
+ app.setConfigs(configs);
+ } else if ("jmx".equals(property.toLowerCase())) {
+ if (quickLinks.isEmpty())
+ quickLinks = sliderAppClient
+ .getQuickLinks(appMasterData.publisherUrl);
+ if (quickLinks != null && quickLinks.containsKey("JMX")) {
+ String jmxUrl = quickLinks.get("JMX");
+ app.setJmx(sliderAppClient.getJmx(jmxUrl));
+ }
+ Map<String, Map<String, String>> configs = sliderAppClient
+ .getConfigs(appMasterData.publisherUrl);
+ app.setConfigs(configs);
+ } else if ("components".equals(property.toLowerCase())) {
+ try {
+ System.setProperty(SliderKeys.HADOOP_USER_NAME, "yarn");
+ ClusterDescription description = sliderClient
+ .getClusterDescription(yarnApp.getName());
+ if (description != null && description.status != null
+ && !description.status.isEmpty()) {
+ Map<String, SliderAppComponent> componentTypeMap = new HashMap<String, SliderAppComponent>();
+ for (Entry<String, Object> e : description.status.entrySet()) {
+ @SuppressWarnings("unchecked")
+ Map<String, Map<String, Map<String, Object>>> componentsObj = (Map<String, Map<String, Map<String, Object>>>) e
+ .getValue();
+ boolean isLive = "live".equals(e.getKey());
+ for (Entry<String, Map<String, Map<String, Object>>> componentEntry : componentsObj
+ .entrySet()) {
+ SliderAppComponent appComponent = componentTypeMap
+ .get(componentEntry.getKey());
+ if (appComponent == null) {
+ appComponent = new SliderAppComponent();
+ appComponent.setComponentName(componentEntry.getKey());
+ appComponent
+ .setActiveContainers(new HashMap<String, Map<String, String>>());
+ appComponent
+ .setCompletedContainers(new HashMap<String, Map<String, String>>());
+ componentTypeMap.put(componentEntry.getKey(),
+ appComponent);
+ }
+ for (Entry<String, Map<String, Object>> containerEntry : componentEntry
+ .getValue().entrySet()) {
+ Map<String, String> containerDataMap = new HashMap<String, String>();
+ String containerId = containerEntry.getKey();
+ Map<String, Object> containerValues = containerEntry
+ .getValue();
+ for (String containerProperty : containerValues
+ .keySet()) {
+ Object containerPropertyValue = containerValues
+ .get(containerProperty);
+ containerDataMap.put(containerProperty,
+ containerPropertyValue.toString());
+ }
+ if (isLive)
+ appComponent.getActiveContainers().put(containerId,
+ containerDataMap);
+ else
+ appComponent.getCompletedContainers().put(
+ containerId, containerDataMap);
+ }
+ appComponent.setInstanceCount(appComponent
+ .getActiveContainers().size()
+ + appComponent.getCompletedContainers().size());
+ }
+ }
+ app.setComponents(componentTypeMap);
+ }
+ } catch (UnknownApplicationInstanceException e) {
+ logger.warn(
+ "Unable to determine app components for "
+ + yarnApp.getName(), e);
+ } catch (YarnException e) {
+ logger.warn(
+ "Unable to determine app components for "
+ + yarnApp.getName(), e);
+ throw new RuntimeException(e.getMessage(), e);
+ } catch (IOException e) {
+ logger.warn(
+ "Unable to determine app components for "
+ + yarnApp.getName(), e);
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+ }
+ }
+ }
+ }
return app;
}
@@ -188,7 +301,12 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
protected SliderClient getSliderClient() {
Configuration sliderClientConfiguration = getSliderClientConfiguration();
if (sliderClientConfiguration != null) {
- SliderClient client = new SliderClient();
+ SliderClient client = new SliderClient() {
+ @Override
+ public String getUsername() throws IOException {
+ return null;
+ }
+ };
try {
sliderClientConfiguration = client.bindArgs(sliderClientConfiguration,
new String[] { "usage" });
@@ -257,15 +375,17 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
@Override
- public List<SliderApp> getSliderApps() throws YarnException, IOException {
+ public List<SliderApp> getSliderApps(Set<String> properties)
+ throws YarnException, IOException {
List<SliderApp> sliderApps = new ArrayList<SliderApp>();
ClassLoader currentClassLoader = Thread.currentThread()
.getContextClassLoader();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
try {
- List<ApplicationReport> yarnApps = getSliderClient().getApplications();
+ SliderClient sliderClient = getSliderClient();
+ List<ApplicationReport> yarnApps = sliderClient.getApplications();
for (ApplicationReport yarnApp : yarnApps) {
- sliderApps.add(mapToSliderApp(yarnApp));
+ sliderApps.add(populateSliderApp(yarnApp, properties, sliderClient));
}
} finally {
Thread.currentThread().setContextClassLoader(currentClassLoader);
http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java
index 139494c..70778af 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java
@@ -85,6 +85,11 @@ public class BaseHttpClient {
}
public JsonElement doGetJson(String path) throws HttpException, IOException {
+ return doGetJson(getUrl(), path);
+ }
+
+ public JsonElement doGetJson(String url, String path) throws HttpException,
+ IOException {
GetMethod get = new GetMethod(url + path);
if (isNeedsAuthentication()) {
get.setDoAuthentication(true);
@@ -101,6 +106,21 @@ public class BaseHttpClient {
return null;
}
+ public String doGet(String path) throws HttpException, IOException {
+ GetMethod get = new GetMethod(url + path);
+ if (isNeedsAuthentication()) {
+ get.setDoAuthentication(true);
+ }
+ int executeMethod = getHttpClient().executeMethod(get);
+ switch (executeMethod) {
+ case HttpStatus.SC_OK:
+ return get.getResponseBodyAsString();
+ default:
+ break;
+ }
+ return null;
+ }
+
private HttpClient getHttpClient() {
if (httpClient == null) {
httpClient = new HttpClient();
http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java
new file mode 100644
index 0000000..db29035
--- /dev/null
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.slider.rest.client;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.httpclient.HttpException;
+import org.apache.log4j.Logger;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+public class SliderAppMasterClient extends BaseHttpClient {
+
+ private static final Logger logger = Logger
+ .getLogger(SliderAppMasterClient.class);
+
+ public static class SliderAppMasterData {
+ public String registryUrl;
+ public String uiUrl;
+ public String managementUrl;
+ public String publisherUrl;
+ }
+
+ public static class SliderAppContainerData {
+ public String hostName;
+ public String containerId;
+ }
+
+ public SliderAppMasterClient(String url) {
+ super(url);
+ }
+
+ public SliderAppMasterData getAppMasterData() {
+ try {
+ String html = doGet("");
+ int from = html.lastIndexOf("<ul>");
+ int to = html.lastIndexOf("</ul>");
+ if (from < to && from > -1) {
+ SliderAppMasterData data = new SliderAppMasterData();
+ String content = html.substring(from, to);
+ content = content.replaceAll("<[^>]*>", "\r\n");
+ String[] splits = content.split("\r\n");
+ for (int i = 0; i < splits.length; i++) {
+ String split = splits[i].trim();
+ if ("Registry Web Service".equals(split))
+ data.registryUrl = splits[i + 1].trim();
+ else if ("Application Master Web UI".equals(split))
+ data.uiUrl = splits[i + 1].trim();
+ else if ("Management REST API".equals(split))
+ data.managementUrl = splits[i + 1].trim();
+ else if ("Publisher Service".equals(split))
+ data.publisherUrl = splits[i + 1].trim();
+ }
+ return data;
+ }
+ } catch (HttpException e) {
+ logger.warn("Unable to determine Ambari clusters", e);
+ throw new RuntimeException(e.getMessage(), e);
+ } catch (IOException e) {
+ logger.warn("Unable to determine Ambari clusters", e);
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ return null;
+ }
+
+ public Map<String, String> getQuickLinks(String providerUrl) {
+ try {
+ JsonElement json = super.doGetJson(providerUrl, "/slider/quicklinks");
+ Map<String, String> quickLinks = new HashMap<String, String>();
+ JsonObject jsonObject = json.getAsJsonObject().get("entries")
+ .getAsJsonObject();
+ for (Entry<String, JsonElement> entry : jsonObject.entrySet()) {
+ if ("org.apache.slider.jmx".equals(entry.getKey())) {
+ quickLinks.put("JMX", entry.getValue().getAsString());
+ } else if ("org.apache.slider.monitor".equals(entry.getKey())) {
+ quickLinks.put("UI", entry.getValue().getAsString());
+ } else if ("org.apache.slider.metrics".equals(entry.getKey())) {
+ quickLinks.put("Metrics", entry.getValue().getAsString());
+ } else {
+ quickLinks.put(entry.getKey(), entry.getValue().getAsString());
+ }
+ }
+ return quickLinks;
+ } catch (HttpException e) {
+ logger.warn("Unable to determine quicklinks from " + providerUrl, e);
+ throw new RuntimeException(e.getMessage(), e);
+ } catch (IOException e) {
+ logger.warn("Unable to determine quicklinks from " + providerUrl, e);
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+
+ public Map<String, Map<String, String>> getConfigs(String providerUrl) {
+ try {
+ Map<String, Map<String, String>> configsMap = new HashMap<String, Map<String, String>>();
+ JsonElement json = super.doGetJson(providerUrl, "/slider");
+ if (json != null) {
+ JsonObject configsJson = json.getAsJsonObject().get("configurations")
+ .getAsJsonObject();
+ for (Entry<String, JsonElement> entry : configsJson.entrySet()) {
+ if ("complete-config".equals(entry.getKey())
+ || "quicklinks".equals(entry.getKey()))
+ continue;
+ JsonElement entryJson = super.doGetJson(providerUrl, "/slider/"
+ + entry.getKey());
+ if (entryJson != null) {
+ JsonObject configsObj = entryJson.getAsJsonObject().get("entries")
+ .getAsJsonObject();
+ if (configsObj != null) {
+ Map<String, String> configs = new HashMap<String, String>();
+ for (Entry<String, JsonElement> e : configsObj.entrySet()) {
+ configs.put(e.getKey(), e.getValue().getAsString());
+ }
+ configsMap.put(entry.getKey(), configs);
+ }
+ }
+ }
+ }
+ return configsMap;
+ } catch (HttpException e) {
+ logger.warn("Unable to determine quicklinks from " + providerUrl, e);
+ throw new RuntimeException(e.getMessage(), e);
+ } catch (IOException e) {
+ logger.warn("Unable to determine quicklinks from " + providerUrl, e);
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Provides only the interesting JMX metric names and values.
+ *
+ * @param jmxUrl
+ * @return
+ */
+ public Map<String, String> getJmx(String jmxUrl) {
+ // TODO Mechanism to filter JMX beans to get only interesting metrics is
+ // needed.
+ return null;
+ }
+
+}