You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/05/30 02:12:44 UTC
git commit: AMBARI-5923. When slider app is frozen and thawed
multiple times,
multiple entries show up in apps table. - string comparison problems
(srimanth)
Repository: ambari
Updated Branches:
refs/heads/branch-1.6.0.slider 8df5a36ad -> 1d22431b6
AMBARI-5923. When slider app is frozen and thawed multiple times, multiple entries show up in apps table. - string comparison problems (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1d22431b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1d22431b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1d22431b
Branch: refs/heads/branch-1.6.0.slider
Commit: 1d22431b6f8182cccd9cb318b85d4bf1c1e3948c
Parents: 8df5a36
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Thu May 29 17:11:25 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Thu May 29 17:11:25 2014 -0700
----------------------------------------------------------------------
.../slider/SliderAppsViewControllerImpl.java | 135 ++++++++++---------
1 file changed, 73 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d22431b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
index 5300008..3f3f24f 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
@@ -97,8 +97,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
private String getAppsFolderPath() {
return viewContext
- .getAmbariProperty(org.apache.ambari.server.configuration.Configuration.RESOURCES_DIR_KEY)
- + "/apps";
+ .getAmbariProperty(org.apache.ambari.server.configuration.Configuration.RESOURCES_DIR_KEY)
+ + "/apps";
}
private String getAppsCreateFolderPath() {
@@ -183,7 +183,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
if (hdfsConfig != null
&& hdfsConfig.containsKey("dfs.namenode.http-address")) {
viewUrls.put("dfs.namenode.http-address",
- "http://" + hdfsConfig.get("dfs.namenode.http-address"));
+ "http://" + hdfsConfig.get("dfs.namenode.http-address"));
}
}
// YARN URL
@@ -195,7 +195,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
if (yarnConfig != null
&& yarnConfig.containsKey("yarn.resourcemanager.webapp.address")) {
viewUrls.put("yarn.resourcemanager.webapp.address", "http://"
- + yarnConfig.get("yarn.resourcemanager.webapp.address"));
+ + yarnConfig.get("yarn.resourcemanager.webapp.address"));
}
}
} else {
@@ -218,7 +218,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
private String getApplicationIdString(ApplicationId appId) {
return Long.toString(appId.getClusterTimestamp()) + "_"
- + Integer.toString(appId.getId());
+ + Integer.toString(appId.getId());
}
private ApplicationId getApplicationId(String appIdString) {
@@ -254,7 +254,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
private SliderApp createSliderAppObject(ApplicationReport yarnApp,
- Set<String> properties, SliderClient sliderClient) {
+ Set<String> properties, SliderClient sliderClient) {
if (yarnApp == null) {
return null;
}
@@ -311,9 +311,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
List<SliderAppType> matchingAppTypes = getSliderAppTypes(null);
if (matchingAppTypes != null && matchingAppTypes.size() > 0) {
for (SliderAppType appType : matchingAppTypes) {
- if ((appType.getTypeName() != null && appType.getTypeName().equalsIgnoreCase(app.getType())) &&
- (appType.getTypeVersion() != null
- && appType.getTypeVersion().equalsIgnoreCase(app.getAppVersion()))) {
+ if ((appType.getTypeName() != null && appType.getTypeName()
+ .equalsIgnoreCase(app.getType()))
+ && (appType.getTypeVersion() != null && appType.getTypeVersion()
+ .equalsIgnoreCase(app.getAppVersion()))) {
matchedAppType = appType;
break;
}
@@ -349,7 +350,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
if (quickLinks != null && quickLinks.containsKey("JMX")) {
String jmxUrl = quickLinks.get("JMX");
if (matchedAppType != null) {
- app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, matchedAppType));
+ app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext,
+ matchedAppType));
}
}
Map<String, Map<String, String>> configs = sliderAppClient
@@ -365,10 +367,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
Map<String, SliderAppComponent> componentTypeMap = new HashMap<String, SliderAppComponent>();
for (Entry<String, Object> e : description.status.entrySet()) {
@SuppressWarnings("unchecked")
- Map<String, Map<String, Map<String, Object>>>
- componentsObj =
- (Map<String, Map<String, Map<String, Object>>>) e
- .getValue();
+ Map<String, Map<String, Map<String, Object>>> componentsObj = (Map<String, Map<String, Map<String, Object>>>) e
+ .getValue();
boolean isLive = "live".equals(e.getKey());
for (Entry<String, Map<String, Map<String, Object>>> componentEntry : componentsObj
.entrySet()) {
@@ -382,7 +382,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
appComponent
.setCompletedContainers(new HashMap<String, Map<String, String>>());
componentTypeMap.put(componentEntry.getKey(),
- appComponent);
+ appComponent);
}
for (Entry<String, Map<String, Object>> containerEntry : componentEntry
.getValue().entrySet()) {
@@ -395,19 +395,19 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
Object containerPropertyValue = containerValues
.get(containerProperty);
containerDataMap.put(containerProperty,
- containerPropertyValue.toString());
+ containerPropertyValue.toString());
}
if (isLive) {
appComponent.getActiveContainers().put(containerId,
- containerDataMap);
+ containerDataMap);
} else {
appComponent.getCompletedContainers().put(
containerId, containerDataMap);
}
}
appComponent.setInstanceCount(appComponent
- .getActiveContainers().size()
- + appComponent.getCompletedContainers().size());
+ .getActiveContainers().size()
+ + appComponent.getCompletedContainers().size());
}
}
app.setComponents(componentTypeMap);
@@ -415,16 +415,16 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
} catch (UnknownApplicationInstanceException e) {
logger.warn(
"Unable to determine app components for "
- + yarnApp.getName(), e);
+ + yarnApp.getName(), e);
} catch (YarnException e) {
logger.warn(
"Unable to determine app components for "
- + yarnApp.getName(), e);
+ + yarnApp.getName(), e);
throw new RuntimeException(e.getMessage(), e);
} catch (IOException e) {
logger.warn(
"Unable to determine app components for "
- + yarnApp.getName(), e);
+ + yarnApp.getName(), e);
throw new RuntimeException(e.getMessage(), e);
}
} else if (property.startsWith(METRICS_PREFIX)) {
@@ -444,8 +444,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
if (quickLinks != null && quickLinks.containsKey("Metrics")) {
String metricsUrl = quickLinks.get("Metrics");
- app.setMetrics(
- sliderAppClient.getGangliaMetrics(metricsUrl, gangliaMetrics, null, viewContext, matchedAppType));
+ app.setMetrics(sliderAppClient.getGangliaMetrics(metricsUrl,
+ gangliaMetrics, null, viewContext, matchedAppType));
}
}
}
@@ -453,9 +453,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
/**
- * Creates a new {@link SliderClient} initialized with appropriate configuration. If configuration was not determined,
- * <code>null</code> is returned.
- *
+ * Creates a new {@link SliderClient} initialized with appropriate
+ * configuration. If configuration was not determined, <code>null</code> is
+ * returned.
+ *
* @return
*/
protected SliderClient getSliderClient() {
@@ -472,7 +473,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
super.serviceInit(conf);
// Override the default FS client to set the super user.
FileSystem fs = FileSystem.get(FileSystem.getDefaultUri(getConfig()),
- getConfig(), "yarn");
+ getConfig(), "yarn");
SliderFileSystem fileSystem = new SliderFileSystem(fs, getConfig());
Field fsField = SliderClient.class
.getDeclaredField("sliderFileSystem");
@@ -482,7 +483,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
};
try {
sliderClientConfiguration = client.bindArgs(sliderClientConfiguration,
- new String[]{"usage"});
+ new String[] { "usage" });
} catch (Exception e) {
logger.warn("Unable to set SliderClient configs", e);
throw new RuntimeException(e.getMessage(), e);
@@ -495,15 +496,16 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
/**
- * Dynamically determines Slider client configuration. If unable to determine, <code>null</code> is returned.
- *
+ * Dynamically determines Slider client configuration. If unable to determine,
+ * <code>null</code> is returned.
+ *
* @return
*/
private Configuration getSliderClientConfiguration() {
AmbariCluster ambariCluster = getAmbariCluster();
if (ambariCluster != null) {
AmbariService zkService = ambariClient.getService(ambariCluster,
- "ZOOKEEPER");
+ "ZOOKEEPER");
if (zkService != null && ambariCluster.getDesiredConfigs() != null
&& ambariCluster.getDesiredConfigs().containsKey("global")
&& ambariCluster.getDesiredConfigs().containsKey("yarn-site")
@@ -538,7 +540,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
yarnConfig.set("yarn.log-aggregation-enable", "true");
yarnConfig.set("yarn.resourcemanager.address", rmAddress);
yarnConfig.set("yarn.resourcemanager.scheduler.address",
- rmSchedulerAddress);
+ rmSchedulerAddress);
yarnConfig.set("fs.defaultFS", hdfsPath);
yarnConfig.set("slider.zookeeper.quorum", zkQuorum.toString());
yarnConfig
@@ -564,11 +566,17 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
List<ApplicationReport> yarnApps = sliderClient.listSliderInstances(null);
for (ApplicationReport yarnApp : yarnApps) {
SliderApp sliderAppObject = createSliderAppObject(yarnApp, properties,
- sliderClient);
+ sliderClient);
if (sliderAppObject != null) {
if (sliderAppsMap.containsKey(sliderAppObject.getName())) {
- if (sliderAppsMap.get(sliderAppObject.getName()).getId()
- .compareTo(sliderAppObject.getId()) < 0) {
+ String mapId = sliderAppsMap.get(sliderAppObject.getName()).getId();
+ String newId = sliderAppObject.getId();
+ ApplicationId mapIdObj = getApplicationId(mapId);
+ ApplicationId newIdObj = getApplicationId(newId);
+ if (mapIdObj.getClusterTimestamp() < newIdObj.getClusterTimestamp()
+ || ((mapIdObj.getClusterTimestamp() == newIdObj
+ .getClusterTimestamp()) && mapIdObj.getId() < newIdObj
+ .getId())) {
sliderAppsMap.put(sliderAppObject.getName(), sliderAppObject);
}
} else {
@@ -641,7 +649,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
try {
ZipFile zipFile = new ZipFile(appZip);
Metainfo metainfo = new MetainfoParser().parse(zipFile
- .getInputStream(zipFile.getEntry("metainfo.xml")));
+ .getInputStream(zipFile.getEntry("metainfo.xml")));
// Create app type object
if (metainfo.getServices() != null
&& metainfo.getServices().size() > 0) {
@@ -685,11 +693,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
// appTypeComponent.setPriority(component.);
if (component.getMinInstanceCount() != null) {
appTypeComponent.setInstanceCount(Integer.parseInt(component
- .getMinInstanceCount()));
+ .getMinInstanceCount()));
}
if (component.getMaxInstanceCount() != null) {
appTypeComponent.setMaxInstanceCount(Integer
- .parseInt(component.getMaxInstanceCount()));
+ .parseInt(component.getMaxInstanceCount()));
}
if (resourcesJson != null) {
JsonElement componentJson = resourcesJson.getAsJsonObject()
@@ -697,10 +705,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
.get(component.getName());
if (componentJson != null
&& componentJson.getAsJsonObject().has(
- "yarn.role.priority")) {
+ "yarn.role.priority")) {
appTypeComponent.setPriority(Integer.parseInt(componentJson
- .getAsJsonObject().get("yarn.role.priority")
- .getAsString()));
+ .getAsJsonObject().get("yarn.role.priority")
+ .getAsString()));
}
}
appTypeComponent.setCategory(component.getCategory());
@@ -709,8 +717,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
appType.setJmxMetrics(readMetrics(zipFile, "jmx_metrics.json"));
appType.setGangliaMetrics(readMetrics(zipFile,
- "ganglia_metrics.json"));
- appType.setSupportedMetrics(getSupportedMetrics(appType.getGangliaMetrics()));
+ "ganglia_metrics.json"));
+ appType.setSupportedMetrics(getSupportedMetrics(appType
+ .getGangliaMetrics()));
appType.setTypeComponents(appTypeComponentList);
appTypes.add(appType);
@@ -726,10 +735,12 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
return appTypes;
}
- private List<String> getSupportedMetrics(Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) {
+ private List<String> getSupportedMetrics(
+ Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) {
Set<String> supportedMetrics = new HashSet<String>();
if (gangliaMetrics != null && gangliaMetrics.size() > 0) {
- for (Map<String, Map<String, Metric>> compMetrics : gangliaMetrics.values()) {
+ for (Map<String, Map<String, Metric>> compMetrics : gangliaMetrics
+ .values()) {
for (Map<String, Metric> metrics : compMetrics.values()) {
supportedMetrics.addAll(metrics.keySet());
}
@@ -739,16 +750,16 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
Map<String, Map<String, Map<String, Metric>>> readMetrics(ZipFile zipFile,
- String fileName) {
+ String fileName) {
Map<String, Map<String, Map<String, Metric>>> metrics = null;
try {
InputStream inputStream = zipFile.getInputStream(zipFile
- .getEntry(fileName));
+ .getEntry(fileName));
ObjectMapper mapper = new ObjectMapper();
metrics = mapper.readValue(inputStream,
- new TypeReference<Map<String, Map<String, Map<String, Metric>>>>() {
- });
+ new TypeReference<Map<String, Map<String, Map<String, Metric>>>>() {
+ });
} catch (IOException e) {
logger.info("Error reading metrics. " + e.getMessage());
}
@@ -788,7 +799,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
appCount = ++createAppCounter;
}
File appCreateFolder = new File(appsCreateFolder,
- Integer.toString(appCount));
+ Integer.toString(appCount));
appCreateFolder.mkdirs();
File appConfigJsonFile = new File(appCreateFolder, "appConfig.json");
File resourcesJsonFile = new File(appCreateFolder, "resources.json");
@@ -799,20 +810,20 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
AmbariCluster cluster = ambariClient.getCluster(clusterInfo);
Map<String, String> coreSiteConfigs = ambariClient.getConfiguration(
clusterInfo, "core-site", cluster.getDesiredConfigs()
- .get("core-site"));
+ .get("core-site"));
String hdfsLocation = coreSiteConfigs.get("fs.defaultFS");
final ActionCreateArgs createArgs = new ActionCreateArgs();
createArgs.template = appConfigJsonFile;
createArgs.resources = resourcesJsonFile;
createArgs.image = new Path(hdfsLocation
- + "/slider/agent/slider-agent.tar.gz");
+ + "/slider/agent/slider-agent.tar.gz");
ClassLoader currentClassLoader = Thread.currentThread()
.getContextClassLoader();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
try {
ApplicationId applicationId = UserGroupInformation.getBestUGI(null,
- "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() {
+ "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() {
public ApplicationId run() throws IOException, YarnException {
SliderClient sliderClient = getSliderClient();
sliderClient.actionCreate(appName, createArgs);
@@ -830,10 +841,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
private void saveAppResources(JsonArray componentsArray,
- File resourcesJsonFile) throws IOException {
+ File resourcesJsonFile) throws IOException {
JsonObject resourcesObj = new JsonObject();
resourcesObj.addProperty("schema",
- "http://example.org/specification/v2.0.0");
+ "http://example.org/specification/v2.0.0");
resourcesObj.add("metadata", new JsonObject());
resourcesObj.add("global", new JsonObject());
JsonObject componentsObj = new JsonObject();
@@ -843,11 +854,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
if (inputComponent.has("id")) {
JsonObject componentValue = new JsonObject();
componentValue.addProperty("yarn.role.priority",
- inputComponent.get("priority").getAsString());
+ inputComponent.get("priority").getAsString());
componentValue.addProperty("yarn.component.instances", inputComponent
.get("instanceCount").getAsString());
componentsObj.add(inputComponent.get("id").getAsString(),
- componentValue);
+ componentValue);
}
}
}
@@ -865,7 +876,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
private void saveAppConfigs(JsonObject configs, JsonArray componentsArray,
- File appConfigJsonFile) throws IOException {
+ File appConfigJsonFile) throws IOException {
JsonObject appConfigs = new JsonObject();
appConfigs.addProperty("schema", "http://example.org/specification/v2.0.0");
appConfigs.add("metadata", new JsonObject());
@@ -876,7 +887,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
JsonObject inputComponent = componentsArray.get(i).getAsJsonObject();
if (inputComponent.has("id")) {
componentsObj.add(inputComponent.get("id").getAsString(),
- new JsonObject());
+ new JsonObject());
}
}
}
@@ -909,7 +920,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
ApplicationId applicationId = UserGroupInformation.getBestUGI(null,
- "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() {
+ "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() {
public ApplicationId run() throws IOException, YarnException {
SliderClient sliderClient = getSliderClient();
ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
@@ -938,7 +949,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
throw new ApplicationNotFoundException(appId);
}
ApplicationId applicationId = UserGroupInformation.getBestUGI(null,
- "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() {
+ "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() {
public ApplicationId run() throws IOException, YarnException {
SliderClient sliderClient = getSliderClient();
ActionThawArgs thawArgs = new ActionThawArgs();