You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/08/15 03:16:31 UTC
git commit: AMBARI-6870. BE: Provide configurations validation via
/validations endpoint on stack-version
Repository: ambari
Updated Branches:
refs/heads/trunk 2489e7711 -> f825ccaec
AMBARI-6870. BE: Provide configurations validation via /validations endpoint on stack-version
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f825ccae
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f825ccae
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f825ccae
Branch: refs/heads/trunk
Commit: f825ccaec685d1400042c9a42ce03bcd94130e81
Parents: 2489e77
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Thu Aug 14 18:08:36 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Thu Aug 14 18:08:36 2014 -0700
----------------------------------------------------------------------
.../stackadvisor/StackAdvisorHelper.java | 21 +-
.../stackadvisor/StackAdvisorRequest.java | 16 +-
.../GetConfigurationRecommnedationCommand.java | 3 +-
.../GetConfigurationValidationCommand.java | 61 +++++
.../commands/StackAdvisorCommand.java | 21 ++
.../internal/StackAdvisorResourceProvider.java | 56 ++++-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 143 +++++++++++
.../stackadvisor/StackAdvisorHelperTest.java | 14 ++
.../stacks/2.0.6/common/test_stack_advisor.py | 248 +++++++++++++++++++
9 files changed, 554 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f825ccae/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
index 156b637..6903c1d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
@@ -25,6 +25,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.St
import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand;
import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand;
import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationRecommnedationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationValidationCommand;
import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand;
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
@@ -74,6 +75,9 @@ public class StackAdvisorHelper {
if (requestType == StackAdvisorRequestType.HOST_GROUPS) {
command = new GetComponentLayoutValidationCommand(recommendationsDir, stackAdvisorScript,
requestId, saRunner);
+ } else if (requestType == StackAdvisorRequestType.CONFIGURATIONS) {
+ command = new GetConfigurationValidationCommand(recommendationsDir, stackAdvisorScript,
+ requestId, saRunner);
} else {
throw new StackAdvisorException(String.format("Unsupported request type, type=%s",
requestType));
@@ -117,21 +121,4 @@ public class StackAdvisorHelper {
return command;
}
- /**
- * Return configurations recommendation based on hosts and services
- * information.
- *
- * @param request the recommendation request
- * @return {@link RecommendationResponse} instance
- * @throws StackAdvisorException in case of stack advisor script errors
- */
- public synchronized RecommendationResponse getConfigurationRecommnedation(
- StackAdvisorRequest request) throws StackAdvisorException {
- requestId += 1;
-
- GetConfigurationRecommnedationCommand command = new GetConfigurationRecommnedationCommand(
- recommendationsDir, stackAdvisorScript, requestId, saRunner);
- return command.invoke(request);
- }
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f825ccae/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index bc1678e..991f198 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -27,8 +27,6 @@ import java.util.Set;
import org.apache.commons.lang.StringUtils;
-import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*;
-
/**
* Stack advisor request.
*/
@@ -42,6 +40,7 @@ public class StackAdvisorRequest {
private Map<String, Set<String>> componentHostsMap = new HashMap<String, Set<String>>();
private Map<String, Set<String>> hostComponents = new HashMap<String, Set<String>>();
private Map<String, Set<String>> hostGroupBindings = new HashMap<String, Set<String>>();
+ private Map<String, Map<String, Map<String, String>>> configurations = new HashMap<String, Map<String, Map<String, String>>>();
public String getStackName() {
return stackName;
@@ -83,6 +82,10 @@ public class StackAdvisorRequest {
return hostGroupBindings;
}
+ public Map<String, Map<String, Map<String, String>>> getConfigurations() {
+ return configurations;
+ }
+
private StackAdvisorRequest(String stackName, String stackVersion) {
this.stackName = stackName;
this.stackVersion = stackVersion;
@@ -125,11 +128,18 @@ public class StackAdvisorRequest {
return this;
}
- public StackAdvisorRequestBuilder forHostsGroupBindings(Map<String, Set<String>> hostGroupBindings) {
+ public StackAdvisorRequestBuilder forHostsGroupBindings(
+ Map<String, Set<String>> hostGroupBindings) {
this.instance.hostGroupBindings = hostGroupBindings;
return this;
}
+ public StackAdvisorRequestBuilder withConfigurations(
+ Map<String, Map<String, Map<String, String>>> configurations) {
+ this.instance.configurations = configurations;
+ return this;
+ }
+
public StackAdvisorRequest build() {
return this.instance;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f825ccae/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
index 48946aa..e65f97f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
@@ -49,7 +49,8 @@ public class GetConfigurationRecommnedationCommand extends
@Override
protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
- if (request.getHosts().isEmpty() || request.getServices().isEmpty()) {
+ if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null
+ || request.getServices().isEmpty()) {
throw new StackAdvisorException("Hosts and services must not be empty");
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f825ccae/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
new file mode 100644
index 0000000..2cd6481
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services.stackadvisor.commands;
+
+import java.io.File;
+
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
+import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
+
+/**
+ * {@link StackAdvisorCommand} implementation for configuration validation.
+ */
+public class GetConfigurationValidationCommand extends StackAdvisorCommand<ValidationResponse> {
+
+ public GetConfigurationValidationCommand(File recommendationsDir, String stackAdvisorScript,
+ int requestId, StackAdvisorRunner saRunner) {
+ super(recommendationsDir, stackAdvisorScript, requestId, saRunner);
+ }
+
+ @Override
+ protected StackAdvisorCommandType getCommandType() {
+ return StackAdvisorCommandType.VALIDATE_CONFIGURATIONS;
+ }
+
+ @Override
+ protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
+ if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null
+ || request.getServices().isEmpty()) {
+ throw new StackAdvisorException("Hosts, services and configurations must not be empty");
+ }
+ }
+
+ @Override
+ protected ValidationResponse updateResponse(StackAdvisorRequest request,
+ ValidationResponse response) {
+ return response;
+ }
+
+ @Override
+ protected String getResultFileName() {
+ return "configurations-validation.json";
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f825ccae/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index 1a30d38..607e337 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -78,6 +78,7 @@ public abstract class StackAdvisorCommand<T> extends BaseService {
private static final String COMPONENT_INFO_PROPETRY = "StackServiceComponents";
private static final String COMPONENT_NAME_PROPERTY = "component_name";
private static final String COMPONENT_HOSTNAMES_PROPETRY = "hostnames";
+ private static final String CONFIGURATIONS_PROPETRY = "configurations";
private File recommendationsDir;
private String stackAdvisorScript;
@@ -133,6 +134,7 @@ public abstract class StackAdvisorCommand<T> extends BaseService {
ObjectNode root = (ObjectNode) this.mapper.readTree(data.servicesJSON);
populateComponentHostsMap(root, request.getComponentHostsMap());
+ populateConfigurations(root, request.getConfigurations());
data.servicesJSON = mapper.writeValueAsString(root);
} catch (Exception e) {
@@ -145,6 +147,25 @@ public abstract class StackAdvisorCommand<T> extends BaseService {
return data;
}
+ private void populateConfigurations(ObjectNode root,
+ Map<String, Map<String, Map<String, String>>> configurations) {
+ ObjectNode configurationsNode = root.putObject(CONFIGURATIONS_PROPETRY);
+ for (String siteName : configurations.keySet()) {
+ ObjectNode siteNode = configurationsNode.putObject(siteName);
+
+ Map<String, Map<String, String>> siteMap = configurations.get(siteName);
+ for (String properties : siteMap.keySet()) {
+ ObjectNode propertiesNode = siteNode.putObject(properties);
+
+ Map<String, String> propertiesMap = siteMap.get(properties);
+ for (String propertyName : propertiesMap.keySet()) {
+ String propertyValue = propertiesMap.get(propertyName);
+ propertiesNode.put(propertyName, propertyValue);
+ }
+ }
+ }
+ }
+
private void populateComponentHostsMap(ObjectNode root, Map<String, Set<String>> componentHostsMap) {
ArrayNode services = (ArrayNode) root.get(SERVICES_PROPETRY);
Iterator<JsonNode> servicesIter = services.getElements();
http://git-wip-us.apache.org/repos/asf/ambari/blob/f825ccae/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
index 95024d2..40b423e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
@@ -95,11 +95,14 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
List<String> services = (List<String>) getRequestProperty(request, SERVICES_PROPERTY);
Map<String, Set<String>> hgComponentsMap = calculateHostGroupComponentsMap(request);
Map<String, Set<String>> hgHostsMap = calculateHostGroupHostsMap(request);
- Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(hgComponentsMap, hgHostsMap);
+ Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(hgComponentsMap,
+ hgHostsMap);
+ Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request);
StackAdvisorRequest saRequest = StackAdvisorRequestBuilder.forStack(stackName, stackVersion)
- .ofType(requestType).forHosts(hosts).forServices(services).forHostComponents(hgComponentsMap)
- .forHostsGroupBindings(hgHostsMap).withComponentHostsMap(componentHostsMap).build();
+ .ofType(requestType).forHosts(hosts).forServices(services)
+ .forHostComponents(hgComponentsMap).forHostsGroupBindings(hgHostsMap)
+ .withComponentHostsMap(componentHostsMap).withConfigurations(configurations).build();
return saRequest;
} catch (Exception e) {
@@ -120,8 +123,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
*/
@SuppressWarnings("unchecked")
private Map<String, Set<String>> calculateHostGroupComponentsMap(Request request) {
- Set<Map<String, Object>> hostGroups =
- (Set<Map<String, Object>>) getRequestProperty(request, BLUEPRINT_HOST_GROUPS_PROPERTY);
+ Set<Map<String, Object>> hostGroups = (Set<Map<String, Object>>) getRequestProperty(request,
+ BLUEPRINT_HOST_GROUPS_PROPERTY);
Map<String, Set<String>> map = new HashMap<String, Set<String>>();
if (hostGroups != null) {
for (Map<String, Object> hostGroup : hostGroups) {
@@ -151,8 +154,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
*/
@SuppressWarnings("unchecked")
private Map<String, Set<String>> calculateHostGroupHostsMap(Request request) {
- Set<Map<String, Object>> bindingHostGroups =
- (Set<Map<String, Object>>) getRequestProperty(request, BINDING_HOST_GROUPS_PROPERTY);
+ Set<Map<String, Object>> bindingHostGroups = (Set<Map<String, Object>>) getRequestProperty(
+ request, BINDING_HOST_GROUPS_PROPERTY);
Map<String, Set<String>> map = new HashMap<String, Set<String>>();
if (bindingHostGroups != null) {
for (Map<String, Object> hostGroup : bindingHostGroups) {
@@ -173,9 +176,46 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
return map;
}
+ private static final String CONFIGURATIONS_PROPERTY_ID = "recommendations/blueprint/configurations/";
+
+ private Map<String, Map<String, Map<String, String>>> calculateConfigurations(Request request) {
+ Map<String, Map<String, Map<String, String>>> configurations = new HashMap<String, Map<String, Map<String, String>>>();
+ Map<String, Object> properties = request.getProperties().iterator().next();
+ for (String property : properties.keySet()) {
+ if (property.startsWith(CONFIGURATIONS_PROPERTY_ID)) {
+ try {
+ String propertyEnd = property.substring(CONFIGURATIONS_PROPERTY_ID.length()); // mapred-site/properties/yarn.app.mapreduce.am.resource.mb
+ String[] propertyPath = propertyEnd.split("/"); // length == 3
+ String siteName = propertyPath[0];
+ String propertiesProperty = propertyPath[1];
+ String propertyName = propertyPath[2];
+
+ Map<String, Map<String, String>> siteMap = configurations.get(siteName);
+ if (siteMap == null) {
+ siteMap = new HashMap<String, Map<String, String>>();
+ configurations.put(siteName, siteMap);
+ }
+
+ Map<String, String> propertiesMap = siteMap.get(propertiesProperty);
+ if (propertiesMap == null) {
+ propertiesMap = new HashMap<String, String>();
+ siteMap.put(propertiesProperty, propertiesMap);
+ }
+
+ String value = (String) properties.get(property);
+ propertiesMap.put(propertyName, value);
+ } catch (Exception e) {
+ LOG.debug(String.format("Error handling configuration property, name = %s", property), e);
+ // do nothing
+ }
+ }
+ }
+ return configurations;
+ }
+
@SuppressWarnings("unchecked")
private Map<String, Set<String>> calculateComponentHostsMap(Map<String, Set<String>> hostGroups,
- Map<String, Set<String>> bindingHostGroups) {
+ Map<String, Set<String>> bindingHostGroups) {
/*
* ClassCastException may occur in case of body inconsistency: property
* missed, etc.
http://git-wip-us.apache.org/repos/asf/ambari/blob/f825ccae/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 402d95b..025caa2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import re
import socket
import sys
@@ -364,8 +365,150 @@ class HDP206StackAdvisor(StackAdvisor):
def validateConfigurations(self, services, hosts):
"""Returns array of Validation objects about issues with configuration values provided in services"""
+ stackName = services["Versions"]["stack_name"]
+ stackVersion = services["Versions"]["stack_version"]
+
+ validations = {
+ "Versions": {"stack_name": stackName, "stack_version": stackVersion},
+ "items": [ ]
+ }
+ items = validations["items"]
+
+ recommendations = self.recommendConfigurations(services, hosts)
+ recommendedDefaults = recommendations["recommendations"]["blueprint"]["configurations"]
+
+ configurations = services["configurations"]
+ for service in services["services"]:
+ serviceName = service["StackServices"]["service_name"]
+ if serviceName == "MAPREDUCE2":
+ mapReduceErrors = validateMapReduce2Configurations(getSiteProperties(configurations, "mapred-site"), recommendedDefaults["mapred-site"]["properties"])
+ items.extend(mapReduceErrors)
+ elif serviceName == "HIVE":
+ hiveErrors = validateHiveConfigurations(getSiteProperties(configurations, "hive-site"), recommendedDefaults["hive-site"]["properties"])
+ items.extend(hiveErrors)
+ elif serviceName == "STORM":
+ oozieErrors = [] #validateStormConfigurations(getSiteProperties(configurations, "storm-site"), recommendedDefaults["storm-site"]["properties"])
+ items.extend(oozieErrors)
+ elif serviceName == "TEZ":
+ tezErrors = validateTezConfigurations(getSiteProperties(configurations, "tez-site"), recommendedDefaults["tez-site"]["properties"])
+ items.extend(tezErrors)
+ elif serviceName == "YARN":
+ yarnErrors = validateYARNConfigurations(getSiteProperties(configurations, "yarn-site"), recommendedDefaults["yarn-site"]["properties"])
+ items.extend(yarnErrors)
+ else:
+ pass
+ return validations
pass
+def getSiteProperties(configurations, siteName):
+ if configurations[siteName] is None:
+ return {}
+ if configurations[siteName]["properties"] is None:
+ return {}
+ return configurations[siteName]["properties"]
+
+def to_number(s):
+ try:
+ return int(re.sub("\D", "", s))
+ except ValueError:
+ return None
+
+def toConfigurationValidationErrors(items, siteName):
+ result = []
+ for item in items:
+ if item["message"] is not None:
+ error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type": siteName, "config-name": item["config-name"] }
+ result.append(error)
+ return result
+
+def validatorLessThenDefaultValue(properties, recommendedDefaults, propertyName):
+ value = to_number(properties[propertyName])
+ if value is None:
+ return "Value should be integer"
+ defaultValue = to_number(recommendedDefaults[propertyName])
+ if defaultValue is None:
+ return None
+ if value < defaultValue:
+ return "Value is less than the recommended default of {0}".format(defaultValue)
+ return None
+
+def validateXmxValue(properties, recommendedDefaults, propertyName):
+ value = properties[propertyName]
+ defaultValue = recommendedDefaults[propertyName]
+ if defaultValue is None:
+ return "Config's default value can't be null or undefined"
+ if not checkXmxValueFormat(value):
+ return 'Invalid value format'
+ valueInt = formatXmxSizeToBytes(getXmxSize(value))
+ defaultValueXmx = getXmxSize(defaultValue)
+ defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
+ if valueInt < defaultValueInt:
+ return "Value is less than the recommended default of -Xmx" + defaultValueXmx
+ return None
+
+def checkXmxValueFormat(value):
+ p = re.compile('-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)?')
+ matches = p.findall(value)
+ return len(matches) == 1
+
+def getXmxSize(value):
+ p = re.compile("-Xmx(\d+)(.?)")
+ result = p.findall(value)[0]
+ if len(result) > 1:
+ # result[1] - is a space or size formatter (b|k|m|g etc)
+ return result[0] + result[1].lower()
+ return result[0]
+
+def formatXmxSizeToBytes(value):
+ value = value.lower()
+ if len(value) == 0:
+ return 0
+ modifier = value[-1]
+
+ if modifier == ' ' or modifier in "0123456789":
+ modifier = 'b'
+ m = {
+ modifier == 'b': 1,
+ modifier == 'k': 1024,
+ modifier == 'm': 1024 * 1024,
+ modifier == 'g': 1024 * 1024 * 1024,
+ modifier == 't': 1024 * 1024 * 1024 * 1024,
+ modifier == 'p': 1024 * 1024 * 1024 * 1024 * 1024
+ }[1]
+ return to_number(value) * m
+
+def validateMapReduce2Configurations(properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
+ {"config-name": 'mapreduce.reduce.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')},
+ {"config-name": 'mapreduce.task.io.sort.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')},
+ {"config-name": 'mapreduce.map.memory.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')},
+ {"config-name": 'mapreduce.reduce.memory.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')},
+ {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
+ {"config-name": 'yarn.app.mapreduce.am.command-opts', "message": validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
+ return toConfigurationValidationErrors(validationItems, "mapred-site")
+
+def validateHiveConfigurations(properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'hive.tez.container.size', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')},
+ {"config-name": 'hive.tez.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')},
+ {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ]
+ return toConfigurationValidationErrors(validationItems, "hive-site")
+
+def validateStormConfigurations(properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'drpc.childopts', "message": validateXmxValue(properties, recommendedDefaults, 'drpc.childopts')},
+ {"config-name": 'ui.childopts', "message": validateXmxValue(properties, recommendedDefaults, 'ui.childopts')},
+ {"config-name": 'logviewer.childopts', "message": validateXmxValue(properties, recommendedDefaults, 'logviewer.childopts')} ]
+ return toConfigurationValidationErrors(validationItems, "storm-site")
+
+def validateTezConfigurations(properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
+ {"config-name": 'tez.am.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ]
+ return toConfigurationValidationErrors(validationItems, "tez-site")
+
+def validateYARNConfigurations(properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
+ {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
+ {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
+ return toConfigurationValidationErrors(validationItems, "yarn-site")
# Helper methods
def getHostForComponent(component, hostsList):
http://git-wip-us.apache.org/repos/asf/ambari/blob/f825ccae/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java
index 6f8c42b..e4cf97a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java
@@ -31,6 +31,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.St
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand;
import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationValidationCommand;
import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand;
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
@@ -149,4 +150,17 @@ public class StackAdvisorHelperTest {
assertEquals(GetComponentLayoutValidationCommand.class, command.getClass());
}
+ @Test
+ public void testCreateValidationCommand_returnsGetConfigurationValidationCommand()
+ throws IOException, StackAdvisorException {
+ Configuration configuration = mock(Configuration.class);
+ StackAdvisorRunner saRunner = mock(StackAdvisorRunner.class);
+ StackAdvisorHelper helper = new StackAdvisorHelper(configuration, saRunner);
+ StackAdvisorRequestType requestType = StackAdvisorRequestType.CONFIGURATIONS;
+
+ StackAdvisorCommand<ValidationResponse> command = helper.createValidationCommand(requestType);
+
+ assertEquals(GetConfigurationValidationCommand.class, command.getClass());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f825ccae/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
new file mode 100644
index 0000000..3139a3e
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -0,0 +1,248 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import socket
+from unittest import TestCase
+
+class TestHDP206StackAdvisor(TestCase):
+
+ def setUp(self):
+ import imp
+ import os
+
+ testDirectory = os.path.dirname(os.path.abspath(__file__))
+ stackAdvisorPath = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/stack_advisor.py')
+ hdp206StackAdvisorPath = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py')
+ hdp206StackAdvisorClassName = 'HDP206StackAdvisor'
+ with open(stackAdvisorPath, 'rb') as fp:
+ stack_advisor = imp.load_module( 'stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE) )
+ with open(hdp206StackAdvisorPath, 'rb') as fp:
+ stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName)
+ self.stackAdvisor = clazz()
+
+ def test_recommendationCardinalityALL(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [{"name": "GANGLIA_MONITOR", "cardinality": "ALL", "category": "SLAVE", "is_master": False}]
+ }
+ ]
+ services = self.prepareServices(servicesInfo)
+ hosts = self.prepareHosts(["host1", "host2"])
+ result = self.stackAdvisor.recommendComponentLayout(services, hosts)
+
+ expectedComponentsHostsMap = {
+ "GANGLIA_MONITOR": ["host1", "host2"]
+ }
+ self.assertHostLayout(expectedComponentsHostsMap, result)
+
+ def test_recommendationAssignmentNotChanged(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [{"name": "GANGLIA_MONITOR", "cardinality": "ALL", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]}]
+ }
+ ]
+ services = self.prepareServices(servicesInfo)
+ hosts = self.prepareHosts(["host1", "host2"])
+ result = self.stackAdvisor.recommendComponentLayout(services, hosts)
+
+ expectedComponentsHostsMap = {
+ "GANGLIA_MONITOR": ["host1"]
+ }
+ self.assertHostLayout(expectedComponentsHostsMap, result)
+
+ def test_recommendationIsNotPreferableOnAmbariServer(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [{"name": "GANGLIA_SERVER", "cardinality": "ALL", "category": "MASTER", "is_master": True}]
+ }
+ ]
+ services = self.prepareServices(servicesInfo)
+ localhost = socket.getfqdn()
+ hosts = self.prepareHosts([localhost, "host2"])
+ result = self.stackAdvisor.recommendComponentLayout(services, hosts)
+
+ expectedComponentsHostsMap = {
+ "GANGLIA_SERVER": ["host2"]
+ }
+ self.assertHostLayout(expectedComponentsHostsMap, result)
+
+ def test_validationNamenodeAndSecondaryNamenode2Hosts(self):
+ servicesInfo = [
+ {
+ "name": "HDFS",
+ "components": [
+ {"name": "NAMENODE", "cardinality": "1-2", "category": "MASTER", "is_master": True, "hostnames": ["host1"]},
+ {"name": "SECONDARY_NAMENODE", "cardinality": "1", "category": "MASTER", "is_master": True, "hostnames": ["host1"]}]
+ }
+ ]
+ services = self.prepareServices(servicesInfo)
+ hosts = self.prepareHosts(["host1", "host2"])
+ result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+ expectedMessages = [
+ "NameNode and Secondary NameNode cannot be hosted on same machine",
+ "NameNode and Secondary NameNode cannot be hosted on same machine",
+ "Host is not used"
+ ]
+ self.assertValidationMessages(expectedMessages, result)
+
+ def test_validationCardinalityALL(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "cardinality": "ALL", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "cardinality": "1-2", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ services = self.prepareServices(servicesInfo)
+ hosts = self.prepareHosts(["host1", "host2"])
+ result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+ expectedMessages = [
+ "Cardinality violation, cardinality=ALL, hosts count=1"
+ ]
+ self.assertValidationMessages(expectedMessages, result)
+
+ def test_validationHostIsNotUsedForNonValuableComponent(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "cardinality": "ALL", "category": "SLAVE", "is_master": False, "hostnames": ["host1", "host2"]},
+ {"name": "GANGLIA_SERVER", "cardinality": "1", "category": "MASTER", "is_master": True, "hostnames": ["host2"]}
+ ]
+ }
+ ]
+ services = self.prepareServices(servicesInfo)
+ hosts = self.prepareHosts(["host1", "host2"])
+ result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+ expectedMessages = [
+ "Host is not used"
+ ]
+ self.assertValidationMessages(expectedMessages, result)
+
+ def test_validationCardinality01TwoHostsAssigned(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_SERVER", "cardinality": "0-1", "category": "MASTER", "is_master": True, "hostnames": ["host1", "host2"]}
+ ]
+ }
+ ]
+ services = self.prepareServices(servicesInfo)
+ hosts = self.prepareHosts(["host1", "host2"])
+ result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+ expectedMessages = [
+ "Cardinality violation, cardinality=0-1, hosts count=2"
+ ]
+ self.assertValidationMessages(expectedMessages, result)
+
+ def test_validationHostIsNotUsed(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_SERVER", "cardinality": "1", "category": "MASTER", "is_master": True, "hostnames": ["host1"]}
+ ]
+ }
+ ]
+ services = self.prepareServices(servicesInfo)
+ hosts = self.prepareHosts(["host1", "host2"])
+ result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+ expectedMessages = [
+ "Host is not used"
+ ]
+ self.assertValidationMessages(expectedMessages, result)
+
+
+ def prepareHosts(self, hostsNames):
+ hosts = { "items": [] }
+ for hostName in hostsNames:
+ nextHost = {"Hosts":{"host_name" : hostName}}
+ hosts["items"].append(nextHost)
+ return hosts
+
+ def prepareServices(self, servicesInfo):
+ services = { "Versions" : { "stack_name" : "HDP", "stack_version" : "2.0.6" } }
+ services["services"] = []
+
+ for serviceInfo in servicesInfo:
+ nextService = {"StackServices":{"service_name" : serviceInfo["name"]}}
+ nextService["components"] = []
+ for component in serviceInfo["components"]:
+ nextComponent = {
+ "StackServiceComponents": {
+ "component_name": component["name"],
+ "cardinality": component["cardinality"],
+ "component_category": component["category"],
+ "is_master": component["is_master"]
+ }
+ }
+ try:
+ nextComponent["StackServiceComponents"]["hostnames"] = component["hostnames"]
+ except KeyError, err:
+ nextComponent["StackServiceComponents"]["hostnames"] = []
+ nextService["components"].append(nextComponent)
+ services["services"].append(nextService)
+
+ return services
+
+ def assertHostLayout(self, componentsHostsMap, recommendation):
+ blueprintMapping = recommendation["recommendations"]["blueprint"]["host_groups"]
+ bindings = recommendation["recommendations"]["blueprint_cluster_binding"]["host_groups"]
+
+ actualComponentHostsMap = {}
+ for hostGroup in blueprintMapping:
+ hostGroupName = hostGroup["name"]
+ hostsInfos = [binding["hosts"] for binding in bindings if binding["name"] == hostGroupName][0]
+ hosts = [info["fqdn"] for info in hostsInfos]
+
+ for component in hostGroup["components"]:
+ componentName = component["name"]
+ try:
+ actualComponentHostsMap[componentName]
+ except KeyError, err:
+ actualComponentHostsMap[componentName] = []
+ for host in hosts:
+ if host not in actualComponentHostsMap[componentName]:
+ actualComponentHostsMap[componentName].append(host)
+
+ for componentName in componentsHostsMap.keys():
+ expectedHosts = componentsHostsMap[componentName]
+ actualHosts = actualComponentHostsMap[componentName]
+ self.checkEqual(expectedHosts, actualHosts)
+
+ def checkEqual(self, l1, l2):
+ if not len(l1) == len(l2) or not sorted(l1) == sorted(l2):
+ raise AssertionError("list1={0}, list2={1}".format(l1, l2))
+
+ def assertValidationMessages(self, expectedMessages, result):
+ realMessages = [item["message"] for item in result["items"]]
+ self.checkEqual(expectedMessages, realMessages)
+
+