You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2016/07/22 12:14:22 UTC
ambari git commit: AMBARI-17835 Maximum validation failure for
'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 988775704 -> b5b9493ff
AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b5b9493f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b5b9493f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b5b9493f
Branch: refs/heads/trunk
Commit: b5b9493ff7da82f342a7b39c2c8d273d763c8eae
Parents: 9887757
Author: Dmytro Sen <ds...@apache.org>
Authored: Fri Jul 22 15:14:13 2016 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Fri Jul 22 15:14:13 2016 +0300
----------------------------------------------------------------------
.../commands/StackAdvisorCommand.java | 6 ++++
.../stacks/HDP/2.2/services/stack_advisor.py | 34 ++++++++++++++++----
.../src/main/resources/stacks/stack_advisor.py | 16 ++++++---
.../commands/StackAdvisorCommandTest.java | 18 +++++++++++
.../stacks/2.2/common/test_stack_advisor.py | 21 +++++-------
5 files changed, 71 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index 6f612af..f93e180 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -98,6 +98,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
private static final String CONFIGURATIONS_PROPERTY = "configurations";
private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties";
+ private static final String STACK_ADVISOR_COMMAND_TYPE_PROPERTY = "stack-advisor-command-type";
private File recommendationsDir;
private String recommendationsArtifactsLifetime;
@@ -163,6 +164,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
populateConfigurations(root, request);
populateConfigGroups(root, request);
populateAmbariServerInfo(root);
+ populateCommandType(root);
data.servicesJSON = mapper.writeValueAsString(root);
} catch (Exception e) {
// should not happen
@@ -183,6 +185,10 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
}
}
+ protected void populateCommandType(ObjectNode root) throws StackAdvisorException {
+ root.put(STACK_ADVISOR_COMMAND_TYPE_PROPERTY, getCommandType().toString());
+ }
+
private void populateConfigurations(ObjectNode root,
StackAdvisorRequest request) {
Map<String, Map<String, Map<String, String>>> configurations =
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index b77980a..416447e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -26,6 +26,8 @@ import socket
import re
import xml.etree.ElementTree as ET
+from stack_advisor import ActionType
+
class HDP22StackAdvisor(HDP21StackAdvisor):
def getServiceConfigurationRecommenderDict(self):
@@ -70,14 +72,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
# Above is the default calculated 'maximum' values derived purely from hosts.
- # However, there are 'maximum' and other attributes that actually change based on the values
- # of other configs. We need to update those values.
+ # However, there are 'maximum' and other attributes that actually should come from services["configurations"] during validations.
# On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
# So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation.
- # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
- # in this case (if 'changed-configurations' are empty).
- if ('changed-configurations' in services.keys() and services['changed-configurations'] and "yarn-site" in services["configurations"]):
+ if (self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS and "yarn-site" in services["configurations"]):
if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]):
putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
@@ -355,8 +354,19 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size, yarnMaxAllocationSize)
putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), container_size))
- putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
- putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+
+ # "maximum" and "minimum" attributes should come from services["configurations"] during validations.
+ # If one of dependencies are in changed-confogurations another can be absent. In this case we need to take both values from 'services'.
+ if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations' in services and services['changed-configurations'] \
+ and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb', services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb', services['changed-configurations']))):
+ if "yarn-site" in services["configurations"]:
+ if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
+ putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
+ if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
+ putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+ else:
+ putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
+ putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
putHiveSiteProperty("hive.prewarm.enabled", "false")
putHiveSiteProperty("hive.prewarm.numcontainers", "3")
@@ -1037,6 +1047,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site")
yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+
+ # "maximum" and "minimum" attributes should come from services["configurations"] during validations.
+ # If one of dependencies are in changed-confogurations another can be absent. In this case we need to take both values from 'services'.
+ if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations' in services and services['changed-configurations'] \
+ and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb', services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb', services['changed-configurations']))):
+ if "yarn-site" in services["configurations"] and "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"] \
+ and "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
+ yarnMinAllocationSize = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
+ yarnMaxAllocationSize = min(30 * int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+
putMapredPropertyAttribute("mapreduce.map.memory.mb", "maximum", yarnMaxAllocationSize)
putMapredPropertyAttribute("mapreduce.map.memory.mb", "minimum", yarnMinAllocationSize)
putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "maximum", yarnMaxAllocationSize)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 437fe4f..380c8f4 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -298,6 +298,12 @@ class StackAdvisor(object):
"""
pass
+class ActionType:
+ RECOMMEND_COMPONENT_LAYOUT_ACTION = 'recommend-component-layout'
+ VALIDATE_COMPONENT_LAYOUT_ACTION = 'validate-component-layout'
+ RECOMMEND_CONFIGURATIONS = 'recommend-configurations'
+ RECOMMEND_CONFIGURATION_DEPENDENCIES = 'recommend-configuration-dependencies'
+ VALIDATE_CONFIGURATIONS = 'validate-configurations'
class DefaultStackAdvisor(StackAdvisor):
"""
@@ -313,7 +319,6 @@ class DefaultStackAdvisor(StackAdvisor):
# Dictionary that maps serviceName or componentName to serviceAdvisor
self.serviceAdvisorsDict = {}
-
def getActiveHosts(self, hosts):
""" Filters the list of specified hosts object and returns
a list of hosts which are not in maintenance mode. """
@@ -987,13 +992,13 @@ class DefaultStackAdvisor(StackAdvisor):
config[configType]["properties"] = {}
def appendProperty(key, value):
# If property exists in changedConfigs, do not override, use user defined property
- if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
+ if self.isPropertyInChangedConfigs(configType, key, changedConfigs):
config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
else:
config[configType]["properties"][key] = str(value)
return appendProperty
- def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
+ def isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
for changedConfig in changedConfigs:
if changedConfig['type']==configType and changedConfig['name']==propertyName:
return True
@@ -1016,7 +1021,7 @@ class DefaultStackAdvisor(StackAdvisor):
def updatePropertyWithCallback(key, value, callback):
# If property exists in changedConfigs, do not override, use user defined property
- if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
+ if self.isPropertyInChangedConfigs(configType, key, changedConfigs):
config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
else:
# Give the callback an empty string if the mapping doesn't exist
@@ -1083,3 +1088,6 @@ class DefaultStackAdvisor(StackAdvisor):
def getServiceNames(self, services):
return [service["StackServices"]["service_name"] for service in services["services"]]
+
+ def getActionType(self, services):
+ return services.get("stack-advisor-command-type", ActionType.RECOMMEND_CONFIGURATIONS)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
index 25d634b..a956789 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
@@ -241,6 +241,24 @@ public class StackAdvisorCommandTest {
}
@Test
+ public void testPopulateCommandType() throws Exception {
+ File file = mock(File.class);
+ String recommendationsArtifactsLifetime = "1w";
+ StackAdvisorRunner stackAdvisorRunner = mock(StackAdvisorRunner.class);
+ AmbariMetaInfo ambariMetaInfo = mock(AmbariMetaInfo.class);
+ StackAdvisorCommand<TestResource> cmd = new TestStackAdvisorCommand(file, recommendationsArtifactsLifetime, "test", 1,
+ stackAdvisorRunner, ambariMetaInfo);
+ ObjectNode objectNode = (ObjectNode) cmd.mapper.readTree("{\"Versions\": " +
+ "{\"stack_name\": \"stack\", \"stack_version\":\"1.0.0\"}}");
+
+ cmd.populateCommandType(objectNode);
+
+ JsonNode commandType = objectNode.get("stack-advisor-command-type");
+ assertNotNull(commandType);
+ assertEquals(StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT.toString(), commandType.getTextValue());
+ }
+
+ @Test
public void testPopulateStackHierarchy_noParents() throws Exception {
File file = mock(File.class);
String recommendationsArtifactsLifetime = "1w";
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 5fe534a..41d8409 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1136,24 +1136,19 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
- # With no 'changed-configurations', we should get updated 'maximum's from recommendations
- # Else update from services.
+ # During a recommendation request, we should get updated 'maximum's from recommendations
+ # During a validation request, update from services.
#
# On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
# So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation.
- # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
- # in this case (if 'changed-configurations' are empty).
- #
- # Test not empty 'changed-configurations':
- services["changed-configurations"] = [
- {
- "type": "yarn-env",
- "name": "min_user_id"
- }
- ]
- services.pop("configurations", None)
+
+ # Test validate-configurations:
+ services.pop("changed-configurations", None)
+ services["stack-advisor-command-type"] = 'validate-configurations'
services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}},
"yarn-env": {"properties": {"min_user_id": "500"}}}
+
+ # Expected recommendations from 'servicess'
expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9'
expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9'
expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321'