You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2016/12/19 16:00:33 UTC
ambari git commit: AMBARI-19236. Stackadvisor: incorrect
recommendations when only one of two dependencies was changed.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 2e154608f -> 6512cc1a0
AMBARI-19236. Stackadvisor: incorrect recommendations when only one of two dependencies was changed.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6512cc1a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6512cc1a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6512cc1a
Branch: refs/heads/branch-2.5
Commit: 6512cc1a08b306737043a525c49f19f8bffb9131
Parents: 2e15460
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Mon Dec 19 17:38:57 2016 +0200
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Mon Dec 19 17:38:57 2016 +0200
----------------------------------------------------------------------
.../src/main/resources/stacks/stack_advisor.py | 19 ++++++-
.../stacks/2.2/common/test_stack_advisor.py | 53 ++++++++++++++++++++
2 files changed, 71 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6512cc1a/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 8148379..8865b70 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -313,6 +313,10 @@ class DefaultStackAdvisor(StackAdvisor):
# Dictionary that maps serviceName or componentName to serviceAdvisor
self.serviceAdvisorsDict = {}
+ # Contains requested properties during 'recommend-configuration-dependencies' request.
+ # It's empty during other requests.
+ self.allRequestedProperties = None
+
def getActiveHosts(self, hosts):
""" Filters the list of specified hosts object and returns
@@ -880,6 +884,7 @@ class DefaultStackAdvisor(StackAdvisor):
return component["StackServiceComponents"]["hostnames"]
def recommendConfigurationDependencies(self, services, hosts):
+ self.allRequestedProperties = self.getAllRequestedProperties(services)
result = self.recommendConfigurations(services, hosts)
return self.filterResult(result, services)
@@ -1000,7 +1005,8 @@ class DefaultStackAdvisor(StackAdvisor):
config[configType]["properties"] = {}
def appendProperty(key, value):
# If property exists in changedConfigs, do not override, use user defined property
- if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
+ if not self.isPropertyRequested(configType, key, changedConfigs) \
+ and configType in userConfigs and key in userConfigs[configType]['properties']:
config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
else:
config[configType]["properties"][key] = str(value)
@@ -1012,6 +1018,17 @@ class DefaultStackAdvisor(StackAdvisor):
return True
return False
+ def isPropertyRequested(self, configType, propertyName, changedConfigs):
+ # When the property depends on more than one property, we need to recalculate it based on the actual values
+ # of all related properties. But "changed-configurations" usually contains only one on the dependent on properties.
+ # So allRequestedProperties is used to avoid recommendations of other properties that are not requested.
+ # Calculations should use user provided values for all properties that we depend on, not only the one that
+ # came in the "changed-configurations".
+ if self.allRequestedProperties:
+ return configType in self.allRequestedProperties and propertyName in self.allRequestedProperties[configType]
+ else:
+ return not self.__isPropertyInChangedConfigs(configType, propertyName, changedConfigs)
+
def updateProperty(self, config, configType, services=None):
userConfigs = {}
changedConfigs = []
http://git-wip-us.apache.org/repos/asf/ambari/blob/6512cc1a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 02d6c80..cad2d1d 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1173,6 +1173,59 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+ def test_multipleDependsOn(self):
+ configurations = {
+ "yarn-env": {
+ "properties": {
+ "min_user_id": "500"
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.nodemanager.resource.memory-mb": "1280",
+ "yarn.scheduler.minimum-allocation-mb": "350",
+ "yarn.scheduler.maximum-allocation-mb": "1000",
+ },
+ },
+ "mapred-site": {
+ "properties": {
+ "mapreduce.map.memory.mb": "0",
+ "mapreduce.reduce.memory.mb": "111"
+ }
+ }
+ }
+ clusterData = {
+ "cpu": 4,
+ "containers" : 5,
+ "ramPerContainer": 256
+ }
+
+ services = {
+ "configurations": configurations,
+ "services": [],
+ "changed-configurations": [
+ {
+ "type": "yarn-site",
+ "name": "yarn.scheduler.maximum-allocation-mb",
+ "old_value": "512"
+ },
+ ]
+
+ }
+ hosts = {}
+
+ # immitate recommend-configuration-dependencies request with only "yarn.scheduler.maximum-allocation-mb" in "changed-configurations"
+ self.stackAdvisor.allRequestedProperties = {'yarn-site': ['yarn.scheduler.maximum-allocation-mb'], 'mapred-site': ['mapreduce.map.memory.mb']}
+
+ self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts)
+
+ # changed-configurations contain only "yarn.scheduler.maximum-allocation-mb".
+ # Ensure that user provided value (350) for "yarn.scheduler.minimum-allocation-mb" is used.
+ # The recommended default for "yarn.scheduler.minimum-allocation-mb" is 256.
+ self.assertEquals(configurations['mapred-site']['properties']['mapreduce.map.memory.mb'], '350') # should not be 256
+
+ # assert that not requested property was not changed
+ self.assertEquals(configurations['mapred-site']['properties']['mapreduce.reduce.memory.mb'], '111')
def test_recommendHiveConfigurationAttributes(self):
self.maxDiff = None