You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/10/24 20:15:30 UTC
[2/2] git commit: AMBARI-7953. Modify HDP-2.2 stack advisor for the
tez config properties (aonishuk)
AMBARI-7953. Modify HDP-2.2 stack advisor for the tez config properties (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d70bc04
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d70bc04
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d70bc04
Branch: refs/heads/branch-1.7.0
Commit: 6d70bc044bc519ba8bf488a54bc51b3c1d8aa2c3
Parents: 7f82db5
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Oct 24 21:15:21 2014 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Oct 24 21:15:21 2014 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.2/services/stack_advisor.py | 18 ++++++
.../stacks/2.2/common/test_stack_advisor.py | 66 ++++++++++++++++++++
2 files changed, 84 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d70bc04/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 19b1065..b291530 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -23,6 +23,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
parentRecommendConfDict = super(HDP22StackAdvisor, self).getServiceConfigurationRecommenderDict()
childRecommendConfDict = {
"HDFS": self.recommendHDFSConfigurations,
+ "TEZ": self.recommendTezConfigurations
}
parentRecommendConfDict.update(childRecommendConfDict)
return parentRecommendConfDict
@@ -31,14 +32,31 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
def recommendHDFSConfigurations(self, configurations, clusterData):
self.putProperty(configurations, "hdfs-site")
+ def recommendTezConfigurations(self, configurations, clusterData):
+ putTezProperty = self.putProperty(configurations, "tez-site")
+ putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory']))
+
+ taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
+ taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory)
+ putTezProperty("tez.task.resource.memory.mb", taskResourceMemory)
+ putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4) if int(taskResourceMemory * 0.4) <= 2147483644 else 2147483644)
+ putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075))
+
def getServiceConfigurationValidators(self):
parentValidators = super(HDP22StackAdvisor, self).getServiceConfigurationValidators()
childValidators = {
"HDFS": ["hdfs-site", self.validateHDFSConfigurations],
+ "TEZ": ["tez-site", self.validateTezConfigurations]
}
parentValidators.update(childValidators)
return parentValidators
+ def validateTezConfigurations(self, properties, recommendedDefaults, configurations):
+ validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
+ {"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')},
+ {"config-name": 'tez.runtime.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.io.sort.mb')},
+ {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},]
+ return self.toConfigurationValidationProblems(validationItems, "tez-site")
def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations):
# We can not access property hadoop.security.authentication from the
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d70bc04/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 3d6b2e6..06c476e 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -42,6 +42,72 @@ class TestHDP22StackAdvisor(TestCase):
clazz = getattr(stack_advisor_impl, hdp22StackAdvisorClassName)
self.stackAdvisor = clazz()
+ def test_recommendTezConfigurations(self):
+ configurations = {}
+ clusterData = {
+ "mapMemory": 3000,
+ "amMemory": 2000,
+ "reduceMemory": 2056,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+ expected = {
+ "tez-site": {
+ "properties": {
+ "tez.am.resource.memory.mb": "4000",
+ "tez.task.resource.memory.mb": "768",
+ "tez.runtime.io.sort.mb": "307",
+ "tez.runtime.unordered.output.buffer.size-mb": "57"
+ }
+ }
+ }
+ self.stackAdvisor.recommendTezConfigurations(configurations, clusterData)
+ self.assertEquals(configurations, expected)
+
+ def test_recommendTezConfigurations_amMemoryMoreThan3072(self):
+ configurations = {}
+ clusterData = {
+ "mapMemory": 4000,
+ "amMemory": 3100,
+ "reduceMemory": 2056,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+ expected = {
+ "tez-site": {
+ "properties": {
+ "tez.am.resource.memory.mb": "3100",
+ "tez.task.resource.memory.mb": "768",
+ "tez.runtime.io.sort.mb": "307",
+ "tez.runtime.unordered.output.buffer.size-mb": "57"
+ }
+ }
+ }
+ self.stackAdvisor.recommendTezConfigurations(configurations, clusterData)
+ self.assertEquals(configurations, expected)
+
+ def test_recommendTezConfigurations_mapMemoryLessThan768(self):
+ configurations = {}
+ clusterData = {
+ "mapMemory": 760,
+ "amMemory": 2000,
+ "reduceMemory": 760,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+ expected = {
+ "tez-site": {
+ "properties": {
+ "tez.am.resource.memory.mb": "4000",
+ "tez.task.resource.memory.mb": "760",
+ "tez.runtime.io.sort.mb": "304",
+ "tez.runtime.unordered.output.buffer.size-mb": "57"
+ }
+ }
+ }
+ self.stackAdvisor.recommendTezConfigurations(configurations, clusterData)
+ self.assertEquals(configurations, expected)
+
def test_validateHDFSConfigurations(self):
self.maxDiff = None