You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/06/01 14:46:47 UTC
ambari git commit: AMBARI-11556. Ambari does not give a warning if
tez.am.resource.memory.mb is greater than yarn max container size (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk ee96fdebc -> 6f13d10ac
AMBARI-11556. Ambari does not give a warning if tez.am.resource.memory.mb is greater than yarn max container size (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6f13d10a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6f13d10a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6f13d10a
Branch: refs/heads/trunk
Commit: 6f13d10ac7b86bb8dc53e7a9da4573fb6116fc49
Parents: ee96fde
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Jun 1 15:46:40 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Jun 1 15:46:40 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.2/services/stack_advisor.py | 16 +++++++++
.../stacks/2.2/common/test_stack_advisor.py | 38 ++++++++++++++++++++
2 files changed, 54 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6f13d10a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 7743fd6..bcadd70 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -582,6 +582,22 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
{"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')},
{"config-name": 'tez.runtime.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.io.sort.mb')},
{"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},]
+
+ tez_site = properties
+ prop_name1 = 'tez.am.resource.memory.mb'
+ prop_name2 = 'tez.task.resource.memory.mb'
+ yarnSiteProperties = getSiteProperties(configurations, "yarn-site")
+ if yarnSiteProperties:
+ yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+ if int(tez_site[prop_name1]) > yarnMaxAllocationSize:
+ validationItems.append({"config-name": prop_name1,
+ "item": self.getWarnItem(
+ "{0} should be less than YARN max allocation size ({1})".format(prop_name1, yarnMaxAllocationSize))})
+ if int(tez_site[prop_name2]) > yarnMaxAllocationSize:
+ validationItems.append({"config-name": prop_name2,
+ "item": self.getWarnItem(
+ "{0} should be less than YARN max allocation size ({1})".format(prop_name2, yarnMaxAllocationSize))})
+
return self.toConfigurationValidationProblems(validationItems, "tez-site")
def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):
http://git-wip-us.apache.org/repos/asf/ambari/blob/6f13d10a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index dffca7b..e87a8d0 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2183,6 +2183,44 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
self.assertEqual("kms://http@myhost1:2222/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+ def test_validateTezConfigurationsEnv(self):
+ configurations = {
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "100",
+ "yarn.scheduler.maximum-allocation-mb": "2048"
+ }
+ }
+ }
+
+ recommendedDefaults = {'tez.task.resource.memory.mb': '1024',
+ 'tez.runtime.io.sort.mb' : '256',
+ 'tez.runtime.unordered.output.buffer.size-mb' : '256',
+ 'tez.am.resource.memory.mb' : '1024'}
+
+ properties = {'tez.task.resource.memory.mb': '2050',
+ 'tez.runtime.io.sort.mb' : '256',
+ 'tez.runtime.unordered.output.buffer.size-mb' : '256',
+ 'tez.am.resource.memory.mb' : '2050'}
+
+
+ res_expected = [{'config-name': 'tez.am.resource.memory.mb',
+ 'config-type': 'tez-site',
+ 'level': 'WARN',
+ 'message': "tez.am.resource.memory.mb should be less than YARN max allocation size (2048)",
+ 'type': 'configuration',
+ 'level': 'WARN'},
+ {'config-name': 'tez.task.resource.memory.mb',
+ 'config-type': 'tez-site',
+ 'level': 'WARN',
+ 'message': "tez.task.resource.memory.mb should be less than YARN max allocation size (2048)",
+ 'type': 'configuration',
+ 'level': 'WARN'}]
+
+ res = self.stackAdvisor.validateTezConfigurations(properties, recommendedDefaults, configurations, '', '')
+ self.assertEquals(res, res_expected)
+
+
def test_validateHDFSConfigurationsEnv(self):
configurations = {}