You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/01/18 05:00:46 UTC
ambari git commit: AMBARI-19578. Fix issues around modifying yarn min
container size (smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 513aa1aad -> a65aa28e7
AMBARI-19578. Fix issues around modifying yarn min container size (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a65aa28e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a65aa28e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a65aa28e
Branch: refs/heads/branch-2.5
Commit: a65aa28e7e70c46db8a7e827f6d31cca1d409a3e
Parents: 513aa1a
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Jan 17 20:59:33 2017 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Jan 17 20:59:33 2017 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 46 ++++++++++++++------
.../stacks/2.0.6/common/test_stack_advisor.py | 21 +++++----
.../stacks/2.2/common/test_stack_advisor.py | 20 ++++-----
.../stacks/2.5/common/test_stack_advisor.py | 44 +++++++++----------
4 files changed, 74 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65aa28e/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index f402f78..968a94f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -124,8 +124,26 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
nodemanagerMinRam = 1048576 # 1TB in mb
if "referenceNodeManagerHost" in clusterData:
nodemanagerMinRam = min(clusterData["referenceNodeManagerHost"]["total_mem"]/1024, nodemanagerMinRam)
+
+ callContext = getCallContext(services)
putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam))))
- putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['minContainerRam']))
+ # read from the supplied config
+ #if 'recommendConfigurations' != callContext and \
+ # "yarn-site" in services["configurations"] and \
+ # "yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]:
+ # putYarnProperty('yarn.nodemanager.resource.memory-mb', int(services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
+ if 'recommendConfigurations' == callContext:
+ putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam))))
+ else:
+ # read from the supplied config
+ if "yarn-site" in services["configurations"] and "yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]:
+ putYarnProperty('yarn.nodemanager.resource.memory-mb', int(services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
+ else:
+ putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam))))
+ pass
+ pass
+
+ putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['yarnMinContainerSize']))
putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
putYarnEnvProperty('min_user_id', self.get_system_min_uid())
@@ -1023,7 +1041,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024)
Logger.info("Memory for YARN apps - cluster[totalAvailableRam]: " + str(cluster["totalAvailableRam"]))
- suggestedMinContainerRam = 1024
+ suggestedMinContainerRam = 1024 # new smaller value for YARN min container
callContext = getCallContext(services)
if services: # its never None but some unit tests pass it as None
@@ -1035,14 +1053,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
str(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]).isdigit():
Logger.info("Using user provided yarn.scheduler.minimum-allocation-mb = " +
str(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
- cluster["minContainerRam"] = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
- Logger.info("Minimum ram per container due to user input - cluster[minContainerRam]: " + str(cluster["minContainerRam"]))
- if cluster["minContainerRam"] > cluster["totalAvailableRam"]:
- cluster["minContainerRam"] = cluster["totalAvailableRam"]
- Logger.info("Minimum ram per container after checking against limit - cluster[minContainerRam]: " + str(cluster["minContainerRam"]))
+ cluster["yarnMinContainerSize"] = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
+ Logger.info("Minimum ram per container due to user input - cluster[yarnMinContainerSize]: " + str(cluster["yarnMinContainerSize"]))
+ if cluster["yarnMinContainerSize"] > cluster["totalAvailableRam"]:
+ cluster["yarnMinContainerSize"] = cluster["totalAvailableRam"]
+ Logger.info("Minimum ram per container after checking against limit - cluster[yarnMinContainerSize]: " + str(cluster["yarnMinContainerSize"]))
pass
- cluster["minContainerSize"] = cluster["minContainerRam"]
- suggestedMinContainerRam = cluster["minContainerRam"]
+ cluster["minContainerSize"] = cluster["yarnMinContainerSize"] # set to what user has suggested as YARN min container size
+ suggestedMinContainerRam = cluster["yarnMinContainerSize"]
pass
pass
pass
@@ -1061,19 +1079,19 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
pass
cluster["ramPerContainer"] = int(abs(cluster["totalAvailableRam"] / cluster["containers"]))
- cluster["minContainerRam"] = min(suggestedMinContainerRam, cluster["ramPerContainer"])
+ cluster["yarnMinContainerSize"] = min(suggestedMinContainerRam, cluster["ramPerContainer"])
Logger.info("Ram per containers before normalization - cluster[ramPerContainer]: " + str(cluster["ramPerContainer"]))
- '''If greater than cluster["minContainerRam"], value will be in multiples of cluster["minContainerRam"]'''
- if cluster["ramPerContainer"] > cluster["minContainerRam"]:
- cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / cluster["minContainerRam"]) * cluster["minContainerRam"]
+ '''If greater than cluster["yarnMinContainerSize"], value will be in multiples of cluster["yarnMinContainerSize"]'''
+ if cluster["ramPerContainer"] > cluster["yarnMinContainerSize"]:
+ cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / cluster["yarnMinContainerSize"]) * cluster["yarnMinContainerSize"]
cluster["mapMemory"] = int(cluster["ramPerContainer"])
cluster["reduceMemory"] = cluster["ramPerContainer"]
cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"])
- Logger.info("Min container size - cluster[minContainerRam]: " + str(cluster["minContainerRam"]))
+ Logger.info("Min container size - cluster[yarnMinContainerSize]: " + str(cluster["yarnMinContainerSize"]))
Logger.info("Available memory for map - cluster[mapMemory]: " + str(cluster["mapMemory"]))
Logger.info("Available memory for reduce - cluster[reduceMemory]: " + str(cluster["reduceMemory"]))
Logger.info("Available memory for am - cluster[amMemory]: " + str(cluster["amMemory"]))
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65aa28e/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index a9473ad..f39703c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -258,7 +258,6 @@ class TestHDP206StackAdvisor(TestCase):
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
- {"message": "Value is less than the recommended default of 510", "level": "WARN"},
{'message': 'Value should be set for yarn.nodemanager.linux-container-executor.group', 'level': 'ERROR'},
{"message": "Value should be integer", "level": "ERROR"},
{"message": "Value should be set", "level": "ERROR"}
@@ -472,7 +471,7 @@ class TestHDP206StackAdvisor(TestCase):
"reservedRam": 4,
"hbaseRam": 4,
"minContainerSize": 1024,
- "minContainerRam" : 1024,
+ "yarnMinContainerSize" : 1024,
"totalAvailableRam": 15360,
"containers": 4,
"ramPerContainer": 3072,
@@ -584,7 +583,7 @@ class TestHDP206StackAdvisor(TestCase):
"reservedRam": 4,
"hbaseRam": 4,
"minContainerSize": 2048,
- "minContainerRam" : 2048,
+ "yarnMinContainerSize" : 2048,
"totalAvailableRam": 15360,
"containers": 4,
"ramPerContainer": 2048,
@@ -649,7 +648,7 @@ class TestHDP206StackAdvisor(TestCase):
"reservedRam": 4,
"hbaseRam": 4,
"minContainerSize": 2048,
- "minContainerRam" : 2048,
+ "yarnMinContainerSize" : 2048,
"totalAvailableRam": 15360,
"containers": 4,
"ramPerContainer": 2048,
@@ -714,7 +713,7 @@ class TestHDP206StackAdvisor(TestCase):
"reservedRam": 4,
"hbaseRam": 4,
"minContainerSize": 4096,
- "minContainerRam" : 4096,
+ "yarnMinContainerSize" : 4096,
"totalAvailableRam": 15360,
"containers": 3,
"ramPerContainer": 4096,
@@ -761,7 +760,7 @@ class TestHDP206StackAdvisor(TestCase):
"reservedRam": 2,
"hbaseRam": 1,
"minContainerSize": 512,
- "minContainerRam" : 512,
+ "yarnMinContainerSize" : 512,
"totalAvailableRam": 3072,
"containers": 6,
"ramPerContainer": 512,
@@ -837,7 +836,7 @@ class TestHDP206StackAdvisor(TestCase):
expected["mapMemory"] = 128
expected["minContainerSize"] = 128
expected["reduceMemory"] = 128
- expected["minContainerRam"] = 128
+ expected["yarnMinContainerSize"] = 128
expected["ram"] = 0
expected["ramPerContainer"] = 128
expected["reservedRam"] = 1
@@ -881,7 +880,7 @@ class TestHDP206StackAdvisor(TestCase):
"mapMemory": 3072,
"reduceMemory": 3072,
"amMemory": 3072,
- "minContainerRam": 1024,
+ "yarnMinContainerSize": 1024,
"referenceHost": hosts["items"][0]["Hosts"]
}
@@ -938,7 +937,7 @@ class TestHDP206StackAdvisor(TestCase):
clusterData = {
"containers" : 5,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
"yarn-env": {
@@ -966,7 +965,7 @@ class TestHDP206StackAdvisor(TestCase):
"mapMemory": 567,
"reduceMemory": 345.6666666666666,
"amMemory": 123.54,
- "minContainerRam": 123.54
+ "yarnMinContainerSize": 123.54
}
expected = {
"mapred-site": {
@@ -1009,7 +1008,7 @@ class TestHDP206StackAdvisor(TestCase):
"mapMemory": 170,
"reduceMemory": 170,
"amMemory": 170,
- "minContainerRam" : 170
+ "yarnMinContainerSize" : 170
}
self.assertEquals(result, expected)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65aa28e/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 0a80703..db71746 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -90,7 +90,7 @@ class TestHDP22StackAdvisor(TestCase):
"reduceMemory": 2056,
"containers": 3,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
"tez-site": {
@@ -184,7 +184,7 @@ class TestHDP22StackAdvisor(TestCase):
"amMemory": 3100,
"reduceMemory": 2056,
"containers": 3,
- "minContainerRam": 256,
+ "yarnMinContainerSize": 256,
"ramPerContainer": 256
}
expected = {
@@ -271,7 +271,7 @@ class TestHDP22StackAdvisor(TestCase):
"reduceMemory": 760,
"containers": 3,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
"tez-site": {
@@ -888,7 +888,7 @@ class TestHDP22StackAdvisor(TestCase):
"cpu": 4,
"containers" : 5,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
"yarn-env": {
@@ -928,7 +928,7 @@ class TestHDP22StackAdvisor(TestCase):
"cpu": 4,
"containers": 5,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
"spark-defaults": {
@@ -961,7 +961,7 @@ class TestHDP22StackAdvisor(TestCase):
"cpu": 4,
"containers" : 5,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
"yarn-env": {
@@ -1223,7 +1223,7 @@ class TestHDP22StackAdvisor(TestCase):
"cpu": 4,
"containers" : 5,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
services = {
@@ -1271,7 +1271,7 @@ class TestHDP22StackAdvisor(TestCase):
"reduceMemory": 2056,
"containers": 3,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
@@ -1762,7 +1762,7 @@ class TestHDP22StackAdvisor(TestCase):
"containers" : 7,
"ramPerContainer": 256,
"totalAvailableRam": 4096,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
"cluster-env": {
@@ -2031,7 +2031,7 @@ class TestHDP22StackAdvisor(TestCase):
"cpu": 4,
"containers" : 5,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
"yarn-env": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65aa28e/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 790c6a7..ad962fd 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -375,7 +375,7 @@ class TestHDP25StackAdvisor(TestCase):
"cpu": 4,
"containers": 5,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
"spark2-defaults": {
@@ -803,7 +803,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
},
- "minContainerRam": 512
+ "yarnMinContainerSize": 512
}
@@ -972,7 +972,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
},
- "minContainerRam": 512
+ "yarnMinContainerSize": 512
}
configurations = {
@@ -1153,7 +1153,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
},
- "minContainerRam": 512
+ "yarnMinContainerSize": 512
}
@@ -1353,7 +1353,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 2048
},
- "minContainerRam": 512
+ "yarnMinContainerSize": 512
}
@@ -1546,7 +1546,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 51200 * 1024
},
- "minContainerRam": 1024
+ "yarnMinContainerSize": 1024
}
configurations = {
@@ -1747,7 +1747,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 40960 * 1024
},
- "minContainerRam": 1024
+ "yarnMinContainerSize": 1024
}
configurations = {
@@ -1942,7 +1942,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 12288 * 1024
},
- "minContainerRam": 341
+ "yarnMinContainerSize": 341
}
@@ -2140,7 +2140,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 204800 * 1024
},
- "minContainerRam": 1024
+ "yarnMinContainerSize": 1024
}
configurations = {
@@ -2341,7 +2341,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 40960 * 1024
},
- "minContainerRam": 1024
+ "yarnMinContainerSize": 1024
}
@@ -2538,7 +2538,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 204800 * 1024
},
- "minContainerRam": 341
+ "yarnMinContainerSize": 341
}
@@ -2734,7 +2734,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 204800 * 1024
},
- "minContainerRam": 1024
+ "yarnMinContainerSize": 1024
}
configurations = {
@@ -2925,7 +2925,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 328960 * 1024
},
- "minContainerRam": 1024
+ "yarnMinContainerSize": 1024
}
@@ -3145,7 +3145,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
},
- "minContainerRam": 512
+ "yarnMinContainerSize": 512
}
configurations = {
@@ -3159,12 +3159,12 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '1.0', 'minimum': '1'})
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '9728')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '204288')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '3584')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '198144')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'], 'true')
self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '4915')
@@ -3370,7 +3370,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
},
- "minContainerRam": 512
+ "yarnMinContainerSize": 512
}
configurations = {
@@ -3563,7 +3563,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 328960 * 1024
},
- "minContainerRam": 1024
+ "yarnMinContainerSize": 1024
}
configurations = {
@@ -3746,7 +3746,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 328960 * 1024
},
- "minContainerRam": 1024
+ "yarnMinContainerSize": 1024
}
configurations = {
@@ -3877,7 +3877,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
},
- "minContainerRam": 512
+ "yarnMinContainerSize": 512
}
configurations = {
@@ -3923,7 +3923,7 @@ class TestHDP25StackAdvisor(TestCase):
"reduceMemory": 2056,
"containers": 3,
"ramPerContainer": 256,
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
expected = {
'application-properties': {
@@ -4659,7 +4659,7 @@ class TestHDP25StackAdvisor(TestCase):
"referenceNodeManagerHost" : {
"total_mem" : 328960 * 1024
},
- "minContainerRam": 256
+ "yarnMinContainerSize": 256
}
hosts = {
"items" : [