You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mp...@apache.org on 2015/06/23 17:50:40 UTC
ambari git commit: AMBARI-11971. Recommended min and max for
slider-config are equal. (Andrew Onischuk via mpapirkovskyy)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 038b13673 -> 550470fbc
AMBARI-11971. Recommended min and max for slider-config are equal. (Andrew Onischuk via mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/550470fb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/550470fb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/550470fb
Branch: refs/heads/branch-2.1
Commit: 550470fbc866c08a76555608660e9c50a9b78809
Parents: 038b136
Author: Myroslav Papirkovskyy <mp...@hortonworks.com>
Authored: Tue Jun 23 18:48:52 2015 +0300
Committer: Myroslav Papirkovskyy <mp...@hortonworks.com>
Committed: Tue Jun 23 18:50:12 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.2/services/stack_advisor.py | 10 +--
.../stacks/2.2/common/test_stack_advisor.py | 79 ++++++++++++++++----
2 files changed, 71 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/550470fb/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index e5b6adb..a085ab2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -318,7 +318,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
container_size = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size, yarnMaxAllocationSize)
- putHiveSiteProperty("hive.tez.container.size", container_size)
+ putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), container_size))
putHiveSiteProperty("hive.prewarm.enabled", "false")
putHiveSiteProperty("hive.prewarm.numcontainers", "3")
putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true")
@@ -577,11 +577,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
putTezProperty = self.putProperty(configurations, "tez-site")
- putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory']))
+ putTezProperty("tez.am.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory'])))
taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory, yarnMaxAllocationSize)
- putTezProperty("tez.task.resource.memory.mb", taskResourceMemory)
+ putTezProperty("tez.task.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), taskResourceMemory))
putTezProperty("tez.runtime.io.sort.mb", min(int(taskResourceMemory * 0.4), 2047))
putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075))
putTezProperty("tez.session.am.dag.submit.timeout.secs", "600")
@@ -642,8 +642,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
min_mapreduce_map_memory_mb = 1536
min_mapreduce_reduce_memory_mb = 1536
min_mapreduce_map_java_opts = 1024
- putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
- putMapredProperty('mapreduce.reduce.memory.mb', max(min_mapreduce_reduce_memory_mb, min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam))))
+ putMapredProperty('mapreduce.map.memory.mb', min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))))
+ putMapredProperty('mapreduce.reduce.memory.mb', min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), max(min_mapreduce_reduce_memory_mb, min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam)))))
mapredMapXmx = int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]));
putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(max(min_mapreduce_map_java_opts, mapredMapXmx)) + "m")
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m")
http://git-wip-us.apache.org/repos/asf/ambari/blob/550470fb/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 436d796..7726b16 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -86,7 +86,8 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"tez-site": {
"properties": {
- "tez.am.resource.memory.mb": "4000",
+ # tez.am.resource.memory.mb must be <= yarn.scheduler.maximum-allocation-mb
+ "tez.am.resource.memory.mb": "2048",
"tez.task.resource.memory.mb": "768",
"tez.runtime.io.sort.mb": "307",
"tez.runtime.unordered.output.buffer.size-mb": "57",
@@ -122,7 +123,8 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"tez-site": {
"properties": {
- "tez.am.resource.memory.mb": "3100",
+ # tez.am.resource.memory.mb must be <= yarn.scheduler.maximum-allocation-mb
+ "tez.am.resource.memory.mb": "2048",
"tez.task.resource.memory.mb": "768",
"tez.runtime.io.sort.mb": "307",
"tez.runtime.unordered.output.buffer.size-mb": "57",
@@ -158,7 +160,8 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"tez-site": {
"properties": {
- "tez.am.resource.memory.mb": "4000",
+ # tez.am.resource.memory.mb must be <= yarn.scheduler.maximum-allocation-mb
+ "tez.am.resource.memory.mb": "2048",
"tez.task.resource.memory.mb": "760",
"tez.runtime.io.sort.mb": "304",
"tez.runtime.unordered.output.buffer.size-mb": "57",
@@ -1255,16 +1258,16 @@ class TestHDP22StackAdvisor(TestCase):
},
"yarn-site": {
"properties": {
- "yarn.nodemanager.resource.memory-mb": "1280",
+ "yarn.nodemanager.resource.memory-mb": "2048",
"yarn.scheduler.minimum-allocation-mb": "100",
- "yarn.scheduler.maximum-allocation-mb": "1280",
+ "yarn.scheduler.maximum-allocation-mb": "2048",
"yarn.nodemanager.resource.cpu-vcores": "2"
},
}
}
clusterData = {
"cpu": 4,
- "containers" : 5,
+ "containers" : 7,
"ramPerContainer": 256,
"totalAvailableRam": 4096,
}
@@ -1286,21 +1289,21 @@ class TestHDP22StackAdvisor(TestCase):
},
"property_attributes": {
'mapreduce.task.io.sort.mb': {'maximum': '2047'},
- 'yarn.app.mapreduce.am.resource.mb': {'maximum': '1280',
+ 'yarn.app.mapreduce.am.resource.mb': {'maximum': '1792',
'minimum': '100'},
- 'mapreduce.map.memory.mb': {'maximum': '1280',
+ 'mapreduce.map.memory.mb': {'maximum': '1792',
'minimum': '100'},
- 'mapreduce.reduce.memory.mb': {'maximum': '1280',
+ 'mapreduce.reduce.memory.mb': {'maximum': '1792',
'minimum': '100'}
}
},
"yarn-site": {
"properties": {
- "yarn.nodemanager.resource.memory-mb": "1280",
+ "yarn.nodemanager.resource.memory-mb": "1792",
"yarn.scheduler.minimum-allocation-mb": "100",
"yarn.scheduler.maximum-allocation-vcores": "1",
"yarn.scheduler.minimum-allocation-vcores": "1",
- "yarn.scheduler.maximum-allocation-mb": "1280",
+ "yarn.scheduler.maximum-allocation-mb": "1792",
"yarn.nodemanager.resource.cpu-vcores": "1"
},
"property_attributes": {
@@ -1308,8 +1311,8 @@ class TestHDP22StackAdvisor(TestCase):
'yarn.nodemanager.resource.cpu-vcores': {'maximum': '2'},
'yarn.scheduler.minimum-allocation-vcores': {'maximum': '1'},
'yarn.scheduler.maximum-allocation-vcores': {'maximum': '1'},
- 'yarn.scheduler.minimum-allocation-mb': {'maximum': '1280'},
- 'yarn.scheduler.maximum-allocation-mb': {'maximum': '1280'}
+ 'yarn.scheduler.minimum-allocation-mb': {'maximum': '1792'},
+ 'yarn.scheduler.maximum-allocation-mb': {'maximum': '1792'}
}
}
}
@@ -1734,6 +1737,56 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+ configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"] = "700"
+
+ expected = {
+ "yarn-env": {
+ "properties": {
+ "min_user_id": "500"
+ }
+ },
+ "mapred-site": {
+ "properties": {
+ "mapreduce.map.memory.mb": "700",
+ "mapreduce.reduce.memory.mb": "1280",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx560m -Dhdp.version=${hdp.version}",
+ "mapreduce.reduce.java.opts": "-Xmx1024m",
+ "yarn.app.mapreduce.am.resource.mb": "700",
+ "mapreduce.map.java.opts": "-Xmx560m",
+ "mapreduce.task.io.sort.mb": "392"
+ },
+ "property_attributes": {
+ 'mapreduce.task.io.sort.mb': {'maximum': '2047'},
+ 'yarn.app.mapreduce.am.resource.mb': {'maximum': '1280',
+ 'minimum': '700'},
+ 'mapreduce.map.memory.mb': {'maximum': '1280',
+ 'minimum': '700'},
+ 'mapreduce.reduce.memory.mb': {'maximum': '1280',
+ 'minimum': '700'}
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.nodemanager.resource.memory-mb": "1280",
+ "yarn.scheduler.minimum-allocation-mb": "700",
+ "yarn.scheduler.maximum-allocation-vcores": "1",
+ "yarn.scheduler.minimum-allocation-vcores": "1",
+ "yarn.scheduler.maximum-allocation-mb": "1280",
+ "yarn.nodemanager.resource.cpu-vcores": "1"
+ },
+ "property_attributes": {
+ 'yarn.nodemanager.resource.memory-mb': {'maximum': '1877'},
+ 'yarn.nodemanager.resource.cpu-vcores': {'maximum': '2'},
+ 'yarn.scheduler.minimum-allocation-vcores': {'maximum': '1'},
+ 'yarn.scheduler.maximum-allocation-vcores': {'maximum': '1'},
+ 'yarn.scheduler.minimum-allocation-mb': {'maximum': '1280'},
+ 'yarn.scheduler.maximum-allocation-mb': {'maximum': '1280'}
+ }
+ }
+ }
+ self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
def test_recommendAmsConfigurations(self):
configurations = {}
clusterData = {}