You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/01/13 23:00:49 UTC
ambari git commit: AMBARI-19338. AM sizing for LLAP - min container
size changes (smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 74de3e0b9 -> 95aa867b2
AMBARI-19338. AM sizing for LLAP - min container size changes (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/95aa867b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/95aa867b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/95aa867b
Branch: refs/heads/branch-2.5
Commit: 95aa867b20a96685606af0f4d44b19c22b3182c7
Parents: 74de3e0
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Jan 12 15:16:33 2017 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Jan 13 14:56:00 2017 -0800
----------------------------------------------------------------------
.../src/main/resources/scripts/stack_advisor.py | 5 +
.../stacks/HDP/2.0.6/services/stack_advisor.py | 68 +++-
.../stacks/HDP/2.2/services/stack_advisor.py | 17 +-
.../stacks/2.0.6/common/test_stack_advisor.py | 383 ++++++++++++++++++-
.../stacks/2.2/common/test_stack_advisor.py | 58 +--
.../stacks/2.3/common/test_stack_advisor.py | 3 +
.../stacks/2.5/common/test_stack_advisor.py | 60 ++-
7 files changed, 521 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/95aa867b/ambari-server/src/main/resources/scripts/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py
index abfab87..56bd34a 100755
--- a/ambari-server/src/main/resources/scripts/stack_advisor.py
+++ b/ambari-server/src/main/resources/scripts/stack_advisor.py
@@ -100,18 +100,23 @@ def main(argv=None):
hosts = stackAdvisor.filterHostMounts(hosts, services)
if action == RECOMMEND_COMPONENT_LAYOUT_ACTION:
+ services['context'] = {'call_type': 'recommendComponentLayout'}
result = stackAdvisor.recommendComponentLayout(services, hosts)
result_file = os.path.join(actionDir, "component-layout.json")
elif action == VALIDATE_COMPONENT_LAYOUT_ACTION:
+ services['context'] = {'call_type': 'validateComponentLayout'}
result = stackAdvisor.validateComponentLayout(services, hosts)
result_file = os.path.join(actionDir, "component-layout-validation.json")
elif action == RECOMMEND_CONFIGURATIONS:
+ services['context'] = {'call_type': 'recommendConfigurations'}
result = stackAdvisor.recommendConfigurations(services, hosts)
result_file = os.path.join(actionDir, "configurations.json")
elif action == RECOMMEND_CONFIGURATION_DEPENDENCIES:
+ services['context'] = {'call_type': 'recommendConfigurationDependencies'}
result = stackAdvisor.recommendConfigurationDependencies(services, hosts)
result_file = os.path.join(actionDir, "configurations.json")
else: # action == VALIDATE_CONFIGURATIONS
+ services['context'] = {'call_type': 'validateConfigurations'}
result = stackAdvisor.validateConfigurations(services, hosts)
result_file = os.path.join(actionDir, "configurations-validation.json")
http://git-wip-us.apache.org/repos/asf/ambari/blob/95aa867b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 480e73d..f402f78 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -125,7 +125,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if "referenceNodeManagerHost" in clusterData:
nodemanagerMinRam = min(clusterData["referenceNodeManagerHost"]["total_mem"]/1024, nodemanagerMinRam)
putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam))))
- putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer']))
+ putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['minContainerRam']))
putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
putYarnEnvProperty('min_user_id', self.get_system_min_uid())
@@ -1010,7 +1010,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
cluster["minContainerSize"] = {
- cluster["ram"] <= 4: 256,
+ cluster["ram"] <= 3: 128,
+ 3 < cluster["ram"] <= 4: 256,
4 < cluster["ram"] <= 8: 512,
8 < cluster["ram"] <= 24: 1024,
24 < cluster["ram"]: 2048
@@ -1020,22 +1021,64 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if cluster["hBaseInstalled"]:
totalAvailableRam -= cluster["hbaseRam"]
cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024)
+ Logger.info("Memory for YARN apps - cluster[totalAvailableRam]: " + str(cluster["totalAvailableRam"]))
+
+ suggestedMinContainerRam = 1024
+ callContext = getCallContext(services)
+
+ if services: # its never None but some unit tests pass it as None
+ if None != getOldValue(self, services, "yarn-site", "yarn.scheduler.minimum-allocation-mb") or \
+ 'recommendConfigurations' != callContext:
+ '''yarn.scheduler.minimum-allocation-mb has changed - then pick this value up'''
+ if "yarn-site" in services["configurations"] and \
+ "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"] and \
+ str(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]).isdigit():
+ Logger.info("Using user provided yarn.scheduler.minimum-allocation-mb = " +
+ str(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
+ cluster["minContainerRam"] = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
+ Logger.info("Minimum ram per container due to user input - cluster[minContainerRam]: " + str(cluster["minContainerRam"]))
+ if cluster["minContainerRam"] > cluster["totalAvailableRam"]:
+ cluster["minContainerRam"] = cluster["totalAvailableRam"]
+ Logger.info("Minimum ram per container after checking against limit - cluster[minContainerRam]: " + str(cluster["minContainerRam"]))
+ pass
+ cluster["minContainerSize"] = cluster["minContainerRam"]
+ suggestedMinContainerRam = cluster["minContainerRam"]
+ pass
+ pass
+ pass
+
+
'''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))'''
- cluster["containers"] = round(max(3,
+ cluster["containers"] = int(round(max(3,
min(2 * cluster["cpu"],
min(ceil(1.8 * cluster["disk"]),
- cluster["totalAvailableRam"] / cluster["minContainerSize"]))))
+ cluster["totalAvailableRam"] / cluster["minContainerSize"])))))
+ Logger.info("Containers per node - cluster[containers]: " + str(cluster["containers"]))
+
+ if cluster["containers"] * cluster["minContainerSize"] > cluster["totalAvailableRam"]:
+ cluster["containers"] = ceil(cluster["totalAvailableRam"] / cluster["minContainerSize"])
+ Logger.info("Modified number of containers based on provided value for yarn.scheduler.minimum-allocation-mb")
+ pass
+
+ cluster["ramPerContainer"] = int(abs(cluster["totalAvailableRam"] / cluster["containers"]))
+ cluster["minContainerRam"] = min(suggestedMinContainerRam, cluster["ramPerContainer"])
+ Logger.info("Ram per containers before normalization - cluster[ramPerContainer]: " + str(cluster["ramPerContainer"]))
+
+ '''If greater than cluster["minContainerRam"], value will be in multiples of cluster["minContainerRam"]'''
+ if cluster["ramPerContainer"] > cluster["minContainerRam"]:
+ cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / cluster["minContainerRam"]) * cluster["minContainerRam"]
- '''ramPerContainers = max(2GB, RAM - reservedRam - hBaseRam) / containers'''
- cluster["ramPerContainer"] = abs(cluster["totalAvailableRam"] / cluster["containers"])
- '''If greater than 1GB, value will be in multiples of 512.'''
- if cluster["ramPerContainer"] > 1024:
- cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / 512) * 512
cluster["mapMemory"] = int(cluster["ramPerContainer"])
cluster["reduceMemory"] = cluster["ramPerContainer"]
cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"])
+ Logger.info("Min container size - cluster[minContainerRam]: " + str(cluster["minContainerRam"]))
+ Logger.info("Available memory for map - cluster[mapMemory]: " + str(cluster["mapMemory"]))
+ Logger.info("Available memory for reduce - cluster[reduceMemory]: " + str(cluster["reduceMemory"]))
+ Logger.info("Available memory for am - cluster[amMemory]: " + str(cluster["amMemory"]))
+
+
return cluster
def getServiceConfigurationValidators(self):
@@ -1901,6 +1944,13 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
service_meta = service_meta[0]
return [item[__stack_service_components]["component_name"] for item in service_meta["components"]]
+def getCallContext(services):
+ if services:
+ if 'context' in services:
+ Logger.info("context : " + str (services['context']))
+ return services['context']['call_type']
+ return ""
+
def getOldValue(self, services, configType, propertyName):
if services:
http://git-wip-us.apache.org/repos/asf/ambari/blob/95aa867b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index cf7134e..d07134e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -1093,7 +1093,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
nodemanagerMinRam = 1048576 # 1TB in mb
if "referenceNodeManagerHost" in clusterData:
nodemanagerMinRam = min(clusterData["referenceNodeManagerHost"]["total_mem"]/1024, nodemanagerMinRam)
- putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
+ putMapredProperty('yarn.app.mapreduce.am.resource.mb', max(int(clusterData['ramPerContainer']),int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"]))) + "m" + " -Dhdp.version=${hdp.version}")
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
min_mapreduce_map_memory_mb = 0
@@ -1103,8 +1103,19 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
min_mapreduce_map_memory_mb = 1536
min_mapreduce_reduce_memory_mb = 1536
min_mapreduce_map_java_opts = 1024
- putMapredProperty('mapreduce.map.memory.mb', min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))))
- putMapredProperty('mapreduce.reduce.memory.mb', min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), max(min_mapreduce_reduce_memory_mb, min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam)))))
+
+ putMapredProperty('mapreduce.map.memory.mb',
+ min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]),
+ max(min_mapreduce_map_memory_mb,
+ max(int(clusterData['ramPerContainer']),
+ int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))))
+ putMapredProperty('mapreduce.reduce.memory.mb',
+ min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]),
+ max(max(min_mapreduce_reduce_memory_mb,
+ int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])),
+ min(2*int(clusterData['ramPerContainer']),
+ int(nodemanagerMinRam)))))
+
mapredMapXmx = int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]));
putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(max(min_mapreduce_map_java_opts, mapredMapXmx)) + "m")
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m")
http://git-wip-us.apache.org/repos/asf/ambari/blob/95aa867b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index d781147..a9473ad 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -258,7 +258,7 @@ class TestHDP206StackAdvisor(TestCase):
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
- {"message": "Value is less than the recommended default of 512", "level": "WARN"},
+ {"message": "Value is less than the recommended default of 510", "level": "WARN"},
{'message': 'Value should be set for yarn.nodemanager.linux-container-executor.group', 'level': 'ERROR'},
{"message": "Value should be integer", "level": "ERROR"},
{"message": "Value should be set", "level": "ERROR"}
@@ -387,6 +387,348 @@ class TestHDP206StackAdvisor(TestCase):
]
self.assertValidationResult(expectedItems, result)
+ def test__getConfigurationClusterSummary_withContext1(self):
+ servicesList = ["HDFS", "YARN"]
+ components = []
+ hosts = {
+ "items" : [
+ {
+ "Hosts" : {
+ "host_name" : "host1",
+ "cpu_count" : 2,
+ "total_mem" : 20471556,
+ "disk_info" : [
+ {
+ "available" : "21052800",
+ "device" : "/dev/vda1",
+ "used" : "3303636",
+ "percent" : "14%",
+ "size" : "25666616",
+ "type" : "ext4",
+ "mountpoint" : "/"
+ },
+ {
+ "available" : "244732200",
+ "device" : "/dev/vdb",
+ "used" : "60508",
+ "percent" : "1%",
+ "size" : "257899908",
+ "type" : "ext4",
+ "mountpoint" : "/grid/0"
+ }
+ ]
+ }
+ }
+ ]
+ }
+
+ services = {"services":
+ [{"StackServices":
+ {"service_name": "YARN",
+ "service_version": "2.6.0.2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "custom_commands": [
+
+ ],
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "service_name": "YARN",
+ "stack_name": "HDP",
+ "stack_version": "2.2",
+ "hostnames": [
+ "host1"
+ ]
+ },
+ "dependencies": [
+ ]
+ }
+ ],
+ }],
+ "configurations": {
+ "yarn-site" : {
+ "properties" : {
+ "yarn.scheduler.minimum-allocation-mb" : "1024",
+ "yarn.scheduler.maximum-allocation-mb" : "2048"
+ }
+ }
+ },
+ "changed-configurations" : [ ]
+ }
+
+ expected = {
+ "hBaseInstalled": False,
+ "components": components,
+ "cpu": 2,
+ "disk": 2,
+ "ram": 19,
+ "reservedRam": 4,
+ "hbaseRam": 4,
+ "minContainerSize": 1024,
+ "minContainerRam" : 1024,
+ "totalAvailableRam": 15360,
+ "containers": 4,
+ "ramPerContainer": 3072,
+ "mapMemory": 3072,
+ "reduceMemory": 3072,
+ "amMemory": 3072,
+ "referenceHost": hosts["items"][0]["Hosts"],
+ "referenceNodeManagerHost": hosts["items"][0]["Hosts"]
+ }
+
+ # Cluster create call
+ result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services)
+ self.assertEquals(result, expected)
+
+ # Validate configuration call - pick user specified value which is same as what was set
+ services = {"services":
+ [{"StackServices":
+ {"service_name": "YARN",
+ "service_version": "2.6.0.2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "custom_commands": [
+
+ ],
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "service_name": "YARN",
+ "stack_name": "HDP",
+ "stack_version": "2.2",
+ "hostnames": [
+ "host1"
+ ]
+ },
+ "dependencies": [
+ ]
+ }
+ ],
+ }],
+ "configurations": {
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "1024",
+ "yarn.scheduler.maximum-allocation-mb": "2048"
+ }
+ }
+ },
+ "changed-configurations": [],
+ "context": {'call_type': 'validateConfigurations'}
+ }
+ result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services)
+ self.assertEquals(result, expected)
+
+ # Validate configuration call - pick user specified value
+ services = {"services":
+ [{"StackServices":
+ {"service_name": "YARN",
+ "service_version": "2.6.0.2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "custom_commands": [
+
+ ],
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "service_name": "YARN",
+ "stack_name": "HDP",
+ "stack_version": "2.2",
+ "hostnames": [
+ "host1"
+ ]
+ },
+ "dependencies": [
+ ]
+ }
+ ],
+ }],
+ "configurations": {
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "2048",
+ "yarn.scheduler.maximum-allocation-mb": "12288"
+ }
+ }
+ },
+ "changed-configurations": [],
+ "context": {'call_type': 'validateConfigurations'}
+ }
+
+ expected_2048 = {
+ "hBaseInstalled": False,
+ "components": components,
+ "cpu": 2,
+ "disk": 2,
+ "ram": 19,
+ "reservedRam": 4,
+ "hbaseRam": 4,
+ "minContainerSize": 2048,
+ "minContainerRam" : 2048,
+ "totalAvailableRam": 15360,
+ "containers": 4,
+ "ramPerContainer": 2048,
+ "mapMemory": 2048,
+ "reduceMemory": 2048,
+ "amMemory": 2048,
+ "referenceHost": hosts["items"][0]["Hosts"],
+ "referenceNodeManagerHost": hosts["items"][0]["Hosts"]
+ }
+ result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services)
+ self.assertEquals(result, expected_2048)
+
+ # Recommend config dependencies call - pick user specified value
+ services = {"services":
+ [{"StackServices":
+ {"service_name": "YARN",
+ "service_version": "2.6.0.2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "custom_commands": [
+
+ ],
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "service_name": "YARN",
+ "stack_name": "HDP",
+ "stack_version": "2.2",
+ "hostnames": [
+ "host1"
+ ]
+ },
+ "dependencies": [
+ ]
+ }
+ ],
+ }],
+ "configurations": {
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "2048",
+ "yarn.scheduler.maximum-allocation-mb": "12288"
+ }
+ }
+ },
+ "changed-configurations": [],
+ "context": {'call_type': 'recommendConfigurationDependencies'}
+ }
+
+ expected_2048 = {
+ "hBaseInstalled": False,
+ "components": components,
+ "cpu": 2,
+ "disk": 2,
+ "ram": 19,
+ "reservedRam": 4,
+ "hbaseRam": 4,
+ "minContainerSize": 2048,
+ "minContainerRam" : 2048,
+ "totalAvailableRam": 15360,
+ "containers": 4,
+ "ramPerContainer": 2048,
+ "mapMemory": 2048,
+ "reduceMemory": 2048,
+ "amMemory": 2048,
+ "referenceHost": hosts["items"][0]["Hosts"],
+ "referenceNodeManagerHost": hosts["items"][0]["Hosts"]
+ }
+ result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services)
+ self.assertEquals(result, expected_2048)
+
+ # Recommend config dependencies call - pick user specified value of 4096 for min
+ services = {"services":
+ [{"StackServices":
+ {"service_name": "YARN",
+ "service_version": "2.6.0.2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "custom_commands": [
+
+ ],
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "service_name": "YARN",
+ "stack_name": "HDP",
+ "stack_version": "2.2",
+ "hostnames": [
+ "host1"
+ ]
+ },
+ "dependencies": [
+ ]
+ }
+ ],
+ }],
+ "configurations": {
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "4096",
+ "yarn.scheduler.maximum-allocation-mb": "12288"
+ }
+ }
+ },
+ "changed-configurations": [],
+ "context": {'call_type': 'recommendConfigurationDependencies'}
+ }
+
+ expected_4096 = {
+ "hBaseInstalled": False,
+ "components": components,
+ "cpu": 2,
+ "disk": 2,
+ "ram": 19,
+ "reservedRam": 4,
+ "hbaseRam": 4,
+ "minContainerSize": 4096,
+ "minContainerRam" : 4096,
+ "totalAvailableRam": 15360,
+ "containers": 3,
+ "ramPerContainer": 4096,
+ "mapMemory": 4096,
+ "reduceMemory": 4096,
+ "amMemory": 4096,
+ "referenceHost": hosts["items"][0]["Hosts"],
+ "referenceNodeManagerHost": hosts["items"][0]["Hosts"]
+ }
+ result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services)
+ self.assertEquals(result, expected_4096)
+
+
+
def test_getConfigurationClusterSummary_withHBaseAnd6gbRam(self):
servicesList = ["HBASE"]
components = []
@@ -419,6 +761,7 @@ class TestHDP206StackAdvisor(TestCase):
"reservedRam": 2,
"hbaseRam": 1,
"minContainerSize": 512,
+ "minContainerRam" : 512,
"totalAvailableRam": 3072,
"containers": 6,
"ramPerContainer": 512,
@@ -487,15 +830,16 @@ class TestHDP206StackAdvisor(TestCase):
})
expected["referenceHost"] = hosts["items"][1]["Hosts"]
expected["referenceNodeManagerHost"] = hosts["items"][1]["Hosts"]
- expected["amMemory"] = 170.66666666666666
- expected["containers"] = 3.0
+ expected["amMemory"] = 128
+ expected["containers"] = 4
expected["cpu"] = 4
expected["totalAvailableRam"] = 512
- expected["mapMemory"] = 170
- expected["minContainerSize"] = 256
- expected["reduceMemory"] = 170.66666666666666
+ expected["mapMemory"] = 128
+ expected["minContainerSize"] = 128
+ expected["reduceMemory"] = 128
+ expected["minContainerRam"] = 128
expected["ram"] = 0
- expected["ramPerContainer"] = 170.66666666666666
+ expected["ramPerContainer"] = 128
expected["reservedRam"] = 1
result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services)
self.assertEquals(result, expected)
@@ -537,6 +881,7 @@ class TestHDP206StackAdvisor(TestCase):
"mapMemory": 3072,
"reduceMemory": 3072,
"amMemory": 3072,
+ "minContainerRam": 1024,
"referenceHost": hosts["items"][0]["Hosts"]
}
@@ -592,7 +937,8 @@ class TestHDP206StackAdvisor(TestCase):
services = {"configurations": configurations, "services": []}
clusterData = {
"containers" : 5,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
"yarn-env": {
@@ -619,7 +965,8 @@ class TestHDP206StackAdvisor(TestCase):
clusterData = {
"mapMemory": 567,
"reduceMemory": 345.6666666666666,
- "amMemory": 123.54
+ "amMemory": 123.54,
+ "minContainerRam": 123.54
}
expected = {
"mapred-site": {
@@ -655,13 +1002,14 @@ class TestHDP206StackAdvisor(TestCase):
"ram": 0,
"reservedRam": 1,
"hbaseRam": 1,
- "minContainerSize": 256,
+ "minContainerSize": 128,
"totalAvailableRam": 512,
"containers": 3,
- "ramPerContainer": 170.66666666666666,
+ "ramPerContainer": 170,
"mapMemory": 170,
- "reduceMemory": 170.66666666666666,
- "amMemory": 170.66666666666666
+ "reduceMemory": 170,
+ "amMemory": 170,
+ "minContainerRam" : 170
}
self.assertEquals(result, expected)
@@ -1171,7 +1519,8 @@ class TestHDP206StackAdvisor(TestCase):
}
clusterData = {
- "totalAvailableRam": 2048
+ "totalAvailableRam": 2048,
+ "totalAvailableRam": 256
}
ambariHostName = socket.getfqdn()
expected = {'oozie-env':
@@ -1210,7 +1559,7 @@ class TestHDP206StackAdvisor(TestCase):
'namenode_heapsize': '1024',
'proxyuser_group': 'users',
'namenode_opt_maxnewsize': '256',
- 'namenode_opt_newsize': '256'}}}
+ 'namenode_opt_newsize': '128'}}}
# Apart from testing other HDFS recommendations, also tests 'hadoop.proxyuser.hive.hosts' config value which includes both HiveServer
# and Hive Server Interactive Host (installed on different host compared to HiveServer).
@@ -1345,7 +1694,7 @@ class TestHDP206StackAdvisor(TestCase):
'namenode_heapsize': '1024',
'proxyuser_group': 'users',
'namenode_opt_maxnewsize': '256',
- 'namenode_opt_newsize': '256'}}}
+ 'namenode_opt_newsize': '128'}}}
# Apart from testing other HDFS recommendations, also tests 'hadoop.proxyuser.hive.hosts' config value which includes both HiveServer
# and Hive Server Interactive Host (installed on same host compared to HiveServer).
@@ -1472,7 +1821,7 @@ class TestHDP206StackAdvisor(TestCase):
'namenode_heapsize': '1024',
'proxyuser_group': 'users',
'namenode_opt_maxnewsize': '256',
- 'namenode_opt_newsize': '256'}}}
+ 'namenode_opt_newsize': '128'}}}
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services3, hosts)
self.assertEquals(configurations, expected)
http://git-wip-us.apache.org/repos/asf/ambari/blob/95aa867b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 02be812..8f44e16 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -86,10 +86,11 @@ class TestHDP22StackAdvisor(TestCase):
}
clusterData = {
"mapMemory": 3000,
- "amMemory": 2000,
+ "amMemory": 2048,
"reduceMemory": 2056,
"containers": 3,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
"tez-site": {
@@ -183,6 +184,7 @@ class TestHDP22StackAdvisor(TestCase):
"amMemory": 3100,
"reduceMemory": 2056,
"containers": 3,
+ "minContainerRam": 256,
"ramPerContainer": 256
}
expected = {
@@ -265,10 +267,11 @@ class TestHDP22StackAdvisor(TestCase):
}
clusterData = {
"mapMemory": 760,
- "amMemory": 2000,
+ "amMemory": 2048,
"reduceMemory": 760,
"containers": 3,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
"tez-site": {
@@ -884,7 +887,8 @@ class TestHDP22StackAdvisor(TestCase):
clusterData = {
"cpu": 4,
"containers" : 5,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
"yarn-env": {
@@ -923,7 +927,8 @@ class TestHDP22StackAdvisor(TestCase):
clusterData = {
"cpu": 4,
"containers": 5,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
"spark-defaults": {
@@ -955,7 +960,8 @@ class TestHDP22StackAdvisor(TestCase):
clusterData = {
"cpu": 4,
"containers" : 5,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
"yarn-env": {
@@ -1216,7 +1222,8 @@ class TestHDP22StackAdvisor(TestCase):
clusterData = {
"cpu": 4,
"containers" : 5,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
services = {
@@ -1263,7 +1270,8 @@ class TestHDP22StackAdvisor(TestCase):
"amMemory": 2000,
"reduceMemory": 2056,
"containers": 3,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
@@ -1754,6 +1762,7 @@ class TestHDP22StackAdvisor(TestCase):
"containers" : 7,
"ramPerContainer": 256,
"totalAvailableRam": 4096,
+ "minContainerRam": 256
}
expected = {
"cluster-env": {
@@ -1772,9 +1781,9 @@ class TestHDP22StackAdvisor(TestCase):
'mapreduce.job.queuename': 'default',
"mapreduce.map.memory.mb": "1536",
"mapreduce.reduce.memory.mb": "1536",
- "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx204m -Dhdp.version=${hdp.version}",
"mapreduce.reduce.java.opts": "-Xmx1228m",
- "yarn.app.mapreduce.am.resource.mb": "100",
+ "yarn.app.mapreduce.am.resource.mb": "256",
"mapreduce.map.java.opts": "-Xmx1228m",
"mapreduce.task.io.sort.mb": "859"
},
@@ -2003,7 +2012,7 @@ class TestHDP22StackAdvisor(TestCase):
"mapreduce.map.memory.mb": "1024",
"mapreduce.reduce.memory.mb": "682",
"yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}",
- "mapreduce.reduce.java.opts": "-Xmx546m",
+ "mapreduce.reduce.java.opts": "-Xmx560m",
"yarn.app.mapreduce.am.resource.mb": "682",
"mapreduce.map.java.opts": "-Xmx546m",
"mapreduce.task.io.sort.mb": "273"
@@ -2021,7 +2030,8 @@ class TestHDP22StackAdvisor(TestCase):
clusterData = {
"cpu": 4,
"containers" : 5,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
"yarn-env": {
@@ -2033,13 +2043,13 @@ class TestHDP22StackAdvisor(TestCase):
"mapred-site": {
"properties": {
'mapreduce.job.queuename': 'default',
- "mapreduce.map.memory.mb": "100",
- "mapreduce.reduce.memory.mb": "200",
- "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}",
- "mapreduce.reduce.java.opts": "-Xmx160m",
- "yarn.app.mapreduce.am.resource.mb": "100",
- "mapreduce.map.java.opts": "-Xmx80m",
- "mapreduce.task.io.sort.mb": "56"
+ "mapreduce.map.memory.mb": "256",
+ "mapreduce.reduce.memory.mb": "512",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx204m -Dhdp.version=${hdp.version}",
+ "mapreduce.reduce.java.opts": "-Xmx409m",
+ "yarn.app.mapreduce.am.resource.mb": "256",
+ "mapreduce.map.java.opts": "-Xmx204m",
+ "mapreduce.task.io.sort.mb": "142"
},
"property_attributes": {
'mapreduce.task.io.sort.mb': {'maximum': '2047'},
@@ -2249,9 +2259,9 @@ class TestHDP22StackAdvisor(TestCase):
"properties": {
'mapreduce.job.queuename': 'default',
"mapreduce.map.memory.mb": "700",
- "mapreduce.reduce.memory.mb": "1280",
+ "mapreduce.reduce.memory.mb": "700",
"yarn.app.mapreduce.am.command-opts": "-Xmx560m -Dhdp.version=${hdp.version}",
- "mapreduce.reduce.java.opts": "-Xmx1024m",
+ "mapreduce.reduce.java.opts": "-Xmx560m",
"yarn.app.mapreduce.am.resource.mb": "700",
"mapreduce.map.java.opts": "-Xmx560m",
"mapreduce.task.io.sort.mb": "392"
@@ -3803,7 +3813,7 @@ class TestHDP22StackAdvisor(TestCase):
"properties": {
"yarn.nodemanager.linux-container-executor.group": "hadoop",
"yarn.nodemanager.resource.memory-mb": "43008",
- "yarn.scheduler.minimum-allocation-mb": "14336",
+ "yarn.scheduler.minimum-allocation-mb": "1024",
"yarn.scheduler.maximum-allocation-vcores": "4",
"yarn.scheduler.minimum-allocation-vcores": "1",
"yarn.nodemanager.resource.cpu-vcores": "4",
@@ -3863,7 +3873,7 @@ class TestHDP22StackAdvisor(TestCase):
"properties": {
"yarn.nodemanager.linux-container-executor.group": "hadoop",
"yarn.nodemanager.resource.memory-mb": "43008",
- "yarn.scheduler.minimum-allocation-mb": "14336",
+ "yarn.scheduler.minimum-allocation-mb": "1024",
"yarn.scheduler.maximum-allocation-vcores": "4",
"yarn.scheduler.minimum-allocation-vcores": "1",
"yarn.nodemanager.resource.cpu-vcores": "4",
http://git-wip-us.apache.org/repos/asf/ambari/blob/95aa867b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 378522e..a9b3c24 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -211,6 +211,9 @@ class TestHDP23StackAdvisor(TestCase):
]
}
services = {
+ "context" : {
+ "call_type" : "recommendConfigurations"
+ },
"services" : [ {
"StackServices":{
"service_name": "YARN",
http://git-wip-us.apache.org/repos/asf/ambari/blob/95aa867b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index a7ccdfe..d4a301c 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -374,7 +374,8 @@ class TestHDP25StackAdvisor(TestCase):
clusterData = {
"cpu": 4,
"containers": 5,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
"spark2-defaults": {
@@ -801,7 +802,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 512,
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
- }
+ },
+ "minContainerRam": 512
}
@@ -969,7 +971,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 512,
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
- }
+ },
+ "minContainerRam": 512
}
configurations = {
@@ -1149,7 +1152,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 512,
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
- }
+ },
+ "minContainerRam": 512
}
@@ -1348,7 +1352,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 512,
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 2048
- }
+ },
+ "minContainerRam": 512
}
@@ -1542,7 +1547,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 1024,
"referenceNodeManagerHost" : {
"total_mem" : 51200 * 1024
- }
+ },
+ "minContainerRam": 1024
}
configurations = {
@@ -1744,7 +1750,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 2048,
"referenceNodeManagerHost" : {
"total_mem" : 40960 * 1024
- }
+ },
+ "minContainerRam": 1024
}
configurations = {
@@ -1940,7 +1947,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 341,
"referenceNodeManagerHost" : {
"total_mem" : 12288 * 1024
- }
+ },
+ "minContainerRam": 341
}
@@ -2138,7 +2146,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 2048,
"referenceNodeManagerHost" : {
"total_mem" : 204800 * 1024
- }
+ },
+ "minContainerRam": 1024
}
configurations = {
@@ -2340,7 +2349,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 3072,
"referenceNodeManagerHost" : {
"total_mem" : 40960 * 1024
- }
+ },
+ "minContainerRam": 1024
}
@@ -2538,7 +2548,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 341,
"referenceNodeManagerHost" : {
"total_mem" : 204800 * 1024
- }
+ },
+ "minContainerRam": 341
}
@@ -2735,7 +2746,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 82240,
"referenceNodeManagerHost" : {
"total_mem" : 204800 * 1024
- }
+ },
+ "minContainerRam": 1024
}
configurations = {
@@ -2927,7 +2939,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 82240,
"referenceNodeManagerHost" : {
"total_mem" : 328960 * 1024
- }
+ },
+ "minContainerRam": 1024
}
@@ -3148,7 +3161,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 512,
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
- }
+ },
+ "minContainerRam": 512
}
configurations = {
@@ -3374,7 +3388,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 512,
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
- }
+ },
+ "minContainerRam": 512
}
configurations = {
@@ -3566,7 +3581,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 82240,
"referenceNodeManagerHost" : {
"total_mem" : 328960 * 1024
- }
+ },
+ "minContainerRam": 1024
}
configurations = {
@@ -3748,7 +3764,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 82240,
"referenceNodeManagerHost" : {
"total_mem" : 328960 * 1024
- }
+ },
+ "minContainerRam": 1024
}
configurations = {
@@ -3880,7 +3897,8 @@ class TestHDP25StackAdvisor(TestCase):
"ramPerContainer": 512,
"referenceNodeManagerHost" : {
"total_mem" : 10240 * 1024
- }
+ },
+ "minContainerRam": 512
}
configurations = {
@@ -3925,7 +3943,8 @@ class TestHDP25StackAdvisor(TestCase):
"amMemory": 2000,
"reduceMemory": 2056,
"containers": 3,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "minContainerRam": 256
}
expected = {
'application-properties': {
@@ -4660,7 +4679,8 @@ class TestHDP25StackAdvisor(TestCase):
"cpu": 4,
"referenceNodeManagerHost" : {
"total_mem" : 328960 * 1024
- }
+ },
+ "minContainerRam": 256
}
hosts = {
"items" : [