You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/04/03 18:03:15 UTC
ambari git commit: AMBARI-10351 BE: YARN stack advisor not
recommending all dependent configs (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk e44c85617 -> b3fc39395
AMBARI-10351 BE: YARN stack advisor not recommending all dependent configs (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b3fc3939
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b3fc3939
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b3fc3939
Branch: refs/heads/trunk
Commit: b3fc39395d82b8b37cb2d2e5b9b8e3edec3cd0bf
Parents: e44c856
Author: Dmytro Sen <ds...@apache.org>
Authored: Fri Apr 3 19:02:58 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Fri Apr 3 19:02:58 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 2 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 18 +-
.../stacks/2.2/common/test_stack_advisor.py | 231 +++++++++++++++++++
ambari-web/pom.xml | 1 +
4 files changed, 242 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3fc3939/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 2405954..b2610ad 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -125,7 +125,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services)
putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer'])))
putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer']))
- putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer'])))
+ putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
putYarnEnvProperty('min_user_id', self.get_system_min_uid())
def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3fc3939/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 87748d2..3a42a3d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -169,17 +169,17 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):
self.recommendYARNConfigurations(configurations, clusterData, services, hosts)
putMapredProperty = self.putProperty(configurations, "mapred-site", services)
- putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory']))
- putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m" + " -Dhdp.version=${hdp.version}")
- putMapredProperty('mapreduce.map.memory.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
- putMapredProperty('mapreduce.reduce.memory.mb', int(clusterData['reduceMemory']))
- putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m")
- putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m")
- putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
+ putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
+ putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"]))) + "m" + " -Dhdp.version=${hdp.version}")
+ putMapredProperty('mapreduce.map.memory.mb', int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
+ putMapredProperty('mapreduce.reduce.memory.mb', int(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
+ putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]))) + "m")
+ putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m")
+ putMapredProperty('mapreduce.task.io.sort.mb', str(int(0.7*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]))))
# Property Attributes
putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site")
- yarnMinAllocationSize = int(clusterData['ramPerContainer'])
- yarnMaxAllocationSize = max(30 * int(clusterData['ramPerContainer']), configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])
+ yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
+ yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
putMapredPropertyAttribute("mapreduce.map.memory.mb", "max", yarnMaxAllocationSize)
putMapredPropertyAttribute("mapreduce.map.memory.mb", "min", yarnMinAllocationSize)
putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "max", yarnMaxAllocationSize)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3fc3939/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index d12c067..0cfc3b6 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -828,6 +828,237 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+ def test_recommendMapredConfigurationAttributes(self):
+ configurations = {
+ "mapred-site": {
+ "properties": {
+ "mapreduce.map.memory.mb": "1024",
+ "mapreduce.reduce.memory.mb": "682",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}",
+ "mapreduce.reduce.java.opts": "-Xmx546m",
+ "yarn.app.mapreduce.am.resource.mb": "682",
+ "mapreduce.map.java.opts": "-Xmx546m",
+ "mapreduce.task.io.sort.mb": "273" }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.nodemanager.resource.memory-mb": "1280",
+ "yarn.scheduler.minimum-allocation-mb": "100",
+ "yarn.scheduler.maximum-allocation-mb": "1280",
+ "yarn.nodemanager.resource.cpu-vcores": "2"
+ },
+ }
+ }
+ clusterData = {
+ "cpu": 4,
+ "containers" : 5,
+ "ramPerContainer": 256
+ }
+ expected = {
+ "yarn-env": {
+ "properties": {
+ "min_user_id": "500"
+ }
+ },
+ "mapred-site": {
+ "properties": {
+ "mapreduce.map.memory.mb": "100",
+ "mapreduce.reduce.memory.mb": "200",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}",
+ "mapreduce.reduce.java.opts": "-Xmx160m",
+ "yarn.app.mapreduce.am.resource.mb": "100",
+ "mapreduce.map.java.opts": "-Xmx80m",
+ "mapreduce.task.io.sort.mb": "70"
+ },
+ "property_attributes": {
+ 'yarn.app.mapreduce.am.resource.mb': {'max': '1280',
+ 'min': '100'},
+ 'mapreduce.map.memory.mb': {'max': '1280',
+ 'min': '100'},
+ 'mapreduce.reduce.memory.mb': {'max': '1280',
+ 'min': '100'}
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.nodemanager.resource.memory-mb": "1280",
+ "yarn.scheduler.minimum-allocation-mb": "100",
+ "yarn.scheduler.maximum-allocation-mb": "1280",
+ "yarn.nodemanager.resource.cpu-vcores": "2"
+ },
+ "property_attributes": {
+ 'yarn.nodemanager.resource.memory-mb': {'max': '1877'},
+ 'yarn.nodemanager.resource.cpu-vcores': {'max': '4'},
+ 'yarn.scheduler.minimum-allocation-vcores': {'max': '2'},
+ 'yarn.scheduler.maximum-allocation-vcores': {'max': '2'},
+ 'yarn.scheduler.minimum-allocation-mb': {'max': '1280'},
+ 'yarn.scheduler.maximum-allocation-mb': {'max': '1280'}
+ }
+ }
+ }
+ services = {
+ "services": [
+ {
+ "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2",
+ "StackServices" : {
+ "service_name" : "MAPREDUCE2",
+ "service_version" : "2.6.0.2.2",
+ "stack_name" : "HDP",
+ "stack_version" : "2.2"
+ },
+ "components" : [ {
+ "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER",
+ "StackServiceComponents" : {
+ "advertise_version" : "true",
+ "cardinality" : "1",
+ "component_category" : "MASTER",
+ "component_name" : "HISTORYSERVER",
+ "custom_commands" : [ ],
+ "display_name" : "History Server",
+ "is_client" : "false",
+ "is_master" : "true",
+ "service_name" : "MAPREDUCE2",
+ "stack_name" : "HDP",
+ "stack_version" : "2.2",
+ "hostnames" : [ "c6402.ambari.apache.org" ]
+ },
+ "auto_deploy" : {
+ "enabled" : "true",
+ "location" : "YARN/RESOURCEMANAGER"
+ },
+ "dependencies" : [ {
+ "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER/dependencies/HDFS_CLIENT",
+ "Dependencies" : {
+ "component_name" : "HDFS_CLIENT",
+ "dependent_component_name" : "HISTORYSERVER",
+ "dependent_service_name" : "MAPREDUCE2",
+ "stack_name" : "HDP",
+ "stack_version" : "2.2"
+ }
+ } ]
+ }]},
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
+ "StackServices": {
+ "service_name": "YARN",
+ "service_version": "2.6.0.2.2",
+ "stack_name": "HDP",
+ "stack_version": "2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1",
+ "component_category": "MASTER",
+ "component_name": "APP_TIMELINE_SERVER",
+ "display_name": "App Timeline Server",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "hostnames": [
+ "c6403.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1-2",
+ "component_category": "MASTER",
+ "component_name": "RESOURCEMANAGER",
+ "display_name": "ResourceManager",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "CLIENT",
+ "component_name": "YARN_CLIENT",
+ "display_name": "YARN Client",
+ "is_client": "true",
+ "is_master": "false",
+ "hostnames": []
+ },
+ "dependencies": []
+ }
+ ]
+ },
+ ],
+ "configurations": configurations,
+ "changed-configurations": [
+ {
+ "type": "yarn-site",
+ "name": "yarn.scheduler.minimum-allocation-mb"
+ },
+ ]
+
+ }
+ hosts = {
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6401.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6402.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6402.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6402.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6403.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6403.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6403.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ }
+ ]
+ }
+ self.maxDiff = None
+ self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
def test_recommendAmsConfigurations(self):
configurations = {}
clusterData = {}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3fc3939/ambari-web/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-web/pom.xml b/ambari-web/pom.xml
index 04f9b5d..aefeff1 100644
--- a/ambari-web/pom.xml
+++ b/ambari-web/pom.xml
@@ -227,6 +227,7 @@
<exclude>app/assets/**</exclude>
<exclude>vendor/**</exclude>
<exclude>node_modules/**</exclude>
+ <exclude>node/**</exclude>
<exclude>npm-debug.log</exclude>
</excludes>
</configuration>