You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/05/12 11:12:21 UTC
ambari git commit: AMBARI-11060 Pig-HA tests fails with
java.lang.OutOfMemoryError: Java heap space (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 7aecacbf7 -> c3690fecf
AMBARI-11060 Pig-HA tests fails with java.lang.OutOfMemoryError: Java heap space (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c3690fec
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c3690fec
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c3690fec
Branch: refs/heads/trunk
Commit: c3690fecf23eb787901a70f077add6b1df54fd5b
Parents: 7aecacb
Author: Dmytro Sen <ds...@apache.org>
Authored: Tue May 12 12:12:08 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Tue May 12 12:12:08 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.2/services/stack_advisor.py | 10 +-
.../stacks/2.2/common/test_stack_advisor.py | 258 +++++++++++++++++++
2 files changed, 266 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c3690fec/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 24248f2..a7aa973 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -576,10 +576,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
nodemanagerMinRam = min(nodemanager["Hosts"]["total_mem"]/1024, nodemanagerMinRam)
putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"]))) + "m" + " -Dhdp.version=${hdp.version}")
- putMapredProperty('mapreduce.map.memory.mb', int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
putMapredProperty('mapreduce.reduce.memory.mb', min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam)))
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ min_mapreduce_map_memory_mb = 0
+ min_mapreduce_map_java_opts = 0
+ if ("PIG" in servicesList):
+ min_mapreduce_map_memory_mb = 1500
+ min_mapreduce_map_java_opts = 1024
+ putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
mapredMapXmx = int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]));
- putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(mapredMapXmx) + "m")
+ putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(max(min_mapreduce_map_java_opts, mapredMapXmx)) + "m")
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m")
putMapredProperty('mapreduce.task.io.sort.mb', str(min(int(0.7*mapredMapXmx), 2047)))
# Property Attributes
http://git-wip-us.apache.org/repos/asf/ambari/blob/c3690fec/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 51574f3..05bded6 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1144,6 +1144,264 @@ class TestHDP22StackAdvisor(TestCase):
self.assertEquals(configurations, expected)
+ def test_recommendMapredConfigurationAttributesWithPigService(self):
+ configurations = {
+ "mapred-site": {
+ "properties": {
+ "mapreduce.map.memory.mb": "1024",
+ "mapreduce.reduce.memory.mb": "682",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}",
+ "mapreduce.reduce.java.opts": "-Xmx546m",
+ "yarn.app.mapreduce.am.resource.mb": "682",
+ "mapreduce.map.java.opts": "-Xmx546m",
+ "mapreduce.task.io.sort.mb": "273"
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.nodemanager.resource.memory-mb": "1280",
+ "yarn.scheduler.minimum-allocation-mb": "100",
+ "yarn.scheduler.maximum-allocation-mb": "1280",
+ "yarn.nodemanager.resource.cpu-vcores": "2"
+ },
+ }
+ }
+ clusterData = {
+ "cpu": 4,
+ "containers" : 5,
+ "ramPerContainer": 256
+ }
+ expected = {
+ "yarn-env": {
+ "properties": {
+ "min_user_id": "500"
+ }
+ },
+ "mapred-site": {
+ "properties": {
+ "mapreduce.map.memory.mb": "1500",
+ "mapreduce.reduce.memory.mb": "200",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}",
+ "mapreduce.reduce.java.opts": "-Xmx160m",
+ "yarn.app.mapreduce.am.resource.mb": "100",
+ "mapreduce.map.java.opts": "-Xmx1200m",
+ "mapreduce.task.io.sort.mb": "840"
+ },
+ "property_attributes": {
+ 'mapreduce.task.io.sort.mb': {'maximum': '2047'},
+ 'yarn.app.mapreduce.am.resource.mb': {'maximum': '1280',
+ 'minimum': '100'},
+ 'mapreduce.map.memory.mb': {'maximum': '1280',
+ 'minimum': '100'},
+ 'mapreduce.reduce.memory.mb': {'maximum': '1280',
+ 'minimum': '100'}
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.nodemanager.resource.memory-mb": "1280",
+ "yarn.scheduler.minimum-allocation-mb": "100",
+ "yarn.scheduler.maximum-allocation-vcores": "1",
+ "yarn.scheduler.minimum-allocation-vcores": "1",
+ "yarn.scheduler.maximum-allocation-mb": "1280",
+ "yarn.nodemanager.resource.cpu-vcores": "1"
+ },
+ "property_attributes": {
+ 'yarn.nodemanager.resource.memory-mb': {'maximum': '1877'},
+ 'yarn.nodemanager.resource.cpu-vcores': {'maximum': '2'},
+ 'yarn.scheduler.minimum-allocation-vcores': {'maximum': '1'},
+ 'yarn.scheduler.maximum-allocation-vcores': {'maximum': '1'},
+ 'yarn.scheduler.minimum-allocation-mb': {'maximum': '1280'},
+ 'yarn.scheduler.maximum-allocation-mb': {'maximum': '1280'}
+ }
+ }
+ }
+ services = {
+ "services": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/PIG",
+ "StackServices": {
+ "service_name": "PIG",
+ "service_version": "2.6.0.2.2",
+ "stack_name": "HDP",
+ "stack_version": "2.2"
+ }, "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "0+",
+ "component_category": "CLIENT",
+ "component_name": "PIG",
+ "display_name": "Pig",
+ "is_client": "true",
+ "is_master": "false",
+ "hostnames": []
+ },
+ "dependencies": []
+ }
+ ]
+ },
+ {
+ "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2",
+ "StackServices" : {
+ "service_name" : "MAPREDUCE2",
+ "service_version" : "2.6.0.2.2",
+ "stack_name" : "HDP",
+ "stack_version" : "2.2"
+ },
+ "components" : [ {
+ "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER",
+ "StackServiceComponents" : {
+ "advertise_version" : "true",
+ "cardinality" : "1",
+ "component_category" : "MASTER",
+ "component_name" : "HISTORYSERVER",
+ "custom_commands" : [ ],
+ "display_name" : "History Server",
+ "is_client" : "false",
+ "is_master" : "true",
+ "service_name" : "MAPREDUCE2",
+ "stack_name" : "HDP",
+ "stack_version" : "2.2",
+ "hostnames" : [ "c6402.ambari.apache.org" ]
+ },
+ "auto_deploy" : {
+ "enabled" : "true",
+ "location" : "YARN/RESOURCEMANAGER"
+ },
+ "dependencies" : [ {
+ "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER/dependencies/HDFS_CLIENT",
+ "Dependencies" : {
+ "component_name" : "HDFS_CLIENT",
+ "dependent_component_name" : "HISTORYSERVER",
+ "dependent_service_name" : "MAPREDUCE2",
+ "stack_name" : "HDP",
+ "stack_version" : "2.2"
+ }
+ } ]
+ }]},
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
+ "StackServices": {
+ "service_name": "YARN",
+ "service_version": "2.6.0.2.2",
+ "stack_name": "HDP",
+ "stack_version": "2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1",
+ "component_category": "MASTER",
+ "component_name": "APP_TIMELINE_SERVER",
+ "display_name": "App Timeline Server",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "hostnames": [
+ "c6403.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1-2",
+ "component_category": "MASTER",
+ "component_name": "RESOURCEMANAGER",
+ "display_name": "ResourceManager",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "CLIENT",
+ "component_name": "YARN_CLIENT",
+ "display_name": "YARN Client",
+ "is_client": "true",
+ "is_master": "false",
+ "hostnames": []
+ },
+ "dependencies": []
+ }
+ ]
+ },
+ ],
+ "configurations": configurations,
+ "changed-configurations": [
+ {
+ "type": "yarn-site",
+ "name": "yarn.scheduler.minimum-allocation-mb"
+ },
+ ]
+
+ }
+ hosts = {
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6401.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6402.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6402.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6402.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6403.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6403.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6403.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ }
+ ]
+ }
+
+ self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
def test_recommendMapredConfigurationAttributes(self):
configurations = {
"mapred-site": {