You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2016/03/17 12:51:25 UTC
ambari git commit: AMBARI-15428. Fix HBase master heap
recommendations.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk df2d304b1 -> dcb4ed803
AMBARI-15428. Fix HBase master heap recommendations.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dcb4ed80
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dcb4ed80
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dcb4ed80
Branch: refs/heads/trunk
Commit: dcb4ed80392219717e11dc6e14b646810fc83f7e
Parents: df2d304
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Thu Mar 17 11:04:03 2016 +0200
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Thu Mar 17 11:04:03 2016 +0200
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 23 +++++++++++--
.../stacks/2.0.6/common/test_stack_advisor.py | 35 ++++++++++----------
.../stacks/2.1/common/test_stack_advisor.py | 33 +++++++++---------
.../stacks/2.2/common/test_stack_advisor.py | 4 +--
.../stacks/2.3/common/test_stack_advisor.py | 4 +--
5 files changed, 58 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcb4ed80/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 8c18421..54d6d6e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -281,9 +281,25 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts):
# recommendations for HBase env config
+
+ # If cluster size is < 100, hbase master heap = 2G
+ # else If cluster size is < 500, hbase master heap = 4G
+ # else hbase master heap = 8G
+ # for small test clusters use 1 gb
+ hostsCount = 0
+ if hosts and "items" in hosts:
+ hostsCount = len(hosts["items"])
+
+ hbaseMasterRam = {
+ hostsCount < 20: 1,
+ 20 <= hostsCount < 100: 2,
+ 100 <= hostsCount < 500: 4,
+ 500 <= hostsCount: 8
+ }[True]
+
putHbaseProperty = self.putProperty(configurations, "hbase-env", services)
putHbaseProperty('hbase_regionserver_heapsize', int(clusterData['hbaseRam']) * 1024)
- putHbaseProperty('hbase_master_heapsize', int(clusterData['hbaseRam']) * 1024)
+ putHbaseProperty('hbase_master_heapsize', hbaseMasterRam * 1024)
# recommendations for HBase site config
putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
@@ -782,7 +798,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
{"os":12, "hbase":16},
{"os":24, "hbase":24},
{"os":32, "hbase":32},
- {"os":64, "hbase":64}
+ {"os":64, "hbase":32}
]
index = {
cluster["ram"] <= 4: 0,
@@ -797,9 +813,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
128 < cluster["ram"] <= 256: 9,
256 < cluster["ram"]: 10
}[1]
+
+
cluster["reservedRam"] = ramRecommendations[index]["os"]
cluster["hbaseRam"] = ramRecommendations[index]["hbase"]
+
cluster["minContainerSize"] = {
cluster["ram"] <= 4: 256,
4 < cluster["ram"] <= 8: 512,
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcb4ed80/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index e15582e..d157936 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -665,23 +665,22 @@ class TestHDP206StackAdvisor(TestCase):
servicesList = ["HBASE"]
configurations = {}
components = []
+ host_item = {
+ "Hosts" : {
+ "cpu_count" : 6,
+ "total_mem" : 50331648,
+ "disk_info" : [
+ {"mountpoint" : "/"},
+ {"mountpoint" : "/dev/shm"},
+ {"mountpoint" : "/vagrant"},
+ {"mountpoint" : "/"},
+ {"mountpoint" : "/dev/shm"},
+ {"mountpoint" : "/vagrant"}
+ ]
+ }
+ }
hosts = {
- "items" : [
- {
- "Hosts" : {
- "cpu_count" : 6,
- "total_mem" : 50331648,
- "disk_info" : [
- {"mountpoint" : "/"},
- {"mountpoint" : "/dev/shm"},
- {"mountpoint" : "/vagrant"},
- {"mountpoint" : "/"},
- {"mountpoint" : "/dev/shm"},
- {"mountpoint" : "/vagrant"}
- ]
- }
- }
- ]
+ "items" : [host_item for i in range(1, 300)]
}
services = {
"services" : [
@@ -707,7 +706,7 @@ class TestHDP206StackAdvisor(TestCase):
},
"hbase-env": {
"properties": {
- "hbase_master_heapsize": "8192",
+ "hbase_master_heapsize": "4096",
"hbase_regionserver_heapsize": "8192",
}
}
@@ -716,7 +715,7 @@ class TestHDP206StackAdvisor(TestCase):
clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, None)
self.assertEquals(clusterData['hbaseRam'], 8)
- self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, None)
+ self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcb4ed80/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
index 7ede5a3..55cf39d 100644
--- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
@@ -283,23 +283,22 @@ class TestHDP21StackAdvisor(TestCase):
servicesList = ["HBASE"]
configurations = {}
components = []
+ host_item = {
+ "Hosts" : {
+ "cpu_count" : 6,
+ "total_mem" : 50331648,
+ "disk_info" : [
+ {"mountpoint" : "/"},
+ {"mountpoint" : "/dev/shm"},
+ {"mountpoint" : "/vagrant"},
+ {"mountpoint" : "/"},
+ {"mountpoint" : "/dev/shm"},
+ {"mountpoint" : "/vagrant"}
+ ]
+ }
+ }
hosts = {
- "items" : [
- {
- "Hosts" : {
- "cpu_count" : 6,
- "total_mem" : 50331648,
- "disk_info" : [
- {"mountpoint" : "/"},
- {"mountpoint" : "/dev/shm"},
- {"mountpoint" : "/vagrant"},
- {"mountpoint" : "/"},
- {"mountpoint" : "/dev/shm"},
- {"mountpoint" : "/vagrant"}
- ]
- }
- }
- ]
+ "items" : [host_item for i in range(1, 600)]
}
services = {
"services" : [
@@ -334,7 +333,7 @@ class TestHDP21StackAdvisor(TestCase):
clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, None)
self.assertEquals(clusterData['hbaseRam'], 8)
- self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, None)
+ self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
def test_recommendHDFSConfigurations(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcb4ed80/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 9070c0a..59eff4d 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2305,7 +2305,7 @@ class TestHDP22StackAdvisor(TestCase):
},
"hbase-env": {
"properties": {
- "hbase_master_heapsize": "8192",
+ "hbase_master_heapsize": "1024",
"hbase_regionserver_heapsize": "8192",
}
}
@@ -2542,7 +2542,7 @@ class TestHDP22StackAdvisor(TestCase):
},
"hbase-env": {
"properties": {
- "hbase_master_heapsize": "8192",
+ "hbase_master_heapsize": "1024",
"hbase_regionserver_heapsize": "8192",
},
"property_attributes": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcb4ed80/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 1af32d1..7a2a37d 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -857,7 +857,7 @@ class TestHDP23StackAdvisor(TestCase):
},
"hbase-env": {
"properties": {
- "hbase_master_heapsize": "114688",
+ "hbase_master_heapsize": "1024",
"hbase_max_direct_memory_size": "94208",
"hbase_regionserver_heapsize": "20480"
}
@@ -987,7 +987,7 @@ class TestHDP23StackAdvisor(TestCase):
expected["hbase-site"]["property_attributes"]["hbase.bucketcache.ioengine"] = {"delete": "true"}
expected["hbase-site"]["property_attributes"]["hbase.bucketcache.percentage.in.combinedcache"] = {"delete": "true"}
expected["hbase-env"]["property_attributes"] = {"hbase_max_direct_memory_size" : {"delete": "true"}}
- expected["hbase-env"]["properties"]["hbase_master_heapsize"] = "4096"
+ expected["hbase-env"]["properties"]["hbase_master_heapsize"] = "1024"
expected["hbase-env"]["properties"]["hbase_regionserver_heapsize"] = "4096"
self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None)
self.assertEquals(configurations, expected)