You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2017/02/23 13:53:40 UTC
ambari git commit: AMBARI-20085 Confusing AMS collector heap size
validation loop (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 986e7a9be -> bb7b83f2b
AMBARI-20085 Confusing AMS collector heap size validation loop (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bb7b83f2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bb7b83f2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bb7b83f2
Branch: refs/heads/trunk
Commit: bb7b83f2bb1b6af2dad38e98ffdcddda13e1cb2b
Parents: 986e7a9
Author: Dmytro Sen <ds...@apache.org>
Authored: Thu Feb 23 15:52:02 2017 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Thu Feb 23 15:52:21 2017 +0200
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 1 +
.../stacks/HDP/2.5/services/stack_advisor.py | 8 ++++++++
.../src/main/resources/stacks/stack_advisor.py | 12 ++++++------
.../python/stacks/2.5/common/test_stack_advisor.py | 16 ++++++++++++++++
.../src/test/resources/stacks/old_stack_advisor.py | 12 ++++++------
.../stacks/HDF/2.0/services/stack_advisor.py | 2 +-
6 files changed, 38 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 5c68f15..61960bb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -1150,6 +1150,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if unusedMemory > 4*gb and collector_needs_increase: # warn user, if more than 4GB RAM is unused
recommended_collector_heapsize = int((unusedMemory - 4*gb)/5) + collector_heapsize * mb
+ recommended_collector_heapsize = min(16*gb, recommended_collector_heapsize) #Make sure heapsize <= 16GB
recommended_collector_heapsize = round_to_n(recommended_collector_heapsize/mb,128) # Round to 128m multiple
if collector_heapsize < recommended_collector_heapsize:
validation_msg = "Consider allocating {0} MB to metrics_collector_heapsize in ams-env to use up some " \
http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 52ada52..6f3dfa7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -142,6 +142,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
return self.toConfigurationValidationProblems(validationItems, "storm-site")
+ def getCardinalitiesDict(self, hosts):
+ result = super(HDP25StackAdvisor, self).getCardinalitiesDict(hosts)
+ min_val = 1
+ if len(hosts["items"]) > 999:
+ min_val = 2
+ result['METRICS_COLLECTOR'] = {"min": min_val}
+ return result
+
def validateAtlasConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
application_properties = self.getSiteProperties(configurations, "application-properties")
validationItems = []
http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 04c6baf..0b81700 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -806,7 +806,7 @@ class DefaultStackAdvisor(StackAdvisor):
return component["StackServiceComponents"]["hostnames"]
if len(hostsList) > 1 and self.isMasterComponentWithMultipleInstances(component):
- hostsCount = self.getMinComponentCount(component)
+ hostsCount = self.getMinComponentCount(component, hosts)
if hostsCount > 1: # get first 'hostsCount' available hosts
hostsForComponent = []
hostIndex = 0
@@ -1460,13 +1460,13 @@ class DefaultStackAdvisor(StackAdvisor):
service = self.getNotValuableComponents()
return componentName in service
- def getMinComponentCount(self, component):
+ def getMinComponentCount(self, component, hosts):
componentName = self.getComponentName(component)
- return self.getComponentCardinality(componentName)["min"]
+ return self.getComponentCardinality(componentName, hosts)["min"]
# Helper dictionaries
- def getComponentCardinality(self, componentName):
- dict = self.getCardinalitiesDict()
+ def getComponentCardinality(self, componentName, hosts):
+ dict = self.getCardinalitiesDict(hosts)
if componentName in dict:
return dict[componentName]
else:
@@ -1508,7 +1508,7 @@ class DefaultStackAdvisor(StackAdvisor):
def getNotPreferableOnServerComponents(self):
return self.notPreferableOnServerComponents
- def getCardinalitiesDict(self):
+ def getCardinalitiesDict(self, hosts):
return self.cardinalitiesDict
def getComponentLayoutSchemes(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 6890ef6..cf1c0ee 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -404,6 +404,12 @@ class TestHDP25StackAdvisor(TestCase):
data = json.load(f)
return data
+ def prepareNHosts(self, host_count):
+ names = []
+ for i in range(0, host_count):
+ names.append("hostname" + str(i))
+ return self.prepareHosts(names)
+
def prepareHosts(self, hostsNames):
hosts = { "items": [] }
for hostName in hostsNames:
@@ -435,6 +441,16 @@ class TestHDP25StackAdvisor(TestCase):
def __getHosts(self, componentsList, componentName):
return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == componentName][0]
+ def test_getCardinalitiesDict(self):
+ hosts = self.prepareNHosts(5)
+ actual = self.stackAdvisor.getCardinalitiesDict(hosts)
+ expected = {'METRICS_COLLECTOR': {'min': 1}}
+ self.assertEquals(actual, expected)
+
+ hosts = self.prepareNHosts(1001)
+ actual = self.stackAdvisor.getCardinalitiesDict(hosts)
+ expected = {'METRICS_COLLECTOR': {'min': 2}}
+ self.assertEquals(actual, expected)
def test_getComponentLayoutValidations_one_hsi_host(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/test/resources/stacks/old_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/old_stack_advisor.py b/ambari-server/src/test/resources/stacks/old_stack_advisor.py
index 8a880e0..ee5674b 100644
--- a/ambari-server/src/test/resources/stacks/old_stack_advisor.py
+++ b/ambari-server/src/test/resources/stacks/old_stack_advisor.py
@@ -359,7 +359,7 @@ class DefaultStackAdvisor(StackAdvisor):
availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)]
if self.isMasterComponentWithMultipleInstances(component):
- hostsCount = self.getMinComponentCount(component)
+ hostsCount = self.getMinComponentCount(component, hosts)
if hostsCount > 1: # get first 'hostsCount' available hosts
if len(availableHosts) < hostsCount:
hostsCount = len(availableHosts)
@@ -524,13 +524,13 @@ class DefaultStackAdvisor(StackAdvisor):
service = self.getNotValuableComponents()
return componentName in service
- def getMinComponentCount(self, component):
+ def getMinComponentCount(self, component, hosts):
componentName = self.getComponentName(component)
- return self.getComponentCardinality(componentName)["min"]
+ return self.getComponentCardinality(componentName, hosts)["min"]
# Helper dictionaries
- def getComponentCardinality(self, componentName):
- return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1})
+ def getComponentCardinality(self, componentName, hosts):
+ return self.getCardinalitiesDict(hosts).get(componentName, {"min": 1, "max": 1})
def getHostForComponent(self, component, hostsList):
componentName = self.getComponentName(component)
@@ -568,7 +568,7 @@ class DefaultStackAdvisor(StackAdvisor):
def getNotPreferableOnServerComponents(self):
return []
- def getCardinalitiesDict(self):
+ def getCardinalitiesDict(self, hosts):
return {}
def getComponentLayoutSchemes(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
index 40cc847..da33b95 100644
--- a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
@@ -1615,7 +1615,7 @@ class HDF20StackAdvisor(DefaultStackAdvisor):
def getNotPreferableOnServerComponents(self):
return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'METRICS_COLLECTOR']
- def getCardinalitiesDict(self):
+ def getCardinalitiesDict(self, hosts):
return {
'ZOOKEEPER_SERVER': {"min": 3},
'METRICS_COLLECTOR': {"min": 1}