You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/09/17 18:39:47 UTC
ambari git commit: AMBARI-13105 Minor optimizations for AMS memory
settings (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk ad810ae46 -> 8ff7db805
AMBARI-13105 Minor optimizations for AMS memory settings (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8ff7db80
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8ff7db80
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8ff7db80
Branch: refs/heads/trunk
Commit: 8ff7db8058cc20fba06b3d28b63187ed87d76e65
Parents: ad810ae
Author: Dmytro Sen <ds...@apache.org>
Authored: Thu Sep 17 19:20:39 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Thu Sep 17 19:39:14 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 95 +++++---
.../stacks/2.0.6/common/test_stack_advisor.py | 221 ++++++++++++++++++-
.../stacks/2.2/common/test_stack_advisor.py | 24 +-
3 files changed, 287 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ff7db80/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 4d1ab23..b2332b0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -288,7 +288,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
total_sinks_count += schCount
collector_heapsize = int(hbase_heapsize/4 if hbase_heapsize > 2048 else 512)
- return collector_heapsize, hbase_heapsize, total_sinks_count
+ return round_to_n(collector_heapsize), round_to_n(hbase_heapsize), total_sinks_count
def recommendAmsConfigurations(self, configurations, clusterData, services, hosts):
putAmsEnvProperty = self.putProperty(configurations, "ams-env", services)
@@ -334,13 +334,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.3)
putTimelineServiceProperty("timeline.metrics.host.aggregator.ttl", 86400)
- # Embedded mode heap size : master + regionserver
- if rootDir.startswith("hdfs://"):
- putHbaseEnvProperty("hbase_master_heapsize", "512m")
- putHbaseEnvProperty("hbase_regionserver_heapsize", str(hbase_heapsize) + "m")
- else:
- putHbaseEnvProperty("hbase_master_heapsize", str(hbase_heapsize) + "m")
-
if len(amsCollectorHosts) > 1:
pass
else:
@@ -354,20 +347,28 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.25)
putAmsHbaseSiteProperty("phoenix.query.maxGlobalMemoryPercentage", 20)
putTimelineServiceProperty("phoenix.query.maxGlobalMemoryPercentage", 30)
- putAmsHbaseSiteProperty("hbase_master_xmn_size", "512m")
- putAmsHbaseSiteProperty("regionserver_xmn_size", "512m")
+ hbase_xmn_size = '512m'
elif total_sinks_count >= 500:
putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
putAmsHbaseSiteProperty("hbase.regionserver.maxlogs", 64)
putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 268435456)
- putAmsHbaseSiteProperty("hbase_master_xmn_size", "512m")
+ hbase_xmn_size = '512m'
elif total_sinks_count >= 250:
- putAmsHbaseSiteProperty("hbase_master_xmn_size", "256m")
+ hbase_xmn_size = '256m'
else:
- putAmsHbaseSiteProperty("hbase_master_xmn_size", "128m")
+ hbase_xmn_size = '128m'
pass
+ # Embedded mode heap size : master + regionserver
+ if rootDir.startswith("hdfs://"):
+ putHbaseEnvProperty("hbase_master_heapsize", "512m")
+ putHbaseEnvProperty("hbase_regionserver_heapsize", str(hbase_heapsize) + "m")
+ putHbaseEnvProperty("regionserver_xmn_size", hbase_xmn_size)
+ else:
+ putHbaseEnvProperty("hbase_master_heapsize", str(hbase_heapsize) + "m")
+ putHbaseEnvProperty("hbase_master_xmn_size", hbase_xmn_size)
+
#split points
scriptDir = os.path.dirname(os.path.abspath(__file__))
metricsDir = os.path.join(scriptDir, '../../../../common-services/AMBARI_METRICS/0.1.0/package')
@@ -630,6 +631,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
rootdir_item = None
op_mode = ams_site.get("timeline.metrics.service.operation.mode")
hbase_rootdir = properties.get("hbase.rootdir")
+ hbase_tmpdir = properties.get("hbase.tmp.dir")
if op_mode == "distributed" and not hbase_rootdir.startswith("hdfs://"):
rootdir_item = self.getWarnItem("In distributed mode hbase.rootdir should point to HDFS. Collector will operate in embedded mode otherwise.")
pass
@@ -646,25 +648,33 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
for host in hosts["items"]:
if host["Hosts"]["host_name"] == collectorHostName:
validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorEnoughDiskSpace(properties, 'hbase.rootdir', host["Hosts"], recommendedDiskSpace)}])
- validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties, 'hbase.rootdir', host["Hosts"])}])
- validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties, 'hbase.tmp.dir', host["Hosts"])}])
+ validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.rootdir', host["Hosts"])}])
+ validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.tmp.dir', host["Hosts"])}])
- dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE")
- # if METRICS_COLLECTOR is co-hosted with DATANODE
- if not hbase_rootdir.startswith("hdfs") and \
- dn_hosts and collectorHostName in dn_hosts:
- # cross-check dfs.datanode.data.dir and hbase.rootdir
- # they shouldn't share same disk partition IO
- hdfs_site = getSiteProperties(configurations, "hdfs-site")
+ if not hbase_rootdir.startswith("hdfs"):
mountPoints = []
for mountPoint in host["Hosts"]["disk_info"]:
mountPoints.append(mountPoint["mountpoint"])
hbase_rootdir_mountpoint = getMountPointForDir(hbase_rootdir, mountPoints)
- if ams_site and hdfs_site and "dfs.datanode.data.dir" in hdfs_site:
- for dfs_datadir in hdfs_site.get("dfs.datanode.data.dir").split(","):
- mountPoints = []
- for mountPoint in host["Hosts"]["disk_info"]:
- mountPoints.append(mountPoint["mountpoint"])
+ hbase_tmpdir_mountpoint = getMountPointForDir(hbase_tmpdir, mountPoints)
+ preferred_mountpoints = self.getPreferredMountPoints(host['Hosts'])
+ # hbase.rootdir and hbase.tmp.dir shouldn't point to the same partition
+ # if multiple preferred_mountpoints exist
+ if hbase_rootdir_mountpoint == hbase_tmpdir_mountpoint and \
+ len(preferred_mountpoints) > 1:
+ item = self.getWarnItem("Consider not using {0} partition for storing metrics temporary data. "
+ "{0} partition is already used as hbase.rootdir to store metrics data".format(hbase_tmpdir_mountpoint))
+ validationItems.extend([{"config-name":'hbase.tmp.dir', "item": item}])
+
+ # if METRICS_COLLECTOR is co-hosted with DATANODE
+ # cross-check dfs.datanode.data.dir and hbase.rootdir
+ # they shouldn't share same disk partition IO
+ dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE")
+ hdfs_site = getSiteProperties(configurations, "hdfs-site")
+ dfs_datadirs = hdfs_site.get("dfs.datanode.data.dir").split(",") if hdfs_site and "dfs.datanode.data.dir" in hdfs_site else []
+ if dn_hosts and collectorHostName in dn_hosts and ams_site and \
+ dfs_datadirs and len(preferred_mountpoints) > len(dfs_datadirs):
+ for dfs_datadir in dfs_datadirs:
dfs_datadir_mountpoint = getMountPointForDir(dfs_datadir, mountPoints)
if dfs_datadir_mountpoint == hbase_rootdir_mountpoint:
item = self.getWarnItem("Consider not using {0} partition for storing metrics data. "
@@ -725,14 +735,26 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
unusedMemory = host["Hosts"]["total_mem"] * 1024 - requiredMemory # in bytes
hbase_rootdir = amsHbaseSite.get("hbase.rootdir")
if unusedMemory > 4294967296: # warn user, if more than 4GB RAM is unused
- propertyToIncrease = "hbase_regionserver_heapsize" if hbase_rootdir.startswith("hdfs://") else "hbase_master_heapsize"
+ heapPropertyToIncrease = "hbase_regionserver_heapsize" if hbase_rootdir.startswith("hdfs://") else "hbase_master_heapsize"
+ xmnPropertyToIncrease = "regionserver_xmn_size" if hbase_rootdir.startswith("hdfs://") else "hbase_master_xmn_size"
collector_heapsize = int((unusedMemory - 4294967296)/5) + to_number(ams_env.get("metrics_collector_heapsize"))*1048576
- hbase_heapsize = int((unusedMemory - 4294967296)*4/5) + to_number(properties.get(propertyToIncrease))*1048576
+ hbase_heapsize = int((unusedMemory - 4294967296)*4/5) + to_number(properties.get(heapPropertyToIncrease))*1048576
+
+ if hbase_heapsize/1048576 > 2048:
+ xmn_size = '512'
+ elif hbase_heapsize/1048576 > 1024:
+ xmn_size = '256'
+ else:
+ xmn_size = '128'
+
msg = "{0} MB RAM is unused on the host {1} based on components " \
- "assigned. Consider allocating {2} MB to metrics_collector_heapsize in ams-env and {3} MB to {4} in ams-hbase-env"
+ "assigned. Consider allocating {2} MB to " \
+ "metrics_collector_heapsize in ams-env, " \
+ "{3} MB to {4} in ams-hbase-env and " \
+ "{5} MB to {6} in ams-hbase-env"
- unusedMemoryHbaseItem = self.getWarnItem(msg.format(unusedMemory/1048576, collectorHostName, collector_heapsize/1048576, hbase_heapsize/1048576, propertyToIncrease))
- validationItems.extend([{"config-name": propertyToIncrease, "item": unusedMemoryHbaseItem}])
+ unusedMemoryHbaseItem = self.getWarnItem(msg.format(unusedMemory/1048576, collectorHostName, collector_heapsize/1048576, hbase_heapsize/1048576, heapPropertyToIncrease, xmn_size, xmnPropertyToIncrease))
+ validationItems.extend([{"config-name": heapPropertyToIncrease, "item": unusedMemoryHbaseItem}])
pass
# Check RS memory in distributed mode since we set default as 512m
@@ -791,11 +813,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
mountPoints.append("/")
return mountPoints
- def validatorNotRootFs(self, properties, propertyName, hostInfo):
+ def validatorNotRootFs(self, properties, recommendedDefaults, propertyName, hostInfo):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
dir = properties[propertyName]
- if dir.startswith("hdfs://"):
+ if dir.startswith("hdfs://") or dir == recommendedDefaults.get(propertyName):
return None
dir = re.sub("^file://", "", dir, count=1)
@@ -1095,7 +1117,7 @@ def getMountPointForDir(dir, mountPoints):
"""
bestMountFound = None
if dir:
- dir = re.sub("^file:///", "", dir, count=1).strip().lower()
+ dir = re.sub("^file://", "", dir, count=1).strip().lower()
# If the path is "/hadoop/hdfs/data", then possible matches for mounts could be
# "/", "/hadoop/hdfs", and "/hadoop/hdfs/data".
@@ -1170,3 +1192,6 @@ def getMemorySizeRequired(components, configurations):
totalMemoryRequired += formatXmxSizeToBytes(heapsize)
return totalMemoryRequired
+
+def round_to_n(mem_size, n=128):
+ return int(round(mem_size / float(n))) * int(n)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ff7db80/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 8b3bee2..5a5c2f0 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -33,8 +33,8 @@ class TestHDP206StackAdvisor(TestCase):
with open(stackAdvisorPath, 'rb') as fp:
stack_advisor = imp.load_module( 'stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE) )
with open(hdp206StackAdvisorPath, 'rb') as fp:
- stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
- clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName)
+ self.stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ clazz = getattr(self.stack_advisor_impl, hdp206StackAdvisorClassName)
self.stackAdvisor = clazz()
self.maxDiff = None
# substitute method in the instance
@@ -894,6 +894,201 @@ class TestHDP206StackAdvisor(TestCase):
res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
self.assertEquals(res, res_expected)
+ def test_validateAmsHbaseSiteConfigurations(self):
+ configurations = {
+ "hdfs-site": {
+ "properties": {
+ 'dfs.datanode.data.dir': "/hadoop/data"
+ }
+ },
+ "ams-site": {
+ "properties": {
+ "timeline.metrics.service.operation.mode": "embedded"
+ }
+ }
+ }
+
+ recommendedDefaults = {
+ 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ properties = {
+ 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ host = {
+ "href" : "/api/v1/hosts/host1",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "host1",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "host1",
+ "rack_info" : "/default-rack",
+ "total_mem" : 2097152,
+ "disk_info": [
+ {
+ "available": str(15<<30), # 15 GB
+ "type": "ext4",
+ "mountpoint": "/"
+ }
+ ]
+ }
+ }
+
+ hosts = {
+ "items" : [
+ host
+ ]
+ }
+
+ services = {
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "AMBARI_METRICS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_COLLECTOR",
+ "hostnames": ["host1"]
+ }
+ }, {
+ "StackServiceComponents": {
+ "component_name": "METRICS_MONITOR",
+ "hostnames": ["host1"]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "HDFS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "DATANODE",
+ "hostnames": ["host1"]
+ }
+ }
+ ]
+ }
+ ],
+ "configurations": configurations
+ }
+
+ # only 1 partition, enough disk space, no warnings
+ res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ expected = []
+ self.assertEquals(res, expected)
+
+
+ # 1 partition, no enough disk space
+ host['Hosts']['disk_info'] = [
+ {
+ "available" : '1',
+ "type" : "ext4",
+ "mountpoint" : "/"
+ }
+ ]
+ res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ expected = [
+ {'config-name': 'hbase.rootdir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'Ambari Metrics disk space requirements not met. '
+ '\nRecommended disk space for partition / is 10G',
+ 'type': 'configuration'
+ }
+ ]
+ self.assertEquals(res, expected)
+
+ # 2 partitions
+ host['Hosts']['disk_info'] = [
+ {
+ "available": str(15<<30), # 15 GB
+ "type" : "ext4",
+ "mountpoint" : "/grid/0"
+ },
+ {
+ "available" : str(15<<30), # 15 GB
+ "type" : "ext4",
+ "mountpoint" : "/"
+ }
+ ]
+ recommendedDefaults = {
+ 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ properties = {
+ 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ expected = []
+ self.assertEquals(res, expected)
+
+ # dfs.dir & hbase.rootdir crosscheck + root partition + hbase.rootdir == hbase.tmp.dir warnings
+ properties = {
+ 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+
+ res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ expected = [
+ {
+ 'config-name': 'hbase.rootdir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'It is not recommended to use root partition for hbase.rootdir',
+ 'type': 'configuration'
+ },
+ {
+ 'config-name': 'hbase.tmp.dir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'Consider not using / partition for storing metrics temporary data. '
+ '/ partition is already used as hbase.rootdir to store metrics data',
+ 'type': 'configuration'
+ },
+ {
+ 'config-name': 'hbase.rootdir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'Consider not using / partition for storing metrics data. '
+ '/ is already used by datanode to store HDFS data',
+ 'type': 'configuration'
+ }
+ ]
+ self.assertEquals(res, expected)
+
+ # incorrect hbase.rootdir in distributed mode
+ properties = {
+ 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ configurations['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed'
+ res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ expected = [
+ {
+ 'config-name': 'hbase.rootdir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'In distributed mode hbase.rootdir should point to HDFS. Collector will operate in embedded mode otherwise.',
+ 'type': 'configuration'
+ }
+ ]
+ self.assertEquals(res, expected)
+
def test_getHostsWithComponent(self):
services = {"services":
[{"StackServices":
@@ -1161,8 +1356,9 @@ class TestHDP206StackAdvisor(TestCase):
}
]}
properties = {"property1": "/var/dir"}
+ recommendedDefaults = {"property1": "/var/dir"}
# only / mountpoint - no warning
- self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, 'property1', hostInfo) == None)
+ self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults, 'property1', hostInfo) == None)
# More preferable /grid/0 mountpoint - warning
hostInfo["disk_info"].append(
{
@@ -1171,8 +1367,8 @@ class TestHDP206StackAdvisor(TestCase):
"mountpoint" : "/grid/0"
}
)
-
- warn = self.stackAdvisor.validatorNotRootFs(properties, 'property1', hostInfo)
+ recommendedDefaults = {"property1": "/grid/0/var/dir"}
+ warn = self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults, 'property1', hostInfo)
self.assertFalse(warn == None)
self.assertEquals({'message': 'It is not recommended to use root partition for property1', 'level': 'WARN'}, warn)
@@ -1184,4 +1380,17 @@ class TestHDP206StackAdvisor(TestCase):
"mountpoint" : "/var"
}
)
- self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, 'property1', hostInfo) == None)
+ self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults, 'property1', hostInfo) == None)
+
+ def test_round_to_n(self):
+ self.assertEquals(self.stack_advisor_impl.round_to_n(0), 0)
+ self.assertEquals(self.stack_advisor_impl.round_to_n(1000), 1024)
+ self.assertEquals(self.stack_advisor_impl.round_to_n(2000), 2048)
+ self.assertEquals(self.stack_advisor_impl.round_to_n(4097), 4096)
+
+ def test_getMountPointForDir(self):
+ self.assertEquals(self.stack_advisor_impl.getMountPointForDir("/var/log", ["/"]), "/")
+ self.assertEquals(self.stack_advisor_impl.getMountPointForDir("/var/log", ["/var", "/"]), "/var")
+ self.assertEquals(self.stack_advisor_impl.getMountPointForDir("file:///var/log", ["/var", "/"]), "/var")
+ self.assertEquals(self.stack_advisor_impl.getMountPointForDir("hdfs:///hdfs_path", ["/var", "/"]), None)
+ self.assertEquals(self.stack_advisor_impl.getMountPointForDir("relative/path", ["/var", "/"]), None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ff7db80/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 4c29dd8..2908d8f 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1977,8 +1977,9 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"ams-hbase-env": {
"properties": {
- "hbase_master_heapsize": "540m"
- }
+ "hbase_master_xmn_size": "128m",
+ "hbase_master_heapsize": "512m"
+ }
},
"ams-env": {
"properties": {
@@ -1993,7 +1994,6 @@ class TestHDP22StackAdvisor(TestCase):
"hfile.block.cache.size": "0.3",
"hbase.rootdir": "file:///var/lib/ambari-metrics-collector/hbase",
"hbase.tmp.dir": "/var/lib/ambari-metrics-collector/hbase-tmp",
- "hbase_master_xmn_size" : "128m"
}
},
"ams-site": {
@@ -2008,7 +2008,7 @@ class TestHDP22StackAdvisor(TestCase):
self.assertEquals(configurations, expected)
# 100-nodes cluster, but still only 1 sink (METRICS_COLLECTOR)
- for i in range(2, 101):
+ for i in range(2, 201):
hosts['items'].extend([{
"Hosts": {
"host_name": "host" + str(i)
@@ -2030,13 +2030,13 @@ class TestHDP22StackAdvisor(TestCase):
{
"StackServiceComponents": {
"component_name": "METRICS_MONITOR",
- "hostnames": ["host" + str(i) for i in range(1, 101)]
+ "hostnames": ["host" + str(i) for i in range(1, 201)]
}
}
]
}
]
- expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1034m'
+ expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1408m'
expected["ams-env"]['properties']['metrics_collector_heapsize'] = '512m'
self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
@@ -2058,7 +2058,7 @@ class TestHDP22StackAdvisor(TestCase):
{
"StackServiceComponents": {
"component_name": "DATANODE",
- "hostnames": ["host" + str(i) for i in range(1, 101)]
+ "hostnames": ["host" + str(i) for i in range(1, 201)]
}
}
]
@@ -2077,7 +2077,7 @@ class TestHDP22StackAdvisor(TestCase):
{
"StackServiceComponents": {
"component_name": "NODEMANAGER",
- "hostnames": ["host" + str(i) for i in range(1, 101)]
+ "hostnames": ["host" + str(i) for i in range(1, 201)]
}
}
]
@@ -2096,16 +2096,16 @@ class TestHDP22StackAdvisor(TestCase):
{
"StackServiceComponents": {
"component_name": "METRICS_MONITOR",
- "hostnames": ["host" + str(i) for i in range(1, 101)]
+ "hostnames": ["host" + str(i) for i in range(1, 201)]
}
}
]
}
]
- expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1601m'
- expected["ams-env"]['properties']['metrics_collector_heapsize'] = '512m'
- # expected["ams-hbase-site"]['properties']['hbase_master_xmn_size'] = '256m'
+ expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2432m'
+ expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '256m'
+ expected["ams-env"]['properties']['metrics_collector_heapsize'] = '640m'
self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)