You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2017/09/13 22:10:24 UTC
ambari git commit: AMBARI-21952 : Refine AMS heap recommendations in
stack advisor. (avijayan)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 410f619de -> 519223db6
AMBARI-21952 : Refine AMS heap recommendations in stack advisor. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/519223db
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/519223db
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/519223db
Branch: refs/heads/branch-2.6
Commit: 519223db63d9311d3f14c8b8fa58452b9f09ad91
Parents: 410f619
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Wed Sep 13 15:10:09 2017 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Wed Sep 13 15:10:19 2017 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 83 ++++---
.../stacks/2.0.6/common/test_stack_advisor.py | 4 +-
.../stacks/2.2/common/test_stack_advisor.py | 232 +++++++++++++++++--
3 files changed, 258 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/519223db/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 7992c95..02f4a51 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -681,18 +681,22 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
schMemoryMap = {
"HDFS": {
"NAMENODE": HEAP_PER_MASTER_COMPONENT,
+ "SECONDARY_NAMENODE": HEAP_PER_MASTER_COMPONENT,
"DATANODE": HEAP_PER_SLAVE_COMPONENT
},
"YARN": {
"RESOURCEMANAGER": HEAP_PER_MASTER_COMPONENT,
+ "NODEMANAGER": HEAP_PER_SLAVE_COMPONENT,
+ "HISTORYSERVER" : HEAP_PER_MASTER_COMPONENT,
+ "APP_TIMELINE_SERVER": HEAP_PER_MASTER_COMPONENT
},
"HBASE": {
"HBASE_MASTER": HEAP_PER_MASTER_COMPONENT,
"HBASE_REGIONSERVER": HEAP_PER_SLAVE_COMPONENT
},
- "ACCUMULO": {
- "ACCUMULO_MASTER": HEAP_PER_MASTER_COMPONENT,
- "ACCUMULO_TSERVER": HEAP_PER_SLAVE_COMPONENT
+ "HIVE": {
+ "HIVE_METASTORE": HEAP_PER_MASTER_COMPONENT,
+ "HIVE_SERVER": HEAP_PER_MASTER_COMPONENT
},
"KAFKA": {
"KAFKA_BROKER": HEAP_PER_MASTER_COMPONENT
@@ -706,6 +710,13 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"AMBARI_METRICS": {
"METRICS_COLLECTOR": HEAP_PER_MASTER_COMPONENT,
"METRICS_MONITOR": HEAP_PER_SLAVE_COMPONENT
+ },
+ "ACCUMULO": {
+ "ACCUMULO_MASTER": HEAP_PER_MASTER_COMPONENT,
+ "ACCUMULO_TSERVER": HEAP_PER_SLAVE_COMPONENT
+ },
+ "LOGSEARCH": {
+ "LOGSEARCH_LOGFEEDER" : HEAP_PER_SLAVE_COMPONENT
}
}
total_sinks_count = 0
@@ -716,9 +727,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
schCount = len(
self.getHostsWithComponent(serviceName, componentName, services,
hosts))
- hbase_heapsize += int((schCount * multiplier) ** 0.9)
+ hbase_heapsize += int((schCount * multiplier))
total_sinks_count += schCount
- collector_heapsize = int(hbase_heapsize/4 if hbase_heapsize > 2048 else 512)
+ collector_heapsize = int(hbase_heapsize/3 if hbase_heapsize > 2048 else 512)
+ hbase_heapsize = min(hbase_heapsize, 32768)
return round_to_n(collector_heapsize), round_to_n(hbase_heapsize), total_sinks_count
@@ -843,7 +855,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putAmsHbaseSiteProperty("phoenix.coprocessor.maxMetaDataCacheSize", 81920000)
putAmsSiteProperty("phoenix.query.maxGlobalMemoryPercentage", 30)
putAmsSiteProperty("timeline.metrics.service.resultset.fetchSize", 10000)
- elif total_sinks_count >= 500:
+ elif total_sinks_count >= 1000:
putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
putAmsHbaseSiteProperty("hbase.regionserver.maxlogs", 64)
@@ -859,17 +871,17 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
# Distributed mode heap size
if operatingMode == "distributed":
- hbase_heapsize = max(hbase_heapsize, 768)
+ hbase_heapsize = max(hbase_heapsize, 1024)
putHbaseEnvProperty("hbase_master_heapsize", "512")
putHbaseEnvProperty("hbase_master_xmn_size", "102") #20% of 512 heap size
putHbaseEnvProperty("hbase_regionserver_heapsize", hbase_heapsize)
- putHbaseEnvProperty("regionserver_xmn_size", round_to_n(0.15*hbase_heapsize,64))
+ putHbaseEnvProperty("regionserver_xmn_size", round_to_n(0.15 * hbase_heapsize,64))
else:
# Embedded mode heap size : master + regionserver
- hbase_rs_heapsize = 768
+ hbase_rs_heapsize = 512
putHbaseEnvProperty("hbase_regionserver_heapsize", hbase_rs_heapsize)
putHbaseEnvProperty("hbase_master_heapsize", hbase_heapsize)
- putHbaseEnvProperty("hbase_master_xmn_size", round_to_n(0.15*(hbase_heapsize+hbase_rs_heapsize),64))
+ putHbaseEnvProperty("hbase_master_xmn_size", round_to_n(0.15*(hbase_heapsize + hbase_rs_heapsize),64))
# If no local DN in distributed mode
if operatingMode == "distributed":
@@ -1212,6 +1224,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
pass
elif len(self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR")) > 1 and op_mode != 'distributed':
correct_op_mode_item = self.getErrorItem("Correct value should be 'distributed' for clusters with more then 1 Metrics collector")
+ elif op_mode == 'embedded':
+ collector_heapsize, hbase_heapsize, total_sinks_count = self.getAmsMemoryRecommendation(services, hosts)
+ if total_sinks_count > 1000:
+ correct_op_mode_item = self.getWarnItem("Number of sinks writing metrics to collector is expected to be more than 1000. "
+ "'Embedded' mode AMS might not be able to handle the load. Consider moving to distributed mode.")
+
validationItems.extend([{"config-name":'timeline.metrics.service.operation.mode', "item": correct_op_mode_item }])
return self.toConfigurationValidationProblems(validationItems, "ams-site")
@@ -1372,6 +1390,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
is_hbase_distributed = amsHbaseSite.get("hbase.cluster.distributed").lower() == 'true'
if is_hbase_distributed:
+
+ if not regionServerItem and hbase_regionserver_heapsize > 32768:
+ regionServerItem = self.getWarnItem("Value is more than the recommended maximum heap size of 32G.")
+ validationItems.extend([{"config-name": "hbase_regionserver_heapsize", "item": regionServerItem}])
+
minMasterXmn = 0.12 * hbase_master_heapsize
maxMasterXmn = 0.2 * hbase_master_heapsize
if hbase_master_xmn_size < minMasterXmn:
@@ -1394,6 +1417,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"(20% of hbase_regionserver_heapsize)"
.format(int(floor(maxRegionServerXmn))))
else:
+
+ if not hbaseMasterHeapsizeItem and (hbase_master_heapsize + hbase_regionserver_heapsize) > 32768:
+ hbaseMasterHeapsizeItem = self.getWarnItem("Value of Master + Regionserver heapsize is more than the recommended maximum heap size of 32G.")
+ validationItems.extend([{"config-name": "hbase_master_heapsize", "item": hbaseMasterHeapsizeItem}])
+
minMasterXmn = 0.12 * (hbase_master_heapsize + hbase_regionserver_heapsize)
maxMasterXmn = 0.2 * (hbase_master_heapsize + hbase_regionserver_heapsize)
if hbase_master_xmn_size < minMasterXmn:
@@ -1440,39 +1468,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
collectorHostName, str(", ".join(hostMasterComponents[collectorHostName]))))
if hbaseMasterHeapsizeItem:
validationItems.extend([{"config-name": "hbase_master_heapsize", "item": hbaseMasterHeapsizeItem}])
-
- # Check for unused RAM on AMS Collector node
- hostComponents = []
- for service in services["services"]:
- for component in service["components"]:
- if component["StackServiceComponents"]["hostnames"] is not None:
- if collectorHostName in component["StackServiceComponents"]["hostnames"]:
- hostComponents.append(component["StackServiceComponents"]["component_name"])
-
- requiredMemory = getMemorySizeRequired(hostComponents, configurations)
- unusedMemory = host["Hosts"]["total_mem"] * 1024 - requiredMemory # in bytes
-
- heapPropertyToIncrease = "hbase_regionserver_heapsize" if is_hbase_distributed else "hbase_master_heapsize"
- xmnPropertyToIncrease = "regionserver_xmn_size" if is_hbase_distributed else "hbase_master_xmn_size"
- hbase_needs_increase = to_number(properties[heapPropertyToIncrease]) * mb < 32 * gb
-
- if unusedMemory > 4*gb and hbase_needs_increase: # warn user, if more than 4GB RAM is unused
-
- recommended_hbase_heapsize = int((unusedMemory - 4*gb)*4/5) + to_number(properties.get(heapPropertyToIncrease))*mb
- recommended_hbase_heapsize = min(32*gb, recommended_hbase_heapsize) #Make sure heapsize <= 32GB
- recommended_hbase_heapsize = round_to_n(recommended_hbase_heapsize/mb,128) # Round to 128m multiple
- if to_number(properties[heapPropertyToIncrease]) < recommended_hbase_heapsize:
- hbaseHeapsizeItem = self.getWarnItem("Consider allocating {0} MB to {1} in ams-hbase-env to use up some "
- "unused memory on host"
- .format(recommended_hbase_heapsize,
- heapPropertyToIncrease))
- validationItems.extend([{"config-name": heapPropertyToIncrease, "item": hbaseHeapsizeItem}])
-
- recommended_xmn_size = round_to_n(0.15*recommended_hbase_heapsize,128)
- if to_number(properties[xmnPropertyToIncrease]) < recommended_xmn_size:
- xmnPropertyToIncreaseItem = self.getWarnItem("Consider allocating {0} MB to use up some unused memory "
- "on host".format(recommended_xmn_size))
- validationItems.extend([{"config-name": xmnPropertyToIncrease, "item": xmnPropertyToIncreaseItem}])
pass
return self.toConfigurationValidationProblems(validationItems, "ams-hbase-env")
@@ -2241,7 +2236,7 @@ def getHeapsizeProperties():
"DATANODE": [{"config-name": "hadoop-env",
"property": "dtnode_heapsize",
"default": "1024m"}],
- "REGIONSERVER": [{"config-name": "hbase-env",
+ "HBASE_REGIONSERVER": [{"config-name": "hbase-env",
"property": "hbase_regionserver_heapsize",
"default": "1024m"}],
"HBASE_MASTER": [{"config-name": "hbase-env",
http://git-wip-us.apache.org/repos/asf/ambari/blob/519223db/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 35af503..9e00704 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -2146,7 +2146,7 @@ class TestHDP206StackAdvisor(TestCase):
'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector',
'hbase_master_heapsize': '512',
'hbase_master_xmn_size': '102',
- 'hbase_regionserver_heapsize': '768',
+ 'hbase_regionserver_heapsize': '1024',
'regionserver_xmn_size': '128'}},
'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true',
'hbase.hregion.memstore.flush.size': '134217728',
@@ -2211,7 +2211,7 @@ class TestHDP206StackAdvisor(TestCase):
'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector',
'hbase_master_heapsize': '512',
'hbase_master_xmn_size': '102',
- 'hbase_regionserver_heapsize': '768',
+ 'hbase_regionserver_heapsize': '1024',
'regionserver_xmn_size': '128'}},
'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true',
'hbase.hregion.memstore.flush.size': '134217728',
http://git-wip-us.apache.org/repos/asf/ambari/blob/519223db/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 9c6bef1..b069828 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2338,9 +2338,9 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"ams-hbase-env": {
"properties": {
- "hbase_master_xmn_size": "192",
+ "hbase_master_xmn_size": "128",
"hbase_master_heapsize": "512",
- "hbase_regionserver_heapsize": "768"
+ "hbase_regionserver_heapsize": "512"
}
},
"ams-grafana-env": {
@@ -2416,14 +2416,14 @@ class TestHDP22StackAdvisor(TestCase):
expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '500'
expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '7'
- expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1408'
- expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '320'
- expected["ams-env"]['properties']['metrics_collector_heapsize'] = '512'
+ expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2560'
+ expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '448'
+ expected["ams-env"]['properties']['metrics_collector_heapsize'] = '896'
self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
- # Still 100 nodes, but with HDFS and YARN services installed on all nodes
+ # 200 nodes, but with HDFS and YARN services installed on all nodes
services['services'] = [
{
"StackServices": {
@@ -2484,14 +2484,22 @@ class TestHDP22StackAdvisor(TestCase):
}
]
- expected["ams-site"]['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'mem_total'
+ expected["ams-site"]['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'dfs.FSNamesystem.FilesTotal,' \
+ 'dfs.datanode.WritesFromRemoteClient,' \
+ 'ipc.IPC.numCallsInReplicationQueue,' \
+ 'mapred.ShuffleMetrics.ShuffleOutputsFailed,' \
+ 'mem_buffered,' \
+ 'read_count,' \
+ 'regionserver.Server.percentFilesLocal,' \
+ 'rpcdetailed.rpcdetailed.RegisterNodeManagerNumOps,' \
+ 'sdisk_vdb_write_count'
expected["ams-site"]['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total'
- expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '500'
- expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '7'
- expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2432'
- expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '512'
- expected["ams-env"]['properties']['metrics_collector_heapsize'] = '640'
+ expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '600'
+ expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '6'
+ expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '6656'
+ expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '1088'
+ expected["ams-env"]['properties']['metrics_collector_heapsize'] = '2176'
self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
@@ -2535,6 +2543,7 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+
# Embedded mode, 8192m master heapsize, more splitpoints recommended
services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192'
expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192'
@@ -2559,12 +2568,11 @@ class TestHDP22StackAdvisor(TestCase):
expected['ams-hbase-site']['properties']['dfs.client.read.shortcircuit'] = 'true'
# Distributed mode, low memory, no splitpoints recommended
- services["configurations"]['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512'
expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'mem_total'
expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total'
- expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512'
+ expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '6656'
expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102'
- expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '384'
+ expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '1024'
expected['ams-site']['properties']['timeline.metrics.service.watcher.disabled'] = 'true'
self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
@@ -2610,6 +2618,200 @@ class TestHDP22StackAdvisor(TestCase):
self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.size'], '700')
self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.commit.interval'], '5')
+ # 500 Nodes with HDFS, YARN, HIVE, STORM, HBASE, KAFKA, AMS
+ node_count = 500
+ hosts = {
+ "items": []
+ }
+ for i in range(1, node_count):
+ hosts['items'].extend([{
+ "Hosts": {
+ "host_name": "host" + str(i)
+ }
+ }])
+
+ services['services'] = [
+ {
+ "StackServices": {
+ "service_name": "HDFS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "NAMENODE",
+ "hostnames": ["host1"]
+ }
+ } ,
+ {
+ "StackServiceComponents": {
+ "component_name": "SECONDARY_NAMENODE",
+ "hostnames": ["host2"]
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DATANODE",
+ "hostnames": ["host" + str(i) for i in range(6, node_count + 1)]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "HBASE"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "HBASE_MASTER",
+ "hostnames": ["host3"]
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "HBASE_REGIONSERVER",
+ "hostnames": ["host" + str(i) for i in range(6, node_count + 1)]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "YARN"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "RESOURCEMANAGER",
+ "hostnames": ["host4"]
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "NODEMANAGER",
+ "hostnames": ["host" + str(i) for i in range(6, node_count + 1)]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "HIVE"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "HIVE_METASTORE",
+ "hostnames": ["host3"]
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "HIVE_SERVER",
+ "hostnames": ["host3"]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "STORM"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "NIMBUS",
+ "hostnames": ["host" + str(i) for i in range(1, 6)]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "KAFKA"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "KAFKA_BROKER",
+ "hostnames": ["host" + str(i) for i in range(1, 6)]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "AMBARI_METRICS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_COLLECTOR",
+ "hostnames": ["host6"]
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_MONITOR",
+ "hostnames": ["host" + str(i) for i in range(6, node_count + 1)]
+ }
+ }
+ ]
+ }
+ ]
+
+ services['configurations'] = {
+ 'core-site': {'properties': {}},
+ 'ams-site': {'properties': {}},
+ 'ams-hbase-site': {'properties': {}},
+ 'ams-hbase-env': {'properties': {}}
+ }
+ services["configurations"]['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed'
+
+ expected['ams-hbase-site']['properties']['hbase.cluster.distributed'] = 'true'
+ expected['ams-hbase-site']['properties']['hbase.rootdir'] = '/user/ams/hbase'
+ expected['ams-hbase-site']['properties']['hbase.zookeeper.property.clientPort'] = '2181'
+
+ expected["ams-site"]['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'default.General.active_calls_api_get_all_databases,' \
+ 'default.General.api_get_database_mean,' \
+ 'default.General.gc.PS-MarkSweep.count,' \
+ 'dfs.FsVolume.TotalDataFileIos,' \
+ 'disk_free,' \
+ 'jvm.JvmMetrics.MemHeapMaxM,' \
+ 'kafka.network.RequestMetrics.RemoteTimeMs.request.Metadata.75percentile,' \
+ 'kafka.network.RequestMetrics.ResponseQueueTimeMs.request.Update.Metadata.mean,' \
+ 'load_one,master.FileSystem.MetaHlogSplitTime_75th_percentile,' \
+ 'metricssystem.MetricsSystem.NumActiveSources,' \
+ 'regionserver.Server.Append_95th_percentile,' \
+ 'regionserver.Server.blockCacheEvictionCount,' \
+ 'rpc.rpc.client.SentBytes,' \
+ 'sdisk_vda1_write_bytes'
+ expected["ams-site"]['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'ipc.IPC.authorizationSuccesses,' \
+ 'metricssystem.MetricsSystem.PublishNumOps'
+
+ expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '700'
+ expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '5'
+ expected["ams-site"]['properties']['timeline.metrics.service.resultset.fetchSize'] = '5000'
+ expected["ams-site"]['properties']['phoenix.query.maxGlobalMemoryPercentage'] = '30'
+
+ expected["ams-env"]['properties']['metrics_collector_heapsize'] = '7040'
+
+ expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '512'
+ expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102'
+
+ expected["ams-hbase-env"]['properties']['hbase_regionserver_heapsize'] = '21120'
+ expected["ams-hbase-env"]['properties']['regionserver_xmn_size'] = '3200'
+
+ expected["ams-hbase-site"]['properties']['phoenix.query.maxGlobalMemoryPercentage'] = '20'
+ expected['ams-hbase-site']['properties']['hbase.hregion.memstore.flush.size'] = '268435456'
+ expected['ams-hbase-site']['properties']['hbase.regionserver.handler.count'] = '60'
+ expected['ams-hbase-site']['properties']['hbase.regionserver.hlog.blocksize'] = '134217728'
+ expected['ams-hbase-site']['properties']['hbase.regionserver.maxlogs'] = '64'
+ expected['ams-hbase-site']['properties']['phoenix.coprocessor.maxMetaDataCacheSize'] = '40960000'
+
+ self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
def test_recommendHbaseConfigurations(self):
servicesList = ["HBASE"]
configurations = {}