You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2016/09/28 13:31:57 UTC
ambari git commit: AMBARI-18098 stack_advisor should recommend AMS
cache size and commit frequency (dsen)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 dbf97c8bc -> 4aa459635
AMBARI-18098 stack_advisor should recommend AMS cache size and commit frequency (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4aa45963
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4aa45963
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4aa45963
Branch: refs/heads/branch-2.5
Commit: 4aa459635b202c7911d30cda4ddd36e5cd794b39
Parents: dbf97c8
Author: Dmytro Sen <ds...@apache.org>
Authored: Thu Aug 11 19:51:11 2016 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Wed Sep 28 16:30:07 2016 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 5 ++-
.../stacks/2.2/common/test_stack_advisor.py | 44 +++++++++++++++++++-
2 files changed, 47 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aa45963/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index e9d7a9c..71953b1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -22,7 +22,7 @@ import os
import sys
import socket
-from math import ceil, floor
+from math import ceil, floor, log
from resource_management.core.logger import Logger
from resource_management.libraries.functions.mounted_dirs_helper import get_mounts_with_multiple_data_dirs
@@ -660,6 +660,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putAmsEnvProperty("metrics_collector_heapsize", collector_heapsize)
+ putAmsSiteProperty("timeline.metrics.cache.size", max(100, int(log(total_sinks_count)) * 100))
+ putAmsSiteProperty("timeline.metrics.cache.commit.interval", min(10, max(12 - int(log(total_sinks_count)), 2)))
+
# blockCache = 0.3, memstore = 0.35, phoenix-server = 0.15, phoenix-client = 0.25
putAmsHbaseSiteProperty("hfile.block.cache.size", 0.3)
putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 134217728)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aa45963/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 388fc3b..0cc4c1f 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2294,7 +2294,9 @@ class TestHDP22StackAdvisor(TestCase):
"timeline.metrics.host.aggregate.splitpoints": "master.FileSystem.MetaHlogSplitTime_75th_percentile",
"timeline.metrics.service.handler.thread.count": "20",
'timeline.metrics.service.webapp.address': 'host1:6188',
- 'timeline.metrics.service.watcher.disabled': 'false'
+ 'timeline.metrics.service.watcher.disabled': 'false',
+ 'timeline.metrics.cache.size': '100',
+ 'timeline.metrics.cache.commit.interval': '10'
}
}
}
@@ -2330,6 +2332,9 @@ class TestHDP22StackAdvisor(TestCase):
]
}
]
+
+ expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '500'
+ expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '7'
expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1408'
expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '320'
expected["ams-env"]['properties']['metrics_collector_heapsize'] = '512'
@@ -2400,6 +2405,9 @@ class TestHDP22StackAdvisor(TestCase):
]
expected["ams-site"]['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'master.Server.numDeadRegionServers'
expected["ams-site"]['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'master.Server.numDeadRegionServers'
+
+ expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '500'
+ expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '7'
expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2432'
expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '512'
expected["ams-env"]['properties']['metrics_collector_heapsize'] = '640'
@@ -2487,6 +2495,40 @@ class TestHDP22StackAdvisor(TestCase):
self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'].split(',')), 13)
self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'].split(',')), 2)
+ # 2000-nodes cluster
+ for i in range(202, 2001):
+ hosts['items'].extend([{
+ "Hosts": {
+ "host_name": "host" + str(i)
+ }
+ }])
+
+ services['services'] = [
+ {
+ "StackServices": {
+ "service_name": "AMBARI_METRICS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_COLLECTOR",
+ "hostnames": ["host1"]
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_MONITOR",
+ "hostnames": ["host" + str(i) for i in range(1, 2001)]
+ }
+ }
+ ]
+ }
+ ]
+
+ self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.size'], '700')
+ self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.commit.interval'], '5')
+
def test_recommendHbaseConfigurations(self):
servicesList = ["HBASE"]
configurations = {}