You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2017/10/09 06:01:31 UTC
[33/50] [abbrv] ambari git commit: AMBARI-22124. Refactor AMS logic
in stack advisors to service advisors.(vbrodetskyi)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f32765d/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py b/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
new file mode 100644
index 0000000..05254fe
--- /dev/null
+++ b/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
@@ -0,0 +1,596 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import imp
+import json
+import os
+from unittest import TestCase
+
+from mock.mock import patch, MagicMock
+
+
+class TestAMBARI_METRICS010ServiceAdvisor(TestCase):
+
+ testDirectory = os.path.dirname(os.path.abspath(__file__))
+ stack_advisor_path = os.path.join(testDirectory, '../../../../main/resources/stacks/stack_advisor.py')
+ with open(stack_advisor_path, 'rb') as fp:
+ imp.load_module('stack_advisor', fp, stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE))
+
+ serviceAdvisorPath = '../../../../main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py'
+ ambariMetrics010ServiceAdvisorPath = os.path.join(testDirectory, serviceAdvisorPath)
+ with open(ambariMetrics010ServiceAdvisorPath, 'rb') as fp:
+ service_advisor_impl = imp.load_module('service_advisor_impl', fp, ambariMetrics010ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+
+ def setUp(self):
+ serviceAdvisorClass = getattr(self.service_advisor_impl, 'AMBARI_METRICSServiceAdvisor')
+ self.serviceAdvisor = serviceAdvisorClass()
+
+
+ def test_recommendAmsConfigurations(self):
+ configurations = {
+ "hadoop-env": {
+ "properties": {
+ "hdfs_user": "hdfs",
+ "proxyuser_group": "users"
+ }
+ }
+ }
+
+ hosts = {
+ "items": [
+ {
+ "href": "/api/v1/hosts/host1",
+ "Hosts": {
+ "cpu_count": 1,
+ "host_name": "c6401.ambari.apache.org",
+ "os_arch": "x86_64",
+ "os_type": "centos6",
+ "ph_cpu_count": 1,
+ "public_host_name": "public.c6401.ambari.apache.org",
+ "rack_info": "/default-rack",
+ "total_mem": 2097152,
+ "disk_info": [{
+ "size": '80000000',
+ "mountpoint": "/"
+ }]
+ }
+ },
+ {
+ "href": "/api/v1/hosts/host2",
+ "Hosts": {
+ "cpu_count": 1,
+ "host_name": "c6402.ambari.apache.org",
+ "os_arch": "x86_64",
+ "os_type": "centos6",
+ "ph_cpu_count": 1,
+ "public_host_name": "public.c6402.ambari.apache.org",
+ "rack_info": "/default-rack",
+ "total_mem": 1048576,
+ "disk_info": [{
+ "size": '800000000',
+ "mountpoint": "/"
+ }]
+ }
+ }
+ ]}
+
+
+ services1 = {
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "HDFS"
+ }, "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "NAMENODE",
+ "hostnames": ["c6401.ambari.apache.org"]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "AMBARI_METRICS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_COLLECTOR",
+ "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
+ }
+ }, {
+ "StackServiceComponents": {
+ "component_name": "METRICS_MONITOR",
+ "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
+ }
+ }
+ ]
+ }],
+ "configurations": configurations,
+ "ambari-server-properties": {"ambari-server.user":"ambari_user"}
+ }
+
+ clusterData = {
+ "totalAvailableRam": 2048
+ }
+
+ expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}},
+ 'ams-grafana-env': {'properties': {},
+ 'property_attributes': {'metrics_grafana_password': {'visible': 'false'}}},
+ 'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector',
+ 'hbase_master_heapsize': '512',
+ 'hbase_master_xmn_size': '102',
+ 'hbase_regionserver_heapsize': '1024',
+ 'regionserver_xmn_size': '128'}},
+ 'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true',
+ 'hbase.hregion.memstore.flush.size': '134217728',
+ 'hbase.regionserver.global.memstore.lowerLimit': '0.3',
+ 'hbase.regionserver.global.memstore.upperLimit': '0.35',
+ 'hbase.rootdir': '/user/ams/hbase',
+ 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp',
+ 'hbase.zookeeper.property.clientPort': '2181',
+ 'hfile.block.cache.size': '0.3'}},
+ 'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10',
+ 'timeline.metrics.cache.size': '100',
+ 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
+ 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
+ 'timeline.metrics.service.handler.thread.count': '20',
+ 'timeline.metrics.service.operation.mode': 'distributed',
+ 'timeline.metrics.service.watcher.disabled': 'true',
+ 'timeline.metrics.service.webapp.address': '0.0.0.0:6188'}},
+ 'hadoop-env': {'properties': {'hdfs_user': 'hdfs',
+ 'proxyuser_group': 'users'}}}
+
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services1, hosts)
+ self.assertEquals(configurations, expected)
+
+ services1 = {
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "HDFS"
+ }, "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "NAMENODE",
+ "hostnames": ["c6401.ambari.apache.org"]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "AMBARI_METRICS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_COLLECTOR",
+ "hostnames": ["c6401.ambari.apache.org"]
+ }
+ }, {
+ "StackServiceComponents": {
+ "component_name": "METRICS_MONITOR",
+ "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
+ }
+ }
+ ]
+ }],
+ "configurations": configurations,
+ "ambari-server-properties": {"ambari-server.user":"ambari_user"}
+ }
+ expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}},
+ 'ams-grafana-env': {'properties': {},
+ 'property_attributes': {'metrics_grafana_password': {'visible': 'false'}}},
+ 'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector',
+ 'hbase_master_heapsize': '512',
+ 'hbase_master_xmn_size': '102',
+ 'hbase_regionserver_heapsize': '1024',
+ 'regionserver_xmn_size': '128'}},
+ 'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true',
+ 'hbase.hregion.memstore.flush.size': '134217728',
+ 'hbase.regionserver.global.memstore.lowerLimit': '0.3',
+ 'hbase.regionserver.global.memstore.upperLimit': '0.35',
+ 'hbase.rootdir': '/user/ams/hbase',
+ 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp',
+ 'hbase.zookeeper.property.clientPort': '2181',
+ 'hfile.block.cache.size': '0.3',
+ 'phoenix.coprocessor.maxMetaDataCacheSize': '20480000'}},
+ 'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10',
+ 'timeline.metrics.cache.size': '100',
+ 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
+ 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
+ 'timeline.metrics.service.handler.thread.count': '20',
+ 'timeline.metrics.service.operation.mode': 'distributed',
+ 'timeline.metrics.service.watcher.disabled': 'true',
+ 'timeline.metrics.service.webapp.address': '0.0.0.0:6188'}},
+ 'hadoop-env': {'properties': {'hdfs_user': 'hdfs',
+ 'proxyuser_group': 'users'}}}
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services1, hosts)
+ self.assertEquals(configurations, expected)
+
+
+ def test_validateAmsSiteConfigurations(self):
+ configurations = {
+ "hdfs-site": {
+ "properties": {
+ 'dfs.datanode.data.dir': "/hadoop/data"
+ }
+ },
+ "core-site": {
+ "properties": {
+ "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020"
+ }
+ },
+ "ams-site": {
+ "properties": {
+ "timeline.metrics.service.operation.mode": "embedded"
+ }
+ }
+ }
+ recommendedDefaults = {
+ 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ properties = {
+ 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false',
+ 'timeline.metrics.service.operation.mode' : 'embedded'
+ }
+ host1 = {
+ "href" : "/api/v1/hosts/host1",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "host1",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "host1",
+ "rack_info" : "/default-rack",
+ "total_mem" : 2097152,
+ "disk_info": [
+ {
+ "available": str(15<<30), # 15 GB
+ "type": "ext4",
+ "mountpoint": "/"
+ }
+ ]
+ }
+ }
+ host2 = {
+ "href" : "/api/v1/hosts/host2",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "host2",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "host2",
+ "rack_info" : "/default-rack",
+ "total_mem" : 2097152,
+ "disk_info": [
+ {
+ "available": str(15<<30), # 15 GB
+ "type": "ext4",
+ "mountpoint": "/"
+ }
+ ]
+ }
+ }
+
+ hosts = {
+ "items" : [
+ host1, host2
+ ]
+ }
+
+ services = {
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "AMBARI_METRICS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_COLLECTOR",
+ "hostnames": ["host1", "host2"]
+ }
+ }, {
+ "StackServiceComponents": {
+ "component_name": "METRICS_MONITOR",
+ "hostnames": ["host1", "host2"]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "HDFS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "DATANODE",
+ "hostnames": ["host1"]
+ }
+ }
+ ]
+ }
+ ],
+ "configurations": configurations
+ }
+ # only 1 partition, enough disk space, no warnings
+ res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts)
+ expected = [{'config-name': 'timeline.metrics.service.operation.mode',
+ 'config-type': 'ams-site',
+ 'level': 'ERROR',
+ 'message': "Correct value should be 'distributed' for clusters with more then 1 Metrics collector",
+ 'type': 'configuration'}]
+ self.assertEquals(res, expected)
+
+
+ services = {
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "AMBARI_METRICS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_COLLECTOR",
+ "hostnames": ["host1"]
+ }
+ }, {
+ "StackServiceComponents": {
+ "component_name": "METRICS_MONITOR",
+ "hostnames": ["host1"]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "HDFS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "DATANODE",
+ "hostnames": ["host1"]
+ }
+ }
+ ]
+ }
+ ],
+ "configurations": configurations
+ }
+ res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts)
+ expected = []
+ self.assertEquals(res, expected)
+
+ def test_validateAmsHbaseSiteConfigurations(self):
+ configurations = {
+ "hdfs-site": {
+ "properties": {
+ 'dfs.datanode.data.dir': "/hadoop/data"
+ }
+ },
+ "core-site": {
+ "properties": {
+ "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020"
+ }
+ },
+ "ams-site": {
+ "properties": {
+ "timeline.metrics.service.operation.mode": "embedded"
+ }
+ }
+ }
+
+ recommendedDefaults = {
+ 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ properties = {
+ 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ host = {
+ "href" : "/api/v1/hosts/host1",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "host1",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "host1",
+ "rack_info" : "/default-rack",
+ "total_mem" : 2097152,
+ "disk_info": [
+ {
+ "available": str(15<<30), # 15 GB
+ "type": "ext4",
+ "mountpoint": "/"
+ }
+ ]
+ }
+ }
+
+ hosts = {
+ "items" : [
+ host
+ ]
+ }
+
+ services = {
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "AMBARI_METRICS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "METRICS_COLLECTOR",
+ "hostnames": ["host1"]
+ }
+ }, {
+ "StackServiceComponents": {
+ "component_name": "METRICS_MONITOR",
+ "hostnames": ["host1"]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices": {
+ "service_name": "HDFS"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "DATANODE",
+ "hostnames": ["host1"]
+ }
+ }
+ ]
+ }
+ ],
+ "configurations": configurations
+ }
+
+ # only 1 partition, enough disk space, no warnings
+ res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts)
+ expected = []
+ self.assertEquals(res, expected)
+
+
+ # 1 partition, no enough disk space
+ host['Hosts']['disk_info'] = [
+ {
+ "available" : '1',
+ "type" : "ext4",
+ "mountpoint" : "/"
+ }
+ ]
+ res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts)
+ expected = [
+ {'config-name': 'hbase.rootdir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'Ambari Metrics disk space requirements not met. '
+ '\nRecommended disk space for partition / is 10G',
+ 'type': 'configuration'
+ }
+ ]
+ self.assertEquals(res, expected)
+
+ # 2 partitions
+ host['Hosts']['disk_info'] = [
+ {
+ "available": str(15<<30), # 15 GB
+ "type" : "ext4",
+ "mountpoint" : "/grid/0"
+ },
+ {
+ "available" : str(15<<30), # 15 GB
+ "type" : "ext4",
+ "mountpoint" : "/"
+ }
+ ]
+ recommendedDefaults = {
+ 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ properties = {
+ 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts)
+ expected = []
+ self.assertEquals(res, expected)
+
+ # dfs.dir & hbase.rootdir crosscheck + root partition + hbase.rootdir == hbase.tmp.dir warnings
+ properties = {
+ 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+
+ res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts)
+ expected = [
+ {
+ 'config-name': 'hbase.rootdir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'It is not recommended to use root partition for hbase.rootdir',
+ 'type': 'configuration'
+ },
+ {
+ 'config-name': 'hbase.tmp.dir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'Consider not using / partition for storing metrics temporary data. '
+ '/ partition is already used as hbase.rootdir to store metrics data',
+ 'type': 'configuration'
+ },
+ {
+ 'config-name': 'hbase.rootdir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'Consider not using / partition for storing metrics data. '
+ '/ is already used by datanode to store HDFS data',
+ 'type': 'configuration'
+ }
+ ]
+ self.assertEquals(res, expected)
+
+ # incorrect hbase.rootdir in distributed mode
+ properties = {
+ 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+ 'hbase.cluster.distributed': 'false'
+ }
+ configurations['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed'
+ res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts)
+ expected = [
+ {
+ 'config-name': 'hbase.rootdir',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'WARN',
+ 'message': 'In distributed mode hbase.rootdir should point to HDFS.',
+ 'type': 'configuration'
+ },
+ {
+ 'config-name': 'hbase.cluster.distributed',
+ 'config-type': 'ams-hbase-site',
+ 'level': 'ERROR',
+ 'message': 'hbase.cluster.distributed property should be set to true for distributed mode',
+ 'type': 'configuration'
+ }
+ ]
+ self.assertEquals(res, expected)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f32765d/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 65b23b0..38d6ecd 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -2315,214 +2315,6 @@ class TestHDP206StackAdvisor(TestCase):
- def test_recommendAmsConfigurations(self):
- configurations = {
- "hadoop-env": {
- "properties": {
- "hdfs_user": "hdfs",
- "proxyuser_group": "users"
- }
- }
- }
-
- hosts = {
- "items": [
- {
- "href": "/api/v1/hosts/host1",
- "Hosts": {
- "cpu_count": 1,
- "host_name": "c6401.ambari.apache.org",
- "os_arch": "x86_64",
- "os_type": "centos6",
- "ph_cpu_count": 1,
- "public_host_name": "public.c6401.ambari.apache.org",
- "rack_info": "/default-rack",
- "total_mem": 2097152,
- "disk_info": [{
- "size": '80000000',
- "mountpoint": "/"
- }]
- }
- },
- {
- "href": "/api/v1/hosts/host2",
- "Hosts": {
- "cpu_count": 1,
- "host_name": "c6402.ambari.apache.org",
- "os_arch": "x86_64",
- "os_type": "centos6",
- "ph_cpu_count": 1,
- "public_host_name": "public.c6402.ambari.apache.org",
- "rack_info": "/default-rack",
- "total_mem": 1048576,
- "disk_info": [{
- "size": '800000000',
- "mountpoint": "/"
- }]
- }
- }
- ]}
-
-
- services1 = {
- "services": [
- {
- "StackServices": {
- "service_name": "HDFS"
- }, "components": [
- {
- "StackServiceComponents": {
- "component_name": "NAMENODE",
- "hostnames": ["c6401.ambari.apache.org"]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
- }
- }, {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
- }
- }
- ]
- }],
- "configurations": configurations,
- "ambari-server-properties": {"ambari-server.user":"ambari_user"}
- }
-
- clusterData = {
- "totalAvailableRam": 2048
- }
-
- expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}},
- 'ams-grafana-env': {'properties': {},
- 'property_attributes': {'metrics_grafana_password': {'visible': 'false'}}},
- 'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector',
- 'hbase_master_heapsize': '512',
- 'hbase_master_xmn_size': '102',
- 'hbase_regionserver_heapsize': '1024',
- 'regionserver_xmn_size': '128'}},
- 'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true',
- 'hbase.hregion.memstore.flush.size': '134217728',
- 'hbase.regionserver.global.memstore.lowerLimit': '0.3',
- 'hbase.regionserver.global.memstore.upperLimit': '0.35',
- 'hbase.rootdir': '/user/ams/hbase',
- 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp',
- 'hbase.zookeeper.property.clientPort': '2181',
- 'hfile.block.cache.size': '0.3'}},
- 'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10',
- 'timeline.metrics.cache.size': '100',
- 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
- 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
- 'timeline.metrics.service.handler.thread.count': '20',
- 'timeline.metrics.service.operation.mode': 'distributed',
- 'timeline.metrics.service.watcher.disabled': 'true',
- 'timeline.metrics.service.webapp.address': '0.0.0.0:6188'}},
- 'hadoop-env': {'properties': {'hdfs_user': 'hdfs',
- 'proxyuser_group': 'users'}}}
-
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services1, hosts)
- self.assertEquals(configurations, expected)
-
- services1 = {
- "services": [
- {
- "StackServices": {
- "service_name": "HDFS"
- }, "components": [
- {
- "StackServiceComponents": {
- "component_name": "NAMENODE",
- "hostnames": ["c6401.ambari.apache.org"]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["c6401.ambari.apache.org"]
- }
- }, {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
- }
- }
- ]
- }],
- "configurations": configurations,
- "ambari-server-properties": {"ambari-server.user":"ambari_user"}
- }
- expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}},
- 'ams-grafana-env': {'properties': {},
- 'property_attributes': {'metrics_grafana_password': {'visible': 'false'}}},
- 'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector',
- 'hbase_master_heapsize': '512',
- 'hbase_master_xmn_size': '102',
- 'hbase_regionserver_heapsize': '1024',
- 'regionserver_xmn_size': '128'}},
- 'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true',
- 'hbase.hregion.memstore.flush.size': '134217728',
- 'hbase.regionserver.global.memstore.lowerLimit': '0.3',
- 'hbase.regionserver.global.memstore.upperLimit': '0.35',
- 'hbase.rootdir': '/user/ams/hbase',
- 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp',
- 'hbase.zookeeper.property.clientPort': '2181',
- 'hfile.block.cache.size': '0.3',
- 'phoenix.coprocessor.maxMetaDataCacheSize': '20480000'}},
- 'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10',
- 'timeline.metrics.cache.size': '100',
- 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
- 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
- 'timeline.metrics.service.handler.thread.count': '20',
- 'timeline.metrics.service.operation.mode': 'distributed',
- 'timeline.metrics.service.watcher.disabled': 'true',
- 'timeline.metrics.service.webapp.address': '0.0.0.0:6188'}},
- 'hadoop-env': {'properties': {'hdfs_user': 'hdfs',
- 'proxyuser_group': 'users'}}}
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services1, hosts)
- self.assertEquals(configurations, expected)
-
- def test_getHostNamesWithComponent(self):
-
- services = {
- "services": [
- {
- "StackServices": {
- "service_name": "SERVICE"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "COMPONENT",
- "hostnames": ["host1","host2","host3"]
- }
- }
- ]
- }
- ],
- "configurations": {}
- }
-
- result = self.stackAdvisor.getHostNamesWithComponent("SERVICE","COMPONENT", services)
- expected = ["host1","host2","host3"]
- self.assertEquals(result, expected)
def test_getZKHostPortString(self):
@@ -2908,374 +2700,6 @@ class TestHDP206StackAdvisor(TestCase):
validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
self.assertEquals(validation_problems, expected)
- def test_validateAmsSiteConfigurations(self):
- configurations = {
- "hdfs-site": {
- "properties": {
- 'dfs.datanode.data.dir': "/hadoop/data"
- }
- },
- "core-site": {
- "properties": {
- "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020"
- }
- },
- "ams-site": {
- "properties": {
- "timeline.metrics.service.operation.mode": "embedded"
- }
- }
- }
- recommendedDefaults = {
- 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
- 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
- 'hbase.cluster.distributed': 'false'
- }
- properties = {
- 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
- 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
- 'hbase.cluster.distributed': 'false',
- 'timeline.metrics.service.operation.mode' : 'embedded'
- }
- host1 = {
- "href" : "/api/v1/hosts/host1",
- "Hosts" : {
- "cpu_count" : 1,
- "host_name" : "host1",
- "os_arch" : "x86_64",
- "os_type" : "centos6",
- "ph_cpu_count" : 1,
- "public_host_name" : "host1",
- "rack_info" : "/default-rack",
- "total_mem" : 2097152,
- "disk_info": [
- {
- "available": str(15<<30), # 15 GB
- "type": "ext4",
- "mountpoint": "/"
- }
- ]
- }
- }
- host2 = {
- "href" : "/api/v1/hosts/host2",
- "Hosts" : {
- "cpu_count" : 1,
- "host_name" : "host2",
- "os_arch" : "x86_64",
- "os_type" : "centos6",
- "ph_cpu_count" : 1,
- "public_host_name" : "host2",
- "rack_info" : "/default-rack",
- "total_mem" : 2097152,
- "disk_info": [
- {
- "available": str(15<<30), # 15 GB
- "type": "ext4",
- "mountpoint": "/"
- }
- ]
- }
- }
-
- hosts = {
- "items" : [
- host1, host2
- ]
- }
-
- services = {
- "services": [
- {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["host1", "host2"]
- }
- }, {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["host1", "host2"]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "HDFS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "DATANODE",
- "hostnames": ["host1"]
- }
- }
- ]
- }
- ],
- "configurations": configurations
- }
- # only 1 partition, enough disk space, no warnings
- res = self.stackAdvisor.validateAmsSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
- expected = [{'config-name': 'timeline.metrics.service.operation.mode',
- 'config-type': 'ams-site',
- 'level': 'ERROR',
- 'message': "Correct value should be 'distributed' for clusters with more then 1 Metrics collector",
- 'type': 'configuration'}]
- self.assertEquals(res, expected)
-
-
- services = {
- "services": [
- {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["host1"]
- }
- }, {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["host1"]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "HDFS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "DATANODE",
- "hostnames": ["host1"]
- }
- }
- ]
- }
- ],
- "configurations": configurations
- }
- res = self.stackAdvisor.validateAmsSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
- expected = []
- self.assertEquals(res, expected)
-
- def test_validateAmsHbaseSiteConfigurations(self):
- configurations = {
- "hdfs-site": {
- "properties": {
- 'dfs.datanode.data.dir': "/hadoop/data"
- }
- },
- "core-site": {
- "properties": {
- "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020"
- }
- },
- "ams-site": {
- "properties": {
- "timeline.metrics.service.operation.mode": "embedded"
- }
- }
- }
-
- recommendedDefaults = {
- 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
- 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
- 'hbase.cluster.distributed': 'false'
- }
- properties = {
- 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
- 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
- 'hbase.cluster.distributed': 'false'
- }
- host = {
- "href" : "/api/v1/hosts/host1",
- "Hosts" : {
- "cpu_count" : 1,
- "host_name" : "host1",
- "os_arch" : "x86_64",
- "os_type" : "centos6",
- "ph_cpu_count" : 1,
- "public_host_name" : "host1",
- "rack_info" : "/default-rack",
- "total_mem" : 2097152,
- "disk_info": [
- {
- "available": str(15<<30), # 15 GB
- "type": "ext4",
- "mountpoint": "/"
- }
- ]
- }
- }
-
- hosts = {
- "items" : [
- host
- ]
- }
-
- services = {
- "services": [
- {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["host1"]
- }
- }, {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["host1"]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "HDFS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "DATANODE",
- "hostnames": ["host1"]
- }
- }
- ]
- }
- ],
- "configurations": configurations
- }
-
- # only 1 partition, enough disk space, no warnings
- res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
- expected = []
- self.assertEquals(res, expected)
-
-
- # 1 partition, no enough disk space
- host['Hosts']['disk_info'] = [
- {
- "available" : '1',
- "type" : "ext4",
- "mountpoint" : "/"
- }
- ]
- res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
- expected = [
- {'config-name': 'hbase.rootdir',
- 'config-type': 'ams-hbase-site',
- 'level': 'WARN',
- 'message': 'Ambari Metrics disk space requirements not met. '
- '\nRecommended disk space for partition / is 10G',
- 'type': 'configuration'
- }
- ]
- self.assertEquals(res, expected)
-
- # 2 partitions
- host['Hosts']['disk_info'] = [
- {
- "available": str(15<<30), # 15 GB
- "type" : "ext4",
- "mountpoint" : "/grid/0"
- },
- {
- "available" : str(15<<30), # 15 GB
- "type" : "ext4",
- "mountpoint" : "/"
- }
- ]
- recommendedDefaults = {
- 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
- 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
- 'hbase.cluster.distributed': 'false'
- }
- properties = {
- 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
- 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
- 'hbase.cluster.distributed': 'false'
- }
- res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
- expected = []
- self.assertEquals(res, expected)
-
- # dfs.dir & hbase.rootdir crosscheck + root partition + hbase.rootdir == hbase.tmp.dir warnings
- properties = {
- 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
- 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
- 'hbase.cluster.distributed': 'false'
- }
-
- res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
- expected = [
- {
- 'config-name': 'hbase.rootdir',
- 'config-type': 'ams-hbase-site',
- 'level': 'WARN',
- 'message': 'It is not recommended to use root partition for hbase.rootdir',
- 'type': 'configuration'
- },
- {
- 'config-name': 'hbase.tmp.dir',
- 'config-type': 'ams-hbase-site',
- 'level': 'WARN',
- 'message': 'Consider not using / partition for storing metrics temporary data. '
- '/ partition is already used as hbase.rootdir to store metrics data',
- 'type': 'configuration'
- },
- {
- 'config-name': 'hbase.rootdir',
- 'config-type': 'ams-hbase-site',
- 'level': 'WARN',
- 'message': 'Consider not using / partition for storing metrics data. '
- '/ is already used by datanode to store HDFS data',
- 'type': 'configuration'
- }
- ]
- self.assertEquals(res, expected)
-
- # incorrect hbase.rootdir in distributed mode
- properties = {
- 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
- 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
- 'hbase.cluster.distributed': 'false'
- }
- configurations['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed'
- res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts)
- expected = [
- {
- 'config-name': 'hbase.rootdir',
- 'config-type': 'ams-hbase-site',
- 'level': 'WARN',
- 'message': 'In distributed mode hbase.rootdir should point to HDFS.',
- 'type': 'configuration'
- },
- {
- 'config-name': 'hbase.cluster.distributed',
- 'config-type': 'ams-hbase-site',
- 'level': 'ERROR',
- 'message': 'hbase.cluster.distributed property should be set to true for distributed mode',
- 'type': 'configuration'
- }
- ]
- self.assertEquals(res, expected)
def test_validateStormSiteConfigurations(self):
configurations = {
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f32765d/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index d6b572e..6f9e18e 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2300,517 +2300,6 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
- def test_recommendAmsConfigurations(self):
- configurations = {}
- clusterData = {}
-
- services = {
- "services": [ {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [{
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["host1"]
- }
-
- }, {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["host1"]
- }
-
- }]
- }],
- "configurations": []
- }
- hosts = {
- "items": [{
- "Hosts": {
- "host_name": "host1",
-
- }
- }]
- }
-
- # 1-node cluster
- expected = {
- "ams-hbase-env": {
- "properties": {
- "hbase_master_xmn_size": "128",
- "hbase_master_heapsize": "512",
- "hbase_regionserver_heapsize": "512"
- }
- },
- "ams-grafana-env": {
- "properties" : {},
- "property_attributes": {
- "metrics_grafana_password": {
- "visible": "false"
- }
- }
- },
- "ams-env": {
- "properties": {
- "metrics_collector_heapsize": "512",
- }
- },
- "ams-hbase-site": {
- "properties": {
- "phoenix.coprocessor.maxMetaDataCacheSize": "20480000",
- "hbase.regionserver.global.memstore.lowerLimit": "0.3",
- "hbase.regionserver.global.memstore.upperLimit": "0.35",
- "hbase.hregion.memstore.flush.size": "134217728",
- "hfile.block.cache.size": "0.3",
- "hbase.cluster.distributed": "false",
- "hbase.rootdir": "file:///var/lib/ambari-metrics-collector/hbase",
- "hbase.tmp.dir": "/var/lib/ambari-metrics-collector/hbase-tmp",
- "hbase.zookeeper.property.clientPort": "61181",
- }
- },
- "ams-site": {
- "properties": {
- "timeline.metrics.cluster.aggregate.splitpoints": "mem_buffered",
- "timeline.metrics.host.aggregate.splitpoints": "mem_buffered",
- "timeline.metrics.service.handler.thread.count": "20",
- 'timeline.metrics.service.webapp.address': '0.0.0.0:6188',
- 'timeline.metrics.service.watcher.disabled': 'false',
- 'timeline.metrics.cache.size': '100',
- 'timeline.metrics.cache.commit.interval': '10'
- }
- }
- }
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
-
- # 100-nodes cluster, but still only 1 sink (METRICS_COLLECTOR)
- for i in range(2, 201):
- hosts['items'].extend([{
- "Hosts": {
- "host_name": "host" + str(i)
- }
- }])
-
- services['services'] = [
- {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["host1"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["host" + str(i) for i in range(1, 201)]
- }
- }
- ]
- }
- ]
-
- expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '500'
- expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '7'
- expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2560'
- expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '448'
- expected["ams-env"]['properties']['metrics_collector_heapsize'] = '896'
-
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
-
- # 200 nodes, but with HDFS and YARN services installed on all nodes
- services['services'] = [
- {
- "StackServices": {
- "service_name": "HDFS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "NAMENODE",
- "hostnames": ["host1"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "DATANODE",
- "hostnames": ["host" + str(i) for i in range(1, 201)]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "YARN"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "RESOURCEMANAGER",
- "hostnames": ["host1"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "NODEMANAGER",
- "hostnames": ["host" + str(i) for i in range(1, 201)]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["host1"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["host" + str(i) for i in range(1, 201)]
- }
- }
- ]
- }
-
- ]
- expected["ams-site"]['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'dfs.FSNamesystem.FilesTotal,' \
- 'dfs.datanode.WritesFromRemoteClient,' \
- 'ipc.IPC.numCallsInReplicationQueue,' \
- 'mapred.ShuffleMetrics.ShuffleOutputsFailed,' \
- 'mem_buffered,' \
- 'read_count,' \
- 'regionserver.Server.percentFilesLocal,' \
- 'rpcdetailed.rpcdetailed.RegisterNodeManagerNumOps,' \
- 'sdisk_vdb_write_count'
- expected["ams-site"]['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total'
-
- expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '600'
- expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '6'
- expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '6656'
- expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '1088'
- expected["ams-env"]['properties']['metrics_collector_heapsize'] = '2176'
-
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
-
- # Test splitpoints, AMS embedded mode
- services['changed-configurations'] = [
- {
- "type": "ams-hbase-env",
- "name": "hbase_master_heapsize",
- "old_value": "1024"
- }
- ]
-
- services['configurations'] = {
- 'core-site': {'properties': {}},
- 'ams-site': {'properties': {}},
- 'ams-hbase-site': {'properties': {}},
- 'ams-hbase-env': {'properties': {}}
- }
-
- # Embedded mode, 512m master heapsize, no splitpoints recommended
- services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512'
- services["configurations"]['ams-hbase-site']['properties']['hbase.regionserver.global.memstore.upperLimit'] = '0.4'
- services["configurations"]['ams-hbase-site']['properties']['hbase.hregion.memstore.flush.size'] = '134217728'
-
- expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'mem_total'
- expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total'
- expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512'
-
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
-
- # Embedded mode, 4096m master heapsize, some splitpoints recommended
- services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '4096'
- expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'dfs.namenode.BlockReportAvgTime,' \
- 'master.AssignmentManger.Assign_mean,' \
- 'regionserver.Server.Append_median,' \
- 'rpcdetailed.rpcdetailed.client.CheckAccessNumOps'
- expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total'
- expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '4096'
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
-
-
- # Embedded mode, 8192m master heapsize, more splitpoints recommended
- services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192'
- expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192'
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'].split(',')), 13)
- self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'].split(',')), 2)
-
- # Test splitpoints, AMS distributed mode
- services['changed-configurations'] = [
- {
- "type": "ams-hbase-env",
- "name": "hbase_regionserver_heapsize",
- "old_value": "512"
- }
- ]
- services["configurations"]['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed'
- services["configurations"]["core-site"]["properties"]["fs.defaultFS"] = 'hdfs://host1:8020'
- expected['ams-hbase-site']['properties']['hbase.cluster.distributed'] = 'true'
- expected['ams-hbase-site']['properties']['hbase.rootdir'] = '/user/ams/hbase'
- expected['ams-hbase-site']['properties']['hbase.zookeeper.property.clientPort'] = '2181'
- expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512'
- expected['ams-hbase-site']['properties']['dfs.client.read.shortcircuit'] = 'true'
-
- # Distributed mode, low memory, no splitpoints recommended
- expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'mem_total'
- expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total'
- expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '6656'
- expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102'
- expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '1024'
- expected['ams-site']['properties']['timeline.metrics.service.watcher.disabled'] = 'true'
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
-
- # Distributed mode, more memory, more splitpoints recommended
- services["configurations"]['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '8192'
- expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '8192'
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'].split(',')), 13)
- self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'].split(',')), 2)
-
- # 2000-nodes cluster
- for i in range(202, 2001):
- hosts['items'].extend([{
- "Hosts": {
- "host_name": "host" + str(i)
- }
- }])
-
- services['services'] = [
- {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["host1"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["host" + str(i) for i in range(1, 2001)]
- }
- }
- ]
- }
- ]
-
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.size'], '700')
- self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.commit.interval'], '5')
-
- # 500 Nodes with HDFS, YARN, HIVE, STORM, HBASE, KAFKA, AMS
- node_count = 500
- hosts = {
- "items": []
- }
- for i in range(1, node_count):
- hosts['items'].extend([{
- "Hosts": {
- "host_name": "host" + str(i)
- }
- }])
-
- services['services'] = [
- {
- "StackServices": {
- "service_name": "HDFS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "NAMENODE",
- "hostnames": ["host1"]
- }
- } ,
- {
- "StackServiceComponents": {
- "component_name": "SECONDARY_NAMENODE",
- "hostnames": ["host2"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "DATANODE",
- "hostnames": ["host" + str(i) for i in range(6, node_count + 1)]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "HBASE"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "HBASE_MASTER",
- "hostnames": ["host3"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "HBASE_REGIONSERVER",
- "hostnames": ["host" + str(i) for i in range(6, node_count + 1)]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "YARN"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "RESOURCEMANAGER",
- "hostnames": ["host4"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "NODEMANAGER",
- "hostnames": ["host" + str(i) for i in range(6, node_count + 1)]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "HIVE"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "HIVE_METASTORE",
- "hostnames": ["host3"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "HIVE_SERVER",
- "hostnames": ["host3"]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "STORM"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "NIMBUS",
- "hostnames": ["host" + str(i) for i in range(1, 6)]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "KAFKA"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "KAFKA_BROKER",
- "hostnames": ["host" + str(i) for i in range(1, 6)]
- }
- }
- ]
- },
- {
- "StackServices": {
- "service_name": "AMBARI_METRICS"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "METRICS_COLLECTOR",
- "hostnames": ["host6"]
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "METRICS_MONITOR",
- "hostnames": ["host" + str(i) for i in range(6, node_count + 1)]
- }
- }
- ]
- }
- ]
-
- services['configurations'] = {
- 'core-site': {'properties': {}},
- 'ams-site': {'properties': {}},
- 'ams-hbase-site': {'properties': {}},
- 'ams-hbase-env': {'properties': {}}
- }
- services["configurations"]['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed'
-
- expected['ams-hbase-site']['properties']['hbase.cluster.distributed'] = 'true'
- expected['ams-hbase-site']['properties']['hbase.rootdir'] = '/user/ams/hbase'
- expected['ams-hbase-site']['properties']['hbase.zookeeper.property.clientPort'] = '2181'
-
- expected["ams-site"]['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'default.General.active_calls_api_get_all_databases,' \
- 'default.General.api_get_database_mean,' \
- 'default.General.gc.PS-MarkSweep.count,' \
- 'dfs.FsVolume.TotalDataFileIos,' \
- 'disk_free,' \
- 'jvm.JvmMetrics.MemHeapMaxM,' \
- 'kafka.network.RequestMetrics.RemoteTimeMs.request.Metadata.75percentile,' \
- 'kafka.network.RequestMetrics.ResponseQueueTimeMs.request.Update.Metadata.mean,' \
- 'load_one,master.FileSystem.MetaHlogSplitTime_75th_percentile,' \
- 'metricssystem.MetricsSystem.NumActiveSources,' \
- 'regionserver.Server.Append_95th_percentile,' \
- 'regionserver.Server.blockCacheEvictionCount,' \
- 'rpc.rpc.client.SentBytes,' \
- 'sdisk_vda1_write_bytes'
- expected["ams-site"]['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'ipc.IPC.authorizationSuccesses,' \
- 'metricssystem.MetricsSystem.PublishNumOps'
-
- expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '700'
- expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '5'
- expected["ams-site"]['properties']['timeline.metrics.service.resultset.fetchSize'] = '5000'
- expected["ams-site"]['properties']['phoenix.query.maxGlobalMemoryPercentage'] = '30'
-
- expected["ams-env"]['properties']['metrics_collector_heapsize'] = '7040'
-
- expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '512'
- expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102'
-
- expected["ams-hbase-env"]['properties']['hbase_regionserver_heapsize'] = '21120'
- expected["ams-hbase-env"]['properties']['regionserver_xmn_size'] = '3200'
-
- expected["ams-hbase-site"]['properties']['phoenix.query.maxGlobalMemoryPercentage'] = '20'
- expected['ams-hbase-site']['properties']['hbase.hregion.memstore.flush.size'] = '268435456'
- expected['ams-hbase-site']['properties']['hbase.regionserver.handler.count'] = '60'
- expected['ams-hbase-site']['properties']['hbase.regionserver.hlog.blocksize'] = '134217728'
- expected['ams-hbase-site']['properties']['hbase.regionserver.maxlogs'] = '64'
- expected['ams-hbase-site']['properties']['phoenix.coprocessor.maxMetaDataCacheSize'] = '40960000'
-
- self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
def test_recommendHbaseConfigurations(self):
servicesList = ["HBASE"]