You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/07/06 13:10:17 UTC
[24/50] [abbrv] ambari git commit: AMBARI-21333 Extract Log Search
part from stack advisor to a separate service advisor (mgergely)
AMBARI-21333 Extract Log Search part from stack advisor to a separate service advisor (mgergely)
Change-Id: Ia65deffc6d72bf0202ab4cff130f1feb22970a4d
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8ba5c3e3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8ba5c3e3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8ba5c3e3
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 8ba5c3e3ae582820828b345bb85da7aa935f637e
Parents: aba2db1
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Tue Jul 4 15:20:52 2017 +0200
Committer: Miklos Gergely <mg...@hortonworks.com>
Committed: Tue Jul 4 15:20:52 2017 +0200
----------------------------------------------------------------------
.../AMBARI_INFRA/0.1.0/service_advisor.py | 133 +++++++++++++
.../LOGSEARCH/0.5.0/service_advisor.py | 188 +++++++++++++++++++
.../stacks/HDP/2.2/services/stack_advisor.py | 66 -------
.../LOGSEARCH/test_service_advisor.py | 164 ++++++++++++++++
.../stacks/2.3/common/test_stack_advisor.py | 123 ------------
5 files changed, 485 insertions(+), 189 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/service_advisor.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/service_advisor.py
new file mode 100644
index 0000000..9b6aace
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/service_advisor.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Python imports
+import imp
+import os
+import traceback
+import re
+import socket
+import fnmatch
+
+
+from resource_management.core.logger import Logger
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+class Ambari_InfraServiceAdvisor(service_advisor.ServiceAdvisor):
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(Ambari_InfraServiceAdvisor, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ # Always call these methods
+ self.modifyMastersWithMultipleInstances()
+ self.modifyCardinalitiesDict()
+ self.modifyHeapSizeProperties()
+ self.modifyNotValuableComponents()
+ self.modifyComponentsNotPreferableOnServer()
+ self.modifyComponentLayoutSchemes()
+
+ def modifyMastersWithMultipleInstances(self):
+ """
+ Modify the set of masters with multiple instances.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyCardinalitiesDict(self):
+ """
+ Modify the dictionary of cardinalities.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyHeapSizeProperties(self):
+ """
+ Modify the dictionary of heap size properties.
+ Must be overriden in child class.
+ """
+ pass
+
+ def modifyNotValuableComponents(self):
+ """
+ Modify the set of components whose host assignment is based on other services.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentsNotPreferableOnServer(self):
+ """
+ Modify the set of components that are not preferable on the server.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentLayoutSchemes(self):
+ """
+ Modify layout scheme dictionaries for components.
+ The scheme dictionary basically maps the number of hosts to
+ host index where component should exist.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def getServiceComponentLayoutValidations(self, services, hosts):
+ """
+ Get a list of errors.
+ Must be overriden in child class.
+ """
+
+ return []
+
+ def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
+ """
+ Entry point.
+ Validate configurations for the service. Return a list of errors.
+ The code for this function should be the same for each Service Advisor.
+ """
+ #Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ pass
+
+ def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
+ """
+ Entry point.
+ Validate configurations for the service. Return a list of errors.
+ The code for this function should be the same for each Service Advisor.
+ """
+ #Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ return []
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
new file mode 100644
index 0000000..6a61b07
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Python imports
+import imp
+import os
+import traceback
+import re
+import socket
+import fnmatch
+import math
+
+
+from resource_management.core.logger import Logger
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+class LogSearchServiceAdvisor(service_advisor.ServiceAdvisor):
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(LogSearchServiceAdvisor, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ # Always call these methods
+ self.modifyMastersWithMultipleInstances()
+ self.modifyCardinalitiesDict()
+ self.modifyHeapSizeProperties()
+ self.modifyNotValuableComponents()
+ self.modifyComponentsNotPreferableOnServer()
+ self.modifyComponentLayoutSchemes()
+
+ def modifyMastersWithMultipleInstances(self):
+ """
+ Modify the set of masters with multiple instances.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyCardinalitiesDict(self):
+ """
+ Modify the dictionary of cardinalities.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyHeapSizeProperties(self):
+ """
+ Modify the dictionary of heap size properties.
+ Must be overriden in child class.
+ """
+ pass
+
+ def modifyNotValuableComponents(self):
+ """
+ Modify the set of components whose host assignment is based on other services.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentsNotPreferableOnServer(self):
+ """
+ Modify the set of components that are not preferable on the server.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentLayoutSchemes(self):
+ """
+ Modify layout scheme dictionaries for components.
+ The scheme dictionary basically maps the number of hosts to
+ host index where component should exist.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def getServiceComponentLayoutValidations(self, services, hosts):
+ """
+ Get a list of errors.
+ Must be overriden in child class.
+ """
+
+ return []
+
+ def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
+ putLogSearchProperty = self.putProperty(configurations, "logsearch-properties", services)
+ putLogSearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties")
+ putLogSearchCommonEnvProperty = self.putProperty(configurations, "logsearch-common-env", services)
+ putLogSearchCommonEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-common-env")
+ putLogSearchEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-env")
+ putLogFeederEnvAttribute = self.putPropertyAttribute(configurations, "logfeeder-env")
+
+ logSearchServerHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SERVER")
+ if logSearchServerHosts is None or len(logSearchServerHosts) == 0:
+ for key in services['configurations']['logsearch-env']['properties']:
+ putLogSearchEnvAttribute(key, 'visible', 'false')
+ for key in services['configurations']['logsearch-properties']['properties']:
+ if key not in ['logsearch.collection.service.logs.numshards', 'logsearch.collection.audit.logs.numshards',
+ 'logsearch.solr.collection.service.logs', 'logsearch.solr.collection.audit.logs',
+ 'logsearch.service.logs.split.interval.mins', 'logsearch.audit.logs.split.interval.mins']:
+ putLogSearchAttribute(key, 'visible', 'false')
+ for key in services['configurations']['logsearch-audit_logs-solrconfig']['properties']:
+ self.putPropertyAttribute(configurations, "logsearch-audit_logs-solrconfig")(key, 'visible', 'false')
+ for key in services['configurations']['logsearch-service_logs-solrconfig']['properties']:
+ self.putPropertyAttribute(configurations, "logsearch-service_logs-solrconfig")(key, 'visible', 'false')
+ for key in services['configurations']['logsearch-log4j']['properties']:
+ self.putPropertyAttribute(configurations, "logsearch-log4j")(key, 'visible', 'false')
+
+ putLogSearchProperty("logsearch.collection.service.logs.numshards", 2)
+ putLogSearchProperty("logsearch.collection.audit.logs.numshards", 2)
+ else:
+ infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR")
+ if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]:
+ replicationReccomendFloat = math.log(len(infraSolrHosts), 5)
+ recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat))
+
+ recommendedMinShards = len(infraSolrHosts)
+ recommendedShards = 2 * len(infraSolrHosts)
+ recommendedMaxShards = 3 * len(infraSolrHosts)
+ else:
+ recommendedReplicationFactor = 2
+
+ recommendedMinShards = 1
+ recommendedShards = 1
+ recommendedMaxShards = 100
+
+ putLogSearchCommonEnvProperty('logsearch_use_external_solr', 'true')
+
+ # recommend number of shard
+ putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
+ putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
+ putLogSearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
+
+ putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
+ putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
+ putLogSearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
+ # recommend replication factor
+ putLogSearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
+ putLogSearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
+
+ kerberos_authentication_enabled = self.isSecurityEnabled(services)
+ if not kerberos_authentication_enabled:
+ putLogSearchCommonEnvProperty('logsearch_external_solr_kerberos_enabled', 'false')
+ putLogSearchCommonEnvAttribute('logsearch_external_solr_kerberos_enabled', 'visible', 'false')
+ putLogSearchEnvAttribute('logsearch_external_solr_kerberos_keytab', 'visible', 'false')
+ putLogSearchEnvAttribute('logsearch_external_solr_kerberos_principal', 'visible', 'false')
+ putLogFeederEnvAttribute('logfeeder_external_solr_kerberos_keytab', 'visible', 'false')
+ putLogFeederEnvAttribute('logfeeder_external_solr_kerberos_principal', 'visible', 'false')
+
+ def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
+ """
+ Entry point.
+ Validate configurations for the service. Return a list of errors.
+ The code for this function should be the same for each Service Advisor.
+ """
+ #Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ return []
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 726514b..2dc1738 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -96,7 +96,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
"STORM": self.recommendStormConfigurations,
"KNOX": self.recommendKnoxConfigurations,
"RANGER": self.recommendRangerConfigurations,
- "LOGSEARCH" : self.recommendLogsearchConfigurations,
"SPARK": self.recommendSparkConfigurations,
"KAFKA": self.recommendKafkaConfigurations,
}
@@ -1054,71 +1053,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
self.mergeValidators(parentValidators, childValidators)
return parentValidators
- def recommendLogsearchConfigurations(self, configurations, clusterData, services, hosts):
- putLogsearchProperty = self.putProperty(configurations, "logsearch-properties", services)
- putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties")
- putLogsearchCommonEnvProperty = self.putProperty(configurations, "logsearch-common-env", services)
- putLogsearchCommonEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-common-env")
- putLogsearchEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-env")
- putLogfeederEnvAttribute = self.putPropertyAttribute(configurations, "logfeeder-env")
-
- logSearchServerHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SERVER")
- if logSearchServerHosts is None or len(logSearchServerHosts) == 0:
- for key in services['configurations']['logsearch-env']['properties']:
- putLogsearchEnvAttribute(key, 'visible', 'false')
- for key in services['configurations']['logsearch-properties']['properties']:
- if key not in ['logsearch.collection.service.logs.numshards', 'logsearch.collection.audit.logs.numshards',
- 'logsearch.solr.collection.service.logs', 'logsearch.solr.collection.audit.logs',
- 'logsearch.service.logs.split.interval.mins', 'logsearch.audit.logs.split.interval.mins']:
- putLogsearchAttribute(key, 'visible', 'false')
- for key in services['configurations']['logsearch-audit_logs-solrconfig']['properties']:
- self.putPropertyAttribute(configurations, "logsearch-audit_logs-solrconfig")(key, 'visible', 'false')
- for key in services['configurations']['logsearch-service_logs-solrconfig']['properties']:
- self.putPropertyAttribute(configurations, "logsearch-service_logs-solrconfig")(key, 'visible', 'false')
- for key in services['configurations']['logsearch-log4j']['properties']:
- self.putPropertyAttribute(configurations, "logsearch-log4j")(key, 'visible', 'false')
-
- putLogsearchProperty("logsearch.collection.service.logs.numshards", 2)
- putLogsearchProperty("logsearch.collection.audit.logs.numshards", 2)
- else:
- infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR")
- if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]:
- replicationReccomendFloat = math.log(len(infraSolrHosts), 5)
- recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat))
-
- recommendedMinShards = len(infraSolrHosts)
- recommendedShards = 2 * len(infraSolrHosts)
- recommendedMaxShards = 3 * len(infraSolrHosts)
- else:
- recommendedReplicationFactor = 2
-
- recommendedMinShards = 1
- recommendedShards = 1
- recommendedMaxShards = 100
-
- putLogsearchCommonEnvProperty('logsearch_use_external_solr', 'true')
-
- # recommend number of shard
- putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
- putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
- putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
-
- putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
- putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
- putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
- # recommend replication factor
- putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
- putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
-
- kerberos_authentication_enabled = self.isSecurityEnabled(services)
- if not kerberos_authentication_enabled:
- putLogsearchCommonEnvProperty('logsearch_external_solr_kerberos_enabled', 'false')
- putLogsearchCommonEnvAttribute('logsearch_external_solr_kerberos_enabled', 'visible', 'false')
- putLogsearchEnvAttribute('logsearch_external_solr_kerberos_keytab', 'visible', 'false')
- putLogsearchEnvAttribute('logsearch_external_solr_kerberos_principal', 'visible', 'false')
- putLogfeederEnvAttribute('logfeeder_external_solr_kerberos_keytab', 'visible', 'false')
- putLogfeederEnvAttribute('logfeeder_external_solr_kerberos_principal', 'visible', 'false')
-
def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
{"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')},
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py b/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
new file mode 100644
index 0000000..3edb03c
--- /dev/null
+++ b/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
@@ -0,0 +1,164 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import imp
+import json
+import os
+from unittest import TestCase
+
+from mock.mock import patch, MagicMock
+
+
+class TestLOGSEARCH050ServiceAdvisor(TestCase):
+
+ testDirectory = os.path.dirname(os.path.abspath(__file__))
+ stack_advisor_path = os.path.join(testDirectory, '../../../../main/resources/stacks/stack_advisor.py')
+ with open(stack_advisor_path, 'rb') as fp:
+ imp.load_module('stack_advisor', fp, stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE))
+
+ serviceAdvisorPath = '../../../../main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py'
+ logserch050ServiceAdvisorPath = os.path.join(testDirectory, serviceAdvisorPath)
+ with open(logserch050ServiceAdvisorPath, 'rb') as fp:
+ service_advisor_impl = imp.load_module('service_advisor_impl', fp, logserch050ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+
+ def setUp(self):
+ serviceAdvisorClass = getattr(self.service_advisor_impl, 'LogSearchServiceAdvisor')
+ self.serviceAdvisor = serviceAdvisorClass()
+
+ def test_recommendLogsearchConfiguration(self):
+ configurations = {
+ "logsearch-properties": {
+ "properties": {
+ "logsearch.collection.service.logs.numshards" : "5",
+ "logsearch.collection.service.logs.replication.factor": "0",
+ "logsearch.collection.audit.logs.numshards" : "5",
+ "logsearch.collection.audit.logs.replication.factor": "0"
+ }
+ }
+ }
+
+ clusterData = {
+ "cpu": 4,
+ "mapMemory": 3000,
+ "amMemory": 2000,
+ "reduceMemory": 2056,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+ expected = {
+ 'logfeeder-env': {
+ 'property_attributes': {
+ 'logfeeder_external_solr_kerberos_keytab': {'visible': 'false'
+ },
+ 'logfeeder_external_solr_kerberos_principal': {'visible': 'false'}
+ }
+ },
+ 'logsearch-common-env': {
+ 'properties': {
+ 'logsearch_external_solr_kerberos_enabled': 'false'
+ },
+ 'property_attributes': {
+ 'logsearch_external_solr_kerberos_enabled': {'visible': 'false'}
+ }
+ },
+ 'logsearch-env': {
+ 'property_attributes': {
+ 'logsearch_external_solr_kerberos_keytab': {'visible': 'false'},
+ 'logsearch_external_solr_kerberos_principal': {'visible': 'false'}
+ }
+ },
+ 'logsearch-properties': {
+ 'properties': {
+ "logsearch.collection.service.logs.numshards" : "2",
+ "logsearch.collection.service.logs.replication.factor": "1",
+ "logsearch.collection.audit.logs.numshards" : "2",
+ "logsearch.collection.audit.logs.replication.factor": "1"
+ },
+ "property_attributes": {
+ "logsearch.collection.service.logs.numshards": {
+ "minimum": "1",
+ "maximum": "3"
+ },
+ "logsearch.collection.audit.logs.numshards": {
+ "minimum": "1",
+ "maximum": "3"
+ }
+ }
+ }
+ }
+ services = {
+ "services": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.3/services/AMBARI_INFRA",
+ "StackServices": {
+ "service_name": "AMBARI_INFRA",
+ "service_version": "2.6.0.2.2",
+ "stack_name": "HDP",
+ "stack_version": "2.3"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1",
+ "component_category": "MASTER",
+ "component_name": "INFRA_SOLR",
+ "display_name": "Infra Solr Instance",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ }
+ ]
+ },
+ ],
+ "configurations": {
+ "logsearch-properties": {
+ "properties": {
+ "logsearch.collection.numshards" : "5",
+ "logsearch.collection.replication.factor": "0"
+ }
+ }
+ },
+ "changed-configurations": [ ]
+
+ }
+
+ hosts = {
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6401.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ }
+ ]
+ }
+ def return_c6401_hostname(services, service_name, component_name):
+ return ["c6401.ambari.apache.org"]
+ self.serviceAdvisor.getComponentHostNames = return_c6401_hostname
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index c28ee2a..9e882d2 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1948,129 +1948,6 @@ class TestHDP23StackAdvisor(TestCase):
self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
- def test_recommendLogsearchConfiguration(self):
- configurations = {
- "logsearch-properties": {
- "properties": {
- "logsearch.collection.service.logs.numshards" : "5",
- "logsearch.collection.service.logs.replication.factor": "0",
- "logsearch.collection.audit.logs.numshards" : "5",
- "logsearch.collection.audit.logs.replication.factor": "0"
- }
- }
- }
-
- clusterData = {
- "cpu": 4,
- "mapMemory": 3000,
- "amMemory": 2000,
- "reduceMemory": 2056,
- "containers": 3,
- "ramPerContainer": 256
- }
- expected = {
- 'logfeeder-env': {
- 'property_attributes': {
- 'logfeeder_external_solr_kerberos_keytab': {'visible': 'false'
- },
- 'logfeeder_external_solr_kerberos_principal': {'visible': 'false'}
- }
- },
- 'logsearch-common-env': {
- 'properties': {
- 'logsearch_external_solr_kerberos_enabled': 'false'
- },
- 'property_attributes': {
- 'logsearch_external_solr_kerberos_enabled': {'visible': 'false'}
- }
- },
- 'logsearch-env': {
- 'property_attributes': {
- 'logsearch_external_solr_kerberos_keytab': {'visible': 'false'},
- 'logsearch_external_solr_kerberos_principal': {'visible': 'false'}
- }
- },
- 'logsearch-properties': {
- 'properties': {
- "logsearch.collection.service.logs.numshards" : "2",
- "logsearch.collection.service.logs.replication.factor": "1",
- "logsearch.collection.audit.logs.numshards" : "2",
- "logsearch.collection.audit.logs.replication.factor": "1"
- },
- "property_attributes": {
- "logsearch.collection.service.logs.numshards": {
- "minimum": "1",
- "maximum": "3"
- },
- "logsearch.collection.audit.logs.numshards": {
- "minimum": "1",
- "maximum": "3"
- }
- }
- }
- }
- services = {
- "services": [
- {
- "href": "/api/v1/stacks/HDP/versions/2.3/services/AMBARI_INFRA",
- "StackServices": {
- "service_name": "AMBARI_INFRA",
- "service_version": "2.6.0.2.2",
- "stack_name": "HDP",
- "stack_version": "2.3"
- },
- "components": [
- {
- "StackServiceComponents": {
- "advertise_version": "false",
- "cardinality": "1",
- "component_category": "MASTER",
- "component_name": "INFRA_SOLR",
- "display_name": "Infra Solr Instance",
- "is_client": "false",
- "is_master": "true",
- "hostnames": []
- },
- "dependencies": []
- }
- ]
- },
- ],
- "configurations": {
- "logsearch-properties": {
- "properties": {
- "logsearch.collection.numshards" : "5",
- "logsearch.collection.replication.factor": "0"
- }
- }
- },
- "changed-configurations": [ ]
-
- }
-
- hosts = {
- "items" : [
- {
- "href" : "/api/v1/hosts/c6401.ambari.apache.org",
- "Hosts" : {
- "cpu_count" : 1,
- "host_name" : "c6401.ambari.apache.org",
- "os_arch" : "x86_64",
- "os_type" : "centos6",
- "ph_cpu_count" : 1,
- "public_host_name" : "c6401.ambari.apache.org",
- "rack_info" : "/default-rack",
- "total_mem" : 1922680
- }
- }
- ]
- }
- def return_c6401_hostname(services, service_name, component_name):
- return ["c6401.ambari.apache.org"]
- self.stackAdvisor.getComponentHostNames = return_c6401_hostname
- self.stackAdvisor.recommendLogsearchConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
-
def test_validateRangerConfigurationsEnv(self):
properties = {
"ranger-kafka-plugin-enabled": "Yes",