You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mi...@apache.org on 2016/06/07 07:20:32 UTC
ambari git commit: AMBARI-16663: Refactor service_advisor apis to
remove passing of stack_advisor (Lav Jain via mithmatt)
Repository: ambari
Updated Branches:
refs/heads/trunk 86bf4dcaa -> cee5c4215
AMBARI-16663: Refactor service_advisor apis to remove passing of stack_advisor (Lav Jain via mithmatt)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cee5c421
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cee5c421
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cee5c421
Branch: refs/heads/trunk
Commit: cee5c421599a8e21bccd95f538186ee651d3b90d
Parents: 86bf4dc
Author: Matt <mm...@pivotal.io>
Authored: Tue Jun 7 00:18:27 2016 -0700
Committer: Matt <mm...@pivotal.io>
Committed: Tue Jun 7 00:18:27 2016 -0700
----------------------------------------------------------------------
.../HAWQ/2.0.0/service_advisor.py | 58 ++---
.../PXF/3.0.0/service_advisor.py | 17 +-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 75 ------
.../main/resources/stacks/service_advisor.py | 169 +------------
.../src/main/resources/stacks/stack_advisor.py | 241 +++++++++++--------
.../stacks/2.3/HAWQ/test_service_advisor.py | 44 ++--
.../stacks/2.3/PXF/test_service_advisor.py | 19 +-
.../stacks/2.3/common/test_stack_advisor.py | 40 +--
8 files changed, 225 insertions(+), 438 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
index 2c78bf1..f0e8b33 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
@@ -36,7 +36,7 @@ except Exception as e:
class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
- def getHostsForMasterComponent(self, stackAdvisor, services, hosts, component, hostsList, hostsComponentsMap):
+ def getHostsForMasterComponent(self, services, hosts, component, hostsList):
if component["StackServiceComponents"]["component_name"] == 'HAWQSTANDBY':
# Do not recommend HAWQSTANDBY on single node cluster, or cluster with no active hosts
if len(hostsList) <= 1:
@@ -56,24 +56,21 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
return availableHosts[:1]
return [ambariServerHost]
- return stackAdvisor.getHostsForMasterComponent(services, hosts, component, hostsList, hostsComponentsMap)
+ return super(HAWQ200ServiceAdvisor, self).getHostsForMasterComponent(services, hosts, component, hostsList)
- def isComponentNotPreferableOnAmbariServerHost(self, componentName):
- return componentName in ('HAWQMASTER', 'HAWQSTANDBY')
+ def getNotPreferableOnServerComponents(self):
+ return ['HAWQMASTER', 'HAWQSTANDBY']
- def getComponentLayoutScheme(self, componentName):
- if componentName == 'HAWQMASTER':
- return {6: 2, 31: 1, "else": 5}
-
- if componentName == 'HAWQSTANDBY':
- return {6: 1, 31: 2, "else": 3}
-
- return None
+ def getComponentLayoutSchemes(self):
+ return {
+ 'HAWQMASTER': {6: 2, 31: 1, "else": 5},
+ 'HAWQSTANDBY': {6: 1, 31: 2, "else": 3}
+ }
- def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents):
+ def colocateService(self, hostsComponentsMap, serviceComponents):
# colocate HAWQSEGMENT with DATANODE, if no hosts have been allocated for HAWQSEGMENT
hawqSegment = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
- if not stackAdvisor.isComponentHostsPopulated(hawqSegment):
+ if not self.isComponentHostsPopulated(hawqSegment):
for hostName in hostsComponentsMap.keys():
hostComponents = hostsComponentsMap[hostName]
if {"name": "DATANODE"} in hostComponents and {"name": "HAWQSEGMENT"} not in hostComponents:
@@ -81,7 +78,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
if {"name": "DATANODE"} not in hostComponents and {"name": "HAWQSEGMENT"} in hostComponents:
hostComponents.remove({"name": "HAWQSEGMENT"})
- def getComponentLayoutValidations(self, stackAdvisor, services, hosts):
+ def getServiceComponentLayoutValidations(self, services, hosts):
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
@@ -107,13 +104,13 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
message = "HAWQ Master and HAWQ Standby Master cannot be deployed on the same host."
items.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
- if len(hawqMasterHosts) == 1 and hostsCount > 1 and stackAdvisor.isLocalHost(hawqMasterHosts[0]):
+ if len(hawqMasterHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]):
message = "The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. " \
"If you are using port 5432 for Postgres, you must either deploy the HAWQ Master on a different host " \
"or configure a different port for the HAWQ Masters in the HAWQ Configuration page."
items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } )
- if len(hawqStandbyHosts) == 1 and hostsCount > 1 and stackAdvisor.isLocalHost(hawqStandbyHosts[0]):
+ if len(hawqStandbyHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]):
message = "The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. " \
"If you are using port 5432 for Postgres, you must either deploy the HAWQ Standby Master on a different host " \
"or configure a different port for the HAWQ Masters in the HAWQ Configuration page."
@@ -121,13 +118,13 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
return items
- def isHawqMasterComponentOnAmbariServer(self, stackAdvisor, services):
+ def isHawqMasterComponentOnAmbariServer(self, services):
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
hawqMasterComponentHosts = [hostname for component in componentsList if component["StackServiceComponents"]["component_name"] in ("HAWQMASTER", "HAWQSTANDBY") for hostname in component["StackServiceComponents"]["hostnames"]]
- return any([stackAdvisor.isLocalHost(host) for host in hawqMasterComponentHosts])
+ return any([self.isLocalHost(host) for host in hawqMasterComponentHosts])
- def getServiceConfigurationRecommendations(self, stackAdvisor, configurations, clusterData, services, hosts):
+ def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services)
# Set dfs.allow.truncate to true
@@ -151,7 +148,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services)
# remove master port when master is colocated with Ambari server
- if self.isHawqMasterComponentOnAmbariServer(stackAdvisor, services) and "hawq_master_address_port" in hawq_site:
+ if self.isHawqMasterComponentOnAmbariServer(services) and "hawq_master_address_port" in hawq_site:
putHawqSiteProperty('hawq_master_address_port', '')
# update query limits if segments are deployed
@@ -170,7 +167,6 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
buckets = factor * numSegments
putHawqSiteProperty('default_hash_table_bucket_number', buckets)
-
# update YARN RM urls with the values from yarn-site if YARN is installed
if "YARN" in servicesList and "yarn-site" in services["configurations"]:
yarn_site = services["configurations"]["yarn-site"]["properties"]
@@ -180,7 +176,6 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
putHawqSiteProperty('hawq_rm_nvcore_limit_perseg', minHawqHostsCoreCount)
-
if "vm.overcommit_memory" in hawq_sysctl_env:
MEM_THRESHOLD = 33554432 # 32GB, minHawqHostsMemory is represented in kB
# Set the value for hawq_rm_memory_limit_perseg based on vm.overcommit value and RAM available on HAWQ Hosts
@@ -236,14 +231,14 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
def getHAWQYARNPropertyMapping(self):
return { "hawq_rm_yarn_address": "yarn.resourcemanager.address", "hawq_rm_yarn_scheduler_address": "yarn.resourcemanager.scheduler.address" }
- def getConfigurationsValidationItems(self, stackAdvisor, configurations, recommendedDefaults, services, hosts):
+ def getConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
siteName = "hawq-site"
method = self.validateHAWQSiteConfigurations
- items = self.validateConfigurationsForSite(stackAdvisor, configurations, recommendedDefaults, services, hosts, siteName, method)
+ items = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method)
siteName = "hdfs-client"
method = self.validateHAWQHdfsClientConfigurations
- resultItems = self.validateConfigurationsForSite(stackAdvisor, configurations, recommendedDefaults, services, hosts, siteName, method)
+ resultItems = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method)
items.extend(resultItems)
return items
@@ -270,13 +265,13 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
"item": self.getErrorItem(
"Multiple directories for " + display_name + " are not allowed.")})
- def validateHAWQSiteConfigurations(self, stackAdvisor, properties, recommendedDefaults, configurations, services, hosts):
+ def validateHAWQSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
hawq_site = properties
validationItems = []
# 1. Check if HAWQ master/standby port numbers don't conflict with Ambari ports. Both Ambari and HAWQ use postgres DB and 5432 port.
- if self.isHawqMasterComponentOnAmbariServer(stackAdvisor, services) and self.isHawqMasterPortConflict(configurations):
+ if self.isHawqMasterComponentOnAmbariServer(services) and self.isHawqMasterPortConflict(configurations):
prop_name = 'hawq_master_address_port'
validationItems.append({"config-name": prop_name,
"item": self.getWarnItem(
@@ -325,9 +320,9 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
message = "Default buckets for Hash Distributed tables parameter value should not be greater than the value of Virtual Segments Limit per Query (Total) parameter, currently set to {0}.".format(hawq_site["hawq_rm_nvseg_perquery_limit"])
validationItems.append({"config-name": "default_hash_table_bucket_number", "item": self.getErrorItem(message)})
- return stackAdvisor.toConfigurationValidationProblems(validationItems, "hawq-site")
+ return self.toConfigurationValidationProblems(validationItems, "hawq-site")
- def validateHAWQHdfsClientConfigurations(self, stackAdvisor, properties, recommendedDefaults, configurations, services, hosts):
+ def validateHAWQHdfsClientConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
hdfs_client = properties
validationItems = []
@@ -345,8 +340,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
message = "{0} should be set to true (checked) for clusters with more than {1} HAWQ Segments"
elif numSegments <= MIN_NUM_SEGMENT_THRESHOLD and value != 'FALSE':
message = "{0} should be set to false (unchecked) for clusters with {1} or less HAWQ Segments"
-
if message:
validationItems.append({"config-name": PROP_NAME, "item": self.getWarnItem(message.format(PROP_NAME, str(MIN_NUM_SEGMENT_THRESHOLD)))})
- return stackAdvisor.toConfigurationValidationProblems(validationItems, "hdfs-client")
+ return self.toConfigurationValidationProblems(validationItems, "hdfs-client")
http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py
index 9b34171..4089b75 100644
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py
@@ -33,10 +33,10 @@ except Exception as e:
class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor):
- def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents):
+ def colocateService(self, hostsComponentsMap, serviceComponents):
# colocate PXF with NAMENODE and DATANODE, if no hosts have been allocated for PXF
pxf = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "PXF"][0]
- if not stackAdvisor.isComponentHostsPopulated(pxf):
+ if not self.isComponentHostsPopulated(pxf):
for hostName in hostsComponentsMap.keys():
hostComponents = hostsComponentsMap[hostName]
if ({"name": "NAMENODE"} in hostComponents or {"name": "DATANODE"} in hostComponents) \
@@ -46,13 +46,12 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor):
and {"name": "PXF"} in hostComponents:
hostsComponentsMap[hostName].remove({"name": "PXF"})
- def getComponentLayoutValidations(self, stackAdvisor, services, hosts):
+ def getServiceComponentLayoutValidations(self, services, hosts):
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
pxfHosts = self.getHosts(componentsList, "PXF")
expectedPxfHosts = set(self.getHosts(componentsList, "NAMENODE") + self.getHosts(componentsList, "DATANODE"))
-
items = []
# Generate WARNING if any PXF is not colocated with NAMENODE or DATANODE
@@ -65,7 +64,7 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor):
return items
- def getServiceConfigurationRecommendations(self, stackAdvisor, configurations, clusterData, services, hosts):
+ def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
if "hbase-env" in services["configurations"]:
hbase_env = services["configurations"]["hbase-env"]["properties"]
if "content" in hbase_env:
@@ -77,7 +76,7 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor):
putHbaseEnvProperty = self.putProperty(configurations, "hbase-env", services)
putHbaseEnvProperty("content", content)
- def validatePXFHBaseEnvConfigurations(self, stackAdvisor, properties, recommendedDefaults, configurations, services, hosts):
+ def validatePXFHBaseEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
# Check if HBASE_CLASSPATH should has the location of pxf-hbase.jar
hbase_env = properties
validationItems = []
@@ -85,10 +84,10 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor):
message = "HBASE_CLASSPATH must contain the location of pxf-hbase.jar"
validationItems.append({"config-name": "content", "item": self.getWarnItem(message)})
- return stackAdvisor.toConfigurationValidationProblems(validationItems, "hbase-env")
+ return self.toConfigurationValidationProblems(validationItems, "hbase-env")
- def getConfigurationsValidationItems(self, stackAdvisor, configurations, recommendedDefaults, services, hosts):
+ def getConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
siteName = "hbase-env"
method = self.validatePXFHBaseEnvConfigurations
- items = self.validateConfigurationsForSite(stackAdvisor, configurations, recommendedDefaults, services, hosts, siteName, method)
+ items = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method)
return items
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 1dedf3d..aa62d1e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -93,72 +93,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"RANGER": self.recommendRangerConfigurations
}
- def putProperty(self, config, configType, services=None):
- userConfigs = {}
- changedConfigs = []
- # if services parameter, prefer values, set by user
- if services:
- if 'configurations' in services.keys():
- userConfigs = services['configurations']
- if 'changed-configurations' in services.keys():
- changedConfigs = services["changed-configurations"]
-
- if configType not in config:
- config[configType] = {}
- if"properties" not in config[configType]:
- config[configType]["properties"] = {}
- def appendProperty(key, value):
- # If property exists in changedConfigs, do not override, use user defined property
- if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
- config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
- else:
- config[configType]["properties"][key] = str(value)
- return appendProperty
-
- def updateProperty(self, config, configType, services=None):
- userConfigs = {}
- changedConfigs = []
- # if services parameter, prefer values, set by user
- if services:
- if 'configurations' in services.keys():
- userConfigs = services['configurations']
- if 'changed-configurations' in services.keys():
- changedConfigs = services["changed-configurations"]
-
- if configType not in config:
- config[configType] = {}
- if"properties" not in config[configType]:
- config[configType]["properties"] = {}
- def updatePropertyWithCallback(key, value, callback):
- # If property exists in changedConfigs, do not override, use user defined property
- if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
- config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
- else:
- # Give the callback an empty string if the mapping doesn't exist
- current_value = ""
- if key in config[configType]["properties"]:
- current_value = config[configType]["properties"][key]
-
- config[configType]["properties"][key] = callback(current_value, value)
- return updatePropertyWithCallback
-
- def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
- for changedConfig in changedConfigs:
- if changedConfig['type']==configType and changedConfig['name']==propertyName:
- return True
- return False
-
- def putPropertyAttribute(self, config, configType):
- if configType not in config:
- config[configType] = {}
- def appendPropertyAttribute(key, attribute, attributeValue):
- if "property_attributes" not in config[configType]:
- config[configType]["property_attributes"] = {}
- if key not in config[configType]["property_attributes"]:
- config[configType]["property_attributes"][key] = {}
- config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue)
- return appendPropertyAttribute
-
def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
putYarnProperty = self.putProperty(configurations, "yarn-site", services)
putYarnPropertyAttribute = self.putPropertyAttribute(configurations, "yarn-site")
@@ -1280,15 +1214,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return self.toConfigurationValidationProblems(validationItems, "ams-hbase-env")
- def validateServiceConfigurations(self, serviceName):
- return self.getServiceConfigurationValidators().get(serviceName, None)
-
- def getWarnItem(self, message):
- return {"level": "WARN", "message": message}
-
- def getErrorItem(self, message):
- return {"level": "ERROR", "message": message}
-
def getPreferredMountPoints(self, hostInfo):
# '/etc/resolv.conf', '/etc/hostname', '/etc/hosts' are docker specific mount points
http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/stacks/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/service_advisor.py b/ambari-server/src/main/resources/stacks/service_advisor.py
index 3d6c293..7a3c7e9 100644
--- a/ambari-server/src/main/resources/stacks/service_advisor.py
+++ b/ambari-server/src/main/resources/stacks/service_advisor.py
@@ -46,55 +46,12 @@ convention listed above.
For examples see: common-services/HAWQ/2.0.0/service_advisor.py
and common-services/PXF/3.0.0/service_advisor.py
"""
-class ServiceAdvisor(object):
- """
- Abstract class implemented by all service advisors.
- """
-
- """
- Provides a scheme for laying out a given component on different number of hosts.
- It should return a json object such as {6: 2, 31: 1, "else": 5}. If a service
- contains more than one component, it may have a statement such as:
-
- if componentName == 'HAWQMASTER':
- return {6: 2, 31: 1, "else": 5}
-
- return None
- """
- def getComponentLayoutScheme(self, componentName):
- return None
+from stack_advisor import DefaultStackAdvisor
+class ServiceAdvisor(DefaultStackAdvisor):
"""
- Returns the cardinality of the component as a json object with min and max attributes.
- Example: {"min": 1, "max": 1}
- """
- def getComponentCardinality(self, componentName):
- return {"min": 1, "max": 1}
-
- """
- Returns True if the component should avoid being configured on the ambari server host.
- """
- def isComponentNotPreferableOnAmbariServerHost(self, componentName):
- return False
-
- """
- Return True if the component is not considered valuable, otherwise returns False.
- Hosts which are only running non valuable components are considered to be not used.
- """
- def isComponentNotValuable(self, component):
- return False
-
- """
- Returns True if the component should use its cardinatity to determine its layout.
- """
- def isComponentUsingCardinalityForLayout(self, componentName):
- return False
-
- """
- Returns True if the component is a master component with multiple instances.
+ Abstract class implemented by all service advisors.
"""
- def isMasterComponentWithMultipleInstances(self, component):
- return False
"""
If any components of the service should be colocated with other services,
@@ -102,7 +59,7 @@ class ServiceAdvisor(object):
# colocate HAWQSEGMENT with DATANODE, if no hosts have been allocated for HAWQSEGMENT
hawqSegment = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
- if not stackAdvisor.isComponentHostsPopulated(hawqSegment):
+ if not self.isComponentHostsPopulated(hawqSegment):
for hostName in hostsComponentsMap.keys():
hostComponents = hostsComponentsMap[hostName]
if {"name": "DATANODE"} in hostComponents and {"name": "HAWQSEGMENT"} not in hostComponents:
@@ -110,7 +67,7 @@ class ServiceAdvisor(object):
if {"name": "DATANODE"} not in hostComponents and {"name": "HAWQSEGMENT"} in hostComponents:
hostComponents.remove({"name": "HAWQSEGMENT"})
"""
- def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents):
+ def colocateService(self, hostsComponentsMap, serviceComponents):
pass
"""
@@ -118,7 +75,7 @@ class ServiceAdvisor(object):
This should be similar to any of the recommendXXXXConfigurations functions in the stack_advisor.py
such as recommendYARNConfigurations().
"""
- def getServiceConfigurationRecommendations(self, stackAdvisor, configurations, clusterSummary, services, hosts):
+ def getServiceConfigurationRecommendations(self, configurations, clusterSummary, services, hosts):
pass
"""
@@ -126,7 +83,7 @@ class ServiceAdvisor(object):
This should detect validation issues which are different than those the stack_advisor.py detects.
The default validations are in stack_advisor.py getComponentLayoutValidations function.
"""
- def getComponentLayoutValidations(self, stackAdvisor, services, hosts):
+ def getServiceComponentLayoutValidations(self, services, hosts):
return []
"""
@@ -134,115 +91,5 @@ class ServiceAdvisor(object):
This should be similar to any of the validateXXXXConfigurations functions in the stack_advisor.py
such as validateHDFSConfigurations.
"""
- def getConfigurationsValidationItems(self, stackAdvisor, configurations, recommendedDefaults, services, hosts):
+ def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
return []
-
- """
- If the service needs to any processing differently than the stack_advisor.py getHostsForMasterComponents
- function, then this logic should be added in this function.
- """
- def getHostsForMasterComponent(self, stackAdvisor, services, hosts, component, hostsList, hostsComponentsMap):
- return stackAdvisor.getHostsForMasterComponent(services, hosts, component, hostsList, hostsComponentsMap)
-
- """
- If the service needs to any processing differently than the stack_advisor.py getHostsForSlaveComponents
- function, then this logic should be added in this function.
- """
- def getHostsForSlaveComponent(self, stackAdvisor, services, hosts, component, hostsList, hostsComponentsMap, freeHosts):
- return stackAdvisor.getHostsForSlaveComponent(services, hosts, component, hostsList, hostsComponentsMap, freeHosts)
-
-
- """
- Utility methods
- """
-
- """
- Utility method used for validation warnings.
- """
- def getWarnItem(self, message):
- return {"level": "WARN", "message": message}
-
- """
- Utility method used for validation errors.
- """
- def getErrorItem(self, message):
- return {"level": "ERROR", "message": message}
-
- """
- Returns the hosts which are running the given component.
- """
- def getHosts(self, componentsList, componentName):
- hostNamesList = [component["hostnames"] for component in componentsList if component["component_name"] == componentName]
- return hostNamesList[0] if len(hostNamesList) > 0 else []
-
- """
- Utility method for setting a configuration property value.
- """
- def putProperty(self, config, configType, services=None):
- userConfigs = {}
- changedConfigs = []
- # if services parameter, prefer values, set by user
- if services:
- if 'configurations' in services.keys():
- userConfigs = services['configurations']
- if 'changed-configurations' in services.keys():
- changedConfigs = services["changed-configurations"]
-
- if configType not in config:
- config[configType] = {}
- if"properties" not in config[configType]:
- config[configType]["properties"] = {}
- def appendProperty(key, value):
- # If property exists in changedConfigs, do not override, use user defined property
- if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
- config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
- else:
- config[configType]["properties"][key] = str(value)
- return appendProperty
-
- """
- Utility method to determine if the configuration property value has been changed.
- """
- def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
- for changedConfig in changedConfigs:
- if changedConfig['type']==configType and changedConfig['name']==propertyName:
- return True
- return False
-
- """
- Utility method for setting a configuration property attribute.
- """
- def putPropertyAttribute(self, config, configType):
- if configType not in config:
- config[configType] = {}
- def appendPropertyAttribute(key, attribute, attributeValue):
- if "property_attributes" not in config[configType]:
- config[configType]["property_attributes"] = {}
- if key not in config[configType]["property_attributes"]:
- config[configType]["property_attributes"][key] = {}
- config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue)
- return appendPropertyAttribute
-
- """
- Utility method to validate configuration settings for a given configuration file.
- This function will call the method provided.
- """
- def validateConfigurationsForSite(self, stackAdvisor, configurations, recommendedDefaults, services, hosts, siteName, method):
- if siteName in recommendedDefaults:
- siteProperties = self.getSiteProperties(configurations, siteName)
- if siteProperties is not None:
- siteRecommendations = recommendedDefaults[siteName]["properties"]
- print("SiteName: %s, method: %s\n" % (siteName, method.__name__))
- print("Site properties: %s\n" % str(siteProperties))
- print("Recommendations: %s\n********\n" % str(siteRecommendations))
- return method(stackAdvisor, siteProperties, siteRecommendations, configurations, services, hosts)
- return []
-
- """
- Utility method used to return all the properties from a given configuration file.
- """
- def getSiteProperties(self, configurations, siteName):
- siteConfig = configurations.get(siteName)
- if siteConfig is None:
- return None
- return siteConfig.get("properties")
http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index beba225..d8685c3 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -17,8 +17,11 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-import socket
+import imp
+import os
import re
+import socket
+import traceback
class StackAdvisor(object):
"""
@@ -296,10 +299,6 @@ class StackAdvisor(object):
pass
-
-
-
-
class DefaultStackAdvisor(StackAdvisor):
"""
Default stack advisor implementation.
@@ -343,32 +342,22 @@ class DefaultStackAdvisor(StackAdvisor):
self.serviceAdvisorsDict[componentName] = self.serviceAdvisorsDict[serviceName]
def instantiateServiceAdvisor(self, service):
- import imp
- import os
- import traceback
serviceName = service["StackServices"]["service_name"]
className = service["StackServices"]["advisor_name"] if "advisor_name" in service["StackServices"] else None
path = service["StackServices"]["advisor_path"] if "advisor_path" in service["StackServices"] else None
- if path is None or className is None:
- return None
-
- if not os.path.exists(path):
- return None
-
- try:
- serviceAdvisor = None
- with open(path, 'rb') as fp:
- serviceAdvisor = imp.load_module('service_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE))
- if hasattr(serviceAdvisor, className):
- print "ServiceAdvisor implementation for service {0} was loaded".format(serviceName)
- clazz = getattr(serviceAdvisor, className)
- return clazz()
+ if path is not None and os.path.exists(path) and className is not None:
+ try:
+ with open(path, 'rb') as fp:
+ serviceAdvisor = imp.load_module('service_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE))
+ if hasattr(serviceAdvisor, className):
+ print "ServiceAdvisor implementation for service {0} was loaded".format(serviceName)
+ return getattr(serviceAdvisor, className)()
- except Exception as e:
- traceback.print_exc()
- print "Failed to load or create ServiceAdvisor implementation for service {0}".format(serviceName)
+ except Exception as e:
+ traceback.print_exc()
+ print "Failed to load or create ServiceAdvisor implementation for service {0}".format(serviceName)
return None
@@ -420,11 +409,8 @@ class DefaultStackAdvisor(StackAdvisor):
serviceAdvisor = self.getServiceAdvisor(serviceName)
for component in masterComponents:
componentName = component["StackServiceComponents"]["component_name"]
- hostsForComponent = []
- if serviceAdvisor is None:
- hostsForComponent = self.getHostsForMasterComponent(services, hosts, component, hostsList, hostsComponentsMap)
- else:
- hostsForComponent = serviceAdvisor.getHostsForMasterComponent(self, services, hosts, component, hostsList, hostsComponentsMap)
+ advisor = serviceAdvisor if serviceAdvisor is not None else self
+ hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component, hostsList)
#extend 'hostsComponentsMap' with 'hostsForComponent'
for hostName in hostsForComponent:
@@ -445,11 +431,8 @@ class DefaultStackAdvisor(StackAdvisor):
serviceAdvisor = self.getServiceAdvisor(serviceName)
for component in slaveClientComponents:
componentName = component["StackServiceComponents"]["component_name"]
- hostsForComponent = []
- if serviceAdvisor is None:
- hostsForComponent = self.getHostsForSlaveComponent(services, hosts, component, hostsList, hostsComponentsMap, freeHosts)
- else:
- hostsForComponent = serviceAdvisor.getHostsForSlaveComponent(self, services, hosts, component, hostsList, hostsComponentsMap, freeHosts)
+ advisor = serviceAdvisor if serviceAdvisor is not None else self
+ hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component, hostsList, freeHosts)
#extend 'hostsComponentsMap' with 'hostsForComponent'
for hostName in hostsForComponent:
@@ -464,7 +447,7 @@ class DefaultStackAdvisor(StackAdvisor):
serviceAdvisor = self.getServiceAdvisor(serviceName)
if serviceAdvisor is not None:
serviceComponents = [component for component in service["components"]]
- serviceAdvisor.colocateService(self, hostsComponentsMap, serviceComponents)
+ serviceAdvisor.colocateService(hostsComponentsMap, serviceComponents)
#prepare 'host-group's from 'hostsComponentsMap'
host_groups = recommendations["blueprint"]["host_groups"]
@@ -477,11 +460,8 @@ class DefaultStackAdvisor(StackAdvisor):
bindings.append( { "name": host_group_name, "hosts": [{ "fqdn": key }] } )
return recommendations
- pass
-
- def getHostsForMasterComponent(self, services, hosts, component, hostsList, hostsComponentsMap):
- componentName = component["StackServiceComponents"]["component_name"]
+ def getHostsForMasterComponent(self, services, hosts, component, hostsList):
if self.isComponentHostsPopulated(component):
return component["StackServiceComponents"]["hostnames"]
@@ -499,18 +479,15 @@ class DefaultStackAdvisor(StackAdvisor):
return [self.getHostForComponent(component, hostsList)]
- def getHostsForSlaveComponent(self, services, hosts, component, hostsList, hostsComponentsMap, freeHosts):
- componentName = component["StackServiceComponents"]["component_name"]
-
+ def getHostsForSlaveComponent(self, services, hosts, component, hostsList, freeHosts):
if component["StackServiceComponents"]["cardinality"] == "ALL":
return hostsList
- componentIsPopulated = self.isComponentHostsPopulated(component)
- if componentIsPopulated:
+ if self.isComponentHostsPopulated(component):
return component["StackServiceComponents"]["hostnames"]
hostsForComponent = []
-
+ componentName = component["StackServiceComponents"]["component_name"]
if self.isSlaveComponent(component):
cardinality = str(component["StackServiceComponents"]["cardinality"])
if self.isComponentUsingCardinalityForLayout(componentName) and cardinality:
@@ -538,10 +515,6 @@ class DefaultStackAdvisor(StackAdvisor):
return hostsForComponent
def isComponentUsingCardinalityForLayout(self, componentName):
- serviceAdvisor = self.getServiceAdvisor(componentName)
- if serviceAdvisor is not None:
- return serviceAdvisor.isComponentUsingCardinalityForLayout(componentName)
-
return False
def createValidationResponse(self, services, validationItems):
@@ -579,12 +552,12 @@ class DefaultStackAdvisor(StackAdvisor):
serviceName = service["StackServices"]["service_name"]
serviceAdvisor = self.getServiceAdvisor(serviceName)
if serviceAdvisor is not None:
- items.extend(serviceAdvisor.getComponentLayoutValidations(self, services, hosts))
+ items.extend(serviceAdvisor.getServiceComponentLayoutValidations(services, hosts))
return items
def getConfigurationClusterSummary(self, servicesList, hosts, components, services):
- pass
+ return {}
def validateClusterConfigurations(self, configurations, services, hosts):
validationItems = []
@@ -653,25 +626,29 @@ class DefaultStackAdvisor(StackAdvisor):
self.validateMinMax(items, recommendedDefaults, configurations)
return items
+ def validateConfigurationsForSite(self, configurations, recommendedDefaults, services, hosts, siteName, method):
+ if siteName in recommendedDefaults:
+ siteProperties = self.getSiteProperties(configurations, siteName)
+ if siteProperties is not None:
+ siteRecommendations = recommendedDefaults[siteName]["properties"]
+ print("SiteName: %s, method: %s" % (siteName, method.__name__))
+ print("Site properties: %s" % str(siteProperties))
+ print("Recommendations: %s" % str(siteRecommendations))
+ return method(siteProperties, siteRecommendations, configurations, services, hosts)
+ return []
+
def getConfigurationsValidationItemsForService(self, configurations, recommendedDefaults, service, services, hosts):
items = []
serviceName = service["StackServices"]["service_name"]
validator = self.validateServiceConfigurations(serviceName)
if validator is not None:
for siteName, method in validator.items():
- if siteName in recommendedDefaults:
- siteProperties = getSiteProperties(configurations, siteName)
- if siteProperties is not None:
- siteRecommendations = recommendedDefaults[siteName]["properties"]
- print("SiteName: %s, method: %s\n" % (siteName, method.__name__))
- print("Site properties: %s\n" % str(siteProperties))
- print("Recommendations: %s\n********\n" % str(siteRecommendations))
- resultItems = method(siteProperties, siteRecommendations, configurations, services, hosts)
- items.extend(resultItems)
+ resultItems = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method)
+ items.extend(resultItems)
serviceAdvisor = self.getServiceAdvisor(serviceName)
if serviceAdvisor is not None:
- items.extend(serviceAdvisor.getConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts))
+ items.extend(serviceAdvisor.getServiceConfigurationsValidationItems(configurations, recommendedDefaults, services, hosts))
return items
@@ -710,7 +687,7 @@ class DefaultStackAdvisor(StackAdvisor):
else:
serviceAdvisor = self.getServiceAdvisor(serviceName)
if serviceAdvisor is not None:
- serviceAdvisor.getServiceConfigurationRecommendations(self, configurations, cgClusterSummary, cgServices, cgHosts)
+ serviceAdvisor.getServiceConfigurationRecommendations(configurations, cgClusterSummary, cgServices, cgHosts)
cgRecommendation = {
"configurations": {},
@@ -781,7 +758,7 @@ class DefaultStackAdvisor(StackAdvisor):
else:
serviceAdvisor = self.getServiceAdvisor(serviceName)
if serviceAdvisor is not None:
- serviceAdvisor.getServiceConfigurationRecommendations(self, configurations, clusterSummary, services, hosts)
+ serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterSummary, services, hosts)
return recommendations
@@ -818,19 +795,11 @@ class DefaultStackAdvisor(StackAdvisor):
def isMasterComponentWithMultipleInstances(self, component):
componentName = self.getComponentName(component)
- serviceAdvisor = self.getServiceAdvisor(componentName)
- if serviceAdvisor is not None:
- return serviceAdvisor.isMasterComponentWithMultipleInstances(componentName)
-
masters = self.getMastersWithMultipleInstances()
return componentName in masters
def isComponentNotValuable(self, component):
componentName = self.getComponentName(component)
- serviceAdvisor = self.getServiceAdvisor(componentName)
- if serviceAdvisor is not None:
- return serviceAdvisor.isComponentNotValuable(componentName)
-
service = self.getNotValuableComponents()
return componentName in service
@@ -840,20 +809,16 @@ class DefaultStackAdvisor(StackAdvisor):
# Helper dictionaries
def getComponentCardinality(self, componentName):
- serviceAdvisor = self.getServiceAdvisor(componentName)
- if serviceAdvisor is not None:
- return serviceAdvisor.getComponentCardinality(componentName)
-
return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1})
def getHostForComponent(self, component, hostsList):
- if (len(hostsList) == 0):
+ if len(hostsList) == 0:
return None
componentName = self.getComponentName(component)
if len(hostsList) != 1:
- scheme = self.getComponentLayoutScheme(componentName)
+ scheme = self.getComponentLayoutSchemes().get(componentName, None)
if scheme is not None:
hostIndex = next((index for key, index in scheme.iteritems() if isinstance(key, ( int, long )) and len(hostsList) < key), scheme['else'])
else:
@@ -863,30 +828,11 @@ class DefaultStackAdvisor(StackAdvisor):
return host
return hostsList[0]
- def getComponentLayoutScheme(self, componentName):
- """
- Provides a scheme for laying out given component on different number of hosts.
- """
- serviceAdvisor = self.getServiceAdvisor(componentName)
- if serviceAdvisor is not None:
- return serviceAdvisor.getComponentLayoutScheme(componentName)
-
- return self.getComponentLayoutSchemes().get(componentName, None)
-
def getComponentName(self, component):
return self.getComponentAttribute(component, "component_name")
- def isComponentNotPreferableOnAmbariServerHost(self, component):
- componentName = self.getComponentName(component)
- serviceAdvisor = self.getServiceAdvisor(componentName)
- if serviceAdvisor is not None:
- return serviceAdvisor.isComponentNotPreferableOnAmbariServerHost(componentName)
-
- service = self.getNotPreferableOnServerComponents()
- return componentName in service
-
def isHostSuitableForComponent(self, host, component):
- return not (self.isComponentNotPreferableOnAmbariServerHost(component) and self.isLocalHost(host))
+ return not (self.getComponentName(component) in self.getNotPreferableOnServerComponents() and self.isLocalHost(host))
def getMastersWithMultipleInstances(self):
return []
@@ -909,13 +855,24 @@ class DefaultStackAdvisor(StackAdvisor):
"""
return {}
+ """
+ Utility method used for validation warnings.
+ """
+ def getWarnItem(self, message):
+ return {"level": "WARN", "message": message}
+
+ """
+ Utility method used for validation errors.
+ """
+ def getErrorItem(self, message):
+ return {"level": "ERROR", "message": message}
+
def getComponentHostNames(self, servicesDict, serviceName, componentName):
for service in servicesDict["services"]:
if service["StackServices"]["service_name"] == serviceName:
for component in service['components']:
if component["StackServiceComponents"]["component_name"] == componentName:
return component["StackServiceComponents"]["hostnames"]
- pass
def recommendConfigurationDependencies(self, services, hosts):
result = self.recommendConfigurations(services, hosts)
@@ -1003,8 +960,84 @@ class DefaultStackAdvisor(StackAdvisor):
return cmp(normalize(version1), normalize(version2))
pass
-def getSiteProperties(configurations, siteName):
- siteConfig = configurations.get(siteName)
- if siteConfig is None:
- return None
- return siteConfig.get("properties")
+ def getSiteProperties(self, configurations, siteName):
+ siteConfig = configurations.get(siteName)
+ if siteConfig is None:
+ return None
+ return siteConfig.get("properties")
+
+ def putProperty(self, config, configType, services=None):
+ userConfigs = {}
+ changedConfigs = []
+ # if services parameter, prefer values, set by user
+ if services:
+ if 'configurations' in services.keys():
+ userConfigs = services['configurations']
+ if 'changed-configurations' in services.keys():
+ changedConfigs = services["changed-configurations"]
+
+ if configType not in config:
+ config[configType] = {}
+ if"properties" not in config[configType]:
+ config[configType]["properties"] = {}
+ def appendProperty(key, value):
+ # If property exists in changedConfigs, do not override, use user defined property
+ if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
+ config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
+ else:
+ config[configType]["properties"][key] = str(value)
+ return appendProperty
+
+ def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
+ for changedConfig in changedConfigs:
+ if changedConfig['type']==configType and changedConfig['name']==propertyName:
+ return True
+ return False
+
+ def updateProperty(self, config, configType, services=None):
+ userConfigs = {}
+ changedConfigs = []
+ # if services parameter, prefer values, set by user
+ if services:
+ if 'configurations' in services.keys():
+ userConfigs = services['configurations']
+ if 'changed-configurations' in services.keys():
+ changedConfigs = services["changed-configurations"]
+
+ if configType not in config:
+ config[configType] = {}
+ if "properties" not in config[configType]:
+ config[configType]["properties"] = {}
+
+ def updatePropertyWithCallback(key, value, callback):
+ # If property exists in changedConfigs, do not override, use user defined property
+ if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
+ config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
+ else:
+ # Give the callback an empty string if the mapping doesn't exist
+ current_value = ""
+ if key in config[configType]["properties"]:
+ current_value = config[configType]["properties"][key]
+
+ config[configType]["properties"][key] = callback(current_value, value)
+
+ return updatePropertyWithCallback
+
+ def putPropertyAttribute(self, config, configType):
+ if configType not in config:
+ config[configType] = {}
+ def appendPropertyAttribute(key, attribute, attributeValue):
+ if "property_attributes" not in config[configType]:
+ config[configType]["property_attributes"] = {}
+ if key not in config[configType]["property_attributes"]:
+ config[configType]["property_attributes"][key] = {}
+ config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue)
+ return appendPropertyAttribute
+
+
+ """
+ Returns the hosts which are running the given component.
+ """
+ def getHosts(self, componentsList, componentName):
+ hostNamesList = [component["hostnames"] for component in componentsList if component["component_name"] == componentName]
+ return hostNamesList[0] if len(hostNamesList) > 0 else []
http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
index 384b8be..12f4fa1 100644
--- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
@@ -28,16 +28,11 @@ class TestHAWQ200ServiceAdvisor(TestCase):
self.testDirectory = os.path.dirname(os.path.abspath(__file__))
stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py')
hawq200ServiceAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/common-services/HAWQ/2.0.0/service_advisor.py')
-
with open(stackAdvisorPath, 'rb') as fp:
- stack_advisor = imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
with open(hawq200ServiceAdvisorPath, 'rb') as fp:
- service_advisor = imp.load_module('stack_advisor_impl', fp, hawq200ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
-
- stackAdvisorClass = getattr(stack_advisor, 'DefaultStackAdvisor')
- self.stackAdvisor = stackAdvisorClass()
-
- serviceAdvisorClass = getattr(service_advisor, 'HAWQ200ServiceAdvisor')
+ service_advisor_impl = imp.load_module('service_advisor_impl', fp, hawq200ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ serviceAdvisorClass = getattr(service_advisor_impl, 'HAWQ200ServiceAdvisor')
self.serviceAdvisor = serviceAdvisorClass()
@@ -85,8 +80,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
# HAWQMASTER is placed on c6403.ambari.apache.org
# There are 4 available hosts in the cluster
# Recommend HAWQSTANDBY on next available host, c6402.ambari.apache.org
- self.stackAdvisor.loadServiceAdvisors(services)
- standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None)
+ standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
self.assertEquals(standbyHosts, ["c6402.ambari.apache.org"])
# Case 2:
@@ -95,7 +89,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
# There are 4 available hosts in the cluster
# Recommend HAWQSTANDBY on next available host, c6403.ambari.apache.org
services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org"]
- standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None)
+ standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
self.assertEquals(standbyHosts, ["c6403.ambari.apache.org"])
# Case 3:
@@ -104,7 +98,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
# There are 2 available hosts in the cluster
# Recommend HAWQSTANDBY on a host which does not have HAWQMASTER, c6401.ambari.apache.org
hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
- standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None)
+ standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
self.assertEquals(standbyHosts, ["c6401.ambari.apache.org"])
# Case 4:
@@ -114,7 +108,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
# Do not recommend HAWQSTANDBY on a single node cluster
hostsList = ["c6401.ambari.apache.org"]
services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6401.ambari.apache.org"]
- standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None)
+ standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
self.assertEquals(standbyHosts, [])
# Case 5:
@@ -126,7 +120,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]
services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org"]
services["services"][0]["components"][1]["StackServiceComponents"]["hostnames"] = ["c6401.ambari.apache.org"]
- standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None)
+ standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
self.assertEquals(standbyHosts, ["c6401.ambari.apache.org"])
@@ -220,33 +214,33 @@ class TestHAWQ200ServiceAdvisor(TestCase):
# HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2
# hawq_global_rm_type: yarn
# Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2")
## Test if vm.overcommit_memory is set correctly
# Case 1: All machines have total_mem above 32GB (total_mem >= 33554432)
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2")
# Case 2: One machine has total_mem below 32GB
hosts["items"][0]["Hosts"]["total_mem"] = 33554431
services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB"
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1")
## Test if hawq_rm_memory_limit_perseg is set correctly
# Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB as vm.overcommit_memory = 1 and in MB
hosts["items"][0]["Hosts"]["total_mem"] = 2097152
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "1536MB")
# Case 2: Minimum host memory is ~ 16 GB, recommended val must be .75% of 16GB as vm.overcommit_memory = 1 and in GB
hosts["items"][0]["Hosts"]["total_mem"] = 16777216
hosts["items"][1]["Hosts"]["total_mem"] = 26777216
hosts["items"][3]["Hosts"]["total_mem"] = 36777216
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "12GB")
# Case 2: Minimum host memory is ~ 64 GB, recommended val must be .75% of 32GB as vm.overcommit_memory = 2 and in GB
@@ -254,21 +248,21 @@ class TestHAWQ200ServiceAdvisor(TestCase):
hosts["items"][1]["Hosts"]["total_mem"] = 77108864
hosts["items"][3]["Hosts"]["total_mem"] = 87108864
services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB"
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "24GB")
# Case 4: Minimum host memory is ~ 512 GB, recommended val must be .85% of 256GB as vm.overcommit_memory = 2 and in GB
hosts["items"][0]["Hosts"]["total_mem"] = 536870912
hosts["items"][1]["Hosts"]["total_mem"] = 636870912
hosts["items"][3]["Hosts"]["total_mem"] = 736870912
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "218GB")
# Case 5: Minimum host memory is ~ 1024 GB, recommended val must be .95% of 512GB as vm.overcommit_memory = 2 and in GB
hosts["items"][0]["Hosts"]["total_mem"] = 1073741824
hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "436GB")
# Case 6: Minimum host memory is ~ 1024 GB, vm.overcommit_ratio = 75, vm.overcommit_memory = 2
@@ -277,7 +271,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 75
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB")
## Test if the properties are set to visible / invisible based on the value of hawq_global_rm_type
@@ -290,7 +284,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
"hawq_rm_yarn_queue_name": "true",
"hawq_rm_yarn_scheduler_address": "true",
"hawq_rm_yarn_address": "true"}
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
for property, status in properties_visible_status.iteritems():
self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status)
@@ -302,6 +296,6 @@ class TestHAWQ200ServiceAdvisor(TestCase):
"hawq_rm_yarn_queue_name": "false",
"hawq_rm_yarn_scheduler_address": "false",
"hawq_rm_yarn_address": "false"}
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
for property, status in properties_visible_status.iteritems():
self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py
index c3a63cc..4ea3bfb 100644
--- a/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py
@@ -27,16 +27,11 @@ class TestPXF300ServiceAdvisor(TestCase):
self.testDirectory = os.path.dirname(os.path.abspath(__file__))
stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py')
pxf300ServiceAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/common-services/PXF/3.0.0/service_advisor.py')
-
with open(stackAdvisorPath, 'rb') as fp:
- stack_advisor = imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
with open(pxf300ServiceAdvisorPath, 'rb') as fp:
- service_advisor = imp.load_module('stack_advisor_impl', fp, pxf300ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
-
- stackAdvisorClass = getattr(stack_advisor, 'DefaultStackAdvisor')
- self.stackAdvisor = stackAdvisorClass()
-
- serviceAdvisorClass = getattr(service_advisor, 'PXF300ServiceAdvisor')
+ service_advisor_impl = imp.load_module('service_advisor_impl', fp, pxf300ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ serviceAdvisorClass = getattr(service_advisor_impl, 'PXF300ServiceAdvisor')
self.serviceAdvisor = serviceAdvisorClass()
self.PXF_PATH = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/pxf/pxf-hbase.jar"
@@ -56,13 +51,13 @@ class TestPXF300ServiceAdvisor(TestCase):
# Case 1: Test pxf-hbase.jar classpath line was added to content
expected = "# Some hbase-env content text\n\n#Add pxf-hbase.jar to HBASE_CLASSPATH\n" + self.PXF_PATH
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, services["configurations"], None, services, None)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(services["configurations"], None, services, None)
self.assertEquals(services["configurations"]["hbase-env"]["properties"]["content"], expected)
# Case 2: Test pxf-hbase.jar classpath line is not added again if content already has it
services["configurations"]["hbase-env"]["properties"]["content"] = self.PXF_PATH
expected = self.PXF_PATH
- self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, services["configurations"], None, services, None)
+ self.serviceAdvisor.getServiceConfigurationRecommendations(services["configurations"], None, services, None)
self.assertEquals(services["configurations"]["hbase-env"]["properties"]["content"], expected)
def test_getConfigurationsValidationItems(self):
@@ -107,10 +102,10 @@ class TestPXF300ServiceAdvisor(TestCase):
"level": "WARN"
}
]
- items = self.serviceAdvisor.getConfigurationsValidationItems(self.stackAdvisor, properties, properties, services, None)
+ items = self.serviceAdvisor.getConfigurationsValidationItems(properties, properties, services, None)
self.assertEquals(items, expected)
# Case 2: No warning should be generated if PXF_PATH is present in hbase-env
properties = services["configurations"]["hbase-env"]["properties"]["content"] = self.PXF_PATH
- items = self.serviceAdvisor.getConfigurationsValidationItems(self.stackAdvisor, properties, properties, services, None)
+ items = self.serviceAdvisor.getConfigurationsValidationItems(properties, properties, services, None)
self.assertEquals(items, [])
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index caf121c..c077a9f 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -2047,7 +2047,7 @@ class TestHDP23StackAdvisor(TestCase):
# Test 1 - with 3 segments
self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3)
serviceAdvisor = self.createHAWQServiceAdvisor()
- serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts)
+ serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(3 * 6))
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false")
@@ -2057,19 +2057,19 @@ class TestHDP23StackAdvisor(TestCase):
# Test 2 - with 100 segments
hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(100)]
- serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts)
+ serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(100 * 5))
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true")
# Test 3 - with 512 segments
hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(512)]
- serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts)
+ serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512")
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true")
# Test 4 - with 513 segments
hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(513)]
- serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts)
+ serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512")
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true")
@@ -2077,7 +2077,7 @@ class TestHDP23StackAdvisor(TestCase):
configurations = {}
services["configurations"]["hawq-site"] = {"properties":{"hawq_global_rm_type": "none"}}
hawqSegmentComponent["hostnames"] = []
- serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts)
+ serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false")
self.assertTrue("default_hash_table_bucket_number" not in configurations["hawq-site"])
@@ -2380,7 +2380,7 @@ class TestHDP23StackAdvisor(TestCase):
}
serviceAdvisor = self.createHAWQServiceAdvisor()
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
problems_dict = {}
for problem in problems:
problems_dict[problem['config-name']] = problem
@@ -2391,7 +2391,7 @@ class TestHDP23StackAdvisor(TestCase):
configurations["hawq-site"] = {"properties": {"hawq_master_directory": "/data/hawq/master",
"hawq_segment_directory": "/data/hawq/segment"}}
properties = configurations["hawq-site"]["properties"]
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
problems_dict = {}
self.assertEqual(len(problems), 0)
expected_warnings = {}
@@ -2400,7 +2400,7 @@ class TestHDP23StackAdvisor(TestCase):
configurations["hawq-site"] = {"properties": {"hawq_master_directory": "/data/hawq/master1,/data/hawq/master2",
"hawq_segment_directory": "/data/hawq/segment1 /data/hawq/segment2"}}
properties = configurations["hawq-site"]["properties"]
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
problems_dict = {}
for problem in problems:
problems_dict[problem['config-name']] = problem
@@ -2454,7 +2454,7 @@ class TestHDP23StackAdvisor(TestCase):
"level": "ERROR"
} ]
"""
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, services["configurations"], services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, services["configurations"], services, hosts)
self.assertEqual(len(problems), 1)
expected = {
"config-type": "hawq-site",
@@ -2468,7 +2468,7 @@ class TestHDP23StackAdvisor(TestCase):
# case 2: hawq_global_rm_type is set as yarn, and YARN service is installed. No validation errors expected.
services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, services["configurations"], services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, services["configurations"], services, hosts)
self.assertEqual(len(problems), 0)
# Test HAWQ Master port conflict with Ambari Server Postgres port
@@ -2481,7 +2481,7 @@ class TestHDP23StackAdvisor(TestCase):
{"hawq_master_address_port": "5432"}
}
}
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 1)
expected = {
"config-name": "hawq_master_address_port",
@@ -2496,19 +2496,19 @@ class TestHDP23StackAdvisor(TestCase):
# case 2: HAWQ Master is placed on Ambari Server and HAWQ Master port is different from Ambari Server Postgres Port
serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=True)
configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432"
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 0)
# case 3: HAWQ Master is not placed on Ambari Server and HAWQ Master port is same as Ambari Server Postgres Port
serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False)
configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "5432"
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 0)
# case 4: HAWQ Master is not placed on Ambari Server and HAWQ Master port is different from Ambari Server Postgres Port
serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False)
configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432"
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 0)
# -------- test query limits warning ----------
@@ -2538,14 +2538,14 @@ class TestHDP23StackAdvisor(TestCase):
'config-name': 'default_hash_table_bucket_number',
'level': 'ERROR'
}
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 1)
self.assertEqual(problems[0], expected)
configurations["hawq-site"] = {"properties": {"default_hash_table_bucket_number": "500",
"hawq_rm_nvseg_perquery_limit": "500"}}
properties = configurations["hawq-site"]["properties"]
- problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 0)
@@ -2578,19 +2578,19 @@ class TestHDP23StackAdvisor(TestCase):
}
serviceAdvisor = self.createHAWQServiceAdvisor()
- problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 1)
self.assertEqual(problems[0], expected)
# 2. Try with 3 hosts
services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["host1", "host2", "host3"]
- problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 1)
self.assertEqual(problems[0], expected)
# 3. Try with 4 hosts - default value
services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["host1", "host2", "host3", "host4"]
- problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 0)
# 4. Try with 4 hosts
@@ -2602,7 +2602,7 @@ class TestHDP23StackAdvisor(TestCase):
'config-name': 'output.replace-datanode-on-failure',
'level': 'WARN'
}
- problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts)
+ problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 1)
self.assertEqual(problems[0], expected)