You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by od...@apache.org on 2015/12/15 02:44:25 UTC
ambari git commit: AMBARI-14339. Stack advisor validations for
HAWQ(Newton Alex via odiachenko).
Repository: ambari
Updated Branches:
refs/heads/trunk 4196801e8 -> 1bc270aac
AMBARI-14339. Stack advisor validations for HAWQ(Newton Alex via odiachenko).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1bc270aa
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1bc270aa
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1bc270aa
Branch: refs/heads/trunk
Commit: 1bc270aac7282aef824b55e46c64c38a51eb9ed0
Parents: 4196801
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Mon Dec 14 17:43:59 2015 -0800
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Mon Dec 14 17:43:59 2015 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.3/services/stack_advisor.py | 132 ++++++++++++++-----
1 file changed, 97 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1bc270aa/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index ecf74b5..5ab05fd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -51,31 +51,25 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
- hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
- hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
+ hawqMasterHosts = [component["StackServiceComponents"]["hostnames"][0] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
+ hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"][0] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
# single node case is not analyzed because HAWQ Standby Master will not be present in single node topology due to logic in createComponentLayoutRecommendations()
- if len(hawqMasterHosts) > 0 and len(hawqStandbyHosts) > 0:
- commonHosts = [host for host in hawqMasterHosts[0] if host in hawqStandbyHosts[0]]
- for host in commonHosts:
- message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host."
- childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": host } )
-
- if len(hawqMasterHosts) > 0 and hostsCount > 1:
- ambariServerHosts = [host for host in hawqMasterHosts[0] if self.isLocalHost(host)]
- for host in ambariServerHosts:
- message = "HAWQ Master and Ambari Server should not be deployed on the same host. " \
- "If you leave them collocated, make sure to set HAWQ Master Port property " \
- "to a value different from the port number used by Ambari Server database."
- childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": host } )
-
- if len(hawqStandbyHosts) > 0 and hostsCount > 1:
- ambariServerHosts = [host for host in hawqStandbyHosts[0] if self.isLocalHost(host)]
- for host in ambariServerHosts:
- message = "HAWQ Standby Master and Ambari Server should not be deployed on the same host. " \
- "If you leave them collocated, make sure to set HAWQ Master Port property " \
- "to a value different from the port number used by Ambari Server database."
- childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": host } )
+ if len(hawqMasterHosts) == 1 and len(hawqStandbyHosts) == 1 and hawqMasterHosts == hawqStandbyHosts:
+ message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host."
+ childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
+
+ if len(hawqMasterHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]):
+ message = "HAWQ Master and Ambari Server should not be deployed on the same host. " \
+ "If you leave them collocated, make sure to set HAWQ Master Port property " \
+ "to a value different from the port number used by Ambari Server database."
+ childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } )
+
+ if len(hawqStandbyHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]):
+ message = "HAWQ Standby Master and Ambari Server should not be deployed on the same host. " \
+ "If you leave them collocated, make sure to set HAWQ Master Port property " \
+ "to a value different from the port number used by Ambari Server database."
+ childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
if "SPARK_THRIFTSERVER" in [service["StackServices"]["service_name"] for service in services["services"]]:
if not "HIVE_SERVER" in [service["StackServices"]["service_name"] for service in services["services"]]:
@@ -117,7 +111,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
"HIVE": self.recommendHIVEConfigurations,
"HBASE": self.recommendHBASEConfigurations,
"KAFKA": self.recommendKAFKAConfigurations,
- "RANGER": self.recommendRangerConfigurations
+ "RANGER": self.recommendRangerConfigurations,
+ "HAWQ": self.recommendHAWQConfigurations
}
parentRecommendConfDict.update(childRecommendConfDict)
return parentRecommendConfDict
@@ -577,18 +572,35 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
if 'yarn-site' in services["configurations"] and 'yarn.resourcemanager.proxy-user-privileges.enabled' in services["configurations"]["yarn-site"]["properties"]:
putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false')
+
+ def isHawqMasterComponentOnAmbariServer(self, services):
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item for sublist in componentsListList for item in sublist]
+ hawqMasterComponentHosts = [component["StackServiceComponents"]["hostnames"][0] for component in componentsList
+ if component["StackServiceComponents"]["component_name"] in ("HAWQMASTER", "HAWQSTANDBY")]
+ return any([self.isLocalHost(host) for host in hawqMasterComponentHosts])
+
+
+ def recommendHAWQConfigurations(self, configurations, clusterData, services, hosts):
+ putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services)
+ if self.isHawqMasterComponentOnAmbariServer(services):
+ if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]:
+ putHawqSiteProperty('hawq_master_address_port', '')
+
+
def getServiceConfigurationValidators(self):
- parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
- childValidators = {
- "HDFS": {"hdfs-site": self.validateHDFSConfigurations},
- "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations,
- "hive-site": self.validateHiveConfigurations},
- "HBASE": {"hbase-site": self.validateHBASEConfigurations},
- "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations},
- "YARN": {"yarn-site": self.validateYARNConfigurations}
- }
- self.mergeValidators(parentValidators, childValidators)
- return parentValidators
+ parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
+ childValidators = {
+ "HDFS": {"hdfs-site": self.validateHDFSConfigurations},
+ "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations,
+ "hive-site": self.validateHiveConfigurations},
+ "HBASE": {"hbase-site": self.validateHBASEConfigurations},
+ "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations},
+ "YARN": {"yarn-site": self.validateYARNConfigurations},
+ "HAWQ": {"hawq-site": self.validateHAWQConfigurations}
+ }
+ self.mergeValidators(parentValidators, childValidators)
+ return parentValidators
def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
super(HDP23StackAdvisor, self).validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
@@ -774,5 +786,55 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
return self.toConfigurationValidationProblems(validationItems, "yarn-site")
+
+ def isHawqMasterPortConflict(self, configurations):
+ prop_name = 'hawq_master_address_port'
+ default_ambari_port = 5432
+ if prop_name in configurations["hawq-site"]["properties"]:
+ portValue = int(configurations["hawq-site"]["properties"][prop_name])
+ return portValue == default_ambari_port
+
+ return False
+
+
+ def validateIfRootDir(self, properties, validationItems, prop_name, display_name):
+ root_dir = '/'
+ if prop_name in properties and properties[prop_name].strip() == root_dir:
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "It is not advisable to have " + display_name + " at " + root_dir +". Consider creating a sub directory for HAWQ")})
+
+
+ def validateHAWQConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ hawq_site = properties
+ validationItems = []
+
+ # 1. Check if HAWQ master/standby port numbers don't conflict with Ambari ports. Both Ambari and HAWQ use postgres DB and 5432 port.
+ if self.isHawqMasterComponentOnAmbariServer(services) and self.isHawqMasterPortConflict(configurations):
+ prop_name = 'hawq_master_address_port'
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "HAWQ Master or Standby Master cannot use the port 5432 when installed on the same host as the Ambari Server. Ambari Postgres DB uses the same port. Please choose a different value (e.g. 10432)")})
+
+ # 2. Check if any data directories are pointing to root dir '/'
+ prop_name = 'hawq_master_directory'
+ display_name = 'HAWQ Master directory'
+ self.validateIfRootDir (properties, validationItems, prop_name, display_name)
+
+ prop_name = 'hawq_master_temp_directory'
+ display_name = 'HAWQ Master temp directory'
+ self.validateIfRootDir (properties, validationItems, prop_name, display_name)
+
+ prop_name = 'hawq_segment_directory'
+ display_name = 'HAWQ Segment directory'
+ self.validateIfRootDir (properties, validationItems, prop_name, display_name)
+
+ prop_name = 'hawq_segment_temp_directory'
+ display_name = 'HAWQ Segment temp directory'
+ self.validateIfRootDir (properties, validationItems, prop_name, display_name)
+
+ return self.toConfigurationValidationProblems(validationItems, "hawq-site")
+
+
def isComponentUsingCardinalityForLayout(self, componentName):
return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']