You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2015/10/26 21:57:03 UTC
[10/10] ambari git commit: AMBARI-13442: stack advisor layout and
validations for HAWQ service master components (adenissov via jaoki)
AMBARI-13442: stack advisor layout and validations for HAWQ service master components (adenissov via jaoki)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/46a20019
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/46a20019
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/46a20019
Branch: refs/heads/trunk
Commit: 46a20019de96f02437720f93f1d442fc33898585
Parents: 7a0aa25
Author: Jun Aoki <ja...@apache.org>
Authored: Mon Oct 26 13:56:44 2015 -0700
Committer: Jun Aoki <ja...@apache.org>
Committed: Mon Oct 26 13:56:44 2015 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.3/services/stack_advisor.py | 70 +
.../python/stacks/2.3/common/hosts-1-host.json | 93 +
.../python/stacks/2.3/common/hosts-3-hosts.json | 269 ++
.../stacks/2.3/common/services-hawq-1-host.json | 2575 ++++++++++++++++++
.../2.3/common/services-hawq-3-hosts.json | 2575 ++++++++++++++++++
.../services-master_ambari_colo-3-hosts.json | 2575 ++++++++++++++++++
.../services-master_standby_colo-3-hosts.json | 2575 ++++++++++++++++++
.../2.3/common/services-nohawq-3-hosts.json | 2214 +++++++++++++++
.../common/services-normal-hawq-3-hosts.json | 2575 ++++++++++++++++++
.../common/services-normal-nohawq-3-hosts.json | 2214 +++++++++++++++
.../services-standby_ambari_colo-3-hosts.json | 2575 ++++++++++++++++++
.../stacks/2.3/common/test_stack_advisor.py | 175 ++
12 files changed, 20485 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/46a20019/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 464f9cc..2a26405 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -22,6 +22,76 @@ import socket
class HDP23StackAdvisor(HDP22StackAdvisor):
+ def createComponentLayoutRecommendations(self, services, hosts):
+ parentComponentLayoutRecommendations = super(HDP23StackAdvisor, self).createComponentLayoutRecommendations(services, hosts)
+
+ # remove HAWQSTANDBY on a single node
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ if len(hostsList) == 1:
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if "HAWQ" in servicesList:
+ components = parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"]
+ components = [ component for component in components if component["name"] != 'HAWQSTANDBY' ]
+ parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"] = components
+
+ return parentComponentLayoutRecommendations
+
+ def getComponentLayoutValidations(self, services, hosts):
+ parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services, hosts)
+
+ if not "HAWQ" in [service["StackServices"]["service_name"] for service in services["services"]]:
+ return parentItems
+
+ childItems = []
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ hostsCount = len(hostsList)
+
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item for sublist in componentsListList for item in sublist]
+ hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
+ hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
+
+ # single node case is not analyzed because HAWQ Standby Master will not be present in single node topology due to logic in createComponentLayoutRecommendations()
+ if len(hawqMasterHosts) > 0 and len(hawqStandbyHosts) > 0:
+ commonHosts = [host for host in hawqMasterHosts[0] if host in hawqStandbyHosts[0]]
+ for host in commonHosts:
+ message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host."
+ childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": host } )
+
+ if len(hawqMasterHosts) > 0 and hostsCount > 1:
+ ambariServerHosts = [host for host in hawqMasterHosts[0] if self.isLocalHost(host)]
+ for host in ambariServerHosts:
+ message = "HAWQ Master and Ambari Server should not be deployed on the same host. " \
+ "If you leave them collocated, make sure to set HAWQ Master Port property " \
+ "to a value different from the port number used by Ambari Server database."
+ childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": host } )
+
+ if len(hawqStandbyHosts) > 0 and hostsCount > 1:
+ ambariServerHosts = [host for host in hawqStandbyHosts[0] if self.isLocalHost(host)]
+ for host in ambariServerHosts:
+ message = "HAWQ Standby Master and Ambari Server should not be deployed on the same host. " \
+ "If you leave them collocated, make sure to set HAWQ Master Port property " \
+ "to a value different from the port number used by Ambari Server database."
+ childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": host } )
+
+ parentItems.extend(childItems)
+ return parentItems
+
+ def getNotPreferableOnServerComponents(self):
+ parentComponents = super(HDP23StackAdvisor, self).getNotPreferableOnServerComponents()
+ parentComponents.extend(['HAWQMASTER', 'HAWQSTANDBY'])
+ return parentComponents
+
+ def getComponentLayoutSchemes(self):
+ parentSchemes = super(HDP23StackAdvisor, self).getComponentLayoutSchemes()
+ # key is max number of cluster hosts + 1, value is index in host list where to put the component
+ childSchemes = {
+ 'HAWQMASTER' : {6: 2, 31: 1, "else": 5},
+ 'HAWQSTANDBY': {6: 1, 31: 2, "else": 3}
+ }
+ parentSchemes.update(childSchemes)
+ return parentSchemes
+
def getServiceConfigurationRecommenderDict(self):
parentRecommendConfDict = super(HDP23StackAdvisor, self).getServiceConfigurationRecommenderDict()
childRecommendConfDict = {
http://git-wip-us.apache.org/repos/asf/ambari/blob/46a20019/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json b/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json
new file mode 100644
index 0000000..5d6a5b6
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json
@@ -0,0 +1,93 @@
+{
+ "href" : "/api/v1/hosts?fields=Hosts/*&Hosts/host_name.in(c6401.ambari.apache.org)",
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "desired_configs" : null,
+ "disk_info" : [
+ {
+ "available" : "481199976",
+ "device" : "/dev/mapper/VolGroup-lv_root",
+ "used" : "5713880",
+ "percent" : "2%",
+ "size" : "512971376",
+ "type" : "ext4",
+ "mountpoint" : "/"
+ },
+ {
+ "available" : "1478456",
+ "device" : "tmpfs",
+ "used" : "0",
+ "percent" : "0%",
+ "size" : "1478456",
+ "type" : "tmpfs",
+ "mountpoint" : "/dev/shm"
+ },
+ {
+ "available" : "438284",
+ "device" : "/dev/sda1",
+ "used" : "31960",
+ "percent" : "7%",
+ "size" : "495844",
+ "type" : "ext4",
+ "mountpoint" : "/boot"
+ },
+ {
+ "available" : "191037148",
+ "device" : "vagrant",
+ "used" : "296051072",
+ "percent" : "61%",
+ "size" : "487088220",
+ "type" : "vboxsf",
+ "mountpoint" : "/vagrant"
+ }
+ ],
+ "host_health_report" : "",
+ "host_name" : "c6401.ambari.apache.org",
+ "host_state" : "HEALTHY",
+ "host_status" : "HEALTHY",
+ "ip" : "192.168.64.101",
+ "last_agent_env" : {
+ "stackFoldersAndFiles" : [ ],
+ "alternatives" : [ ],
+ "existingUsers" : [ ],
+ "existingRepos" : [ ],
+ "installedPackages" : [ ],
+ "hostHealth" : {
+ "activeJavaProcs" : [ ],
+ "agentTimeStampAtReporting" : 1445288922364,
+ "serverTimeStampAtReporting" : 1445288922425,
+ "liveServices" : [
+ {
+ "desc" : "",
+ "name" : "ntpd",
+ "status" : "Healthy"
+ }
+ ]
+ },
+ "umask" : 18,
+ "transparentHugePage" : "always",
+ "firewallRunning" : false,
+ "firewallName" : "iptables",
+ "reverseLookup" : true
+ },
+ "last_heartbeat_time" : 1445288922425,
+ "last_registration_time" : 1445288888619,
+ "os_arch" : "x86_64",
+ "os_family" : "redhat6",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "recovery_report" : {
+ "summary" : "DISABLED",
+ "component_reports" : [ ]
+ },
+ "recovery_summary" : "DISABLED",
+ "total_mem" : 2956916
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/46a20019/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json b/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json
new file mode 100644
index 0000000..3c0511e
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json
@@ -0,0 +1,269 @@
+{
+ "href" : "/api/v1/hosts?fields=Hosts/*&Hosts/host_name.in(c6401.ambari.apache.org,c6402.ambari.apache.org,c6403.ambari.apache.org)",
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "desired_configs" : null,
+ "disk_info" : [
+ {
+ "available" : "481199416",
+ "device" : "/dev/mapper/VolGroup-lv_root",
+ "used" : "5714440",
+ "percent" : "2%",
+ "size" : "512971376",
+ "type" : "ext4",
+ "mountpoint" : "/"
+ },
+ {
+ "available" : "1478456",
+ "device" : "tmpfs",
+ "used" : "0",
+ "percent" : "0%",
+ "size" : "1478456",
+ "type" : "tmpfs",
+ "mountpoint" : "/dev/shm"
+ },
+ {
+ "available" : "438284",
+ "device" : "/dev/sda1",
+ "used" : "31960",
+ "percent" : "7%",
+ "size" : "495844",
+ "type" : "ext4",
+ "mountpoint" : "/boot"
+ },
+ {
+ "available" : "191426876",
+ "device" : "vagrant",
+ "used" : "295661344",
+ "percent" : "61%",
+ "size" : "487088220",
+ "type" : "vboxsf",
+ "mountpoint" : "/vagrant"
+ }
+ ],
+ "host_health_report" : "",
+ "host_name" : "c6401.ambari.apache.org",
+ "host_state" : "HEALTHY",
+ "host_status" : "HEALTHY",
+ "ip" : "192.168.64.101",
+ "last_agent_env" : {
+ "stackFoldersAndFiles" : [ ],
+ "alternatives" : [ ],
+ "existingUsers" : [ ],
+ "existingRepos" : [ ],
+ "installedPackages" : [ ],
+ "hostHealth" : {
+ "activeJavaProcs" : [ ],
+ "agentTimeStampAtReporting" : 1445290054919,
+ "serverTimeStampAtReporting" : 1445290054980,
+ "liveServices" : [
+ {
+ "desc" : "",
+ "name" : "ntpd",
+ "status" : "Healthy"
+ }
+ ]
+ },
+ "umask" : 18,
+ "transparentHugePage" : "always",
+ "firewallRunning" : false,
+ "firewallName" : "iptables",
+ "reverseLookup" : true
+ },
+ "last_heartbeat_time" : 1445290077648,
+ "last_registration_time" : 1445289898391,
+ "os_arch" : "x86_64",
+ "os_family" : "redhat6",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "recovery_report" : {
+ "summary" : "DISABLED",
+ "component_reports" : [ ]
+ },
+ "recovery_summary" : "DISABLED",
+ "total_mem" : 2956916
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6402.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "desired_configs" : null,
+ "disk_info" : [
+ {
+ "available" : "482615388",
+ "device" : "/dev/mapper/VolGroup-lv_root",
+ "used" : "4298468",
+ "percent" : "1%",
+ "size" : "512971376",
+ "type" : "ext4",
+ "mountpoint" : "/"
+ },
+ {
+ "available" : "1478456",
+ "device" : "tmpfs",
+ "used" : "0",
+ "percent" : "0%",
+ "size" : "1478456",
+ "type" : "tmpfs",
+ "mountpoint" : "/dev/shm"
+ },
+ {
+ "available" : "438284",
+ "device" : "/dev/sda1",
+ "used" : "31960",
+ "percent" : "7%",
+ "size" : "495844",
+ "type" : "ext4",
+ "mountpoint" : "/boot"
+ },
+ {
+ "available" : "191430912",
+ "device" : "vagrant",
+ "used" : "295657308",
+ "percent" : "61%",
+ "size" : "487088220",
+ "type" : "vboxsf",
+ "mountpoint" : "/vagrant"
+ }
+ ],
+ "host_health_report" : "",
+ "host_name" : "c6402.ambari.apache.org",
+ "host_state" : "HEALTHY",
+ "host_status" : "HEALTHY",
+ "ip" : "192.168.64.102",
+ "last_agent_env" : {
+ "stackFoldersAndFiles" : [ ],
+ "alternatives" : [ ],
+ "existingUsers" : [ ],
+ "existingRepos" : [ ],
+ "installedPackages" : [ ],
+ "hostHealth" : {
+ "activeJavaProcs" : [ ],
+ "agentTimeStampAtReporting" : 1445290076132,
+ "serverTimeStampAtReporting" : 1445290076343,
+ "liveServices" : [
+ {
+ "desc" : "",
+ "name" : "ntpd",
+ "status" : "Healthy"
+ }
+ ]
+ },
+ "umask" : 18,
+ "transparentHugePage" : "always",
+ "firewallRunning" : false,
+ "firewallName" : "iptables",
+ "reverseLookup" : true
+ },
+ "last_heartbeat_time" : 1445290086404,
+ "last_registration_time" : 1445290032489,
+ "os_arch" : "x86_64",
+ "os_family" : "redhat6",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6402.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "recovery_report" : {
+ "summary" : "DISABLED",
+ "component_reports" : [ ]
+ },
+ "recovery_summary" : "DISABLED",
+ "total_mem" : 2956916
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6403.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "desired_configs" : null,
+ "disk_info" : [
+ {
+ "available" : "482615388",
+ "device" : "/dev/mapper/VolGroup-lv_root",
+ "used" : "4298468",
+ "percent" : "1%",
+ "size" : "512971376",
+ "type" : "ext4",
+ "mountpoint" : "/"
+ },
+ {
+ "available" : "1478456",
+ "device" : "tmpfs",
+ "used" : "0",
+ "percent" : "0%",
+ "size" : "1478456",
+ "type" : "tmpfs",
+ "mountpoint" : "/dev/shm"
+ },
+ {
+ "available" : "438284",
+ "device" : "/dev/sda1",
+ "used" : "31960",
+ "percent" : "7%",
+ "size" : "495844",
+ "type" : "ext4",
+ "mountpoint" : "/boot"
+ },
+ {
+ "available" : "191430912",
+ "device" : "vagrant",
+ "used" : "295657308",
+ "percent" : "61%",
+ "size" : "487088220",
+ "type" : "vboxsf",
+ "mountpoint" : "/vagrant"
+ }
+ ],
+ "host_health_report" : "",
+ "host_name" : "c6403.ambari.apache.org",
+ "host_state" : "HEALTHY",
+ "host_status" : "HEALTHY",
+ "ip" : "192.168.64.103",
+ "last_agent_env" : {
+ "stackFoldersAndFiles" : [ ],
+ "alternatives" : [ ],
+ "existingUsers" : [ ],
+ "existingRepos" : [ ],
+ "installedPackages" : [ ],
+ "hostHealth" : {
+ "activeJavaProcs" : [ ],
+ "agentTimeStampAtReporting" : 1445290073223,
+ "serverTimeStampAtReporting" : 1445290073435,
+ "liveServices" : [
+ {
+ "desc" : "",
+ "name" : "ntpd",
+ "status" : "Healthy"
+ }
+ ]
+ },
+ "umask" : 18,
+ "transparentHugePage" : "always",
+ "firewallRunning" : false,
+ "firewallName" : "iptables",
+ "reverseLookup" : true
+ },
+ "last_heartbeat_time" : 1445290083492,
+ "last_registration_time" : 1445290030522,
+ "os_arch" : "x86_64",
+ "os_family" : "redhat6",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6403.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "recovery_report" : {
+ "summary" : "DISABLED",
+ "component_reports" : [ ]
+ },
+ "recovery_summary" : "DISABLED",
+ "total_mem" : 2956916
+ }
+ }
+ ]
+}
\ No newline at end of file