You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/08/19 08:34:43 UTC
[2/3] ambari git commit: AMBARI-18157. Hive service check is failing
after cluster Kerberization (dgrinenko via dlysnichenko)
AMBARI-18157. Hive service check is failing after cluster Kerberization (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2ad7c802
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2ad7c802
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2ad7c802
Branch: refs/heads/trunk
Commit: 2ad7c8024403be251235e49898d0ef1203ad6563
Parents: 356feed
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Aug 19 11:33:37 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Aug 19 11:33:37 2016 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 91 ++++++++++++++++++++
.../stacks/HDP/2.1/services/stack_advisor.py | 24 +++++-
.../stacks/2.0.6/common/test_stack_advisor.py | 53 ++++++++++++
.../stacks/2.1/common/test_stack_advisor.py | 83 ++++++++++++++++++
4 files changed, 250 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ad7c802/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 7133c27..7aeda19 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -1679,6 +1679,97 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
leafQueueNames.add(leafQueueName)
return leafQueueNames
+ def get_service_component_meta(self, service, component, services):
+ """
+ Function retrieve service component meta information as dict from services.json
+ If no service or component found, would be returned empty dict
+
+ Return value example:
+ "advertise_version" : true,
+ "bulk_commands_display_name" : "",
+ "bulk_commands_master_component_name" : "",
+ "cardinality" : "1+",
+ "component_category" : "CLIENT",
+ "component_name" : "HBASE_CLIENT",
+ "custom_commands" : [ ],
+ "decommission_allowed" : false,
+ "display_name" : "HBase Client",
+ "has_bulk_commands_definition" : false,
+ "is_client" : true,
+ "is_master" : false,
+ "reassign_allowed" : false,
+ "recovery_enabled" : false,
+ "service_name" : "HBASE",
+ "stack_name" : "HDP",
+ "stack_version" : "2.5",
+ "hostnames" : [ "host1", "host2" ]
+
+ :type service str
+ :type component str
+ :type services dict
+ :rtype dict
+ """
+ __stack_services = "StackServices"
+ __stack_service_components = "StackServiceComponents"
+
+ if not services:
+ return {}
+
+ service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
+ if len(service_meta) == 0:
+ return {}
+
+ service_meta = service_meta[0]
+ component_meta = [item for item in service_meta["components"] if item[__stack_service_components]["component_name"] == component]
+
+ if len(component_meta) == 0:
+ return {}
+
+ return component_meta[0][__stack_service_components]
+
+ def is_secured_cluster(self, services):
+ """
+ Detects if cluster is secured or not
+ :type services dict
+ :rtype bool
+ """
+ return services and "cluster-env" in services["configurations"] and\
+ "security_enabled" in services["configurations"]["cluster-env"]["properties"] and\
+ services["configurations"]["cluster-env"]["properties"]["security_enabled"].lower() == "true"
+
+ def get_services_list(self, services):
+ """
+ Returns available services as list
+
+ :type services dict
+ :rtype list
+ """
+ if not services:
+ return []
+
+ return [service["StackServices"]["service_name"] for service in services["services"]]
+
+ def get_components_list(self, service, services):
+ """
+ Return list of components for specific service
+ :type service str
+ :type services dict
+ :rtype list
+ """
+ __stack_services = "StackServices"
+ __stack_service_components = "StackServiceComponents"
+
+ if not services:
+ return []
+
+ service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
+ if len(service_meta) == 0:
+ return []
+
+ service_meta = service_meta[0]
+ return [item[__stack_service_components]["component_name"] for item in service_meta["components"]]
+
+
def getOldValue(self, services, configType, propertyName):
if services:
if 'changed-configurations' in services.keys():
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ad7c802/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 34b4614..9678dc1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -107,7 +107,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
dbConnection = self.getDBConnectionString(hiveEnvProperties['hive_database']).format(hiveServerHost['Hosts']['host_name'], hiveSiteProperties['ambari.hive.db.schema.name'])
putHiveProperty('javax.jdo.option.ConnectionURL', dbConnection)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ servicesList = self.get_services_list(services)
if "PIG" in servicesList:
ambari_user = self.getAmbariUser(services)
ambariHostName = socket.getfqdn()
@@ -120,6 +120,28 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
+ if self.is_secured_cluster(services):
+ appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services)
+
+ def updateCallback(originalValue, newValue):
+ """
+ :type originalValue str
+ :type newValue list
+ """
+ if originalValue and not originalValue.isspace():
+ hosts = originalValue.split(',')
+
+ if newValue:
+ hosts.extend(newValue)
+
+ result = ','.join(set(hosts))
+ return result
+ else:
+ return ','.join(set(newValue))
+
+ meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services)
+ if "hostnames" in meta:
+ appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', meta["hostnames"], updateCallback)
def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
putTezProperty = self.putProperty(configurations, "tez-site")
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ad7c802/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index e28389e..a70922f 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -188,6 +188,59 @@ class TestHDP206StackAdvisor(TestCase):
]
self.assertValidationResult(expectedItems, result)
+ def test_get_components_list(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ expected = sorted(["GANGLIA_MONITOR", "GANGLIA_SERVER"])
+
+ services = self.prepareServices(servicesInfo)
+ result = sorted(self.stackAdvisor.get_components_list("GANGLIA", services))
+
+ self.assertEqual(expected, result)
+
+ def test_get_services_list(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ expected = ["GANGLIA"]
+
+ services = self.prepareServices(servicesInfo)
+ result = self.stackAdvisor.get_services_list(services)
+
+ self.assertEqual(expected, result)
+
+ def test_get_service_component_meta(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ expected = ["host1"]
+
+ services = self.prepareServices(servicesInfo)
+ result = self.stackAdvisor.get_service_component_meta("GANGLIA", "GANGLIA_MONITOR", services)
+
+ self.assertEquals(True, "hostnames" in result)
+ self.assertEqual(expected, result["hostnames"])
+
+
def test_validationWarnMessagesIfLessThanDefault(self):
servicesInfo = [
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ad7c802/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
index e174461..7835262 100644
--- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
@@ -270,6 +270,89 @@ class TestHDP21StackAdvisor(TestCase):
self.assertEquals(configurations['hive-site']['properties']['javax.jdo.option.ConnectionURL'], "jdbc:mysql://example.com/hive_name")
self.assertEquals(configurations['hive-site']['properties']['javax.jdo.option.ConnectionDriverName'], "com.mysql.jdbc.Driver")
+ def test_recommendHiveConfigurationsSecure(self):
+ services = {
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "HIVE",
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "WEBHCAT_SERVER",
+ "service_name": "HIVE",
+ "hostnames": ["example.com"]
+ }
+ }
+ ]
+ }
+ ],
+ "configurations": {}
+ }
+
+ configurations = {
+ "hive-site": {
+ "properties": {
+ "javax.jdo.option.ConnectionDriverName": "",
+ "ambari.hive.db.schema.name": "hive_name",
+ "javax.jdo.option.ConnectionURL": "jdbc:mysql://localhost/hive?createDatabaseIfNotExist=true"
+ }
+ },
+ "hive-env": {
+ "properties": {
+ "hive_database": "New MySQL Database"
+ }
+ },
+ "cluster-env": {
+ "properties": {
+ "security_enabled": "true"
+ }
+ }
+ }
+
+ services['configurations'] = configurations
+ clusterData = {
+ "mapMemory": 3000,
+ "reduceMemory": 2056,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+ services['changed-configurations'] = []
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "host_name": "example.com"
+ }
+ },
+ {
+ "Hosts": {
+ "host_name": "example.org"
+ }
+ }
+ ]
+ }
+
+ # new mysql
+ self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals("core-site" in configurations, True)
+ self.assertEqual("hadoop.proxyuser.HTTP.hosts" in configurations["core-site"]["properties"], True)
+ self.assertEqual(configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"] == "example.com", True)
+
+ newhost_list = ["example.com", "example.org"]
+ services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = newhost_list
+ configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"] = ""
+
+ self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals("core-site" in configurations, True)
+ self.assertEqual("hadoop.proxyuser.HTTP.hosts" in configurations["core-site"]["properties"], True)
+
+ fetch_list = sorted(configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"].split(","))
+ self.assertEqual(sorted(newhost_list), fetch_list)
+
+
+
def test_recommendHiveConfigurations_containersRamIsLess(self):
configurations = {}
clusterData = {