You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2016/11/09 21:50:09 UTC
[12/13] ambari git commit: AMBARI-18671. Ranger KMS should add proxy
users for yarn and livy (Mugdha Varadkar via smohanty)
AMBARI-18671. Ranger KMS should add proxy users for yarn and livy (Mugdha Varadkar via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2a0c9414
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2a0c9414
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2a0c9414
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 2a0c9414b4c1a6244ff3af29227072b6c9d6f43e
Parents: f33dd9e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Nov 9 13:06:32 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Nov 9 13:09:18 2016 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.3/services/stack_advisor.py | 119 +++++++++++++++----
.../stacks/HDP/2.5/services/stack_advisor.py | 15 ++-
.../stacks/2.0.6/common/test_stack_advisor.py | 29 +++++
.../stacks/2.3/common/test_stack_advisor.py | 28 +++++
.../stacks/2.5/common/test_stack_advisor.py | 9 +-
5 files changed, 164 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index d65248b..4e2a994 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -436,6 +436,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site")
putRangerKmsAuditProperty = self.putProperty(configurations, "ranger-kms-audit", services)
+ security_enabled = self.isSecurityEnabled(services)
+ putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services)
+ putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']):
@@ -477,6 +480,95 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
default_fs = services['configurations']['core-site']['properties']['fs.defaultFS']
putRangerKmsAuditProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit'))
+ required_services = [{'service' : 'YARN', 'config-type': 'yarn-env', 'property-name': 'yarn_user', 'proxy-category': ['hosts', 'users', 'groups']},
+ {'service' : 'SPARK', 'config-type': 'livy-env', 'property-name': 'livy_user', 'proxy-category': ['hosts', 'users', 'groups']}]
+
+ required_services_for_secure = [{'service' : 'HIVE', 'config-type': 'hive-env', 'property-name': 'hive_user', 'proxy-category': ['hosts', 'users']},
+ {'service' : 'OOZIE', 'config-type': 'oozie-env', 'property-name': 'oozie_user', 'proxy-category': ['hosts', 'users']}]
+
+ if security_enabled:
+ required_services.extend(required_services_for_secure)
+
+ # recommendations for kms proxy related properties
+ self.recommendKMSProxyUsers(configurations, services, hosts, required_services)
+
+ ambari_user = self.getAmbariUser(services)
+ if security_enabled:
+ # adding for ambari user
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), '*')
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), '*')
+ # adding for HTTP
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.users', '*')
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.hosts', '*')
+ else:
+ self.deleteKMSProxyUsers(configurations, services, hosts, required_services_for_secure)
+ # deleting ambari user proxy properties
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), 'delete', 'true')
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), 'delete', 'true')
+ # deleting HTTP proxy properties
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.hosts', 'delete', 'true')
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.users', 'delete', 'true')
+
+ def recommendKMSProxyUsers(self, configurations, services, hosts, requiredServices):
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services)
+ putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
+
+ if 'forced-configurations' not in services:
+ services["forced-configurations"] = []
+
+ for index in range(len(requiredServices)):
+ service = requiredServices[index]['service']
+ config_type = requiredServices[index]['config-type']
+ property_name = requiredServices[index]['property-name']
+ proxy_category = requiredServices[index]['proxy-category']
+
+ if service in servicesList:
+ if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']:
+ service_user = services['configurations'][config_type]['properties'][property_name]
+ service_old_user = getOldValue(self, services, config_type, property_name)
+
+ if 'groups' in proxy_category:
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.groups'.format(service_user), '*')
+ if 'hosts' in proxy_category:
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(service_user), '*')
+ if 'users' in proxy_category:
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(service_user), '*')
+
+ if service_old_user is not None and service_user != service_old_user:
+ if 'groups' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_old_user), 'delete', 'true')
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_old_user)})
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_user)})
+ if 'hosts' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_old_user), 'delete', 'true')
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_old_user)})
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_user)})
+ if 'users' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_old_user), 'delete', 'true')
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_old_user)})
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_user)})
+
+ def deleteKMSProxyUsers(self, configurations, services, hosts, requiredServices):
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
+
+ for index in range(len(requiredServices)):
+ service = requiredServices[index]['service']
+ config_type = requiredServices[index]['config-type']
+ property_name = requiredServices[index]['property-name']
+ proxy_category = requiredServices[index]['proxy-category']
+
+ if service in servicesList:
+ if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']:
+ service_user = services['configurations'][config_type]['properties'][property_name]
+
+ if 'groups' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_user), 'delete', 'true')
+ if 'hosts' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_user), 'delete', 'true')
+ if 'users' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_user), 'delete', 'true')
def getOracleDBConnectionHostPort(self, db_type, db_host, rangerDbName):
connection_string = self.getDBConnectionHostPort(db_type, db_host)
@@ -696,16 +788,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
else:
putYarnSitePropertyAttributes('yarn.authorization-provider', 'delete', 'true')
- if 'yarn-site' in services["configurations"] and 'yarn.resourcemanager.proxy-user-privileges.enabled' in services["configurations"]["yarn-site"]["properties"]:
- if self.isSecurityEnabled(services):
- # enable proxy-user privileges for secure clusters for long-running services (spark streaming etc)
- putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'true')
- if 'RANGER_KMS' in servicesList:
- # disable proxy-user privileges on secure clusters as it does not work with TDE
- putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false')
- else:
- putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false')
-
def recommendSqoopConfigurations(self, configurations, clusterData, services, hosts):
putSqoopSiteProperty = self.putProperty(configurations, "sqoop-site", services)
@@ -753,8 +835,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
"HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations,
"hive-site": self.validateHiveConfigurations},
"HBASE": {"hbase-site": self.validateHBASEConfigurations},
- "KAFKA": {"kafka-broker": self.validateKAFKAConfigurations},
- "YARN": {"yarn-site": self.validateYARNConfigurations},
+ "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations},
"RANGER": {"admin-properties": self.validateRangerAdminConfigurations,
"ranger-env": self.validateRangerConfigurationsEnv}
}
@@ -950,20 +1031,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
return self.toConfigurationValidationProblems(validationItems, "kafka-broker")
- def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- yarn_site = properties
- validationItems = []
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if 'RANGER_KMS' in servicesList and 'KERBEROS' in servicesList:
- yarn_resource_proxy_enabled = yarn_site['yarn.resourcemanager.proxy-user-privileges.enabled']
- if yarn_resource_proxy_enabled.lower() == 'true':
- validationItems.append({"config-name": 'yarn.resourcemanager.proxy-user-privileges.enabled',
- "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled " \
- "property value as false under yarn-site"
- )})
-
- return self.toConfigurationValidationProblems(validationItems, "yarn-site")
-
def isComponentUsingCardinalityForLayout(self, componentName):
return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 9ceebb9..dfb7b0c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -1699,16 +1699,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts):
super(HDP25StackAdvisor, self).recommendRangerKMSConfigurations(configurations, clusterData, services, hosts)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services)
- if 'ranger-env' in services['configurations'] and 'ranger_user' in services['configurations']['ranger-env']['properties']:
- rangerUser = services['configurations']['ranger-env']['properties']['ranger_user']
+ security_enabled = self.isSecurityEnabled(services)
+ required_services = [{'service' : 'RANGER', 'config-type': 'ranger-env', 'property-name': 'ranger_user', 'proxy-category': ['hosts', 'users', 'groups']}]
- if 'kms-site' in services['configurations'] and 'KERBEROS' in servicesList:
- putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.groups'.format(rangerUser), '*')
- putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(rangerUser), '*')
- putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(rangerUser), '*')
+ if security_enabled:
+ # recommendations for kms proxy related properties
+ self.recommendKMSProxyUsers(configurations, services, hosts, required_services)
+ else:
+ self.deleteKMSProxyUsers(configurations, services, hosts, required_services)
def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
super(HDP25StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 0a73c55..125c9ca 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -2648,3 +2648,32 @@ class TestHDP206StackAdvisor(TestCase):
self.assertFalse(recommendations['blueprint']['host_groups'][0]['components'])
# Assert that DATANODE is placed on host-group-2
self.assertEquals(recommendations['blueprint']['host_groups'][1]['components'][0]['name'], 'DATANODE')
+
+ def test_validateYARNConfigurations(self):
+ configurations = {
+ "cluster-env": {
+ "properties": {
+ "user_group": "hadoop",
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ 'yarn.nodemanager.resource.memory-mb' : '12288',
+ 'yarn.scheduler.minimum-allocation-mb' : '3072',
+ 'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+ 'yarn.scheduler.maximum-allocation-mb': '12288'
+ }
+ }
+ }
+ services = {'configurations': {} }
+ recommendedDefaults = {'yarn.nodemanager.resource.memory-mb' : '12288',
+ 'yarn.scheduler.minimum-allocation-mb' : '3072',
+ 'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+ 'yarn.scheduler.maximum-allocation-mb': '12288'}
+ properties = {'yarn.nodemanager.resource.memory-mb' : '12288',
+ 'yarn.scheduler.minimum-allocation-mb' : '3072',
+ 'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+ 'yarn.scheduler.maximum-allocation-mb': '12288'}
+
+ res = self.stackAdvisor.validateYARNConfigurations(properties, recommendedDefaults, configurations, services, {})
+ self.assertFalse(res)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index ad6842f..39cbbce 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1543,6 +1543,9 @@ class TestHDP23StackAdvisor(TestCase):
def test_recommendRangerKMSConfigurations(self):
clusterData = {}
services = {
+ "ambari-server-properties": {
+ "ambari-server.user": "root"
+ },
"Versions": {
"stack_version" : "2.3",
},
@@ -1583,6 +1586,11 @@ class TestHDP23StackAdvisor(TestCase):
'db_host' : 'c6401.ambari.apache.org:1521:XE',
'db_name' : "XE"
}
+ },
+ 'cluster-env': {
+ 'properties': {
+ 'security_enabled': 'false'
+ }
}
},
"forced-configurations": []
@@ -1604,6 +1612,16 @@ class TestHDP23StackAdvisor(TestCase):
'ranger-kms-audit': {
'properties': {
}
+ },
+ 'kms-site': {
+ 'properties': {
+ },
+ 'property_attributes': {
+ 'hadoop.kms.proxyuser.HTTP.hosts': {'delete': 'true'},
+ 'hadoop.kms.proxyuser.HTTP.users': {'delete': 'true'},
+ 'hadoop.kms.proxyuser.root.hosts': {'delete': 'true'},
+ 'hadoop.kms.proxyuser.root.users': {'delete': 'true'}
+ }
}
}
@@ -1618,6 +1636,8 @@ class TestHDP23StackAdvisor(TestCase):
"service_name": "KERBEROS"
}
})
+ services['configurations']['cluster-env']['properties']['security_enabled'] = "true"
+ services['configurations']['cluster-env']['properties']['ambari_principal_name'] = "ambari-cl1@EXAMPLE.COM"
expected = {
'kms-properties': {
@@ -1637,6 +1657,14 @@ class TestHDP23StackAdvisor(TestCase):
'ranger-kms-audit': {
'properties': {
}
+ },
+ 'kms-site': {
+ 'properties': {
+ 'hadoop.kms.proxyuser.HTTP.hosts': '*',
+ 'hadoop.kms.proxyuser.HTTP.users': '*',
+ 'hadoop.kms.proxyuser.ambari-cl1.hosts': '*',
+ 'hadoop.kms.proxyuser.ambari-cl1.users': '*'
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 6fe357a..7124307 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -446,9 +446,9 @@ class TestHDP25StackAdvisor(TestCase):
def test_validateYarnConfigurations(self):
properties = {'enable_hive_interactive': 'true',
'hive_server_interactive_host': 'c6401.ambari.apache.org',
- 'hive.tez.container.size': '2048'}
+ 'hive.tez.container.size': '2048', "yarn.nodemanager.linux-container-executor.group": "hadoop"}
recommendedDefaults = {'enable_hive_interactive': 'true',
- "hive_server_interactive_host": "c6401.ambari.apache.org"}
+ "hive_server_interactive_host": "c6401.ambari.apache.org", "yarn.nodemanager.linux-container-executor.group": "hadoop"}
configurations = {
"hive-interactive-env": {
"properties": {'enable_hive_interactive': 'true', "hive_server_interactive_host": "c6401.ambari.apache.org"}
@@ -461,6 +461,11 @@ class TestHDP25StackAdvisor(TestCase):
},
"yarn-site": {
"properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "false"}
+ },
+ "cluster-env": {
+ "properties": {
+ "user_group": "hadoop",
+ }
}
}
services = self.load_json("services-normal-his-valid.json")