You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2016/03/28 10:50:02 UTC
ambari git commit: AMBARI-15587. improve ranger kms install
integeration (alexantonenko)
Repository: ambari
Updated Branches:
refs/heads/trunk 12b59857e -> aa033d0c1
AMBARI-15587. improve ranger kms install integeration (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aa033d0c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aa033d0c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aa033d0c
Branch: refs/heads/trunk
Commit: aa033d0c152acdc8d1f7640ccd60f77b9710ed80
Parents: 12b5985
Author: Alex Antonenko <hi...@gmail.com>
Authored: Fri Mar 25 18:49:59 2016 +0200
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Mon Mar 28 11:49:57 2016 +0300
----------------------------------------------------------------------
.../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 10 ++
.../0.5.0.2.3/package/scripts/params.py | 7 +-
.../stacks/HDP/2.3/services/stack_advisor.py | 14 ++-
.../stacks/2.3/common/test_stack_advisor.py | 104 +++++++++++++++++++
4 files changed, 132 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa033d0c/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
index 92fe529..11a705a 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
@@ -256,6 +256,16 @@ def kms(upgrade_type=None):
content=params.kms_log4j,
mode=0644
)
+ if params.stack_is_hdp23_or_further and params.security_enabled:
+ # core-site.xml linking required by setup for HDFS encryption
+ XmlConfig("core-site.xml",
+ conf_dir=params.kms_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0644
+ )
def copy_jdbc_connector(stack_version=None):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa033d0c/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
index 30eda0b..ae4591e 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
@@ -18,6 +18,7 @@ limitations under the License.
"""
import os
+from resource_management.libraries.functions import conf_select
from resource_management.libraries.script import Script
from resource_management.libraries.functions.version import format_stack_version, compare_versions
from resource_management.libraries.functions.format import format
@@ -33,11 +34,13 @@ stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
stack_version_formatted = format_stack_version(stack_version_unformatted)
stack_is_hdp23_or_further = Script.is_stack_greater_or_equal("2.3")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+security_enabled = config['configurations']['cluster-env']['security_enabled']
if stack_is_hdp23_or_further:
kms_home = '/usr/hdp/current/ranger-kms'
kms_conf_dir = '/usr/hdp/current/ranger-kms/conf'
-
+
kms_log_dir = default("/configurations/kms-env/kms_log_dir", "/var/log/ranger/kms")
java_home = config['hostLevelParams']['java_home']
kms_user = default("/configurations/kms-env/kms_user", "kms")
@@ -203,4 +206,4 @@ if current_host in ranger_kms_hosts:
check_db_connection_jar_name = "DBConnectionVerification.jar"
check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}")
ranger_kms_jdbc_connection_url = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.url']
-ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver']
\ No newline at end of file
+ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver']
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa033d0c/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 92d18e2..741011c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -460,6 +460,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
putRangerKmsDbksProperty = self.putProperty(configurations, "dbks-site", services)
putRangerKmsProperty = self.putProperty(configurations, "kms-properties", services)
+ kmsEnvProperties = getSiteProperties(services['configurations'], 'kms-env')
+ putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
+ putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site")
if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']):
@@ -492,6 +495,15 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
for key in rangerKmsDbProperties:
putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key))
+ if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList:
+ kmsUser = kmsEnvProperties['kms_user']
+ kmsUserOld = getOldValue(self, services, 'kms-env', 'kms_user')
+ putCoreSiteProperty('hadoop.proxyuser.{0}.groups'.format(kmsUser), '*')
+ if kmsUserOld is not None and kmsUser != kmsUserOld:
+ putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true')
+ services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)})
+ services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUser)})
+
def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
super(HDP23StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts)
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
@@ -1025,7 +1037,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
validationItems.append({"config-name": PROP_NAME, "item": self.getWarnItem(message.format(PROP_NAME, str(limit)))})
return self.toConfigurationValidationProblems(validationItems, "hdfs-client")
-
+
def isComponentUsingCardinalityForLayout(self, componentName):
return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa033d0c/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 04c69c4..d415b6f 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -2215,3 +2215,107 @@ class TestHDP23StackAdvisor(TestCase):
problems = self.stackAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
self.assertEqual(len(problems), 1)
self.assertEqual(problems[0], expected)
+
+ def test_recommendRangerKMSConfigurations(self):
+ clusterData = {}
+ services = {
+ "Versions": {
+ "stack_version" : "2.3",
+ },
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "RANGER",
+ "service_version": "0.5.0.2.3"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "RANGER_ADMIN",
+ "hostnames": ["host1"]
+ }
+ }
+ ]
+ }
+ ],
+ "configurations": {
+ "kms-env": {
+ "properties": {
+ "kms_user": "kmsname"
+ }
+ },
+ "core-site": {
+ "properties": {
+ }
+ }
+ },
+ "forced-configurations": []
+ }
+ expected = {
+ 'kms-properties': {
+ 'properties': {}
+ },
+ 'dbks-site': {
+ 'properties': {}
+ },
+ 'core-site': {
+ 'properties': {
+ }
+ }
+ }
+
+ # non kerberized cluster. There should be no proxyuser configs
+ recommendedConfigurations = {}
+ self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
+ self.assertEquals(recommendedConfigurations, expected)
+
+ # kerberized cluster
+ services['services'].append({
+ "StackServices": {
+ "service_name": "KERBEROS"
+ }
+ })
+
+ expected = {
+ 'kms-properties': {
+ 'properties': {}
+ },
+ 'dbks-site': {
+ 'properties': {}
+ },
+ 'core-site': {
+ 'properties': {
+ 'hadoop.proxyuser.kmsname.groups': '*'
+ }
+ }
+ }
+
+ # on kerberized cluster property should be recommended
+ recommendedConfigurations = {}
+ self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
+ self.assertEquals(recommendedConfigurations, expected)
+
+ recommendedConfigurations = {}
+ services['changed-configurations'] = [
+ {
+ 'type': 'kms-env',
+ 'name': 'kms_user',
+ 'old_value': 'kmsname'
+ }
+ ]
+ services['configurations']['kms-env']['properties']['kms_user'] = 'kmsnew'
+
+ expected['core-site'] = {
+ 'properties': {
+ 'hadoop.proxyuser.kmsnew.groups': '*'
+ },
+ 'property_attributes': {
+ 'hadoop.proxyuser.kmsname.groups': {
+ 'delete': 'true'
+ }
+ }
+ }
+
+ # kms_user was changed, old property should be removed
+ self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
+ self.assertEquals(recommendedConfigurations, expected)