You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2016/06/10 20:18:01 UTC
ambari git commit: AMBARI-17054. Configure Atlas Ranger Plugin - with
test fixes. (Gautam Borad via srimanth)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 08bc4c465 -> 0038e3bd9
AMBARI-17054. Configure Atlas Ranger Plugin - with test fixes. (Gautam Borad via srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0038e3bd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0038e3bd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0038e3bd
Branch: refs/heads/branch-2.4
Commit: 0038e3bd9a2cbc725543cb948669f28d9e765649
Parents: 08bc4c4
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Fri Jun 10 13:17:41 2016 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Fri Jun 10 13:17:41 2016 -0700
----------------------------------------------------------------------
.../libraries/functions/constants.py | 1 +
.../package/scripts/metadata_server.py | 24 ++--
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 98 +++++++++++++++
.../package/scripts/setup_ranger_atlas.py | 70 +++++++++++
.../0.1.0.2.3/package/scripts/status_params.py | 4 +
.../RANGER/0.6.0/configuration/ranger-env.xml | 23 +++-
.../RANGER/0.6.0/themes/theme_version_3.json | 28 ++++-
.../HDP/2.0.6/properties/stack_features.json | 5 +
.../configuration/application-properties.xml | 12 ++
.../ATLAS/configuration/ranger-atlas-audit.xml | 122 +++++++++++++++++++
.../ranger-atlas-plugin-properties.xml | 77 ++++++++++++
.../ranger-atlas-policymgr-ssl.xml | 67 ++++++++++
.../configuration/ranger-atlas-security.xml | 64 ++++++++++
.../stacks/HDP/2.5/services/stack_advisor.py | 20 ++-
.../stacks/2.5/common/test_stack_advisor.py | 23 +++-
15 files changed, 626 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 555a215..7e85115 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -86,3 +86,4 @@ class StackFeature:
RANGER_USERSYNC_PASSWORD_JCEKS = "ranger_usersync_password_jceks"
LOGSEARCH_SUPPORT = "logsearch_support"
HBASE_HOME_DIRECTORY = "hbase_home_directory"
+ ATLAS_RANGER_PLUGIN_SUPPORT = "atlas_ranger_plugin_support"
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
index 1e9e7a7..bf3125e 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
@@ -29,6 +29,8 @@ from resource_management.libraries.functions.stack_features import check_stack_f
from resource_management.libraries.functions import StackFeature
import os
import shutil
+from resource_management.core.logger import Logger
+from setup_ranger_atlas import setup_ranger_atlas
class MetadataServer(Script):
@@ -68,7 +70,13 @@ class MetadataServer(Script):
daemon_cmd = format('source {params.conf_dir}/atlas-env.sh ; {params.metadata_start_script}')
no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p `cat {params.pid_file}` >/dev/null 2>&1')
-
+
+ if params.stack_supports_atlas_ranger_plugin:
+ Logger.info('Atlas plugin is enabled, configuring Atlas plugin.')
+ setup_ranger_atlas(upgrade_type = upgrade_type)
+ else:
+ Logger.info('Atlas plugin is not supported or enabled.')
+
try:
Execute(daemon_cmd,
user=params.metadata_user,
@@ -82,7 +90,7 @@ class MetadataServer(Script):
import params
env.set_params(params)
daemon_cmd = format('source {params.conf_dir}/atlas-env.sh; {params.metadata_stop_script}')
-
+
try:
Execute(daemon_cmd,
user=params.metadata_user,
@@ -90,7 +98,7 @@ class MetadataServer(Script):
except:
show_logs(params.log_dir, params.metadata_user)
raise
-
+
File(params.pid_file, action="delete")
def status(self, env):
@@ -113,9 +121,9 @@ class MetadataServer(Script):
props_read_check = ['atlas.authentication.keytab',
'atlas.http.authentication.kerberos.keytab']
atlas_site_expectations = build_expectations('application',
- props_value_check,
- props_empty_check,
- props_read_check)
+ props_value_check,
+ props_empty_check,
+ props_read_check)
atlas_expectations = {}
atlas_expectations.update(atlas_site_expectations)
@@ -152,11 +160,11 @@ class MetadataServer(Script):
issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
self.put_structured_out({"securityState": "UNSECURED"})
-
+
def get_log_folder(self):
import params
return params.log_dir
-
+
def get_user(self):
import params
return params.metadata_user
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 66c07b1..634677f 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -24,6 +24,9 @@ from resource_management.libraries.functions import format
from resource_management.libraries.functions.default import default
import status_params
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.is_empty import is_empty
# server configurations
config = Script.get_config()
@@ -158,3 +161,98 @@ for host in zookeeper_hosts:
index += 1
if index < len(zookeeper_hosts):
zookeeper_quorum += ","
+
+
+# Atlas Ranger plugin configurations
+stack_supports_atlas_ranger_plugin = stack_version_formatted and check_stack_feature(StackFeature.ATLAS_RANGER_PLUGIN_SUPPORT, stack_version_formatted)
+stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted)
+retry_enabled = default("/commandParams/command_retry_enabled", False)
+
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
+enable_ranger_atlas = False
+
+
+
+
+if has_ranger_admin and stack_supports_atlas_ranger_plugin:
+ # for create_hdfs_directory
+ hadoop_bin_dir = status_params.hadoop_bin_dir
+ namenode_host = set(default("/clusterHostInfo/namenode_host", []))
+ has_namenode = not len(namenode_host) == 0
+ hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
+ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None
+ hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None
+ hdfs_site = config['configurations']['hdfs-site']
+ default_fs = config['configurations']['core-site']['fs.defaultFS']
+ dfs_type = default("/commandParams/dfs_type", "")
+
+ import functools
+ from resource_management.libraries.resources.hdfs_resource import HdfsResource
+ from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+ #create partial functions with common arguments for every HdfsResource call
+ #to create hdfs directory we need to call params.HdfsResource in code
+
+ HdfsResource = functools.partial(
+ HdfsResource,
+ user = hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources(),
+ dfs_type = dfs_type
+ )
+
+ repo_name = str(config['clusterName']) + '_atlas'
+ ssl_keystore_password = unicode(config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'])
+ ssl_truststore_password = unicode(config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'])
+ credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
+ xa_audit_hdfs_is_enabled = default('/configurations/ranger-atlas-audit/xasecure.audit.destination.hdfs', False)
+ enable_ranger_atlas = config['configurations']['ranger-atlas-plugin-properties']['ranger-atlas-plugin-enabled']
+ enable_ranger_atlas = not is_empty(enable_ranger_atlas) and enable_ranger_atlas.lower() == 'yes'
+ policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+ atlas_hosts = sorted(default('/clusterHostInfo/atlas_server_hosts', []))
+ metadata_server_host = atlas_hosts[0]
+ metadata_server_url = format('{metadata_protocol}://{metadata_server_host}:{metadata_port}')
+
+ downloaded_custom_connector = None
+ driver_curl_source = None
+ driver_curl_target = None
+
+ ranger_env = config['configurations']['ranger-env']
+ ranger_plugin_properties = config['configurations']['ranger-atlas-plugin-properties']
+
+ ranger_atlas_audit = config['configurations']['ranger-atlas-audit']
+ ranger_atlas_audit_attrs = config['configuration_attributes']['ranger-atlas-audit']
+ ranger_atlas_security = config['configurations']['ranger-atlas-security']
+ ranger_atlas_security_attrs = config['configuration_attributes']['ranger-atlas-security']
+ ranger_atlas_policymgr_ssl = config['configurations']['ranger-atlas-policymgr-ssl']
+ ranger_atlas_policymgr_ssl_attrs = config['configuration_attributes']['ranger-atlas-policymgr-ssl']
+
+ policy_user = config['configurations']['ranger-atlas-plugin-properties']['policy_user']
+
+ atlas_repository_configuration = {
+ 'username' : config['configurations']['ranger-atlas-plugin-properties']['REPOSITORY_CONFIG_USERNAME'],
+ 'password' : unicode(config['configurations']['ranger-atlas-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']),
+ 'atlas.rest.address' : metadata_server_url,
+ 'commonNameForCertificate' : config['configurations']['ranger-atlas-plugin-properties']['common.name.for.certificate'],
+ 'ambari.service.check.user' : policy_user
+ }
+ if security_enabled:
+ atlas_repository_configuration['policy.download.auth.users'] = metadata_user
+ atlas_repository_configuration['tag.download.auth.users'] = metadata_user
+
+ atlas_ranger_plugin_repo = {
+ 'isEnabled': 'true',
+ 'configs': atlas_repository_configuration,
+ 'description': 'atlas repo',
+ 'name': repo_name,
+ 'type': 'atlas',
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py
new file mode 100644
index 0000000..f5d7f38
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from resource_management.core.logger import Logger
+
+def setup_ranger_atlas(upgrade_type=None):
+ import params
+
+ if params.has_ranger_admin:
+
+ from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
+
+ if params.retry_enabled:
+ Logger.info("ATLAS: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("ATLAS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
+ if params.enable_ranger_atlas and params.xa_audit_hdfs_is_enabled:
+ if params.has_namenode:
+ params.HdfsResource("/ranger/audit",
+ type="directory",
+ action="create_on_execute",
+ owner=params.metadata_user,
+ group=params.user_group,
+ mode=0755,
+ recursive_chmod=True
+ )
+ params.HdfsResource("/ranger/audit/atlas",
+ type="directory",
+ action="create_on_execute",
+ owner=params.metadata_user,
+ group=params.user_group,
+ mode=0700,
+ recursive_chmod=True
+ )
+ params.HdfsResource(None, action="execute")
+
+ setup_ranger_plugin('atlas-server', 'atlas',None,
+ params.downloaded_custom_connector, params.driver_curl_source,
+ params.driver_curl_target, params.java64_home,
+ params.repo_name, params.atlas_ranger_plugin_repo,
+ params.ranger_env, params.ranger_plugin_properties,
+ params.policy_user, params.policymgr_mgr_url,
+ params.enable_ranger_atlas, conf_dict=params.conf_dir,
+ component_user=params.metadata_user, component_group=params.user_group, cache_service_list=['atlas'],
+ plugin_audit_properties=params.config['configurations']['ranger-atlas-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-atlas-audit'],
+ plugin_security_properties=params.config['configurations']['ranger-atlas-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-atlas-security'],
+ plugin_policymgr_ssl_properties=params.config['configurations']['ranger-atlas-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-atlas-policymgr-ssl'],
+ component_list=['atlas-server'], audit_db_is_enabled=False,
+ credential_file=params.credential_file, xa_audit_db_password=None,
+ ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
+ api_version = 'v2', skip_if_rangeradmin_down = not params.retry_enabled, is_security_enabled = params.security_enabled,
+ is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
+ component_user_principal=params.atlas_jaas_principal if params.security_enabled else None,
+ component_user_keytab=params.atlas_keytab_path if params.security_enabled else None)
+ else:
+ Logger.info('Ranger admin not installed')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
index 4c54214..0b0d2ae 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
@@ -21,6 +21,8 @@ import os
from resource_management import Script
from resource_management.libraries.functions import get_kinit_path, format
from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
config = Script.get_config()
@@ -38,3 +40,5 @@ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executab
tmp_dir = Script.get_tmp_dir()
stack_name = default("/hostLevelParams/stack_name", None)
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-env.xml b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-env.xml
index 6eb312f..960c575 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-env.xml
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-env.xml
@@ -40,4 +40,25 @@
<value>1</value>
<on-ambari-upgrade add="true"/>
</property>
-</configuration>
+ <property>
+ <name>ranger-atlas-plugin-enabled</name>
+ <value>No</value>
+ <display-name>Atlas Ranger Plugin</display-name>
+ <description>Enable Atlas Ranger plugin</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>Yes</value>
+ <label>ON</label>
+ </entry>
+ <entry>
+ <value>No</value>
+ <label>OFF</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/common-services/RANGER/0.6.0/themes/theme_version_3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/themes/theme_version_3.json b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/themes/theme_version_3.json
index 0f7b0c0..3f50774 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/themes/theme_version_3.json
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/themes/theme_version_3.json
@@ -78,6 +78,26 @@
"configuration-layout": "default",
"configs": [
{
+ "config": "ranger-env/ranger-atlas-plugin-enabled",
+ "subsection-name": "section-ranger-plugin-row1-col2",
+ "depends-on": [
+ {
+ "resource": "service",
+ "if": "ATLAS",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
"config": "ranger-tagsync-site/ranger.tagsync.source.atlas",
"subsection-name": "subsection-ranger-tagsync-row1-col1"
},
@@ -353,6 +373,12 @@
}
},
{
+ "config": "ranger-env/ranger-atlas-plugin-enabled",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
"config": "ranger-ugsync-site/ranger.usersync.user.searchenabled",
"widget": {
"type": "toggle"
@@ -366,4 +392,4 @@
}
]
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index 734d5b4..8ad53da 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -240,6 +240,11 @@
"name": "spark_livy",
"description": "Livy as slave component of spark",
"min_version": "2.5.0.0"
+ },
+ {
+ "name": "atlas_ranger_plugin_support",
+ "description": "Atlas Ranger plugin support",
+ "min_version": "2.5.0.0"
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
index 11e636f..20f3173 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
@@ -209,4 +209,16 @@
<deleted>true</deleted>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>atlas.authorizer.impl</name>
+ <description>
+ Atlas authorizer class
+ </description>
+ <depends-on>
+ <property>
+ <type>ranger-atlas-plugin-properties</type>
+ <name>ranger-atlas-plugin-enabled</name>
+ </property>
+ </depends-on>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-audit.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-audit.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-audit.xml
new file mode 100644
index 0000000..9c4ad88
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-audit.xml
@@ -0,0 +1,122 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+ <property>
+ <name>xasecure.audit.is.enabled</name>
+ <value>true</value>
+ <description>Is Audit enabled?</description>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs</name>
+ <value>true</value>
+ <display-name>Audit to HDFS</display-name>
+ <description>Is Audit to HDFS enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.hdfs</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ <value>hdfs://NAMENODE_HOSTNAME:8020/ranger/audit</value>
+ <description>HDFS folder to write audit to, make sure the service user has requried permissions</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs.batch.filespool.dir</name>
+ <value>/var/log/hadoop/hdfs/audit/hdfs/spool</value>
+ <description>/var/log/hadoop/hdfs/audit/hdfs/spool</description>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.solr</name>
+ <value>false</value>
+ <display-name>Audit to SOLR</display-name>
+ <description>Is Solr audit enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.solr</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.solr.urls</name>
+ <value></value>
+ <description>Solr URL</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.urls</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.solr.zookeepers</name>
+ <value>NONE</value>
+ <description>Solr Zookeeper string</description>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.zookeepers</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.solr.batch.filespool.dir</name>
+ <value>/var/log/atlas/audit/solr/spool</value>
+ <description>/var/log/atlas/audit/solr/spool</description>
+ </property>
+
+ <property>
+ <name>xasecure.audit.provider.summary.enabled</name>
+ <value>false</value>
+ <display-name>Audit provider summary enabled</display-name>
+ <description>Enable Summary audit?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-plugin-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-plugin-properties.xml
new file mode 100644
index 0000000..2fa9448
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-plugin-properties.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+
+ <property>
+ <name>policy_user</name>
+ <value>ambari-qa</value>
+ <display-name>Policy user for Atlas</display-name>
+ <description>This user must be system user and also present at Ranger
+ admin portal</description>
+ </property>
+
+ <property>
+ <name>common.name.for.certificate</name>
+ <value></value>
+ <description>Common name for certificate, this value should match what is specified in repo within ranger admin</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>ranger-atlas-plugin-enabled</name>
+ <value>No</value>
+ <display-name>Enable Ranger for Atlas</display-name>
+ <description>Enable ranger Atlas plugin</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>ranger-atlas-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>REPOSITORY_CONFIG_USERNAME</name>
+ <value>atlas</value>
+ <display-name>Ranger repository config user</display-name>
+ <description>Used for repository creation on ranger admin
+ </description>
+ </property>
+
+ <property>
+ <name>REPOSITORY_CONFIG_PASSWORD</name>
+ <value>atlas</value>
+ <display-name>Ranger repository config password</display-name>
+ <property-type>PASSWORD</property-type>
+ <description>Used for repository creation on ranger admin
+ </description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-policymgr-ssl.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-policymgr-ssl.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-policymgr-ssl.xml
new file mode 100644
index 0000000..41c8e6a
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-policymgr-ssl.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore</name>
+ <value>/usr/hdp/current/atlas-server/conf/ranger-plugin-keystore.jks</value>
+ <description>Java Keystore files</description>
+ </property>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.password</name>
+ <value>myKeyFilePassword</value>
+ <property-type>PASSWORD</property-type>
+ <description>password for keystore</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore</name>
+ <value>/usr/hdp/current/atlas-server/conf/ranger-plugin-truststore.jks</value>
+ <description>java truststore file</description>
+ </property>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.password</name>
+ <value>changeit</value>
+ <property-type>PASSWORD</property-type>
+ <description>java truststore password</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.credential.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>java keystore credential file</description>
+ </property>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.credential.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>java truststore credential file</description>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-security.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-security.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-security.xml
new file mode 100644
index 0000000..f520455
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/ranger-atlas-security.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>ranger.plugin.atlas.service.name</name>
+ <value>{{repo_name}}</value>
+ <description>Name of the Ranger service containing Atlas policies</description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.atlas.policy.source.impl</name>
+ <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value>
+ <description>Class to retrieve policies from the source</description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.atlas.policy.rest.url</name>
+ <value>{{policymgr_mgr_url}}</value>
+ <description>URL to Ranger Admin</description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.atlas.policy.rest.ssl.config.file</name>
+ <value>/usr/hdp/current/atlas-server/conf/ranger-policymgr-ssl.xml</value>
+ <description>Path to the file containing SSL details to contact Ranger Admin</description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.atlas.policy.pollIntervalMs</name>
+ <value>30000</value>
+ <description>How often to poll for changes in policies?</description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.atlas.policy.cache.dir</name>
+ <value>/etc/ranger/{{repo_name}}/policycache</value>
+ <description>Directory where Ranger policies are cached after successful retrieval from the source</description>
+ </property>
+
+ <property>
+ <name>xasecure.add-hadoop-authorization</name>
+ <value>true</value>
+ <description>Enable/Disable the default hadoop authorization (based on rwxrwxrwx permission on the resource) if Ranger Authorization fails.</description>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index df6c65c..3049517 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -173,6 +173,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
def recommendAtlasConfigurations(self, configurations, clusterData, services, hosts):
putAtlasApplicationProperty = self.putProperty(configurations, "application-properties", services)
+ putAtlasRangerPluginProperty = self.putProperty(configurations, "ranger-atlas-plugin-properties", services)
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
@@ -272,6 +273,22 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putAtlasApplicationProperty('atlas.graph.storage.hostname', "")
putAtlasApplicationProperty('atlas.audit.hbase.zookeeper.quorum', "")
+ if "ranger-env" in services["configurations"] and "ranger-atlas-plugin-properties" in services["configurations"] and \
+ "ranger-atlas-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]:
+ ranger_atlas_plugin_enabled = services["configurations"]["ranger-env"]["properties"]["ranger-atlas-plugin-enabled"]
+ putAtlasRangerPluginProperty('ranger-atlas-plugin-enabled', ranger_atlas_plugin_enabled)
+
+ ranger_atlas_plugin_enabled = ''
+ if 'ranger-atlas-plugin-properties' in configurations and 'ranger-atlas-plugin-enabled' in configurations['ranger-atlas-plugin-properties']['properties']:
+ ranger_atlas_plugin_enabled = configurations['ranger-atlas-plugin-properties']['properties']['ranger-atlas-plugin-enabled']
+ elif 'ranger-atlas-plugin-properties' in services['configurations'] and 'ranger-atlas-plugin-enabled' in services['configurations']['ranger-atlas-plugin-properties']['properties']:
+ ranger_atlas_plugin_enabled = services['configurations']['ranger-atlas-plugin-properties']['properties']['ranger-atlas-plugin-enabled']
+
+ if ranger_atlas_plugin_enabled and (ranger_atlas_plugin_enabled.lower() == 'Yes'.lower()):
+ putAtlasApplicationProperty('atlas.authorizer.impl','org.apache.ranger.authorization.atlas.authorizer.RangerAtlasAuthorizer')
+ else:
+ putAtlasApplicationProperty('atlas.authorizer.impl','org.apache.atlas.authorize.SimpleAtlasAuthorizer')
+
def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP25StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts)
putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
@@ -1265,7 +1282,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
{'service_name': 'KNOX', 'audit_file': 'ranger-knox-audit'},
{'service_name': 'KAFKA', 'audit_file': 'ranger-kafka-audit'},
{'service_name': 'STORM', 'audit_file': 'ranger-storm-audit'},
- {'service_name': 'RANGER_KMS', 'audit_file': 'ranger-kms-audit'}
+ {'service_name': 'RANGER_KMS', 'audit_file': 'ranger-kms-audit'},
+ {'service_name': 'ATLAS', 'audit_file': 'ranger-atlas-audit'}
]
for item in range(len(ranger_services)):
http://git-wip-us.apache.org/repos/asf/ambari/blob/0038e3bd/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 5a54548..4e6dcda 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -5686,6 +5686,11 @@ class TestHDP25StackAdvisor(TestCase):
"properties": {
"logsearch_solr_znode": "/logsearch"
}
+ },
+ 'ranger-atlas-plugin-properties': {
+ 'properties': {
+ 'ranger-atlas-plugin-enabled':'No'
+ }
}
}
clusterData = {
@@ -5705,13 +5710,19 @@ class TestHDP25StackAdvisor(TestCase):
"atlas.kafka.bootstrap.servers": "c6401.ambari.apache.org:6667",
"atlas.kafka.zookeeper.connect": "c6401.ambari.apache.org",
'atlas.server.address.id1': "c6401.ambari.apache.org:21000",
- 'atlas.server.ids': "id1"
+ 'atlas.server.ids': "id1",
+ 'atlas.authorizer.impl':'org.apache.ranger.authorization.atlas.authorizer.RangerAtlasAuthorizer'
}
},
"logsearch-solr-env": {
"properties": {
"logsearch_solr_znode": "/logsearch"
}
+ },
+ 'ranger-atlas-plugin-properties': {
+ 'properties': {
+ 'ranger-atlas-plugin-enabled':'Yes'
+ }
}
}
services = {
@@ -5864,6 +5875,11 @@ class TestHDP25StackAdvisor(TestCase):
"zookeeper.connect": "c6401.ambari.apache.org",
"port": "6667"
}
+ },
+ 'ranger-atlas-plugin-properties': {
+ 'properties': {
+ 'ranger-atlas-plugin-enabled':'No'
+ }
}
},
"changed-configurations": [ ]
@@ -5888,6 +5904,11 @@ class TestHDP25StackAdvisor(TestCase):
}
self.stackAdvisor.recommendAtlasConfigurations(configurations, clusterData, services, hosts)
+ # test for Ranger Atlas plugin disabled
+ self.assertEquals(configurations['application-properties']['properties']['atlas.authorizer.impl'], 'org.apache.atlas.authorize.SimpleAtlasAuthorizer', 'Test atlas.authorizer.impl with Ranger Atlas plugin is disabled ')
+
+ configurations['ranger-atlas-plugin-properties']['properties']['ranger-atlas-plugin-enabled'] = 'Yes'
+ configurations['application-properties']['properties']['atlas.authorizer.impl'] = 'org.apache.ranger.authorization.atlas.authorizer.RangerAtlasAuthorizer'
self.assertEquals(configurations, expected)
services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}