You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mu...@apache.org on 2017/01/13 07:11:44 UTC
ambari git commit: AMBARI-19360 : Ranger policies not syncing for
storm, kafka and knox (Vishal Suvagia via mugdha)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 6b1d05ae8 -> a8aa8ab17
AMBARI-19360 : Ranger policies not syncing for storm, kafka and knox (Vishal Suvagia via mugdha)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a8aa8ab1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a8aa8ab1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a8aa8ab1
Branch: refs/heads/branch-2.5
Commit: a8aa8ab174185f5ecd0859db187f57891bf84a8f
Parents: 6b1d05a
Author: Vishal Suvagia <vi...@yahoo.com>
Authored: Fri Jan 13 11:13:46 2017 +0530
Committer: Mugdha Varadkar <mu...@apache.org>
Committed: Fri Jan 13 12:41:10 2017 +0530
----------------------------------------------------------------------
.../libraries/functions/constants.py | 1 +
.../functions/setup_ranger_plugin_xml.py | 12 +++++++
.../KAFKA/0.8.1/package/scripts/params.py | 1 +
.../0.8.1/package/scripts/setup_ranger_kafka.py | 6 ++++
.../0.5.0.2.2/package/scripts/params_linux.py | 1 +
.../package/scripts/setup_ranger_knox.py | 9 +++++
.../STORM/0.9.1/package/scripts/params_linux.py | 1 +
.../0.9.1/package/scripts/setup_ranger_storm.py | 35 +++++++++++++++++++-
.../HDP/2.0.6/properties/stack_features.json | 5 +++
9 files changed, 70 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index f4837fb..6fba224 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -108,3 +108,4 @@ class StackFeature:
RANGER_TAGSYNC_SSL_XML_SUPPORT="ranger_tagsync_ssl_xml_support"
RANGER_SOLR_CONFIG_SUPPORT='ranger_solr_config_support'
HIVE_INTERACTIVE_ATLAS_HOOK_REQUIRED="hive_interactive_atlas_hook_required"
+ CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT='core_site_for_ranger_plugins'
http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
index ba6061a..6561928 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
@@ -33,6 +33,7 @@ from resource_management.core.source import DownloadSource, InlineTemplate
from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2
from resource_management.core.utils import PasswordString
from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.format import format
def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar,
component_downloaded_custom_connector, component_driver_curl_source,
@@ -215,3 +216,14 @@ def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_versio
group = component_group,
mode = 0640
)
+
+
+def setup_core_site_for_required_plugins(component_user, component_group, create_core_site_path, config):
+ XmlConfig('core-site.xml',
+ conf_dir=create_core_site_path,
+ configurations=config['configurations']['core-site'],
+ configuration_attributes=config['configuration_attributes']['core-site'],
+ owner=component_user,
+ group=component_group,
+ mode=0644
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
index 41970f8..82849c8 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
@@ -58,6 +58,7 @@ version_for_stack_feature_checks = get_stack_feature_version(config)
stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
+stack_supports_core_site_for_ranger_plugin = check_stack_feature(StackFeature.CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT, version_for_stack_feature_checks)
# When downgrading the 'version' and 'current_version' are both pointing to the downgrade-target version
# downgrade_from_version provides the source-version the downgrade is happening from
http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
index 3a9f51e..528dec2 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
@@ -17,6 +17,7 @@ limitations under the License.
from resource_management.core.logger import Logger
from resource_management.core.resources import File, Execute
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_core_site_for_required_plugins
def setup_ranger_kafka():
import params
@@ -80,5 +81,10 @@ def setup_ranger_kafka():
group = params.user_group,
mode = 0755
)
+ if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.has_namenode and params.security_enabled:
+ Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions")
+ setup_core_site_for_required_plugins(component_user=params.kafka_user,component_group=params.user_group,create_core_site_path = params.conf_dir, config = params.config)
+ else:
+ Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations")
else:
Logger.info('Ranger admin not installed')
http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
index 4a23bca..d245178 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
@@ -60,6 +60,7 @@ version_for_stack_feature_checks = get_stack_feature_version(config)
stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
+stack_supports_core_site_for_ranger_plugin = check_stack_feature(StackFeature.CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT, version_for_stack_feature_checks)
# This is the version whose state is CURRENT. During an RU, this is the source version.
# DO NOT format it since we need the build number too.
http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
index d4eb6b9..7601dfa 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
@@ -18,6 +18,9 @@ limitations under the License.
"""
from resource_management.core.logger import Logger
+from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_core_site_for_required_plugins
+
+
def setup_ranger_knox(upgrade_type=None):
import params
@@ -95,5 +98,11 @@ def setup_ranger_knox(upgrade_type=None):
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble)
+ if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_knox and params.has_namenode and params.security_enabled:
+ Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating core-site.xml from namenode configuraitions")
+ setup_core_site_for_required_plugins(component_user=params.knox_user, component_group=params.knox_group,create_core_site_path = params.knox_conf_dir, config = params.config)
+ else:
+ Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations")
+
else:
Logger.info('Ranger admin not installed')
http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
index 67d5769..c167d7c 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
@@ -66,6 +66,7 @@ stack_version_formatted = status_params.stack_version_formatted
stack_supports_ru = stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted)
stack_supports_storm_kerberos = stack_version_formatted and check_stack_feature(StackFeature.STORM_KERBEROS, stack_version_formatted)
stack_supports_storm_ams = stack_version_formatted and check_stack_feature(StackFeature.STORM_AMS, stack_version_formatted)
+stack_supports_core_site_for_ranger_plugin = check_stack_feature(StackFeature.CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT, stack_version_formatted)
# get the correct version to use for checking stack features
version_for_stack_feature_checks = get_stack_feature_version(config)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py
index caafd55..e81d62a 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py
@@ -18,6 +18,10 @@ limitations under the License.
"""
from resource_management.core.logger import Logger
+from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_core_site_for_required_plugins
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions.format import format
+from resource_management.core.resources import File, Directory
def setup_ranger_storm(upgrade_type=None):
"""
@@ -93,8 +97,37 @@ def setup_ranger_storm(upgrade_type=None):
plugin_security_properties=params.config['configurations']['ranger-storm-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-storm-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-storm-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-storm-policymgr-ssl'],
component_list=['storm-client', 'storm-nimbus'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
- credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
+ credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble)
+
+
+ site_files_create_path = format('{storm_component_home_dir}/extlib-daemon/ranger-storm-plugin-impl/conf')
+ Directory(site_files_create_path,
+ owner = params.storm_user,
+ group = params.user_group,
+ mode=0775,
+ create_parents = True,
+ cd_access = 'a'
+ )
+
+ if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_storm and params.has_namenode and params.security_enabled:
+ Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions")
+ setup_core_site_for_required_plugins(component_user=params.storm_user,component_group=params.user_group,create_core_site_path = site_files_create_path, config = params.config)
+ if len(params.namenode_hosts) > 1:
+ Logger.info('Ranger Storm plugin is enabled along with security and NameNode is HA , creating hdfs-site.xml')
+ XmlConfig("hdfs-site.xml",
+ conf_dir=site_files_create_path,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.storm_user,
+ group=params.user_group,
+ mode=0644
+ )
+ else:
+ Logger.info('Ranger Storm plugin is not enabled or security is disabled, removing hdfs-site.xml')
+ File(format('{site_files_create_path}/hdfs-site.xml'), action="delete")
+ else:
+ Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations")
else:
Logger.info('Ranger admin not installed')
http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index c3044af..a753f89 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -343,6 +343,11 @@
"name": "hive_interactive_atlas_hook_required",
"description": "Registering Atlas Hook for Hive Interactive.",
"min_version": "2.6.0.0"
+ },
+ {
+ "name": "core_site_for_ranger_plugins",
+ "description": "Adding core-site.xml in when Ranger plugin is enabled for Storm, Kafka, and Knox.",
+ "min_version": "2.6.0.0"
}
]
}