You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/01/18 15:57:47 UTC
[14/50] [abbrv] ambari git commit: AMBARI-19044 Install & configure
Ranger plugin components independently of Ranger admin components (mugdha)
AMBARI-19044 Install & configure Ranger plugin components independently of Ranger admin components (mugdha)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1524fd77
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1524fd77
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1524fd77
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 1524fd775d4b25d0896c648cb1bbc8ed3644a73d
Parents: 8b22dd0
Author: Mugdha Varadkar <mu...@apache.org>
Authored: Tue Jan 17 17:08:02 2017 +0530
Committer: Mugdha Varadkar <mu...@apache.org>
Committed: Tue Jan 17 18:16:38 2017 +0530
----------------------------------------------------------------------
.../libraries/functions/constants.py | 3 +
.../functions/setup_ranger_plugin_xml.py | 47 +++-
.../server/upgrade/UpgradeCatalog250.java | 37 ++++
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 71 ++++--
.../package/scripts/setup_ranger_atlas.py | 4 +-
.../0.96.0.2.0/package/scripts/params_linux.py | 163 +++++++-------
.../package/scripts/setup_ranger_hbase.py | 4 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 166 +++++++-------
.../package/scripts/setup_ranger_hdfs.py | 44 ++--
.../0.12.0.2.0/package/scripts/params_linux.py | 161 +++++++-------
.../package/scripts/setup_ranger_hive.py | 6 +-
.../KAFKA/0.8.1/package/scripts/params.py | 126 +++++------
.../0.8.1/package/scripts/setup_ranger_kafka.py | 4 +-
.../0.9.0/configuration/ranger-kafka-audit.xml | 32 +--
.../ranger-kafka-plugin-properties.xml | 14 +-
.../ranger-kafka-policymgr-ssl.xml | 12 +-
.../configuration/ranger-kafka-security.xml | 18 +-
.../ranger-knox-plugin-properties.xml | 12 +-
.../0.5.0.2.2/package/scripts/params_linux.py | 155 ++++++-------
.../package/scripts/setup_ranger_knox.py | 5 +-
.../configuration/ranger-kms-security.xml | 6 +
.../0.10.0/configuration/ranger-storm-audit.xml | 32 +--
.../ranger-storm-policymgr-ssl.xml | 12 +-
.../configuration/ranger-storm-security.xml | 18 +-
.../STORM/0.9.1/package/scripts/params_linux.py | 161 +++++++-------
.../0.9.1/package/scripts/setup_ranger_storm.py | 4 +-
.../ranger-storm-plugin-properties.xml | 71 ++++++
.../2.1.0.2.0/package/scripts/params_linux.py | 215 ++++++++++---------
.../package/scripts/resourcemanager.py | 2 +-
.../package/scripts/setup_ranger_yarn.py | 4 +-
.../HDP/2.0.6/properties/stack_features.json | 17 +-
.../ranger-hbase-plugin-properties.xml | 10 +-
.../ranger-hdfs-plugin-properties.xml | 12 +-
.../ranger-hive-plugin-properties.xml | 10 +-
.../ranger-knox-plugin-properties.xml | 2 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 38 ++--
.../HBASE/configuration/ranger-hbase-audit.xml | 32 +--
.../ranger-hbase-policymgr-ssl.xml | 12 +-
.../configuration/ranger-hbase-security.xml | 20 +-
.../configuration/ranger-hdfs-policymgr-ssl.xml | 12 +-
.../HDFS/configuration/ranger-hdfs-security.xml | 20 +-
.../HIVE/configuration/ranger-hive-audit.xml | 32 +--
.../configuration/ranger-hive-policymgr-ssl.xml | 12 +-
.../HIVE/configuration/ranger-hive-security.xml | 20 +-
.../ranger-kafka-policymgr-ssl.xml | 4 +-
.../KNOX/configuration/ranger-knox-audit.xml | 32 +--
.../configuration/ranger-knox-policymgr-ssl.xml | 12 +-
.../KNOX/configuration/ranger-knox-security.xml | 18 +-
.../ranger-storm-policymgr-ssl.xml | 4 +-
.../configuration/ranger-storm-security.xml | 2 +-
.../YARN/configuration/ranger-yarn-audit.xml | 32 +--
.../ranger-yarn-plugin-properties.xml | 12 +-
.../configuration/ranger-yarn-policymgr-ssl.xml | 12 +-
.../YARN/configuration/ranger-yarn-security.xml | 18 +-
.../stacks/HDP/2.3/services/stack_advisor.py | 34 +++
.../ATLAS/configuration/ranger-atlas-audit.xml | 6 +-
.../ranger-atlas-plugin-properties.xml | 58 ++++-
.../ranger-atlas-policymgr-ssl.xml | 12 +-
.../configuration/ranger-atlas-security.xml | 20 +-
.../ranger-hbase-plugin-properties.xml | 71 ++++++
.../ranger-hdfs-plugin-properties.xml | 50 ++++-
.../ranger-hive-plugin-properties.xml | 71 ++++++
.../HIVE/configuration/ranger-hive-security.xml | 2 +-
.../ranger-kafka-plugin-properties.xml | 71 ++++++
.../ranger-knox-plugin-properties.xml | 71 ++++++
.../ranger-storm-policymgr-ssl.xml | 4 +-
.../configuration/ranger-storm-security.xml | 2 +-
.../ranger-yarn-plugin-properties.xml | 71 ++++++
.../stacks/HDP/2.5/services/stack_advisor.py | 7 +
.../server/upgrade/UpgradeCatalog250Test.java | 110 ++++++++++
.../stacks/2.0.6/configs/altfs_plus_hdfs.json | 6 +-
.../python/stacks/2.0.6/configs/default.json | 10 +-
.../stacks/2.0.6/configs/default_client.json | 3 +-
.../2.0.6/configs/default_hive_nn_ha.json | 3 +-
.../2.0.6/configs/default_hive_nn_ha_2.json | 3 +-
.../2.0.6/configs/default_hive_non_hdfs.json | 3 +-
.../2.0.6/configs/default_no_install.json | 3 +-
.../2.0.6/configs/default_with_bucket.json | 4 +-
.../2.0.6/configs/ha_bootstrap_active_node.json | 2 +-
.../configs/ha_bootstrap_standby_node.json | 2 +-
...ha_bootstrap_standby_node_initial_start.json | 2 +-
...dby_node_initial_start_dfs_nameservices.json | 2 +-
.../python/stacks/2.0.6/configs/ha_default.json | 4 +-
.../python/stacks/2.0.6/configs/ha_secured.json | 2 +-
.../python/stacks/2.0.6/configs/hbase-2.2.json | 4 +-
.../2.0.6/configs/hbase-rs-2.2-phoenix.json | 4 +-
.../stacks/2.0.6/configs/hbase-rs-2.2.json | 4 +-
.../python/stacks/2.0.6/configs/nn_ru_lzo.json | 2 +-
.../python/stacks/2.0.6/configs/secured.json | 12 +-
.../stacks/2.0.6/configs/secured_client.json | 3 +-
.../stacks/2.1/configs/default-storm-start.json | 2 +-
.../test/python/stacks/2.1/configs/default.json | 5 +-
.../stacks/2.1/configs/secured-storm-start.json | 2 +-
.../test/python/stacks/2.1/configs/secured.json | 5 +-
.../stacks/2.2/common/test_stack_advisor.py | 53 ++++-
.../test/python/stacks/2.2/configs/default.json | 6 +-
.../python/stacks/2.2/configs/hive-upgrade.json | 3 +-
.../stacks/2.3/common/test_stack_advisor.py | 3 +-
.../python/stacks/2.5/configs/hsi_default.json | 3 +-
.../test/python/stacks/2.5/configs/hsi_ha.json | 3 +-
.../controllers/main/service/info/configs.js | 4 +-
.../app/controllers/wizard/step7_controller.js | 4 +-
102 files changed, 1889 insertions(+), 946 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 56af615..6895e34 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -106,6 +106,9 @@ class StackFeature:
ZKFC_VERSION_ADVERTISED = "zkfc_version_advertised"
PHOENIX_CORE_HDFS_SITE_REQUIRED = "phoenix_core_hdfs_site_required"
RANGER_TAGSYNC_SSL_XML_SUPPORT="ranger_tagsync_ssl_xml_support"
+ RANGER_XML_CONFIGURATION = "ranger_xml_configuration"
+ KAFKA_RANGER_PLUGIN_SUPPORT = "kafka_ranger_plugin_support"
+ YARN_RANGER_PLUGIN_SUPPORT = "yarn_ranger_plugin_support"
RANGER_SOLR_CONFIG_SUPPORT='ranger_solr_config_support'
HIVE_INTERACTIVE_ATLAS_HOOK_REQUIRED="hive_interactive_atlas_hook_required"
CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT='core_site_for_ranger_plugins'
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
index 6561928..a12116d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
@@ -17,8 +17,7 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-__all__ = ["setup_ranger_plugin"]
-
+__all__ = ["setup_ranger_plugin", "get_audit_configs"]
import os
import ambari_simplejson as json
@@ -34,6 +33,7 @@ from resource_management.libraries.functions.ranger_functions_v2 import Rangerad
from resource_management.core.utils import PasswordString
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.default import default
def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar,
component_downloaded_custom_connector, component_driver_curl_source,
@@ -164,8 +164,8 @@ def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar,
group = component_group,
mode=0744)
- #This should be done by rpm
- #setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)
+ # creating symblink should be done by rpm package
+ # setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)
setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password, ssl_keystore_password,
@@ -176,7 +176,6 @@ def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar,
action="delete"
)
-
def setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list):
stack_root = Script.get_stack_root()
@@ -217,7 +216,6 @@ def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_versio
mode = 0640
)
-
def setup_core_site_for_required_plugins(component_user, component_group, create_core_site_path, config):
XmlConfig('core-site.xml',
conf_dir=create_core_site_path,
@@ -227,3 +225,40 @@ def setup_core_site_for_required_plugins(component_user, component_group, create
group=component_group,
mode=0644
)
+
+def get_audit_configs(config):
+ xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'].lower()
+ xa_db_host = config['configurations']['admin-properties']['db_host']
+ xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+
+ if xa_audit_db_flavor == 'mysql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "com.mysql.jdbc.Driver"
+ elif xa_audit_db_flavor == 'oracle':
+ jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
+ colon_count = xa_db_host.count(':')
+ if colon_count == 2 or colon_count == 0:
+ audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
+ else:
+ audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
+ jdbc_driver = "oracle.jdbc.OracleDriver"
+ elif xa_audit_db_flavor == 'postgres':
+ jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "org.postgresql.Driver"
+ elif xa_audit_db_flavor == 'mssql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
+ jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+ elif xa_audit_db_flavor == 'sqla':
+ jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
+ jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
+
+ return jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index 29e1f17..6638379 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -166,6 +166,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
updateLogSearchConfigs();
updateAmbariInfraConfigs();
updateYarnSite();
+ updateRangerUrlConfigs();
addManageServiceAutoStartPermissions();
}
@@ -881,4 +882,40 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
"CLUSTER.OPERATOR:CLUSTER");
addRoleAuthorization("CLUSTER.MANAGE_AUTO_START", "Manage service auto-start configuration", roles);
}
+
+ /**
+ * Updates Ranger admin url for Ranger plugin supported configs.
+ *
+ * @throws AmbariException
+ */
+ protected void updateRangerUrlConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) {
+
+ Config ranger_admin_properties = cluster.getDesiredConfigByType("admin-properties");
+ if(null != ranger_admin_properties) {
+ String policyUrl = ranger_admin_properties.getProperties().get("policymgr_external_url");
+ if (null != policyUrl) {
+ updateRangerUrl(cluster, "ranger-hdfs-security", "ranger.plugin.hdfs.policy.rest.url", policyUrl);
+ updateRangerUrl(cluster, "ranger-hive-security", "ranger.plugin.hive.policy.rest.url", policyUrl);
+ updateRangerUrl(cluster, "ranger-hbase-security", "ranger.plugin.hbase.policy.rest.url", policyUrl);
+ updateRangerUrl(cluster, "ranger-knox-security", "ranger.plugin.knox.policy.rest.url", policyUrl);
+ updateRangerUrl(cluster, "ranger-storm-security", "ranger.plugin.storm.policy.rest.url", policyUrl);
+ updateRangerUrl(cluster, "ranger-yarn-security", "ranger.plugin.yarn.policy.rest.url", policyUrl);
+ updateRangerUrl(cluster, "ranger-kafka-security", "ranger.plugin.kafka.policy.rest.url", policyUrl);
+ updateRangerUrl(cluster, "ranger-atlas-security", "ranger.plugin.atlas.policy.rest.url", policyUrl);
+ updateRangerUrl(cluster, "ranger-kms-security", "ranger.plugin.kms.policy.rest.url", policyUrl);
+ }
+ }
+ }
+ }
+
+ protected void updateRangerUrl(Cluster cluster, String configType, String configProperty, String policyUrl) throws AmbariException {
+ Config componentSecurity = cluster.getDesiredConfigByType(configType);
+ if(componentSecurity != null && componentSecurity.getProperties().containsKey(configProperty)) {
+ Map<String, String> updateProperty = new HashMap<>();
+ updateProperty.put(configProperty, policyUrl);
+ updateConfigurationPropertiesForCluster(cluster, configType, updateProperty, true, false);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 94193be..c74d046 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -219,17 +219,7 @@ for host in zookeeper_hosts:
if index < len(zookeeper_hosts):
zookeeper_quorum += ","
-
-# Atlas Ranger plugin configurations
-stack_supports_atlas_ranger_plugin = check_stack_feature(StackFeature.ATLAS_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks)
-stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
stack_supports_atlas_hdfs_site_on_namenode_ha = check_stack_feature(StackFeature.ATLAS_HDFS_SITE_ON_NAMENODE_HA, version_for_stack_feature_checks)
-retry_enabled = default("/commandParams/command_retry_enabled", False)
-
-ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
-has_ranger_admin = not len(ranger_admin_hosts) == 0
-xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
-enable_ranger_atlas = False
atlas_server_xmx = default("configurations/atlas-env/atlas_server_xmx", 2048)
atlas_server_max_new_size = default("configurations/atlas-env/atlas_server_max_new_size", 614)
@@ -237,9 +227,6 @@ atlas_server_max_new_size = default("configurations/atlas-env/atlas_server_max_n
hbase_master_hosts = default('/clusterHostInfo/hbase_master_hosts', [])
has_hbase_master = not len(hbase_master_hosts) == 0
-ranger_admin_hosts = default('/clusterHostInfo/ranger_admin_hosts', [])
-has_ranger_admin = not len(ranger_admin_hosts) == 0
-
atlas_hbase_setup = format("{exec_tmp_dir}/atlas_hbase_setup.rb")
atlas_kafka_setup = format("{exec_tmp_dir}/atlas_kafka_acl.sh")
atlas_graph_storage_hbase_table = default('/configurations/application-properties/atlas.graph.storage.hbase.table', None)
@@ -247,7 +234,6 @@ atlas_audit_hbase_tablename = default('/configurations/application-properties/at
hbase_user_keytab = default('/configurations/hbase-env/hbase_user_keytab', None)
hbase_principal_name = default('/configurations/hbase-env/hbase_principal_name', None)
-enable_ranger_hbase = False
# ToDo: Kafka port to Atlas
# Used while upgrading the stack in a kerberized cluster and running kafka-acls.sh
@@ -289,7 +275,29 @@ if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, version_for_stack_fea
namenode_host = set(default("/clusterHostInfo/namenode_host", []))
has_namenode = not len(namenode_host) == 0
-if has_ranger_admin and stack_supports_atlas_ranger_plugin:
+# ranger altas plugin section start
+
+# ranger host
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+
+retry_enabled = default("/commandParams/command_retry_enabled", False)
+
+stack_supports_atlas_ranger_plugin = check_stack_feature(StackFeature.ATLAS_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks)
+stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
+
+# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature
+xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks)
+
+# ranger atlas plugin enabled property
+enable_ranger_atlas = default("/configurations/ranger-atlas-plugin-properties/ranger-atlas-plugin-enabled", "No")
+enable_ranger_atlas = True if enable_ranger_atlas.lower() == "yes" else False
+
+# ranger hbase plugin enabled property
+enable_ranger_hbase = default("/configurations/ranger-hbase-plugin-properties/ranger-hbase-plugin-enabled", "No")
+enable_ranger_hbase = True if enable_ranger_hbase.lower() == 'yes' else False
+
+if stack_supports_atlas_ranger_plugin and enable_ranger_atlas:
# for create_hdfs_directory
hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None
@@ -320,27 +328,42 @@ if has_ranger_admin and stack_supports_atlas_ranger_plugin:
dfs_type = dfs_type
)
+ # ranger atlas service/repository name
repo_name = str(config['clusterName']) + '_atlas'
repo_name_value = config['configurations']['ranger-atlas-security']['ranger.plugin.atlas.service.name']
if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
repo_name = repo_name_value
- ssl_keystore_password = unicode(config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'])
- ssl_truststore_password = unicode(config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'])
+
+ ssl_keystore_password = config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']
+ ssl_truststore_password = config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']
credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
xa_audit_hdfs_is_enabled = default('/configurations/ranger-atlas-audit/xasecure.audit.destination.hdfs', False)
- enable_ranger_atlas = config['configurations']['ranger-atlas-plugin-properties']['ranger-atlas-plugin-enabled']
- enable_ranger_atlas = not is_empty(enable_ranger_atlas) and enable_ranger_atlas.lower() == 'yes'
- enable_ranger_hbase = config['configurations']['ranger-hbase-plugin-properties']['ranger-hbase-plugin-enabled']
- enable_ranger_hbase = not is_empty(enable_ranger_hbase) and enable_ranger_hbase.lower() == 'yes'
- policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+
+ # get ranger policy url
+ policymgr_mgr_url = config['configurations']['ranger-atlas-security']['ranger.plugin.atlas.policy.rest.url']
+
+ if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
downloaded_custom_connector = None
driver_curl_source = None
driver_curl_target = None
ranger_env = config['configurations']['ranger-env']
- ranger_plugin_properties = config['configurations']['ranger-atlas-plugin-properties']
+ # create ranger-env config having external ranger credential properties
+ if not has_ranger_admin and enable_ranger_atlas:
+ external_admin_username = default('/configurations/ranger-atlas-plugin-properties/external_admin_username', 'admin')
+ external_admin_password = default('/configurations/ranger-atlas-plugin-properties/external_admin_password', 'admin')
+ external_ranger_admin_username = default('/configurations/ranger-atlas-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin')
+ external_ranger_admin_password = default('/configurations/ranger-atlas-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin')
+ ranger_env = {}
+ ranger_env['admin_username'] = external_admin_username
+ ranger_env['admin_password'] = external_admin_password
+ ranger_env['ranger_admin_username'] = external_ranger_admin_username
+ ranger_env['ranger_admin_password'] = external_ranger_admin_password
+
+ ranger_plugin_properties = config['configurations']['ranger-atlas-plugin-properties']
ranger_atlas_audit = config['configurations']['ranger-atlas-audit']
ranger_atlas_audit_attrs = config['configuration_attributes']['ranger-atlas-audit']
ranger_atlas_security = config['configurations']['ranger-atlas-security']
@@ -357,6 +380,7 @@ if has_ranger_admin and stack_supports_atlas_ranger_plugin:
'commonNameForCertificate' : config['configurations']['ranger-atlas-plugin-properties']['common.name.for.certificate'],
'ambari.service.check.user' : policy_user
}
+
if security_enabled:
atlas_repository_configuration['policy.download.auth.users'] = metadata_user
atlas_repository_configuration['tag.download.auth.users'] = metadata_user
@@ -368,3 +392,4 @@ if has_ranger_admin and stack_supports_atlas_ranger_plugin:
'name': repo_name,
'type': 'atlas',
}
+# ranger atlas plugin section end
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py
index f5d7f38..c47c75c 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py
@@ -19,7 +19,7 @@ from resource_management.core.logger import Logger
def setup_ranger_atlas(upgrade_type=None):
import params
- if params.has_ranger_admin:
+ if params.enable_ranger_atlas:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
@@ -67,4 +67,4 @@ def setup_ranger_atlas(upgrade_type=None):
component_user_principal=params.atlas_jaas_principal if params.security_enabled else None,
component_user_keytab=params.atlas_keytab_path if params.security_enabled else None)
else:
- Logger.info('Ranger admin not installed')
\ No newline at end of file
+ Logger.info('Ranger Atlas plugin is not enabled')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index e27fd72..268d81c 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -41,6 +41,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.expect import expect
from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames
+from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs
# server configurations
config = Script.get_config()
@@ -232,8 +233,6 @@ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-
-
hdfs_site = config['configurations']['hdfs-site']
default_fs = config['configurations']['core-site']['fs.defaultFS']
@@ -258,87 +257,90 @@ HdfsResource = functools.partial(
dfs_type = dfs_type
)
-# ranger host
-ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
-has_ranger_admin = not len(ranger_admin_hosts) == 0
-xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
-ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
-
-# ranger hbase properties
-policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
-if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
- policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
-xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
-xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
-xa_db_host = config['configurations']['admin-properties']['db_host']
-repo_name = str(config['clusterName']) + '_hbase'
-repo_name_value = config['configurations']['ranger-hbase-security']['ranger.plugin.hbase.service.name']
-if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
- repo_name = repo_name_value
-
-common_name_for_certificate = config['configurations']['ranger-hbase-plugin-properties']['common.name.for.certificate']
-
zookeeper_znode_parent = config['configurations']['hbase-site']['zookeeper.znode.parent']
hbase_zookeeper_quorum = config['configurations']['hbase-site']['hbase.zookeeper.quorum']
hbase_zookeeper_property_clientPort = config['configurations']['hbase-site']['hbase.zookeeper.property.clientPort']
hbase_security_authentication = config['configurations']['hbase-site']['hbase.security.authentication']
hadoop_security_authentication = config['configurations']['core-site']['hadoop.security.authentication']
-repo_config_username = config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+# ranger hbase plugin section start
-ranger_env = config['configurations']['ranger-env']
-ranger_plugin_properties = config['configurations']['ranger-hbase-plugin-properties']
-policy_user = config['configurations']['ranger-hbase-plugin-properties']['policy_user']
-
-#For curl command in ranger plugin to get db connector
+# to get db connector jar
jdk_location = config['hostLevelParams']['jdk_location']
-java_share_dir = '/usr/share/java'
-enable_ranger_hbase = False
-if has_ranger_admin:
- enable_ranger_hbase = (config['configurations']['ranger-hbase-plugin-properties']['ranger-hbase-plugin-enabled'].lower() == 'yes')
+
+# ranger host
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+
+# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env introduced, using stack feature
+xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks)
+
+# ambari-server hostname
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+# ranger hbase plugin enabled property
+enable_ranger_hbase = default("/configurations/ranger-hbase-plugin-properties/ranger-hbase-plugin-enabled", "No")
+enable_ranger_hbase = True if enable_ranger_hbase.lower() == 'yes' else False
+
+# ranger hbase properties
+if enable_ranger_hbase:
+ # get ranger policy url
+ policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+ if xml_configurations_supported:
+ policymgr_mgr_url = config['configurations']['ranger-hbase-security']['ranger.plugin.hbase.policy.rest.url']
+
+ if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+
+ # ranger audit db user
+ xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+
+ # ranger hbase service/repository name
+ repo_name = str(config['clusterName']) + '_hbase'
+ repo_name_value = config['configurations']['ranger-hbase-security']['ranger.plugin.hbase.service.name']
+ if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
+ repo_name = repo_name_value
+
+ common_name_for_certificate = config['configurations']['ranger-hbase-plugin-properties']['common.name.for.certificate']
+ repo_config_username = config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+ ranger_plugin_properties = config['configurations']['ranger-hbase-plugin-properties']
+ policy_user = config['configurations']['ranger-hbase-plugin-properties']['policy_user']
+ repo_config_password = config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']
+
+ # ranger-env config
+ ranger_env = config['configurations']['ranger-env']
+
+ # create ranger-env config having external ranger credential properties
+ if not has_ranger_admin and enable_ranger_hbase:
+ external_admin_username = default('/configurations/ranger-hbase-plugin-properties/external_admin_username', 'admin')
+ external_admin_password = default('/configurations/ranger-hbase-plugin-properties/external_admin_password', 'admin')
+ external_ranger_admin_username = default('/configurations/ranger-hbase-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin')
+ external_ranger_admin_password = default('/configurations/ranger-hbase-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin')
+ ranger_env = {}
+ ranger_env['admin_username'] = external_admin_username
+ ranger_env['admin_password'] = external_admin_password
+ ranger_env['ranger_admin_username'] = external_ranger_admin_username
+ ranger_env['ranger_admin_password'] = external_ranger_admin_password
+
xa_audit_db_password = ''
- if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
- xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
- repo_config_password = unicode(config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
- xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
+ if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin:
+ xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password']
+
+ downloaded_custom_connector = None
previous_jdbc_jar_name = None
+ driver_curl_source = None
+ driver_curl_target = None
+ previous_jdbc_jar = None
+
+ if has_ranger_admin and stack_supports_ranger_audit_db:
+ xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR']
+ jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config)
- if stack_supports_ranger_audit_db:
- if xa_audit_db_flavor == 'mysql':
- jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
- audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
- jdbc_driver = "com.mysql.jdbc.Driver"
- elif xa_audit_db_flavor == 'oracle':
- jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
- colon_count = xa_db_host.count(':')
- if colon_count == 2 or colon_count == 0:
- audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
- else:
- audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
- jdbc_driver = "oracle.jdbc.OracleDriver"
- elif xa_audit_db_flavor == 'postgres':
- jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
- audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
- jdbc_driver = "org.postgresql.Driver"
- elif xa_audit_db_flavor == 'mssql':
- jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
- audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
- jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
- elif xa_audit_db_flavor == 'sqla':
- jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
- audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
- jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
-
- downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- driver_curl_target = format("{stack_root}/current/{component_directory}/lib/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- previous_jdbc_jar = format("{stack_root}/current/{component_directory}/lib/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- sql_connector_jar = ''
+ downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_target = format("{stack_root}/current/{component_directory}/lib/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ previous_jdbc_jar = format("{stack_root}/current/{component_directory}/lib/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ sql_connector_jar = ''
if security_enabled:
master_principal = config['configurations']['hbase-site']['hbase.master.kerberos.principal']
@@ -385,23 +387,24 @@ if has_ranger_admin:
if stack_supports_ranger_kerberos and security_enabled and 'hbase-master' in component_directory.lower():
ranger_hbase_principal = master_jaas_princ
ranger_hbase_keytab = master_keytab_path
- elif stack_supports_ranger_kerberos and security_enabled and 'hbase-regionserver' in component_directory.lower():
+ elif stack_supports_ranger_kerberos and security_enabled and 'hbase-regionserver' in component_directory.lower():
ranger_hbase_principal = regionserver_jaas_princ
ranger_hbase_keytab = regionserver_keytab_path
xa_audit_db_is_enabled = False
- ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
if xml_configurations_supported and stack_supports_ranger_audit_db:
xa_audit_db_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.db']
- xa_audit_hdfs_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
- ssl_keystore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
- ssl_truststore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
- credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
- #For SQLA explicitly disable audit to DB for Ranger
- if xa_audit_db_flavor == 'sqla':
+ xa_audit_hdfs_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else False
+ ssl_keystore_password = config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None
+ ssl_truststore_password = config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None
+ credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
+
+ # for SQLA explicitly disable audit to DB for Ranger
+ if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla':
xa_audit_db_is_enabled = False
+# ranger hbase plugin section end
create_hbase_home_directory = check_stack_feature(StackFeature.HBASE_HOME_DIRECTORY, stack_version_formatted)
hbase_home_directory = format("/user/{hbase_user}")
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
index 0d73e39..d32dce1 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
@@ -22,7 +22,7 @@ from resource_management.core.logger import Logger
def setup_ranger_hbase(upgrade_type=None, service_name="hbase-master"):
import params
- if params.has_ranger_admin:
+ if params.enable_ranger_hbase:
stack_version = None
@@ -103,4 +103,4 @@ def setup_ranger_hbase(upgrade_type=None, service_name="hbase-master"):
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble)
else:
- Logger.info('Ranger admin not installed')
+ Logger.info('Ranger HBase plugin is not enabled')
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 21e7b68..31431b9 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -44,7 +44,7 @@ from resource_management.libraries.functions.get_lzo_packages import get_lzo_pac
from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs
from resource_management.libraries.functions import is_empty
from resource_management.libraries.functions.get_architecture import get_architecture
-
+from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
@@ -392,95 +392,100 @@ dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
-# ranger host
-ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
-has_ranger_admin = not len(ranger_admin_hosts) == 0
-xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
-ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
-
-#ranger hdfs properties
-policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
-if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
- policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
-xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
-xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
-xa_db_host = config['configurations']['admin-properties']['db_host']
-repo_name = str(config['clusterName']) + '_hadoop'
-repo_name_value = config['configurations']['ranger-hdfs-security']['ranger.plugin.hdfs.service.name']
-if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
- repo_name = repo_name_value
-
hadoop_security_authentication = config['configurations']['core-site']['hadoop.security.authentication']
hadoop_security_authorization = config['configurations']['core-site']['hadoop.security.authorization']
fs_default_name = config['configurations']['core-site']['fs.defaultFS']
hadoop_security_auth_to_local = config['configurations']['core-site']['hadoop.security.auth_to_local']
-hadoop_rpc_protection = config['configurations']['ranger-hdfs-plugin-properties']['hadoop.rpc.protection']
-common_name_for_certificate = config['configurations']['ranger-hdfs-plugin-properties']['common.name.for.certificate']
-
-repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
if security_enabled:
sn_principal_name = default("/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/_HOST@EXAMPLE.COM")
sn_principal_name = sn_principal_name.replace('_HOST',hostname.lower())
-ranger_env = config['configurations']['ranger-env']
-ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties']
-policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user']
-
-#For curl command in ranger plugin to get db connector
+# for curl command in ranger plugin to get db connector
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
is_https_enabled = is_https_enabled_in_hdfs(config['configurations']['hdfs-site']['dfs.http.policy'],
config['configurations']['hdfs-site']['dfs.https.enable'])
-if has_ranger_admin:
- enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes')
+# ranger hdfs plugin section start
+
+# ranger host
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+
+# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature
+xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks)
+
+# ambari-server hostname
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+# ranger hdfs plugin enabled property
+enable_ranger_hdfs = default("/configurations/ranger-hdfs-plugin-properties/ranger-hdfs-plugin-enabled", "No")
+enable_ranger_hdfs = True if enable_ranger_hdfs.lower() == 'yes' else False
+
+# get ranger hdfs properties if enable_ranger_hdfs is True
+if enable_ranger_hdfs:
+ # ranger policy url
+ policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+ if xml_configurations_supported:
+ policymgr_mgr_url = config['configurations']['ranger-hdfs-security']['ranger.plugin.hdfs.policy.rest.url']
+
+ if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+
+ # ranger audit db user
+ xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+
+ # ranger hdfs service name
+ repo_name = str(config['clusterName']) + '_hadoop'
+ repo_name_value = config['configurations']['ranger-hdfs-security']['ranger.plugin.hdfs.service.name']
+ if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
+ repo_name = repo_name_value
+
+ hadoop_rpc_protection = config['configurations']['ranger-hdfs-plugin-properties']['hadoop.rpc.protection']
+ common_name_for_certificate = config['configurations']['ranger-hdfs-plugin-properties']['common.name.for.certificate']
+ repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+
+ # ranger-env config
+ ranger_env = config['configurations']['ranger-env']
+
+ # create ranger-env config having external ranger credential properties
+ if not has_ranger_admin and enable_ranger_hdfs:
+ external_admin_username = default('/configurations/ranger-hdfs-plugin-properties/external_admin_username', 'admin')
+ external_admin_password = default('/configurations/ranger-hdfs-plugin-properties/external_admin_password', 'admin')
+ external_ranger_admin_username = default('/configurations/ranger-hdfs-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin')
+ external_ranger_admin_password = default('/configurations/ranger-hdfs-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin')
+ ranger_env = {}
+ ranger_env['admin_username'] = external_admin_username
+ ranger_env['admin_password'] = external_admin_password
+ ranger_env['ranger_admin_username'] = external_ranger_admin_username
+ ranger_env['ranger_admin_password'] = external_ranger_admin_password
+
+ ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties']
+ policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user']
+ repo_config_password = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']
+
xa_audit_db_password = ''
- if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
- xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
- repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
- xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
+ if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin:
+ xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password']
+
+ downloaded_custom_connector = None
previous_jdbc_jar_name = None
+ driver_curl_source = None
+ driver_curl_target = None
+ previous_jdbc_jar = None
+
+ # to get db connector related properties
+ if has_ranger_admin and stack_supports_ranger_audit_db:
+ xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR']
+ jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config)
- if stack_supports_ranger_audit_db:
-
- if xa_audit_db_flavor == 'mysql':
- jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
- audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
- jdbc_driver = "com.mysql.jdbc.Driver"
- elif xa_audit_db_flavor == 'oracle':
- jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
- colon_count = xa_db_host.count(':')
- if colon_count == 2 or colon_count == 0:
- audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
- else:
- audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
- jdbc_driver = "oracle.jdbc.OracleDriver"
- elif xa_audit_db_flavor == 'postgres':
- jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
- audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
- jdbc_driver = "org.postgresql.Driver"
- elif xa_audit_db_flavor == 'mssql':
- jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
- audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
- jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
- elif xa_audit_db_flavor == 'sqla':
- jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
- audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
- jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
-
- downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- driver_curl_target = format("{hadoop_lib_home}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- previous_jdbc_jar = format("{hadoop_lib_home}/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
-
- sql_connector_jar = ''
+ downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_target = format("{hadoop_lib_home}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ previous_jdbc_jar = format("{hadoop_lib_home}/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ sql_connector_jar = ''
hdfs_ranger_plugin_config = {
'username': repo_config_username,
@@ -504,6 +509,7 @@ if has_ranger_admin:
'repositoryType': 'hdfs',
'assetType': '1'
}
+
if stack_supports_ranger_kerberos and security_enabled:
hdfs_ranger_plugin_config['policy.download.auth.users'] = hdfs_user
hdfs_ranger_plugin_config['tag.download.auth.users'] = hdfs_user
@@ -520,14 +526,16 @@ if has_ranger_admin:
}
xa_audit_db_is_enabled = False
- ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
if xml_configurations_supported and stack_supports_ranger_audit_db:
xa_audit_db_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.db']
- xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
- ssl_keystore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
- ssl_truststore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
- credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
- #For SQLA explicitly disable audit to DB for Ranger
- if xa_audit_db_flavor == 'sqla':
+ xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else False
+ ssl_keystore_password = config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None
+ ssl_truststore_password = config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None
+ credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
+
+ # for SQLA explicitly disable audit to DB for Ranger
+ if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla':
xa_audit_db_is_enabled = False
+
+# ranger hdfs plugin section end
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
index e3aff9d..47c6e35 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
@@ -29,8 +29,7 @@ from resource_management.libraries.functions.format import format
def setup_ranger_hdfs(upgrade_type=None):
import params
- if params.has_ranger_admin:
-
+ if params.enable_ranger_hdfs:
stack_version = None
@@ -93,29 +92,28 @@ def setup_ranger_hdfs(upgrade_type=None):
target_file = source_file + ".bak"
Execute(("mv", source_file, target_file), sudo=True, only_if=format("test -f {source_file}"))
else:
- Logger.info('Ranger admin not installed')
+ Logger.info('Ranger Hdfs plugin is not enabled')
def create_ranger_audit_hdfs_directories():
import params
- if params.has_ranger_admin:
- if params.xml_configurations_supported and params.enable_ranger_hdfs and params.xa_audit_hdfs_is_enabled:
- params.HdfsResource("/ranger/audit",
- type="directory",
- action="create_on_execute",
- owner=params.hdfs_user,
- group=params.hdfs_user,
- mode=0755,
- recursive_chmod=True,
- )
- params.HdfsResource("/ranger/audit/hdfs",
- type="directory",
- action="create_on_execute",
- owner=params.hdfs_user,
- group=params.hdfs_user,
- mode=0700,
- recursive_chmod=True,
- )
- params.HdfsResource(None, action="execute")
+ if params.enable_ranger_hdfs and params.xml_configurations_supported and params.xa_audit_hdfs_is_enabled:
+ params.HdfsResource("/ranger/audit",
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ group=params.hdfs_user,
+ mode=0755,
+ recursive_chmod=True,
+ )
+ params.HdfsResource("/ranger/audit/hdfs",
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ group=params.hdfs_user,
+ mode=0700,
+ recursive_chmod=True,
+ )
+ params.HdfsResource(None, action="execute")
else:
- Logger.info('Ranger admin not installed')
+ Logger.info('Ranger Hdfs plugin is not enabled')
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 62fdbfd..9185f78 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -43,6 +43,7 @@ from resource_management.libraries.functions.expect import expect
from resource_management.libraries import functions
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames
+from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs
# Default log4j version; put config files under /etc/hive/conf
log4j_version = '1'
@@ -641,84 +642,85 @@ if has_hive_interactive:
hive_server2_zookeeper_namespace = config['configurations']['hive-site']['hive.server2.zookeeper.namespace']
hive_zookeeper_quorum = config['configurations']['hive-site']['hive.zookeeper.quorum']
-# ranger host
-ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
-has_ranger_admin = not len(ranger_admin_hosts) == 0
-xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
-
-#ranger hive properties
-policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
-if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
- policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
-xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
-xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
-xa_db_host = config['configurations']['admin-properties']['db_host']
-repo_name = str(config['clusterName']) + '_hive'
-repo_name_value = config['configurations']['ranger-hive-security']['ranger.plugin.hive.service.name']
-if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
- repo_name = repo_name_value
-
-jdbc_driver_class_name = config['configurations']['ranger-hive-plugin-properties']['jdbc.driverClassName']
-common_name_for_certificate = config['configurations']['ranger-hive-plugin-properties']['common.name.for.certificate']
-
-repo_config_username = config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
-
-ranger_env = config['configurations']['ranger-env']
-ranger_plugin_properties = config['configurations']['ranger-hive-plugin-properties']
-policy_user = config['configurations']['ranger-hive-plugin-properties']['policy_user']
+if security_enabled:
+ hive_principal = hive_server_principal.replace('_HOST',hostname.lower())
+ hive_keytab = config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab']
hive_cluster_token_zkstore = default("/configurations/hive-site/hive.cluster.delegation.token.store.zookeeper.znode", None)
jaas_file = os.path.join(hive_config_dir, 'zkmigrator_jaas.conf')
zkdtsm_pattern = '/zkdtsm_*'
hive_zk_namespace = default("/configurations/hive-site/hive.zookeeper.namespace", None)
-if security_enabled:
- hive_principal = hive_server_principal.replace('_HOST',hostname.lower())
- hive_keytab = config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab']
+# ranger hive plugin section start
+
+# ranger host
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
-#For curl command in ranger plugin to get db connector
-if has_ranger_admin:
- enable_ranger_hive = (config['configurations']['hive-env']['hive_security_authorization'].lower() == 'ranger')
- repo_config_password = unicode(config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
- xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
+# ranger hive plugin enabled property
+enable_ranger_hive = config['configurations']['hive-env']['hive_security_authorization'].lower() == 'ranger'
+
+# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature
+xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks)
+
+# get ranger hive properties if enable_ranger_hive is True
+if enable_ranger_hive:
+ # get ranger policy url
+ policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+ if xml_configurations_supported:
+ policymgr_mgr_url = config['configurations']['ranger-hive-security']['ranger.plugin.hive.policy.rest.url']
+
+ if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+
+ # ranger audit db user
+ xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+
+ # ranger hive service name
+ repo_name = str(config['clusterName']) + '_hive'
+ repo_name_value = config['configurations']['ranger-hive-security']['ranger.plugin.hive.service.name']
+ if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
+ repo_name = repo_name_value
+
+ jdbc_driver_class_name = config['configurations']['ranger-hive-plugin-properties']['jdbc.driverClassName']
+ common_name_for_certificate = config['configurations']['ranger-hive-plugin-properties']['common.name.for.certificate']
+ repo_config_username = config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+
+ # ranger-env config
+ ranger_env = config['configurations']['ranger-env']
+
+ # create ranger-env config having external ranger credential properties
+ if not has_ranger_admin and enable_ranger_hive:
+ external_admin_username = default('/configurations/ranger-hive-plugin-properties/external_admin_username', 'admin')
+ external_admin_password = default('/configurations/ranger-hive-plugin-properties/external_admin_password', 'admin')
+ external_ranger_admin_username = default('/configurations/ranger-hive-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin')
+ external_ranger_admin_password = default('/configurations/ranger-hive-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin')
+ ranger_env = {}
+ ranger_env['admin_username'] = external_admin_username
+ ranger_env['admin_password'] = external_admin_password
+ ranger_env['ranger_admin_username'] = external_ranger_admin_username
+ ranger_env['ranger_admin_password'] = external_ranger_admin_password
+
+ ranger_plugin_properties = config['configurations']['ranger-hive-plugin-properties']
+ policy_user = config['configurations']['ranger-hive-plugin-properties']['policy_user']
+ repo_config_password = config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']
+
+ ranger_downloaded_custom_connector = None
ranger_previous_jdbc_jar_name = None
+ ranger_driver_curl_source = None
+ ranger_driver_curl_target = None
+ ranger_previous_jdbc_jar = None
+
+ # to get db connector related properties
+ if has_ranger_admin and stack_supports_ranger_audit_db:
+ xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR']
+ ranger_jdbc_jar_name, ranger_previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config)
- if stack_supports_ranger_audit_db:
- if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql':
- ranger_jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
- ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
- audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
- jdbc_driver = "com.mysql.jdbc.Driver"
- elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle':
- ranger_jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
- ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
- colon_count = xa_db_host.count(':')
- if colon_count == 2 or colon_count == 0:
- audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
- else:
- audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
- jdbc_driver = "oracle.jdbc.OracleDriver"
- elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres':
- ranger_jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
- ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
- audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
- jdbc_driver = "org.postgresql.Driver"
- elif xa_audit_db_flavor and xa_audit_db_flavor == 'mssql':
- ranger_jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
- ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
- audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
- jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
- elif xa_audit_db_flavor and xa_audit_db_flavor == 'sqla':
- ranger_jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
- ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
- audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
- jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
-
- ranger_downloaded_custom_connector = format("{tmp_dir}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- ranger_driver_curl_target = format("{hive_lib}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- ranger_previous_jdbc_jar = format("{hive_lib}/{ranger_previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- sql_connector_jar = ''
+ ranger_downloaded_custom_connector = format("{tmp_dir}/{ranger_jdbc_jar_name}")
+ ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_jar_name}")
+ ranger_driver_curl_target = format("{hive_lib}/{ranger_jdbc_jar_name}")
+ ranger_previous_jdbc_jar = format("{hive_lib}/{ranger_previous_jdbc_jar_name}")
+ sql_connector_jar = ''
ranger_hive_url = format("{hive_url}/default;principal={hive_principal}") if security_enabled else hive_url
if stack_supports_ranger_hive_jdbc_url_change:
@@ -757,20 +759,21 @@ if has_ranger_admin:
'type': 'hive'
}
+ xa_audit_db_password = ''
+ if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin:
+ xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password']
xa_audit_db_is_enabled = False
- xa_audit_db_password = ''
- if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
- xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
- ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
if xml_configurations_supported and stack_supports_ranger_audit_db:
xa_audit_db_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.db']
- xa_audit_hdfs_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
- ssl_keystore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
- ssl_truststore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
- credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
- #For SQLA explicitly disable audit to DB for Ranger
- if xa_audit_db_flavor == 'sqla':
+ xa_audit_hdfs_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else False
+ ssl_keystore_password = config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None
+ ssl_truststore_password = config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None
+ credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
+
+ # for SQLA explicitly disable audit to DB for Ranger
+ if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla':
xa_audit_db_is_enabled = False
+# ranger hive plugin section end
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
index 81a4e3e..80bd7c8 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
@@ -22,7 +22,7 @@ from resource_management.core.logger import Logger
def setup_ranger_hive(upgrade_type = None):
import params
- if params.has_ranger_admin:
+ if params.enable_ranger_hive:
stack_version = None
@@ -34,7 +34,7 @@ def setup_ranger_hive(upgrade_type = None):
else:
Logger.info("Hive: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
- if params.xml_configurations_supported and params.enable_ranger_hive and params.xa_audit_hdfs_is_enabled:
+ if params.xml_configurations_supported and params.xa_audit_hdfs_is_enabled:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
@@ -95,4 +95,4 @@ def setup_ranger_hive(upgrade_type = None):
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble)
else:
- Logger.info('Ranger admin not installed')
+ Logger.info('Ranger Hive plugin is not enabled')
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
index 82849c8..6c7ff69 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
@@ -34,6 +34,7 @@ from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs
# server configurations
config = Script.get_config()
@@ -166,41 +167,66 @@ else:
kafka_jaas_principal = None
kafka_keytab_path = None
-# *********************** RANGER PLUGIN CHANGES ***********************
+# for curl command in ranger plugin to get db connector
+jdk_location = config['hostLevelParams']['jdk_location']
+
+# ranger kafka plugin section start
+
# ranger host
-# **********************************************************************
ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
has_ranger_admin = not len(ranger_admin_hosts) == 0
-xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
+
+# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature
+xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks)
+
+# ambari-server hostname
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
ranger_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir","/var/log/ranger/admin")
-is_supported_kafka_ranger = config['configurations']['kafka-env']['is_supported_kafka_ranger']
-#ranger kafka properties
-if has_ranger_admin and is_supported_kafka_ranger:
+# ranger kafka plugin enabled property
+enable_ranger_kafka = default("configurations/ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled", "No")
+enable_ranger_kafka = True if enable_ranger_kafka.lower() == 'yes' else False
- enable_ranger_kafka = config['configurations']['ranger-kafka-plugin-properties']['ranger-kafka-plugin-enabled']
- enable_ranger_kafka = not is_empty(enable_ranger_kafka) and enable_ranger_kafka.lower() == 'yes'
- policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
- if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
+# ranger kafka-plugin supported flag, instead of dependending on is_supported_kafka_ranger/kafka-env.xml, using stack feature
+is_supported_kafka_ranger = check_stack_feature(StackFeature.KAFKA_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks)
+
+# ranger kafka properties
+if enable_ranger_kafka and is_supported_kafka_ranger:
+ # get ranger policy url
+ policymgr_mgr_url = config['configurations']['ranger-kafka-security']['ranger.plugin.kafka.policy.rest.url']
+
+ if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
- xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR']
- xa_audit_db_flavor = xa_audit_db_flavor.lower() if xa_audit_db_flavor else None
- xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+
+ # ranger audit db user
xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+
xa_audit_db_password = ''
- if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
- xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
- xa_db_host = config['configurations']['admin-properties']['db_host']
+ if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin:
+ xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password']
+
+ # ranger kafka service/repository name
repo_name = str(config['clusterName']) + '_kafka'
repo_name_value = config['configurations']['ranger-kafka-security']['ranger.plugin.kafka.service.name']
if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
repo_name = repo_name_value
ranger_env = config['configurations']['ranger-env']
- ranger_plugin_properties = config['configurations']['ranger-kafka-plugin-properties']
+ # create ranger-env config having external ranger credential properties
+ if not has_ranger_admin and enable_ranger_kafka:
+ external_admin_username = default('/configurations/ranger-kafka-plugin-properties/external_admin_username', 'admin')
+ external_admin_password = default('/configurations/ranger-kafka-plugin-properties/external_admin_password', 'admin')
+ external_ranger_admin_username = default('/configurations/ranger-kafka-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin')
+ external_ranger_admin_password = default('/configurations/ranger-kafka-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin')
+ ranger_env = {}
+ ranger_env['admin_username'] = external_admin_username
+ ranger_env['admin_password'] = external_admin_password
+ ranger_env['ranger_admin_username'] = external_ranger_admin_username
+ ranger_env['ranger_admin_password'] = external_ranger_admin_password
+
+ ranger_plugin_properties = config['configurations']['ranger-kafka-plugin-properties']
ranger_kafka_audit = config['configurations']['ranger-kafka-audit']
ranger_kafka_audit_attrs = config['configuration_attributes']['ranger-kafka-audit']
ranger_kafka_security = config['configurations']['ranger-kafka-security']
@@ -212,7 +238,7 @@ if has_ranger_admin and is_supported_kafka_ranger:
ranger_plugin_config = {
'username' : config['configurations']['ranger-kafka-plugin-properties']['REPOSITORY_CONFIG_USERNAME'],
- 'password' : unicode(config['configurations']['ranger-kafka-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']),
+ 'password' : config['configurations']['ranger-kafka-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'],
'zookeeper.connect' : config['configurations']['ranger-kafka-plugin-properties']['zookeeper.connect'],
'commonNameForCertificate' : config['configurations']['ranger-kafka-plugin-properties']['common.name.for.certificate']
}
@@ -232,64 +258,40 @@ if has_ranger_admin and is_supported_kafka_ranger:
ranger_plugin_config['tag.download.auth.users'] = kafka_user
ranger_plugin_config['ambari.service.check.user'] = policy_user
- #For curl command in ranger plugin to get db connector
- jdk_location = config['hostLevelParams']['jdk_location']
- java_share_dir = '/usr/share/java'
+ downloaded_custom_connector = None
previous_jdbc_jar_name = None
+ driver_curl_source = None
+ driver_curl_target = None
+ previous_jdbc_jar = None
- if stack_supports_ranger_audit_db:
- if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql':
- jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
- audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
- jdbc_driver = "com.mysql.jdbc.Driver"
- elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle':
- jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
- colon_count = xa_db_host.count(':')
- if colon_count == 2 or colon_count == 0:
- audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
- else:
- audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
- jdbc_driver = "oracle.jdbc.OracleDriver"
- elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres':
- jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
- audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
- jdbc_driver = "org.postgresql.Driver"
- elif xa_audit_db_flavor and xa_audit_db_flavor == 'mssql':
- jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
- audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
- jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
- elif xa_audit_db_flavor and xa_audit_db_flavor == 'sqla':
- jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
- previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
- audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
- jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
-
- downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- driver_curl_target = format("{kafka_home}/libs/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
- previous_jdbc_jar = format("{kafka_home}/libs/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ if has_ranger_admin and stack_supports_ranger_audit_db:
+ xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR']
+ jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config)
+
+ downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_target = format("{kafka_home}/libs/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ previous_jdbc_jar = format("{kafka_home}/libs/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
xa_audit_db_is_enabled = False
- ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
if xml_configurations_supported and stack_supports_ranger_audit_db:
xa_audit_db_is_enabled = config['configurations']['ranger-kafka-audit']['xasecure.audit.destination.db']
+
xa_audit_hdfs_is_enabled = default('/configurations/ranger-kafka-audit/xasecure.audit.destination.hdfs', False)
- ssl_keystore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
- ssl_truststore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
- credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
+ ssl_keystore_password = config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None
+ ssl_truststore_password = config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None
+ credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
stack_version = get_stack_version('kafka-broker')
setup_ranger_env_sh_source = format('{stack_root}/{stack_version}/ranger-kafka-plugin/install/conf.templates/enable/kafka-ranger-env.sh')
setup_ranger_env_sh_target = format("{conf_dir}/kafka-ranger-env.sh")
- #For SQLA explicitly disable audit to DB for Ranger
- if xa_audit_db_flavor == 'sqla':
+ # for SQLA explicitly disable audit to DB for Ranger
+ if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla':
xa_audit_db_is_enabled = False
+# ranger kafka plugin section end
+
namenode_hosts = default("/clusterHostInfo/namenode_host", [])
has_namenode = not len(namenode_hosts) == 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
index 528dec2..e9719aa 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
@@ -22,7 +22,7 @@ from resource_management.libraries.functions.setup_ranger_plugin_xml import setu
def setup_ranger_kafka():
import params
- if params.has_ranger_admin:
+ if params.enable_ranger_kafka:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
@@ -87,4 +87,4 @@ def setup_ranger_kafka():
else:
Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations")
else:
- Logger.info('Ranger admin not installed')
+ Logger.info('Ranger Kafka plugin is not enabled')