You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2015/10/22 19:19:01 UTC
ambari git commit: AMBARI-13421. Blueprints: install for Ranger
Components (ranger-admin, ranger-usersync,
ranger-kms) (Sebastian Toader via smohanty)
Repository: ambari
Updated Branches:
refs/heads/trunk 1ff22dffe -> c4c833842
AMBARI-13421. Blueprints: install for Ranger Components (ranger-admin, ranger-usersync, ranger-kms) (Sebastian Toader via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c4c83384
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c4c83384
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c4c83384
Branch: refs/heads/trunk
Commit: c4c833842978d321c0d08c319b2f308d5861e323
Parents: 1ff22df
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Oct 22 10:17:07 2015 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Oct 22 10:17:07 2015 -0700
----------------------------------------------------------------------
.../libraries/functions/ranger_functions.py | 8 +-
.../libraries/functions/ranger_functions_v2.py | 39 +++++--
.../libraries/functions/setup_ranger_plugin.py | 6 +-
.../functions/setup_ranger_plugin_xml.py | 6 +-
.../java/org/apache/ambari/server/Role.java | 6 ++
.../HBASE/0.96.0.2.0/package/scripts/params.py | 3 +-
.../package/scripts/setup_ranger_hbase.py | 7 +-
.../HDFS/2.1.0.2.0/package/scripts/params.py | 1 +
.../package/scripts/setup_ranger_hdfs.py | 7 +-
.../HIVE/0.12.0.2.0/package/scripts/params.py | 1 +
.../package/scripts/setup_ranger_hive.py | 9 +-
.../KAFKA/0.8.1.2.2/package/scripts/params.py | 1 +
.../package/scripts/setup_ranger_kafka.py | 8 +-
.../KNOX/0.5.0.2.2/package/scripts/params.py | 3 +-
.../package/scripts/setup_ranger_knox.py | 9 +-
.../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 1 -
.../STORM/0.9.1.2.1/package/scripts/params.py | 2 +-
.../package/scripts/setup_ranger_storm.py | 7 +-
.../YARN/2.1.0.2.0/package/scripts/params.py | 2 +-
.../package/scripts/setup_ranger_yarn.py | 10 +-
.../stacks/HDP/2.2/role_command_order.json | 13 ++-
.../ambari/server/stack/StackManagerTest.java | 105 +++++++++++++++++++
22 files changed, 218 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
index de58976..dcf59c1 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
@@ -30,7 +30,7 @@ import re
class Rangeradmin:
sInstance = None
- def __init__(self, url='http://localhost:6080'):
+ def __init__(self, url='http://localhost:6080', skip_if_rangeradmin_down = True):
self.baseUrl = url
self.urlLogin = self.baseUrl + '/login.jsp'
@@ -41,6 +41,10 @@ class Rangeradmin:
self.urlGroups = self.baseUrl + '/service/xusers/groups'
self.urlUsers = self.baseUrl + '/service/xusers/users'
self.urlSecUsers = self.baseUrl + '/service/xusers/secure/users'
+ self.skip_if_rangeradmin_down = skip_if_rangeradmin_down
+
+ if self.skip_if_rangeradmin_down:
+ Logger.info("Rangeradmin: Skip ranger admin if it's down !")
def get_repository_by_name_urllib2(self, name, component, status, usernamepassword):
"""
@@ -121,6 +125,8 @@ class Rangeradmin:
raise Fail('{0} Repository creation failed in Ranger admin'.format(component.title()))
else:
raise Fail('Ambari admin user creation failed')
+ elif not self.skip_if_rangeradmin_down:
+ raise Fail("Connection failed to Ranger Admin !")
def create_repository_urllib2(self, data, usernamepassword, policy_user):
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
index 81658bf..b79f6d8 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
@@ -27,12 +27,10 @@ from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.format import format
import re
-
class RangeradminV2:
sInstance = None
- def __init__(self, url='http://localhost:6080'):
-
+ def __init__(self, url='http://localhost:6080', skip_if_rangeradmin_down = True):
self.base_url = url
self.url_login = self.base_url + '/login.jsp'
self.url_login_post = self.base_url + '/j_spring_security_check'
@@ -42,6 +40,10 @@ class RangeradminV2:
self.url_groups = self.base_url + '/service/xusers/groups'
self.url_users = self.base_url + '/service/xusers/users'
self.url_sec_users = self.base_url + '/service/xusers/secure/users'
+ self.skip_if_rangeradmin_down = skip_if_rangeradmin_down
+
+ if self.skip_if_rangeradmin_down:
+ Logger.info("RangeradminV2: Skip ranger admin if it's down !")
def get_repository_by_name_urllib2(self, name, component, status, usernamepassword):
"""
@@ -88,21 +90,36 @@ class RangeradminV2:
ambari_ranger_password = unicode(ambari_ranger_password)
admin_password = unicode(admin_password)
ambari_username_password_for_ranger = format('{ambari_ranger_admin}:{ambari_ranger_password}')
+
if response_code is not None and response_code == 200:
user_resp_code = self.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, format("{admin_uname}:{admin_password}"))
if user_resp_code is not None and user_resp_code == 200:
- repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger)
- if repo is not None:
- Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name']))
- else:
- response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger)
- if response is not None:
- Logger.info('{0} Repository created in Ranger admin'.format(component.title()))
+ retryCount = 0
+ while retryCount <= 5:
+ repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger)
+ if repo is not None:
+ Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name']))
+ break
else:
- Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title()))
+ response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger)
+ if response is not None:
+ Logger.info('{0} Repository created in Ranger admin'.format(component.title()))
+ break
+ else:
+ if retryCount < 5:
+ Logger.info("Retry Repository Creation is being called")
+ time.sleep(30) # delay for 30 seconds
+ retryCount += 1
+ else:
+ Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title()))
+ raise Fail('{0} Repository creation failed in Ranger admin'.format(component.title()))
else:
Logger.error('Ambari admin user creation failed')
+ raise Fail('Ambari admin user creation failed')
+ elif not self.skip_if_rangeradmin_down:
+ raise Fail("Connection failed to Ranger Admin !")
+
def create_repository_urllib2(self, data, usernamepassword):
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py
index e4a19aa..e5e4266 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py
@@ -37,7 +37,7 @@ def setup_ranger_plugin(component_select_name, service_name,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
- plugin_enabled, component_user, component_group, api_version=None, **kwargs):
+ plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down = True, **kwargs):
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source),
mode = 0644
@@ -68,9 +68,9 @@ def setup_ranger_plugin(component_select_name, service_name,
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'),)
if api_version == 'v2' and api_version is not None:
- ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url)
+ ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
else:
- ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url)
+ ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
index 0b404a9..29ffe0d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
@@ -44,7 +44,7 @@ def setup_ranger_plugin(component_select_name, service_name,
plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes,
component_list, audit_db_is_enabled, credential_file,
xa_audit_db_password, ssl_truststore_password,
- ssl_keystore_password, api_version=None, hdp_version_override = None):
+ ssl_keystore_password, api_version=None, hdp_version_override = None, skip_if_rangeradmin_down = True):
if audit_db_is_enabled:
File(component_downloaded_custom_connector,
@@ -68,9 +68,9 @@ def setup_ranger_plugin(component_select_name, service_name,
if plugin_enabled:
if api_version == 'v2' and api_version is not None:
- ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url)
+ ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
else:
- ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url)
+ ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/java/org/apache/ambari/server/Role.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/Role.java b/ambari-server/src/main/java/org/apache/ambari/server/Role.java
index df60988..f72cc5b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/Role.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/Role.java
@@ -113,6 +113,12 @@ public class Role {
public static final Role METRICS_MONITOR = valueOf("METRICS_MONITOR");
public static final Role AMS_SERVICE_CHECK = valueOf("AMBARI_METRICS_SERVICE_CHECK");
public static final Role ACCUMULO_CLIENT = valueOf("ACCUMULO_CLIENT");
+ public static final Role RANGER_ADMIN = valueOf("RANGER_ADMIN");
+ public static final Role RANGER_USERSYNC = valueOf("RANGER_USERSYNC");
+ public static final Role KNOX_GATEWAY = valueOf("KNOX_GATEWAY");
+ public static final Role KAFKA_BROKER = valueOf("KAFKA_BROKER");
+ public static final Role NIMBUS = valueOf("NIMBUS");
+ public static final Role RANGER_KMS_SERVER = valueOf("RANGER_KMS_SERVER");
public static final Role INSTALL_PACKAGES = valueOf("install_packages");
public static final Role UPDATE_REPO = valueOf("update_repo");
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py
index 36dd07f..f3208ce 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py
@@ -25,4 +25,5 @@ if OSCheck.is_windows_family():
else:
from params_linux import *
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
\ No newline at end of file
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+retryAble = default("/commandParams/command_retry_enabled", False)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
index 6b4dfaa..8f4a6d0 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
@@ -34,6 +34,11 @@ def setup_ranger_hbase(rolling_upgrade = False):
if rolling_upgrade:
hdp_version = params.version
+ if params.retryAble:
+ Logger.info("HBase: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("HBase: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
setup_ranger_plugin('hbase-client', 'hbase',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
@@ -48,6 +53,6 @@ def setup_ranger_hbase(rolling_upgrade = False):
component_list=['hbase-client', 'hbase-master', 'hbase-regionserver'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
- hdp_version_override = hdp_version)
+ hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py
index b89eefd..7514918 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py
@@ -26,3 +26,4 @@ else:
host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
nfsgateway_heapsize = config['configurations']['hadoop-env']['nfsgateway_heapsize']
+retryAble = default("/commandParams/command_retry_enabled", False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
index 6a64b2f..bd158ec 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
@@ -34,6 +34,11 @@ def setup_ranger_hdfs(upgrade_type=None):
if upgrade_type is not None:
hdp_version = params.version
+ if params.retryAble:
+ Logger.info("HDFS: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("HDFS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
setup_ranger_plugin('hadoop-client', 'hdfs',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java_home,
@@ -48,6 +53,6 @@ def setup_ranger_hdfs(upgrade_type=None):
component_list=['hadoop-client'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
- hdp_version_override = hdp_version)
+ hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
index 36f7983..f10a3f3 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
@@ -26,3 +26,4 @@ else:
from params_linux import *
host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+retryAble = default("/commandParams/command_retry_enabled", False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
index b0f0c3f..5fdaa70 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
@@ -32,7 +32,12 @@ def setup_ranger_hive(rolling_upgrade = False):
hdp_version = None
if rolling_upgrade:
hdp_version = params.version
-
+
+ if params.retryAble:
+ Logger.info("Hive: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("Hive: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
setup_ranger_plugin('hive-server2', 'hive',
params.ranger_downloaded_custom_connector, params.ranger_driver_curl_source,
params.ranger_driver_curl_target, params.java64_home,
@@ -47,6 +52,6 @@ def setup_ranger_hive(rolling_upgrade = False):
component_list=['hive-client', 'hive-metastore', 'hive-server2'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
- hdp_version_override = hdp_version)
+ hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
index dc0c087..da8333a 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
@@ -32,6 +32,7 @@ import status_params
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_name = default("/hostLevelParams/stack_name", None)
+retryAble = default("/commandParams/command_retry_enabled", False)
version = default("/commandParams/version", None)
host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py
index 540bb9a..c210791 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py
@@ -24,6 +24,12 @@ def setup_ranger_kafka():
if params.has_ranger_admin:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
+
+ if params.retryAble:
+ Logger.info("Kafka: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("Kafka: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
setup_ranger_plugin('kafka-broker', 'kafka',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
@@ -38,7 +44,7 @@ def setup_ranger_kafka():
component_list=['kafka-broker'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
- api_version = 'v2')
+ api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble)
if params.enable_ranger_kafka:
Execute(('cp', '--remove-destination', params.setup_ranger_env_sh_source, params.setup_ranger_env_sh_target),
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py
index 8fe1028..14e021d 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py
@@ -26,4 +26,5 @@ if OSCheck.is_windows_family():
else:
from params_linux import *
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
\ No newline at end of file
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+retryAble = default("/commandParams/command_retry_enabled", False)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
index f1319b3..1efe9e0 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
@@ -33,7 +33,12 @@ def setup_ranger_knox(rolling_upgrade = False):
if rolling_upgrade:
hdp_version = params.version
- setup_ranger_plugin('knox-server', 'knox',
+ if params.retryAble:
+ Logger.info("Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
+ setup_ranger_plugin('knox-server', 'knox',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java_home,
params.repo_name, params.knox_ranger_plugin_repo,
@@ -47,6 +52,6 @@ def setup_ranger_knox(rolling_upgrade = False):
component_list=['knox-server'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
- hdp_version_override = hdp_version)
+ hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
index 570b2b7..4c5bd1a 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
@@ -264,7 +264,6 @@ def enable_kms_plugin():
import params
if params.has_ranger_admin:
-
ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
ambari_username_password_for_ranger = format("{ambari_ranger_admin}:{ambari_ranger_password}")
response_code = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py
index 1e591f4..f10a3f3 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py
@@ -26,4 +26,4 @@ else:
from params_linux import *
host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
-
+retryAble = default("/commandParams/command_retry_enabled", False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py
index d874ba3..5d90f5b 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py
@@ -33,6 +33,11 @@ def setup_ranger_storm(rolling_upgrade = False):
if rolling_upgrade:
hdp_version = params.version
+ if params.retryAble:
+ Logger.info("Storm: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("Storm: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
setup_ranger_plugin('storm-nimbus', 'storm',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
@@ -47,6 +52,6 @@ def setup_ranger_storm(rolling_upgrade = False):
component_list=['storm-client', 'storm-nimbus'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
- hdp_version_override = hdp_version)
+ hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
index 5695e83..073e84f 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
@@ -28,4 +28,4 @@ else:
from params_linux import *
host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
-
+retryAble = default("/commandParams/command_retry_enabled", False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py
index c8b12df..5db65d0d 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py
@@ -22,8 +22,12 @@ def setup_ranger_yarn():
if params.has_ranger_admin:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
-
-
+
+ if params.retryAble:
+ Logger.info("YARN: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("YARN: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
setup_ranger_plugin('hadoop-yarn-resourcemanager', 'yarn',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
@@ -38,7 +42,7 @@ def setup_ranger_yarn():
component_list=['hadoop-yarn-resourcemanager'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
- api_version = 'v2'
+ api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble
)
else:
Logger.info('Ranger admin not installed')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
index 3571a6c..3beed16 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
@@ -21,7 +21,16 @@
"FLUME_SERVICE_CHECK-SERVICE_CHECK": ["FLUME_HANDLER-START"],
"FALCON_SERVICE_CHECK-SERVICE_CHECK": ["FALCON_SERVER-START"],
"SLIDER_SERVICE_CHECK-SERVICE_CHECK" : ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
- "KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START"],
- "KAFKA_SERVICE_CHECK-SERVICE_CHECK": ["KAFKA_BROKER-START"]
+ "KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START"],
+ "KAFKA_SERVICE_CHECK-SERVICE_CHECK": ["KAFKA_BROKER-START"],
+ "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"],
+ "ZOOKEEPER_SERVER-START" : ["RANGER_USERSYNC-START"],
+ "DATANODE-START" : ["RANGER_USERSYNC-START"],
+ "NAMENODE-START" : ["RANGER_USERSYNC-START"],
+ "KNOX_GATEWAY-START": ["RANGER_USERSYNC-START"],
+ "RESOURCEMANAGER-START" : ["RANGER_USERSYNC-START"],
+ "NIMBUS-START": ["RANGER_USERSYNC-START"],
+ "HBASE_MASTER-START": ["RANGER_USERSYNC-START"],
+ "HIVE_SERVER-START" : ["RANGER_USERSYNC-START"]
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java
index be8b073..6a13ab2 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java
@@ -41,6 +41,8 @@ import java.util.Map;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.Role;
+import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.metadata.ActionMetadata;
import org.apache.ambari.server.orm.dao.MetainfoDAO;
@@ -54,6 +56,7 @@ import org.apache.ambari.server.state.ServiceOsSpecific;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.stack.MetricDefinition;
import org.apache.ambari.server.state.stack.OsFamily;
+import org.apache.ambari.server.state.stack.StackRoleCommandOrder;
import org.apache.commons.lang.StringUtils;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -667,5 +670,107 @@ public class StackManagerTest {
}
}
+ @Test
+ public void testServicesWithRangerPluginRoleCommandOrder() throws AmbariException {
+ // Given
+ String stackRoot = ClassLoader.getSystemClassLoader().getResource("stacks").getPath().replace("test-classes","classes");
+ String commonServices = ClassLoader.getSystemClassLoader().getResource("common-services").getPath().replace("test-classes","classes");
+
+ MetainfoDAO metaInfoDao = createNiceMock(MetainfoDAO.class);
+ StackDAO stackDao = createNiceMock(StackDAO.class);
+ ActionMetadata actionMetadata = createNiceMock(ActionMetadata.class);
+ Configuration config = createNiceMock(Configuration.class);
+
+ expect(config.getSharedResourcesDirPath()).andReturn(
+ ClassLoader.getSystemClassLoader().getResource("").getPath()).anyTimes();
+
+ replay(config, metaInfoDao, stackDao, actionMetadata);
+
+ OsFamily osFamily = new OsFamily(config);
+
+ StackManager stackManager = new StackManager(new File(stackRoot), new File(commonServices), osFamily, metaInfoDao, actionMetadata, stackDao);
+
+ String rangerUserSyncRoleCommand = Role.RANGER_USERSYNC + "-" + RoleCommand.START;
+ String rangerAdminRoleCommand = Role.RANGER_ADMIN + "-" + RoleCommand.START;
+
+ // When
+ StackInfo hdp = stackManager.getStack("HDP", "2.3");
+ Map<String, Object> rco = hdp.getRoleCommandOrder().getContent();
+
+ // Then
+ // verify that services that have ranger plugin are after ranger admin in the role command order sequence
+ // as these services require ranger admin and ranger user sync to up upfront
+ Map<String, Object> generalDeps = (Map<String, Object>)rco.get("general_deps");
+
+ // HDFS
+ String nameNodeRoleCommand = Role.NAMENODE + "-" + RoleCommand.START;
+ ArrayList<String> nameNodeBlockers = (ArrayList<String>)generalDeps.get(nameNodeRoleCommand);
+
+ assertTrue(nameNodeRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, nameNodeBlockers.contains(rangerUserSyncRoleCommand));
+
+ String dataNodeRoleCommand = Role.DATANODE + "-" + RoleCommand.START;
+ ArrayList<String> dataNodeBlockers = (ArrayList<String>)generalDeps.get(dataNodeRoleCommand);
+
+ assertTrue(dataNodeRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, dataNodeBlockers.contains(rangerUserSyncRoleCommand));
+
+ // YARN
+ String resourceManagerCommandRoleCommand = Role.RESOURCEMANAGER + "-" + RoleCommand.START;
+ ArrayList<String> resourceManagerBlockers = (ArrayList<String>)generalDeps.get(resourceManagerCommandRoleCommand);
+
+ assertTrue(resourceManagerCommandRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, resourceManagerBlockers.contains(rangerUserSyncRoleCommand));
+
+
+ // HBase
+ String hbaseRoleCommand = Role.HBASE_MASTER + "-" + RoleCommand.START;
+ ArrayList<String> hbaseBlockers = (ArrayList<String>)generalDeps.get(hbaseRoleCommand);
+
+ assertTrue(hbaseRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, hbaseBlockers.contains(rangerUserSyncRoleCommand));
+
+ // Knox
+ String knoxRoleCommand = Role.KNOX_GATEWAY + "-" + RoleCommand.START;
+ ArrayList<String> knoxBlockers = (ArrayList<String>)generalDeps.get(knoxRoleCommand);
+
+ assertTrue(knoxRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, knoxBlockers.contains(rangerUserSyncRoleCommand));
+
+ // Kafka
+ String kafkaRoleCommand = Role.KAFKA_BROKER + "-" + RoleCommand.START;
+ ArrayList<String> kafkaBlockers = (ArrayList<String>)generalDeps.get(kafkaRoleCommand);
+
+ assertTrue(Role.KAFKA_BROKER + "-" + RoleCommand.START + " should be dependent of " + rangerUserSyncRoleCommand, kafkaBlockers.contains(rangerUserSyncRoleCommand));
+
+ // Hive
+ String hiveRoleCommand = Role.HIVE_SERVER + "-" + RoleCommand.START;
+ ArrayList<String> hiveBlockers = (ArrayList<String>)generalDeps.get(hiveRoleCommand);
+
+ assertTrue(hiveRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, hiveBlockers.contains(rangerUserSyncRoleCommand));
+
+ // Storm
+ String stormRoleCommand = Role.NIMBUS + "-" + RoleCommand.START;
+ ArrayList<String> stormBlockers = (ArrayList<String>)generalDeps.get(stormRoleCommand);
+
+ assertTrue(stormRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, stormBlockers.contains(rangerUserSyncRoleCommand));
+
+ // Ranger KMS
+ String kmsRoleCommand = Role.RANGER_KMS_SERVER + "-" + RoleCommand.START;
+ ArrayList<String> rangerKmsBlockers = (ArrayList<String>)generalDeps.get(kmsRoleCommand);
+
+ assertTrue(kmsRoleCommand + " should be dependent of " + rangerAdminRoleCommand, rangerKmsBlockers.contains(rangerAdminRoleCommand));
+
+ // Ranger User Sync
+ ArrayList<String> rangerUserSyncBlockers = (ArrayList<String>)generalDeps.get(rangerUserSyncRoleCommand);
+
+ assertTrue(rangerUserSyncRoleCommand + " should be dependent of " + rangerAdminRoleCommand, rangerUserSyncBlockers.contains(rangerAdminRoleCommand));
+ assertTrue(rangerUserSyncRoleCommand + " should be dependent of " + kmsRoleCommand, rangerUserSyncBlockers.contains(kmsRoleCommand));
+
+ // Zookeeper Server
+ String zookeeperServerRoleCommand = Role.ZOOKEEPER_SERVER + "-" + RoleCommand.START;
+ ArrayList<String> zookeeperBlockers = (ArrayList<String>)generalDeps.get(zookeeperServerRoleCommand);
+
+ assertTrue(zookeeperServerRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, zookeeperBlockers.contains(rangerUserSyncRoleCommand));
+
+
+ }
+
+
//todo: component override assertions
}