You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by yu...@apache.org on 2015/01/27 21:49:28 UTC
[2/2] ambari git commit: AMBARI-9188. Adhere to ambari's 2-space
formatting rule. AMBARI-9285. Add default values for ranger plugin
properties. AMBARI-9286. Stack advisor recommendations. (Gautam Borad via
yusaku)
AMBARI-9188. Adhere to ambari's 2-space formatting rule.
AMBARI-9285. Add default values for ranger plugin properties.
AMBARI-9286. Stack advisor recommendations.
(Gautam Borad via yusaku)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4cbf3a87
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4cbf3a87
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4cbf3a87
Branch: refs/heads/trunk
Commit: 4cbf3a876bafcea62cd2e84062523b5088b3ab42
Parents: 46b23d4
Author: Yusaku Sako <yu...@hortonworks.com>
Authored: Tue Jan 27 12:44:24 2015 -0800
Committer: Yusaku Sako <yu...@hortonworks.com>
Committed: Tue Jan 27 12:44:24 2015 -0800
----------------------------------------------------------------------
.../libraries/functions/ranger_functions.py | 219 ++++++++----
.../HBASE/0.96.0.2.0/package/scripts/params.py | 12 +
.../package/scripts/setup_ranger_hbase.py | 300 ++++++++--------
.../HDFS/2.1.0.2.0/package/scripts/params.py | 24 +-
.../package/scripts/setup_ranger_hdfs.py | 350 +++++++++++--------
.../HIVE/0.12.0.2.0/package/scripts/params.py | 20 +-
.../package/scripts/setup_ranger_hive.py | 291 ++++++++-------
.../KNOX/0.5.0.2.2/package/scripts/params.py | 24 +-
.../package/scripts/setup_ranger_knox.py | 294 +++++++++-------
.../RANGER/0.4.0/configuration/ranger-env.xml | 16 +-
.../RANGER/0.4.0/configuration/ranger-site.xml | 67 ++++
.../common-services/RANGER/0.4.0/metainfo.xml | 1 +
.../RANGER/0.4.0/package/scripts/params.py | 12 +
.../0.4.0/package/scripts/ranger_admin.py | 56 +--
.../0.4.0/package/scripts/ranger_usersync.py | 54 +--
.../0.4.0/package/scripts/service_check.py | 5 +-
.../0.4.0/package/scripts/setup_ranger.py | 277 +++++++++------
.../STORM/0.9.1.2.1/metainfo.xml | 1 +
.../STORM/0.9.1.2.1/package/scripts/nimbus.py | 3 +-
.../STORM/0.9.1.2.1/package/scripts/params.py | 26 ++
.../package/scripts/setup_ranger_storm.py | 206 +++++++++++
.../0.9.1.2.1/package/scripts/ui_server.py | 4 +-
.../ranger-storm-plugin-properties.xml | 150 ++++++++
.../stacks/HDP/2.2/services/stack_advisor.py | 16 +-
.../stacks/2.1/configs/default-storm-start.json | 3 +
.../stacks/2.1/configs/secured-storm-start.json | 3 +
26 files changed, 1619 insertions(+), 815 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cbf3a87/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
index 4036611..4d04928 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
@@ -24,21 +24,23 @@ import json
from resource_management.core.logger import Logger
import urllib2, base64, httplib
+
class Rangeradmin:
sInstance = None
- def __init__(self, url= 'http://localhost:6080'):
-
- self.baseUrl = url
- self.urlLogin = self.baseUrl + '/login.jsp'
+
+ def __init__(self, url='http://localhost:6080'):
+
+ self.baseUrl = url
+ self.urlLogin = self.baseUrl + '/login.jsp'
self.urlLoginPost = self.baseUrl + '/j_spring_security_check'
- self.urlRepos = self.baseUrl + '/service/assets/assets'
- self.urlReposPub = self.baseUrl + '/service/public/api/repository'
- self.urlPolicies = self.baseUrl + '/service/public/api/policy'
- self.urlGroups = self.baseUrl + '/service/xusers/groups'
- self.urlUsers = self.baseUrl + '/service/xusers/users'
- self.urlSecUsers = self.baseUrl + '/service/xusers/secure/users'
-
- self.session = None
+ self.urlRepos = self.baseUrl + '/service/assets/assets'
+ self.urlReposPub = self.baseUrl + '/service/public/api/repository'
+ self.urlPolicies = self.baseUrl + '/service/public/api/policy'
+ self.urlGroups = self.baseUrl + '/service/xusers/groups'
+ self.urlUsers = self.baseUrl + '/service/xusers/users'
+ self.urlSecUsers = self.baseUrl + '/service/xusers/secure/users'
+
+ self.session = None
self.isLoggedIn = False
def get_repository_by_name_urllib2(self, name, component, status, usernamepassword):
@@ -46,11 +48,11 @@ class Rangeradmin:
searchRepoURL = self.urlReposPub + "?name=" + name + "&type=" + component + "&status=" + status
request = urllib2.Request(searchRepoURL)
base64string = base64.encodestring(usernamepassword).replace('\n', '')
- request.add_header("Content-Type", "application/json")
- request.add_header("Accept", "application/json")
- request.add_header("Authorization", "Basic %s" % base64string)
+ request.add_header("Content-Type", "application/json")
+ request.add_header("Accept", "application/json")
+ request.add_header("Authorization", "Basic %s" % base64string)
result = urllib2.urlopen(request)
- response_code = result.getcode()
+ response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response['vXRepositories']) > 0:
@@ -63,10 +65,10 @@ class Rangeradmin:
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
- Logger.error("HTTP Code: %s" % e.code)
- Logger.error("HTTP Data: %s" % e.read())
+ Logger.error("HTTP Code: %s" % e.code)
+ Logger.error("HTTP Data: %s" % e.read())
else:
- Logger.error("Error : %s" % (e.reason))
+ Logger.error("Error : %s" % (e.reason))
return None
except httplib.BadStatusLine:
Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
@@ -81,93 +83,96 @@ class Rangeradmin:
"Content-Type": "application/json"
}
request = urllib2.Request(searchRepoURL, data, headers)
- request.add_header("Authorization", "Basic %s" % base64string)
+ request.add_header("Authorization", "Basic %s" % base64string)
result = urllib2.urlopen(request)
- response_code = result.getcode()
+ response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
- if response_code == 200 :
+ if response_code == 200:
Logger.info('Repository created Successfully')
- #Get Policies
- repoData = json.loads(data)
- repoName = repoData['name']
+ # Get Policies
+ repoData = json.loads(data)
+ repoName = repoData['name']
typeOfPolicy = repoData['repositoryType']
##Get Policies by repo name
- policyList = self.get_policy_by_repo_name(name=repoName, component=typeOfPolicy, status="true", usernamepassword=usernamepassword)
- if (len(policyList)) > 0 :
+ policyList = self.get_policy_by_repo_name(name=repoName, component=typeOfPolicy, status="true",
+ usernamepassword=usernamepassword)
+ if (len(policyList)) > 0:
policiesUpdateCount = 0
for policy in policyList:
- updatedPolicyObj = self.get_policy_params(typeOfPolicy,policy)
- policyResCode, policyResponse = self.update_ranger_policy(updatedPolicyObj['id'], json.dumps(updatedPolicyObj), usernamepassword)
+ updatedPolicyObj = self.get_policy_params(typeOfPolicy, policy)
+ policyResCode, policyResponse = self.update_ranger_policy(updatedPolicyObj['id'],
+ json.dumps(updatedPolicyObj), usernamepassword)
if policyResCode == 200:
- policiesUpdateCount = policiesUpdateCount+1
+ policiesUpdateCount = policiesUpdateCount + 1
else:
- Logger.info('Policy Update failed')
- ##Check for count of updated policies
+ Logger.info('Policy Update failed')
+ ##Check for count of updated policies
if len(policyList) == policiesUpdateCount:
- Logger.info("Ranger Repository created successfully and policies updated successfully providing ambari-qa user all permissions")
+ Logger.info(
+ "Ranger Repository created successfully and policies updated successfully providing ambari-qa user all permissions")
return response
else:
return None
else:
Logger.info("Policies not found for the newly created Repository")
- return None
+ return None
else:
Logger.info('Repository creation failed')
- return None
+ return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
- Logger.error("HTTP Code: %s" % e.code)
- Logger.error("HTTP Data: %s" % e.read())
+ Logger.error("HTTP Code: %s" % e.code)
+ Logger.error("HTTP Data: %s" % e.read())
else:
- Logger.error("Error: %s" % (e.reason))
+ Logger.error("Error: %s" % (e.reason))
return None
except httplib.BadStatusLine:
Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
return None
- def check_ranger_login_urllib2(self, url,usernamepassword ):
+ def check_ranger_login_urllib2(self, url, usernamepassword):
try:
request = urllib2.Request(url)
base64string = base64.encodestring(usernamepassword).replace('\n', '')
- request.add_header("Content-Type", "application/json")
- request.add_header("Accept", "application/json")
- request.add_header("Authorization", "Basic %s" % base64string)
+ request.add_header("Content-Type", "application/json")
+ request.add_header("Accept", "application/json")
+ request.add_header("Authorization", "Basic %s" % base64string)
result = urllib2.urlopen(request)
response = result.read()
- response_code = result.getcode()
+ response_code = result.getcode()
return response_code, response
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
- Logger.error("HTTP Code: %s" % e.code)
- Logger.error("HTTP Data: %s" % e.read())
+ Logger.error("HTTP Code: %s" % e.code)
+ Logger.error("HTTP Data: %s" % e.read())
else:
- Logger.error("Error : %s" % (e.reason))
+ Logger.error("Error : %s" % (e.reason))
return None, None
except httplib.BadStatusLine, e:
Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
- return None, None
+ return None, None
def get_policy_by_repo_name(self, name, component, status, usernamepassword):
try:
searchPolicyURL = self.urlPolicies + "?repositoryName=" + name + "&repositoryType=" + component + "&isEnabled=" + status
request = urllib2.Request(searchPolicyURL)
base64string = base64.encodestring(usernamepassword).replace('\n', '')
- request.add_header("Content-Type", "application/json")
- request.add_header("Accept", "application/json")
- request.add_header("Authorization", "Basic %s" % base64string)
+ request.add_header("Content-Type", "application/json")
+ request.add_header("Accept", "application/json")
+ request.add_header("Authorization", "Basic %s" % base64string)
result = urllib2.urlopen(request)
- response_code = result.getcode()
+ response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response['vXPolicies']) > 0:
- return response['vXPolicies']
+ return response['vXPolicies']
else:
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
- Logger.error("HTTP Code: %s" % e.code)
- Logger.error("HTTP Data: %s" % e.read())
+ Logger.error("HTTP Code: %s" % e.code)
+ Logger.error("HTTP Data: %s" % e.read())
else:
- Logger.error("Error: %s" % (e.reason))
+ Logger.error("Error: %s" % (e.reason))
return None
except httplib.BadStatusLine:
Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
@@ -175,19 +180,19 @@ class Rangeradmin:
def update_ranger_policy(self, policyId, data, usernamepassword):
try:
- searchRepoURL = self.urlPolicies +"/"+str(policyId)
+ searchRepoURL = self.urlPolicies + "/" + str(policyId)
base64string = base64.encodestring('%s' % (usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(searchRepoURL, data, headers)
- request.add_header("Authorization", "Basic %s" % base64string)
+ request.add_header("Authorization", "Basic %s" % base64string)
request.get_method = lambda: 'PUT'
result = urllib2.urlopen(request)
- response_code = result.getcode()
+ response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
- if response_code == 200 :
+ if response_code == 200:
Logger.info('Policy updated Successfully')
return response_code, response
else:
@@ -195,27 +200,101 @@ class Rangeradmin:
return None, None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
- Logger.error("HTTP Code: %s" % e.code)
- Logger.error("HTTP Data: %s" % e.read())
+ Logger.error("HTTP Code: %s" % e.code)
+ Logger.error("HTTP Data: %s" % e.read())
else:
- Logger.error("Error: %s" % (e.reason))
+ Logger.error("Error: %s" % (e.reason))
return None, None
except httplib.BadStatusLine:
Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
return None, None
- def get_policy_params(self, typeOfPolicy,policyObj):
-
+ def get_policy_params(self, typeOfPolicy, policyObj):
+
typeOfPolicy = typeOfPolicy.lower()
if typeOfPolicy == "hdfs":
- policyObj['permMapList'] = [{'userList':['ambari-qa'],'permList': ['Read','Write', 'Execute', 'Admin']}]
+ policyObj['permMapList'] = [{'userList': ['ambari-qa'], 'permList': ['Read', 'Write', 'Execute', 'Admin']}]
elif typeOfPolicy == "hive":
- policyObj['permMapList'] = [{'userList':['ambari-qa'], 'permList':[ 'Select','Update', 'Create', 'Drop', 'Alter', 'Index', 'Lock', 'All', 'Admin' ]}]
+ policyObj['permMapList'] = [{'userList': ['ambari-qa'],
+ 'permList': ['Select', 'Update', 'Create', 'Drop', 'Alter', 'Index', 'Lock', 'All',
+ 'Admin']}]
elif typeOfPolicy == "hbase":
- policyObj['permMapList'] = [{'userList':['ambari-qa'],'permList':[ 'Read', 'Write', 'Create', 'Admin']}]
+ policyObj['permMapList'] = [{'userList': ['ambari-qa'], 'permList': ['Read', 'Write', 'Create', 'Admin']}]
elif typeOfPolicy == "knox":
- policyObj['permMapList'] = [{'userList':['ambari-qa'], 'permList': ['Allow','Admin']}]
- elif typeOfPolicy == "storm" :
- policyObj['permMapList'] = [{'userList':['ambari-qa'], 'permList':[ 'Submit Topology', 'File Upload', 'Get Nimbus Conf', 'Get Cluster Info', 'File Download', 'Kill Topology', 'Rebalance', 'Activate','Deactivate', 'Get Topology Conf', 'Get Topology', 'Get User Topology', 'Get Topology Info', 'Upload New Credential', 'Admin']}]
+ policyObj['permMapList'] = [{'userList': ['ambari-qa'], 'permList': ['Allow', 'Admin']}]
+ elif typeOfPolicy == "storm":
+ policyObj['permMapList'] = [{'userList': ['ambari-qa', 'storm'],
+ 'permList': ['SubmitTopology', 'FileUpload', 'GetNimbusConf', 'GetClusterInfo',
+ 'FileDownload', 'KillTopology', 'Rebalance', 'Activate', 'Deactivate',
+ 'GetTopologyConf', 'GetTopology', 'GetUserTopology',
+ 'GetTopologyInfo', 'UploadNewCredential', 'Admin']}]
return policyObj
+
+ def create_ambari_admin_user(self,ambari_admin_username, ambari_admin_password,usernamepassword):
+ try:
+ url = self.urlUsers + '?startIndex=0'
+ request = urllib2.Request(url)
+ base64string = base64.encodestring(usernamepassword).replace('\n', '')
+ request.add_header("Content-Type", "application/json")
+ request.add_header("Accept", "application/json")
+ request.add_header("Authorization", "Basic %s" % base64string)
+ result = urllib2.urlopen(request)
+ response_code = result.getcode()
+ response = json.loads(result.read())
+ if response_code == 200 and len(response['vXUsers']) > 0:
+ ambari_admin_username = ambari_admin_username
+ flag_ambari_admin_present = False
+ for vxuser in response['vXUsers']:
+ rangerlist_username = vxuser['name']
+ if rangerlist_username == ambari_admin_username:
+ flag_ambari_admin_present = True
+ break
+ else:
+ flag_ambari_admin_present = False
+
+ if flag_ambari_admin_present:
+ Logger.info(ambari_admin_username + ' user already exists, using existing user from configurations.')
+ return ambari_admin_username,ambari_admin_password
+ else:
+ Logger.info(ambari_admin_username + ' user is not present, creating user using given configurations')
+ url = self.urlSecUsers
+ admin_user = dict()
+ admin_user['status'] = 1
+ admin_user['userRoleList'] = ['ROLE_SYS_ADMIN']
+ admin_user['name'] = ambari_admin_username
+ admin_user['password'] = ambari_admin_password
+ admin_user['description'] = ambari_admin_username
+ admin_user['firstName'] = ambari_admin_username
+ data = json.dumps(admin_user)
+ base64string = base64.encodestring('%s' % (usernamepassword)).replace('\n', '')
+ headers = {
+ 'Accept': 'application/json',
+ "Content-Type": "application/json"
+ }
+ request = urllib2.Request(url, data, headers)
+ request.add_header("Authorization", "Basic %s" % base64string)
+ result = urllib2.urlopen(request)
+ response_code = result.getcode()
+ response = json.loads(json.JSONEncoder().encode(result.read()))
+ if response_code == 200 and response is not None:
+ Logger.info('Ambari admin user creation successful.')
+ else:
+ Logger.info('Ambari admin user creation failed,setting username and password as blank')
+ ambari_admin_username = ''
+ ambari_admin_password = ''
+ return ambari_admin_username,ambari_admin_password
+ else:
+ return '',''
+
+ except urllib2.URLError, e:
+ if isinstance(e, urllib2.HTTPError):
+ Logger.error("HTTP Code: %s" % e.code)
+ Logger.error("HTTP Data: %s" % e.read())
+ return '',''
+ else:
+ Logger.error("Error: %s" % (e.reason))
+ return '',''
+ except httplib.BadStatusLine:
+ Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
+ return '',''
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cbf3a87/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py
index 74cc0a4..a50b985 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py
@@ -164,6 +164,7 @@ if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
region_drainer = format("/usr/hdp/current/hbase-{role_root}/bin/draining_servers.rb")
hbase_cmd = format("/usr/hdp/current/hbase-{role_root}/bin/hbase")
+user_input = default("/configurations/ranger-hbase-plugin-properties/ranger-hbase-plugin-enabled","no")
if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
# Setting Flag value for ranger hbase plugin
enable_ranger_hbase = False
@@ -176,3 +177,14 @@ if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
# ranger host
ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
has_ranger_admin = not len(ranger_admin_hosts) == 0
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+jdk_location = config['hostLevelParams']['jdk_location']
+java_share_dir = '/usr/share/java'
+jdbc_jar_name = "mysql-connector-java.jar"
+
+downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}")
+
+driver_curl_source = format("{jdk_location}/{jdbc_jar_name}")
+driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cbf3a87/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
index 6daa0bf..9ef9fec 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py
@@ -22,166 +22,186 @@ import fileinput
import subprocess
import json
import re
+import os
from resource_management import *
from resource_management.libraries.functions.ranger_functions import Rangeradmin
from resource_management.core.logger import Logger
def setup_ranger_hbase(env):
- import params
- env.set_params(params)
-
- if params.has_ranger_admin:
- try:
- command = 'hdp-select status hbase-client'
- return_code, hdp_output = shell.call(command, timeout=20)
- except Exception, e:
- Logger.error(str(e))
- raise Fail('Unable to execute hdp-select command to retrieve the version.')
-
- if return_code != 0:
- raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
-
- hdp_version = re.sub('hbase-client - ', '', hdp_output)
- match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
-
- if match is None:
- raise Fail('Failed to get extracted version')
-
- file_path = '/usr/hdp/'+ hdp_version +'/ranger-hbase-plugin/install.properties'
-
- ranger_hbase_dict = ranger_hbase_properties(params)
- hbase_repo_data = hbase_repo_properties(params)
-
- write_properties_to_file(file_path, ranger_hbase_dict)
-
- if params.enable_ranger_hbase:
- cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh enable-hbase-plugin.sh')
- ranger_adm_obj = Rangeradmin(url=ranger_hbase_dict['POLICY_MGR_URL'])
- response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(ranger_hbase_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')
-
- if response_code is not None and response_code == 200:
- repo = ranger_adm_obj.get_repository_by_name_urllib2(ranger_hbase_dict['REPOSITORY_NAME'], 'hbase', 'true', 'admin:admin')
-
- if repo and repo['name'] == ranger_hbase_dict['REPOSITORY_NAME']:
- Logger.info('Hbase Repository exist')
- else:
- response = ranger_adm_obj.create_repository_urllib2(hbase_repo_data, 'admin:admin')
- if response is not None:
- Logger.info('Hbase Repository created in Ranger admin')
- else:
- Logger.info('Hbase Repository creation failed in Ranger admin')
+ import params
+ env.set_params(params)
+
+ if params.has_ranger_admin:
+
+ environment = {"no_proxy": format("{params.ambari_server_hostname}")}
+
+ Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
+ params.downloaded_custom_connector),
+ not_if=format("test -f {params.downloaded_custom_connector}"),
+ path=["/bin", "/usr/bin/"],
+ environment=environment,
+ sudo=True)
+
+ if not os.path.isfile(params.driver_curl_target):
+ Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
+ path=["/bin", "/usr/bin/"],
+ sudo=True)
+
+ try:
+ command = 'hdp-select status hbase-client'
+ return_code, hdp_output = shell.call(command, timeout=20)
+ except Exception, e:
+ Logger.error(str(e))
+ raise Fail('Unable to execute hdp-select command to retrieve the version.')
+
+ if return_code != 0:
+ raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
+
+ hdp_version = re.sub('hbase-client - ', '', hdp_output)
+ match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
+
+ if match is None:
+ raise Fail('Failed to get extracted version')
+
+ file_path = '/usr/hdp/'+ hdp_version +'/ranger-hbase-plugin/install.properties'
+
+ ranger_hbase_dict = ranger_hbase_properties(params)
+ hbase_repo_data = hbase_repo_properties(params)
+
+ write_properties_to_file(file_path, ranger_hbase_dict)
+
+ if params.enable_ranger_hbase:
+ cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh enable-hbase-plugin.sh')
+ ranger_adm_obj = Rangeradmin(url=ranger_hbase_dict['POLICY_MGR_URL'])
+ response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(ranger_hbase_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')
+
+ if response_code is not None and response_code == 200:
+ ambari_ranger_admin = params.config['configurations']['ranger-env']['ranger_admin_username']
+ ambari_ranger_password = params.config['configurations']['ranger-env']['ranger_admin_password']
+ ambari_ranger_admin,ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, 'admin:admin')
+ ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
+ if ambari_ranger_admin != '' and ambari_ranger_password != '':
+ repo = ranger_adm_obj.get_repository_by_name_urllib2(ranger_hbase_dict['REPOSITORY_NAME'], 'hbase', 'true', ambari_username_password_for_ranger)
+ if repo and repo['name'] == ranger_hbase_dict['REPOSITORY_NAME']:
+ Logger.info('Hbase Repository exist')
+ else:
+ response = ranger_adm_obj.create_repository_urllib2(hbase_repo_data, ambari_username_password_for_ranger)
+ if response is not None:
+ Logger.info('Hbase Repository created in Ranger admin')
else:
- Logger.info('Ranger service is not started on given host')
+ Logger.info('Hbase Repository creation failed in Ranger admin')
else:
- cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh disable-hbase-plugin.sh')
-
- Execute(cmd, environment={'JAVA_HOME': params.java64_home}, logoutput=True)
+ Logger.info('Ambari admin username and password are blank ')
+ else:
+ Logger.info('Ranger service is not started on given host')
else:
- Logger.info('Ranger admin not installed')
+ cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh disable-hbase-plugin.sh')
+
+ Execute(cmd, environment={'JAVA_HOME': params.java64_home}, logoutput=True)
+ else:
+ Logger.info('Ranger admin not installed')
def write_properties_to_file(file_path, value):
- for key in value:
- modify_config(file_path, key, value[key])
+ for key in value:
+ modify_config(file_path, key, value[key])
def modify_config(filepath, variable, setting):
- var_found = False
- already_set = False
- V=str(variable)
- S=str(setting)
- # use quotes if setting has spaces #
- if ' ' in S:
- S = '%s' % S
-
- for line in fileinput.input(filepath, inplace = 1):
- # process lines that look like config settings #
- if not line.lstrip(' ').startswith('#') and '=' in line:
- _infile_var = str(line.split('=')[0].rstrip(' '))
- _infile_set = str(line.split('=')[1].lstrip(' ').rstrip())
- # only change the first matching occurrence #
- if var_found == False and _infile_var.rstrip(' ') == V:
- var_found = True
- # don't change it if it is already set #
- if _infile_set.lstrip(' ') == S:
- already_set = True
- else:
- line = "%s=%s\n" % (V, S)
-
- sys.stdout.write(line)
-
- # Append the variable if it wasn't found #
- if not var_found:
- with open(filepath, "a") as f:
- f.write("%s=%s\n" % (V, S))
- elif already_set == True:
- pass
- else:
- pass
+ var_found = False
+ already_set = False
+ V=str(variable)
+ S=str(setting)
+ # use quotes if setting has spaces #
+ if ' ' in S:
+ S = '%s' % S
+ for line in fileinput.input(filepath, inplace = 1):
+ # process lines that look like config settings #
+ if not line.lstrip(' ').startswith('#') and '=' in line:
+ _infile_var = str(line.split('=')[0].rstrip(' '))
+ _infile_set = str(line.split('=')[1].lstrip(' ').rstrip())
+ # only change the first matching occurrence #
+ if var_found == False and _infile_var.rstrip(' ') == V:
+ var_found = True
+ # don't change it if it is already set #
+ if _infile_set.lstrip(' ') == S:
+ already_set = True
+ else:
+ line = "%s=%s\n" % (V, S)
+ sys.stdout.write(line)
+
+ # Append the variable if it wasn't found #
+ if not var_found:
+ with open(filepath, "a") as f:
+ f.write("%s=%s\n" % (V, S))
+ elif already_set == True:
+ pass
+ else:
+ pass
- return
+ return
def ranger_hbase_properties(params):
- ranger_hbase_properties = dict()
-
- ranger_hbase_properties['POLICY_MGR_URL'] = params.config['configurations']['admin-properties']['policymgr_external_url']
- ranger_hbase_properties['SQL_CONNECTOR_JAR'] = params.config['configurations']['admin-properties']['SQL_CONNECTOR_JAR']
- ranger_hbase_properties['XAAUDIT.DB.FLAVOUR'] = params.config['configurations']['admin-properties']['DB_FLAVOR']
- ranger_hbase_properties['XAAUDIT.DB.DATABASE_NAME'] = params.config['configurations']['admin-properties']['audit_db_name']
- ranger_hbase_properties['XAAUDIT.DB.USER_NAME'] = params.config['configurations']['admin-properties']['audit_db_user']
- ranger_hbase_properties['XAAUDIT.DB.PASSWORD'] = params.config['configurations']['admin-properties']['audit_db_password']
- ranger_hbase_properties['XAAUDIT.DB.HOSTNAME'] = params.config['configurations']['admin-properties']['db_host']
- ranger_hbase_properties['REPOSITORY_NAME'] = params.config['clusterName'] + '_hbase'
-
- ranger_hbase_properties['XAAUDIT.DB.IS_ENABLED'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.DB.IS_ENABLED']
-
- ranger_hbase_properties['XAAUDIT.HDFS.IS_ENABLED'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.IS_ENABLED']
- ranger_hbase_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINATION_DIRECTORY']
- ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY']
- ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY']
- ranger_hbase_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FILE']
- ranger_hbase_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS']
- ranger_hbase_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS']
- ranger_hbase_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS']
- ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FILE']
- ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS']
- ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS']
- ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT']
+ ranger_hbase_properties = dict()
+
+ ranger_hbase_properties['POLICY_MGR_URL'] = params.config['configurations']['admin-properties']['policymgr_external_url']
+ ranger_hbase_properties['SQL_CONNECTOR_JAR'] = params.config['configurations']['admin-properties']['SQL_CONNECTOR_JAR']
+ ranger_hbase_properties['XAAUDIT.DB.FLAVOUR'] = params.config['configurations']['admin-properties']['DB_FLAVOR']
+ ranger_hbase_properties['XAAUDIT.DB.DATABASE_NAME'] = params.config['configurations']['admin-properties']['audit_db_name']
+ ranger_hbase_properties['XAAUDIT.DB.USER_NAME'] = params.config['configurations']['admin-properties']['audit_db_user']
+ ranger_hbase_properties['XAAUDIT.DB.PASSWORD'] = params.config['configurations']['admin-properties']['audit_db_password']
+ ranger_hbase_properties['XAAUDIT.DB.HOSTNAME'] = params.config['configurations']['admin-properties']['db_host']
+ ranger_hbase_properties['REPOSITORY_NAME'] = str(params.config['clusterName']) + '_hbase'
+
+ ranger_hbase_properties['XAAUDIT.DB.IS_ENABLED'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.DB.IS_ENABLED']
+
+ ranger_hbase_properties['XAAUDIT.HDFS.IS_ENABLED'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.IS_ENABLED']
+ ranger_hbase_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINATION_DIRECTORY']
+ ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY']
+ ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY']
+ ranger_hbase_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FILE']
+ ranger_hbase_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS']
+ ranger_hbase_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS']
+ ranger_hbase_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS']
+ ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FILE']
+ ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS']
+ ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS']
+ ranger_hbase_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = params.config['configurations']['ranger-hbase-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT']
- ranger_hbase_properties['SSL_KEYSTORE_FILE_PATH'] = params.config['configurations']['ranger-hbase-plugin-properties']['SSL_KEYSTORE_FILE_PATH']
- ranger_hbase_properties['SSL_KEYSTORE_PASSWORD'] = params.config['configurations']['ranger-hbase-plugin-properties']['SSL_KEYSTORE_PASSWORD']
- ranger_hbase_properties['SSL_TRUSTSTORE_FILE_PATH'] = params.config['configurations']['ranger-hbase-plugin-properties']['SSL_TRUSTSTORE_FILE_PATH']
- ranger_hbase_properties['SSL_TRUSTSTORE_PASSWORD'] = params.config['configurations']['ranger-hbase-plugin-properties']['SSL_TRUSTSTORE_PASSWORD']
-
- ranger_hbase_properties['UPDATE_XAPOLICIES_ON_GRANT_REVOKE'] = params.config['configurations']['ranger-hbase-plugin-properties']['UPDATE_XAPOLICIES_ON_GRANT_REVOKE']
+ ranger_hbase_properties['SSL_KEYSTORE_FILE_PATH'] = params.config['configurations']['ranger-hbase-plugin-properties']['SSL_KEYSTORE_FILE_PATH']
+ ranger_hbase_properties['SSL_KEYSTORE_PASSWORD'] = params.config['configurations']['ranger-hbase-plugin-properties']['SSL_KEYSTORE_PASSWORD']
+ ranger_hbase_properties['SSL_TRUSTSTORE_FILE_PATH'] = params.config['configurations']['ranger-hbase-plugin-properties']['SSL_TRUSTSTORE_FILE_PATH']
+ ranger_hbase_properties['SSL_TRUSTSTORE_PASSWORD'] = params.config['configurations']['ranger-hbase-plugin-properties']['SSL_TRUSTSTORE_PASSWORD']
+
+ ranger_hbase_properties['UPDATE_XAPOLICIES_ON_GRANT_REVOKE'] = params.config['configurations']['ranger-hbase-plugin-properties']['UPDATE_XAPOLICIES_ON_GRANT_REVOKE']
- return ranger_hbase_properties
+ return ranger_hbase_properties
def hbase_repo_properties(params):
- config_dict = dict()
- config_dict['username'] = params.config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
- config_dict['password'] = params.config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']
- config_dict['hadoop.security.authentication'] = params.config['configurations']['core-site']['hadoop.security.authentication']
- config_dict['hbase.security.authentication'] = params.config['configurations']['hbase-site']['hbase.security.authentication']
- config_dict['hbase.zookeeper.property.clientPort'] = params.config['configurations']['hbase-site']['hbase.zookeeper.property.clientPort']
- config_dict['hbase.zookeeper.quorum'] = params.config['configurations']['hbase-site']['hbase.zookeeper.quorum']
- config_dict['zookeeper.znode.parent'] = params.config['configurations']['hbase-site']['zookeeper.znode.parent']
-
- if params.config['configurations']['cluster-env']['security_enabled']:
- config_dict['hbase.master.kerberos.principal'] = params.config['configurations']['hbase-site']['hbase.master.kerberos.principal']
- else:
- config_dict['hbase.master.kerberos.principal'] = ''
-
- repo= dict()
- repo['isActive'] = "true"
- repo['config'] = json.dumps(config_dict)
- repo['description'] = "hbase repo"
- repo['name'] = params.config['clusterName'] + "_hbase"
- repo['repositoryType'] = "Hbase"
- repo['assetType'] = '2'
-
- data = json.dumps(repo)
-
- return data
+ config_dict = dict()
+ config_dict['username'] = params.config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+ config_dict['password'] = params.config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']
+ config_dict['hadoop.security.authentication'] = params.config['configurations']['core-site']['hadoop.security.authentication']
+ config_dict['hbase.security.authentication'] = params.config['configurations']['hbase-site']['hbase.security.authentication']
+ config_dict['hbase.zookeeper.property.clientPort'] = params.config['configurations']['hbase-site']['hbase.zookeeper.property.clientPort']
+ config_dict['hbase.zookeeper.quorum'] = params.config['configurations']['hbase-site']['hbase.zookeeper.quorum']
+ config_dict['zookeeper.znode.parent'] = params.config['configurations']['hbase-site']['zookeeper.znode.parent']
+
+ if params.config['configurations']['cluster-env']['security_enabled']:
+ config_dict['hbase.master.kerberos.principal'] = params.config['configurations']['hbase-site']['hbase.master.kerberos.principal']
+ else:
+ config_dict['hbase.master.kerberos.principal'] = ''
+
+ repo= dict()
+ repo['isActive'] = "true"
+ repo['config'] = json.dumps(config_dict)
+ repo['description'] = "hbase repo"
+ repo['name'] = str(params.config['clusterName']) + "_hbase"
+ repo['repositoryType'] = "Hbase"
+ repo['assetType'] = '2'
+
+ data = json.dumps(repo)
+
+ return data
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cbf3a87/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py
index 12a010d..b185058 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py
@@ -306,13 +306,25 @@ mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefi
# ranger host
ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+user_input = default("/configurations/ranger-hdfs-plugin-properties/ranger-hdfs-plugin-enabled", "no")
has_ranger_admin = not len(ranger_admin_hosts) == 0
if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
- # setting flag value for ranger hdfs plugin
+ # setting flag value for ranger hdfs plugin
+ enable_ranger_hdfs = False
+ user_input = config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled']
+ if user_input.lower() == 'yes':
+ enable_ranger_hdfs = True
+ elif user_input.lower() == 'no':
enable_ranger_hdfs = False
- user_input = config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled']
- if user_input.lower() == 'yes':
- enable_ranger_hdfs = True
- elif user_input.lower() == 'no':
- enable_ranger_hdfs = False
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+jdk_location = config['hostLevelParams']['jdk_location']
+java_share_dir = '/usr/share/java'
+jdbc_jar_name = "mysql-connector-java.jar"
+
+downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}")
+
+driver_curl_source = format("{jdk_location}/{jdbc_jar_name}")
+driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cbf3a87/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
index d93ba65..6bf04f2 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
@@ -22,170 +22,228 @@ import fileinput
import subprocess
import json
import re
+import os
from resource_management import *
from resource_management.libraries.functions.ranger_functions import Rangeradmin
from resource_management.core.logger import Logger
-def setup_ranger_hdfs(env):
- import params
- env.set_params(params)
-
- if params.has_ranger_admin:
- try:
- command = 'hdp-select status hadoop-client'
- return_code, hdp_output = shell.call(command, timeout=20)
- except Exception, e:
- Logger.error(str(e))
- raise Fail('Unable to execute hdp-select command to retrieve the version.')
-
- if return_code != 0:
- raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
-
- hdp_version = re.sub('hadoop-client - ', '', hdp_output)
- match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
-
- if match is None:
- raise Fail('Failed to get extracted version')
-
- file_path = '/usr/hdp/'+ hdp_version +'/ranger-hdfs-plugin/install.properties'
- ranger_hdfs_dict = ranger_hdfs_properties(params)
- hdfs_repo_data = hdfs_repo_properties(params)
-
- write_properties_to_file(file_path, ranger_hdfs_dict)
-
- if params.enable_ranger_hdfs:
- cmd = format('cd /usr/hdp/{hdp_version}/ranger-hdfs-plugin/ && sh enable-hdfs-plugin.sh')
- ranger_adm_obj = Rangeradmin(url=ranger_hdfs_dict['POLICY_MGR_URL'])
- response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(ranger_hdfs_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')
-
- if response_code is not None and response_code == 200:
- repo = ranger_adm_obj.get_repository_by_name_urllib2(ranger_hdfs_dict['REPOSITORY_NAME'], 'hdfs', 'true', 'admin:admin')
-
- if repo and repo['name'] == ranger_hdfs_dict['REPOSITORY_NAME']:
- Logger.info('HDFS Repository exist')
- else:
- response = ranger_adm_obj.create_repository_urllib2(hdfs_repo_data, 'admin:admin')
- if response is not None:
- Logger.info('HDFS Repository created in Ranger Admin')
- else:
- Logger.info('HDFS Repository creation failed in Ranger Admin')
+def setup_ranger_hdfs(env):
+ import params
+
+ env.set_params(params)
+
+ if params.has_ranger_admin:
+
+ environment = {"no_proxy": format("{params.ambari_server_hostname}")}
+
+ Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
+ params.downloaded_custom_connector),
+ not_if=format("test -f {params.downloaded_custom_connector}"),
+ path=["/bin", "/usr/bin/"],
+ environment=environment,
+ sudo=True)
+
+ if not os.path.isfile(params.driver_curl_target):
+ Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
+ path=["/bin", "/usr/bin/"],
+ sudo=True)
+
+ try:
+ command = 'hdp-select status hadoop-client'
+ return_code, hdp_output = shell.call(command, timeout=20)
+ except Exception, e:
+ Logger.error(str(e))
+ raise Fail('Unable to execute hdp-select command to retrieve the version.')
+
+ if return_code != 0:
+ raise Fail(
+ 'Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
+
+ hdp_version = re.sub('hadoop-client - ', '', hdp_output)
+ match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
+
+ if match is None:
+ raise Fail('Failed to get extracted version')
+
+ file_path = '/usr/hdp/' + hdp_version + '/ranger-hdfs-plugin/install.properties'
+
+ ranger_hdfs_dict = ranger_hdfs_properties(params)
+ hdfs_repo_data = hdfs_repo_properties(params)
+
+ write_properties_to_file(file_path, ranger_hdfs_dict)
+
+ if params.enable_ranger_hdfs:
+ cmd = format('cd /usr/hdp/{hdp_version}/ranger-hdfs-plugin/ && sh enable-hdfs-plugin.sh')
+ ranger_adm_obj = Rangeradmin(url=ranger_hdfs_dict['POLICY_MGR_URL'])
+ response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(
+ ranger_hdfs_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')
+
+ if response_code is not None and response_code == 200:
+ ambari_ranger_admin = params.config['configurations']['ranger-env']['ranger_admin_username']
+ ambari_ranger_password = params.config['configurations']['ranger-env']['ranger_admin_password']
+ ambari_ranger_admin,ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, 'admin:admin')
+ ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
+ if ambari_ranger_admin != '' and ambari_ranger_password != '':
+ repo = ranger_adm_obj.get_repository_by_name_urllib2(ranger_hdfs_dict['REPOSITORY_NAME'], 'hdfs', 'true', ambari_username_password_for_ranger)
+ if repo and repo['name'] == ranger_hdfs_dict['REPOSITORY_NAME']:
+ Logger.info('HDFS Repository exist')
+ else:
+ response = ranger_adm_obj.create_repository_urllib2(hdfs_repo_data, ambari_username_password_for_ranger)
+ if response is not None:
+ Logger.info('HDFS Repository created in Ranger Admin')
else:
- Logger.info('Ranger service is not started on given host')
+ Logger.info('HDFS Repository creation failed in Ranger Admin')
else:
- cmd = format('cd /usr/hdp/{hdp_version}/ranger-hdfs-plugin/ && sh disable-hdfs-plugin.sh')
-
- Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True)
+ Logger.info('Ambari admin username and password are blank ')
+ else:
+ Logger.info('Ranger service is not started on given host')
else:
- Logger.info('Ranger admin not installed')
+ cmd = format('cd /usr/hdp/{hdp_version}/ranger-hdfs-plugin/ && sh disable-hdfs-plugin.sh')
+
+ Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True)
+ else:
+ Logger.info('Ranger admin not installed')
def write_properties_to_file(file_path, value):
- for key in value:
- modify_config(file_path, key, value[key])
+ for key in value:
+ modify_config(file_path, key, value[key])
def modify_config(filepath, variable, setting):
- var_found = False
- already_set = False
- V=str(variable)
- S=str(setting)
- # use quotes if setting has spaces #
- if ' ' in S:
- S = '%s' % S
-
- for line in fileinput.input(filepath, inplace = 1):
- # process lines that look like config settings #
- if not line.lstrip(' ').startswith('#') and '=' in line:
- _infile_var = str(line.split('=')[0].rstrip(' '))
- _infile_set = str(line.split('=')[1].lstrip(' ').rstrip())
- # only change the first matching occurrence #
- if var_found == False and _infile_var.rstrip(' ') == V:
- var_found = True
- # don't change it if it is already set #
- if _infile_set.lstrip(' ') == S:
- already_set = True
- else:
- line = "%s=%s\n" % (V, S)
-
- sys.stdout.write(line)
-
- # Append the variable if it wasn't found #
- if not var_found:
- with open(filepath, "a") as f:
- f.write("%s=%s\n" % (V, S))
- elif already_set == True:
- pass
- else:
- pass
-
- return
+ var_found = False
+ already_set = False
+ V = str(variable)
+ S = str(setting)
+ # use quotes if setting has spaces #
+ if ' ' in S:
+ S = '%s' % S
+
+ for line in fileinput.input(filepath, inplace=1):
+ # process lines that look like config settings #
+ if not line.lstrip(' ').startswith('#') and '=' in line:
+ _infile_var = str(line.split('=')[0].rstrip(' '))
+ _infile_set = str(line.split('=')[1].lstrip(' ').rstrip())
+ # only change the first matching occurrence #
+ if var_found == False and _infile_var.rstrip(' ') == V:
+ var_found = True
+ # don't change it if it is already set #
+ if _infile_set.lstrip(' ') == S:
+ already_set = True
+ else:
+ line = "%s=%s\n" % (V, S)
-def ranger_hdfs_properties(params):
- ranger_hdfs_properties = dict()
-
- ranger_hdfs_properties['POLICY_MGR_URL'] = params.config['configurations']['admin-properties']['policymgr_external_url']
- ranger_hdfs_properties['SQL_CONNECTOR_JAR'] = params.config['configurations']['admin-properties']['SQL_CONNECTOR_JAR']
- ranger_hdfs_properties['XAAUDIT.DB.FLAVOUR'] = params.config['configurations']['admin-properties']['DB_FLAVOR']
- ranger_hdfs_properties['XAAUDIT.DB.DATABASE_NAME'] = params.config['configurations']['admin-properties']['audit_db_name']
- ranger_hdfs_properties['XAAUDIT.DB.USER_NAME'] = params.config['configurations']['admin-properties']['audit_db_user']
- ranger_hdfs_properties['XAAUDIT.DB.PASSWORD'] = params.config['configurations']['admin-properties']['audit_db_password']
- ranger_hdfs_properties['XAAUDIT.DB.HOSTNAME'] = params.config['configurations']['admin-properties']['db_host']
- ranger_hdfs_properties['REPOSITORY_NAME'] = params.config['clusterName'] + '_hadoop'
-
- ranger_hdfs_properties['XAAUDIT.DB.IS_ENABLED'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.DB.IS_ENABLED']
-
- ranger_hdfs_properties['XAAUDIT.HDFS.IS_ENABLED'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.IS_ENABLED']
- ranger_hdfs_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.DESTINATION_DIRECTORY']
- ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY']
- ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY']
- ranger_hdfs_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FILE']
- ranger_hdfs_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS']
- ranger_hdfs_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS']
- ranger_hdfs_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS']
- ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FILE']
- ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS']
- ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS']
- ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT']
-
-
- ranger_hdfs_properties['SSL_KEYSTORE_FILE_PATH'] = params.config['configurations']['ranger-hdfs-plugin-properties']['SSL_KEYSTORE_FILE_PATH']
- ranger_hdfs_properties['SSL_KEYSTORE_PASSWORD'] = params.config['configurations']['ranger-hdfs-plugin-properties']['SSL_KEYSTORE_PASSWORD']
- ranger_hdfs_properties['SSL_TRUSTSTORE_FILE_PATH'] = params.config['configurations']['ranger-hdfs-plugin-properties']['SSL_TRUSTSTORE_FILE_PATH']
- ranger_hdfs_properties['SSL_TRUSTSTORE_PASSWORD'] = params.config['configurations']['ranger-hdfs-plugin-properties']['SSL_TRUSTSTORE_PASSWORD']
-
- return ranger_hdfs_properties
+ sys.stdout.write(line)
+ # Append the variable if it wasn't found #
+ if not var_found:
+ with open(filepath, "a") as f:
+ f.write("%s=%s\n" % (V, S))
+ elif already_set == True:
+ pass
+ else:
+ pass
-def hdfs_repo_properties(params):
+ return
- config_dict = dict()
- config_dict['username'] = params.config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
- config_dict['password'] = params.config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']
- config_dict['hadoop.security.authentication'] = params.config['configurations']['core-site']['hadoop.security.authentication']
- config_dict['hadoop.security.authorization'] = params.config['configurations']['core-site']['hadoop.security.authorization']
- config_dict['fs.default.name'] = params.config['configurations']['core-site']['fs.defaultFS']
- config_dict['hadoop.security.auth_to_local'] = params.config['configurations']['core-site']['hadoop.security.auth_to_local']
- config_dict['hadoop.rpc.protection'] = params.config['configurations']['ranger-hdfs-plugin-properties']['hadoop.rpc.protection']
- config_dict['commonNameForCertificate'] = params.config['configurations']['ranger-hdfs-plugin-properties']['common.name.for.certificate']
-
- if params.config['configurations']['cluster-env']['security_enabled']:
- config_dict['dfs.datanode.kerberos.principal'] = params.config['configurations']['hdfs-site']['dfs.datanode.kerberos.principal']
- config_dict['dfs.namenode.kerberos.principal'] = params.config['configurations']['hdfs-site']['dfs.namenode.kerberos.principal']
- config_dict['dfs.secondary.namenode.kerberos.principal'] = params.config['configurations']['hdfs-site']['dfs.secondary.namenode.kerberos.principal']
- else:
- config_dict['dfs.datanode.kerberos.principal'] = ''
- config_dict['dfs.namenode.kerberos.principal'] = ''
- config_dict['dfs.secondary.namenode.kerberos.principal'] = ''
- repo= dict()
- repo['isActive'] = "true"
- repo['config'] = json.dumps(config_dict)
- repo['description'] = "hdfs repo"
- repo['name'] = params.config['clusterName'] + "_hadoop"
- repo['repositoryType'] = "Hdfs"
- repo['assetType'] = '1'
+def ranger_hdfs_properties(params):
+ ranger_hdfs_properties = dict()
+
+ ranger_hdfs_properties['POLICY_MGR_URL'] = params.config['configurations']['admin-properties'][
+ 'policymgr_external_url']
+ ranger_hdfs_properties['SQL_CONNECTOR_JAR'] = params.config['configurations']['admin-properties']['SQL_CONNECTOR_JAR']
+ ranger_hdfs_properties['XAAUDIT.DB.FLAVOUR'] = params.config['configurations']['admin-properties']['DB_FLAVOR']
+ ranger_hdfs_properties['XAAUDIT.DB.DATABASE_NAME'] = params.config['configurations']['admin-properties'][
+ 'audit_db_name']
+ ranger_hdfs_properties['XAAUDIT.DB.USER_NAME'] = params.config['configurations']['admin-properties']['audit_db_user']
+ ranger_hdfs_properties['XAAUDIT.DB.PASSWORD'] = params.config['configurations']['admin-properties'][
+ 'audit_db_password']
+ ranger_hdfs_properties['XAAUDIT.DB.HOSTNAME'] = params.config['configurations']['admin-properties']['db_host']
+ ranger_hdfs_properties['REPOSITORY_NAME'] = str(params.config['clusterName']) + '_hadoop'
+
+ ranger_hdfs_properties['XAAUDIT.DB.IS_ENABLED'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'XAAUDIT.DB.IS_ENABLED']
+
+ ranger_hdfs_properties['XAAUDIT.HDFS.IS_ENABLED'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'XAAUDIT.HDFS.IS_ENABLED']
+ ranger_hdfs_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.DESTINATION_DIRECTORY']
+ ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY']
+ ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY']
+ ranger_hdfs_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FILE']
+ ranger_hdfs_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS']
+ ranger_hdfs_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS']
+ ranger_hdfs_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS']
+ ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FILE']
+ ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS']
+ ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS']
+ ranger_hdfs_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = \
+ params.config['configurations']['ranger-hdfs-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT']
+
+ ranger_hdfs_properties['SSL_KEYSTORE_FILE_PATH'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'SSL_KEYSTORE_FILE_PATH']
+ ranger_hdfs_properties['SSL_KEYSTORE_PASSWORD'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'SSL_KEYSTORE_PASSWORD']
+ ranger_hdfs_properties['SSL_TRUSTSTORE_FILE_PATH'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'SSL_TRUSTSTORE_FILE_PATH']
+ ranger_hdfs_properties['SSL_TRUSTSTORE_PASSWORD'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'SSL_TRUSTSTORE_PASSWORD']
+
+ return ranger_hdfs_properties
- data = json.dumps(repo)
- return data
+def hdfs_repo_properties(params):
+ config_dict = dict()
+ config_dict['username'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'REPOSITORY_CONFIG_USERNAME']
+ config_dict['password'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'REPOSITORY_CONFIG_PASSWORD']
+ config_dict['hadoop.security.authentication'] = params.config['configurations']['core-site'][
+ 'hadoop.security.authentication']
+ config_dict['hadoop.security.authorization'] = params.config['configurations']['core-site'][
+ 'hadoop.security.authorization']
+ config_dict['fs.default.name'] = params.config['configurations']['core-site']['fs.defaultFS']
+ config_dict['hadoop.security.auth_to_local'] = params.config['configurations']['core-site'][
+ 'hadoop.security.auth_to_local']
+ config_dict['hadoop.rpc.protection'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'hadoop.rpc.protection']
+ config_dict['commonNameForCertificate'] = params.config['configurations']['ranger-hdfs-plugin-properties'][
+ 'common.name.for.certificate']
+
+ if params.config['configurations']['cluster-env']['security_enabled']:
+ config_dict['dfs.datanode.kerberos.principal'] = params.config['configurations']['hdfs-site'][
+ 'dfs.datanode.kerberos.principal']
+ config_dict['dfs.namenode.kerberos.principal'] = params.config['configurations']['hdfs-site'][
+ 'dfs.namenode.kerberos.principal']
+ config_dict['dfs.secondary.namenode.kerberos.principal'] = params.config['configurations']['hdfs-site'][
+ 'dfs.secondary.namenode.kerberos.principal']
+ else:
+ config_dict['dfs.datanode.kerberos.principal'] = ''
+ config_dict['dfs.namenode.kerberos.principal'] = ''
+ config_dict['dfs.secondary.namenode.kerberos.principal'] = ''
+
+ repo = dict()
+ repo['isActive'] = "true"
+ repo['config'] = json.dumps(config_dict)
+ repo['description'] = "hdfs repo"
+ repo['name'] = str(params.config['clusterName']) + "_hadoop"
+ repo['repositoryType'] = "Hdfs"
+ repo['assetType'] = '1'
+
+ data = json.dumps(repo)
+
+ return data
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cbf3a87/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
index bebb2fd..134bf1f 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
@@ -329,12 +329,20 @@ HdfsDirectory = functools.partial(
# ranger host
ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+user_input = default("/configurations/ranger-hive-plugin-properties/ranger-hive-plugin-enabled", "no")
has_ranger_admin = not len(ranger_admin_hosts) == 0
if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >=0:
- # setting flag value for ranger hive plugin
+ # setting flag value for ranger hive plugin
+ enable_ranger_hive = False
+ user_input = config['configurations']['ranger-hive-plugin-properties']['ranger-hive-plugin-enabled']
+ if user_input.lower() == 'yes':
+ enable_ranger_hive = True
+ elif user_input.lower() == 'no':
enable_ranger_hive = False
- user_input = config['configurations']['ranger-hive-plugin-properties']['ranger-hive-plugin-enabled']
- if user_input.lower() == 'yes':
- enable_ranger_hive = True
- elif user_input.lower() == 'no':
- enable_ranger_hive = False
+
+ranger_jdbc_jar_name = "mysql-connector-java.jar"
+
+ranger_downloaded_custom_connector = format("{tmp_dir}/{ranger_jdbc_jar_name}")
+
+ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_jar_name}")
+ranger_driver_curl_target = format("{java_share_dir}/{ranger_jdbc_jar_name}")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cbf3a87/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
index cc6a1d4..a766103 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
@@ -22,161 +22,184 @@ import fileinput
import subprocess
import json
import re
+import os
from resource_management import *
from resource_management.libraries.functions.ranger_functions import Rangeradmin
from resource_management.core.logger import Logger
def setup_ranger_hive(env):
- import params
- env.set_params(params)
-
- if params.has_ranger_admin:
- try:
- command = 'hdp-select status hive-server2'
- return_code, hdp_output = shell.call(command, timeout=20)
- except Exception, e:
- Logger.error(str(e))
- raise Fail('Unable to execute hdp-select command to retrieve the version.')
-
- if return_code != 0:
- raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
-
- hdp_version = re.sub('hive-server2 - ', '', hdp_output)
- match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
-
- if match is None:
- raise Fail('Failed to get extracted version')
-
- file_path = '/usr/hdp/'+ hdp_version +'/ranger-hive-plugin/install.properties'
-
- ranger_hive_dict = ranger_hive_properties(params)
- hive_repo_data = hive_repo_properties(params)
-
- write_properties_to_file(file_path, ranger_hive_dict)
-
- if params.enable_ranger_hive:
- cmd = format('cd /usr/hdp/{hdp_version}/ranger-hive-plugin/ && sh enable-hive-plugin.sh')
- ranger_adm_obj = Rangeradmin(url=ranger_hive_dict['POLICY_MGR_URL'])
- response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(ranger_hive_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')
-
- if response_code is not None and response_code == 200:
- repo = ranger_adm_obj.get_repository_by_name_urllib2(ranger_hive_dict['REPOSITORY_NAME'], 'hive', 'true', 'admin:admin')
-
- if repo and repo['name'] == ranger_hive_dict['REPOSITORY_NAME']:
- Logger.info('Hive Repository exist')
- else:
- response = ranger_adm_obj.create_repository_urllib2(hive_repo_data, 'admin:admin')
- if response is not None:
- Logger.info('Hive Repository created in Ranger Admin')
- else:
- Logger.info('Hive Repository creation failed in Ranger Admin')
- else:
- Logger.info('Ranger service is not started on given host')
+ import params
+ env.set_params(params)
+
+ if params.has_ranger_admin:
+
+ environment = {"no_proxy": format("{params.ambari_server_hostname}")}
+
+ Execute(('curl', '-kf', '-x', "", '--retry', '10', params.ranger_driver_curl_source, '-o',
+ params.ranger_downloaded_custom_connector),
+ not_if=format("test -f {params.ranger_downloaded_custom_connector}"),
+ path=["/bin", "/usr/bin/"],
+ environment=environment,
+ sudo=True)
+
+ if not os.path.isfile(params.ranger_driver_curl_target):
+ Execute(('cp', '--remove-destination', params.ranger_downloaded_custom_connector, params.ranger_driver_curl_target),
+ path=["/bin", "/usr/bin/"],
+ sudo=True)
+
+ try:
+ command = 'hdp-select status hive-server2'
+ return_code, hdp_output = shell.call(command, timeout=20)
+ except Exception, e:
+ Logger.error(str(e))
+ raise Fail('Unable to execute hdp-select command to retrieve the version.')
+
+ if return_code != 0:
+ raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
+
+ hdp_version = re.sub('hive-server2 - ', '', hdp_output)
+ match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
+
+ if match is None:
+ raise Fail('Failed to get extracted version')
+
+ file_path = '/usr/hdp/'+ hdp_version +'/ranger-hive-plugin/install.properties'
+
+ ranger_hive_dict = ranger_hive_properties(params)
+ hive_repo_data = hive_repo_properties(params)
+
+ write_properties_to_file(file_path, ranger_hive_dict)
+
+ if params.enable_ranger_hive:
+ cmd = format('cd /usr/hdp/{hdp_version}/ranger-hive-plugin/ && sh enable-hive-plugin.sh')
+ ranger_adm_obj = Rangeradmin(url=ranger_hive_dict['POLICY_MGR_URL'])
+ response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(ranger_hive_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')
+
+ if response_code is not None and response_code == 200:
+ ambari_ranger_admin = params.config['configurations']['ranger-env']['ranger_admin_username']
+ ambari_ranger_password = params.config['configurations']['ranger-env']['ranger_admin_password']
+ ambari_ranger_admin,ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, 'admin:admin')
+ ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
+ if ambari_ranger_admin != '' and ambari_ranger_password != '':
+ repo = ranger_adm_obj.get_repository_by_name_urllib2(ranger_hive_dict['REPOSITORY_NAME'], 'hive', 'true', ambari_username_password_for_ranger)
+
+ if repo and repo['name'] == ranger_hive_dict['REPOSITORY_NAME']:
+ Logger.info('Hive Repository exist')
+ else:
+ response = ranger_adm_obj.create_repository_urllib2(hive_repo_data,ambari_username_password_for_ranger)
+ if response is not None:
+ Logger.info('Hive Repository created in Ranger Admin')
+ else:
+ Logger.info('Hive Repository creation failed in Ranger Admin')
else:
- cmd = format('cd /usr/hdp/{hdp_version}/ranger-hive-plugin/ && sh disable-hive-plugin.sh')
-
- Execute(cmd, environment={'JAVA_HOME': params.java64_home}, logoutput=True)
+ Logger.info('Ambari admin username and password are blank ')
+ else:
+ Logger.info('Ranger service is not started on given host')
else:
- Logger.info('Ranger admin not installed')
+ cmd = format('cd /usr/hdp/{hdp_version}/ranger-hive-plugin/ && sh disable-hive-plugin.sh')
+
+ Execute(cmd, environment={'JAVA_HOME': params.java64_home}, logoutput=True)
+ else:
+ Logger.info('Ranger admin not installed')
def write_properties_to_file(file_path, value):
- for key in value:
- modify_config(file_path, key, value[key])
+ for key in value:
+ modify_config(file_path, key, value[key])
def modify_config(filepath, variable, setting):
- var_found = False
- already_set = False
- V=str(variable)
- S=str(setting)
- # use quotes if setting has spaces #
- if ' ' in S:
- S = '%s' % S
-
- for line in fileinput.input(filepath, inplace = 1):
- # process lines that look like config settings #
- if not line.lstrip(' ').startswith('#') and '=' in line:
- _infile_var = str(line.split('=')[0].rstrip(' '))
- _infile_set = str(line.split('=')[1].lstrip(' ').rstrip())
- # only change the first matching occurrence #
- if var_found == False and _infile_var.rstrip(' ') == V:
- var_found = True
- # don't change it if it is already set #
- if _infile_set.lstrip(' ') == S:
- already_set = True
- else:
- line = "%s=%s\n" % (V, S)
-
- sys.stdout.write(line)
-
- # Append the variable if it wasn't found #
- if not var_found:
- with open(filepath, "a") as f:
- f.write("%s=%s\n" % (V, S))
- elif already_set == True:
- pass
- else:
- pass
+ var_found = False
+ already_set = False
+ V=str(variable)
+ S=str(setting)
+ # use quotes if setting has spaces #
+ if ' ' in S:
+ S = '%s' % S
+
+ for line in fileinput.input(filepath, inplace = 1):
+ # process lines that look like config settings #
+ if not line.lstrip(' ').startswith('#') and '=' in line:
+ _infile_var = str(line.split('=')[0].rstrip(' '))
+ _infile_set = str(line.split('=')[1].lstrip(' ').rstrip())
+ # only change the first matching occurrence #
+ if var_found == False and _infile_var.rstrip(' ') == V:
+ var_found = True
+ # don't change it if it is already set #
+ if _infile_set.lstrip(' ') == S:
+ already_set = True
+ else:
+ line = "%s=%s\n" % (V, S)
- return
+ sys.stdout.write(line)
-def ranger_hive_properties(params):
- ranger_hive_properties = dict()
-
- ranger_hive_properties['POLICY_MGR_URL'] = params.config['configurations']['admin-properties']['policymgr_external_url']
- ranger_hive_properties['SQL_CONNECTOR_JAR'] = params.config['configurations']['admin-properties']['SQL_CONNECTOR_JAR']
- ranger_hive_properties['XAAUDIT.DB.FLAVOUR'] = params.config['configurations']['admin-properties']['DB_FLAVOR']
- ranger_hive_properties['XAAUDIT.DB.DATABASE_NAME'] = params.config['configurations']['admin-properties']['audit_db_name']
- ranger_hive_properties['XAAUDIT.DB.USER_NAME'] = params.config['configurations']['admin-properties']['audit_db_user']
- ranger_hive_properties['XAAUDIT.DB.PASSWORD'] = params.config['configurations']['admin-properties']['audit_db_password']
- ranger_hive_properties['XAAUDIT.DB.HOSTNAME'] = params.config['configurations']['admin-properties']['db_host']
- ranger_hive_properties['REPOSITORY_NAME'] = params.config['clusterName'] + '_hive'
-
- ranger_hive_properties['XAAUDIT.DB.IS_ENABLED'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.DB.IS_ENABLED']
-
- ranger_hive_properties['XAAUDIT.HDFS.IS_ENABLED'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.IS_ENABLED']
- ranger_hive_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINATION_DIRECTORY']
- ranger_hive_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY']
- ranger_hive_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY']
- ranger_hive_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FILE']
- ranger_hive_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS']
- ranger_hive_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS']
- ranger_hive_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS']
- ranger_hive_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FILE']
- ranger_hive_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS']
- ranger_hive_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS']
- ranger_hive_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT']
-
-
- ranger_hive_properties['SSL_KEYSTORE_FILE_PATH'] = params.config['configurations']['ranger-hive-plugin-properties']['SSL_KEYSTORE_FILE_PATH']
- ranger_hive_properties['SSL_KEYSTORE_PASSWORD'] = params.config['configurations']['ranger-hive-plugin-properties']['SSL_KEYSTORE_PASSWORD']
- ranger_hive_properties['SSL_TRUSTSTORE_FILE_PATH'] = params.config['configurations']['ranger-hive-plugin-properties']['SSL_TRUSTSTORE_FILE_PATH']
- ranger_hive_properties['SSL_TRUSTSTORE_PASSWORD'] = params.config['configurations']['ranger-hive-plugin-properties']['SSL_TRUSTSTORE_PASSWORD']
+ # Append the variable if it wasn't found #
+ if not var_found:
+ with open(filepath, "a") as f:
+ f.write("%s=%s\n" % (V, S))
+ elif already_set == True:
+ pass
+ else:
+ pass
- ranger_hive_properties['UPDATE_XAPOLICIES_ON_GRANT_REVOKE'] = params.config['configurations']['ranger-hive-plugin-properties']['UPDATE_XAPOLICIES_ON_GRANT_REVOKE']
+ return
- return ranger_hive_properties
+def ranger_hive_properties(params):
+ ranger_hive_properties = dict()
+
+ ranger_hive_properties['POLICY_MGR_URL'] = params.config['configurations']['admin-properties']['policymgr_external_url']
+ ranger_hive_properties['SQL_CONNECTOR_JAR'] = params.config['configurations']['admin-properties']['SQL_CONNECTOR_JAR']
+ ranger_hive_properties['XAAUDIT.DB.FLAVOUR'] = params.config['configurations']['admin-properties']['DB_FLAVOR']
+ ranger_hive_properties['XAAUDIT.DB.DATABASE_NAME'] = params.config['configurations']['admin-properties']['audit_db_name']
+ ranger_hive_properties['XAAUDIT.DB.USER_NAME'] = params.config['configurations']['admin-properties']['audit_db_user']
+ ranger_hive_properties['XAAUDIT.DB.PASSWORD'] = params.config['configurations']['admin-properties']['audit_db_password']
+ ranger_hive_properties['XAAUDIT.DB.HOSTNAME'] = params.config['configurations']['admin-properties']['db_host']
+ ranger_hive_properties['REPOSITORY_NAME'] = str(params.config['clusterName']) + '_hive'
+
+ ranger_hive_properties['XAAUDIT.DB.IS_ENABLED'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.DB.IS_ENABLED']
+
+ ranger_hive_properties['XAAUDIT.HDFS.IS_ENABLED'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.IS_ENABLED']
+ ranger_hive_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINATION_DIRECTORY']
+ ranger_hive_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY']
+ ranger_hive_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY']
+ ranger_hive_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FILE']
+ ranger_hive_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS']
+ ranger_hive_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS']
+ ranger_hive_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS']
+ ranger_hive_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FILE']
+ ranger_hive_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS']
+ ranger_hive_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS']
+ ranger_hive_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = params.config['configurations']['ranger-hive-plugin-properties']['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT']
+
+
+ ranger_hive_properties['SSL_KEYSTORE_FILE_PATH'] = params.config['configurations']['ranger-hive-plugin-properties']['SSL_KEYSTORE_FILE_PATH']
+ ranger_hive_properties['SSL_KEYSTORE_PASSWORD'] = params.config['configurations']['ranger-hive-plugin-properties']['SSL_KEYSTORE_PASSWORD']
+ ranger_hive_properties['SSL_TRUSTSTORE_FILE_PATH'] = params.config['configurations']['ranger-hive-plugin-properties']['SSL_TRUSTSTORE_FILE_PATH']
+ ranger_hive_properties['SSL_TRUSTSTORE_PASSWORD'] = params.config['configurations']['ranger-hive-plugin-properties']['SSL_TRUSTSTORE_PASSWORD']
+
+ ranger_hive_properties['UPDATE_XAPOLICIES_ON_GRANT_REVOKE'] = params.config['configurations']['ranger-hive-plugin-properties']['UPDATE_XAPOLICIES_ON_GRANT_REVOKE']
+
+ return ranger_hive_properties
def hive_repo_properties(params):
- hive_host = params.config['clusterHostInfo']['hive_server_host'][0]
+ hive_host = params.config['clusterHostInfo']['hive_server_host'][0]
- config_dict = dict()
- config_dict['username'] = params.config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
- config_dict['password'] = params.config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']
- config_dict['jdbc.driverClassName'] = params.config['configurations']['ranger-hive-plugin-properties']['jdbc.driverClassName']
- config_dict['jdbc.url'] = 'jdbc:hive2://' + hive_host + ':10000'
- config_dict['commonNameForCertificate'] = params.config['configurations']['ranger-hive-plugin-properties']['common.name.for.certificate']
+ config_dict = dict()
+ config_dict['username'] = params.config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+ config_dict['password'] = params.config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']
+ config_dict['jdbc.driverClassName'] = params.config['configurations']['ranger-hive-plugin-properties']['jdbc.driverClassName']
+ config_dict['jdbc.url'] = 'jdbc:hive2://' + hive_host + ':10000'
+ config_dict['commonNameForCertificate'] = params.config['configurations']['ranger-hive-plugin-properties']['common.name.for.certificate']
- repo= dict()
- repo['isActive'] = "true"
- repo['config'] = json.dumps(config_dict)
- repo['description'] = "hive repo"
- repo['name'] = params.config['clusterName'] + '_hive'
- repo['repositoryType'] = "Hive"
- repo['assetType'] = '3'
+ repo= dict()
+ repo['isActive'] = "true"
+ repo['config'] = json.dumps(config_dict)
+ repo['description'] = "hive repo"
+ repo['name'] = str(params.config['clusterName']) + '_hive'
+ repo['repositoryType'] = "Hive"
+ repo['assetType'] = '3'
- data = json.dumps(repo)
+ data = json.dumps(repo)
- return data
+ return data