You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ga...@apache.org on 2016/06/29 14:39:48 UTC
[6/6] ambari git commit: AMBARI-17333. ranger kms repo creation is
failing after ranger kms is installed(Mugdha Varadkar via gautam)
AMBARI-17333. ranger kms repo creation is failing after ranger kms is installed(Mugdha Varadkar via gautam)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2360560f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2360560f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2360560f
Branch: refs/heads/trunk
Commit: 2360560fb39d36888db6372dab678b1d501ebd18
Parents: c40ec58
Author: Gautam Borad <ga...@apache.org>
Authored: Tue Jun 28 11:32:20 2016 +0530
Committer: Gautam Borad <ga...@apache.org>
Committed: Wed Jun 29 20:09:11 2016 +0530
----------------------------------------------------------------------
.../libraries/functions/ranger_functions_v2.py | 8 +-
.../common-services/RANGER/0.6.0/kerberos.json | 2 +-
.../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 8 +-
.../0.5.0.2.3/package/scripts/params.py | 35 +-
.../stacks/2.5/RANGER_KMS/test_kms_server.py | 712 +++++++++++++++
.../stacks/2.5/configs/ranger-kms-default.json | 803 +++++++++++++++++
.../stacks/2.5/configs/ranger-kms-secured.json | 873 +++++++++++++++++++
7 files changed, 2419 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2360560f/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
index 05beadb..5c3a3bb 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
@@ -331,7 +331,7 @@ class RangeradminV2:
@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=None)
- def get_repository_by_name_curl(self, component_user,component_user_keytab,component_user_principal,name, component, status):
+ def get_repository_by_name_curl(self, component_user, component_user_keytab, component_user_principal, name, component, status, is_keyadmin = False):
"""
:param component_user: service user for which call is to be made
:param component_user_keytab: keytab of service user
@@ -344,6 +344,8 @@ class RangeradminV2:
"""
try:
search_repo_url = self.url_repos_pub + "?serviceName=" + name + "&serviceType=" + component + "&isEnabled=" + status
+ if is_keyadmin:
+ search_repo_url = '{0}&suser=keyadmin'.format(search_repo_url)
response,error_message,time_in_millis = self.call_curl_request(component_user,component_user_keytab,component_user_principal,search_repo_url,False,request_method='GET')
response_stripped = response[1:len(response) - 1]
if response_stripped and len(response_stripped) > 0:
@@ -360,7 +362,7 @@ class RangeradminV2:
@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=None)
- def create_repository_curl(self,component_user,component_user_keytab,component_user_principal,name, data,policy_user):
+ def create_repository_curl(self, component_user, component_user_keytab, component_user_principal, name, data, policy_user, is_keyadmin = False):
"""
:param component_user: service user for which call is to be made
:param component_user_keytab: keytab of service user
@@ -371,6 +373,8 @@ class RangeradminV2:
"""
try:
search_repo_url = self.url_repos_pub
+ if is_keyadmin:
+ search_repo_url = '{0}?suser=keyadmin'.format(search_repo_url)
header = 'Content-Type: application/json'
method = 'POST'
http://git-wip-us.apache.org/repos/asf/ambari/blob/2360560f/ambari-server/src/main/resources/common-services/RANGER/0.6.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/kerberos.json b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/kerberos.json
index c633230..91a0032 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/kerberos.json
@@ -119,7 +119,7 @@
"atlas.jaas.KafkaClient.option.keyTab": "{{tagsync_keytab_path}}",
"atlas.jaas.KafkaClient.option.principal": "{{tagsync_jaas_principal}}",
"atlas.kafka.sasl.kerberos.service.name": "kafka",
- "atlas.kafka.security.protocol": "SASL_PLAINTEXT"
+ "atlas.kafka.security.protocol": "PLAINTEXTSASL"
}
}
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/2360560f/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
index a4a38c1..133760b 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
@@ -380,13 +380,13 @@ def enable_kms_plugin():
if not ranger_flag:
Logger.error('Error in Get/Create service for Ranger Kms.')
- current_datetime = datetime.now()
+ current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
File(format('{kms_conf_dir}/ranger-security.xml'),
owner = params.kms_user,
group = params.kms_group,
mode = 0644,
- content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
+ content = format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')
)
Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')],
@@ -565,12 +565,12 @@ def check_ranger_service_support_kerberos():
response_code = ranger_adm_obj.check_ranger_login_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, policymgr_mgr_url, True)
if response_code is not None and response_code[0] == 200:
- get_repo_name_response = ranger_adm_obj.get_repository_by_name_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.repo_name, 'kms', 'true')
+ get_repo_name_response = ranger_adm_obj.get_repository_by_name_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.repo_name, 'kms', 'true', is_keyadmin = True)
if get_repo_name_response is not None:
Logger.info('KMS repository {0} exist'.format(get_repo_name_response['name']))
return True
else:
- create_repo_response = ranger_adm_obj.create_repository_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.repo_name, json.dumps(params.kms_ranger_plugin_repo), None)
+ create_repo_response = ranger_adm_obj.create_repository_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.repo_name, json.dumps(params.kms_ranger_plugin_repo), None, is_keyadmin = True)
if create_repo_response is not None and len(create_repo_response) > 0:
return True
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/2360560f/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
index 26e9c8b..dce6576 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
@@ -25,6 +25,7 @@ from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.get_bare_principal import get_bare_principal
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
@@ -200,17 +201,6 @@ kms_plugin_config = {
'provider' : format('kms://http@{kms_host}:{kms_port}/kms')
}
-if stack_supports_ranger_kerberos:
- kms_plugin_config['policy.download.auth.users'] = 'keyadmin'
-
-kms_ranger_plugin_repo = {
- 'isEnabled' : 'true',
- 'configs' : kms_plugin_config,
- 'description' : 'kms repo',
- 'name' : repo_name,
- 'type' : 'kms'
-}
-
xa_audit_db_is_enabled = False
if stack_supports_ranger_audit_db:
xa_audit_db_is_enabled = config['configurations']['ranger-kms-audit']['xasecure.audit.destination.db']
@@ -241,10 +231,25 @@ hms_partition_passwd = default("/configurations/kms-env/hsm_partition_password",
# kms kerberos from stack 2.5 onward
rangerkms_keytab = config['configurations']['dbks-site']['ranger.ks.kerberos.keytab']
-if stack_supports_ranger_kerberos and security_enabled:
- rangerkms_principal = default("/configurations/dbks-site/ranger.ks.kerberos.principal", None)
- if rangerkms_principal is not None:
- rangerkms_principal = rangerkms_principal.replace('_HOST', kms_host.lower())
+rangerkms_bare_principal = 'rangerkms'
+
+if stack_supports_ranger_kerberos:
+ if security_enabled:
+ rangerkms_principal = default("/configurations/dbks-site/ranger.ks.kerberos.principal", None)
+ if rangerkms_principal is not None:
+ rangerkms_bare_principal = get_bare_principal(rangerkms_principal)
+ rangerkms_principal = rangerkms_principal.replace('_HOST', kms_host.lower())
+ kms_plugin_config['policy.download.auth.users'] = format('keyadmin,{rangerkms_bare_principal}')
+ else:
+ kms_plugin_config['policy.download.auth.users'] = 'keyadmin'
+
+kms_ranger_plugin_repo = {
+ 'isEnabled' : 'true',
+ 'configs' : kms_plugin_config,
+ 'description' : 'kms repo',
+ 'name' : repo_name,
+ 'type' : 'kms'
+}
# ranger kms pid
user_group = config['configurations']['cluster-env']['user_group']
http://git-wip-us.apache.org/repos/asf/ambari/blob/2360560f/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py b/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
new file mode 100644
index 0000000..70e3d42
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
@@ -0,0 +1,712 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import json
+from datetime import datetime
+from mock.mock import MagicMock, patch
+from stacks.utils.RMFTestCase import *
+from only_for_platform import not_for_platform, PLATFORM_WINDOWS
+from resource_management.libraries.functions.ranger_functions import Rangeradmin
+from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2
+
+@not_for_platform(PLATFORM_WINDOWS)
+class TestRangerKMS(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "RANGER_KMS/0.5.0.2.3/package"
+ STACK_VERSION = "2.5"
+
+ @patch("os.path.isfile")
+ def test_configure_default(self, isfile_mock):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/kms_server.py",
+ classname = "KmsServer",
+ command = "configure",
+ config_file="ranger-kms-default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default()
+ self.assertTrue(isfile_mock.called)
+ self.assertNoMoreResources()
+
+ @patch("resource_management.libraries.functions.ranger_functions.Rangeradmin.check_ranger_login_urllib2", new=MagicMock(return_value=200))
+ @patch("resource_management.libraries.functions.ranger_functions.Rangeradmin.create_ambari_admin_user", new=MagicMock(return_value=200))
+ @patch("kms.get_repo")
+ @patch("kms.create_repo")
+ @patch("os.path.isfile")
+ def test_start_default(self, get_repo_mock, create_repo_mock, isfile_mock):
+
+ get_repo_mock.return_value = True
+ create_repo_mock.return_value = True
+
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/kms_server.py",
+ classname = "KmsServer",
+ command = "start",
+ config_file="ranger-kms-default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default()
+
+ # TODO confirm repo call
+
+ current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/ranger-security.xml',
+ owner = 'kms',
+ group = 'kms',
+ content = '<ranger>\n<enabled>{0}</enabled>\n</ranger>'.format(current_datetime),
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Directory', '/etc/ranger/c1_kms',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0775,
+ create_parents = True
+ )
+
+ self.assertResourceCalled('Directory', '/etc/ranger/c1_kms/policycache',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0775,
+ create_parents = True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/c1_kms/policycache/kms_c1_kms.json',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0644
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-kms-audit.xml',
+ mode = 0744,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['ranger-kms-audit'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-kms-audit']
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-kms-security.xml',
+ mode = 0744,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['ranger-kms-security'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-kms-security']
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-policymgr-ssl.xml',
+ mode = 0744,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['ranger-kms-policymgr-ssl'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-kms-policymgr-ssl']
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/hdp/current/ranger-kms/ranger_credential_helper.py', '-l', '/usr/hdp/current/ranger-kms/cred/lib/*', '-f', '/etc/ranger/c1_kms/cred.jceks', '-k', 'sslKeyStore', '-v', 'myKeyFilePassword', '-c', '1'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo=True
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/hdp/current/ranger-kms/ranger_credential_helper.py', '-l', '/usr/hdp/current/ranger-kms/cred/lib/*', '-f', '/etc/ranger/c1_kms/cred.jceks', '-k', 'sslTrustStore', '-v', 'changeit', '-c', '1'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/c1_kms/cred.jceks',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0640
+ )
+
+ self.assertResourceCalled('Directory', '/tmp/jce_dir',
+ create_parents = True,
+ )
+
+ self.assertResourceCalled('File', '/tmp/jce_dir/UnlimitedJCEPolicyJDK7.zip',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//UnlimitedJCEPolicyJDK7.zip'),
+ mode = 0644,
+ )
+
+ self.assertResourceCalled('File', '/usr/jdk64/jdk1.7.0_45/jre/lib/security/local_policy.jar',
+ action = ["delete"]
+ )
+
+ self.assertResourceCalled('File', '/usr/jdk64/jdk1.7.0_45/jre/lib/security/US_export_policy.jar',
+ action = ["delete"]
+ )
+
+ self.assertResourceCalled('Execute', ("unzip", "-o", "-j", "-q", "/tmp/jce_dir/UnlimitedJCEPolicyJDK7.zip", "-d", "/usr/jdk64/jdk1.7.0_45/jre/lib/security"),
+ only_if = 'test -e /usr/jdk64/jdk1.7.0_45/jre/lib/security && test -f /tmp/jce_dir/UnlimitedJCEPolicyJDK7.zip',
+ path=['/bin/', '/usr/bin'],
+ sudo=True
+ )
+
+ self.assertResourceCalled('Execute', '/usr/hdp/current/ranger-kms/ranger-kms start',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ not_if = 'ps -ef | grep proc_rangerkms | grep -v grep',
+ user = 'kms'
+ )
+
+ self.assertTrue(isfile_mock.called)
+ self.assertNoMoreResources()
+
+ def test_stop_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/kms_server.py",
+ classname = "KmsServer",
+ command = "stop",
+ config_file="ranger-kms-default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', '/usr/hdp/current/ranger-kms/ranger-kms stop',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ user = 'kms'
+ )
+ self.assertResourceCalled('File', '/var/run/ranger_kms/rangerkms.pid',
+ action = ['delete']
+ )
+ self.assertNoMoreResources()
+
+ def assert_configure_default(self):
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-kms/conf',
+ owner = 'kms',
+ group = 'kms',
+ create_parents = True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java-old.jar',
+ action = ['delete'],
+ )
+
+ self.assertResourceCalled('File', '/tmp/mysql-connector-java.jar',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar'),
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-kms/ews/lib',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/tmp/mysql-connector-java.jar',
+ '/usr/hdp/current/ranger-kms/ews/webapp/lib'),
+ path=['/bin', '/usr/bin/'],
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java.jar',
+ mode = 0644
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/current/ranger-kms/install.properties',
+ properties = self.getConfig()['configurations']['kms-properties'],
+ owner = 'kms'
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/current/ranger-kms/install.properties',
+ properties = {'SQL_CONNECTOR_JAR': '/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java.jar'},
+ owner = 'kms'
+ )
+
+ self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+ content=DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+ mode=0644,
+ )
+
+ self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6401.ambari.apache.org:3306/rangerkms01\' rangerkms01 rangerkms01 com.mysql.jdbc.Driver',
+ path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10, environment = {}
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-kms/ews/webapp/WEB-INF/classes/lib',
+ mode = 0755,
+ owner = 'kms',
+ group = 'kms'
+ )
+
+ self.assertResourceCalled('Execute', ('cp', '/usr/hdp/current/ranger-kms/ranger-kms-initd', '/etc/init.d/ranger-kms'),
+ not_if=format('ls /etc/init.d/ranger-kms'),
+ only_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms-initd'),
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/etc/init.d/ranger-kms',
+ mode=0755,
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-kms/',
+ owner = 'kms',
+ group = 'kms',
+ recursive_ownership = True,
+ )
+
+ self.assertResourceCalled('Directory', '/var/run/ranger_kms',
+ mode=0755,
+ owner = 'kms',
+ group = 'hadoop',
+ cd_access = "a",
+ create_parents=True
+ )
+
+ self.assertResourceCalled('Directory', '/var/log/ranger/kms',
+ owner = 'kms',
+ group = 'kms',
+ cd_access = 'a',
+ create_parents = True,
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/ranger-kms-env-logdir.sh',
+ content = format("export RANGER_KMS_LOG_DIR=/var/log/ranger/kms"),
+ owner = 'kms',
+ group = 'kms',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-kms/ranger-kms', '/usr/bin/ranger-kms'),
+ not_if=format('ls /usr/bin/ranger-kms'),
+ only_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms'),
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/usr/bin/ranger-kms',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-kms/ranger-kms', '/usr/bin/ranger-kms-services.sh'),
+ not_if=format('ls /usr/bin/ranger-kms-services.sh'),
+ only_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms'),
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/usr/bin/ranger-kms-services.sh',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-kms/ranger-kms-initd', '/usr/hdp/current/ranger-kms/ranger-kms-services.sh'),
+ not_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms-services.sh'),
+ only_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms-initd'),
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/ranger-kms-services.sh',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Directory', '/var/log/ranger/kms',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0775
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/jdk64/jdk1.7.0_45/bin/java', '-cp', '/usr/hdp/current/ranger-kms/cred/lib/*', 'org.apache.ranger.credentialapi.buildks', 'create', 'ranger.ks.jdbc.password', '-value', 'rangerkms01', '-provider', 'jceks://file/etc/ranger/kms/rangerkms.jceks'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/kms/rangerkms.jceks',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0640
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/jdk64/jdk1.7.0_45/bin/java', '-cp', '/usr/hdp/current/ranger-kms/cred/lib/*', 'org.apache.ranger.credentialapi.buildks', 'create', 'ranger.ks.masterkey.password', '-value', 'StrongPassword01', '-provider', 'jceks://file/etc/ranger/kms/rangerkms.jceks'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/kms/rangerkms.jceks',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0640
+ )
+
+ self.assertResourceCalled('XmlConfig', 'dbks-site.xml',
+ mode=0644,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['dbks-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['dbks-site']
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-kms-site.xml',
+ mode = 0644,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['ranger-kms-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-kms-site']
+ )
+
+ self.assertResourceCalled('XmlConfig', 'kms-site.xml',
+ mode = 0644,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['kms-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['kms-site']
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/kms-log4j.properties',
+ mode = 0644,
+ owner = 'kms',
+ group = 'kms',
+ content = self.getConfig()['configurations']['kms-log4j']['content']
+ )
+
+ @patch("os.path.isfile")
+ def test_configure_secured(self, isfile_mock):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/kms_server.py",
+ classname = "KmsServer",
+ command = "configure",
+ config_file="ranger-kms-secured.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_secured()
+ self.assertTrue(isfile_mock.called)
+ self.assertNoMoreResources()
+
+ @patch("resource_management.libraries.functions.ranger_functions_v2.RangeradminV2.check_ranger_login_curl", new=MagicMock(return_value=(200, '', '')))
+ @patch("resource_management.libraries.functions.ranger_functions_v2.RangeradminV2.get_repository_by_name_curl", new=MagicMock(return_value=({'name': 'c1_kms'})))
+ @patch("resource_management.libraries.functions.ranger_functions_v2.RangeradminV2.create_repository_curl", new=MagicMock(return_value=({'name': 'c1_kms'})))
+ @patch("os.path.isfile")
+ def test_start_secured(self, isfile_mock):
+
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/kms_server.py",
+ classname = "KmsServer",
+ command = "start",
+ config_file="ranger-kms-secured.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_secured()
+
+ # TODO repo call in secure
+
+ current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/ranger-security.xml',
+ owner = 'kms',
+ group = 'kms',
+ content = '<ranger>\n<enabled>{0}</enabled>\n</ranger>'.format(current_datetime),
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Directory', '/etc/ranger/c1_kms',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0775,
+ create_parents = True
+ )
+
+ self.assertResourceCalled('Directory', '/etc/ranger/c1_kms/policycache',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0775,
+ create_parents = True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/c1_kms/policycache/kms_c1_kms.json',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0644
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-kms-audit.xml',
+ mode = 0744,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['ranger-kms-audit'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-kms-audit']
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-kms-security.xml',
+ mode = 0744,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['ranger-kms-security'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-kms-security']
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-policymgr-ssl.xml',
+ mode = 0744,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['ranger-kms-policymgr-ssl'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-kms-policymgr-ssl']
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/hdp/current/ranger-kms/ranger_credential_helper.py', '-l', '/usr/hdp/current/ranger-kms/cred/lib/*', '-f', '/etc/ranger/c1_kms/cred.jceks', '-k', 'sslKeyStore', '-v', 'myKeyFilePassword', '-c', '1'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo=True
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/hdp/current/ranger-kms/ranger_credential_helper.py', '-l', '/usr/hdp/current/ranger-kms/cred/lib/*', '-f', '/etc/ranger/c1_kms/cred.jceks', '-k', 'sslTrustStore', '-v', 'changeit', '-c', '1'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/c1_kms/cred.jceks',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0640
+ )
+
+ self.assertResourceCalled('Directory', '/tmp/jce_dir',
+ create_parents = True,
+ )
+
+ self.assertResourceCalled('File', '/tmp/jce_dir/UnlimitedJCEPolicyJDK7.zip',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//UnlimitedJCEPolicyJDK7.zip'),
+ mode = 0644,
+ )
+
+ self.assertResourceCalled('File', '/usr/jdk64/jdk1.7.0_45/jre/lib/security/local_policy.jar',
+ action = ["delete"]
+ )
+
+ self.assertResourceCalled('File', '/usr/jdk64/jdk1.7.0_45/jre/lib/security/US_export_policy.jar',
+ action = ["delete"]
+ )
+
+ self.assertResourceCalled('Execute', ("unzip", "-o", "-j", "-q", "/tmp/jce_dir/UnlimitedJCEPolicyJDK7.zip", "-d", "/usr/jdk64/jdk1.7.0_45/jre/lib/security"),
+ only_if = 'test -e /usr/jdk64/jdk1.7.0_45/jre/lib/security && test -f /tmp/jce_dir/UnlimitedJCEPolicyJDK7.zip',
+ path=['/bin/', '/usr/bin'],
+ sudo=True
+ )
+
+ self.assertResourceCalled('Execute', '/usr/hdp/current/ranger-kms/ranger-kms start',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ not_if = 'ps -ef | grep proc_rangerkms | grep -v grep',
+ user = 'kms'
+ )
+
+ self.assertTrue(isfile_mock.called)
+ self.assertNoMoreResources()
+
+ def assert_configure_secured(self):
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-kms/conf',
+ owner = 'kms',
+ group = 'kms',
+ create_parents = True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java-old.jar',
+ action = ['delete'],
+ )
+
+ self.assertResourceCalled('File', '/tmp/mysql-connector-java.jar',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar'),
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-kms/ews/lib',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/tmp/mysql-connector-java.jar',
+ '/usr/hdp/current/ranger-kms/ews/webapp/lib'),
+ path=['/bin', '/usr/bin/'],
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java.jar',
+ mode = 0644
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/current/ranger-kms/install.properties',
+ properties = self.getConfig()['configurations']['kms-properties'],
+ owner = 'kms'
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/current/ranger-kms/install.properties',
+ properties = {'SQL_CONNECTOR_JAR': '/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java.jar'},
+ owner = 'kms'
+ )
+
+ self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+ content=DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+ mode=0644,
+ )
+
+ self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6401.ambari.apache.org:3306/rangerkms01\' rangerkms01 rangerkms01 com.mysql.jdbc.Driver',
+ path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10, environment = {}
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-kms/ews/webapp/WEB-INF/classes/lib',
+ mode = 0755,
+ owner = 'kms',
+ group = 'kms'
+ )
+
+ self.assertResourceCalled('Execute', ('cp', '/usr/hdp/current/ranger-kms/ranger-kms-initd', '/etc/init.d/ranger-kms'),
+ not_if=format('ls /etc/init.d/ranger-kms'),
+ only_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms-initd'),
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/etc/init.d/ranger-kms',
+ mode=0755,
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-kms/',
+ owner = 'kms',
+ group = 'kms',
+ recursive_ownership = True,
+ )
+
+ self.assertResourceCalled('Directory', '/var/run/ranger_kms',
+ mode=0755,
+ owner = 'kms',
+ group = 'hadoop',
+ cd_access = "a",
+ create_parents=True
+ )
+
+ self.assertResourceCalled('Directory', '/var/log/ranger/kms',
+ owner = 'kms',
+ group = 'kms',
+ cd_access = 'a',
+ create_parents = True,
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/ranger-kms-env-logdir.sh',
+ content = format("export RANGER_KMS_LOG_DIR=/var/log/ranger/kms"),
+ owner = 'kms',
+ group = 'kms',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-kms/ranger-kms', '/usr/bin/ranger-kms'),
+ not_if=format('ls /usr/bin/ranger-kms'),
+ only_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms'),
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/usr/bin/ranger-kms',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-kms/ranger-kms', '/usr/bin/ranger-kms-services.sh'),
+ not_if=format('ls /usr/bin/ranger-kms-services.sh'),
+ only_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms'),
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/usr/bin/ranger-kms-services.sh',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-kms/ranger-kms-initd', '/usr/hdp/current/ranger-kms/ranger-kms-services.sh'),
+ not_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms-services.sh'),
+ only_if=format('ls /usr/hdp/current/ranger-kms/ranger-kms-initd'),
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/ranger-kms-services.sh',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Directory', '/var/log/ranger/kms',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0775
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/jdk64/jdk1.7.0_45/bin/java', '-cp', '/usr/hdp/current/ranger-kms/cred/lib/*', 'org.apache.ranger.credentialapi.buildks', 'create', 'ranger.ks.jdbc.password', '-value', 'rangerkms01', '-provider', 'jceks://file/etc/ranger/kms/rangerkms.jceks'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/kms/rangerkms.jceks',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0640
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/jdk64/jdk1.7.0_45/bin/java', '-cp', '/usr/hdp/current/ranger-kms/cred/lib/*', 'org.apache.ranger.credentialapi.buildks', 'create', 'ranger.ks.masterkey.password', '-value', 'StrongPassword01', '-provider', 'jceks://file/etc/ranger/kms/rangerkms.jceks'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo=True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/kms/rangerkms.jceks',
+ owner = 'kms',
+ group = 'kms',
+ mode = 0640
+ )
+
+ self.assertResourceCalled('XmlConfig', 'dbks-site.xml',
+ mode=0644,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['dbks-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['dbks-site']
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-kms-site.xml',
+ mode = 0644,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['ranger-kms-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-kms-site']
+ )
+
+ self.assertResourceCalled('XmlConfig', 'kms-site.xml',
+ mode = 0644,
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['kms-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['kms-site']
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/kms-log4j.properties',
+ mode = 0644,
+ owner = 'kms',
+ group = 'kms',
+ content = self.getConfig()['configurations']['kms-log4j']['content']
+ )
+
+ self.assertResourceCalled('XmlConfig', 'core-site.xml',
+ owner = 'kms',
+ group = 'kms',
+ conf_dir = '/usr/hdp/current/ranger-kms/conf',
+ configurations = self.getConfig()['configurations']['core-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+ mode = 0644
+ )
+