You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/05/03 16:46:58 UTC
[15/53] [abbrv] ambari git commit: AMBARI-20779 Create Ranger KMS
HDFS audit folder as part of install (mugdha)
AMBARI-20779 Create Ranger KMS HDFS audit folder as part of install (mugdha)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9dd542be
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9dd542be
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9dd542be
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 9dd542be2c7bb3b12ea50c8902f3d8e1413eebf3
Parents: 4857a6e
Author: Mugdha Varadkar <mu...@apache.org>
Authored: Fri Apr 28 13:35:27 2017 +0530
Committer: Gautam Borad <ga...@apache.org>
Committed: Sun Apr 30 23:23:08 2017 +0530
----------------------------------------------------------------------
.../RANGER/0.5.0/role_command_order.json | 2 +-
.../RANGER_KMS/0.5.0.2.3/metainfo.xml | 9 ++
.../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 20 ++++
.../0.5.0.2.3/package/scripts/params.py | 34 ++++++-
.../0.5.0.2.3/role_command_order.json | 2 +-
.../stacks/HDP/2.3/role_command_order.json | 4 +-
.../stacks/2.5/RANGER_KMS/test_kms_server.py | 98 ++++++++++++++++++++
.../stacks/2.5/configs/ranger-kms-secured.json | 2 +-
8 files changed, 165 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json
index df62dfd..073a077 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json
@@ -1,6 +1,6 @@
{
"general_deps" : {
"_comment" : "dependencies for RANGER",
- "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"]
+ "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START"]
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml
index b20201c..bc0aa74 100644
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml
@@ -45,6 +45,15 @@
<primary>true</primary>
</log>
</logs>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
index 423cdec..5a25b92 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
@@ -506,6 +506,26 @@ def enable_kms_plugin():
mode = 0640
)
+ # create ranger kms audit directory
+ if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
+ params.HdfsResource("/ranger/audit",
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ group=params.hdfs_user,
+ mode=0755,
+ recursive_chmod=True
+ )
+ params.HdfsResource("/ranger/audit/kms",
+ type="directory",
+ action="create_on_execute",
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0750,
+ recursive_chmod=True
+ )
+ params.HdfsResource(None, action="execute")
+
if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
Logger.info('Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml')
XmlConfig("hdfs-site.xml",
http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
index 9fe0a61..2445f2e 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
@@ -29,6 +29,9 @@ from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.get_bare_principal import get_bare_principal
from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import get_kinit_path
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
@@ -296,4 +299,33 @@ xa_audit_hdfs_is_enabled = default("/configurations/ranger-kms-audit/xasecure.au
namenode_host = default("/clusterHostInfo/namenode_host", [])
# need this to capture cluster name from where ranger kms plugin is enabled
-cluster_name = config['clusterName']
\ No newline at end of file
+cluster_name = config['clusterName']
+
+has_namenode = len(namenode_host) > 0
+
+hdfs_user = default("/configurations/hadoop-env/hdfs_user", None)
+hdfs_user_keytab = default("/configurations/hadoop-env/hdfs_user_keytab", None)
+hdfs_principal_name = default("/configurations/hadoop-env/hdfs_principal_name", None)
+default_fs = default("/configurations/core-site/fs.defaultFS", None)
+hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+import functools
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs
+)
+
+local_component_list = default("/localComponents", [])
+has_hdfs_client_on_node = 'HDFS_CLIENT' in local_component_list
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json
index 006d177..7ddab41 100644
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json
@@ -1,7 +1,7 @@
{
"general_deps" : {
"_comment" : "dependencies for RANGER-KMS",
- "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"],
+ "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"],
"RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"]
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
index 4f279c6..5b1b50e 100755
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
@@ -4,8 +4,8 @@
"general_deps" : {
"_comment" : "dependencies for all cases",
"MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
- "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"],
- "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"],
+ "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START"],
+ "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"],
"RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"],
"PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"],
"SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"],
http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py b/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
index 7082a33..3d03c5f 100644
--- a/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
+++ b/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
@@ -151,6 +151,55 @@ class TestRangerKMS(RMFTestCase):
mode = 0640
)
+ self.assertResourceCalled('HdfsResource', '/ranger/audit',
+ type = 'directory',
+ action = ['create_on_execute'],
+ owner = 'hdfs',
+ group = 'hdfs',
+ mode = 0755,
+ recursive_chmod = True,
+ user = 'hdfs',
+ security_enabled = False,
+ keytab = None,
+ kinit_path_local = '/usr/bin/kinit',
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ principal_name = None,
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+ )
+
+ self.assertResourceCalled('HdfsResource', '/ranger/audit/kms',
+ type = 'directory',
+ action = ['create_on_execute'],
+ owner = 'kms',
+ group = 'kms',
+ mode = 0750,
+ recursive_chmod = True,
+ user = 'hdfs',
+ security_enabled = False,
+ keytab = None,
+ kinit_path_local = '/usr/bin/kinit',
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ principal_name = None,
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+ )
+
+ self.assertResourceCalled('HdfsResource', None,
+ action = ['execute'],
+ user = 'hdfs',
+ security_enabled = False,
+ keytab = None,
+ kinit_path_local = '/usr/bin/kinit',
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ principal_name = None,
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+ )
+
self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/hdfs-site.xml',
action = ['delete'],
)
@@ -537,6 +586,55 @@ class TestRangerKMS(RMFTestCase):
mode = 0640
)
+ self.assertResourceCalled('HdfsResource', '/ranger/audit',
+ type = 'directory',
+ action = ['create_on_execute'],
+ owner = 'hdfs',
+ group = 'hdfs',
+ mode = 0755,
+ recursive_chmod = True,
+ user = 'hdfs',
+ security_enabled = True,
+ keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+ kinit_path_local = '/usr/bin/kinit',
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ principal_name = 'hdfs-cl1@EXAMPLE.COM',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+ )
+
+ self.assertResourceCalled('HdfsResource', '/ranger/audit/kms',
+ type = 'directory',
+ action = ['create_on_execute'],
+ owner = 'kms',
+ group = 'kms',
+ mode = 0750,
+ recursive_chmod = True,
+ user = 'hdfs',
+ security_enabled = True,
+ keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+ kinit_path_local = '/usr/bin/kinit',
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ principal_name = 'hdfs-cl1@EXAMPLE.COM',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+ )
+
+ self.assertResourceCalled('HdfsResource', None,
+ action = ['execute'],
+ user = 'hdfs',
+ security_enabled = True,
+ keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+ kinit_path_local = '/usr/bin/kinit',
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ principal_name = 'hdfs-cl1@EXAMPLE.COM',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+ )
+
self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/hdfs-site.xml',
action = ['delete'],
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json b/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json
index 7054e8f..4e7d857 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json
@@ -751,7 +751,7 @@
"hdfs_log_dir_prefix": "/var/log/hadoop",
"hdfs_user_nofile_limit": "128000",
"hdfs_user": "hdfs",
- "hdfs_principal_name": "hdfs-test_cluster01@EXAMPLE.COM",
+ "hdfs_principal_name": "hdfs-cl1@EXAMPLE.COM",
"keyserver_host": " ",
"namenode_opt_maxnewsize": "128m",
"hdfs_user_keytab": "/etc/security/keytabs/hdfs.headless.keytab",