You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by lp...@apache.org on 2017/07/06 16:17:25 UTC
[03/26] ambari git commit: AMBARI-21058 HDP 3.0 - Changing common
service version for Ranger & Ranger Kms (mugdha)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/metainfo.xml
new file mode 100644
index 0000000..d328d89
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/metainfo.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>RANGER_KMS</name>
+ <displayName>Ranger KMS</displayName>
+ <comment>Key Management Server</comment>
+ <version>1.0.0.3.0</version>
+ <components>
+
+ <component>
+ <name>RANGER_KMS_SERVER</name>
+ <displayName>Ranger KMS Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <commandScript>
+ <script>scripts/kms_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <logs>
+ <log>
+ <logId>ranger_kms</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ </component>
+ </components>
+
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>ranger_${stack_version}-kms</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+ <packages>
+ <package>
+ <name>ranger-${stack_version}-kms</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <configuration-dependencies>
+ <config-type>kms-properties</config-type>
+ <config-type>kms-site</config-type>
+ <config-type>kms-log4j</config-type>
+ <config-type>dbks-site</config-type>
+ <config-type>ranger-kms-site</config-type>
+ <config-type>ranger-kms-audit</config-type>
+ <config-type>ranger-kms-policymgr-ssl</config-type>
+ <config-type>ranger-kms-security</config-type>
+ </configuration-dependencies>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>RANGER</service>
+ <service>HDFS</service>
+ </requiredServices>
+
+ <themes>
+ <theme>
+ <fileName>theme_version_1.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms.py
new file mode 100755
index 0000000..6e4a171
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms.py
@@ -0,0 +1,675 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+import fileinput
+import os
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+import urllib2, base64, httplib
+from StringIO import StringIO as BytesIO
+from datetime import datetime
+from resource_management.core.resources.system import File, Directory, Execute
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.resources.modify_properties_file import ModifyPropertiesFile
+from resource_management.core.source import DownloadSource, InlineTemplate
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.ranger_functions import Rangeradmin
+from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2
+from resource_management.libraries.functions.decorator import safe_retry
+from resource_management.core.utils import PasswordString
+from resource_management.core.shell import as_sudo
+import re
+import time
+import socket
+
+def password_validation(password, key):
+ import params
+ if password.strip() == "":
+ raise Fail("Blank password is not allowed for {0} property. Please enter valid password.".format(key))
+ if re.search("[\\\`'\"]",password):
+ raise Fail("{0} password contains one of the unsupported special characters like \" ' \ `".format(key))
+ else:
+ Logger.info("Password validated")
+
+def setup_kms_db(stack_version=None):
+ import params
+
+ if params.has_ranger_admin:
+
+ kms_home = params.kms_home
+ version = params.version
+ if stack_version is not None:
+ kms_home = format("{stack_root}/{stack_version}/ranger-kms")
+ version = stack_version
+
+ password_validation(params.kms_master_key_password, 'KMS master key')
+
+ copy_jdbc_connector(stack_version=version)
+
+ env_dict = {'RANGER_KMS_HOME':kms_home, 'JAVA_HOME': params.java_home}
+ if params.db_flavor.lower() == 'sqla':
+ env_dict = {'RANGER_KMS_HOME':kms_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH':params.ld_library_path}
+
+ dba_setup = format('ambari-python-wrap {kms_home}/dba_script.py -q')
+ db_setup = format('ambari-python-wrap {kms_home}/db_setup.py')
+
+ if params.create_db_user:
+ Logger.info('Setting up Ranger KMS DB and DB User')
+ Execute(dba_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10)
+ else:
+ Logger.info('Separate DBA property not set. Assuming Ranger KMS DB and DB User exists!')
+ Execute(db_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10)
+
+def setup_java_patch():
+ import params
+
+ if params.has_ranger_admin:
+
+ kms_home = params.kms_home
+ setup_java_patch = format('ambari-python-wrap {kms_home}/db_setup.py -javapatch')
+
+ env_dict = {'RANGER_KMS_HOME':kms_home, 'JAVA_HOME': params.java_home}
+ if params.db_flavor.lower() == 'sqla':
+ env_dict = {'RANGER_KMS_HOME':kms_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH':params.ld_library_path}
+
+ Execute(setup_java_patch, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10)
+
+ kms_lib_path = format('{kms_home}/ews/webapp/lib/')
+ files = os.listdir(kms_lib_path)
+ hadoop_jar_files = []
+
+ for x in files:
+ if x.startswith('hadoop-common') and x.endswith('.jar'):
+ hadoop_jar_files.append(x)
+
+ if len(hadoop_jar_files) != 0:
+ for f in hadoop_jar_files:
+ Execute((format('{java_home}/bin/jar'),'-uf', format('{kms_home}/ews/webapp/lib/{f}'), format('{kms_home}/ews/webapp/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory')),
+ user=params.kms_user)
+
+ File(format('{kms_home}/ews/webapp/lib/{f}'), owner=params.kms_user, group=params.kms_group)
+
+def do_keystore_setup(cred_provider_path, credential_alias, credential_password):
+ import params
+
+ if cred_provider_path is not None:
+ java_bin = format('{java_home}/bin/java')
+ file_path = format('jceks://file{cred_provider_path}')
+ cmd = (java_bin, '-cp', params.cred_lib_path, 'org.apache.ranger.credentialapi.buildks', 'create', credential_alias, '-value', PasswordString(credential_password), '-provider', file_path)
+ Execute(cmd,
+ environment={'JAVA_HOME': params.java_home},
+ logoutput=True,
+ sudo=True,
+ )
+
+ File(cred_provider_path,
+ owner = params.kms_user,
+ group = params.kms_group,
+ mode = 0640
+ )
+
+def kms(upgrade_type=None):
+ import params
+
+ if params.has_ranger_admin:
+
+ Directory(params.kms_conf_dir,
+ owner = params.kms_user,
+ group = params.kms_group,
+ create_parents = True
+ )
+
+ Directory("/etc/security/serverKeys",
+ create_parents = True,
+ cd_access = "a"
+ )
+
+ Directory("/etc/ranger/kms",
+ create_parents = True,
+ cd_access = "a"
+ )
+
+ copy_jdbc_connector()
+
+ File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
+ content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
+ mode = 0644,
+ )
+
+ cp = format("{check_db_connection_jar}")
+ if params.db_flavor.lower() == 'sqla':
+ cp = cp + os.pathsep + format("{kms_home}/ews/webapp/lib/sajdbc4.jar")
+ else:
+ path_to_jdbc = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
+ if not os.path.isfile(path_to_jdbc):
+ path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + \
+ params.default_connectors_map[params.db_flavor.lower()] if params.db_flavor.lower() in params.default_connectors_map else None
+ if not os.path.isfile(path_to_jdbc):
+ path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + "*"
+ error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.db_flavor] + \
+ " in ranger kms lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
+ Logger.error(error_message)
+
+ cp = cp + os.pathsep + path_to_jdbc
+
+ db_connection_check_command = format(
+ "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}")
+
+ env_dict = {}
+ if params.db_flavor.lower() == 'sqla':
+ env_dict = {'LD_LIBRARY_PATH':params.ld_library_path}
+
+ Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10, environment=env_dict)
+
+ if params.xa_audit_db_is_enabled and params.driver_source is not None and not params.driver_source.endswith("/None"):
+ if params.xa_previous_jdbc_jar and os.path.isfile(params.xa_previous_jdbc_jar):
+ File(params.xa_previous_jdbc_jar, action='delete')
+
+ File(params.downloaded_connector_path,
+ content = DownloadSource(params.driver_source),
+ mode = 0644
+ )
+
+ Execute(('cp', '--remove-destination', params.downloaded_connector_path, params.driver_target),
+ path=["/bin", "/usr/bin/"],
+ sudo=True)
+
+ File(params.driver_target, mode=0644)
+
+ Directory(os.path.join(params.kms_home, 'ews', 'webapp', 'WEB-INF', 'classes', 'lib'),
+ mode=0755,
+ owner=params.kms_user,
+ group=params.kms_group
+ )
+
+ Execute(('cp',format('{kms_home}/ranger-kms-initd'),'/etc/init.d/ranger-kms'),
+ not_if=format('ls /etc/init.d/ranger-kms'),
+ only_if=format('ls {kms_home}/ranger-kms-initd'),
+ sudo=True)
+
+ File('/etc/init.d/ranger-kms',
+ mode = 0755
+ )
+
+ Directory(format('{kms_home}/'),
+ owner = params.kms_user,
+ group = params.kms_group,
+ recursive_ownership = True,
+ )
+
+ Directory(params.ranger_kms_pid_dir,
+ mode=0755,
+ owner = params.kms_user,
+ group = params.user_group,
+ cd_access = "a",
+ create_parents=True
+ )
+
+ if params.stack_supports_pid:
+ File(format('{kms_conf_dir}/ranger-kms-env-piddir.sh'),
+ content = format("export RANGER_KMS_PID_DIR_PATH={ranger_kms_pid_dir}\nexport KMS_USER={kms_user}"),
+ owner = params.kms_user,
+ group = params.kms_group,
+ mode=0755
+ )
+
+ Directory(params.kms_log_dir,
+ owner = params.kms_user,
+ group = params.kms_group,
+ cd_access = 'a',
+ create_parents=True,
+ mode=0755
+ )
+
+ File(format('{kms_conf_dir}/ranger-kms-env-logdir.sh'),
+ content = format("export RANGER_KMS_LOG_DIR={kms_log_dir}"),
+ owner = params.kms_user,
+ group = params.kms_group,
+ mode=0755
+ )
+
+ Execute(('ln','-sf', format('{kms_home}/ranger-kms'),'/usr/bin/ranger-kms'),
+ not_if=format('ls /usr/bin/ranger-kms'),
+ only_if=format('ls {kms_home}/ranger-kms'),
+ sudo=True)
+
+ File('/usr/bin/ranger-kms', mode = 0755)
+
+ Execute(('ln','-sf', format('{kms_home}/ranger-kms'),'/usr/bin/ranger-kms-services.sh'),
+ not_if=format('ls /usr/bin/ranger-kms-services.sh'),
+ only_if=format('ls {kms_home}/ranger-kms'),
+ sudo=True)
+
+ File('/usr/bin/ranger-kms-services.sh', mode = 0755)
+
+ Execute(('ln','-sf', format('{kms_home}/ranger-kms-initd'),format('{kms_home}/ranger-kms-services.sh')),
+ not_if=format('ls {kms_home}/ranger-kms-services.sh'),
+ only_if=format('ls {kms_home}/ranger-kms-initd'),
+ sudo=True)
+
+ File(format('{kms_home}/ranger-kms-services.sh'), mode = 0755)
+
+ Directory(params.kms_log_dir,
+ owner = params.kms_user,
+ group = params.kms_group,
+ mode = 0775
+ )
+
+ do_keystore_setup(params.credential_provider_path, params.jdbc_alias, params.db_password)
+ do_keystore_setup(params.credential_provider_path, params.masterkey_alias, params.kms_master_key_password)
+ if params.stack_support_kms_hsm and params.enable_kms_hsm:
+ do_keystore_setup(params.credential_provider_path, params.hms_partition_alias, unicode(params.hms_partition_passwd))
+ if params.stack_supports_ranger_kms_ssl and params.ranger_kms_ssl_enabled:
+ do_keystore_setup(params.ranger_kms_cred_ssl_path, params.ranger_kms_ssl_keystore_alias, params.ranger_kms_ssl_passwd)
+
+ # remove plain-text password from xml configs
+ dbks_site_copy = {}
+ dbks_site_copy.update(params.config['configurations']['dbks-site'])
+
+ for prop in params.dbks_site_password_properties:
+ if prop in dbks_site_copy:
+ dbks_site_copy[prop] = "_"
+
+ XmlConfig("dbks-site.xml",
+ conf_dir=params.kms_conf_dir,
+ configurations=dbks_site_copy,
+ configuration_attributes=params.config['configuration_attributes']['dbks-site'],
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0644
+ )
+
+ ranger_kms_site_copy = {}
+ ranger_kms_site_copy.update(params.config['configurations']['ranger-kms-site'])
+ if params.stack_supports_ranger_kms_ssl:
+ # remove plain-text password from xml configs
+ for prop in params.ranger_kms_site_password_properties:
+ if prop in ranger_kms_site_copy:
+ ranger_kms_site_copy[prop] = "_"
+
+ XmlConfig("ranger-kms-site.xml",
+ conf_dir=params.kms_conf_dir,
+ configurations=ranger_kms_site_copy,
+ configuration_attributes=params.config['configuration_attributes']['ranger-kms-site'],
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0644
+ )
+
+ XmlConfig("kms-site.xml",
+ conf_dir=params.kms_conf_dir,
+ configurations=params.config['configurations']['kms-site'],
+ configuration_attributes=params.config['configuration_attributes']['kms-site'],
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0644
+ )
+
+ File(os.path.join(params.kms_conf_dir, "kms-log4j.properties"),
+ owner=params.kms_user,
+ group=params.kms_group,
+ content=InlineTemplate(params.kms_log4j),
+ mode=0644
+ )
+ if params.security_enabled:
+ # core-site.xml linking required by setup for HDFS encryption
+ XmlConfig("core-site.xml",
+ conf_dir=params.kms_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0644
+ )
+ else:
+ File(format('{kms_conf_dir}/core-site.xml'), action="delete")
+
+def copy_jdbc_connector(stack_version=None):
+ import params
+
+ if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
+ error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \
+ Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(params.db_flavor, params.jdk_location)
+ Logger.error(error_message)
+
+ if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
+ if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
+ File(params.previous_jdbc_jar, action='delete')
+
+ kms_home = params.kms_home
+ if stack_version is not None:
+ kms_home = format("{stack_root}/{stack_version}/ranger-kms")
+
+ driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
+
+ File(params.downloaded_custom_connector,
+ content = DownloadSource(params.driver_curl_source),
+ mode = 0644
+ )
+
+ Directory(os.path.join(kms_home, 'ews', 'lib'),
+ mode=0755
+ )
+
+ if params.db_flavor.lower() == 'sqla':
+ Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
+
+ Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
+ path=["/bin", "/usr/bin/"],
+ sudo=True)
+
+ Directory(params.jdbc_libs_dir,
+ cd_access="a",
+ create_parents=True)
+
+ Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
+ path=["/bin", "/usr/bin/"])
+
+ File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'), mode=0644)
+ else:
+ Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
+ path=["/bin", "/usr/bin/"],
+ sudo=True)
+
+ File(os.path.join(kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644)
+
+ ModifyPropertiesFile(format("{kms_home}/install.properties"),
+ properties = params.config['configurations']['kms-properties'],
+ owner = params.kms_user
+ )
+
+ if params.db_flavor.lower() == 'sqla':
+ ModifyPropertiesFile(format("{kms_home}/install.properties"),
+ properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/sajdbc4.jar')},
+ owner = params.kms_user,
+ )
+ else:
+ ModifyPropertiesFile(format("{kms_home}/install.properties"),
+ properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
+ owner = params.kms_user,
+ )
+
+def enable_kms_plugin():
+
+ import params
+
+ if params.has_ranger_admin:
+
+ ranger_flag = False
+
+ if params.stack_supports_ranger_kerberos and params.security_enabled:
+ if not is_empty(params.rangerkms_principal) and params.rangerkms_principal != '':
+ ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal)
+ else:
+ ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.spengo_keytab, params.spnego_principal)
+ else:
+ ranger_flag = check_ranger_service()
+
+ if not ranger_flag:
+ Logger.error('Error in Get/Create service for Ranger Kms.')
+
+ current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+ File(format('{kms_conf_dir}/ranger-security.xml'),
+ owner = params.kms_user,
+ group = params.kms_group,
+ mode = 0644,
+ content = format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')
+ )
+
+ Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')],
+ owner = params.kms_user,
+ group = params.kms_group,
+ mode=0775,
+ create_parents = True
+ )
+
+ File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')),
+ owner = params.kms_user,
+ group = params.kms_group,
+ mode = 0644
+ )
+
+ # remove plain-text password from xml configs
+ plugin_audit_properties_copy = {}
+ plugin_audit_properties_copy.update(params.config['configurations']['ranger-kms-audit'])
+
+ if params.plugin_audit_password_property in plugin_audit_properties_copy:
+ plugin_audit_properties_copy[params.plugin_audit_password_property] = "crypted"
+
+ XmlConfig("ranger-kms-audit.xml",
+ conf_dir=params.kms_conf_dir,
+ configurations=plugin_audit_properties_copy,
+ configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'],
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0744)
+
+ XmlConfig("ranger-kms-security.xml",
+ conf_dir=params.kms_conf_dir,
+ configurations=params.config['configurations']['ranger-kms-security'],
+ configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'],
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0744)
+
+ # remove plain-text password from xml configs
+ ranger_kms_policymgr_ssl_copy = {}
+ ranger_kms_policymgr_ssl_copy.update(params.config['configurations']['ranger-kms-policymgr-ssl'])
+
+ for prop in params.kms_plugin_password_properties:
+ if prop in ranger_kms_policymgr_ssl_copy:
+ ranger_kms_policymgr_ssl_copy[prop] = "crypted"
+
+ XmlConfig("ranger-policymgr-ssl.xml",
+ conf_dir=params.kms_conf_dir,
+ configurations=ranger_kms_policymgr_ssl_copy,
+ configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'],
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0744)
+
+ if params.xa_audit_db_is_enabled:
+ cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1')
+ Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
+
+ cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1')
+ Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
+
+ cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1')
+ Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
+
+ File(params.credential_file,
+ owner = params.kms_user,
+ group = params.kms_group,
+ mode = 0640
+ )
+
+ # create ranger kms audit directory
+ if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
+ params.HdfsResource("/ranger/audit",
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ group=params.hdfs_user,
+ mode=0755,
+ recursive_chmod=True
+ )
+ params.HdfsResource("/ranger/audit/kms",
+ type="directory",
+ action="create_on_execute",
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0750,
+ recursive_chmod=True
+ )
+ params.HdfsResource(None, action="execute")
+
+ if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
+ Logger.info('Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml')
+ XmlConfig("hdfs-site.xml",
+ conf_dir=params.kms_conf_dir,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.kms_user,
+ group=params.kms_group,
+ mode=0644
+ )
+ else:
+ File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
+
+def setup_kms_jce():
+ import params
+
+ if params.jce_name is not None:
+ Directory(params.jce_source_dir,
+ create_parents = True
+ )
+
+ jce_target = format('{jce_source_dir}/{jce_name}')
+
+ File(jce_target,
+ content = DownloadSource(format('{jdk_location}/{jce_name}')),
+ mode = 0644,
+ )
+
+ File([format("{java_home}/jre/lib/security/local_policy.jar"), format("{java_home}/jre/lib/security/US_export_policy.jar")],
+ action = "delete",
+ )
+
+ unzip_cmd = ("unzip", "-o", "-j", "-q", jce_target, "-d", format("{java_home}/jre/lib/security"))
+
+ Execute(unzip_cmd,
+ only_if = format("test -e {java_home}/jre/lib/security && test -f {jce_target}"),
+ path = ['/bin/','/usr/bin'],
+ sudo = True
+ )
+ else:
+ Logger.warning("Required jce policy zip is not available, need to setup manually")
+
+def check_ranger_service():
+ import params
+
+ policymgr_mgr_url = params.policymgr_mgr_url
+ if policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+ ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url)
+ ambari_username_password_for_ranger = format("{ambari_ranger_admin}:{ambari_ranger_password}")
+ response_code = ranger_adm_obj.check_ranger_login_urllib2(policymgr_mgr_url)
+
+ if response_code is not None and response_code == 200:
+ user_resp_code = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
+ if user_resp_code is not None and user_resp_code == 200:
+ get_repo_flag = get_repo(policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger)
+ if not get_repo_flag:
+ return create_repo(policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger)
+ else:
+ return True
+ else:
+ return False
+ else:
+ Logger.error('Ranger service is not reachable')
+ return False
+
+@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=False)
+def create_repo(url, data, usernamepassword):
+ try:
+ base_url = url + '/service/public/v2/api/service'
+ base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
+ headers = {
+ 'Accept': 'application/json',
+ "Content-Type": "application/json"
+ }
+ request = urllib2.Request(base_url, data, headers)
+ request.add_header("Authorization", "Basic {0}".format(base64string))
+ result = urllib2.urlopen(request, timeout=20)
+ response_code = result.getcode()
+ response = json.loads(json.JSONEncoder().encode(result.read()))
+ if response_code == 200:
+ Logger.info('Repository created Successfully')
+ return True
+ else:
+ Logger.info('Repository not created')
+ return False
+ except urllib2.URLError, e:
+ if isinstance(e, urllib2.HTTPError):
+ raise Fail("Error creating service. Http status code - {0}. \n {1}".format(e.code, e.read()))
+ else:
+ raise Fail("Error creating service. Reason - {0}.".format(e.reason))
+ except socket.timeout as e:
+ raise Fail("Error creating service. Reason - {0}".format(e))
+
+@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=False)
+def get_repo(url, name, usernamepassword):
+ try:
+ base_url = url + '/service/public/v2/api/service?serviceName=' + name + '&serviceType=kms&isEnabled=true'
+ request = urllib2.Request(base_url)
+ base64string = base64.encodestring(usernamepassword).replace('\n', '')
+ request.add_header("Content-Type", "application/json")
+ request.add_header("Accept", "application/json")
+ request.add_header("Authorization", "Basic {0}".format(base64string))
+ result = urllib2.urlopen(request, timeout=20)
+ response_code = result.getcode()
+ response = json.loads(result.read())
+ if response_code == 200 and len(response) > 0:
+ for repo in response:
+ if repo.get('name').lower() == name.lower() and repo.has_key('name'):
+ Logger.info('KMS repository exist')
+ return True
+ else:
+ Logger.info('KMS repository doesnot exist')
+ return False
+ else:
+ Logger.info('KMS repository doesnot exist')
+ return False
+ except urllib2.URLError, e:
+ if isinstance(e, urllib2.HTTPError):
+ raise Fail("Error getting {0} service. Http status code - {1}. \n {2}".format(name, e.code, e.read()))
+ else:
+ raise Fail("Error getting {0} service. Reason - {1}.".format(name, e.reason))
+ except socket.timeout as e:
+ raise Fail("Error creating service. Reason - {0}".format(e))
+
+def check_ranger_service_support_kerberos(user, keytab, principal):
+ import params
+
+ policymgr_mgr_url = params.policymgr_mgr_url
+ if policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+ ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url)
+ response_code = ranger_adm_obj.check_ranger_login_curl(user, keytab, principal, policymgr_mgr_url, True)
+
+ if response_code is not None and response_code[0] == 200:
+ get_repo_name_response = ranger_adm_obj.get_repository_by_name_curl(user, keytab, principal, params.repo_name, 'kms', 'true', is_keyadmin = True)
+ if get_repo_name_response is not None:
+ Logger.info('KMS repository {0} exist'.format(get_repo_name_response['name']))
+ return True
+ else:
+ create_repo_response = ranger_adm_obj.create_repository_curl(user, keytab, principal, params.repo_name, json.dumps(params.kms_ranger_plugin_repo), None, is_keyadmin = True)
+ if create_repo_response is not None and len(create_repo_response) > 0:
+ return True
+ else:
+ return False
+ else:
+ Logger.error('Ranger service is not reachable')
+ return False
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_server.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_server.py
new file mode 100755
index 0000000..613931b
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_server.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.script import Script
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.libraries.functions.format import format
+from resource_management.core.logger import Logger
+from resource_management.core import shell
+from resource_management.libraries.functions.default import default
+from kms import kms, setup_kms_db, setup_java_patch, enable_kms_plugin, setup_kms_jce
+from kms_service import kms_service
+import upgrade
+
+class KmsServer(Script):
+
+ def get_component_name(self):
+ return "ranger-kms"
+
+ def install(self, env):
+ self.install_packages(env)
+ import params
+ env.set_params(params)
+
+ setup_kms_db()
+ self.configure(env)
+ setup_java_patch()
+
+ def stop(self, env, upgrade_type=None):
+ import params
+
+ env.set_params(params)
+ kms_service(action = 'stop', upgrade_type=upgrade_type)
+ if params.stack_supports_pid:
+ File(params.ranger_kms_pid_file,
+ action = "delete"
+ )
+
+ def start(self, env, upgrade_type=None):
+ import params
+
+ env.set_params(params)
+ self.configure(env)
+ enable_kms_plugin()
+ setup_kms_jce()
+ kms_service(action = 'start', upgrade_type=upgrade_type)
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ if status_params.stack_supports_pid:
+ check_process_status(status_params.ranger_kms_pid_file)
+ return
+
+ cmd = 'ps -ef | grep proc_rangerkms | grep -v grep'
+ code, output = shell.call(cmd, timeout=20)
+ if code != 0:
+ Logger.debug('KMS process not running')
+ raise ComponentIsNotRunning()
+ pass
+
+ def configure(self, env):
+ import params
+
+ env.set_params(params)
+ kms()
+
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ upgrade.prestart(env, "ranger-kms")
+ kms(upgrade_type=upgrade_type)
+ setup_java_patch()
+
+ def setup_ranger_kms_database(self, env):
+ import params
+ env.set_params(params)
+
+ upgrade_stack = stack_select._get_upgrade_stack()
+ if upgrade_stack is None:
+ raise Fail('Unable to determine the stack and stack version')
+
+ stack_version = upgrade_stack[1]
+ Logger.info(format('Setting Ranger KMS database schema, using version {stack_version}'))
+ setup_kms_db(stack_version=stack_version)
+
+ def get_log_folder(self):
+ import params
+ return params.kms_log_dir
+
+ def get_user(self):
+ import params
+ return params.kms_user
+
+if __name__ == "__main__":
+ KmsServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_service.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_service.py
new file mode 100644
index 0000000..2ff48c3
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_service.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core import shell
+from resource_management.libraries.functions.format import format
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.show_logs import show_logs
+from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING
+from resource_management.libraries.functions.constants import Direction
+import os
+
+def kms_service(action='start', upgrade_type=None):
+ import params
+
+ env_dict = {'JAVA_HOME': params.java_home}
+ if params.db_flavor.lower() == 'sqla':
+ env_dict = {'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH': params.ld_library_path}
+
+ if action == 'start':
+ no_op_test = format('ps -ef | grep proc_rangerkms | grep -v grep')
+ cmd = format('{kms_home}/ranger-kms start')
+ try:
+ Execute(cmd, not_if=no_op_test, environment=env_dict, user=format('{kms_user}'))
+ except:
+ show_logs(params.kms_log_dir, params.kms_user)
+ raise
+ elif action == 'stop':
+ if upgrade_type == UPGRADE_TYPE_NON_ROLLING and params.upgrade_direction == Direction.UPGRADE:
+ if os.path.isfile(format('{kms_home}/ranger-kms')):
+ File(format('{kms_home}/ranger-kms'),
+ owner=params.kms_user,
+ group = params.kms_group
+ )
+ cmd = format('{kms_home}/ranger-kms stop')
+ try:
+ Execute(cmd, environment=env_dict, user=format('{kms_user}'))
+ except:
+ show_logs(params.kms_log_dir, params.kms_user)
+ raise
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/params.py
new file mode 100755
index 0000000..003eee1
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/params.py
@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.stack_features import get_stack_feature_version
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.get_bare_principal import get_bare_principal
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import get_kinit_path
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+stack_root = Script.get_stack_root()
+
+stack_name = default("/hostLevelParams/stack_name", None)
+version = default("/commandParams/version", None)
+upgrade_direction = default("/commandParams/upgrade_direction", None)
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+# get the correct version to use for checking stack features
+version_for_stack_feature_checks = get_stack_feature_version(config)
+
+stack_supports_config_versioning = check_stack_feature(StackFeature.CONFIG_VERSIONING, version_for_stack_feature_checks)
+stack_support_kms_hsm = check_stack_feature(StackFeature.RANGER_KMS_HSM_SUPPORT, version_for_stack_feature_checks)
+stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
+stack_supports_pid = check_stack_feature(StackFeature.RANGER_KMS_PID_SUPPORT, version_for_stack_feature_checks)
+stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
+stack_supports_ranger_kms_ssl = check_stack_feature(StackFeature.RANGER_KMS_SSL, version_for_stack_feature_checks)
+
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+if stack_supports_config_versioning:
+ kms_home = format('{stack_root}/current/ranger-kms')
+ kms_conf_dir = format('{stack_root}/current/ranger-kms/conf')
+
+kms_log_dir = default("/configurations/kms-env/kms_log_dir", "/var/log/ranger/kms")
+java_home = config['hostLevelParams']['java_home']
+kms_user = default("/configurations/kms-env/kms_user", "kms")
+kms_group = default("/configurations/kms-env/kms_group", "kms")
+
+ranger_kms_audit_log_maxfilesize = default('/configurations/kms-log4j/ranger_kms_audit_log_maxfilesize',256)
+ranger_kms_audit_log_maxbackupindex = default('/configurations/kms-log4j/ranger_kms_audit_log_maxbackupindex',20)
+ranger_kms_log_maxfilesize = default('/configurations/kms-log4j/ranger_kms_log_maxfilesize',256)
+ranger_kms_log_maxbackupindex = default('/configurations/kms-log4j/ranger_kms_log_maxbackupindex',20)
+
+jdk_location = config['hostLevelParams']['jdk_location']
+kms_log4j = config['configurations']['kms-log4j']['content']
+
+# ranger host
+ranger_admin_hosts = config['clusterHostInfo']['ranger_admin_hosts'][0]
+has_ranger_admin = len(ranger_admin_hosts) > 0
+kms_host = config['clusterHostInfo']['ranger_kms_server_hosts'][0]
+kms_port = config['configurations']['kms-env']['kms_port']
+
+create_db_user = config['configurations']['kms-env']['create_db_user']
+
+#kms properties
+db_flavor = (config['configurations']['kms-properties']['DB_FLAVOR']).lower()
+db_host = config['configurations']['kms-properties']['db_host']
+db_name = config['configurations']['kms-properties']['db_name']
+db_user = config['configurations']['kms-properties']['db_user']
+db_password = unicode(config['configurations']['kms-properties']['db_password'])
+kms_master_key_password = unicode(config['configurations']['kms-properties']['KMS_MASTER_KEY_PASSWD'])
+credential_provider_path = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.credential.provider.path']
+jdbc_alias = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.credential.alias']
+masterkey_alias = config['configurations']['dbks-site']['ranger.ks.masterkey.credential.alias']
+repo_name = str(config['clusterName']) + '_kms'
+repo_name_value = config['configurations']['ranger-kms-security']['ranger.plugin.kms.service.name']
+if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
+ repo_name = repo_name_value
+cred_lib_path = os.path.join(kms_home,"cred","lib","*")
+cred_setup_prefix = (format('{kms_home}/ranger_credential_helper.py'), '-l', cred_lib_path)
+credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
+
+if has_ranger_admin:
+ policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+ if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+ xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
+ xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+ xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+ xa_audit_db_password = ''
+ if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+ xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password']
+ xa_db_host = config['configurations']['admin-properties']['db_host']
+
+ admin_uname = config['configurations']['ranger-env']['admin_username']
+ admin_password = config['configurations']['ranger-env']['admin_password']
+ ambari_ranger_admin = config['configurations']['ranger-env']['ranger_admin_username']
+ ambari_ranger_password = config['configurations']['ranger-env']['ranger_admin_password']
+ admin_uname_password = format("{admin_uname}:{admin_password}")
+ ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
+
+default_connectors_map = { "mssql":"sqljdbc4.jar",
+ "mysql":"mysql-connector-java.jar",
+ "postgres":"postgresql-jdbc.jar",
+ "oracle":"ojdbc.jar",
+ "sqla":"sajdbc4.jar"}
+
+java_share_dir = '/usr/share/java'
+jdbc_jar_name = None
+previous_jdbc_jar_name = None
+if db_flavor == 'mysql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
+ db_jdbc_url = format('jdbc:log4jdbc:mysql://{db_host}/{db_name}')
+ db_jdbc_driver = "com.mysql.jdbc.Driver"
+ jdbc_dialect = "org.eclipse.persistence.platform.database.MySQLPlatform"
+elif db_flavor == 'oracle':
+ jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
+ colon_count = db_host.count(':')
+ if colon_count == 2 or colon_count == 0:
+ db_jdbc_url = format('jdbc:oracle:thin:@{db_host}')
+ else:
+ db_jdbc_url = format('jdbc:oracle:thin:@//{db_host}')
+ db_jdbc_driver = "oracle.jdbc.OracleDriver"
+ jdbc_dialect = "org.eclipse.persistence.platform.database.OraclePlatform"
+elif db_flavor == 'postgres':
+ jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
+ db_jdbc_url = format('jdbc:postgresql://{db_host}/{db_name}')
+ db_jdbc_driver = "org.postgresql.Driver"
+ jdbc_dialect = "org.eclipse.persistence.platform.database.PostgreSQLPlatform"
+elif db_flavor == 'mssql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
+ db_jdbc_url = format('jdbc:sqlserver://{db_host};databaseName={db_name}')
+ db_jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+ jdbc_dialect = "org.eclipse.persistence.platform.database.SQLServerPlatform"
+elif db_flavor == 'sqla':
+ jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
+ db_jdbc_url = format('jdbc:sqlanywhere:database={db_name};host={db_host}')
+ db_jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
+ jdbc_dialect = "org.eclipse.persistence.platform.database.SQLAnywherePlatform"
+
+downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}")
+
+driver_curl_source = format("{jdk_location}/{jdbc_jar_name}")
+driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
+previous_jdbc_jar = format("{kms_home}/ews/webapp/lib/{previous_jdbc_jar_name}")
+ews_lib_jar_path = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
+
+if db_flavor == 'sqla':
+ downloaded_custom_connector = format("{tmp_dir}/sqla-client-jdbc.tar.gz")
+ jar_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/java/sajdbc4.jar")
+ libs_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/native/lib64/*")
+ jdbc_libs_dir = format("{kms_home}/native/lib64")
+ ld_library_path = format("{jdbc_libs_dir}")
+
+if has_ranger_admin:
+ xa_previous_jdbc_jar_name = None
+ if stack_supports_ranger_audit_db:
+ if xa_audit_db_flavor == 'mysql':
+ jdbc_jar = default("/hostLevelParams/custom_mysql_jdbc_name", None)
+ xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "com.mysql.jdbc.Driver"
+ elif xa_audit_db_flavor == 'oracle':
+ jdbc_jar = default("/hostLevelParams/custom_oracle_jdbc_name", None)
+ xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
+ colon_count = xa_db_host.count(':')
+ if colon_count == 2 or colon_count == 0:
+ audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
+ else:
+ audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
+ jdbc_driver = "oracle.jdbc.OracleDriver"
+ elif xa_audit_db_flavor == 'postgres':
+ jdbc_jar = default("/hostLevelParams/custom_postgres_jdbc_name", None)
+ xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "org.postgresql.Driver"
+ elif xa_audit_db_flavor == 'mssql':
+ jdbc_jar = default("/hostLevelParams/custom_mssql_jdbc_name", None)
+ xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
+ jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+ elif xa_audit_db_flavor == 'sqla':
+ jdbc_jar = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
+ xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
+ jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
+
+ downloaded_connector_path = format("{tmp_dir}/{jdbc_jar}") if stack_supports_ranger_audit_db else None
+ driver_source = format("{jdk_location}/{jdbc_jar}") if stack_supports_ranger_audit_db else None
+ driver_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar}") if stack_supports_ranger_audit_db else None
+ xa_previous_jdbc_jar = format("{kms_home}/ews/webapp/lib/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+
+repo_config_username = config['configurations']['kms-properties']['REPOSITORY_CONFIG_USERNAME']
+repo_config_password = unicode(config['configurations']['kms-properties']['REPOSITORY_CONFIG_PASSWORD'])
+
+kms_plugin_config = {
+ 'username' : repo_config_username,
+ 'password' : repo_config_password,
+ 'provider' : format('kms://http@{kms_host}:{kms_port}/kms')
+}
+
+xa_audit_db_is_enabled = False
+if stack_supports_ranger_audit_db:
+ xa_audit_db_is_enabled = config['configurations']['ranger-kms-audit']['xasecure.audit.destination.db']
+ssl_keystore_password = unicode(config['configurations']['ranger-kms-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'])
+ssl_truststore_password = unicode(config['configurations']['ranger-kms-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'])
+
+#For SQLA explicitly disable audit to DB for Ranger
+if xa_audit_db_flavor == 'sqla':
+ xa_audit_db_is_enabled = False
+
+current_host = config['hostname']
+ranger_kms_hosts = config['clusterHostInfo']['ranger_kms_server_hosts']
+if current_host in ranger_kms_hosts:
+ kms_host = current_host
+
+check_db_connection_jar_name = "DBConnectionVerification.jar"
+check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}")
+ranger_kms_jdbc_connection_url = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.url']
+ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver']
+
+jce_name = default("/hostLevelParams/jce_name", None)
+jce_source_dir = format('{tmp_dir}/jce_dir')
+
+#kms hsm support
+enable_kms_hsm = default("/configurations/dbks-site/ranger.ks.hsm.enabled", False)
+hms_partition_alias = default("/configurations/dbks-site/ranger.ks.hsm.partition.password.alias", "ranger.kms.hsm.partition.password")
+hms_partition_passwd = default("/configurations/kms-env/hsm_partition_password", None)
+
+# kms kerberos from stack 2.5 onward
+rangerkms_bare_principal = 'rangerkms'
+
+if stack_supports_ranger_kerberos:
+ if security_enabled:
+ rangerkms_principal = config['configurations']['dbks-site']['ranger.ks.kerberos.principal']
+ rangerkms_keytab = config['configurations']['dbks-site']['ranger.ks.kerberos.keytab']
+ if not is_empty(rangerkms_principal) and rangerkms_principal != '':
+ rangerkms_bare_principal = get_bare_principal(rangerkms_principal)
+ rangerkms_principal = rangerkms_principal.replace('_HOST', kms_host.lower())
+ kms_plugin_config['policy.download.auth.users'] = format('keyadmin,{rangerkms_bare_principal}')
+
+custom_ranger_service_config = generate_ranger_service_config(config['configurations']['kms-properties'])
+if len(custom_ranger_service_config) > 0:
+ kms_plugin_config.update(custom_ranger_service_config)
+
+kms_ranger_plugin_repo = {
+ 'isEnabled' : 'true',
+ 'configs' : kms_plugin_config,
+ 'description' : 'kms repo',
+ 'name' : repo_name,
+ 'type' : 'kms'
+}
+
+# ranger kms pid
+user_group = config['configurations']['cluster-env']['user_group']
+ranger_kms_pid_dir = default("/configurations/kms-env/ranger_kms_pid_dir", "/var/run/ranger_kms")
+ranger_kms_pid_file = format('{ranger_kms_pid_dir}/rangerkms.pid')
+
+if security_enabled:
+ spengo_keytab = config['configurations']['kms-site']['hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab']
+ spnego_principal = config['configurations']['kms-site']['hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal']
+ spnego_principal = spnego_principal.replace('_HOST', current_host.lower())
+
+plugin_audit_password_property = 'xasecure.audit.destination.db.password'
+kms_plugin_password_properties = ['xasecure.policymgr.clientssl.keystore.password', 'xasecure.policymgr.clientssl.truststore.password']
+dbks_site_password_properties = ['ranger.db.encrypt.key.password', 'ranger.ks.jpa.jdbc.password', 'ranger.ks.hsm.partition.password']
+ranger_kms_site_password_properties = ['ranger.service.https.attrib.keystore.pass']
+ranger_kms_cred_ssl_path = config['configurations']['ranger-kms-site']['ranger.credential.provider.path']
+ranger_kms_ssl_keystore_alias = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.keystore.credential.alias']
+ranger_kms_ssl_passwd = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.keystore.pass']
+ranger_kms_ssl_enabled = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.ssl.enabled']
+
+xa_audit_hdfs_is_enabled = default("/configurations/ranger-kms-audit/xasecure.audit.destination.hdfs", False)
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+
+# need this to capture cluster name from where ranger kms plugin is enabled
+cluster_name = config['clusterName']
+
+has_namenode = len(namenode_host) > 0
+
+hdfs_user = default("/configurations/hadoop-env/hdfs_user", None)
+hdfs_user_keytab = default("/configurations/hadoop-env/hdfs_user_keytab", None)
+hdfs_principal_name = default("/configurations/hadoop-env/hdfs_principal_name", None)
+default_fs = default("/configurations/core-site/fs.defaultFS", None)
+hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+import functools
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs
+)
+
+local_component_list = default("/localComponents", [])
+has_hdfs_client_on_node = 'HDFS_CLIENT' in local_component_list
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/service_check.py
new file mode 100644
index 0000000..84e4e73
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/service_check.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+from resource_management.core.logger import Logger
+from resource_management.core import shell
+from resource_management.core.exceptions import ComponentIsNotRunning
+
+
+class KmsServiceCheck(Script):
+ def service_check(self, env):
+ import params
+
+ env.set_params(params)
+ cmd = 'ps -ef | grep proc_rangerkms | grep -v grep'
+ code, output = shell.call(cmd, timeout=20)
+ if code == 0:
+ Logger.info('KMS process up and running')
+ else:
+ Logger.debug('KMS process not running')
+ raise ComponentIsNotRunning()
+
+if __name__ == "__main__":
+ KmsServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/status_params.py
new file mode 100644
index 0000000..34d0082
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/status_params.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+stack_name = default("/hostLevelParams/stack_name", None)
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+stack_supports_pid = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KMS_PID_SUPPORT, stack_version_formatted)
+ranger_kms_pid_dir = default("/configurations/kms-env/ranger_kms_pid_dir", "/var/run/ranger_kms")
+ranger_kms_pid_file = format('{ranger_kms_pid_dir}/rangerkms.pid')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/upgrade.py
new file mode 100644
index 0000000..8478bb8
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/upgrade.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.format import format
+
+def prestart(env, stack_component):
+ import params
+
+ if params.version and params.stack_supports_config_versioning:
+ conf_select.select(params.stack_name, stack_component, params.version)
+ stack_select.select(stack_component, params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/templates/input.config-ranger-kms.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/templates/input.config-ranger-kms.json.j2 b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/templates/input.config-ranger-kms.json.j2
new file mode 100644
index 0000000..306fade
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/templates/input.config-ranger-kms.json.j2
@@ -0,0 +1,48 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"ranger_kms",
+ "rowtype":"service",
+ "path":"{{default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms')}}/kms.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ranger_kms"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/role_command_order.json b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/role_command_order.json
new file mode 100644
index 0000000..7ddab41
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/role_command_order.json
@@ -0,0 +1,7 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for RANGER-KMS",
+ "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"],
+ "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py
new file mode 100644
index 0000000..b81e05b
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py
@@ -0,0 +1,358 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Python imports
+import imp
+import os
+import traceback
+import re
+import socket
+import fnmatch
+
+
+from resource_management.core.logger import Logger
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"}
+
+class Ranger_KMSServiceAdvisor(service_advisor.ServiceAdvisor):
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(Ranger_KMSServiceAdvisor, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ # Always call these methods
+ self.modifyMastersWithMultipleInstances()
+ self.modifyCardinalitiesDict()
+ self.modifyHeapSizeProperties()
+ self.modifyNotValuableComponents()
+ self.modifyComponentsNotPreferableOnServer()
+ self.modifyComponentLayoutSchemes()
+
+ def modifyMastersWithMultipleInstances(self):
+ """
+ Modify the set of masters with multiple instances.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyCardinalitiesDict(self):
+ """
+ Modify the dictionary of cardinalities.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyHeapSizeProperties(self):
+ """
+ Modify the dictionary of heap size properties.
+ Must be overriden in child class.
+ """
+ pass
+
+ def modifyNotValuableComponents(self):
+ """
+ Modify the set of components whose host assignment is based on other services.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentsNotPreferableOnServer(self):
+ """
+ Modify the set of components that are not preferable on the server.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentLayoutSchemes(self):
+ """
+ Modify layout scheme dictionaries for components.
+ The scheme dictionary basically maps the number of hosts to
+ host index where component should exist.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def getServiceComponentLayoutValidations(self, services, hosts):
+ """
+ Get a list of errors.
+ Must be overriden in child class.
+ """
+
+ return []
+
+ def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
+ """
+ Entry point.
+ Must be overriden in child class.
+ """
+ # Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ recommender = RangerKMSRecommender()
+ recommender.recommendRangerKMSConfigurationsFromHDP23(configurations, clusterData, services, hosts)
+ recommender.recommendRangerKMSConfigurationsFromHDP25(configurations, clusterData, services, hosts)
+ recommender.recommendRangerKMSConfigurationsFromHDP26(configurations, clusterData, services, hosts)
+
+ def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
+ """
+ Entry point.
+ Validate configurations for the service. Return a list of errors.
+ The code for this function should be the same for each Service Advisor.
+ """
+ # Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ validator = RangerKMSValidator()
+ # Calls the methods of the validator using arguments,
+ # method(siteProperties, siteRecommendations, configurations, services, hosts)
+ return validator.validateListOfConfigUsingMethod(configurations, recommendedDefaults, services, hosts, validator.validators)
+
+class RangerKMSRecommender(service_advisor.ServiceAdvisor):
+ """
+ RangerKMS Recommender suggests properties when adding the service for the first time or modifying configs via the UI.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(RangerKMSRecommender, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ def recommendRangerKMSConfigurationsFromHDP23(self, configurations, clusterData, services, hosts):
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putRangerKmsDbksProperty = self.putProperty(configurations, "dbks-site", services)
+ putRangerKmsProperty = self.putProperty(configurations, "kms-properties", services)
+ kmsEnvProperties = self.getSiteProperties(services['configurations'], 'kms-env')
+ putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
+ putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site")
+ putRangerKmsAuditProperty = self.putProperty(configurations, "ranger-kms-audit", services)
+ security_enabled = self.isSecurityEnabled(services)
+ putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services)
+ putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
+
+ if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']):
+
+ rangerKmsDbFlavor = services['configurations']["kms-properties"]["properties"]["DB_FLAVOR"]
+
+ if ('db_host' in services['configurations']['kms-properties']['properties']) and ('db_name' in services['configurations']['kms-properties']['properties']):
+
+ rangerKmsDbHost = services['configurations']["kms-properties"]["properties"]["db_host"]
+ rangerKmsDbName = services['configurations']["kms-properties"]["properties"]["db_name"]
+
+ ranger_kms_db_url_dict = {
+ 'MYSQL': {'ranger.ks.jpa.jdbc.driver': 'com.mysql.jdbc.Driver',
+ 'ranger.ks.jpa.jdbc.url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName},
+ 'ORACLE': {'ranger.ks.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver',
+ 'ranger.ks.jpa.jdbc.url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost, rangerKmsDbName)},
+ 'POSTGRES': {'ranger.ks.jpa.jdbc.driver': 'org.postgresql.Driver',
+ 'ranger.ks.jpa.jdbc.url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName},
+ 'MSSQL': {'ranger.ks.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver',
+ 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';databaseName=' + rangerKmsDbName},
+ 'SQLA': {'ranger.ks.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver',
+ 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';database=' + rangerKmsDbName}
+ }
+
+ rangerKmsDbProperties = ranger_kms_db_url_dict.get(rangerKmsDbFlavor, ranger_kms_db_url_dict['MYSQL'])
+ for key in rangerKmsDbProperties:
+ putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key))
+
+ if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList:
+ kmsUser = kmsEnvProperties['kms_user']
+ kmsUserOld = self.getOldValue(services, 'kms-env', 'kms_user')
+ self.put_proxyuser_value(kmsUser, '*', is_groups=True, services=services, configurations=configurations, put_function=putCoreSiteProperty)
+ if kmsUserOld is not None and kmsUser != kmsUserOld:
+ putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true')
+ services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)})
+ services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUser)})
+
+ if "HDFS" in servicesList:
+ if 'core-site' in services['configurations'] and ('fs.defaultFS' in services['configurations']['core-site']['properties']):
+ default_fs = services['configurations']['core-site']['properties']['fs.defaultFS']
+ putRangerKmsAuditProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit'))
+
+ required_services = [{'service' : 'YARN', 'config-type': 'yarn-env', 'property-name': 'yarn_user', 'proxy-category': ['hosts', 'users', 'groups']},
+ {'service' : 'SPARK', 'config-type': 'livy-env', 'property-name': 'livy_user', 'proxy-category': ['hosts', 'users', 'groups']}]
+
+ required_services_for_secure = [{'service' : 'HIVE', 'config-type': 'hive-env', 'property-name': 'hive_user', 'proxy-category': ['hosts', 'users']},
+ {'service' : 'OOZIE', 'config-type': 'oozie-env', 'property-name': 'oozie_user', 'proxy-category': ['hosts', 'users']}]
+
+ if security_enabled:
+ required_services.extend(required_services_for_secure)
+
+ # recommendations for kms proxy related properties
+ self.recommendKMSProxyUsers(configurations, services, hosts, required_services)
+
+ ambari_user = self.getAmbariUser(services)
+ if security_enabled:
+ # adding for ambari user
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), '*')
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), '*')
+ # adding for HTTP
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.users', '*')
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.hosts', '*')
+ else:
+ self.deleteKMSProxyUsers(configurations, services, hosts, required_services_for_secure)
+ # deleting ambari user proxy properties
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), 'delete', 'true')
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), 'delete', 'true')
+ # deleting HTTP proxy properties
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.hosts', 'delete', 'true')
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.users', 'delete', 'true')
+
+ def recommendRangerKMSConfigurationsFromHDP25(self, configurations, clusterData, services, hosts):
+
+ security_enabled = self.isSecurityEnabled(services)
+ required_services = [{'service' : 'RANGER', 'config-type': 'ranger-env', 'property-name': 'ranger_user', 'proxy-category': ['hosts', 'users', 'groups']}]
+
+ if security_enabled:
+ # recommendations for kms proxy related properties
+ self.recommendKMSProxyUsers(configurations, services, hosts, required_services)
+ else:
+ self.deleteKMSProxyUsers(configurations, services, hosts, required_services)
+
+ def recommendRangerKMSConfigurationsFromHDP26(self, configurations, clusterData, services, hosts):
+ putRangerKmsEnvProperty = self.putProperty(configurations, "kms-env", services)
+
+ ranger_kms_ssl_enabled = False
+ ranger_kms_ssl_port = "9393"
+ if 'ranger-kms-site' in services['configurations'] and 'ranger.service.https.attrib.ssl.enabled' in services['configurations']['ranger-kms-site']['properties']:
+ ranger_kms_ssl_enabled = services['configurations']['ranger-kms-site']['properties']['ranger.service.https.attrib.ssl.enabled'].lower() == "true"
+
+ if 'ranger-kms-site' in services['configurations'] and 'ranger.service.https.port' in services['configurations']['ranger-kms-site']['properties']:
+ ranger_kms_ssl_port = services['configurations']['ranger-kms-site']['properties']['ranger.service.https.port']
+
+ if ranger_kms_ssl_enabled:
+ putRangerKmsEnvProperty("kms_port", ranger_kms_ssl_port)
+ else:
+ putRangerKmsEnvProperty("kms_port", "9292")
+
+ def getDBConnectionHostPort(self, db_type, db_host):
+ connection_string = ""
+ if db_type is None or db_type == "":
+ return connection_string
+ else:
+ colon_count = db_host.count(':')
+ if colon_count == 0:
+ if DB_TYPE_DEFAULT_PORT_MAP.has_key(db_type):
+ connection_string = db_host + ":" + DB_TYPE_DEFAULT_PORT_MAP[db_type]
+ else:
+ connection_string = db_host
+ elif colon_count == 1:
+ connection_string = db_host
+ elif colon_count == 2:
+ connection_string = db_host
+
+ return connection_string
+
+ def getOracleDBConnectionHostPort(self, db_type, db_host, rangerDbName):
+ connection_string = self.getDBConnectionHostPort(db_type, db_host)
+ colon_count = db_host.count(':')
+ if colon_count == 1 and '/' in db_host:
+ connection_string = "//" + connection_string
+ elif colon_count == 0 or colon_count == 1:
+ connection_string = "//" + connection_string + "/" + rangerDbName if rangerDbName else "//" + connection_string
+
+ return connection_string
+
+ def recommendKMSProxyUsers(self, configurations, services, hosts, requiredServices):
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services)
+ putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
+
+ if 'forced-configurations' not in services:
+ services["forced-configurations"] = []
+
+ for index in range(len(requiredServices)):
+ service = requiredServices[index]['service']
+ config_type = requiredServices[index]['config-type']
+ property_name = requiredServices[index]['property-name']
+ proxy_category = requiredServices[index]['proxy-category']
+
+ if service in servicesList:
+ if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']:
+ service_user = services['configurations'][config_type]['properties'][property_name]
+ service_old_user = self.getOldValue(services, config_type, property_name)
+
+ if 'groups' in proxy_category:
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.groups'.format(service_user), '*')
+ if 'hosts' in proxy_category:
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(service_user), '*')
+ if 'users' in proxy_category:
+ putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(service_user), '*')
+
+ if service_old_user is not None and service_user != service_old_user:
+ if 'groups' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_old_user), 'delete', 'true')
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_old_user)})
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_user)})
+ if 'hosts' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_old_user), 'delete', 'true')
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_old_user)})
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_user)})
+ if 'users' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_old_user), 'delete', 'true')
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_old_user)})
+ services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_user)})
+
+ def deleteKMSProxyUsers(self, configurations, services, hosts, requiredServices):
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
+
+ for index in range(len(requiredServices)):
+ service = requiredServices[index]['service']
+ config_type = requiredServices[index]['config-type']
+ property_name = requiredServices[index]['property-name']
+ proxy_category = requiredServices[index]['proxy-category']
+
+ if service in servicesList:
+ if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']:
+ service_user = services['configurations'][config_type]['properties'][property_name]
+
+ if 'groups' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_user), 'delete', 'true')
+ if 'hosts' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_user), 'delete', 'true')
+ if 'users' in proxy_category:
+ putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_user), 'delete', 'true')
+
+class RangerKMSValidator(service_advisor.ServiceAdvisor):
+ """
+ RangerKMS Validator checks the correctness of properties whenever the service is first added or the user attempts to
+ change configs via the UI.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(RangerKMSValidator, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ self.validators = []