You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2017/05/17 20:51:42 UTC
[08/25] ambari git commit: AMBARI-21032. HDP 3.0 TP - create service
definition for Knox with configs, kerberos, widgets, etc.(vbrodetsky)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/knox_ldap.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/knox_ldap.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/knox_ldap.py
new file mode 100644
index 0000000..b6f1b89
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/knox_ldap.py
@@ -0,0 +1,59 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources.service import ServiceConfig
+from resource_management.core.resources.system import File
+from ambari_commons import OSConst
+from resource_management.core.source import InlineTemplate
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+def _ldap_common():
+ import params
+
+ File(os.path.join(params.knox_conf_dir, 'ldap-log4j.properties'),
+ mode=params.mode,
+ group=params.knox_group,
+ owner=params.knox_user,
+ content=InlineTemplate(params.ldap_log4j)
+ )
+
+ File(os.path.join(params.knox_conf_dir, 'users.ldif'),
+ mode=params.mode,
+ group=params.knox_group,
+ owner=params.knox_user,
+ content=params.users_ldif
+ )
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def ldap():
+ import params
+
+ # Manually overriding service logon user & password set by the installation package
+ ServiceConfig(params.knox_ldap_win_service_name,
+ action="change_user",
+ username = params.knox_user,
+ password = Script.get_password(params.knox_user))
+
+ _ldap_common()
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def ldap():
+ _ldap_common()
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params.py
new file mode 100644
index 0000000..ad1a1dc
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params.py
@@ -0,0 +1,29 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+retryAble = default("/commandParams/command_retry_enabled", False)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params_linux.py
new file mode 100644
index 0000000..4558069
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params_linux.py
@@ -0,0 +1,457 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import status_params
+
+from resource_management.core.logger import Logger
+
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_port_from_url import get_port_from_url
+from resource_management.libraries.functions.get_stack_version import get_stack_version
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.script.script import Script
+from status_params import *
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import stack_select, conf_select
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.stack_features import get_stack_feature_version
+from resource_management.libraries.functions.constants import StackFeature
+from resource_management.libraries.functions import is_empty
+from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config
+
+# server configurations
+config = Script.get_config()
+stack_root = Script.get_stack_root()
+
+tmp_dir = Script.get_tmp_dir()
+stack_name = status_params.stack_name
+upgrade_direction = default("/commandParams/upgrade_direction", None)
+version = default("/commandParams/version", None)
+# E.g., 2.3.2.0
+version_formatted = format_stack_version(version)
+
+# E.g., 2.3
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+# get the correct version to use for checking stack features
+version_for_stack_feature_checks = get_stack_feature_version(config)
+
+stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
+stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
+stack_supports_core_site_for_ranger_plugin = check_stack_feature(StackFeature.CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT, version_for_stack_feature_checks)
+
+# This is the version whose state is CURRENT. During an RU, this is the source version.
+# DO NOT format it since we need the build number too.
+upgrade_from_version = default("/hostLevelParams/current_version", None)
+
+# server configurations
+# Default value used in HDP 2.3.0.0 and earlier.
+knox_data_dir = '/var/lib/knox/data'
+
+# Important, it has to be strictly greater than 2.3.0.0!!!
+Logger.info(format("Stack version to use is {version_formatted}"))
+if version_formatted and check_stack_feature(StackFeature.KNOX_VERSIONED_DATA_DIR, version_formatted):
+ # This is the current version. In the case of a Rolling Upgrade, it will be the newer version.
+ # In the case of a Downgrade, it will be the version downgrading to.
+ # This is always going to be a symlink to /var/lib/knox/data_${version}
+ knox_data_dir = format('{stack_root}/{version}/knox/data')
+ Logger.info(format("Detected stack with version {version}, will use knox_data_dir = {knox_data_dir}"))
+
+
+knox_master_secret_path = format('{knox_data_dir}/security/master')
+knox_cert_store_path = format('{knox_data_dir}/security/keystores/gateway.jks')
+knox_user = default("/configurations/knox-env/knox_user", "knox")
+
+# server configurations
+knox_data_dir = '/var/lib/knox/data'
+knox_logs_dir = '/var/log/knox'
+
+# default parameters
+knox_bin = '/usr/bin/gateway'
+knox_conf_dir = '/etc/knox/conf'
+ldap_bin = '/usr/lib/knox/bin/ldap.sh'
+knox_client_bin = '/usr/lib/knox/bin/knoxcli.sh'
+
+# HDP 2.2+ parameters
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ knox_bin = format('{stack_root}/current/knox-server/bin/gateway.sh')
+ knox_conf_dir = format('{stack_root}/current/knox-server/conf')
+ ldap_bin = format('{stack_root}/current/knox-server/bin/ldap.sh')
+ knox_client_bin = format('{stack_root}/current/knox-server/bin/knoxcli.sh')
+ knox_master_secret_path = format('{stack_root}/current/knox-server/data/security/master')
+ knox_cert_store_path = format('{stack_root}/current/knox-server/data/security/keystores/gateway.jks')
+ knox_data_dir = format('{stack_root}/current/knox-server/data/')
+
+knox_group = default("/configurations/knox-env/knox_group", "knox")
+mode = 0644
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+dfs_ha_enabled = False
+dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.internal.nameservices', None)
+if dfs_ha_nameservices is None:
+ dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.nameservices', None)
+dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None)
+
+namenode_rpc = None
+
+if dfs_ha_namenode_ids:
+ dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",")
+ dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list)
+ if dfs_ha_namenode_ids_array_len > 1:
+ dfs_ha_enabled = True
+if dfs_ha_enabled:
+ for nn_id in dfs_ha_namemodes_ids_list:
+ nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')]
+ if hostname.lower() in nn_host.lower():
+ namenode_id = nn_id
+ namenode_rpc = nn_host
+ # With HA enabled namenode_address is recomputed
+ namenode_address = format('hdfs://{dfs_ha_nameservices}')
+
+namenode_port_map = {}
+if dfs_ha_enabled:
+ for nn_id in dfs_ha_namemodes_ids_list:
+ nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.http-address.{dfs_ha_nameservices}.{nn_id}')]
+ nn_host_parts = nn_host.split(':')
+ namenode_port_map[nn_host_parts[0]] = nn_host_parts[1]
+
+
+namenode_hosts = default("/clusterHostInfo/namenode_host", None)
+if type(namenode_hosts) is list:
+ namenode_host = namenode_hosts[0]
+else:
+ namenode_host = namenode_hosts
+
+has_namenode = not namenode_host == None
+namenode_http_port = "50070"
+namenode_rpc_port = "8020"
+
+if has_namenode:
+ if 'dfs.namenode.http-address' in config['configurations']['hdfs-site']:
+ namenode_http_port = get_port_from_url(config['configurations']['hdfs-site']['dfs.namenode.http-address'])
+ if dfs_ha_enabled and namenode_rpc:
+ namenode_rpc_port = get_port_from_url(namenode_rpc)
+ else:
+ if 'dfs.namenode.rpc-address' in config['configurations']['hdfs-site']:
+ namenode_rpc_port = get_port_from_url(config['configurations']['hdfs-site']['dfs.namenode.rpc-address'])
+
+webhdfs_service_urls = ""
+
+def buildUrlElement(protocol, hdfs_host, port, servicePath) :
+ openTag = "<url>"
+ closeTag = "</url>"
+ proto = protocol + "://"
+ newLine = "\n"
+ if hdfs_host is None or port is None:
+ return ""
+ else:
+ return openTag + proto + hdfs_host + ":" + port + servicePath + closeTag + newLine
+
+namenode_host_keys = namenode_port_map.keys();
+if len(namenode_host_keys) > 0:
+ for host in namenode_host_keys:
+ webhdfs_service_urls += buildUrlElement("http", host, namenode_port_map[host], "/webhdfs")
+else:
+ webhdfs_service_urls = buildUrlElement("http", namenode_host, namenode_http_port, "/webhdfs")
+
+
+rm_hosts = default("/clusterHostInfo/rm_host", None)
+if type(rm_hosts) is list:
+ rm_host = rm_hosts[0]
+else:
+ rm_host = rm_hosts
+has_rm = not rm_host == None
+
+jt_rpc_port = "8050"
+rm_port = "8080"
+
+if has_rm:
+ if 'yarn.resourcemanager.address' in config['configurations']['yarn-site']:
+ jt_rpc_port = get_port_from_url(config['configurations']['yarn-site']['yarn.resourcemanager.address'])
+
+ if 'yarn.resourcemanager.webapp.address' in config['configurations']['yarn-site']:
+ rm_port = get_port_from_url(config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'])
+
+hive_http_port = default('/configurations/hive-site/hive.server2.thrift.http.port', "10001")
+hive_http_path = default('/configurations/hive-site/hive.server2.thrift.http.path', "cliservice")
+hive_server_hosts = default("/clusterHostInfo/hive_server_host", None)
+if type(hive_server_hosts) is list:
+ hive_server_host = hive_server_hosts[0]
+else:
+ hive_server_host = hive_server_hosts
+
+templeton_port = default('/configurations/webhcat-site/templeton.port', "50111")
+webhcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", None)
+if type(webhcat_server_hosts) is list:
+ webhcat_server_host = webhcat_server_hosts[0]
+else:
+ webhcat_server_host = webhcat_server_hosts
+
+hbase_master_port = default('/configurations/hbase-site/hbase.rest.port', "8080")
+hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", None)
+if type(hbase_master_hosts) is list:
+ hbase_master_host = hbase_master_hosts[0]
+else:
+ hbase_master_host = hbase_master_hosts
+
+oozie_server_hosts = default("/clusterHostInfo/oozie_server", None)
+if type(oozie_server_hosts) is list:
+ oozie_server_host = oozie_server_hosts[0]
+else:
+ oozie_server_host = oozie_server_hosts
+
+has_oozie = not oozie_server_host == None
+oozie_server_port = "11000"
+
+if has_oozie:
+ oozie_server_port = get_port_from_url(config['configurations']['oozie-site']['oozie.base.url'])
+
+# Knox managed properties
+knox_managed_pid_symlink= format('{stack_root}/current/knox-server/pids')
+
+#knox log4j
+knox_gateway_log_maxfilesize = default('/configurations/gateway-log4j/knox_gateway_log_maxfilesize',256)
+knox_gateway_log_maxbackupindex = default('/configurations/gateway-log4j/knox_gateway_log_maxbackupindex',20)
+knox_ldap_log_maxfilesize = default('/configurations/ldap-log4j/knox_ldap_log_maxfilesize',256)
+knox_ldap_log_maxbackupindex = default('/configurations/ldap-log4j/knox_ldap_log_maxbackupindex',20)
+
+# server configurations
+knox_master_secret = config['configurations']['knox-env']['knox_master_secret']
+knox_host_name = config['clusterHostInfo']['knox_gateway_hosts'][0]
+knox_host_name_in_cluster = config['hostname']
+knox_host_port = config['configurations']['gateway-site']['gateway.port']
+topology_template = config['configurations']['topology']['content']
+admin_topology_template = default('/configurations/admin-topology/content', None)
+knoxsso_topology_template = config['configurations']['knoxsso-topology']['content']
+gateway_log4j = config['configurations']['gateway-log4j']['content']
+ldap_log4j = config['configurations']['ldap-log4j']['content']
+users_ldif = config['configurations']['users-ldif']['content']
+java_home = config['hostLevelParams']['java_home']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+if security_enabled:
+ knox_keytab_path = config['configurations']['knox-env']['knox_keytab_path']
+ _hostname_lowercase = config['hostname'].lower()
+ knox_principal_name = config['configurations']['knox-env']['knox_principal_name'].replace('_HOST',_hostname_lowercase)
+
+# for curl command in ranger plugin to get db connector
+jdk_location = config['hostLevelParams']['jdk_location']
+
+# ranger knox plugin start section
+
+# ranger host
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+
+# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature
+xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks)
+
+# ambari-server hostname
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+# ranger knox plugin enabled property
+enable_ranger_knox = default("/configurations/ranger-knox-plugin-properties/ranger-knox-plugin-enabled", "No")
+enable_ranger_knox = True if enable_ranger_knox.lower() == 'yes' else False
+
+# get ranger knox properties if enable_ranger_knox is True
+if enable_ranger_knox:
+ # get ranger policy url
+ policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+ if xml_configurations_supported:
+ policymgr_mgr_url = config['configurations']['ranger-knox-security']['ranger.plugin.knox.policy.rest.url']
+
+ if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+
+ # ranger audit db user
+ xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+
+ # ranger knox service/repositry name
+ repo_name = str(config['clusterName']) + '_knox'
+ repo_name_value = config['configurations']['ranger-knox-security']['ranger.plugin.knox.service.name']
+ if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
+ repo_name = repo_name_value
+
+ knox_home = config['configurations']['ranger-knox-plugin-properties']['KNOX_HOME']
+ common_name_for_certificate = config['configurations']['ranger-knox-plugin-properties']['common.name.for.certificate']
+ repo_config_username = config['configurations']['ranger-knox-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+
+ # ranger-env config
+ ranger_env = config['configurations']['ranger-env']
+
+ # create ranger-env config having external ranger credential properties
+ if not has_ranger_admin and enable_ranger_knox:
+ external_admin_username = default('/configurations/ranger-knox-plugin-properties/external_admin_username', 'admin')
+ external_admin_password = default('/configurations/ranger-knox-plugin-properties/external_admin_password', 'admin')
+ external_ranger_admin_username = default('/configurations/ranger-knox-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin')
+ external_ranger_admin_password = default('/configurations/ranger-knox-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin')
+ ranger_env = {}
+ ranger_env['admin_username'] = external_admin_username
+ ranger_env['admin_password'] = external_admin_password
+ ranger_env['ranger_admin_username'] = external_ranger_admin_username
+ ranger_env['ranger_admin_password'] = external_ranger_admin_password
+
+ ranger_plugin_properties = config['configurations']['ranger-knox-plugin-properties']
+ policy_user = config['configurations']['ranger-knox-plugin-properties']['policy_user']
+ repo_config_password = config['configurations']['ranger-knox-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']
+
+ xa_audit_db_password = ''
+ if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin:
+ xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password']
+
+ downloaded_custom_connector = None
+ previous_jdbc_jar_name = None
+ driver_curl_source = None
+ driver_curl_target = None
+ previous_jdbc_jar = None
+
+ if has_ranger_admin and stack_supports_ranger_audit_db:
+ xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR']
+ jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config)
+
+ downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_target = format("{stack_root}/current/knox-server/ext/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ previous_jdbc_jar = format("{stack_root}/current/knox-server/ext/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ sql_connector_jar = ''
+
+ knox_ranger_plugin_config = {
+ 'username': repo_config_username,
+ 'password': repo_config_password,
+ 'knox.url': format("https://{knox_host_name}:{knox_host_port}/gateway/admin/api/v1/topologies"),
+ 'commonNameForCertificate': common_name_for_certificate
+ }
+
+ knox_ranger_plugin_repo = {
+ 'isActive': 'true',
+ 'config': json.dumps(knox_ranger_plugin_config),
+ 'description': 'knox repo',
+ 'name': repo_name,
+ 'repositoryType': 'knox',
+ 'assetType': '5',
+ }
+
+ custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties)
+ if len(custom_ranger_service_config) > 0:
+ knox_ranger_plugin_config.update(custom_ranger_service_config)
+
+ if stack_supports_ranger_kerberos and security_enabled:
+ knox_ranger_plugin_config['policy.download.auth.users'] = knox_user
+ knox_ranger_plugin_config['tag.download.auth.users'] = knox_user
+
+ if stack_supports_ranger_kerberos:
+ knox_ranger_plugin_config['ambari.service.check.user'] = policy_user
+
+ knox_ranger_plugin_repo = {
+ 'isEnabled': 'true',
+ 'configs': knox_ranger_plugin_config,
+ 'description': 'knox repo',
+ 'name': repo_name,
+ 'type': 'knox'
+ }
+
+ xa_audit_db_is_enabled = False
+ if xml_configurations_supported and stack_supports_ranger_audit_db:
+ xa_audit_db_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.destination.db']
+
+ xa_audit_hdfs_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else False
+ ssl_keystore_password = config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None
+ ssl_truststore_password = config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None
+ credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
+
+ # for SQLA explicitly disable audit to DB for Ranger
+ if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor == 'sqla':
+ xa_audit_db_is_enabled = False
+
+# need this to capture cluster name from where ranger knox plugin is enabled
+cluster_name = config['clusterName']
+
+# ranger knox plugin end section
+
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None
+hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None
+default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources()
+)
+
+druid_coordinator_urls = ""
+if "druid-coordinator" in config['configurations']:
+ port = config['configurations']['druid-coordinator']['druid.port']
+ for host in config['clusterHostInfo']['druid_coordinator_hosts']:
+ druid_coordinator_urls += buildUrlElement("http", host, port, "")
+
+druid_overlord_urls = ""
+if "druid-overlord" in config['configurations']:
+ port = config['configurations']['druid-overlord']['druid.port']
+ for host in config['clusterHostInfo']['druid_overlord_hosts']:
+ druid_overlord_urls += buildUrlElement("http", host, port, "")
+
+druid_broker_urls = ""
+if "druid-broker" in config['configurations']:
+ port = config['configurations']['druid-broker']['druid.port']
+ for host in config['clusterHostInfo']['druid_broker_hosts']:
+ druid_broker_urls += buildUrlElement("http", host, port, "")
+
+druid_router_urls = ""
+if "druid-router" in config['configurations']:
+ port = config['configurations']['druid-router']['druid.port']
+ for host in config['clusterHostInfo']['druid_router_hosts']:
+ druid_router_urls += buildUrlElement("http", host, port, "")
+
+zeppelin_ui_urls = ""
+zeppelin_ws_urls = ""
+websocket_support = "false"
+if "zeppelin-config" in config['configurations']:
+ port = config['configurations']['zeppelin-config']['zeppelin.server.port']
+ protocol = "https" if config['configurations']['zeppelin-config']['zeppelin.ssl'] else "http"
+ host = config['clusterHostInfo']['zeppelin_master_hosts'][0]
+ zeppelin_ui_urls += buildUrlElement(protocol, host, port, "")
+ zeppelin_ws_urls += buildUrlElement("ws", host, port, "/ws")
+ websocket_support = "true"
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params_windows.py
new file mode 100644
index 0000000..631146d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/params_windows.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.script import Script
+import os
+from status_params import *
+
+# server configurations
+config = Script.get_config()
+
+stack_root = None
+knox_home = None
+knox_conf_dir = None
+knox_logs_dir = None
+knox_bin = None
+ldap_bin = None
+knox_client_bin = None
+knox_data_dir = None
+
+knox_master_secret_path = None
+knox_cert_store_path = None
+
+try:
+ stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
+ knox_home = os.environ['KNOX_HOME']
+ knox_conf_dir = os.environ['KNOX_CONF_DIR']
+ knox_logs_dir = os.environ['KNOX_LOG_DIR']
+ knox_bin = os.path.join(knox_home, 'bin', 'gateway.exe')
+ ldap_bin = os.path.join(knox_home, 'bin', 'ldap.exe')
+ knox_client_bin = os.path.join(knox_home, 'bin', 'knoxcli.cmd')
+ knox_data_dir = os.path.join(knox_home, 'data')
+
+ knox_master_secret_path = os.path.join(knox_data_dir, 'security', 'master')
+ knox_cert_store_path = os.path.join(knox_data_dir, 'security', 'keystores', 'gateway.jks')
+except:
+ pass
+
+knox_host_port = config['configurations']['gateway-site']['gateway.port']
+knox_host_name = config['clusterHostInfo']['knox_gateway_hosts'][0]
+knox_host_name_in_cluster = config['hostname']
+knox_master_secret = config['configurations']['knox-env']['knox_master_secret']
+topology_template = config['configurations']['topology']['content']
+admin_topology_template = default('/configurations/admin-topology/content', None)
+knoxsso_topology_template = config['configurations']['knoxsso-topology']['content']
+gateway_log4j = config['configurations']['gateway-log4j']['content']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+ldap_log4j = config['configurations']['ldap-log4j']['content']
+users_ldif = config['configurations']['users-ldif']['content']
+
+hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
+knox_user = hadoop_user
+hdfs_user = hadoop_user
+knox_group = None
+mode = None
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/service_check.py
new file mode 100644
index 0000000..a2134d6
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/service_check.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources.system import Execute, File
+from resource_management.libraries.functions.format import format
+from resource_management.core.source import StaticFile
+import sys
+import os
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+
+class KnoxServiceCheck(Script):
+ def service_check(self, env):
+ pass
+
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class KnoxServiceCheckWindows(KnoxServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ temp_dir = os.path.join(os.path.dirname(params.knox_home), "temp")
+ validateKnoxFileName = "validateKnoxStatus.py"
+ validateKnoxFilePath = os.path.join(temp_dir, validateKnoxFileName)
+ python_executable = sys.executable
+ validateStatusCmd = "%s %s -p %s -n %s" % (python_executable, validateKnoxFilePath, params.knox_host_port, params.knox_host_name)
+
+ print "Test connectivity to knox server"
+
+ File(validateKnoxFilePath,
+ content=StaticFile(validateKnoxFileName)
+ )
+
+ Execute(validateStatusCmd,
+ tries=3,
+ try_sleep=5,
+ timeout=5,
+ logoutput=True
+ )
+
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class KnoxServiceCheckDefault(KnoxServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ validateKnoxFileName = "validateKnoxStatus.py"
+ validateKnoxFilePath = format("{tmp_dir}/{validateKnoxFileName}")
+ python_executable = sys.executable
+ validateStatusCmd = format("{python_executable} {validateKnoxFilePath} -p {knox_host_port} -n {knox_host_name}")
+ if params.security_enabled:
+ kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
+ smoke_cmd = format("{kinit_cmd} {validateStatusCmd}")
+ else:
+ smoke_cmd = validateStatusCmd
+
+ print "Test connectivity to knox server"
+
+ File(validateKnoxFilePath,
+ content=StaticFile(validateKnoxFileName),
+ mode=0755
+ )
+
+ Execute(smoke_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ user=params.smokeuser,
+ timeout=5,
+ logoutput=True
+ )
+
+
+if __name__ == "__main__":
+ KnoxServiceCheck().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/setup_ranger_knox.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/setup_ranger_knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/setup_ranger_knox.py
new file mode 100644
index 0000000..c486ef7
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/setup_ranger_knox.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_core_site_for_required_plugins
+from resource_management.core.resources import File
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions.format import format
+
+def setup_ranger_knox(upgrade_type=None):
+ import params
+
+ if params.enable_ranger_knox:
+
+ stack_version = None
+ if upgrade_type is not None:
+ stack_version = params.version
+
+ if params.retryAble:
+ Logger.info("Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
+ if params.xml_configurations_supported and params.enable_ranger_knox and params.xa_audit_hdfs_is_enabled:
+ if params.has_namenode:
+ params.HdfsResource("/ranger/audit",
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ group=params.hdfs_user,
+ mode=0755,
+ recursive_chmod=True
+ )
+ params.HdfsResource("/ranger/audit/knox",
+ type="directory",
+ action="create_on_execute",
+ owner=params.knox_user,
+ group=params.knox_user,
+ mode=0700,
+ recursive_chmod=True
+ )
+ params.HdfsResource(None, action="execute")
+
+ if params.namenode_hosts is not None and len(params.namenode_hosts) > 1:
+ Logger.info('Ranger Knox plugin is enabled in NameNode HA environment along with audit to Hdfs enabled, creating hdfs-site.xml')
+ XmlConfig("hdfs-site.xml",
+ conf_dir=params.knox_conf_dir,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.knox_user,
+ group=params.knox_group,
+ mode=0644
+ )
+ else:
+ File(format('{knox_conf_dir}/hdfs-site.xml'), action="delete")
+
+ if params.xml_configurations_supported:
+ api_version=None
+ if params.stack_supports_ranger_kerberos:
+ api_version='v2'
+ from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
+ setup_ranger_plugin('knox-server', 'knox', params.previous_jdbc_jar,
+ params.downloaded_custom_connector, params.driver_curl_source,
+ params.driver_curl_target, params.java_home,
+ params.repo_name, params.knox_ranger_plugin_repo,
+ params.ranger_env, params.ranger_plugin_properties,
+ params.policy_user, params.policymgr_mgr_url,
+ params.enable_ranger_knox, conf_dict=params.knox_conf_dir,
+ component_user=params.knox_user, component_group=params.knox_group, cache_service_list=['knox'],
+ plugin_audit_properties=params.config['configurations']['ranger-knox-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-knox-audit'],
+ plugin_security_properties=params.config['configurations']['ranger-knox-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-knox-security'],
+ plugin_policymgr_ssl_properties=params.config['configurations']['ranger-knox-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-knox-policymgr-ssl'],
+ component_list=['knox-server'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
+ credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
+ ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
+ stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble,api_version=api_version,
+ is_security_enabled = params.security_enabled,
+ is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
+ component_user_principal=params.knox_principal_name if params.security_enabled else None,
+ component_user_keytab=params.knox_keytab_path if params.security_enabled else None)
+ else:
+ from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
+ setup_ranger_plugin('knox-server', 'knox', params.previous_jdbc_jar,
+ params.downloaded_custom_connector, params.driver_curl_source,
+ params.driver_curl_target, params.java_home,
+ params.repo_name, params.knox_ranger_plugin_repo,
+ params.ranger_env, params.ranger_plugin_properties,
+ params.policy_user, params.policymgr_mgr_url,
+ params.enable_ranger_knox, conf_dict=params.knox_conf_dir,
+ component_user=params.knox_user, component_group=params.knox_group, cache_service_list=['knox'],
+ plugin_audit_properties=params.config['configurations']['ranger-knox-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-knox-audit'],
+ plugin_security_properties=params.config['configurations']['ranger-knox-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-knox-security'],
+ plugin_policymgr_ssl_properties=params.config['configurations']['ranger-knox-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-knox-policymgr-ssl'],
+ component_list=['knox-server'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
+ credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
+ ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
+ stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble)
+ if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_knox and params.has_namenode and params.security_enabled:
+ Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating core-site.xml from namenode configuraitions")
+ setup_core_site_for_required_plugins(component_user=params.knox_user, component_group=params.knox_group,create_core_site_path = params.knox_conf_dir, config = params.config)
+ else:
+ Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations")
+
+ else:
+ Logger.info('Ranger Knox plugin is not enabled')
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/status_params.py
new file mode 100644
index 0000000..3cbd920
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/status_params.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.script.script import Script
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
+
+
+config = Script.get_config()
+stack_root = Script.get_stack_root()
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+if OSCheck.is_windows_family():
+ knox_gateway_win_service_name = "gateway"
+ knox_ldap_win_service_name = "ldap"
+else:
+ knox_conf_dir = '/etc/knox/conf'
+ if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ knox_conf_dir = format('{stack_root}/current/knox-server/conf')
+ knox_pid_dir = config['configurations']['knox-env']['knox_pid_dir']
+ knox_pid_file = format("{knox_pid_dir}/gateway.pid")
+ ldap_pid_file = format("{knox_pid_dir}/ldap.pid")
+
+ security_enabled = config['configurations']['cluster-env']['security_enabled']
+ if security_enabled:
+ knox_keytab_path = config['configurations']['knox-env']['knox_keytab_path']
+ knox_principal_name = config['configurations']['knox-env']['knox_principal_name']
+ else:
+ knox_keytab_path = None
+ knox_principal_name = None
+
+ hostname = config['hostname'].lower()
+ knox_user = default("/configurations/knox-env/knox_user", "knox")
+ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+ temp_dir = Script.get_tmp_dir()
+
+stack_name = default("/hostLevelParams/stack_name", None)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/upgrade.py
new file mode 100644
index 0000000..917f340
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/scripts/upgrade.py
@@ -0,0 +1,118 @@
+
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+import tempfile
+
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions import tar_archive
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.script.script import Script
+
+
+BACKUP_TEMP_DIR = "knox-upgrade-backup"
+BACKUP_DATA_ARCHIVE = "knox-data-backup.tar"
+STACK_ROOT_DEFAULT = Script.get_stack_root()
+
+def backup_data():
+ """
+ Backs up the knox data as part of the upgrade process.
+ :return: Returns the path to the absolute backup directory.
+ """
+ Logger.info('Backing up Knox data directory before upgrade...')
+ directoryMappings = _get_directory_mappings_during_upgrade()
+
+ Logger.info("Directory mappings to backup: {0}".format(str(directoryMappings)))
+
+ absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
+ if not os.path.isdir(absolute_backup_dir):
+ os.makedirs(absolute_backup_dir)
+
+ for directory in directoryMappings:
+ if not os.path.isdir(directory):
+ raise Fail("Unable to backup missing directory {0}".format(directory))
+
+ archive = os.path.join(absolute_backup_dir, directoryMappings[directory])
+ Logger.info('Compressing {0} to {1}'.format(directory, archive))
+
+ if os.path.exists(archive):
+ os.remove(archive)
+
+ # backup the directory, following symlinks instead of including them
+ tar_archive.archive_directory_dereference(archive, directory)
+
+ return absolute_backup_dir
+
+
+def seed_current_data_directory():
+ """
+ HDP stack example:
+
+ Knox uses "versioned" data directories in some stacks:
+ /usr/hdp/2.2.0.0-1234/knox/data -> /var/lib/knox/data
+ /usr/hdp/2.3.0.0-4567/knox/data -> /var/lib/knox/data-2.3.0.0-4567
+
+ If the stack being upgraded to supports versioned data directories for Knox, then we should
+ seed the data from the prior version. This is mainly because Knox keeps things like keystores
+ in the data directory and if those aren't copied over then it will re-create self-signed
+ versions. This side-effect behavior causes loss of service in clusters where Knox is using
+ custom keystores.
+
+ cp -R -p -f /usr/hdp/<old>/knox-server/data/. /usr/hdp/current/knox-server/data
+ :return:
+ """
+ import params
+
+ if params.version is None or params.upgrade_from_version is None:
+ raise Fail("The source and target versions are required")
+
+ if check_stack_feature(StackFeature.KNOX_VERSIONED_DATA_DIR, params.version):
+ Logger.info("Seeding Knox data from prior version...")
+
+ # <stack-root>/2.3.0.0-1234/knox/data/.
+ source_data_dir = os.path.join(params.stack_root, params.upgrade_from_version, "knox", "data", ".")
+
+ # <stack-root>/current/knox-server/data
+ target_data_dir = os.path.join(params.stack_root, "current", "knox-server", "data")
+
+ # recursive copy, overwriting, and preserving attributes
+ Execute(("cp", "-R", "-p", "-f", source_data_dir, target_data_dir), sudo = True)
+
+
+def _get_directory_mappings_during_upgrade():
+ """
+ Gets a dictionary of directory to archive name that represents the
+ directories that need to be backed up and their output tarball archive targets
+ :return: the dictionary of directory to tarball mappings
+ """
+ import params
+
+ # the data directory is always a symlink to the "correct" data directory in /var/lib/knox
+ # such as /var/lib/knox/data or /var/lib/knox/data-2.4.0.0-1234
+ knox_data_dir = STACK_ROOT_DEFAULT + '/current/knox-server/data'
+
+ directories = { knox_data_dir: BACKUP_DATA_ARCHIVE }
+
+ Logger.info(format("Knox directories to backup:\n{directories}"))
+ return directories
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/templates/input.config-knox.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/templates/input.config-knox.json.j2 b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/templates/input.config-knox.json.j2
new file mode 100644
index 0000000..6d7cf72
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/templates/input.config-knox.json.j2
@@ -0,0 +1,60 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"knox_gateway",
+ "rowtype":"service",
+ "path":"/var/log/knox/gateway.log"
+ },
+ {
+ "type":"knox_cli",
+ "rowtype":"service",
+ "path":"/var/log/knox/knoxcli.log"
+ },
+ {
+ "type":"knox_ldap",
+ "rowtype":"service",
+ "path":"/var/log/knox/ldap.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "knox_gateway",
+ "knox_cli",
+ "knox_ldap"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/templates/krb5JAASLogin.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/templates/krb5JAASLogin.conf.j2 b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/templates/krb5JAASLogin.conf.j2
new file mode 100644
index 0000000..fa3237b
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/package/templates/krb5JAASLogin.conf.j2
@@ -0,0 +1,30 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+com.sun.security.jgss.initiate {
+com.sun.security.auth.module.Krb5LoginModule required
+renewTGT=true
+doNotPrompt=true
+useKeyTab=true
+keyTab="{{knox_keytab_path}}"
+principal="{{knox_principal_name}}"
+isInitiator=true
+storeKey=true
+useTicketCache=true
+client=true;
+};
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/role_command_order.json b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/role_command_order.json
new file mode 100644
index 0000000..c0475e4
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.3.0/role_command_order.json
@@ -0,0 +1,7 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for KNOX",
+ "KNOX_GATEWAY-START" : ["RANGER_USERSYNC-START", "NAMENODE-START"],
+ "KNOX_SERVICE_CHECK-SERVICE_CHECK" : ["KNOX_GATEWAY-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9adffcf7/ambari-server/src/main/resources/stacks/HDP/3.0/services/KNOX/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/KNOX/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/KNOX/metainfo.xml
new file mode 100644
index 0000000..d8054b1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/KNOX/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>KNOX</name>
+ <version>0.5.0.3.0</version>
+ <extends>common-services/KNOX/0.5.0.3.0</extends>
+ </service>
+ </services>
+</metainfo>