You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2016/09/14 05:26:48 UTC
[1/8] ambari git commit: AMBARI-18385: Add HDF management pack
(jluniya)
Repository: ambari
Updated Branches:
refs/heads/trunk 532b689f3 -> 37e71db74
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/widgets.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/widgets.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/widgets.json
new file mode 100644
index 0000000..3176354
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/widgets.json
@@ -0,0 +1,95 @@
+{
+ "layouts": [
+ {
+ "layout_name": "default_system_heatmap",
+ "display_name": "Heatmaps",
+ "section_name": "SYSTEM_HEATMAPS",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "Host Disk Space Used %",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "disk_free",
+ "metric_path": "metrics/disk/disk_free",
+ "service_name": "STACK"
+ },
+ {
+ "name": "disk_total",
+ "metric_path": "metrics/disk/disk_total",
+ "service_name": "STACK"
+ }
+ ],
+ "values": [
+ {
+ "name": "Host Disk Space Used %",
+ "value": "${((disk_total-disk_free)/disk_total)*100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "Host Memory Used %",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "mem_total",
+ "metric_path": "metrics/memory/mem_total",
+ "service_name": "STACK"
+ },
+ {
+ "name": "mem_free",
+ "metric_path": "metrics/memory/mem_free",
+ "service_name": "STACK"
+ },
+ {
+ "name": "mem_cached",
+ "metric_path": "metrics/memory/mem_cached",
+ "service_name": "STACK"
+ }
+ ],
+ "values": [
+ {
+ "name": "Host Memory Used %",
+ "value": "${((mem_total-mem_free-mem_cached)/mem_total)*100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "Host CPU Wait IO %",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "cpu_wio",
+ "metric_path": "metrics/cpu/cpu_wio",
+ "service_name": "STACK"
+ }
+ ],
+ "values": [
+ {
+ "name": "Host Memory Used %",
+ "value": "${cpu_wio*100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/pom.xml b/contrib/management-packs/pom.xml
index c8acf6b..f5ec2bb 100644
--- a/contrib/management-packs/pom.xml
+++ b/contrib/management-packs/pom.xml
@@ -35,6 +35,7 @@
</properties>
<modules>
<module>microsoft-r_mpack</module>
+ <module>hdf-ambari-mpack</module>
</modules>
<build>
<pluginManagement>
@@ -72,6 +73,7 @@
<exclude>**/.tmp/**</exclude>
<exclude>**/tmp/**</exclude>
<exclude>**/*.json</exclude>
+ <exclude>**/fast-hdfs-resource.jar</exclude>
<exclude>**/*.svg</exclude>
<exclude>**/vendor/**</exclude>
<exclude>**/bin</exclude>
@@ -150,6 +152,7 @@
</activation>
<modules>
<module>microsoft-r_mpack</module>
+ <module>hdf-ambari-mpack</module>
</modules>
</profile>
</profiles>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fc6a2ccc..7ca4ba4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -301,6 +301,7 @@
<exclude>ambari-web/node_modules/**</exclude>
<!--Contributions-->
+ <exclude>contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/fast-hdfs-resource.jar</exclude>
<exclude>contrib/ambari-scom/management-pack/Hadoop_MP/**</exclude>
<exclude>contrib/addons/test/dataServices/jmx/data/cluster_configuration.json.nohbase</exclude>
<exclude>contrib/ambari-scom/msi/src/GUI_Ambari.sln</exclude>
[6/8] ambari git commit: AMBARI-18385: Add HDF management pack
(jluniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py
new file mode 100644
index 0000000..362a40b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys, nifi_ca_util, os, pwd, grp, signal, time, glob, socket, json
+from resource_management.core import sudo
+from resource_management import *
+from subprocess import call
+from setup_ranger_nifi import setup_ranger_nifi
+
+reload(sys)
+sys.setdefaultencoding('utf8')
+
+class Master(Script):
+ def install(self, env):
+
+ import params
+ import status_params
+
+ self.install_packages(env)
+
+ Directory([params.nifi_node_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ #update the configs specified by user
+ self.configure(env, True)
+
+ Execute('touch ' + params.nifi_node_log_file, user=params.nifi_user)
+
+ def configure(self, env, isInstall=False, is_starting = False):
+ import params
+ import status_params
+ env.set_params(params)
+ env.set_params(status_params)
+
+ #create the log, pid, conf dirs if not already present
+ Directory([status_params.nifi_pid_dir, params.nifi_node_log_dir, params.nifi_internal_dir, params.nifi_database_dir, params.nifi_flowfile_repo_dir, params.nifi_content_repo_dir_default, params.nifi_provenance_repo_dir_default, params.nifi_config_dir, params.nifi_flow_config_dir, params.nifi_state_dir, params.lib_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ # On some OS this folder may not exist, so we will create it before pushing files there
+ Directory(params.limits_conf_dir,
+ create_parents = True,
+ owner='root',
+ group='root'
+ )
+
+ File(os.path.join(params.limits_conf_dir, 'nifi.conf'),
+ owner='root',
+ group='root',
+ mode=0644,
+ content=Template("nifi.conf.j2")
+ )
+
+
+ ca_client_script = nifi_ca_util.get_toolkit_script('tls-toolkit.sh')
+ File(ca_client_script, mode=0755)
+
+
+ if params.nifi_ca_host and params.nifi_ssl_enabled:
+ ca_client_json = os.path.realpath(os.path.join(params.nifi_config_dir, 'nifi-certificate-authority-client.json'))
+ File(ca_client_json,
+ owner = params.nifi_user,
+ group = params.nifi_group,
+ mode = 0600)
+ ca_client_dict = nifi_ca_util.load(ca_client_json)
+ if is_starting:
+ if params.nifi_toolkit_tls_regenerate:
+ nifi_ca_util.move_keystore_truststore(ca_client_dict)
+ ca_client_dict = {}
+ else:
+ nifi_ca_util.move_keystore_truststore_if_necessary(ca_client_dict, params.nifi_ca_client_config)
+ nifi_ca_util.overlay(ca_client_dict, params.nifi_ca_client_config)
+ nifi_ca_util.dump(ca_client_json, ca_client_dict)
+ if is_starting:
+ Execute('JAVA_HOME='+params.jdk64_home+' '+ca_client_script+' client -F -f '+ca_client_json, user=params.nifi_user)
+ nifi_ca_util.update_nifi_properties(nifi_ca_util.load(ca_client_json), params.nifi_properties)
+
+ #write out nifi.properties
+ PropertiesFile(params.nifi_config_dir + '/nifi.properties',
+ properties = params.nifi_properties,
+ mode = 0400,
+ owner = params.nifi_user,
+ group = params.nifi_group)
+
+ #write out boostrap.conf
+ bootstrap_content=InlineTemplate(params.nifi_boostrap_content)
+ File(format("{params.nifi_config_dir}/bootstrap.conf"), content=bootstrap_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out logback.xml
+ logback_content=InlineTemplate(params.nifi_node_logback_content)
+ File(format("{params.nifi_config_dir}/logback.xml"), content=logback_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out state-management.xml
+ statemgmt_content=InlineTemplate(params.nifi_state_management_content)
+ File(format("{params.nifi_config_dir}/state-management.xml"), content=statemgmt_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out authorizers file
+ authorizers_content=InlineTemplate(params.nifi_authorizers_content)
+ File(format("{params.nifi_config_dir}/authorizers.xml"), content=authorizers_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out login-identity-providers.xml
+ login_identity_providers_content=InlineTemplate(params.nifi_login_identity_providers_content)
+ File(format("{params.nifi_config_dir}/login-identity-providers.xml"), content=login_identity_providers_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out nifi-env in bin as 0755 (see BUG-61769)
+ env_content=InlineTemplate(params.nifi_env_content)
+ File(format("{params.bin_dir}/nifi-env.sh"), content=env_content, owner=params.nifi_user, group=params.nifi_group, mode=0755)
+
+ #write out bootstrap-notification-services.xml
+ boostrap_notification_content=InlineTemplate(params.nifi_boostrap_notification_content)
+ File(format("{params.nifi_config_dir}/bootstrap-notification-services.xml"), content=boostrap_notification_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ def stop(self, env):
+ import params
+ import status_params
+
+ Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh stop >> ' + params.nifi_node_log_file, user=params.nifi_user)
+ if os.path.isfile(status_params.nifi_node_pid_file):
+ sudo.unlink(status_params.nifi_node_pid_file)
+
+ def start(self, env):
+ import params
+ import status_params
+ self.configure(env, is_starting = True)
+ setup_ranger_nifi(upgrade_type=None)
+
+ # Write out flow.xml.gz to internal dir only if AMS installed (must be writable by Nifi)
+ # only during first install. It is used to automate setup of Ambari metrics reporting task in Nifi
+ if params.metrics_collector_host and params.nifi_ambari_reporting_enabled and self.check_is_fresh_install(self):
+ Execute('echo "First time setup so generating flow.xml.gz" >> ' + params.nifi_node_log_file, user=params.nifi_user)
+ flow_content=InlineTemplate(params.nifi_flow_content)
+ File(format("{params.nifi_flow_config_dir}/flow.xml"), content=flow_content, owner=params.nifi_user, group=params.nifi_group, mode=0600)
+ Execute(format("cd {params.nifi_flow_config_dir}; mv flow.xml.gz flow_$(date +%d-%m-%Y).xml.gz ;"),user=params.nifi_user,ignore_failures=True)
+ Execute(format("cd {params.nifi_flow_config_dir}; gzip flow.xml;"), user=params.nifi_user)
+
+
+ Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh start >> ' + params.nifi_node_log_file, user=params.nifi_user)
+ #If nifi pid file not created yet, wait a bit
+ if not os.path.isfile(status_params.nifi_pid_dir+'/nifi.pid'):
+ Execute ('sleep 5')
+
+
+ def status(self, env):
+ import status_params
+ check_process_status(status_params.nifi_node_pid_file)
+
+
+ def check_is_fresh_install(self, env):
+ """
+ Checks if fresh nifi install by checking if zk dir exists
+ :return:
+ """
+ import params, re
+ from resource_management.core import shell
+ from resource_management.core.exceptions import Fail
+ from resource_management.core.logger import Logger
+
+ ZK_CONNECT_ERROR = "ConnectionLoss"
+ ZK_NODE_NOT_EXIST = "Node does not exist"
+
+ zookeeper_queried = False
+ is_fresh_nifi_install = True
+
+ # For every zk server try to find nifi zk dir
+ zookeeper_server_list = params.config['clusterHostInfo']['zookeeper_hosts']
+ for zookeeper_server in zookeeper_server_list:
+ # Determine where the zkCli.sh shell script is
+ zk_command_location = os.path.join(params.stack_root, "current", "zookeeper-client", "bin", "zkCli.sh")
+ if params.stack_version_buildnum is not None:
+ zk_command_location = os.path.join(params.stack_root, params.stack_version_buildnum, "zookeeper", "bin", "zkCli.sh")
+
+ # create the ZooKeeper query command e.g.
+ # /usr/hdf/current/zookeeper-client/bin/zkCli.sh -server node:2181 ls /nifi
+ command = "{0} -server {1}:{2} ls {3}".format(
+ zk_command_location, zookeeper_server, params.zookeeper_port, params.nifi_znode)
+
+ # echo 'ls /nifi' | /usr/hdf/current/zookeeper-client/bin/zkCli.sh -server node:2181
+ #command = "echo 'ls {3}' | {0} -server {1}:{2}".format(
+ # zk_command_location, zookeeper_server, params.zookeeper_port, params.nifi_znode)
+
+ Logger.info("Running command: " + command)
+
+ code, out = shell.call(command, logoutput=True, quiet=False, timeout=20)
+ if not out or re.search(ZK_CONNECT_ERROR, out):
+ Logger.info("Unable to query Zookeeper: " + zookeeper_server + ". Skipping and trying next ZK server")
+ continue
+ elif re.search(ZK_NODE_NOT_EXIST, out):
+ Logger.info("Nifi ZNode does not exist, so must be fresh install of Nifi: " + params.nifi_znode)
+ zookeeper_queried = True
+ is_fresh_nifi_install = True
+ break
+ else:
+ Logger.info("Nifi ZNode already exists, so must not be a fresh install of Nifi: " + params.nifi_znode)
+ zookeeper_queried = True
+ is_fresh_nifi_install = False
+ break
+
+ # fail if the ZK data could not be queried
+ if not zookeeper_queried:
+ raise Fail("Unable to query for znode on on any of the following ZooKeeper hosts: {0}. Please ensure Zookeepers are started and retry".format(
+ zookeeper_server_list))
+ else:
+ return is_fresh_nifi_install
+
+
+if __name__ == "__main__":
+ Master().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py
new file mode 100644
index 0000000..bc343e4
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import nifi_ca_util, os, time
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.resources.system import Directory, Execute
+from resource_management.core.sudo import kill, read_file, path_isfile, unlink
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources import File
+from signal import SIGTERM, SIGKILL
+
+class CertificateAuthority(Script):
+ def install(self, env):
+ import params
+ import status_params
+
+ self.install_packages(env)
+
+ #Be sure ca script is in cache
+ nifi_ca_util.get_toolkit_script('tls-toolkit.sh')
+
+ def configure(self, env):
+ import params
+ import status_params
+ env.set_params(params)
+ env.set_params(status_params)
+
+ #create the log, pid, conf dirs if not already present
+ Directory([status_params.nifi_pid_dir, params.nifi_node_log_dir, params.nifi_config_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True
+ )
+
+ ca_json = os.path.join(params.nifi_config_dir, 'nifi-certificate-authority.json')
+ ca_dict = nifi_ca_util.load(ca_json)
+ nifi_ca_util.overlay(ca_dict, params.nifi_ca_config)
+ nifi_ca_util.dump(ca_json, ca_dict)
+
+ Directory([params.nifi_config_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ def invalidate_ca_server(self, env):
+ import params
+ ca_json = os.path.join(params.nifi_config_dir, 'nifi-certificate-authority.json')
+ nifi_ca_util.move_store(nifi_ca_util.load(ca_json), 'keyStore')
+ unlink(ca_json)
+
+ def status(self, env):
+ import status_params
+ check_process_status(status_params.nifi_ca_pid_file)
+
+ def start(self, env):
+ import params
+ import status_params
+
+ self.configure(env)
+ ca_server_script = nifi_ca_util.get_toolkit_script('tls-toolkit.sh')
+ run_ca_script = os.path.join(os.path.dirname(__file__), 'run_ca.sh')
+ Directory([params.nifi_config_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ File(ca_server_script, mode=0755)
+ File(run_ca_script, mode=0755)
+ Execute((run_ca_script, params.jdk64_home, ca_server_script, params.nifi_config_dir + '/nifi-certificate-authority.json', params.nifi_ca_log_file_stdout, params.nifi_ca_log_file_stderr, status_params.nifi_ca_pid_file), user=params.nifi_user)
+ if not os.path.isfile(status_params.nifi_ca_pid_file):
+ raise Exception('Expected pid file to exist')
+
+ def stop(self, env):
+ import status_params
+
+ if path_isfile(status_params.nifi_ca_pid_file):
+ try:
+ self.status(env)
+ pid = int(read_file(status_params.nifi_ca_pid_file))
+ for i in range(25):
+ kill(pid, SIGTERM)
+ time.sleep(1)
+ self.status(env)
+ kill(pid, SIGKILL)
+ time.sleep(5)
+ self.status(env)
+ except ComponentIsNotRunning:
+ unlink(status_params.nifi_ca_pid_file)
+
+if __name__ == "__main__":
+ CertificateAuthority().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py
new file mode 100644
index 0000000..6807491
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import json, nifi_constants, os
+from resource_management.core import sudo
+from resource_management.core.resources.system import File
+
+script_dir = os.path.dirname(__file__)
+files_dir = os.path.realpath(os.path.join(os.path.dirname(script_dir), 'files'))
+
+def load(config_json):
+ if sudo.path_isfile(config_json):
+ contents = sudo.read_file(config_json)
+ if len(contents) > 0:
+ return json.loads(contents)
+ return {}
+
+def dump(config_json, config_dict):
+ import params
+ File(config_json,
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ mode=0600,
+ content=json.dumps(config_dict, sort_keys=True, indent=4)
+ )
+
+def overlay(config_dict, overlay_dict):
+ for k, v in overlay_dict.iteritems():
+ if v or k not in config_dict:
+ config_dict[k] = v
+
+def get_toolkit_script(scriptName, scriptDir = files_dir):
+ nifiToolkitDir = None
+ for dir in os.listdir(scriptDir):
+ if dir.startswith('nifi-toolkit-'):
+ nifiToolkitDir = os.path.join(scriptDir, dir)
+
+ if nifiToolkitDir is None:
+ raise Exception("Couldn't find nifi toolkit directory in " + scriptDir)
+ result = nifiToolkitDir + '/bin/' + scriptName
+ if not sudo.path_isfile(result):
+ raise Exception("Couldn't find file " + result)
+ return result
+
+def update_nifi_properties(client_dict, nifi_properties):
+ nifi_properties[nifi_constants.NIFI_SECURITY_KEYSTORE_TYPE] = client_dict['keyStoreType']
+ nifi_properties[nifi_constants.NIFI_SECURITY_KEYSTORE_PASSWD] = client_dict['keyStorePassword']
+ nifi_properties[nifi_constants.NIFI_SECURITY_KEY_PASSWD] = client_dict['keyPassword']
+ nifi_properties[nifi_constants.NIFI_SECURITY_TRUSTSTORE_TYPE] = client_dict['trustStoreType']
+ nifi_properties[nifi_constants.NIFI_SECURITY_TRUSTSTORE_PASSWD] = client_dict['trustStorePassword']
+
+def store_exists(client_dict, key):
+ if key not in client_dict:
+ return False
+ return sudo.path_isfile(client_dict[key])
+
+def different(one, two, key):
+ if key not in one:
+ return False
+ if len(one[key]) == 0:
+ return False
+ if key not in two:
+ return False
+ if len(two[key]) == 0:
+ return False
+ return one[key] != two[key]
+
+def move_keystore_truststore_if_necessary(orig_client_dict, new_client_dict):
+ if not (store_exists(new_client_dict, 'keyStore') or store_exists(new_client_dict, 'trustStore')):
+ return
+ if different(orig_client_dict, new_client_dict, 'keyStoreType'):
+ move_keystore_truststore(new_client_dict)
+ elif different(orig_client_dict, new_client_dict, 'keyStorePassword'):
+ move_keystore_truststore(new_client_dict)
+ elif different(orig_client_dict, new_client_dict, 'keyPassword'):
+ move_keystore_truststore(new_client_dict)
+ elif different(orig_client_dict, new_client_dict, 'trustStoreType'):
+ move_keystore_truststore(new_client_dict)
+ elif different(orig_client_dict, new_client_dict, 'trustStorePassword'):
+ move_keystore_truststore(new_client_dict)
+
+def move_keystore_truststore(client_dict):
+ move_store(client_dict, 'keyStore')
+ move_store(client_dict, 'trustStore')
+
+def move_store(client_dict, key):
+ if store_exists(client_dict, key):
+ num = 0
+ name = client_dict[key]
+ while sudo.path_isfile(name + '.bak.' + str(num)):
+ num += 1
+ sudo.copy(name, name + '.bak.' + str(num))
+ sudo.unlink(name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py
new file mode 100644
index 0000000..632a473
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+NIFI_SECURITY_KEY_PASSWD = 'nifi.security.keyPasswd'
+NIFI_SECURITY_KEYSTORE_PASSWD = 'nifi.security.keystorePasswd'
+NIFI_SECURITY_KEYSTORE_TYPE = 'nifi.security.keystoreType'
+NIFI_SECURITY_TRUSTSTORE_PASSWD = 'nifi.security.truststorePasswd'
+NIFI_SECURITY_TRUSTSTORE_TYPE = 'nifi.security.truststoreType'
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py
new file mode 100644
index 0000000..087cef4
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py
@@ -0,0 +1,445 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.libraries.script.script import Script
+import sys, os, glob, socket, re
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set
+
+# server configurations
+config = Script.get_config()
+stack_root = Script.get_stack_root()
+stack_version_buildnum = default("/commandParams/version", None)
+
+#nifi_install_dir = '/usr/hdf/current/nifi'
+nifi_install_dir = os.path.join(stack_root, "current", "nifi")
+if stack_version_buildnum is not None:
+ nifi_install_dir = os.path.join(stack_root, stack_version_buildnum, "nifi")
+
+# params from nifi-ambari-config
+nifi_initial_mem = config['configurations']['nifi-ambari-config']['nifi.initial_mem']
+nifi_max_mem = config['configurations']['nifi-ambari-config']['nifi.max_mem']
+nifi_ambari_reporting_frequency = config['configurations']['nifi-ambari-config']['nifi.ambari_reporting_frequency']
+nifi_ambari_reporting_enabled = config['configurations']['nifi-ambari-config']['nifi.ambari_reporting_enabled']
+
+# note: nifi.node.port and nifi.node.ssl.port must be defined in same xml file for quicklinks to work
+nifi_node_port = config['configurations']['nifi-ambari-config']['nifi.node.port']
+nifi_node_ssl_port = config['configurations']['nifi-ambari-config']['nifi.node.ssl.port']
+nifi_node_protocol_port = config['configurations']['nifi-ambari-config']['nifi.node.protocol.port']
+
+nifi_znode = config['configurations']['nifi-ambari-config']['nifi.nifi_znode']
+
+nifi_internal_dir=config['configurations']['nifi-ambari-config']['nifi.internal.dir']
+nifi_state_dir=config['configurations']['nifi-ambari-config']['nifi.state.dir']
+nifi_database_dir=config['configurations']['nifi-ambari-config']['nifi.database.dir']
+nifi_flowfile_repo_dir=config['configurations']['nifi-ambari-config']['nifi.flowfile.repository.dir']
+nifi_content_repo_dir_default=config['configurations']['nifi-ambari-config']['nifi.content.repository.dir.default']
+nifi_provenance_repo_dir_default=config['configurations']['nifi-ambari-config']['nifi.provenance.repository.dir.default']
+nifi_config_dir = config['configurations']['nifi-ambari-config']['nifi.config.dir']
+nifi_flow_config_dir = config['configurations']['nifi-ambari-config']['nifi.flow.config.dir']
+nifi_sensitive_props_key = config['configurations']['nifi-ambari-config']['nifi.sensitive.props.key']
+
+
+nifi_flow_config_dir = nifi_flow_config_dir.replace('{nifi_internal_dir}',nifi_internal_dir)
+nifi_state_dir = nifi_state_dir.replace('{nifi_internal_dir}',nifi_internal_dir)
+nifi_config_dir = nifi_config_dir.replace('{nifi_install_dir}',nifi_install_dir)
+
+master_configs = config['clusterHostInfo']
+nifi_master_hosts = master_configs['nifi_master_hosts']
+
+# detect if running in single (sandbox) box
+nifi_num_nodes = len(master_configs['nifi_master_hosts'])
+#if nifi_num_nodes > 1:
+# nifi_is_node='true'
+#else:
+# nifi_is_node='false'
+#nifi_node_hosts = ",".join(master_configs['nifi_master_hosts'])
+
+# In sandbox scenario, Ambari should still setup nifi in clustered mode for now
+nifi_is_node='true'
+
+nifi_node_dir=nifi_install_dir
+bin_dir = os.path.join(*[nifi_node_dir,'bin'])
+lib_dir = os.path.join(*[nifi_node_dir,'lib'])
+
+nifi_ca_host = None
+if 'nifi_ca_hosts' in master_configs:
+ nifi_ca_hosts = master_configs['nifi_ca_hosts']
+ if len(nifi_ca_hosts) > 0:
+ nifi_ca_host = nifi_ca_hosts[0]
+
+# params from nifi-ambari-ssl-config
+
+nifi_ssl_enabled = config['configurations']['nifi-ambari-ssl-config']['nifi.node.ssl.isenabled']
+nifi_keystore = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystore']
+nifi_keystoreType = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystoreType']
+nifi_keystorePasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystorePasswd']
+nifi_keyPasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keyPasswd']
+nifi_truststore = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststore']
+nifi_truststoreType = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststoreType']
+nifi_truststorePasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststorePasswd']
+nifi_needClientAuth = config['configurations']['nifi-ambari-ssl-config']['nifi.security.needClientAuth']
+nifi_initial_admin_id = config['configurations']['nifi-ambari-ssl-config']['nifi.initial.admin.identity']
+nifi_ssl_config_content = config['configurations']['nifi-ambari-ssl-config']['content']
+
+#property that is set to hostname regardless of whether SSL enabled
+nifi_node_host = socket.getfqdn()
+
+nifi_truststore = nifi_truststore.replace('{nifi_node_ssl_host}',nifi_node_host)
+nifi_keystore = nifi_keystore.replace('{nifi_node_ssl_host}',nifi_node_host)
+
+#populate properties whose values depend on whether SSL enabled
+nifi_keystore = nifi_keystore.replace('{{nifi_config_dir}}',nifi_config_dir)
+nifi_truststore = nifi_truststore.replace('{{nifi_config_dir}}',nifi_config_dir)
+
+if nifi_ssl_enabled:
+ nifi_node_ssl_host = nifi_node_host
+ nifi_node_port = ""
+else:
+ nifi_node_nonssl_host = nifi_node_host
+ nifi_node_ssl_port = ""
+
+nifi_ca_parent_config = config['configurations']['nifi-ambari-ssl-config']
+nifi_use_ca = nifi_ca_parent_config['nifi.toolkit.tls.token']
+nifi_toolkit_dn_prefix = nifi_ca_parent_config['nifi.toolkit.dn.prefix']
+nifi_toolkit_dn_suffix = nifi_ca_parent_config['nifi.toolkit.dn.suffix']
+nifi_toolkit_tls_regenerate = nifi_ca_parent_config['nifi.toolkit.tls.regenerate']
+nifi_ca_log_file_stdout = config['configurations']['nifi-env']['nifi_node_log_dir'] + '/nifi-ca.stdout'
+nifi_ca_log_file_stderr = config['configurations']['nifi-env']['nifi_node_log_dir'] + '/nifi-ca.stderr'
+
+nifi_ca_config = {
+ "days" : int(nifi_ca_parent_config['nifi.toolkit.tls.helper.days']),
+ "keyStore" : nifi_config_dir + '/nifi-certificate-authority-keystore.jks',
+ "token" : nifi_ca_parent_config['nifi.toolkit.tls.token'],
+ "caHostname" : nifi_ca_host,
+ "port" : int(nifi_ca_parent_config['nifi.toolkit.tls.port'])
+}
+
+if nifi_ca_host:
+ nifi_ca_config['dn'] = nifi_toolkit_dn_prefix + nifi_ca_host + nifi_toolkit_dn_suffix
+
+nifi_ca_client_config = {
+ "days" : int(nifi_ca_parent_config['nifi.toolkit.tls.helper.days']),
+ "keyStore" : nifi_keystore,
+ "keyStoreType" : nifi_keystoreType,
+ "keyStorePassword" : nifi_keystorePasswd,
+ "keyPassword" : nifi_keyPasswd,
+ "token" : nifi_ca_parent_config['nifi.toolkit.tls.token'],
+ "dn" : nifi_toolkit_dn_prefix + nifi_node_host + nifi_toolkit_dn_suffix,
+ "port" : int(nifi_ca_parent_config['nifi.toolkit.tls.port']),
+ "caHostname" : nifi_ca_host,
+ "trustStore" : nifi_truststore,
+ "trustStoreType" : nifi_truststoreType,
+ "trustStorePassword": nifi_truststorePasswd
+}
+
+# params from nifi-env
+nifi_user = config['configurations']['nifi-env']['nifi_user']
+nifi_group = config['configurations']['nifi-env']['nifi_group']
+
+nifi_node_log_dir = config['configurations']['nifi-env']['nifi_node_log_dir']
+nifi_node_log_file = os.path.join(nifi_node_log_dir,'nifi-setup.log')
+
+# limits related params
+limits_conf_dir = '/etc/security/limits.d'
+nifi_user_nofile_limit = config['configurations']['nifi-env']['nifi_user_nofile_limit']
+nifi_user_nproc_limit = config['configurations']['nifi-env']['nifi_user_nproc_limit']
+
+# params from nifi-boostrap
+nifi_env_content = config['configurations']['nifi-env']['content']
+
+
+# params from nifi-logback
+nifi_master_logback_content = config['configurations']['nifi-master-logback-env']['content']
+nifi_node_logback_content = config['configurations']['nifi-node-logback-env']['content']
+
+# params from nifi-properties-env
+nifi_master_properties_content = config['configurations']['nifi-master-properties-env']['content']
+nifi_properties = config['configurations']['nifi-properties'].copy()
+
+#kerberos params
+nifi_kerberos_authentication_expiration = config['configurations']['nifi-properties']['nifi.kerberos.spnego.authentication.expiration']
+nifi_kerberos_realm = default("/configurations/kerberos-env/realm", None)
+
+# params from nifi-flow
+nifi_flow_content = config['configurations']['nifi-flow-env']['content']
+
+# params from nifi-state-management-env
+nifi_state_management_content = config['configurations']['nifi-state-management-env']['content']
+
+# params from nifi-authorizers-env
+nifi_authorizers_content = config['configurations']['nifi-authorizers-env']['content']
+
+# params from nifi-login-identity-providers-env
+nifi_login_identity_providers_content = config['configurations']['nifi-login-identity-providers-env']['content']
+
+# params from nifi-boostrap
+nifi_boostrap_content = config['configurations']['nifi-bootstrap-env']['content']
+
+# params from nifi-bootstrap-notification-services-env
+nifi_boostrap_notification_content = config['configurations']['nifi-bootstrap-notification-services-env']['content']
+
+#autodetect jdk home
+jdk64_home=config['hostLevelParams']['java_home']
+
+#autodetect ambari server for metrics
+if 'metrics_collector_hosts' in config['clusterHostInfo']:
+ metrics_collector_host = str(config['clusterHostInfo']['metrics_collector_hosts'][0])
+ metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address']))
+else:
+ metrics_collector_host = ''
+ metrics_collector_port = ''
+
+
+#detect zookeeper_quorum
+zookeeper_port=default('/configurations/zoo.cfg/clientPort', None)
+#get comma separated list of zookeeper hosts from clusterHostInfo
+index = 0
+zookeeper_quorum=""
+for host in config['clusterHostInfo']['zookeeper_hosts']:
+ zookeeper_quorum += host + ":"+str(zookeeper_port)
+ index += 1
+ if index < len(config['clusterHostInfo']['zookeeper_hosts']):
+ zookeeper_quorum += ","
+
+
+#setup ranger configuration
+
+retryAble = default("/commandParams/command_retry_enabled", False)
+version = default("/commandParams/version", None)
+namenode_hosts = default("/clusterHostInfo/namenode_host", None)
+
+if type(namenode_hosts) is list:
+ namenode_host = namenode_hosts[0]
+else:
+ namenode_host = namenode_hosts
+
+has_namenode = not namenode_host == None
+
+
+nifi_authorizer = 'file-provider'
+
+nifi_host_name = config['hostname']
+nifi_host_port = config['configurations']['nifi-ambari-config']['nifi.node.port']
+java_home = config['hostLevelParams']['java_home']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+if security_enabled:
+ _hostname_lowercase = nifi_host_name.lower()
+ nifi_properties['nifi.kerberos.service.principal'] = nifi_properties['nifi.kerberos.service.principal'].replace('_HOST',_hostname_lowercase)
+ nifi_properties['nifi.kerberos.spnego.principal'] = nifi_properties['nifi.kerberos.spnego.principal'].replace('_HOST',_hostname_lowercase)
+
+# ranger host
+# E.g., 2.3
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted)
+stack_supports_ranger_audit_db = stack_version_formatted and check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, stack_version_formatted)
+
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+# ranger nifi properties
+policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+
+if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+
+xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
+xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
+xa_db_host = config['configurations']['admin-properties']['db_host']
+repo_name = str(config['clusterName']) + '_nifi'
+
+repo_config_username = config['configurations']['ranger-nifi-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+
+ranger_env = config['configurations']['ranger-env']
+ranger_plugin_properties = config['configurations']['ranger-nifi-plugin-properties']
+policy_user = config['configurations']['ranger-nifi-plugin-properties']['policy_user']
+
+#For curl command in ranger plugin to get db connector
+jdk_location = config['hostLevelParams']['jdk_location']
+java_share_dir = '/usr/share/java'
+
+if has_ranger_admin:
+ enable_ranger_nifi = (config['configurations']['ranger-nifi-plugin-properties']['ranger-nifi-plugin-enabled'].lower() == 'yes')
+ xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+ repo_config_password = unicode(config['configurations']['ranger-nifi-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
+ xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
+ previous_jdbc_jar_name= None
+
+ if stack_supports_ranger_audit_db:
+ if xa_audit_db_flavor == 'mysql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "com.mysql.jdbc.Driver"
+ elif xa_audit_db_flavor == 'oracle':
+ jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
+ colon_count = xa_db_host.count(':')
+ if colon_count == 2 or colon_count == 0:
+ audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
+ else:
+ audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
+ jdbc_driver = "oracle.jdbc.OracleDriver"
+ elif xa_audit_db_flavor == 'postgres':
+ jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "org.postgresql.Driver"
+ elif xa_audit_db_flavor == 'mssql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
+ jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+ elif xa_audit_db_flavor == 'sqla':
+ jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
+ jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
+
+ downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+
+ driver_curl_target = format("{stack_root}/current/nifi/ext/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ previous_jdbc_jar = format("{stack_root}/current/nifi/ext/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ sql_connector_jar = ''
+
+ ssl_keystore_password = unicode(config['configurations']['ranger-nifi-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
+ ssl_truststore_password = unicode(config['configurations']['ranger-nifi-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
+ credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
+ credential_file_type = 'jceks'
+ ranger_admin_username = config['configurations']['ranger-env']['ranger_admin_username']
+ ranger_admin_password = config['configurations']['ranger-env']['ranger_admin_password']
+
+ #create ranger service's nifi client properties
+ nifi_authentication = config['configurations']['ranger-nifi-plugin-properties']['nifi.authentication']
+ ranger_id_owner_for_certificate = config['configurations']['ranger-nifi-plugin-properties']['owner.for.certificate']
+ nifi_id_owner_for_certificate = config['configurations']['ranger-nifi-policymgr-ssl']['owner.for.certificate']
+ regex = r"(CN)=([a-zA-Z0-9\.\-\* ]*)"
+ match = re.search(regex, nifi_id_owner_for_certificate)
+ common_name_for_certificate = match.group(2) if match else 'NONE'
+
+ if nifi_authentication == 'SSL':
+
+ nifi_ranger_plugin_config = {
+ 'nifi.authentication': nifi_authentication,
+ 'nifi.url': format("https://{nifi_host_name}:{nifi_node_ssl_port}/nifi-api/resources"),
+ 'nifi.ssl.keystore': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystore'],
+ 'nifi.ssl.keystoreType':config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystoreType'],
+ 'nifi.ssl.keystorePassword': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystorePassword'],
+ 'nifi.ssl.truststore': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststore'],
+ 'nifi.ssl.truststoreType': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststoreType'],
+ 'nifi.ssl.truststorePassword': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststorePassword'],
+ 'commonNameForCertificate': common_name_for_certificate
+ }
+ else:
+ nifi_ranger_plugin_config = {
+ 'nifi.authentication': nifi_authentication,
+ 'nifi.url': format("https://{nifi_host_name}:{nifi_host_port}/nifi-api/resources"),
+ 'commonNameForCertificate': common_name_for_certificate
+ }
+
+ nifi_ranger_plugin_repo = {
+ 'isActive': 'true',
+ 'config': json.dumps(nifi_ranger_plugin_config),
+ 'description': 'nifi repo',
+ 'name': repo_name,
+ 'repositoryType': 'nifi',
+ 'assetType': '5'
+ }
+
+ # used in nifi authorizers
+ ranger_admin_identity = ranger_id_owner_for_certificate
+
+ if stack_supports_ranger_kerberos and security_enabled:
+ nifi_ranger_plugin_config['policy.download.auth.users'] = nifi_user
+ nifi_ranger_plugin_config['tag.download.auth.users'] = nifi_user
+ ranger_nifi_principal = config['configurations']['nifi-properties']['nifi.kerberos.service.principal'].replace('_HOST',_hostname_lowercase)
+ ranger_nifi_keytab = config['configurations']['nifi-properties']['nifi.kerberos.service.keytab.location']
+
+ if stack_supports_ranger_kerberos:
+ nifi_ranger_plugin_config['ambari.service.check.user'] = policy_user
+
+ nifi_ranger_plugin_repo = {
+ 'isEnabled': 'true',
+ 'configs': nifi_ranger_plugin_config,
+ 'description': 'nifi repo',
+ 'name': repo_name,
+ 'type': 'nifi'
+ }
+
+ xa_audit_db_is_enabled = False
+ ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
+
+ if xml_configurations_supported and stack_supports_ranger_audit_db:
+ xa_audit_db_is_enabled = config['configurations']['ranger-nifi-audit']['xasecure.audit.destination.db']
+
+ xa_audit_hdfs_is_enabled = default('/configurations/ranger-nifi-audit/xasecure.audit.destination.hdfs', False)
+
+
+ #For SQLA explicitly disable audit to DB for Ranger
+ if xa_audit_db_flavor == 'sqla':
+ xa_audit_db_is_enabled = False
+
+ nifi_authorizer = 'ranger-provider'
+
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None
+hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None
+default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources()
+)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh
new file mode 100644
index 0000000..c5863fa
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Launches NiFi CA server
+# $1 -> JAVA_HOME
+# $2 -> tls-toolkit.sh path
+# $3 -> config json
+# $4 -> stdout log
+# $5 -> stderr log
+# $6 -> pid file
+
+die() {
+ echo "$1"
+ exit 1
+}
+
+read -r -d '' WAIT_FOR_LOG << 'EOF'
+ STDOUT_LOG="$0"
+ until [ -f "$STDOUT_LOG" ]; do
+ echo "Waiting for $STDOUT_LOG to exist"
+ sleep 1;
+ done
+EOF
+
+read -r -d '' WAIT_FOR_CHILD << 'EOF'
+ SHELL_PID="$0"
+ PID_FILE="$1"
+ CHILD_PROCESS="$(pgrep -P "$SHELL_PID" java)"
+ while [ -z "$CHILD_PROCESS" ]; do
+ echo "Waiting for child java process to exist"
+ sleep 1;
+ CHILD_PROCESS="$(pgrep -P "$SHELL_PID" java)"
+ done
+ echo "$CHILD_PROCESS" > "$PID_FILE"
+EOF
+
+JAVA_HOME="$1" nohup "$2" server -F -f "$3" > "$4" 2> "$5" < /dev/null &
+SHELL_PID="$!"
+
+timeout 30 bash -c "$WAIT_FOR_LOG" "$4" || die "Timed out while waiting for $4"
+
+timeout 30 bash -c "$WAIT_FOR_CHILD" "$SHELL_PID" "$6" || die "Timed out while waiting for child java process to exist"
+
+#Want to wait until Jetty starts
+#See http://superuser.com/questions/270529/monitoring-a-file-until-a-string-is-found#answer-900134
+( tail -f -n +1 "$4" & ) | timeout 180 grep -q "Server Started" || die "Timed out while waiting for CA server to start"
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py
new file mode 100644
index 0000000..535592f
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import urllib2
+import httplib
+
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script.script import Script
+
+from ambari_commons.inet_utils import openurl
+from ambari_commons.exceptions import TimeoutError
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions.decorator import retry
+
+class NifiServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ Logger.info("Running Nifi service check")
+ for nifi_master_host in params.nifi_master_hosts:
+ url = ""
+ if params.nifi_ssl_enabled:
+ url = "https://{0}:{1}/nifi".format(nifi_master_host, params.nifi_node_ssl_port)
+ else:
+ url = "http://{0}:{1}/nifi".format(nifi_master_host, params.nifi_node_port)
+ Logger.info("Checking Nifi portal {0} status".format(url))
+ NifiServiceCheck.check_nifi_portal(url)
+
+ @staticmethod
+ @retry(times=15, sleep_time=5, max_sleep_time=20, backoff_factor=2, err_class=Fail)
+ def check_nifi_portal(url):
+ try:
+ request = urllib2.Request(url)
+ result = openurl(request, timeout=20)
+ response_code = result.getcode()
+ if response_code == 200 or response_code == 401:
+ Logger.info("Nifi portal {0} is up. Response code {1}".format(url, response_code))
+ else:
+ raise Fail("Error connecting to {0}. Response code {1}".format(url, response_code))
+ except urllib2.URLError, e:
+ if isinstance(e, urllib2.HTTPError):
+ if e.code == 401:
+ Logger.info("Nifi portal {0} is up. Response code {1}".format(url, e.code))
+ else:
+ raise Fail("Error connecting to {0}. Http status code - {1}. \n {2}".format(url, e.code, e.read()))
+ elif e.reason and "violation of protocol" in str(e.reason):
+ Logger.info("Ignore certificate validation error - {0}".format(e.reason))
+ pass
+ else:
+ raise Fail("Error connecting to {0}. Reason - {1}.".format(url, e.reason))
+ except httplib.BadStatusLine:
+ raise Fail("Error connecting to {0}. Reason - Not Reachable".format(url))
+ except TimeoutError:
+ raise Fail("Error connecting to {0}. Reason - Timeout".format(url))
+
+if __name__ == "__main__":
+ NifiServiceCheck().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py
new file mode 100644
index 0000000..91a6885
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.core.logger import Logger
+from resource_management.core.resources import File
+
+def setup_ranger_nifi(upgrade_type=None):
+ import params, os
+
+ if params.has_ranger_admin and params.enable_ranger_nifi:
+
+ stack_version = None
+ if upgrade_type is not None:
+ stack_version = params.version
+
+ if params.retryAble:
+ Logger.info("nifi: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("nifi: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
+
+ api_version=None
+ if params.stack_supports_ranger_kerberos:
+ api_version='v2'
+ from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
+ setup_ranger_plugin('nifi', 'nifi', params.previous_jdbc_jar,
+ params.downloaded_custom_connector, params.driver_curl_source,
+ params.driver_curl_target, params.java_home,
+ params.repo_name, params.nifi_ranger_plugin_repo,
+ params.ranger_env, params.ranger_plugin_properties,
+ params.policy_user, params.policymgr_mgr_url,
+ params.enable_ranger_nifi, conf_dict=params.nifi_config_dir,
+ component_user=params.nifi_user, component_group=params.nifi_group, cache_service_list=['nifi'],
+ plugin_audit_properties=params.config['configurations']['ranger-nifi-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-nifi-audit'],
+ plugin_security_properties=params.config['configurations']['ranger-nifi-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-nifi-security'],
+ plugin_policymgr_ssl_properties=params.config['configurations']['ranger-nifi-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-nifi-policymgr-ssl'],
+ component_list=[], audit_db_is_enabled=params.xa_audit_db_is_enabled,
+ credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
+ ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
+ stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble,api_version=api_version,
+ is_security_enabled = params.security_enabled,
+ is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
+ component_user_principal=params.ranger_nifi_principal if params.security_enabled else None,
+ component_user_keytab=params.ranger_nifi_keytab if params.security_enabled else None)
+
+ #change permissions of ranger xml that were written to 0400
+ File(os.path.join(params.nifi_config_dir, 'ranger-nifi-audit.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
+ File(os.path.join(params.nifi_config_dir, 'ranger-nifi-security.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
+ File(os.path.join(params.nifi_config_dir, 'ranger-policymgr-ssl.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ else:
+ Logger.info('Ranger admin not installed')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py
new file mode 100755
index 0000000..9748661
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import sys, os
+
+config = Script.get_config()
+
+nifi_pid_dir=config['configurations']['nifi-env']['nifi_pid_dir']
+#nifi_master_pid_file=nifi_pid_dir + '/nifi-master.pid'
+nifi_node_pid_file=nifi_pid_dir + '/nifi.pid'
+nifi_ca_pid_file=nifi_pid_dir + '/nifi-ca.pid'
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2
new file mode 100644
index 0000000..97f5711
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2
@@ -0,0 +1,36 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{nifi_user}} - nofile {{nifi_user_nofile_limit}}
+{{nifi_user}} - nproc {{nifi_user_nproc_limit}}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json
new file mode 100644
index 0000000..bd27705
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json
@@ -0,0 +1,35 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"https",
+ "checks":[
+ {
+ "property":"nifi.node.ssl.isenabled",
+ "desired":"true",
+ "site":"nifi-ambari-ssl-config"
+ }
+ ]
+ },
+
+ "links": [
+ {
+ "name": "nifi_ui",
+ "label": "Nifi UI",
+ "requires_user_name": "false",
+ "component_name": "NIFI_MASTER",
+ "url":"%@://%@:%@/nifi",
+ "port":{
+ "http_property": "nifi.node.port",
+ "http_default_port": "9090",
+ "https_property": "nifi.node.ssl.port",
+ "https_default_port": "9091",
+ "regex": "^(\\d+)$",
+ "site": "nifi-ambari-config"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json
new file mode 100644
index 0000000..cd6791b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json
@@ -0,0 +1,463 @@
+{
+ "layouts": [
+ {
+ "layout_name": "default_nifi_dashboard",
+ "display_name": "Standard NiFi Dashboard",
+ "section_name": "NIFI_SUMMARY",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "FlowFiles Received Last 5 mins",
+ "description": "The number of FlowFiles received in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "FlowFilesReceivedLast5Minutes",
+ "metric_path": "metrics/nifi/FlowFilesReceivedLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "FlowFiles Received",
+ "value": "${FlowFilesReceivedLast5Minutes}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Received Last 5 mins",
+ "description": "The number of MBs received in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesReceivedLast5Minutes",
+ "metric_path": "metrics/nifi/BytesReceivedLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Received",
+ "value": "${BytesReceivedLast5Minutes/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "FlowFiles Sent Last 5 mins",
+ "description": "The number of FlowFiles sent in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "FlowFilesSentLast5Minutes",
+ "metric_path": "metrics/nifi/FlowFilesSentLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "FlowFiles Sent",
+ "value": "${FlowFilesSentLast5Minutes}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Sent Last 5 mins",
+ "description": "The number of MBs sent in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesSentLast5Minutes",
+ "metric_path": "metrics/nifi/BytesSentLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Sent",
+ "value": "${BytesSentLast5Minutes/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "FlowFiles Queued",
+ "description": "The number of FlowFiles queued.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "FlowFilesQueued",
+ "metric_path": "metrics/nifi/FlowFilesQueued",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "FlowFiles Queued",
+ "value": "${FlowFilesQueued}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Queued",
+ "description": "The number of MBs queued.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesQueued",
+ "metric_path": "metrics/nifi/BytesQueued",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Queued",
+ "value": "${BytesQueued/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Read Last 5 mins",
+ "description": "The number of MBs read in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesReadLast5Minutes",
+ "metric_path": "metrics/nifi/BytesReadLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Read",
+ "value": "${BytesReadLast5Minutes/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Written Last 5 mins",
+ "description": "The number of MBs written in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesWrittenLast5Minutes",
+ "metric_path": "metrics/nifi/BytesWrittenLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Written",
+ "value": "${BytesWrittenLast5Minutes/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Active Threads",
+ "description": "The number of active threads.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "ActiveThreads",
+ "metric_path": "metrics/nifi/ActiveThreads",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Active Threads",
+ "value": "${ActiveThreads}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Total Task Duration Milliseconds",
+ "description": "The total task duration in Milliseconds.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "TotalTaskDurationNanoSeconds",
+ "metric_path": "metrics/nifi/TotalTaskDurationNanoSeconds",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Total Task Duration Milliseconds",
+ "value": "${TotalTaskDurationNanoSeconds/1000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "JVM Heap Used (MBs)",
+ "description": "Number of MBs being used on the JVM Heap.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.heap_used",
+ "metric_path": "metrics/jvm/heap_used",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "JVM Heap Used",
+ "value": "${jvm.heap_used/1024/1024}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "100000",
+ "error_threshold": "200000"
+ }
+ },
+ {
+ "widget_name": "JVM Heap Usage",
+ "description": "Precentage of JVM heap being used.",
+ "widget_type": "GAUGE",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.heap_usage",
+ "metric_path": "metrics/jvm/heap_usage",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "JVM Heap Usage",
+ "value": "${jvm.heap_usage}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "0.8",
+ "error_threshold": "0.9"
+ }
+ },
+ {
+ "widget_name": "JVM File Descriptor Usage",
+ "description": "Precentage of file descriptors being used.",
+ "widget_type": "GAUGE",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.file_descriptor_usage",
+ "metric_path": "metrics/jvm/file_descriptor_usage",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "JVM File Descriptor Usage",
+ "value": "${jvm.file_descriptor_usage}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "0.8",
+ "error_threshold": "0.9"
+ }
+ },
+ {
+ "widget_name": "Thread Count",
+ "description": "Number of threads in the JVM.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.thread_count",
+ "metric_path": "metrics/jvm/thread_count",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Thread Count",
+ "value": "${jvm.thread_count}"
+ }
+ ],
+ "properties": {
+ }
+ },
+ {
+ "widget_name": "Daemon Thread Count",
+ "description": "Number of daemon threads in the JVM.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.daemon_thread_count",
+ "metric_path": "metrics/jvm/daemon_thread_count",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Daemon Thread Count",
+ "value": "${jvm.daemon_thread_count}"
+ }
+ ],
+ "properties": {
+ }
+ },
+ {
+ "widget_name": "Runnable Threads",
+ "description": "Number of runnable threads in the JVM.",
+ "widget_type": "GRAPH",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "jvm.thread_states.runnable",
+ "metric_path": "metrics/jvm/thread_states/runnable",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Runnable Threads",
+ "value": "${jvm.thread_states.runnable}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Blocked Threads",
+ "description": "Number of blocked threads in the JVM.",
+ "widget_type": "GRAPH",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "jvm.thread_states.blocked",
+ "metric_path": "metrics/jvm/thread_states/blocked",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Blocked Threads",
+ "value": "${jvm.thread_states.blocked}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Timed-Waiting Threads",
+ "description": "Number of timed-waiting threads in the JVM.",
+ "widget_type": "GRAPH",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "jvm.thread_states.timed_waiting",
+ "metric_path": "metrics/jvm/thread_states/timed_waiting",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Timed-Waiting Threads",
+ "value": "${jvm.thread_states.timed_waiting}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Terminated Threads",
+ "description": "Number of terminated threads in the JVM.",
+ "widget_type": "GRAPH",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "jvm.thread_states.terminated",
+ "metric_path": "metrics/jvm/thread_states/terminated",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Terminated Threads",
+ "value": "${jvm.thread_states.terminated}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py
new file mode 100644
index 0000000..8f0743b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import sys
+import os
+from ambari_server.serverConfiguration import get_ambari_properties, get_resources_location
+from resource_management.core import sudo
+
+def main():
+ properties = get_ambari_properties()
+ if properties == -1:
+ print >> sys.stderr, "Error getting ambari properties"
+ return -1
+
+ resources_location = get_resources_location(properties)
+ views_dir = os.path.join(resources_location, "views")
+
+ for file in os.listdir(views_dir):
+ path = os.path.join(views_dir, file)
+ if os.path.isfile(path):
+ if "ambari-admin" in path or "storm-view" in path:
+ print "Keeping views jar : " + path
+ else:
+ print "Deleting views jar : " + path
+ sudo.unlink(path)
+ else:
+ print "Deleting views directory : " + path
+ sudo.rmtree(path)
+ return 0
+
+if __name__ == "__main__":
+ exit (main())
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json
new file mode 100644
index 0000000..865561e
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json
@@ -0,0 +1,30 @@
+{
+ "type" : "full-release",
+ "name" : "hdf-ambari-mpack",
+ "version": "${project.version}",
+ "description" : "HDF Ambari Management Pack",
+ "prerequisites": {
+ "min-ambari-version" : "${minAmbariVersion}",
+ "max-ambari-version" : "${maxAmbariVersion}"
+ },
+ "hooks": [
+ {
+ "name": "after-install",
+ "type": "python",
+ "script": "hooks/after_install.py"
+ }
+ ],
+ "artifacts": [
+ {
+ "name" : "hdf-service-definitions",
+ "type" : "service-definitions",
+ "source_dir": "common-services"
+ },
+ {
+ "name" : "hdf-stack-definitions",
+ "type" : "stack-definitions",
+ "source_dir": "stacks"
+ }
+ ]
+}
+
[3/8] ambari git commit: AMBARI-18385: Add HDF management pack
(jluniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/themes/theme_version_4.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/themes/theme_version_4.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/themes/theme_version_4.json
new file mode 100644
index 0000000..379e980
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/themes/theme_version_4.json
@@ -0,0 +1,480 @@
+{
+ "name": "default",
+ "description": "Default theme for Ranger service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "ranger_tagsync",
+ "display-name": "Ranger Tagsync",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-tagsync-file",
+ "display-name": "File Tag Source",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-ranger-tagsync-row1-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "name": "ranger_plugin",
+ "display-name": "Ranger Plugin",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-ranger-plugin",
+ "display-name": "Ranger Plugin",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "3",
+ "section-columns": "3",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "section-ranger-plugin-row1-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "section-ranger-plugin-row1-col2",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "section-ranger-plugin-row1-col3",
+ "row-index": "0",
+ "column-index": "2",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "name" : "ranger_audit_settings",
+ "display-name" : "Ranger Audit",
+ "layout" : {
+ "sections" : [
+ {
+ "subsections" : [
+ {
+ "name" : "subsection-ranger-solr-row1-col1",
+ "removed" : false,
+ "row-index" : "0",
+ "column-span" : "1",
+ "row-span" : "1",
+ "column-index" : "0"
+ }
+ ],
+ "name" : "section-ranger-audit-solr",
+ "removed" : false,
+ "display-name" : "Audit to Solr",
+ "row-index" : "0",
+ "section-rows" : "1",
+ "column-span" : "1",
+ "section-columns" : "1",
+ "column-index" : "0",
+ "row-span" : "1"
+ }
+ ],
+ "tab-rows" : "2",
+ "tab-columns" : "2"
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "ranger-tagsync-site/ranger.tagsync.source.file",
+ "subsection-name": "subsection-ranger-tagsync-row1-col1"
+ },
+ {
+ "config": "ranger-tagsync-site/ranger.tagsync.source.file.check.interval.millis",
+ "subsection-name": "subsection-ranger-tagsync-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "ranger-tagsync-site/ranger.tagsync.source.file"
+ ],
+ "if": "${ranger-tagsync-site/ranger.tagsync.source.file}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "ranger-tagsync-site/ranger.tagsync.source.file.filename",
+ "subsection-name": "subsection-ranger-tagsync-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "ranger-tagsync-site/ranger.tagsync.source.file"
+ ],
+ "if": "${ranger-tagsync-site/ranger.tagsync.source.file}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "ranger-env/ranger-nifi-plugin-enabled",
+ "subsection-name": "section-ranger-plugin-row1-col1",
+ "depends-on": [
+ {
+ "resource": "service",
+ "if": "NIFI",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "ranger-env/ranger-storm-plugin-enabled",
+ "subsection-name": "section-ranger-plugin-row1-col2",
+ "depends-on": [
+ {
+ "resource": "service",
+ "if": "STORM",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "ranger-env/ranger-kafka-plugin-enabled",
+ "subsection-name": "section-ranger-plugin-row1-col3",
+ "depends-on": [
+ {
+ "resource": "service",
+ "if": "KAFKA",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config" : "ranger-env/xasecure.audit.destination.solr",
+ "removed" : false,
+ "subsection-name" : "subsection-ranger-solr-row1-col1"
+ },
+ {
+ "config" : "ranger-env/is_solrCloud_enabled",
+ "removed" : false,
+ "subsection-name" : "subsection-ranger-solr-row1-col1",
+ "depends-on" : [
+ {
+ "configs" : [
+ "ranger-env/xasecure.audit.destination.solr"
+ ],
+ "then" : {
+ "property_value_attributes" : {
+ "visible" : true
+ }
+ },
+ "if" : "${ranger-env/xasecure.audit.destination.solr}",
+ "else" : {
+ "property_value_attributes" : {
+ "visible" : false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "ranger-env/is_external_solrCloud_enabled",
+ "removed" : false,
+ "subsection-name": "subsection-ranger-solr-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "ranger-env/xasecure.audit.destination.solr",
+ "ranger-env/is_solrCloud_enabled"
+ ],
+ "if": "${ranger-env/xasecure.audit.destination.solr} && ${ranger-env/is_solrCloud_enabled}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "ranger-env/is_external_solrCloud_kerberos",
+ "removed" : false,
+ "subsection-name": "subsection-ranger-solr-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "ranger-env/xasecure.audit.destination.solr",
+ "ranger-env/is_solrCloud_enabled",
+ "ranger-env/is_external_solrCloud_enabled"
+ ],
+ "if": "${ranger-env/xasecure.audit.destination.solr} && ${ranger-env/is_solrCloud_enabled} && ${ranger-env/is_external_solrCloud_enabled}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config" : "ranger-admin-site/ranger.audit.solr.urls",
+ "removed" : false,
+ "subsection-name" : "subsection-ranger-solr-row1-col1",
+ "depends-on" : [
+ {
+ "configs" : [
+ "ranger-env/is_solrCloud_enabled",
+ "ranger-env/xasecure.audit.destination.solr"
+ ],
+ "then" : {
+ "property_value_attributes" : {
+ "visible" : true
+ }
+ },
+ "if" : "${ranger-env/is_solrCloud_enabled} === false && ${ranger-env/xasecure.audit.destination.solr}",
+ "else" : {
+ "property_value_attributes" : {
+ "visible" : false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config" : "ranger-admin-site/ranger.audit.solr.zookeepers",
+ "removed" : false,
+ "subsection-name" : "subsection-ranger-solr-row1-col1",
+ "depends-on" : [
+ {
+ "configs" : [
+ "ranger-env/is_solrCloud_enabled",
+ "ranger-env/xasecure.audit.destination.solr"
+ ],
+ "then" : {
+ "property_value_attributes" : {
+ "visible" : true
+ }
+ },
+ "if" : "${ranger-env/is_solrCloud_enabled} && ${ranger-env/xasecure.audit.destination.solr}",
+ "else" : {
+ "property_value_attributes" : {
+ "visible" : false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config" : "ranger-admin-site/ranger.audit.solr.username",
+ "removed" : false,
+ "subsection-name" : "subsection-ranger-solr-row1-col1",
+ "depends-on" : [
+ {
+ "configs" : [
+ "ranger-env/xasecure.audit.destination.solr"
+ ],
+ "then" : {
+ "property_value_attributes" : {
+ "visible" : true
+ }
+ },
+ "if" : "${ranger-env/xasecure.audit.destination.solr}",
+ "else" : {
+ "property_value_attributes" : {
+ "visible" : false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config" : "ranger-admin-site/ranger.audit.solr.password",
+ "removed" : false,
+ "subsection-name" : "subsection-ranger-solr-row1-col1",
+ "depends-on" : [
+ {
+ "configs" : [
+ "ranger-env/xasecure.audit.destination.solr"
+ ],
+ "then" : {
+ "property_value_attributes" : {
+ "visible" : true
+ }
+ },
+ "if" : "${ranger-env/xasecure.audit.destination.solr}",
+ "else" : {
+ "property_value_attributes" : {
+ "visible" : false
+ }
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "ranger-tagsync-site/ranger.tagsync.source.file.check.interval.millis",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "ranger-tagsync-site/ranger.tagsync.source.file.filename",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "ranger-tagsync-site/ranger.tagsync.source.file",
+ "widget": {
+ "type": "checkbox"
+ }
+ },
+ {
+ "config": "ranger-env/ranger-nifi-plugin-enabled",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "ranger-env/ranger-kafka-plugin-enabled",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "ranger-env/ranger-storm-plugin-enabled",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config" : "ranger-env/xasecure.audit.destination.solr",
+ "widget" : {
+ "type" : "toggle"
+ }
+ },
+ {
+ "config" : "ranger-env/is_solrCloud_enabled",
+ "widget" : {
+ "type" : "toggle"
+ }
+ },
+ {
+ "config" : "ranger-admin-site/ranger.audit.solr.urls",
+ "widget" : {
+ "type" : "text-field"
+ }
+ },
+ {
+ "config" : "ranger-admin-site/ranger.audit.solr.zookeepers",
+ "widget" : {
+ "type" : "text-field"
+ }
+ },
+ {
+ "config" : "ranger-admin-site/ranger.audit.solr.username",
+ "widget" : {
+ "type" : "text-field"
+ }
+ },
+ {
+ "config" : "ranger-admin-site/ranger.audit.solr.password",
+ "widget" : {
+ "type" : "password"
+ }
+ },
+ {
+ "config": "ranger-admin-site/ranger.is.solr.kerberised",
+ "widget": {
+ "type": "toggle"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-audit.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-audit.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-audit.xml
new file mode 100644
index 0000000..a210fea
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-audit.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-policymgr-ssl.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-policymgr-ssl.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-policymgr-ssl.xml
new file mode 100644
index 0000000..16c4753
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-policymgr-ssl.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore</name>
+ <value>/usr/hdf/current/storm-client/conf/ranger-plugin-keystore.jks</value>
+ <description>Java Keystore files</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore</name>
+ <value>/usr/hdf/current/storm-client/conf/ranger-plugin-truststore.jks</value>
+ <description>java truststore file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-security.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-security.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-security.xml
new file mode 100644
index 0000000..1be8e4a
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/ranger-storm-security.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+ <property>
+ <name>ranger.plugin.storm.policy.rest.ssl.config.file</name>
+ <value>/usr/hdf/current/storm-client/conf/ranger-policymgr-ssl.xml</value>
+ <description>Path to the file containing SSL details to contact Ranger Admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/storm-site.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/storm-site.xml
new file mode 100644
index 0000000..455a8f2
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/configuration/storm-site.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<!--
+ censed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="true" xmlns:xi="http://www.w3.org/2001/XInclude">
+ <property>
+ <name>java.library.path</name>
+ <value>/usr/local/lib:/opt/local/lib:/usr/lib:/usr/hdf/current/storm-client/lib</value>
+ <description>This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers)
+ for the java.library.path value. java.library.path tells the JVM where
+ to look for native libraries. It is necessary to set this config correctly since
+ Storm uses the ZeroMQ and JZMQ native libs. </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nimbus.childopts</name>
+ <value>-Xmx1024m _JAAS_PLACEHOLDER -javaagent:/usr/hdf/current/storm-nimbus/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/hdf/current/storm-nimbus/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM</value>
+ <description>This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon.</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>worker.childopts</name>
+ <value>-Xmx768m _JAAS_PLACEHOLDER -javaagent:/usr/hdf/current/storm-client/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdf/current/storm-client/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM</value>
+ <description>The jvm opts provided to workers launched by this supervisor. All \"%ID%\" substrings are replaced with an identifier for this worker.</description>
+ <value-attributes>
+ <type>multiLine</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>supervisor.childopts</name>
+ <value>-Xmx256m _JAAS_PLACEHOLDER -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port={{jmxremote_port}} -javaagent:/usr/hdf/current/storm-supervisor/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdf/current/storm-supervisor/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM</value>
+ <description>This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon.</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/metainfo.xml
new file mode 100644
index 0000000..500c7dc
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/STORM/metainfo.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>STORM</name>
+ <version>1.0.1.2.0</version>
+ <extends>common-services/STORM/1.0.1</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/ZOOKEEPER/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/ZOOKEEPER/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/ZOOKEEPER/metainfo.xml
new file mode 100644
index 0000000..2db589a
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/ZOOKEEPER/metainfo.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ZOOKEEPER</name>
+ <version>3.4.6.2.0</version>
+ <extends>common-services/ZOOKEEPER/3.4.6</extends>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>amazon2015,redhat6,redhat7,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>zookeeper_${stack_version}</name>
+ </package>
+ <package>
+ <name>zookeeper_${stack_version}-server</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+ <packages>
+ <package>
+ <name>zookeeper-${stack_version}</name>
+ </package>
+ <package>
+ <name>zookeeper-${stack_version}-server</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ </service>
+ </services>
+</metainfo>
[4/8] ambari git commit: AMBARI-18385: Add HDF management pack
(jluniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/rack_awareness.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/rack_awareness.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/rack_awareness.py
new file mode 100644
index 0000000..548f051
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/rack_awareness.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.resources import File
+from resource_management.core.source import StaticFile, Template
+from resource_management.libraries.functions import format
+
+
+def create_topology_mapping():
+ import params
+
+ File(params.net_topology_mapping_data_file_path,
+ content=Template("topology_mappings.data.j2"),
+ owner=params.hdfs_user,
+ group=params.user_group,
+ only_if=format("test -d {net_topology_script_dir}"))
+
+def create_topology_script():
+ import params
+
+ File(params.net_topology_script_file_path,
+ content=StaticFile('topology_script.py'),
+ mode=0755,
+ only_if=format("test -d {net_topology_script_dir}"))
+
+def create_topology_script_and_mapping():
+ import params
+ if params.has_hadoop_env:
+ create_topology_mapping()
+ create_topology_script()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/shared_initialization.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/shared_initialization.py
new file mode 100644
index 0000000..ba9c8fb
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/shared_initialization.py
@@ -0,0 +1,175 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil
+
+from resource_management import *
+
+def setup_hadoop():
+ """
+ Setup hadoop files and directories
+ """
+ import params
+
+ Execute(("setenforce","0"),
+ only_if="test -f /selinux/enforce",
+ not_if="(! which getenforce ) || (which getenforce && getenforce | grep -q Disabled)",
+ sudo=True,
+ )
+
+ #directories
+ if params.has_namenode or params.dfs_type == 'HCFS':
+ Directory(params.hdfs_log_dir_prefix,
+ create_parents = True,
+ owner='root',
+ group=params.user_group,
+ mode=0775,
+ cd_access='a',
+ )
+ if params.has_namenode:
+ Directory(params.hadoop_pid_dir_prefix,
+ create_parents = True,
+ owner='root',
+ group='root',
+ cd_access='a',
+ )
+ Directory(params.hadoop_tmp_dir,
+ create_parents = True,
+ owner=params.hdfs_user,
+ cd_access='a',
+ )
+ #files
+ if params.security_enabled:
+ tc_owner = "root"
+ else:
+ tc_owner = params.hdfs_user
+
+ # if WebHDFS is not enabled we need this jar to create hadoop folders.
+ if params.host_sys_prepped:
+ print "Skipping copying of fast-hdfs-resource.jar as host is sys prepped"
+ elif params.dfs_type == 'HCFS' or not WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs):
+ # for source-code of jar goto contrib/fast-hdfs-resource
+ File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"),
+ mode=0644,
+ content=StaticFile("fast-hdfs-resource.jar")
+ )
+
+ if os.path.exists(params.hadoop_conf_dir):
+ File(os.path.join(params.hadoop_conf_dir, 'commons-logging.properties'),
+ owner=tc_owner,
+ content=Template('commons-logging.properties.j2')
+ )
+
+ health_check_template_name = "health_check"
+ File(os.path.join(params.hadoop_conf_dir, health_check_template_name),
+ owner=tc_owner,
+ content=Template(health_check_template_name + ".j2")
+ )
+
+ log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties")
+ if (params.log4j_props != None):
+ File(log4j_filename,
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ content=params.log4j_props
+ )
+ elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))):
+ File(log4j_filename,
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ )
+
+ File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"),
+ owner=params.hdfs_user,
+ group=params.user_group,
+ content=Template("hadoop-metrics2.properties.j2")
+ )
+
+ if params.dfs_type == 'HCFS' and params.has_core_site and 'ECS_CLIENT' in params.component_list:
+ create_dirs()
+
+
+def setup_configs():
+ """
+ Creates configs for services HDFS mapred
+ """
+ import params
+
+ if params.has_namenode or params.dfs_type == 'HCFS':
+ if os.path.exists(params.hadoop_conf_dir):
+ File(params.task_log4j_properties_location,
+ content=StaticFile("task-log4j.properties"),
+ mode=0755
+ )
+
+ if os.path.exists(os.path.join(params.hadoop_conf_dir, 'configuration.xsl')):
+ File(os.path.join(params.hadoop_conf_dir, 'configuration.xsl'),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+ if os.path.exists(os.path.join(params.hadoop_conf_dir, 'masters')):
+ File(os.path.join(params.hadoop_conf_dir, 'masters'),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+
+ generate_include_file()
+
+
+def generate_include_file():
+ import params
+
+ if params.has_namenode and params.dfs_hosts and params.has_slaves:
+ include_hosts_list = params.slave_hosts
+ File(params.dfs_hosts,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+
+def create_javahome_symlink():
+ if os.path.exists("/usr/jdk/jdk1.6.0_31") and not os.path.exists("/usr/jdk64/jdk1.6.0_31"):
+ Directory("/usr/jdk64/",
+ create_parents = True,
+ )
+ Link("/usr/jdk/jdk1.6.0_31",
+ to="/usr/jdk64/jdk1.6.0_31",
+ )
+
+def create_dirs():
+ import params
+ params.HdfsResource(params.hdfs_tmp_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ mode=0777
+ )
+ params.HdfsResource(params.smoke_hdfs_user_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.smoke_user,
+ mode=params.smoke_hdfs_user_mode
+ )
+ params.HdfsResource(None,
+ action="execute"
+ )
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/commons-logging.properties.j2
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/commons-logging.properties.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/commons-logging.properties.j2
new file mode 100644
index 0000000..2197ba5
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/commons-logging.properties.j2
@@ -0,0 +1,43 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+#Logging Implementation
+
+#Log4J
+org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger
+
+#JDK Logger
+#org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/exclude_hosts_list.j2
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/exclude_hosts_list.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/exclude_hosts_list.j2
new file mode 100644
index 0000000..1adba80
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/exclude_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in hdfs_exclude_file %}
+{{host}}
+{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
new file mode 100644
index 0000000..fcd9b23
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
@@ -0,0 +1,104 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# syntax: [prefix].[source|sink|jmx].[instance].[options]
+# See package.html for org.apache.hadoop.metrics2 for details
+
+{% if has_ganglia_server %}
+*.period=60
+
+*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
+*.sink.ganglia.period=10
+
+# default for supportsparse is false
+*.sink.ganglia.supportsparse=true
+
+.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
+.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
+
+# Hook up to the server
+namenode.sink.ganglia.servers={{ganglia_server_host}}:8661
+datanode.sink.ganglia.servers={{ganglia_server_host}}:8659
+jobtracker.sink.ganglia.servers={{ganglia_server_host}}:8662
+tasktracker.sink.ganglia.servers={{ganglia_server_host}}:8658
+maptask.sink.ganglia.servers={{ganglia_server_host}}:8660
+reducetask.sink.ganglia.servers={{ganglia_server_host}}:8660
+resourcemanager.sink.ganglia.servers={{ganglia_server_host}}:8664
+nodemanager.sink.ganglia.servers={{ganglia_server_host}}:8657
+historyserver.sink.ganglia.servers={{ganglia_server_host}}:8666
+journalnode.sink.ganglia.servers={{ganglia_server_host}}:8654
+nimbus.sink.ganglia.servers={{ganglia_server_host}}:8649
+supervisor.sink.ganglia.servers={{ganglia_server_host}}:8650
+
+resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
+
+{% endif %}
+
+{% if has_metric_collector %}
+
+*.period={{metrics_collection_period}}
+*.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar
+*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+*.sink.timeline.period={{metrics_collection_period}}
+*.sink.timeline.sendInterval={{metrics_report_interval}}000
+*.sink.timeline.slave.host.name = {{hostname}}
+
+# HTTPS properties
+*.sink.timeline.truststore.path = {{metric_truststore_path}}
+*.sink.timeline.truststore.type = {{metric_truststore_type}}
+*.sink.timeline.truststore.password = {{metric_truststore_password}}
+
+datanode.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+namenode.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+resourcemanager.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+nodemanager.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+jobhistoryserver.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+journalnode.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+maptask.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+reducetask.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+applicationhistoryserver.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+
+resourcemanager.sink.timeline.tagsForPrefix.yarn=Queue
+
+{% if is_nn_client_port_configured %}
+# Namenode rpc ports customization
+namenode.sink.timeline.metric.rpc.client.port={{nn_rpc_client_port}}
+{% endif %}
+{% if is_nn_dn_port_configured %}
+namenode.sink.timeline.metric.rpc.datanode.port={{nn_rpc_dn_port}}
+{% endif %}
+{% if is_nn_healthcheck_port_configured %}
+namenode.sink.timeline.metric.rpc.healthcheck.port={{nn_rpc_healthcheck_port}}
+{% endif %}
+
+{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/health_check.j2
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/health_check.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/health_check.j2
new file mode 100644
index 0000000..0a03d17
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/health_check.j2
@@ -0,0 +1,81 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+#!/bin/bash
+#
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+err=0;
+
+function check_disks {
+
+ for m in `awk '$3~/ext3/ {printf" %s ",$2}' /etc/fstab` ; do
+ fsdev=""
+ fsdev=`awk -v m=$m '$2==m {print $1}' /proc/mounts`;
+ if [ -z "$fsdev" -a "$m" != "/mnt" ] ; then
+ msg_="$msg_ $m(u)"
+ else
+ msg_="$msg_`awk -v m=$m '$2==m { if ( $4 ~ /^ro,/ ) {printf"%s(ro)",$2 } ; }' /proc/mounts`"
+ fi
+ done
+
+ if [ -z "$msg_" ] ; then
+ echo "disks ok" ; exit 0
+ else
+ echo "$msg_" ; exit 2
+ fi
+
+}
+
+# Run all checks
+for check in disks ; do
+ msg=`check_${check}` ;
+ if [ $? -eq 0 ] ; then
+ ok_msg="$ok_msg$msg,"
+ else
+ err_msg="$err_msg$msg,"
+ fi
+done
+
+if [ ! -z "$err_msg" ] ; then
+ echo -n "ERROR $err_msg "
+fi
+if [ ! -z "$ok_msg" ] ; then
+ echo -n "OK: $ok_msg"
+fi
+
+echo
+
+# Success!
+exit 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/include_hosts_list.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/include_hosts_list.j2
new file mode 100644
index 0000000..4a9e713
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in slave_hosts %}
+{{host}}
+{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/topology_mappings.data.j2
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/topology_mappings.data.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/topology_mappings.data.j2
new file mode 100644
index 0000000..15034d6
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/topology_mappings.data.j2
@@ -0,0 +1,24 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+ #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+[network_topology]
+{% for host in all_hosts %}
+{% if host in slave_hosts %}
+{{host}}={{all_racks[loop.index-1]}}
+{{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}}
+{% endif %}
+{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/kerberos.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/kerberos.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/kerberos.json
new file mode 100644
index 0000000..c5ebe20
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/kerberos.json
@@ -0,0 +1,64 @@
+{
+ "properties": {
+ "realm": "${kerberos-env/realm}",
+ "keytab_dir": "/etc/security/keytabs",
+ "additional_realms": ""
+ },
+ "identities": [
+ {
+ "name": "spnego",
+ "principal": {
+ "value": "HTTP/_HOST@${realm}",
+ "type" : "service"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/spnego.service.keytab",
+ "owner": {
+ "name": "root",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ }
+ }
+ },
+ {
+ "name": "smokeuser",
+ "principal": {
+ "value": "${cluster-env/smokeuser}-${cluster_name|toLower()}@${realm}",
+ "type" : "user",
+ "configuration": "cluster-env/smokeuser_principal_name",
+ "local_username" : "${cluster-env/smokeuser}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/smokeuser.headless.keytab",
+ "owner": {
+ "name": "${cluster-env/smokeuser}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ },
+ "configuration": "cluster-env/smokeuser_keytab"
+ }
+ },
+ {
+ "name": "ambari-server",
+ "principal": {
+ "value": "ambari-server-${cluster_name|toLower()}@${realm}",
+ "type" : "user",
+ "configuration": "cluster-env/ambari_principal_name"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/ambari.server.keytab",
+ "owner": {
+ "name": "root",
+ "access": "r"
+ }
+ }
+ }
+ ]
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/metainfo.xml
new file mode 100644
index 0000000..50c5f75
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/metainfo.xml
@@ -0,0 +1,23 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <versions>
+ <active>true</active>
+ </versions>
+ <minJdk>1.7</minJdk>
+ <maxJdk>1.8</maxJdk>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_features.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_features.json
new file mode 100644
index 0000000..645e357
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_features.json
@@ -0,0 +1,281 @@
+{
+ "stack_features": [
+ {
+ "name": "snappy",
+ "description": "Snappy compressor/decompressor support",
+ "min_version": "0.0.0.0",
+ "max_version": "0.2.0.0"
+ },
+ {
+ "name": "lzo",
+ "description": "LZO libraries support",
+ "min_version": "0.2.1.0"
+ },
+ {
+ "name": "express_upgrade",
+ "description": "Express upgrade support",
+ "min_version": "0.1.0.0"
+ },
+ {
+ "name": "rolling_upgrade",
+ "description": "Rolling upgrade support",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "config_versioning",
+ "description": "Configurable versions support",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "datanode_non_root",
+ "description": "DataNode running as non-root support (AMBARI-7615)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "remove_ranger_hdfs_plugin_env",
+ "description": "HDFS removes Ranger env files (AMBARI-14299)",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "ranger",
+ "description": "Ranger Service support",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "ranger_tagsync_component",
+ "description": "Ranger Tagsync component support (AMBARI-14383)",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "phoenix",
+ "description": "Phoenix Service support",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "nfs",
+ "description": "NFS support",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "tez_for_spark",
+ "description": "Tez dependency for Spark",
+ "min_version": "0.2.0.0",
+ "max_version": "0.3.0.0"
+ },
+ {
+ "name": "timeline_state_store",
+ "description": "Yarn application timeline-service supports state store property (AMBARI-11442)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "copy_tarball_to_hdfs",
+ "description": "Copy tarball to HDFS support (AMBARI-12113)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "spark_16plus",
+ "description": "Spark 1.6+",
+ "min_version": "1.2.0.0"
+ },
+ {
+ "name": "spark_thriftserver",
+ "description": "Spark Thrift Server",
+ "min_version": "0.3.2.0"
+ },
+ {
+ "name": "storm_kerberos",
+ "description": "Storm Kerberos support (AMBARI-7570)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "storm_ams",
+ "description": "Storm AMS integration (AMBARI-10710)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "create_kafka_broker_id",
+ "description": "Ambari should create Kafka Broker Id (AMBARI-12678)",
+ "min_version": "0.2.0.0",
+ "max_version": "0.3.0.0"
+ },
+ {
+ "name": "kafka_listeners",
+ "description": "Kafka listeners (AMBARI-10984)",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "kafka_kerberos",
+ "description": "Kafka Kerberos support (AMBARI-10984)",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "pig_on_tez",
+ "description": "Pig on Tez support (AMBARI-7863)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "ranger_usersync_non_root",
+ "description": "Ranger Usersync as non-root user (AMBARI-10416)",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "ranger_audit_db_support",
+ "description": "Ranger Audit to DB support",
+ "min_version": "0.2.0.0",
+ "max_version": "2.0.0.0"
+ },
+ {
+ "name": "accumulo_kerberos_user_auth",
+ "description": "Accumulo Kerberos User Auth (AMBARI-10163)",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "knox_versioned_data_dir",
+ "description": "Use versioned data dir for Knox (AMBARI-13164)",
+ "min_version": "0.3.2.0"
+ },
+ {
+ "name": "knox_sso_topology",
+ "description": "Knox SSO Topology support (AMBARI-13975)",
+ "min_version": "0.3.8.0"
+ },
+ {
+ "name": "atlas_rolling_upgrade",
+ "description": "Rolling upgrade support for Atlas",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "oozie_admin_user",
+ "description": "Oozie install user as an Oozie admin user (AMBARI-7976)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "oozie_create_hive_tez_configs",
+ "description": "Oozie create configs for Ambari Hive and Tez deployments (AMBARI-8074)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "oozie_setup_shared_lib",
+ "description": "Oozie setup tools used to shared Oozie lib to HDFS (AMBARI-7240)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "oozie_host_kerberos",
+ "description": "Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)",
+ "min_version": "0.0.0.0",
+ "max_version": "0.2.0.0"
+ },
+ {
+ "name": "falcon_extensions",
+ "description": "Falcon Extension",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "hive_metastore_upgrade_schema",
+ "description": "Hive metastore upgrade schema support (AMBARI-11176)",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "hive_server_interactive",
+ "description": "Hive server interactive support (AMBARI-15573)",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "hive_webhcat_specific_configs",
+ "description": "Hive webhcat specific configurations support (AMBARI-12364)",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "hive_purge_table",
+ "description": "Hive purge table support (AMBARI-12260)",
+ "min_version": "0.3.0.0"
+ },
+ {
+ "name": "hive_server2_kerberized_env",
+ "description": "Hive server2 working on kerberized environment (AMBARI-13749)",
+ "min_version": "0.2.3.0",
+ "max_version": "0.2.5.0"
+ },
+ {
+ "name": "hive_env_heapsize",
+ "description": "Hive heapsize property defined in hive-env (AMBARI-12801)",
+ "min_version": "0.2.0.0"
+ },
+ {
+ "name": "ranger_kms_hsm_support",
+ "description": "Ranger KMS HSM support (AMBARI-15752)",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "ranger_log4j_support",
+ "description": "Ranger supporting log-4j properties (AMBARI-15681)",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "ranger_kerberos_support",
+ "description": "Ranger Kerberos support",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "hive_metastore_site_support",
+ "description": "Hive Metastore site support",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "ranger_usersync_password_jceks",
+ "description": "Saving Ranger Usersync credentials in jceks",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "ranger_install_infra_client",
+ "description": "Ambari Infra Service support",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "hbase_home_directory",
+ "description": "Hbase home directory in HDFS needed for HBASE backup",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "spark_livy",
+ "description": "Livy as slave component of spark",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "atlas_ranger_plugin_support",
+ "description": "Atlas Ranger plugin support",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "atlas_conf_dir_in_path",
+ "description": "Prepend the Atlas conf dir (/etc/atlas/conf) to the classpath of Storm and Falcon",
+ "min_version": "0.3.0.0",
+ "max_version": "0.4.99.99"
+ },
+ {
+ "name": "atlas_upgrade_support",
+ "description": "Atlas supports express and rolling upgrades",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "ranger_pid_support",
+ "description": "Ranger Service support pid generation AMBARI-16756",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "ranger_kms_pid_support",
+ "description": "Ranger KMS Service support pid generation",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "ranger_admin_password_change",
+ "description": "Allow ranger admin credentials to be specified during cluster creation (AMBARI-17000)",
+ "min_version": "2.0.0.0"
+ },
+ {
+ "name": "storm_metrics_apache_classes",
+ "description": "Metrics sink for Storm that uses Apache class names",
+ "min_version": "2.0.0.0"
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_tools.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_tools.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_tools.json
new file mode 100644
index 0000000..fb6e1b2
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_tools.json
@@ -0,0 +1,4 @@
+{
+ "stack_selector": ["hdf-select", "/usr/bin/hdf-select", "hdf-select"],
+ "conf_selector": ["conf-select", "/usr/bin/conf-select", "conf-select"]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/repos/repoinfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/repos/repoinfo.xml
new file mode 100644
index 0000000..3f8fd9e
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/repos/repoinfo.xml
@@ -0,0 +1,104 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<reposinfo>
+ <latest>http://s3.amazonaws.com/dev.hortonworks.com/HDF/hdf_urlinfo.json</latest>
+ <os family="redhat6">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDF/centos6/2.x/updates/2.0.0.0</baseurl>
+ <repoid>HDF-2.0</repoid>
+ <reponame>HDF</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ </repo>
+ </os>
+ <os family="redhat7">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDF/centos7/2.x/updates/2.0.0.0</baseurl>
+ <repoid>HDF-2.0</repoid>
+ <reponame>HDF</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos7</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ </repo>
+ </os>
+ <os family="suse11">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDF/suse11sp3/2.x/updates/2.0.0.0</baseurl>
+ <repoid>HDF-2.0</repoid>
+ <reponame>HDF</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/suse11sp3</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ </repo>
+ </os>
+ <os family="suse12">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDF/suse11sp3/2.x/updates/2.0.0.0</baseurl>
+ <repoid>HDF-2.0</repoid>
+ <reponame>HDF</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/suse11sp3</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ </repo>
+ </os>
+ <os family="ubuntu12">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDF/ubuntu12/2.x/updates/2.0.0.0</baseurl>
+ <repoid>HDF-2.0</repoid>
+ <reponame>HDF</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/ubuntu12</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ </repo>
+ </os>
+ <os family="debian7">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDF/debian7/2.x/updates/2.0.0.0</baseurl>
+ <repoid>HDF-2.0</repoid>
+ <reponame>HDF</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/debian6</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ </repo>
+ </os>
+ <os family="ubuntu14">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDF/ubuntu14/2.x/updates/2.0.0.0</baseurl>
+ <repoid>HDF-2.0</repoid>
+ <reponame>HDF</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/ubuntu12</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ </repo>
+ </os>
+</reposinfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/role_command_order.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/role_command_order.json
new file mode 100644
index 0000000..96afb81
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/role_command_order.json
@@ -0,0 +1,55 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
+ "ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
+ "ZOOKEEPER_SERVER-STOP" : ["METRICS_COLLECTOR-STOP"],
+ "METRICS_COLLECTOR-START": ["ZOOKEEPER_SERVER-START"],
+ "AMBARI_METRICS_SERVICE_CHECK-SERVICE_CHECK": ["METRICS_COLLECTOR-START"],
+ "METRICS_GRAFANA-START": ["METRICS_COLLECTOR-START"],
+ "METRICS_COLLECTOR-STOP": ["METRICS_GRAFANA-STOP"],
+ "NIMBUS-START" : ["ZOOKEEPER_SERVER-START"],
+ "SUPERVISOR-START" : ["NIMBUS-START"],
+ "STORM_UI_SERVER-START" : ["NIMBUS-START"],
+ "DRPC_SERVER-START" : ["NIMBUS-START"],
+ "STORM_REST_API-START" : ["NIMBUS-START", "STORM_UI_SERVER-START", "SUPERVISOR-START", "DRPC_SERVER-START"],
+ "STORM_SERVICE_CHECK-SERVICE_CHECK": ["NIMBUS-START", "SUPERVISOR-START", "STORM_UI_SERVER-START",
+ "DRPC_SERVER-START"],
+ "NIMBUS-STOP" : ["SUPERVISOR-STOP", "STORM_UI_SERVER-STOP", "DRPC_SERVER-STOP"],
+ "RANGER_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_ADMIN-START"],
+ "RANGER_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_USERSYNC-START"],
+ "KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START"],
+ "KAFKA_SERVICE_CHECK-SERVICE_CHECK": ["KAFKA_BROKER-START"],
+ "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START"],
+ "NIMBUS-START": ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START"],
+ "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"],
+ "RANGER_ADMIN-START" : ["ZOOKEEPER_SERVER-START", "INFRA_SOLR-START"],
+ "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"],
+ "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"],
+ "KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"],
+ "NIMBUS-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"],
+ "NIFI_MASTER-START": ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NIFI_CA-START"],
+ "NIFI_SERVICE_CHECK-SERVICE_CHECK": ["NIFI_MASTER-START"],
+ "STORM_UI_SERVER-START" : ["NIMBUS-START", "NAMENODE-START"],
+
+ "METRICS_COLLECTOR-RESTART": ["ZOOKEEPER_SERVER-RESTART"],
+ "METRICS_GRAFANA-RESTART": ["METRICS_COLLECTOR-RESTART"],
+ "NIMBUS-RESTART" : ["ZOOKEEPER_SERVER-RESTART"],
+ "SUPERVISOR-RESTART" : ["NIMBUS-RESTART"],
+ "STORM_UI_SERVER-RESTART" : ["NIMBUS-RESTART"],
+ "DRPC_SERVER-RESTART" : ["NIMBUS-RESTART"],
+ "STORM_REST_API-RESTART" : ["NIMBUS-RESTART", "STORM_UI_SERVER-RESTART", "SUPERVISOR-RESTART", "DRPC_SERVER-RESTART"],
+ "KAFKA_BROKER-RESTART" : ["ZOOKEEPER_SERVER-RESTART", "RANGER_USERSYNC-RESTART"],
+ "RANGER_USERSYNC-RESTART" : ["RANGER_ADMIN-RESTART"],
+ "NIMBUS-RESTART": ["ZOOKEEPER_SERVER-RESTART", "RANGER_USERSYNC-RESTART"],
+ "RANGER_USERSYNC-RESTART" : ["RANGER_ADMIN-RESTART", "RANGER_KMS_SERVER-RESTART"],
+ "RANGER_ADMIN-RESTART" : ["ZOOKEEPER_SERVER-RESTART", "INFRA_SOLR-RESTART"],
+ "RANGER_KMS_SERVER-RESTART" : ["RANGER_ADMIN-RESTART"],
+ "KAFKA_BROKER-RESTART" : ["ZOOKEEPER_SERVER-RESTART", "RANGER_USERSYNC-RESTART", "NAMENODE-RESTART"],
+ "NIMBUS-RESTART" : ["ZOOKEEPER_SERVER-RESTART", "RANGER_USERSYNC-RESTART", "NAMENODE-RESTART"],
+ "NIFI_MASTER-RESTART": ["ZOOKEEPER_SERVER-RESTART", "RANGER_USERSYNC-RESTART"],
+ "STORM_UI_SERVER-RESTART" : ["NIMBUS-RESTART", "NAMENODE-RESTART"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_INFRA/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_INFRA/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_INFRA/metainfo.xml
new file mode 100644
index 0000000..635d525
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_INFRA/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>AMBARI_INFRA</name>
+ <extends>common-services/AMBARI_INFRA/0.1.0</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_INFRA/role_command_order.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_INFRA/role_command_order.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_INFRA/role_command_order.json
new file mode 100755
index 0000000..7246e9e
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_INFRA/role_command_order.json
@@ -0,0 +1,7 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for ambari infra",
+ "INFRA_SOLR-START" : ["ZOOKEEPER_SERVER-START"],
+ "AMBARI_INFRA_SERVICE_CHECK-SERVICE_CHECK": ["INFRA_SOLR-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_METRICS/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_METRICS/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_METRICS/metainfo.xml
new file mode 100644
index 0000000..5b1d62f
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/AMBARI_METRICS/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>AMBARI_METRICS</name>
+ <extends>common-services/AMBARI_METRICS/0.1.0</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/configuration/ranger-kafka-audit.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/configuration/ranger-kafka-audit.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/configuration/ranger-kafka-audit.xml
new file mode 100644
index 0000000..c4b9bdf
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/configuration/ranger-kafka-audit.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+ <property>
+ <name>xasecure.audit.destination.db</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.db.jdbc.url</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.db.user</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.db.password</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.db.jdbc.driver</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.credential.provider.file</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.db.batch.filespool.dir</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/configuration/ranger-kafka-policymgr-ssl.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/configuration/ranger-kafka-policymgr-ssl.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/configuration/ranger-kafka-policymgr-ssl.xml
new file mode 100644
index 0000000..158cb67
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/configuration/ranger-kafka-policymgr-ssl.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore</name>
+ <value>/usr/hdf/current/kafka-broker/config/ranger-plugin-keystore.jks</value>
+ <description>Java Keystore files</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore</name>
+ <value>/usr/hdf/current/kafka-broker/config/ranger-plugin-truststore.jks</value>
+ <description>java truststore file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/kerberos.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/kerberos.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/kerberos.json
new file mode 100644
index 0000000..e1e6461
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/kerberos.json
@@ -0,0 +1,69 @@
+{
+ "services": [
+ {
+ "name": "KAFKA",
+ "identities": [
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "configurations": [
+ {
+ "kafka-broker": {
+ "authorizer.class.name": "kafka.security.auth.SimpleAclAuthorizer",
+ "principal.to.local.class":"kafka.security.auth.KerberosPrincipalToLocal",
+ "super.users": "user:${kafka-env/kafka_user}",
+ "security.inter.broker.protocol": "PLAINTEXTSASL",
+ "zookeeper.set.acl": "true"
+ }
+ },
+ {
+ "ranger-kafka-audit": {
+ "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",
+ "xasecure.audit.jaas.Client.loginModuleControlFlag": "required",
+ "xasecure.audit.jaas.Client.option.useKeyTab": "true",
+ "xasecure.audit.jaas.Client.option.storeKey": "false",
+ "xasecure.audit.jaas.Client.option.serviceName": "solr",
+ "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "KAFKA_BROKER",
+ "identities": [
+ {
+ "name": "kafka_broker",
+ "principal": {
+ "value": "${kafka-env/kafka_user}/_HOST@${realm}",
+ "type": "service",
+ "configuration": "kafka-env/kafka_principal_name"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/kafka.service.keytab",
+ "owner": {
+ "name": "${kafka-env/kafka_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "kafka-env/kafka_keytab"
+ }
+ },
+ {
+ "name": "/KAFKA/KAFKA_BROKER/kafka_broker",
+ "principal": {
+ "configuration": "ranger-kafka-audit/xasecure.audit.jaas.Client.option.principal"
+ },
+ "keytab": {
+ "configuration": "ranger-kafka-audit/xasecure.audit.jaas.Client.option.keyTab"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/metainfo.xml
new file mode 100644
index 0000000..c9871c3
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KAFKA/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>KAFKA</name>
+ <version>0.10.0.2.0</version>
+ <extends>common-services/KAFKA/0.9.0</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KERBEROS/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KERBEROS/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KERBEROS/metainfo.xml
new file mode 100644
index 0000000..25cfcc6
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/KERBEROS/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>KERBEROS</name>
+ <extends>common-services/KERBEROS/1.10.3-10</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/LOGSEARCH/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/LOGSEARCH/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/LOGSEARCH/metainfo.xml
new file mode 100644
index 0000000..df697dc
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/LOGSEARCH/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>LOGSEARCH</name>
+ <extends>common-services/LOGSEARCH/0.5.0</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/LOGSEARCH/role_command_order.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/LOGSEARCH/role_command_order.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/LOGSEARCH/role_command_order.json
new file mode 100644
index 0000000..8098875
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/LOGSEARCH/role_command_order.json
@@ -0,0 +1,10 @@
+{
+ "general_deps" : {
+ "_comment": "dependencies for logsearch",
+ "LOGSEARCH_SERVER-START": ["INFRA_SOLR-START", "ZOOKEEPER_SERVER-START"],
+ "LOGSEARCH_LOGFEEDER-START": ["INFRA_SOLR-START", "LOGSEARCH_SERVER-START"],
+ "LOGSEARCH_SERVICE_CHECK-SERVICE_CHECK": ["INFRA_SOLR-START", "LOGSEARCH_SERVER-START"],
+ "LOGSEARCH_SERVER-RESTART" : ["INFRA_SOLR-RESTART"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/NIFI/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/NIFI/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/NIFI/metainfo.xml
new file mode 100755
index 0000000..2cd4c05
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/NIFI/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>NIFI</name>
+ <version>1.0.0.2.0</version>
+ <extends>common-services/NIFI/1.0.0</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-admin-site.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-admin-site.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-admin-site.xml
new file mode 100644
index 0000000..188cc8c
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-admin-site.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+
+ <property>
+ <name>ranger.plugins.atlas.serviceuser</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>ranger.plugins.nifi.serviceuser</name>
+ <value>nifi</value>
+ <depends-on>
+ <property>
+ <type>nifi-env</type>
+ <name>nifi_user</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>ranger.supportedcomponents</name>
+ <value>storm,kafka,nifi</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-env.xml
new file mode 100644
index 0000000..8672c76
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-env.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="true" supports_adding_forbidden="true">
+
+ <property>
+ <name>ranger-atlas-plugin-enabled</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>ranger-nifi-plugin-enabled</name>
+ <value>No</value>
+ <display-name>NIFI Ranger Plugin</display-name>
+ <description>Enable NIFI Ranger plugin</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>Yes</value>
+ <label>ON</label>
+ </entry>
+ <entry>
+ <value>No</value>
+ <label>OFF</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-tagsync-site.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-tagsync-site.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-tagsync-site.xml
new file mode 100644
index 0000000..5e3cea6
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-tagsync-site.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="true">
+
+ <property>
+ <name>ranger.tagsync.source.atlas</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.atlasrest</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.atlasrest.download.interval.millis</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.atlasrest.endpoint</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.atlasrest.keystore.filename</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.atlasrest.username</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>ranger.tagsync.dest.ranger.ssl.config.filename</name>
+ <value>/usr/hdf/current/ranger-tagsync/conf/mytruststore.jks</value>
+ <description>Truststore used for tagsync, required if tagsync -> ranger admin communication is SSL enabled</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>ranger.tagsync.keystore.filename</name>
+ <value>/usr/hdf/current/ranger-tagsync/conf/rangertagsync.jceks</value>
+ <description>Keystore file</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-ugsync-site.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-ugsync-site.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-ugsync-site.xml
new file mode 100644
index 0000000..35b6499
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/ranger-ugsync-site.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+
+ <property>
+ <name>ranger.usersync.keystore.file</name>
+ <value>/usr/hdf/current/ranger-usersync/conf/unixauthservice.jks</value>
+ <description>Keystore file used for usersync</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>ranger.usersync.truststore.file</name>
+ <value>/usr/hdf/current/ranger-usersync/conf/mytruststore.jks</value>
+ <description>Truststore used for usersync, required if usersync -> ranger admin communication is SSL enabled</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>ranger.usersync.ldap.bindkeystore</name>
+ <value></value>
+ <description>Set same value as ranger.usersync.keystore.file property i.e default value /usr/hdf/current/ranger-usersync/conf/ugsync.jceks</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>ranger.usersync.credstore.filename</name>
+ <value>/usr/hdf/current/ranger-usersync/conf/ugsync.jceks</value>
+ <description>Credential store file name for user sync, specify full path</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>ranger.usersync.policymgr.keystore</name>
+ <value>/usr/hdf/current/ranger-usersync/conf/ugsync.jceks</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/tagsync-application-properties.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/tagsync-application-properties.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/tagsync-application-properties.xml
new file mode 100644
index 0000000..8ef8b53
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/configuration/tagsync-application-properties.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="true" supports_adding_forbidden="true">
+ <property>
+ <name>atlas.kafka.entities.group.id</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>atlas.kafka.bootstrap.servers</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>atlas.kafka.zookeeper.connect</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/metainfo.xml
new file mode 100644
index 0000000..7d035eb
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/RANGER/metainfo.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>RANGER</name>
+ <extends>common-services/RANGER/0.6.0</extends>
+ <version>0.6.0.2.0</version>
+ <components>
+ <component>
+ <name>RANGER_TAGSYNC</name>
+ <cardinality>0</cardinality>
+ </component>
+ </components>
+ <themes>
+ <theme>
+ <fileName>theme_version_4.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+ </service>
+ </services>
+</metainfo>
[5/8] ambari git commit: AMBARI-18385: Add HDF management pack
(jluniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/configuration/cluster-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/configuration/cluster-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/configuration/cluster-env.xml
new file mode 100644
index 0000000..9114222
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/configuration/cluster-env.xml
@@ -0,0 +1,210 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>recovery_enabled</name>
+ <value>true</value>
+ <description>Auto start enabled or not for this cluster.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_type</name>
+ <value>AUTO_START</value>
+ <description>Auto start type.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_lifetime_max_count</name>
+ <value>1024</value>
+ <description>Auto start lifetime maximum count of recovery attempt allowed per host component. This is reset when agent is restarted.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_max_count</name>
+ <value>6</value>
+ <description>Auto start maximum count of recovery attempt allowed per host component in a window. This is reset when agent is restarted.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_window_in_minutes</name>
+ <value>60</value>
+ <description>Auto start recovery window size in minutes.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_retry_interval</name>
+ <value>5</value>
+ <description>Auto start recovery retry gap between tries per host component.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>security_enabled</name>
+ <value>false</value>
+ <description>Hadoop Security</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kerberos_domain</name>
+ <value>EXAMPLE.COM</value>
+ <description>Kerberos realm.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>ignore_groupsusers_create</name>
+ <display-name>Skip group modifications during install</display-name>
+ <value>false</value>
+ <property-type>ADDITIONAL_USER_PROPERTY</property-type>
+ <description>Whether to ignore failures on users and group creation</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>smokeuser</name>
+ <display-name>Smoke User</display-name>
+ <value>ambari-qa</value>
+ <property-type>USER</property-type>
+ <description>User executing service checks</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>smokeuser_keytab</name>
+ <value>/etc/security/keytabs/smokeuser.headless.keytab</value>
+ <description>Path to smoke test user keytab file</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>user_group</name>
+ <display-name>Hadoop Group</display-name>
+ <value>hadoop</value>
+ <property-type>GROUP</property-type>
+ <description>Hadoop user group.</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>repo_suse_rhel_template</name>
+ <value>[{{repo_id}}]
+name={{repo_id}}
+{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}
+
+path=/
+enabled=1
+gpgcheck=0</value>
+ <description>Template of repositories for rhel and suse.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>repo_ubuntu_template</name>
+ <value>{{package_type}} {{base_url}} {{components}}</value>
+ <description>Template of repositories for ubuntu.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>override_uid</name>
+ <value>true</value>
+ <property-type>ADDITIONAL_USER_PROPERTY</property-type>
+ <display-name>Have Ambari manage UIDs</display-name>
+ <description>Have Ambari manage UIDs</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>fetch_nonlocal_groups</name>
+ <value>true</value>
+ <display-name>Ambari fetch nonlocal groups</display-name>
+ <description>Ambari requires fetching all the groups. This can be slow
+ on envs with enabled ldap. Setting this option to false will enable Ambari,
+ to skip user/group management connected with ldap groups.</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>managed_hdfs_resource_property_names</name>
+ <value></value>
+ <description>Comma separated list of property names with HDFS resource paths.
+ Resource from this list will be managed even if it is marked as not managed in the stack</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>stack_tools</name>
+ <value></value>
+ <description>Stack specific tools</description>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>stack_tools.json</property-file-name>
+ <property-file-type>json</property-file-type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>stack_features</name>
+ <value></value>
+ <description>List of features supported by the stack</description>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>stack_features.json</property-file-name>
+ <property-file-type>json</property-file-type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>stack_root</name>
+ <value>/usr/hdf</value>
+ <description>Stack root folder</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>alerts_repeat_tolerance</name>
+ <value>1</value>
+ <description>The number of consecutive alerts required to transition an alert from the SOFT to the HARD state.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/hook.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/hook.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/hook.py
new file mode 100644
index 0000000..8a583b3
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/hook.py
@@ -0,0 +1,37 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.hook import Hook
+from shared_initialization import link_configs
+from shared_initialization import setup_config
+from shared_initialization import setup_stack_symlinks
+
+class AfterInstallHook(Hook):
+
+ def hook(self, env):
+ import params
+
+ env.set_params(params)
+ setup_stack_symlinks()
+ setup_config()
+
+ link_configs(self.stroutfile)
+
+if __name__ == "__main__":
+ AfterInstallHook().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/params.py
new file mode 100644
index 0000000..6244d3e
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/params.py
@@ -0,0 +1,101 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions import default
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import format_jvm_option
+from resource_management.libraries.functions.version import format_stack_version
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+dfs_type = default("/commandParams/dfs_type", "")
+
+is_parallel_execution_enabled = int(default("/agentConfigParams/agent/parallel_execution", 0)) == 1
+
+sudo = AMBARI_SUDO_BINARY
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+# current host stack version
+current_version = default("/hostLevelParams/current_version", None)
+
+# default hadoop params
+mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
+hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec")
+hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
+
+# HDF 0.2+ params
+if Script.is_stack_greater_or_equal("0.2"):
+ mapreduce_libs_path = "/usr/hdf/current/hadoop-mapreduce-client/*"
+
+ # not supported in HDF 0.2+
+ hadoop_conf_empty_dir = None
+
+versioned_stack_root = '/usr/hdf/current'
+
+#security params
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+#java params
+java_home = config['hostLevelParams']['java_home']
+
+#hadoop params
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
+hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger']
+
+jsvc_path = "/usr/lib/bigtop-utils"
+
+hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
+namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
+namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
+namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
+
+jtnode_opt_newsize = "200m"
+jtnode_opt_maxnewsize = "200m"
+jtnode_heapsize = "1024m"
+ttnode_heapsize = "1024m"
+
+dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
+mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+
+#users and groups
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+user_group = config['configurations']['cluster-env']['user_group']
+
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+has_namenode = not len(namenode_host) == 0
+
+if has_namenode or dfs_type == 'HCFS':
+ hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
+
+link_configs_lock_file = os.path.join(tmp_dir, "link_configs_lock_file")
+stack_select_lock_file = os.path.join(tmp_dir, "stack_select_lock_file")
+
+upgrade_suspended = default("/roleParams/upgrade_suspended", False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/shared_initialization.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/shared_initialization.py
new file mode 100644
index 0000000..87c48ef
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -0,0 +1,108 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+
+import ambari_simplejson as json
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.fcntl_based_process_lock import FcntlBasedProcessLock
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.script import Script
+
+
+def setup_stack_symlinks():
+ """
+ Invokes <stack-selector-tool> set all against a calculated fully-qualified, "normalized" version based on a
+ stack version, such as "0.3". This should always be called after a component has been
+ installed to ensure that all HDF pointers are correct. The stack upgrade logic does not
+ interact with this since it's done via a custom command and will not trigger this hook.
+ :return:
+ """
+ import params
+ if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '0.2') >= 0:
+ # try using the exact version first, falling back in just the stack if it's not defined
+ # which would only be during an intial cluster installation
+ version = params.current_version if params.current_version is not None else params.stack_version_unformatted
+
+ if not params.upgrade_suspended:
+ # On parallel command execution this should be executed by a single process at a time.
+ with FcntlBasedProcessLock(params.stack_select_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
+ stack_select.select_all(version)
+
+def setup_config():
+ import params
+ stackversion = params.stack_version_unformatted
+ Logger.info("FS Type: {0}".format(params.dfs_type))
+
+ is_hadoop_conf_dir_present = False
+ if hasattr(params, "hadoop_conf_dir") and params.hadoop_conf_dir is not None and os.path.exists(params.hadoop_conf_dir):
+ is_hadoop_conf_dir_present = True
+ else:
+ Logger.warning("Parameter hadoop_conf_dir is missing or directory does not exist. This is expected if this host does not have any Hadoop components.")
+
+ if is_hadoop_conf_dir_present and (params.has_namenode or stackversion.find('Gluster') >= 0 or params.dfs_type == 'HCFS'):
+ # create core-site only if the hadoop config diretory exists
+ XmlConfig("core-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.hdfs_user,
+ group=params.user_group,
+ only_if=format("ls {hadoop_conf_dir}"))
+
+
+def load_version(struct_out_file):
+ """
+ Load version from file. Made a separate method for testing
+ """
+ json_version = None
+ try:
+ if os.path.exists(struct_out_file):
+ with open(struct_out_file, 'r') as fp:
+ json_info = json.load(fp)
+ json_version = json_info['version']
+ except:
+ pass
+
+ return json_version
+
+
+def link_configs(struct_out_file):
+ """
+ Links configs, only on a fresh install of HDF-0.3 and higher
+ """
+ import params
+
+ if not Script.is_stack_greater_or_equal("0.3"):
+ Logger.info("Can only link configs for HDF-0.3 and higher.")
+ return
+
+ json_version = load_version(struct_out_file)
+
+ if not json_version:
+ Logger.info("Could not load 'version' from {0}".format(struct_out_file))
+ return
+
+ # On parallel command execution this should be executed by a single process at a time.
+ with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
+ for k, v in conf_select.get_package_dirs().iteritems():
+ conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/files/changeToSecureUid.sh
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/files/changeToSecureUid.sh b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/files/changeToSecureUid.sh
new file mode 100644
index 0000000..08542c4
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/files/changeToSecureUid.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+username=$1
+directories=$2
+
+function find_available_uid() {
+ for ((i=1001; i<=2000; i++))
+ do
+ grep -q $i /etc/passwd
+ if [ "$?" -ne 0 ]
+ then
+ newUid=$i
+ break
+ fi
+ done
+}
+
+find_available_uid
+
+if [ $newUid -eq 0 ]
+then
+ echo "Failed to find Uid between 1000 and 2000"
+ exit 1
+fi
+
+set -e
+
+dir_array=($(echo $directories | sed 's/,/\n/g'))
+old_uid=$(id -u $username)
+sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
+echo "Changing uid of $username from $old_uid to $newUid"
+echo "Changing directory permisions for ${dir_array[@]}"
+$sudo_prefix usermod -u $newUid $username && for dir in ${dir_array[@]} ; do ls $dir 2> /dev/null && echo "Changing permission for $dir" && $sudo_prefix chown -Rh $newUid $dir ; done
+exit 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/hook.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/hook.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/hook.py
new file mode 100644
index 0000000..c34be0b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/hook.py
@@ -0,0 +1,36 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from shared_initialization import *
+
+class BeforeAnyHook(Hook):
+
+ def hook(self, env):
+ import params
+ env.set_params(params)
+
+ setup_users()
+ if params.has_namenode or params.dfs_type == 'HCFS':
+ setup_hadoop_env()
+ setup_java()
+
+if __name__ == "__main__":
+ BeforeAnyHook().execute()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/params.py
new file mode 100644
index 0000000..02ef721
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/params.py
@@ -0,0 +1,235 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import collections
+import re
+import os
+
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions import default
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import format_jvm_option
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.expect import expect
+from ambari_commons.os_check import OSCheck
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+dfs_type = default("/commandParams/dfs_type", "")
+
+artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
+jdk_name = default("/hostLevelParams/jdk_name", None)
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+jdk_location = config['hostLevelParams']['jdk_location']
+
+sudo = AMBARI_SUDO_BINARY
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+restart_type = default("/commandParams/restart_type", "")
+version = default("/commandParams/version", None)
+# Handle upgrade and downgrade
+if (restart_type.lower() == "rolling_upgrade" or restart_type.lower() == "nonrolling_upgrade") and version:
+ stack_version_formatted = format_stack_version(version)
+
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+
+# Some datanode settings
+dfs_dn_addr = default('/configurations/hdfs-site/dfs.datanode.address', None)
+dfs_dn_http_addr = default('/configurations/hdfs-site/dfs.datanode.http.address', None)
+dfs_dn_https_addr = default('/configurations/hdfs-site/dfs.datanode.https.address', None)
+dfs_http_policy = default('/configurations/hdfs-site/dfs.http.policy', None)
+secure_dn_ports_are_in_use = False
+
+def get_port(address):
+ """
+ Extracts port from the address like 0.0.0.0:1019
+ """
+ if address is None:
+ return None
+ m = re.search(r'(?:http(?:s)?://)?([\w\d.]*):(\d{1,5})', address)
+ if m is not None:
+ return int(m.group(2))
+ else:
+ return None
+
+def is_secure_port(port):
+ """
+ Returns True if port is root-owned at *nix systems
+ """
+ if port is not None:
+ return port < 1024
+ else:
+ return False
+
+# hadoop default params
+mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
+
+# upgrades would cause these directories to have a version instead of "current"
+# which would cause a lot of problems when writing out hadoop-env.sh; instead
+# force the use of "current" in the hook
+hadoop_home = stack_select.get_hadoop_dir("home", force_latest_on_upgrade=True)
+hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec", force_latest_on_upgrade=True)
+
+hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
+hadoop_secure_dn_user = hdfs_user
+hadoop_dir = "/etc/hadoop"
+versioned_stack_root = '/usr/hdf/current'
+hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir")
+datanode_max_locked_memory = config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory']
+is_datanode_max_locked_memory_set = not is_empty(config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory'])
+
+# HDF 0.2+ params
+if Script.is_stack_greater_or_equal("0.2"):
+ mapreduce_libs_path = "/usr/hdf/current/hadoop-mapreduce-client/*"
+
+ # not supported in HDF 0.2+
+ hadoop_conf_empty_dir = None
+
+ if not security_enabled:
+ hadoop_secure_dn_user = '""'
+ else:
+ dfs_dn_port = get_port(dfs_dn_addr)
+ dfs_dn_http_port = get_port(dfs_dn_http_addr)
+ dfs_dn_https_port = get_port(dfs_dn_https_addr)
+ # We try to avoid inability to start datanode as a plain user due to usage of root-owned ports
+ if dfs_http_policy == "HTTPS_ONLY":
+ secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_https_port)
+ elif dfs_http_policy == "HTTP_AND_HTTPS":
+ secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port) or is_secure_port(dfs_dn_https_port)
+ else: # params.dfs_http_policy == "HTTP_ONLY" or not defined:
+ secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port)
+ if secure_dn_ports_are_in_use:
+ hadoop_secure_dn_user = hdfs_user
+ else:
+ hadoop_secure_dn_user = '""'
+
+#hadoop params
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
+hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger']
+
+jsvc_path = "/usr/lib/bigtop-utils"
+
+hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
+namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
+namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
+namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
+
+jtnode_opt_newsize = "200m"
+jtnode_opt_maxnewsize = "200m"
+jtnode_heapsize = "1024m"
+ttnode_heapsize = "1024m"
+
+dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
+nfsgateway_heapsize = config['configurations']['hadoop-env']['nfsgateway_heapsize']
+mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+hadoop_env_sh_template = config['configurations']['hadoop-env']['content']
+
+#users and groups
+hbase_user = config['configurations']['hbase-env']['hbase_user']
+smoke_user = config['configurations']['cluster-env']['smokeuser']
+gmetad_user = config['configurations']['ganglia-env']["gmetad_user"]
+gmond_user = config['configurations']['ganglia-env']["gmond_user"]
+tez_user = config['configurations']['tez-env']["tez_user"]
+oozie_user = config['configurations']['oozie-env']["oozie_user"]
+falcon_user = config['configurations']['falcon-env']["falcon_user"]
+ranger_user = config['configurations']['ranger-env']["ranger_user"]
+nifi_user = config['configurations']['nifi-env']["nifi_user"]
+
+user_group = config['configurations']['cluster-env']['user_group']
+
+ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
+oozie_servers = default("/clusterHostInfo/oozie_server", [])
+falcon_server_hosts = default("/clusterHostInfo/falcon_server_hosts", [])
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+nifi_master_hosts = default("/clusterHostInfo/nifi_master_hosts", [])
+
+has_namenode = not len(namenode_host) == 0
+has_ganglia_server = not len(ganglia_server_hosts) == 0
+has_tez = 'tez-site' in config['configurations']
+has_hbase_masters = not len(hbase_master_hosts) == 0
+has_oozie_server = not len(oozie_servers) == 0
+has_falcon_server_hosts = not len(falcon_server_hosts) == 0
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+has_nifi_master = not len(nifi_master_hosts) == 0
+
+if has_namenode or dfs_type == 'HCFS':
+ hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
+
+hbase_tmp_dir = "/tmp/hbase-hbase"
+
+proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users")
+ranger_group = config['configurations']['ranger-env']['ranger_group']
+nifi_group = config['configurations']['nifi-env']["nifi_group"]
+dfs_cluster_administrators_group = config['configurations']['hdfs-site']["dfs.cluster.administrators"]
+
+ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False)
+fetch_nonlocal_groups = config['configurations']['cluster-env']["fetch_nonlocal_groups"]
+
+smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}")
+if has_hbase_masters:
+ hbase_user_dirs = format("/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}")
+#repo params
+repo_info = config['hostLevelParams']['repo_info']
+service_repo_info = default("/hostLevelParams/service_repo_info",None)
+
+user_to_groups_dict = collections.defaultdict(lambda:[user_group])
+user_to_groups_dict[smoke_user] = [proxyuser_group]
+if has_ganglia_server:
+ user_to_groups_dict[gmond_user] = [gmond_user]
+ user_to_groups_dict[gmetad_user] = [gmetad_user]
+if has_tez:
+ user_to_groups_dict[tez_user] = [proxyuser_group]
+if has_oozie_server:
+ user_to_groups_dict[oozie_user] = [proxyuser_group]
+if has_falcon_server_hosts:
+ user_to_groups_dict[falcon_user] = [proxyuser_group]
+if has_ranger_admin:
+ user_to_groups_dict[ranger_user] = [ranger_group]
+if has_nifi_master:
+ user_to_groups_dict[nifi_user] = [nifi_group]
+
+user_to_gid_dict = collections.defaultdict(lambda:user_group)
+
+user_list = json.loads(config['hostLevelParams']['user_list'])
+group_list = json.loads(config['hostLevelParams']['group_list'])
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+
+tez_am_view_acls = config['configurations']['tez-site']["tez.am.view-acls"]
+override_uid = str(default("/configurations/cluster-env/override_uid", "true")).lower()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/shared_initialization.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/shared_initialization.py
new file mode 100644
index 0000000..e8e8b08
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-ANY/scripts/shared_initialization.py
@@ -0,0 +1,224 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import getpass
+import tempfile
+from copy import copy
+from resource_management.libraries.functions.version import compare_versions
+from resource_management import *
+
+def setup_users():
+ """
+ Creates users before cluster installation
+ """
+ import params
+ should_create_users_and_groups = not params.host_sys_prepped and not params.ignore_groupsusers_create
+
+ if should_create_users_and_groups:
+ for group in params.group_list:
+ Group(group,
+ )
+
+ for user in params.user_list:
+ User(user,
+ gid = params.user_to_gid_dict[user],
+ groups = params.user_to_groups_dict[user],
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups
+ )
+
+ if params.override_uid == "true":
+ set_uid(params.smoke_user, params.smoke_user_dirs)
+ else:
+ Logger.info('Skipping setting uid for smoke user as host is sys prepped')
+ else:
+ Logger.info('Skipping creation of User and Group as host is sys prepped or ignore_groupsusers_create flag is on')
+ pass
+
+
+ if params.has_hbase_masters:
+ Directory (params.hbase_tmp_dir,
+ owner = params.hbase_user,
+ mode=0775,
+ create_parents = True,
+ cd_access="a",
+ )
+ if not params.host_sys_prepped and params.override_uid == "true":
+ set_uid(params.hbase_user, params.hbase_user_dirs)
+ else:
+ Logger.info('Skipping setting uid for hbase user as host is sys prepped')
+ pass
+
+ if not params.host_sys_prepped:
+ if params.has_namenode:
+ if should_create_users_and_groups:
+ create_dfs_cluster_admins()
+ if params.has_tez and params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '0.3') >= 0:
+ if should_create_users_and_groups:
+ create_tez_am_view_acls()
+ else:
+ Logger.info('Skipping setting dfs cluster admin and tez view acls as host is sys prepped')
+
+def create_dfs_cluster_admins():
+ """
+ dfs.cluster.administrators support format <comma-delimited list of usernames><space><comma-delimited list of group names>
+ """
+ import params
+
+ groups_list = create_users_and_groups(params.dfs_cluster_administrators_group)
+
+ User(params.hdfs_user,
+ groups = params.user_to_groups_dict[params.hdfs_user] + groups_list,
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups
+ )
+
+def create_tez_am_view_acls():
+
+ """
+ tez.am.view-acls support format <comma-delimited list of usernames><space><comma-delimited list of group names>
+ """
+ import params
+
+ if not params.tez_am_view_acls.startswith("*"):
+ create_users_and_groups(params.tez_am_view_acls)
+
+def create_users_and_groups(user_and_groups):
+
+ import params
+
+ parts = re.split('\s', user_and_groups)
+ if len(parts) == 1:
+ parts.append("")
+
+ users_list = parts[0].split(",") if parts[0] else []
+ groups_list = parts[1].split(",") if parts[1] else []
+
+ if users_list:
+ User(users_list,
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups
+ )
+
+ if groups_list:
+ Group(copy(groups_list),
+ )
+ return groups_list
+
+def set_uid(user, user_dirs):
+ """
+ user_dirs - comma separated directories
+ """
+ import params
+
+ File(format("{tmp_dir}/changeUid.sh"),
+ content=StaticFile("changeToSecureUid.sh"),
+ mode=0555)
+ ignore_groupsusers_create_str = str(params.ignore_groupsusers_create).lower()
+ Execute(format("{tmp_dir}/changeUid.sh {user} {user_dirs}"),
+ not_if = format("(test $(id -u {user}) -gt 1000) || ({ignore_groupsusers_create_str})"))
+
+def setup_hadoop_env():
+ import params
+ stackversion = params.stack_version_unformatted
+ Logger.info("FS Type: {0}".format(params.dfs_type))
+ if params.has_namenode or stackversion.find('Gluster') >= 0 or params.dfs_type == 'HCFS':
+ if params.security_enabled:
+ tc_owner = "root"
+ else:
+ tc_owner = params.hdfs_user
+
+ # create /etc/hadoop
+ Directory(params.hadoop_dir, mode=0755)
+
+ # HDF < 0.2 used a conf -> conf.empty symlink for /etc/hadoop/
+ if Script.is_stack_less_than("0.2"):
+ Directory(params.hadoop_conf_empty_dir, create_parents = True, owner="root",
+ group=params.user_group )
+
+ Link(params.hadoop_conf_dir, to=params.hadoop_conf_empty_dir,
+ not_if=format("ls {hadoop_conf_dir}"))
+
+ # write out hadoop-env.sh, but only if the directory exists
+ if os.path.exists(params.hadoop_conf_dir):
+ File(os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh'), owner=tc_owner,
+ group=params.user_group,
+ content=InlineTemplate(params.hadoop_env_sh_template))
+
+ # Create tmp dir for java.io.tmpdir
+ # Handle a situation when /tmp is set to noexec
+ Directory(params.hadoop_java_io_tmpdir,
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=0777
+ )
+
+def setup_java():
+ """
+ Installs jdk using specific params, that comes from ambari-server
+ """
+ import params
+
+ java_exec = format("{java_home}/bin/java")
+
+ if not os.path.isfile(java_exec):
+ if not params.jdk_name: # if custom jdk is used.
+ raise Fail(format("Unable to access {java_exec}. Confirm you have copied jdk to this host."))
+
+ jdk_curl_target = format("{tmp_dir}/{jdk_name}")
+ java_dir = os.path.dirname(params.java_home)
+
+ Directory(params.artifact_dir,
+ create_parents = True,
+ )
+
+ File(jdk_curl_target,
+ content = DownloadSource(format("{jdk_location}/{jdk_name}")),
+ not_if = format("test -f {jdk_curl_target}")
+ )
+
+ tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=params.tmp_dir)
+
+ try:
+ if params.jdk_name.endswith(".bin"):
+ chmod_cmd = ("chmod", "+x", jdk_curl_target)
+ install_cmd = format("cd {tmp_java_dir} && echo A | {jdk_curl_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}")
+ elif params.jdk_name.endswith(".gz"):
+ chmod_cmd = ("chmod","a+x", java_dir)
+ install_cmd = format("cd {tmp_java_dir} && tar -xf {jdk_curl_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}")
+
+ Directory(java_dir
+ )
+
+ Execute(chmod_cmd,
+ sudo = True,
+ )
+
+ Execute(install_cmd,
+ )
+
+ finally:
+ Directory(tmp_java_dir, action="delete")
+
+ File(format("{java_home}/bin/java"),
+ mode=0755,
+ cd_access="a",
+ )
+ Execute(('chmod', '-R', '755', params.java_home),
+ sudo = True,
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/hook.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/hook.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/hook.py
new file mode 100644
index 0000000..ce17776
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/hook.py
@@ -0,0 +1,37 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from shared_initialization import *
+from repo_initialization import *
+
+class BeforeInstallHook(Hook):
+
+ def hook(self, env):
+ import params
+
+ self.run_custom_hook('before-ANY')
+ env.set_params(params)
+
+ install_repos()
+ install_packages()
+
+if __name__ == "__main__":
+ BeforeInstallHook().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/params.py
new file mode 100644
index 0000000..6193c11
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/params.py
@@ -0,0 +1,113 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management.libraries.functions.version import format_stack_version, compare_versions
+from resource_management.core.system import System
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import default, format
+from resource_management.libraries.functions.expect import expect
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+sudo = AMBARI_SUDO_BINARY
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+#users and groups
+hbase_user = config['configurations']['hbase-env']['hbase_user']
+smoke_user = config['configurations']['cluster-env']['smokeuser']
+gmetad_user = config['configurations']['ganglia-env']["gmetad_user"]
+gmond_user = config['configurations']['ganglia-env']["gmond_user"]
+tez_user = config['configurations']['tez-env']["tez_user"]
+
+user_group = config['configurations']['cluster-env']['user_group']
+proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users")
+
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+
+# repo templates
+repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
+repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
+
+#hosts
+hostname = config["hostname"]
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+rm_host = default("/clusterHostInfo/rm_host", [])
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+oozie_servers = default("/clusterHostInfo/oozie_server", [])
+hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
+hive_server_host = default("/clusterHostInfo/hive_server_host", [])
+hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
+hs_host = default("/clusterHostInfo/hs_host", [])
+jtnode_host = default("/clusterHostInfo/jtnode_host", [])
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
+storm_server_hosts = default("/clusterHostInfo/nimbus_hosts", [])
+falcon_host = default('/clusterHostInfo/falcon_server_hosts', [])
+
+has_sqoop_client = 'sqoop-env' in config['configurations']
+has_namenode = not len(namenode_host) == 0
+has_hs = not len(hs_host) == 0
+has_resourcemanager = not len(rm_host) == 0
+has_slaves = not len(slave_hosts) == 0
+has_oozie_server = not len(oozie_servers) == 0
+has_hcat_server_host = not len(hcat_server_hosts) == 0
+has_hive_server_host = not len(hive_server_host) == 0
+has_hbase_masters = not len(hbase_master_hosts) == 0
+has_zk_host = not len(zk_hosts) == 0
+has_ganglia_server = not len(ganglia_server_hosts) == 0
+has_storm_server = not len(storm_server_hosts) == 0
+has_falcon_server = not len(falcon_host) == 0
+has_tez = 'tez-site' in config['configurations']
+
+is_namenode_master = hostname in namenode_host
+is_jtnode_master = hostname in jtnode_host
+is_rmnode_master = hostname in rm_host
+is_hsnode_master = hostname in hs_host
+is_hbase_master = hostname in hbase_master_hosts
+is_slave = hostname in slave_hosts
+if has_ganglia_server:
+ ganglia_server_host = ganglia_server_hosts[0]
+
+hbase_tmp_dir = "/tmp/hbase-hbase"
+
+#security params
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+#java params
+java_home = config['hostLevelParams']['java_home']
+artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
+jdk_name = default("/hostLevelParams/jdk_name", None) # None when jdk is already installed by user
+jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user
+jce_location = config['hostLevelParams']['jdk_location']
+jdk_location = config['hostLevelParams']['jdk_location']
+ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False)
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+
+smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}")
+if has_hbase_masters:
+ hbase_user_dirs = format("/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}")
+#repo params
+repo_info = config['hostLevelParams']['repo_info']
+service_repo_info = default("/hostLevelParams/service_repo_info",None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/repo_initialization.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/repo_initialization.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/repo_initialization.py
new file mode 100644
index 0000000..c37c6bd
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/repo_initialization.py
@@ -0,0 +1,68 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.resources.repository import Repository
+from resource_management.core.logger import Logger
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+
+# components_lits = repoName + postfix
+_UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
+
+def _alter_repo(action, repo_string, repo_template):
+ """
+ @param action: "delete" or "create"
+ @param repo_string: e.g. "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDF/centos6/2.x/updates/0.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDF-2.0._\",\"repoName\":\"HDF\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDF/centos6/2.x/updates/0.0.6.0\"}]"
+ """
+ repo_dicts = json.loads(repo_string)
+
+ if not isinstance(repo_dicts, list):
+ repo_dicts = [repo_dicts]
+
+ if 0 == len(repo_dicts):
+ Logger.info("Repository list is empty. Ambari may not be managing the repositories.")
+ else:
+ Logger.info("Initializing {0} repositories".format(str(len(repo_dicts))))
+
+ for repo in repo_dicts:
+ if not 'baseUrl' in repo:
+ repo['baseUrl'] = None
+ if not 'mirrorsList' in repo:
+ repo['mirrorsList'] = None
+
+ ubuntu_components = [ repo['repoName'] ] + _UBUNTU_REPO_COMPONENTS_POSTFIX
+
+ Repository(repo['repoId'],
+ action = action,
+ base_url = repo['baseUrl'],
+ mirror_list = repo['mirrorsList'],
+ repo_file_name = repo['repoName'],
+ repo_template = repo_template,
+ components = ubuntu_components, # ubuntu specific
+ )
+
+def install_repos():
+ import params
+ if params.host_sys_prepped:
+ return
+
+ template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu
+ _alter_repo("create", params.repo_info, template)
+ if params.service_repo_info:
+ _alter_repo("create", params.service_repo_info, template)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/shared_initialization.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/shared_initialization.py
new file mode 100644
index 0000000..f07aea5
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-INSTALL/scripts/shared_initialization.py
@@ -0,0 +1,37 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management.libraries.functions import stack_tools
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.core.resources.packaging import Package
+
+def install_packages():
+ import params
+ if params.host_sys_prepped:
+ return
+
+ packages = ['unzip', 'curl']
+ if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '0.2') >= 0:
+ stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
+ packages.append(stack_selector_package)
+ Package(packages,
+ retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
+ retry_count=params.agent_stack_retry_count)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-RESTART/scripts/hook.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-RESTART/scripts/hook.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-RESTART/scripts/hook.py
new file mode 100644
index 0000000..14b9d99
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-RESTART/scripts/hook.py
@@ -0,0 +1,29 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+class BeforeRestartHook(Hook):
+
+ def hook(self, env):
+ self.run_custom_hook('before-START')
+
+if __name__ == "__main__":
+ BeforeRestartHook().execute()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/checkForFormat.sh b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/checkForFormat.sh
new file mode 100644
index 0000000..68aa96d
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/checkForFormat.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+export hdfs_user=$1
+shift
+export conf_dir=$1
+shift
+export bin_dir=$1
+shift
+export mark_dir=$1
+shift
+export name_dirs=$*
+
+export EXIT_CODE=0
+export command="namenode -format"
+export list_of_non_empty_dirs=""
+
+mark_file=/var/run/hadoop/hdfs/namenode-formatted
+if [[ -f ${mark_file} ]] ; then
+ /var/lib/ambari-agent/ambari-sudo.sh rm -f ${mark_file}
+ /var/lib/ambari-agent/ambari-sudo.sh mkdir -p ${mark_dir}
+fi
+
+if [[ ! -d $mark_dir ]] ; then
+ for dir in `echo $name_dirs | tr ',' ' '` ; do
+ echo "NameNode Dirname = $dir"
+ cmd="ls $dir | wc -l | grep -q ^0$"
+ eval $cmd
+ if [[ $? -ne 0 ]] ; then
+ (( EXIT_CODE = $EXIT_CODE + 1 ))
+ list_of_non_empty_dirs="$list_of_non_empty_dirs $dir"
+ fi
+ done
+
+ if [[ $EXIT_CODE == 0 ]] ; then
+ /var/lib/ambari-agent/ambari-sudo.sh su ${hdfs_user} - -s /bin/bash -c "export PATH=$PATH:$bin_dir ; yes Y | hdfs --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
+ else
+ echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
+ fi
+else
+ echo "${mark_dir} exists. Namenode DFS already formatted"
+fi
+
+exit $EXIT_CODE
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/fast-hdfs-resource.jar
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/fast-hdfs-resource.jar b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/fast-hdfs-resource.jar
new file mode 100644
index 0000000..c90890b
Binary files /dev/null and b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/fast-hdfs-resource.jar differ
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/task-log4j.properties
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/task-log4j.properties b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/task-log4j.properties
new file mode 100644
index 0000000..7e12962
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/task-log4j.properties
@@ -0,0 +1,134 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file rolled daily:
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.iscleanup=false
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+# Rolling File Appender
+#
+
+#log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Logfile size and and 30-day backups
+#log4j.appender.RFA.MaxFileSize=1MB
+#log4j.appender.RFA.MaxBackupIndex=30
+
+#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+# Custom Logging levels
+
+hadoop.metrics.log.level=INFO
+#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}
+
+# Jets3t library
+log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+
+#
+# Null Appender
+# Trap security logger on the hadoop client side
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
+
+# Removes "deprecated" messages
+log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/topology_script.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/topology_script.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/topology_script.py
new file mode 100644
index 0000000..0f7a55c
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/files/topology_script.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import sys, os
+from string import join
+import ConfigParser
+
+
+DEFAULT_RACK = "/default-rack"
+DATA_FILE_NAME = os.path.dirname(os.path.abspath(__file__)) + "/topology_mappings.data"
+SECTION_NAME = "network_topology"
+
+class TopologyScript():
+
+ def load_rack_map(self):
+ try:
+ #RACK_MAP contains both host name vs rack and ip vs rack mappings
+ mappings = ConfigParser.ConfigParser()
+ mappings.read(DATA_FILE_NAME)
+ return dict(mappings.items(SECTION_NAME))
+ except ConfigParser.NoSectionError:
+ return {}
+
+ def get_racks(self, rack_map, args):
+ if len(args) == 1:
+ return DEFAULT_RACK
+ else:
+ return join([self.lookup_by_hostname_or_ip(input_argument, rack_map) for input_argument in args[1:]],)
+
+ def lookup_by_hostname_or_ip(self, hostname_or_ip, rack_map):
+ #try looking up by hostname
+ rack = rack_map.get(hostname_or_ip)
+ if rack is not None:
+ return rack
+ #try looking up by ip
+ rack = rack_map.get(self.extract_ip(hostname_or_ip))
+ #try by localhost since hadoop could be passing in 127.0.0.1 which might not be mapped
+ return rack if rack is not None else rack_map.get("localhost.localdomain", DEFAULT_RACK)
+
+ #strips out port and slashes in case hadoop passes in something like 127.0.0.1/127.0.0.1:50010
+ def extract_ip(self, container_string):
+ return container_string.split("/")[0].split(":")[0]
+
+ def execute(self, args):
+ rack_map = self.load_rack_map()
+ rack = self.get_racks(rack_map, args)
+ print rack
+
+if __name__ == "__main__":
+ TopologyScript().execute(sys.argv)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/hook.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/hook.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/hook.py
new file mode 100644
index 0000000..f21e4b1
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/hook.py
@@ -0,0 +1,39 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from rack_awareness import create_topology_script_and_mapping
+from shared_initialization import setup_hadoop, setup_configs, create_javahome_symlink
+
+class BeforeStartHook(Hook):
+
+ def hook(self, env):
+ import params
+
+ self.run_custom_hook('before-ANY')
+ env.set_params(params)
+
+ setup_hadoop()
+ setup_configs()
+ create_javahome_symlink()
+ create_topology_script_and_mapping()
+
+if __name__ == "__main__":
+ BeforeStartHook().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py
new file mode 100644
index 0000000..6a70675
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py
@@ -0,0 +1,318 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import default
+from resource_management.libraries.functions import format_jvm_option
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.version import format_stack_version, compare_versions
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+
+config = Script.get_config()
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+dfs_type = default("/commandParams/dfs_type", "")
+hadoop_conf_dir = "/etc/hadoop/conf"
+
+component_list = default("/localComponents", [])
+
+hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir']
+
+# hadoop default params
+mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
+
+hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec")
+hadoop_lib_home = stack_select.get_hadoop_dir("lib")
+hadoop_bin = stack_select.get_hadoop_dir("sbin")
+hadoop_home = '/usr'
+create_lib_snappy_symlinks = True
+
+# HDF 0.2+ params
+if Script.is_stack_greater_or_equal("0.2"):
+ mapreduce_libs_path = "/usr/hdf/current/hadoop-mapreduce-client/*"
+ hadoop_home = stack_select.get_hadoop_dir("home")
+ create_lib_snappy_symlinks = False
+
+current_service = config['serviceName']
+
+#security params
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+#users and groups
+has_hadoop_env = 'hadoop-env' in config['configurations']
+mapred_user = config['configurations']['mapred-env']['mapred_user']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+yarn_user = config['configurations']['yarn-env']['yarn_user']
+
+user_group = config['configurations']['cluster-env']['user_group']
+
+#hosts
+hostname = config["hostname"]
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+rm_host = default("/clusterHostInfo/rm_host", [])
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+oozie_servers = default("/clusterHostInfo/oozie_server", [])
+hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
+hive_server_host = default("/clusterHostInfo/hive_server_host", [])
+hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
+hs_host = default("/clusterHostInfo/hs_host", [])
+jtnode_host = default("/clusterHostInfo/jtnode_host", [])
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
+
+has_namenode = not len(namenode_host) == 0
+has_resourcemanager = not len(rm_host) == 0
+has_slaves = not len(slave_hosts) == 0
+has_oozie_server = not len(oozie_servers) == 0
+has_hcat_server_host = not len(hcat_server_hosts) == 0
+has_hive_server_host = not len(hive_server_host) == 0
+has_hbase_masters = not len(hbase_master_hosts) == 0
+has_zk_host = not len(zk_hosts) == 0
+has_ganglia_server = not len(ganglia_server_hosts) == 0
+has_metric_collector = not len(ams_collector_hosts) == 0
+
+is_namenode_master = hostname in namenode_host
+is_jtnode_master = hostname in jtnode_host
+is_rmnode_master = hostname in rm_host
+is_hsnode_master = hostname in hs_host
+is_hbase_master = hostname in hbase_master_hosts
+is_slave = hostname in slave_hosts
+if has_ganglia_server:
+ ganglia_server_host = ganglia_server_hosts[0]
+if has_metric_collector:
+ if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
+ metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
+ else:
+ metric_collector_host = ams_collector_hosts[0]
+ if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ else:
+ metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188")
+ if metric_collector_web_address.find(':') != -1:
+ metric_collector_port = metric_collector_web_address.split(':')[1]
+ else:
+ metric_collector_port = '6188'
+ if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
+ metric_collector_protocol = 'https'
+ else:
+ metric_collector_protocol = 'http'
+ metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
+ metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
+ metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
+
+ pass
+metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60)
+metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10)
+
+#hadoop params
+
+if has_namenode or dfs_type == 'HCFS':
+ hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}")
+ hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
+ task_log4j_properties_location = os.path.join(hadoop_conf_dir, "task-log4j.properties")
+
+hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+hbase_tmp_dir = "/tmp/hbase-hbase"
+#db params
+server_db_name = config['hostLevelParams']['db_name']
+db_driver_filename = config['hostLevelParams']['db_driver_filename']
+oracle_driver_url = config['hostLevelParams']['oracle_jdbc_url']
+mysql_driver_url = config['hostLevelParams']['mysql_jdbc_url']
+ambari_server_resources = config['hostLevelParams']['jdk_location']
+oracle_driver_symlink_url = format("{ambari_server_resources}oracle-jdbc-driver.jar")
+mysql_driver_symlink_url = format("{ambari_server_resources}mysql-jdbc-driver.jar")
+
+ambari_db_rca_url = config['hostLevelParams']['ambari_db_rca_url'][0]
+ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver'][0]
+ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username'][0]
+ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password'][0]
+
+if has_namenode and 'rca_enabled' in config['configurations']['hadoop-env']:
+ rca_enabled = config['configurations']['hadoop-env']['rca_enabled']
+else:
+ rca_enabled = False
+rca_disabled_prefix = "###"
+if rca_enabled == True:
+ rca_prefix = ""
+else:
+ rca_prefix = rca_disabled_prefix
+
+#hadoop-env.sh
+java_home = config['hostLevelParams']['java_home']
+
+jsvc_path = "/usr/lib/bigtop-utils"
+
+hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
+namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
+namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
+namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
+
+jtnode_opt_newsize = "200m"
+jtnode_opt_maxnewsize = "200m"
+jtnode_heapsize = "1024m"
+ttnode_heapsize = "1024m"
+
+dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
+mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+
+#log4j.properties
+
+yarn_log_dir_prefix = default("/configurations/yarn-env/yarn_log_dir_prefix","/var/log/hadoop-yarn")
+
+dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None)
+
+#log4j.properties
+if (('hdfs-log4j' in config['configurations']) and ('content' in config['configurations']['hdfs-log4j'])):
+ log4j_props = config['configurations']['hdfs-log4j']['content']
+ if (('yarn-log4j' in config['configurations']) and ('content' in config['configurations']['yarn-log4j'])):
+ log4j_props += config['configurations']['yarn-log4j']['content']
+else:
+ log4j_props = None
+
+refresh_topology = False
+command_params = config["commandParams"] if "commandParams" in config else None
+if command_params is not None:
+ refresh_topology = bool(command_params["refresh_topology"]) if "refresh_topology" in command_params else False
+
+ambari_libs_dir = "/var/lib/ambari-agent/lib"
+is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+
+#host info
+all_hosts = default("/clusterHostInfo/all_hosts", [])
+all_racks = default("/clusterHostInfo/all_racks", [])
+all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", [])
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+
+#topology files
+net_topology_script_file_path = "/etc/hadoop/conf/topology_script.py"
+net_topology_script_dir = os.path.dirname(net_topology_script_file_path)
+net_topology_mapping_data_file_name = 'topology_mappings.data'
+net_topology_mapping_data_file_path = os.path.join(net_topology_script_dir, net_topology_mapping_data_file_name)
+
+#Added logic to create /tmp and /user directory for HCFS stack.
+has_core_site = 'core-site' in config['configurations']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+kinit_path_local = get_kinit_path()
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None)
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+smoke_user = config['configurations']['cluster-env']['smokeuser']
+smoke_hdfs_user_dir = format("/user/{smoke_user}")
+smoke_hdfs_user_mode = 0770
+
+
+##### Namenode RPC ports - metrics config section start #####
+
+# Figure out the rpc ports for current namenode
+nn_rpc_client_port = None
+nn_rpc_dn_port = None
+nn_rpc_healthcheck_port = None
+
+namenode_id = None
+namenode_rpc = None
+
+dfs_ha_enabled = False
+dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.internal.nameservices', None)
+if dfs_ha_nameservices is None:
+ dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.nameservices', None)
+dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None)
+
+dfs_ha_namemodes_ids_list = []
+other_namenode_id = None
+
+if dfs_ha_namenode_ids:
+ dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",")
+ dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list)
+ if dfs_ha_namenode_ids_array_len > 1:
+ dfs_ha_enabled = True
+
+if dfs_ha_enabled:
+ for nn_id in dfs_ha_namemodes_ids_list:
+ nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')]
+ if hostname in nn_host:
+ namenode_id = nn_id
+ namenode_rpc = nn_host
+ pass
+ pass
+else:
+ namenode_rpc = default('/configurations/hdfs-site/dfs.namenode.rpc-address', None)
+
+if namenode_rpc:
+ nn_rpc_client_port = namenode_rpc.split(':')[1].strip()
+
+if dfs_ha_enabled:
+ dfs_service_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.servicerpc-address.{dfs_ha_nameservices}.{namenode_id}'), None)
+ dfs_lifeline_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.lifeline.rpc-address.{dfs_ha_nameservices}.{namenode_id}'), None)
+else:
+ dfs_service_rpc_address = default('/configurations/hdfs-site/dfs.namenode.servicerpc-address', None)
+ dfs_lifeline_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.lifeline.rpc-address'), None)
+
+if dfs_service_rpc_address:
+ nn_rpc_dn_port = dfs_service_rpc_address.split(':')[1].strip()
+
+if dfs_lifeline_rpc_address:
+ nn_rpc_healthcheck_port = dfs_lifeline_rpc_address.split(':')[1].strip()
+
+is_nn_client_port_configured = False if nn_rpc_client_port is None else True
+is_nn_dn_port_configured = False if nn_rpc_dn_port is None else True
+is_nn_healthcheck_port_configured = False if nn_rpc_healthcheck_port is None else True
+
+##### end #####
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources(),
+ dfs_type = dfs_type
+)
[2/8] ambari git commit: AMBARI-18385: Add HDF management pack
(jluniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
new file mode 100644
index 0000000..40cc847
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
@@ -0,0 +1,1828 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import math
+import re
+import os
+import sys
+from math import ceil, floor
+
+from stack_advisor import DefaultStackAdvisor
+
+DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"}
+
+class HDF20StackAdvisor(DefaultStackAdvisor):
+
+ def getComponentLayoutValidations(self, services, hosts):
+ """Returns array of Validation objects about issues with hostnames components assigned to"""
+ items = super(HDF20StackAdvisor, self).getComponentLayoutValidations(services, hosts)
+
+ # Use a set for fast lookup
+ hostsSet = set(super(HDF20StackAdvisor, self).getActiveHosts([host["Hosts"] for host in hosts["items"]])) #[host["Hosts"]["host_name"] for host in hosts["items"]]
+ hostsCount = len(hostsSet)
+
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item for sublist in componentsListList for item in sublist]
+
+ # Validating cardinality
+ for component in componentsList:
+ if component["StackServiceComponents"]["cardinality"] is not None:
+ componentName = component["StackServiceComponents"]["component_name"]
+ componentDisplayName = component["StackServiceComponents"]["display_name"]
+ componentHosts = []
+ if component["StackServiceComponents"]["hostnames"] is not None:
+ componentHosts = [componentHost for componentHost in component["StackServiceComponents"]["hostnames"] if componentHost in hostsSet]
+ componentHostsCount = len(componentHosts)
+ cardinality = str(component["StackServiceComponents"]["cardinality"])
+ # cardinality types: null, 1+, 1-2, 1, ALL
+ message = None
+ if "+" in cardinality:
+ hostsMin = int(cardinality[:-1])
+ if componentHostsCount < hostsMin:
+ message = "At least {0} {1} components should be installed in cluster.".format(hostsMin, componentDisplayName)
+ elif "-" in cardinality:
+ nums = cardinality.split("-")
+ hostsMin = int(nums[0])
+ hostsMax = int(nums[1])
+ if componentHostsCount > hostsMax or componentHostsCount < hostsMin:
+ message = "Between {0} and {1} {2} components should be installed in cluster.".format(hostsMin, hostsMax, componentDisplayName)
+ elif "ALL" == cardinality:
+ if componentHostsCount != hostsCount:
+ message = "{0} component should be installed on all hosts in cluster.".format(componentDisplayName)
+ else:
+ if componentHostsCount != int(cardinality):
+ message = "Exactly {0} {1} components should be installed in cluster.".format(int(cardinality), componentDisplayName)
+
+ if message is not None:
+ items.append({"type": 'host-component', "level": 'ERROR', "message": message, "component-name": componentName})
+
+ # Validating host-usage
+ usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)]
+ usedHostsList = [item for sublist in usedHostsListList for item in sublist]
+ nonUsedHostsList = [item for item in hostsSet if item not in usedHostsList]
+ for host in nonUsedHostsList:
+ items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not used', "host": str(host) } )
+
+ return items
+
+ def getServiceConfigurationRecommenderDict(self):
+ return {
+ "KAFKA": self.recommendKAFKAConfigurations,
+ "NIFI": self.recommendNIFIConfigurations,
+ "STORM": self.recommendStormConfigurations,
+ "AMBARI_METRICS": self.recommendAmsConfigurations,
+ "RANGER": self.recommendRangerConfigurations,
+ "LOGSEARCH" : self.recommendLogsearchConfigurations
+ }
+
+ def putProperty(self, config, configType, services=None):
+ userConfigs = {}
+ changedConfigs = []
+ # if services parameter, prefer values, set by user
+ if services:
+ if 'configurations' in services.keys():
+ userConfigs = services['configurations']
+ if 'changed-configurations' in services.keys():
+ changedConfigs = services["changed-configurations"]
+
+ if configType not in config:
+ config[configType] = {}
+ if"properties" not in config[configType]:
+ config[configType]["properties"] = {}
+ def appendProperty(key, value):
+ # If property exists in changedConfigs, do not override, use user defined property
+ if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
+ config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
+ else:
+ config[configType]["properties"][key] = str(value)
+ return appendProperty
+
+ def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
+ for changedConfig in changedConfigs:
+ if changedConfig['type']==configType and changedConfig['name']==propertyName:
+ return True
+ return False
+
+ def putPropertyAttribute(self, config, configType):
+ if configType not in config:
+ config[configType] = {}
+ def appendPropertyAttribute(key, attribute, attributeValue):
+ if "property_attributes" not in config[configType]:
+ config[configType]["property_attributes"] = {}
+ if key not in config[configType]["property_attributes"]:
+ config[configType]["property_attributes"][key] = {}
+ config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue)
+ return appendPropertyAttribute
+
+ def recommendKAFKAConfigurations(self, configurations, clusterData, services, hosts):
+ kafka_broker = getServicesSiteProperties(services, "kafka-broker")
+
+ # kerberos security for kafka is decided from `security.inter.broker.protocol` property value
+ security_enabled = (kafka_broker is not None and 'security.inter.broker.protocol' in kafka_broker
+ and 'SASL' in kafka_broker['security.inter.broker.protocol'])
+ putKafkaBrokerProperty = self.putProperty(configurations, "kafka-broker", services)
+ putKafkaLog4jProperty = self.putProperty(configurations, "kafka-log4j", services)
+ putKafkaBrokerAttributes = self.putPropertyAttribute(configurations, "kafka-broker")
+
+ #If AMS is part of Services, use the KafkaTimelineMetricsReporter for metric reporting. Default is ''.
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if "AMBARI_METRICS" in servicesList:
+ putKafkaBrokerProperty('kafka.metrics.reporters', 'org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter')
+
+ if "ranger-env" in services["configurations"] and "ranger-kafka-plugin-properties" in services["configurations"] and \
+ "ranger-kafka-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]:
+ putKafkaRangerPluginProperty = self.putProperty(configurations, "ranger-kafka-plugin-properties", services)
+ rangerEnvKafkaPluginProperty = services["configurations"]["ranger-env"]["properties"]["ranger-kafka-plugin-enabled"]
+ putKafkaRangerPluginProperty("ranger-kafka-plugin-enabled", rangerEnvKafkaPluginProperty)
+
+ if 'ranger-kafka-plugin-properties' in services['configurations'] and ('ranger-kafka-plugin-enabled' in services['configurations']['ranger-kafka-plugin-properties']['properties']):
+ kafkaLog4jRangerLines = [{
+ "name": "log4j.appender.rangerAppender",
+ "value": "org.apache.log4j.DailyRollingFileAppender"
+ },
+ {
+ "name": "log4j.appender.rangerAppender.DatePattern",
+ "value": "'.'yyyy-MM-dd-HH"
+ },
+ {
+ "name": "log4j.appender.rangerAppender.File",
+ "value": "${kafka.logs.dir}/ranger_kafka.log"
+ },
+ {
+ "name": "log4j.appender.rangerAppender.layout",
+ "value": "org.apache.log4j.PatternLayout"
+ },
+ {
+ "name": "log4j.appender.rangerAppender.layout.ConversionPattern",
+ "value": "%d{ISO8601} %p [%t] %C{6} (%F:%L) - %m%n"
+ },
+ {
+ "name": "log4j.logger.org.apache.ranger",
+ "value": "INFO, rangerAppender"
+ }]
+
+ rangerPluginEnabled=''
+ if 'ranger-kafka-plugin-properties' in configurations and 'ranger-kafka-plugin-enabled' in configurations['ranger-kafka-plugin-properties']['properties']:
+ rangerPluginEnabled = configurations['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled']
+ elif 'ranger-kafka-plugin-properties' in services['configurations'] and 'ranger-kafka-plugin-enabled' in services['configurations']['ranger-kafka-plugin-properties']['properties']:
+ rangerPluginEnabled = services['configurations']['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled']
+
+ if rangerPluginEnabled and rangerPluginEnabled.lower() == "Yes".lower():
+ # recommend authorizer.class.name
+ putKafkaBrokerProperty("authorizer.class.name", 'org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer')
+ # change kafka-log4j when ranger plugin is installed
+
+ if 'kafka-log4j' in services['configurations'] and 'content' in services['configurations']['kafka-log4j']['properties']:
+ kafkaLog4jContent = services['configurations']['kafka-log4j']['properties']['content']
+ for item in range(len(kafkaLog4jRangerLines)):
+ if kafkaLog4jRangerLines[item]["name"] not in kafkaLog4jContent:
+ kafkaLog4jContent+= '\n' + kafkaLog4jRangerLines[item]["name"] + '=' + kafkaLog4jRangerLines[item]["value"]
+ putKafkaLog4jProperty("content",kafkaLog4jContent)
+
+
+ else:
+ # Kerberized Cluster with Ranger plugin disabled
+ if security_enabled and 'kafka-broker' in services['configurations'] and 'authorizer.class.name' in services['configurations']['kafka-broker']['properties'] and \
+ services['configurations']['kafka-broker']['properties']['authorizer.class.name'] == 'org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer':
+ putKafkaBrokerProperty("authorizer.class.name", 'kafka.security.auth.SimpleAclAuthorizer')
+ # Non-kerberos Cluster with Ranger plugin disabled
+ else:
+ putKafkaBrokerAttributes('authorizer.class.name', 'delete', 'true')
+
+ # Non-Kerberos Cluster without Ranger
+ elif not security_enabled:
+ putKafkaBrokerAttributes('authorizer.class.name', 'delete', 'true')
+
+ def getOracleDBConnectionHostPort(self, db_type, db_host, rangerDbName):
+ connection_string = self.getDBConnectionHostPort(db_type, db_host)
+ colon_count = db_host.count(':')
+ if colon_count == 1 and '/' in db_host:
+ connection_string = "//" + connection_string
+ elif colon_count == 0 or colon_count == 1:
+ connection_string = "//" + connection_string + "/" + rangerDbName if rangerDbName else "//" + connection_string
+
+ return connection_string
+
+ def getDBConnectionHostPort(self, db_type, db_host):
+ connection_string = ""
+ if db_type is None or db_type == "":
+ return connection_string
+ else:
+ colon_count = db_host.count(':')
+ if colon_count == 0:
+ if DB_TYPE_DEFAULT_PORT_MAP.has_key(db_type):
+ connection_string = db_host + ":" + DB_TYPE_DEFAULT_PORT_MAP[db_type]
+ else:
+ connection_string = db_host
+ elif colon_count == 1:
+ connection_string = db_host
+ elif colon_count == 2:
+ connection_string = db_host
+
+ return connection_string
+
+ def recommendNIFIConfigurations(self, configurations, clusterData, services, hosts):
+ nifi = getServicesSiteProperties(services, "nifi")
+
+ if "ranger-env" in services["configurations"] and "ranger-nifi-plugin-properties" in services["configurations"] and \
+ "ranger-nifi-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]:
+ putNiFiRangerPluginProperty = self.putProperty(configurations, "ranger-nifi-plugin-properties", services)
+ rangerEnvNiFiPluginProperty = services["configurations"]["ranger-env"]["properties"]["ranger-nifi-plugin-enabled"]
+ putNiFiRangerPluginProperty("ranger-nifi-plugin-enabled", rangerEnvNiFiPluginProperty)
+
+ if rangerEnvNiFiPluginProperty == 'Yes' and \
+ "nifi.authentication" in services["configurations"]["ranger-nifi-plugin-properties"]["properties"] and \
+ "nifi.node.ssl.isenabled" in services["configurations"]["nifi-ambari-ssl-config"]["properties"]:
+ nifiAmbariSSLConfig = 'SSL' if services["configurations"]["nifi-ambari-ssl-config"]["properties"]["nifi.node.ssl.isenabled"] == 'true' else 'NONE'
+ putNiFiRangerPluginProperty("nifi.authentication",nifiAmbariSSLConfig)
+
+ def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
+
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putRangerEnvProperty = self.putProperty(configurations, "ranger-env", services)
+ putRangerAdminProperty = self.putProperty(configurations, "admin-properties", services)
+ putRangerAdminSiteProperty = self.putProperty(configurations, "ranger-admin-site", services)
+ putRangerUgsyncSite = self.putProperty(configurations, "ranger-ugsync-site", services)
+ putTagsyncAppProperty = self.putProperty(configurations, "tagsync-application-properties", services)
+ putTagsyncSiteProperty = self.putProperty(configurations, "ranger-tagsync-site", services)
+
+ # get zookeeper hosts
+ zookeeper_host_port = self.getZKHostPortString(services)
+
+ # Build policymgr_external_url
+ protocol = 'http'
+ ranger_admin_host = 'localhost'
+ port = '6080'
+
+ # Check if http is disabled. For HDF this can be checked in ranger-admin-site/ranger.service.http.enabled
+ if ('ranger-admin-site' in services['configurations'] and 'ranger.service.http.enabled' in services['configurations']['ranger-admin-site']['properties'] \
+ and services['configurations']['ranger-admin-site']['properties']['ranger.service.http.enabled'].lower() == 'false'):
+ # HTTPS protocol is used
+ protocol = 'https'
+ if 'ranger-admin-site' in services['configurations'] and \
+ 'ranger.service.https.port' in services['configurations']['ranger-admin-site']['properties']:
+ port = services['configurations']['ranger-admin-site']['properties']['ranger.service.https.port']
+ else:
+ # HTTP protocol is used
+ if 'ranger-admin-site' in services['configurations'] and \
+ 'ranger.service.http.port' in services['configurations']['ranger-admin-site']['properties']:
+ port = services['configurations']['ranger-admin-site']['properties']['ranger.service.http.port']
+
+ ranger_admin_hosts = self.getComponentHostNames(services, "RANGER", "RANGER_ADMIN")
+ if ranger_admin_hosts:
+ if len(ranger_admin_hosts) > 1 \
+ and services['configurations'] \
+ and 'admin-properties' in services['configurations'] and 'policymgr_external_url' in services['configurations']['admin-properties']['properties'] \
+ and services['configurations']['admin-properties']['properties']['policymgr_external_url'] \
+ and services['configurations']['admin-properties']['properties']['policymgr_external_url'].strip():
+
+ # in case of HA deployment keep the policymgr_external_url specified in the config
+ policymgr_external_url = services['configurations']['admin-properties']['properties']['policymgr_external_url']
+ else:
+ ranger_admin_host = ranger_admin_hosts[0]
+ policymgr_external_url = "{0}://{1}:{2}".format(protocol, ranger_admin_host, port)
+
+ putRangerAdminProperty('policymgr_external_url', policymgr_external_url)
+
+ cluster_env = getServicesSiteProperties(services, "cluster-env")
+ security_enabled = cluster_env is not None and "security_enabled" in cluster_env and \
+ cluster_env["security_enabled"].lower() == "true"
+ if "ranger-env" in configurations and not security_enabled:
+ putRangerEnvProperty("ranger-storm-plugin-enabled", "No")
+
+ if 'admin-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['admin-properties']['properties']) \
+ and ('db_host' in services['configurations']['admin-properties']['properties']) and ('db_name' in services['configurations']['admin-properties']['properties']):
+
+ rangerDbFlavor = services['configurations']["admin-properties"]["properties"]["DB_FLAVOR"]
+ rangerDbHost = services['configurations']["admin-properties"]["properties"]["db_host"]
+ rangerDbName = services['configurations']["admin-properties"]["properties"]["db_name"]
+ ranger_db_url_dict = {
+ 'MYSQL': {'ranger.jpa.jdbc.driver': 'com.mysql.jdbc.Driver',
+ 'ranger.jpa.jdbc.url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/' + rangerDbName},
+ 'ORACLE': {'ranger.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver',
+ 'ranger.jpa.jdbc.url': 'jdbc:oracle:thin:@//' + self.getOracleDBConnectionHostPort(rangerDbFlavor, rangerDbHost, rangerDbName)},
+ 'POSTGRES': {'ranger.jpa.jdbc.driver': 'org.postgresql.Driver',
+ 'ranger.jpa.jdbc.url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/' + rangerDbName},
+ 'MSSQL': {'ranger.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver',
+ 'ranger.jpa.jdbc.url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';databaseName=' + rangerDbName},
+ 'SQLA': {'ranger.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver',
+ 'ranger.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';database=' + rangerDbName}
+ }
+ rangerDbProperties = ranger_db_url_dict.get(rangerDbFlavor, ranger_db_url_dict['MYSQL'])
+ for key in rangerDbProperties:
+ putRangerAdminSiteProperty(key, rangerDbProperties.get(key))
+
+ if 'admin-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['admin-properties']['properties']) \
+ and ('db_host' in services['configurations']['admin-properties']['properties']):
+
+ rangerDbFlavor = services['configurations']["admin-properties"]["properties"]["DB_FLAVOR"]
+ rangerDbHost = services['configurations']["admin-properties"]["properties"]["db_host"]
+ ranger_db_privelege_url_dict = {
+ 'MYSQL': {'ranger_privelege_user_jdbc_url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost)},
+ 'ORACLE': {'ranger_privelege_user_jdbc_url': 'jdbc:oracle:thin:@//' + self.getOracleDBConnectionHostPort(rangerDbFlavor, rangerDbHost, None)},
+ 'POSTGRES': {'ranger_privelege_user_jdbc_url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/postgres'},
+ 'MSSQL': {'ranger_privelege_user_jdbc_url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';'},
+ 'SQLA': {'ranger_privelege_user_jdbc_url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';'}
+ }
+ rangerPrivelegeDbProperties = ranger_db_privelege_url_dict.get(rangerDbFlavor, ranger_db_privelege_url_dict['MYSQL'])
+ for key in rangerPrivelegeDbProperties:
+ putRangerEnvProperty(key, rangerPrivelegeDbProperties.get(key))
+
+ # Recommend ldap settings based on ambari.properties configuration
+ if 'ambari-server-properties' in services and \
+ 'ambari.ldap.isConfigured' in services['ambari-server-properties'] and \
+ services['ambari-server-properties']['ambari.ldap.isConfigured'].lower() == "true":
+ serverProperties = services['ambari-server-properties']
+ if 'authentication.ldap.baseDn' in serverProperties:
+ putRangerUgsyncSite('ranger.usersync.ldap.searchBase', serverProperties['authentication.ldap.baseDn'])
+ if 'authentication.ldap.groupMembershipAttr' in serverProperties:
+ putRangerUgsyncSite('ranger.usersync.group.memberattributename', serverProperties['authentication.ldap.groupMembershipAttr'])
+ if 'authentication.ldap.groupNamingAttr' in serverProperties:
+ putRangerUgsyncSite('ranger.usersync.group.nameattribute', serverProperties['authentication.ldap.groupNamingAttr'])
+ if 'authentication.ldap.groupObjectClass' in serverProperties:
+ putRangerUgsyncSite('ranger.usersync.group.objectclass', serverProperties['authentication.ldap.groupObjectClass'])
+ if 'authentication.ldap.managerDn' in serverProperties:
+ putRangerUgsyncSite('ranger.usersync.ldap.binddn', serverProperties['authentication.ldap.managerDn'])
+ if 'authentication.ldap.primaryUrl' in serverProperties:
+ ldap_protocol = 'ldap://'
+ if 'authentication.ldap.useSSL' in serverProperties and serverProperties['authentication.ldap.useSSL'] == 'true':
+ ldap_protocol = 'ldaps://'
+ ldapUrl = ldap_protocol + serverProperties['authentication.ldap.primaryUrl'] if serverProperties['authentication.ldap.primaryUrl'] else serverProperties['authentication.ldap.primaryUrl']
+ putRangerUgsyncSite('ranger.usersync.ldap.url', ldapUrl)
+ if 'authentication.ldap.userObjectClass' in serverProperties:
+ putRangerUgsyncSite('ranger.usersync.ldap.user.objectclass', serverProperties['authentication.ldap.userObjectClass'])
+ if 'authentication.ldap.usernameAttribute' in serverProperties:
+ putRangerUgsyncSite('ranger.usersync.ldap.user.nameattribute', serverProperties['authentication.ldap.usernameAttribute'])
+
+
+ # Recommend Ranger Authentication method
+ authMap = {
+ 'org.apache.ranger.unixusersync.process.UnixUserGroupBuilder': 'UNIX',
+ 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': 'LDAP'
+ }
+
+ if 'ranger-ugsync-site' in services['configurations'] and 'ranger.usersync.source.impl.class' in services['configurations']["ranger-ugsync-site"]["properties"]:
+ rangerUserSyncClass = services['configurations']["ranger-ugsync-site"]["properties"]["ranger.usersync.source.impl.class"]
+ if rangerUserSyncClass in authMap:
+ rangerAuthMethod = authMap.get(rangerUserSyncClass)
+ putRangerAdminSiteProperty('ranger.authentication.method', rangerAuthMethod)
+
+ # Recommend Ambari Infra Solr properties
+ is_solr_cloud_enabled = False
+ if 'ranger-env' in services['configurations'] and 'is_solrCloud_enabled' in services['configurations']["ranger-env"]["properties"]:
+ is_solr_cloud_enabled = services['configurations']["ranger-env"]["properties"]["is_solrCloud_enabled"] == "true"
+
+ is_external_solr_cloud_enabled = False
+ if 'ranger-env' in services['configurations'] and 'is_external_solrCloud_enabled' in services['configurations']['ranger-env']['properties']:
+ is_external_solr_cloud_enabled = services['configurations']['ranger-env']['properties']['is_external_solrCloud_enabled'] == 'true'
+
+ ranger_audit_zk_port = ''
+
+
+ if 'AMBARI_INFRA' in servicesList and zookeeper_host_port and is_solr_cloud_enabled and not is_external_solr_cloud_enabled:
+ zookeeper_host_port = zookeeper_host_port.split(',')
+ zookeeper_host_port.sort()
+ zookeeper_host_port = ",".join(zookeeper_host_port)
+ infra_solr_znode = '/infra-solr'
+
+ if 'infra-solr-env' in services['configurations'] and \
+ ('infra_solr_znode' in services['configurations']['infra-solr-env']['properties']):
+ infra_solr_znode = services['configurations']['infra-solr-env']['properties']['infra_solr_znode']
+ ranger_audit_zk_port = '{0}{1}'.format(zookeeper_host_port, infra_solr_znode)
+ putRangerAdminSiteProperty('ranger.audit.solr.zookeepers', ranger_audit_zk_port)
+ elif zookeeper_host_port and is_solr_cloud_enabled and is_external_solr_cloud_enabled:
+ ranger_audit_zk_port = '{0}/{1}'.format(zookeeper_host_port, 'ranger_audits')
+ putRangerAdminSiteProperty('ranger.audit.solr.zookeepers', ranger_audit_zk_port)
+ else:
+ putRangerAdminSiteProperty('ranger.audit.solr.zookeepers', 'NONE')
+
+ # Recommend Ranger supported service's audit properties
+ ranger_services = [
+ {'service_name': 'KAFKA', 'audit_file': 'ranger-kafka-audit'},
+ {'service_name': 'STORM', 'audit_file': 'ranger-storm-audit'},
+ {'service_name': 'NIFI', 'audit_file': 'ranger-nifi-audit'}
+ ]
+
+ for item in range(len(ranger_services)):
+ if ranger_services[item]['service_name'] in servicesList:
+ component_audit_file = ranger_services[item]['audit_file']
+ if component_audit_file in services["configurations"]:
+ ranger_audit_dict = [
+ {'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.solr', 'target_configname': 'xasecure.audit.destination.solr'},
+ {'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.urls', 'target_configname': 'xasecure.audit.destination.solr.urls'},
+ {'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.zookeepers', 'target_configname': 'xasecure.audit.destination.solr.zookeepers'}
+ ]
+ putRangerAuditProperty = self.putProperty(configurations, component_audit_file, services)
+
+ for item in ranger_audit_dict:
+ if item['filename'] in services["configurations"] and item['configname'] in services["configurations"][item['filename']]["properties"]:
+ if item['filename'] in configurations and item['configname'] in configurations[item['filename']]["properties"]:
+ rangerAuditProperty = configurations[item['filename']]["properties"][item['configname']]
+ else:
+ rangerAuditProperty = services["configurations"][item['filename']]["properties"][item['configname']]
+ putRangerAuditProperty(item['target_configname'], rangerAuditProperty)
+
+ ranger_plugins_serviceuser = [
+ {'service_name': 'STORM', 'file_name': 'storm-env', 'config_name': 'storm_user', 'target_configname': 'ranger.plugins.storm.serviceuser'},
+ {'service_name': 'KAFKA', 'file_name': 'kafka-env', 'config_name': 'kafka_user', 'target_configname': 'ranger.plugins.kafka.serviceuser'},
+ {'service_name': 'NIFI', 'file_name': 'nifi-env', 'config_name': 'nifi_user', 'target_configname': 'ranger.plugins.nifi.serviceuser'}
+ ]
+
+ for item in range(len(ranger_plugins_serviceuser)):
+ if ranger_plugins_serviceuser[item]['service_name'] in servicesList:
+ file_name = ranger_plugins_serviceuser[item]['file_name']
+ config_name = ranger_plugins_serviceuser[item]['config_name']
+ target_configname = ranger_plugins_serviceuser[item]['target_configname']
+ if file_name in services["configurations"] and config_name in services["configurations"][file_name]["properties"]:
+ service_user = services["configurations"][file_name]["properties"][config_name]
+ putRangerAdminSiteProperty(target_configname, service_user)
+
+
+ has_ranger_tagsync = False
+ if 'RANGER' in servicesList:
+ ranger_tagsync_host = self.getComponentHostNames(services, "RANGER", "RANGER_TAGSYNC")
+ has_ranger_tagsync = len(ranger_tagsync_host) > 0
+
+ if 'ATLAS' in servicesList and has_ranger_tagsync:
+ putTagsyncSiteProperty('ranger.tagsync.source.atlas', 'true')
+
+ if zookeeper_host_port and has_ranger_tagsync:
+ putTagsyncAppProperty('atlas.kafka.zookeeper.connect', zookeeper_host_port)
+
+ if 'KAFKA' in servicesList and has_ranger_tagsync:
+ kafka_hosts = self.getHostNamesWithComponent("KAFKA", "KAFKA_BROKER", services)
+ kafka_port = '6667'
+ if 'kafka-broker' in services['configurations'] and (
+ 'port' in services['configurations']['kafka-broker']['properties']):
+ kafka_port = services['configurations']['kafka-broker']['properties']['port']
+ kafka_host_port = []
+ for i in range(len(kafka_hosts)):
+ kafka_host_port.append(kafka_hosts[i] + ':' + kafka_port)
+
+ final_kafka_host = ",".join(kafka_host_port)
+ putTagsyncAppProperty('atlas.kafka.bootstrap.servers', final_kafka_host)
+
+ def getAmsMemoryRecommendation(self, services, hosts):
+ # MB per sink in hbase heapsize
+ HEAP_PER_MASTER_COMPONENT = 50
+ HEAP_PER_SLAVE_COMPONENT = 10
+
+ schMemoryMap = {
+ "KAFKA": {
+ "KAFKA_BROKER": HEAP_PER_MASTER_COMPONENT
+ },
+ "STORM": {
+ "NIMBUS": HEAP_PER_MASTER_COMPONENT,
+ },
+ "AMBARI_METRICS": {
+ "METRICS_COLLECTOR": HEAP_PER_MASTER_COMPONENT,
+ "METRICS_MONITOR": HEAP_PER_SLAVE_COMPONENT
+ }
+ }
+ total_sinks_count = 0
+ # minimum heap size
+ hbase_heapsize = 500
+ for serviceName, componentsDict in schMemoryMap.items():
+ for componentName, multiplier in componentsDict.items():
+ schCount = len(
+ self.getHostsWithComponent(serviceName, componentName, services,
+ hosts))
+ hbase_heapsize += int((schCount * multiplier) ** 0.9)
+ total_sinks_count += schCount
+ collector_heapsize = int(hbase_heapsize/4 if hbase_heapsize > 2048 else 512)
+
+ return round_to_n(collector_heapsize), round_to_n(hbase_heapsize), total_sinks_count
+
+ def recommendStormConfigurations(self, configurations, clusterData, services, hosts):
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putStormSiteProperty = self.putProperty(configurations, "storm-site", services)
+ putStormStartupProperty = self.putProperty(configurations, "storm-site", services)
+ putStormSiteAttributes = self.putPropertyAttribute(configurations, "storm-site")
+
+ # Storm AMS integration
+ if 'AMBARI_METRICS' in servicesList:
+ putStormSiteProperty('metrics.reporter.register', 'org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter')
+
+ storm_site = getServicesSiteProperties(services, "storm-site")
+ security_enabled = (storm_site is not None and "storm.zookeeper.superACL" in storm_site)
+ if "ranger-env" in services["configurations"] and "ranger-storm-plugin-properties" in services["configurations"] and \
+ "ranger-storm-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]:
+ putStormRangerPluginProperty = self.putProperty(configurations, "ranger-storm-plugin-properties", services)
+ rangerEnvStormPluginProperty = services["configurations"]["ranger-env"]["properties"]["ranger-storm-plugin-enabled"]
+ putStormRangerPluginProperty("ranger-storm-plugin-enabled", rangerEnvStormPluginProperty)
+
+ rangerPluginEnabled = ''
+ if 'ranger-storm-plugin-properties' in configurations and 'ranger-storm-plugin-enabled' in configurations['ranger-storm-plugin-properties']['properties']:
+ rangerPluginEnabled = configurations['ranger-storm-plugin-properties']['properties']['ranger-storm-plugin-enabled']
+ elif 'ranger-storm-plugin-properties' in services['configurations'] and 'ranger-storm-plugin-enabled' in services['configurations']['ranger-storm-plugin-properties']['properties']:
+ rangerPluginEnabled = services['configurations']['ranger-storm-plugin-properties']['properties']['ranger-storm-plugin-enabled']
+
+ nonRangerClass = 'backtype.storm.security.auth.authorizer.SimpleACLAuthorizer'
+ rangerServiceVersion=''
+ if 'RANGER' in servicesList:
+ rangerServiceVersion = [service['StackServices']['service_version'] for service in services["services"] if service['StackServices']['service_name'] == 'RANGER'][0]
+
+ if rangerServiceVersion and rangerServiceVersion == '0.4.0':
+ rangerClass = 'com.xasecure.authorization.storm.authorizer.XaSecureStormAuthorizer'
+ else:
+ rangerClass = 'org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer'
+ # Cluster is kerberized
+ if security_enabled:
+ if rangerPluginEnabled and (rangerPluginEnabled.lower() == 'Yes'.lower()):
+ putStormSiteProperty('nimbus.authorizer',rangerClass)
+ elif (services["configurations"]["storm-site"]["properties"]["nimbus.authorizer"] == rangerClass):
+ putStormSiteProperty('nimbus.authorizer', nonRangerClass)
+ else:
+ putStormSiteAttributes('nimbus.authorizer', 'delete', 'true')
+
+ if "storm-site" in services["configurations"]:
+ # atlas
+ notifier_plugin_property = "storm.topology.submission.notifier.plugin.class"
+ if notifier_plugin_property in services["configurations"]["storm-site"]["properties"]:
+ notifier_plugin_value = services["configurations"]["storm-site"]["properties"][notifier_plugin_property]
+ if notifier_plugin_value is None:
+ notifier_plugin_value = " "
+ else:
+ notifier_plugin_value = " "
+
+ include_atlas = "ATLAS" in servicesList
+ atlas_hook_class = "org.apache.atlas.storm.hook.StormAtlasHook"
+ if include_atlas and atlas_hook_class not in notifier_plugin_value:
+ if notifier_plugin_value == " ":
+ notifier_plugin_value = atlas_hook_class
+ else:
+ notifier_plugin_value = notifier_plugin_value + "," + atlas_hook_class
+ if not include_atlas and atlas_hook_class in notifier_plugin_value:
+ application_classes = []
+ for application_class in notifier_plugin_value.split(","):
+ if application_class != atlas_hook_class and application_class != " ":
+ application_classes.append(application_class)
+ if application_classes:
+ notifier_plugin_value = ",".join(application_classes)
+ else:
+ notifier_plugin_value = " "
+ if notifier_plugin_value != " ":
+ putStormStartupProperty(notifier_plugin_property, notifier_plugin_value)
+
+ def recommendAmsConfigurations(self, configurations, clusterData, services, hosts):
+ putAmsEnvProperty = self.putProperty(configurations, "ams-env", services)
+ putAmsHbaseSiteProperty = self.putProperty(configurations, "ams-hbase-site", services)
+ putAmsSiteProperty = self.putProperty(configurations, "ams-site", services)
+ putHbaseEnvProperty = self.putProperty(configurations, "ams-hbase-env", services)
+ putGrafanaProperty = self.putProperty(configurations, "ams-grafana-env", services)
+ putGrafanaPropertyAttribute = self.putPropertyAttribute(configurations, "ams-grafana-env")
+
+ amsCollectorHosts = self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR")
+
+ if 'cluster-env' in services['configurations'] and \
+ 'metrics_collector_vip_host' in services['configurations']['cluster-env']['properties']:
+ metric_collector_host = services['configurations']['cluster-env']['properties']['metrics_collector_vip_host']
+ else:
+ metric_collector_host = 'localhost' if len(amsCollectorHosts) == 0 else amsCollectorHosts[0]
+
+ putAmsSiteProperty("timeline.metrics.service.webapp.address", str(metric_collector_host) + ":6188")
+
+ log_dir = "/var/log/ambari-metrics-collector"
+ if "ams-env" in services["configurations"]:
+ if "metrics_collector_log_dir" in services["configurations"]["ams-env"]["properties"]:
+ log_dir = services["configurations"]["ams-env"]["properties"]["metrics_collector_log_dir"]
+ putHbaseEnvProperty("hbase_log_dir", log_dir)
+
+ defaultFs = 'file:///'
+ if "core-site" in services["configurations"] and \
+ "fs.defaultFS" in services["configurations"]["core-site"]["properties"]:
+ defaultFs = services["configurations"]["core-site"]["properties"]["fs.defaultFS"]
+
+ operatingMode = "embedded"
+ if "ams-site" in services["configurations"]:
+ if "timeline.metrics.service.operation.mode" in services["configurations"]["ams-site"]["properties"]:
+ operatingMode = services["configurations"]["ams-site"]["properties"]["timeline.metrics.service.operation.mode"]
+
+ if operatingMode == "distributed":
+ putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'true')
+ putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 259200)
+ putAmsHbaseSiteProperty("hbase.cluster.distributed", 'true')
+ else:
+ putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'false')
+ putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 86400)
+ putAmsHbaseSiteProperty("hbase.cluster.distributed", 'false')
+
+ rootDir = "file:///var/lib/ambari-metrics-collector/hbase"
+ tmpDir = "/var/lib/ambari-metrics-collector/hbase-tmp"
+ zk_port_default = []
+ if "ams-hbase-site" in services["configurations"]:
+ if "hbase.rootdir" in services["configurations"]["ams-hbase-site"]["properties"]:
+ rootDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.rootdir"]
+ if "hbase.tmp.dir" in services["configurations"]["ams-hbase-site"]["properties"]:
+ tmpDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.tmp.dir"]
+ if "hbase.zookeeper.property.clientPort" in services["configurations"]["ams-hbase-site"]["properties"]:
+ zk_port_default = services["configurations"]["ams-hbase-site"]["properties"]["hbase.zookeeper.property.clientPort"]
+
+ # Skip recommendation item if default value is present
+ if operatingMode == "distributed" and not "{{zookeeper_clientPort}}" in zk_port_default:
+ zkPort = self.getZKPort(services)
+ putAmsHbaseSiteProperty("hbase.zookeeper.property.clientPort", zkPort)
+ elif operatingMode == "embedded" and not "{{zookeeper_clientPort}}" in zk_port_default:
+ putAmsHbaseSiteProperty("hbase.zookeeper.property.clientPort", "61181")
+
+ mountpoints = ["/"]
+ for collectorHostName in amsCollectorHosts:
+ for host in hosts["items"]:
+ if host["Hosts"]["host_name"] == collectorHostName:
+ mountpoints = self.getPreferredMountPoints(host["Hosts"])
+ break
+ isLocalRootDir = rootDir.startswith("file://") or (defaultFs.startswith("file://") and rootDir.startswith("/"))
+ if isLocalRootDir:
+ rootDir = re.sub("^file:///|/", "", rootDir, count=1)
+ rootDir = "file://" + os.path.join(mountpoints[0], rootDir)
+ tmpDir = re.sub("^file:///|/", "", tmpDir, count=1)
+ if len(mountpoints) > 1 and isLocalRootDir:
+ tmpDir = os.path.join(mountpoints[1], tmpDir)
+ else:
+ tmpDir = os.path.join(mountpoints[0], tmpDir)
+ putAmsHbaseSiteProperty("hbase.tmp.dir", tmpDir)
+
+ if operatingMode == "distributed":
+ putAmsHbaseSiteProperty("hbase.rootdir", defaultFs + "/user/ams/hbase")
+
+ if operatingMode == "embedded":
+ if isLocalRootDir:
+ putAmsHbaseSiteProperty("hbase.rootdir", rootDir)
+ else:
+ putAmsHbaseSiteProperty("hbase.rootdir", "file:///var/lib/ambari-metrics-collector/hbase")
+
+ collector_heapsize, hbase_heapsize, total_sinks_count = self.getAmsMemoryRecommendation(services, hosts)
+
+ putAmsEnvProperty("metrics_collector_heapsize", collector_heapsize)
+
+ # blockCache = 0.3, memstore = 0.35, phoenix-server = 0.15, phoenix-client = 0.25
+ putAmsHbaseSiteProperty("hfile.block.cache.size", 0.3)
+ putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 134217728)
+ putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", 0.35)
+ putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.3)
+
+ if len(amsCollectorHosts) > 1:
+ pass
+ else:
+ # blockCache = 0.3, memstore = 0.3, phoenix-server = 0.2, phoenix-client = 0.3
+ if total_sinks_count >= 2000:
+ putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
+ putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
+ putAmsHbaseSiteProperty("hbase.regionserver.maxlogs", 64)
+ putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 268435456)
+ putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", 0.3)
+ putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.25)
+ putAmsHbaseSiteProperty("phoenix.query.maxGlobalMemoryPercentage", 20)
+ putAmsHbaseSiteProperty("phoenix.coprocessor.maxMetaDataCacheSize", 81920000)
+ putAmsSiteProperty("phoenix.query.maxGlobalMemoryPercentage", 30)
+ putAmsSiteProperty("timeline.metrics.service.resultset.fetchSize", 10000)
+ elif total_sinks_count >= 500:
+ putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
+ putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
+ putAmsHbaseSiteProperty("hbase.regionserver.maxlogs", 64)
+ putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 268435456)
+ putAmsHbaseSiteProperty("phoenix.coprocessor.maxMetaDataCacheSize", 40960000)
+ putAmsSiteProperty("timeline.metrics.service.resultset.fetchSize", 5000)
+ else:
+ putAmsHbaseSiteProperty("phoenix.coprocessor.maxMetaDataCacheSize", 20480000)
+ pass
+
+ metrics_api_handlers = min(50, max(20, int(total_sinks_count / 100)))
+ putAmsSiteProperty("timeline.metrics.service.handler.thread.count", metrics_api_handlers)
+
+ # Distributed mode heap size
+ if operatingMode == "distributed":
+ hbase_heapsize = max(hbase_heapsize, 768)
+ putHbaseEnvProperty("hbase_master_heapsize", "512")
+ putHbaseEnvProperty("hbase_master_xmn_size", "102") #20% of 512 heap size
+ putHbaseEnvProperty("hbase_regionserver_heapsize", hbase_heapsize)
+ putHbaseEnvProperty("regionserver_xmn_size", round_to_n(0.15*hbase_heapsize,64))
+ else:
+ # Embedded mode heap size : master + regionserver
+ hbase_rs_heapsize = 768
+ putHbaseEnvProperty("hbase_regionserver_heapsize", hbase_rs_heapsize)
+ putHbaseEnvProperty("hbase_master_heapsize", hbase_heapsize)
+ putHbaseEnvProperty("hbase_master_xmn_size", round_to_n(0.15*(hbase_heapsize+hbase_rs_heapsize),64))
+
+ # If no local DN in distributed mode
+ if operatingMode == "distributed":
+ dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE")
+ # call by Kerberos wizard sends only the service being affected
+ # so it is possible for dn_hosts to be None but not amsCollectorHosts
+ if dn_hosts and len(dn_hosts) > 0:
+ if set(amsCollectorHosts).intersection(dn_hosts):
+ collector_cohosted_with_dn = "true"
+ else:
+ collector_cohosted_with_dn = "false"
+ putAmsHbaseSiteProperty("dfs.client.read.shortcircuit", collector_cohosted_with_dn)
+
+ #split points
+ scriptDir = os.path.dirname(os.path.abspath(__file__))
+ metricsDir = os.path.join(scriptDir, '../../../../common-services/AMBARI_METRICS/0.1.0/package')
+ serviceMetricsDir = os.path.join(metricsDir, 'files', 'service-metrics')
+ sys.path.append(os.path.join(metricsDir, 'scripts'))
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+ from split_points import FindSplitPointsForAMSRegions
+
+ ams_hbase_site = None
+ ams_hbase_env = None
+
+ # Overriden properties form the UI
+ if "ams-hbase-site" in services["configurations"]:
+ ams_hbase_site = services["configurations"]["ams-hbase-site"]["properties"]
+ if "ams-hbase-env" in services["configurations"]:
+ ams_hbase_env = services["configurations"]["ams-hbase-env"]["properties"]
+
+ # Recommendations
+ if not ams_hbase_site:
+ ams_hbase_site = configurations["ams-hbase-site"]["properties"]
+ if not ams_hbase_env:
+ ams_hbase_env = configurations["ams-hbase-env"]["properties"]
+
+ split_point_finder = FindSplitPointsForAMSRegions(
+ ams_hbase_site, ams_hbase_env, serviceMetricsDir, operatingMode, servicesList)
+
+ result = split_point_finder.get_split_points()
+ precision_splits = ' '
+ aggregate_splits = ' '
+ if result.precision:
+ precision_splits = result.precision
+ if result.aggregate:
+ aggregate_splits = result.aggregate
+ putAmsSiteProperty("timeline.metrics.host.aggregate.splitpoints", ','.join(precision_splits))
+ putAmsSiteProperty("timeline.metrics.cluster.aggregate.splitpoints", ','.join(aggregate_splits))
+
+ component_grafana_exists = False
+ for service in services['services']:
+ if 'components' in service:
+ for component in service['components']:
+ if 'StackServiceComponents' in component:
+ # If Grafana is installed the hostnames would indicate its location
+ if 'METRICS_GRAFANA' in component['StackServiceComponents']['component_name'] and\
+ len(component['StackServiceComponents']['hostnames']) != 0:
+ component_grafana_exists = True
+ break
+ pass
+
+ if not component_grafana_exists:
+ putGrafanaPropertyAttribute("metrics_grafana_password", "visible", "false")
+
+ pass
+
+ def getHostNamesWithComponent(self, serviceName, componentName, services):
+ """
+ Returns the list of hostnames on which service component is installed
+ """
+ if services is not None and serviceName in [service["StackServices"]["service_name"] for service in services["services"]]:
+ service = [serviceEntry for serviceEntry in services["services"] if serviceEntry["StackServices"]["service_name"] == serviceName][0]
+ components = [componentEntry for componentEntry in service["components"] if componentEntry["StackServiceComponents"]["component_name"] == componentName]
+ if (len(components) > 0 and len(components[0]["StackServiceComponents"]["hostnames"]) > 0):
+ componentHostnames = components[0]["StackServiceComponents"]["hostnames"]
+ return componentHostnames
+ return []
+
+ def getHostsWithComponent(self, serviceName, componentName, services, hosts):
+ if services is not None and hosts is not None and serviceName in [service["StackServices"]["service_name"] for service in services["services"]]:
+ service = [serviceEntry for serviceEntry in services["services"] if serviceEntry["StackServices"]["service_name"] == serviceName][0]
+ components = [componentEntry for componentEntry in service["components"] if componentEntry["StackServiceComponents"]["component_name"] == componentName]
+ if (len(components) > 0 and len(components[0]["StackServiceComponents"]["hostnames"]) > 0):
+ componentHostnames = components[0]["StackServiceComponents"]["hostnames"]
+ componentHosts = [host for host in hosts["items"] if host["Hosts"]["host_name"] in componentHostnames]
+ return componentHosts
+ return []
+
+ def getHostWithComponent(self, serviceName, componentName, services, hosts):
+ componentHosts = self.getHostsWithComponent(serviceName, componentName, services, hosts)
+ if (len(componentHosts) > 0):
+ return componentHosts[0]
+ return None
+
+ def getHostComponentsByCategories(self, hostname, categories, services, hosts):
+ components = []
+ if services is not None and hosts is not None:
+ for service in services["services"]:
+ components.extend([componentEntry for componentEntry in service["components"]
+ if componentEntry["StackServiceComponents"]["component_category"] in categories
+ and hostname in componentEntry["StackServiceComponents"]["hostnames"]])
+ return components
+
+ def getZKHostPortString(self, services, include_port=True):
+ """
+ Returns the comma delimited string of zookeeper server host with the configure port installed in a cluster
+ Example: zk.host1.org:2181,zk.host2.org:2181,zk.host3.org:2181
+ include_port boolean param -> If port is also needed.
+ """
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ include_zookeeper = "ZOOKEEPER" in servicesList
+ zookeeper_host_port = ''
+
+ if include_zookeeper:
+ zookeeper_hosts = self.getHostNamesWithComponent("ZOOKEEPER", "ZOOKEEPER_SERVER", services)
+ zookeeper_host_port_arr = []
+
+ if include_port:
+ zookeeper_port = self.getZKPort(services)
+ for i in range(len(zookeeper_hosts)):
+ zookeeper_host_port_arr.append(zookeeper_hosts[i] + ':' + zookeeper_port)
+ else:
+ for i in range(len(zookeeper_hosts)):
+ zookeeper_host_port_arr.append(zookeeper_hosts[i])
+
+ zookeeper_host_port = ",".join(zookeeper_host_port_arr)
+ return zookeeper_host_port
+
+ def getZKPort(self, services):
+ zookeeper_port = '2181' #default port
+ if 'zoo.cfg' in services['configurations'] and ('clientPort' in services['configurations']['zoo.cfg']['properties']):
+ zookeeper_port = services['configurations']['zoo.cfg']['properties']['clientPort']
+ return zookeeper_port
+
+ def getConfigurationClusterSummary(self, servicesList, hosts, components, services):
+
+ hBaseInstalled = False
+ cluster = {
+ "cpu": 0,
+ "disk": 0,
+ "ram": 0,
+ "hBaseInstalled": hBaseInstalled,
+ "components": components
+ }
+
+ if len(hosts["items"]) > 0:
+ nodeManagerHosts = self.getHostsWithComponent("YARN", "NODEMANAGER", services, hosts)
+ # NodeManager host with least memory is generally used in calculations as it will work in larger hosts.
+ if nodeManagerHosts is not None and len(nodeManagerHosts) > 0:
+ nodeManagerHost = nodeManagerHosts[0];
+ for nmHost in nodeManagerHosts:
+ if nmHost["Hosts"]["total_mem"] < nodeManagerHost["Hosts"]["total_mem"]:
+ nodeManagerHost = nmHost
+ host = nodeManagerHost["Hosts"]
+ cluster["referenceNodeManagerHost"] = host
+ else:
+ host = hosts["items"][0]["Hosts"]
+ cluster["referenceHost"] = host
+ cluster["cpu"] = host["cpu_count"]
+ cluster["disk"] = len(host["disk_info"])
+ cluster["ram"] = int(host["total_mem"] / (1024 * 1024))
+
+ ramRecommendations = [
+ {"os":1, "hbase":1},
+ {"os":2, "hbase":1},
+ {"os":2, "hbase":2},
+ {"os":4, "hbase":4},
+ {"os":6, "hbase":8},
+ {"os":8, "hbase":8},
+ {"os":8, "hbase":8},
+ {"os":12, "hbase":16},
+ {"os":24, "hbase":24},
+ {"os":32, "hbase":32},
+ {"os":64, "hbase":32}
+ ]
+ index = {
+ cluster["ram"] <= 4: 0,
+ 4 < cluster["ram"] <= 8: 1,
+ 8 < cluster["ram"] <= 16: 2,
+ 16 < cluster["ram"] <= 24: 3,
+ 24 < cluster["ram"] <= 48: 4,
+ 48 < cluster["ram"] <= 64: 5,
+ 64 < cluster["ram"] <= 72: 6,
+ 72 < cluster["ram"] <= 96: 7,
+ 96 < cluster["ram"] <= 128: 8,
+ 128 < cluster["ram"] <= 256: 9,
+ 256 < cluster["ram"]: 10
+ }[1]
+
+
+ cluster["reservedRam"] = ramRecommendations[index]["os"]
+ cluster["hbaseRam"] = ramRecommendations[index]["hbase"]
+
+
+ cluster["minContainerSize"] = {
+ cluster["ram"] <= 4: 256,
+ 4 < cluster["ram"] <= 8: 512,
+ 8 < cluster["ram"] <= 24: 1024,
+ 24 < cluster["ram"]: 2048
+ }[1]
+
+ totalAvailableRam = cluster["ram"] - cluster["reservedRam"]
+ if cluster["hBaseInstalled"]:
+ totalAvailableRam -= cluster["hbaseRam"]
+ cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024)
+ '''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))'''
+ cluster["containers"] = round(max(3,
+ min(2 * cluster["cpu"],
+ min(ceil(1.8 * cluster["disk"]),
+ cluster["totalAvailableRam"] / cluster["minContainerSize"]))))
+
+ '''ramPerContainers = max(2GB, RAM - reservedRam - hBaseRam) / containers'''
+ cluster["ramPerContainer"] = abs(cluster["totalAvailableRam"] / cluster["containers"])
+ '''If greater than 1GB, value will be in multiples of 512.'''
+ if cluster["ramPerContainer"] > 1024:
+ cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / 512) * 512
+
+ cluster["mapMemory"] = int(cluster["ramPerContainer"])
+ cluster["reduceMemory"] = cluster["ramPerContainer"]
+ cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"])
+
+ return cluster
+
+ def getServiceConfigurationValidators(self):
+ return {
+ "KAFKA": {"ranger-kafka-plugin-properties": self.validateKafkaRangerPluginConfigurations,
+ "kafka-broker": self.validateKAFKAConfigurations},
+ "STORM": {"storm-site": self.validateStormConfigurations,
+ "ranger-storm-plugin-properties": self.validateStormRangerPluginConfigurations},
+ "AMBARI_METRICS": {"ams-hbase-site": self.validateAmsHbaseSiteConfigurations,
+ "ams-hbase-env": self.validateAmsHbaseEnvConfigurations,
+ "ams-site": self.validateAmsSiteConfigurations},
+ "RANGER": {"ranger-env": self.validateRangerConfigurationsEnv,
+ "admin-properties": self.validateRangerAdminConfigurations,
+ "ranger-tagsync-site": self.validateRangerTagsyncConfigurations},
+ "NIFI": {"ranger-nifi-plugin-properties": self.validateNiFiRangerPluginConfigurations,
+ "nifi-ambari-ssl-config": self.validateNiFiSslProperties }
+ }
+
+ def recommendLogsearchConfigurations(self, configurations, clusterData, services, hosts):
+ putLogsearchProperty = self.putProperty(configurations, "logsearch-properties", services)
+ infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR")
+
+ if infraSolrHosts is not None and len(infraSolrHosts) > 0 \
+ and "logsearch-properties" in services["configurations"]:
+ recommendedMinShards = len(infraSolrHosts)
+ recommendedShards = 2 * len(infraSolrHosts)
+ recommendedMaxShards = 3 * len(infraSolrHosts)
+ # recommend number of shard
+ putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties")
+ putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
+ putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
+ putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
+
+ putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
+ putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
+ putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
+ # recommend replication factor
+ replicationReccomendFloat = math.log(len(infraSolrHosts), 5)
+ recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat))
+ putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
+ putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
+
+ def validateMinMax(self, items, recommendedDefaults, configurations):
+
+ # required for casting to the proper numeric type before comparison
+ def convertToNumber(number):
+ try:
+ return int(number)
+ except ValueError:
+ return float(number)
+
+ for configName in configurations:
+ validationItems = []
+ if configName in recommendedDefaults and "property_attributes" in recommendedDefaults[configName]:
+ for propertyName in recommendedDefaults[configName]["property_attributes"]:
+ if propertyName in configurations[configName]["properties"]:
+ if "maximum" in recommendedDefaults[configName]["property_attributes"][propertyName] and \
+ propertyName in recommendedDefaults[configName]["properties"]:
+ userValue = convertToNumber(configurations[configName]["properties"][propertyName])
+ maxValue = convertToNumber(recommendedDefaults[configName]["property_attributes"][propertyName]["maximum"])
+ if userValue > maxValue:
+ validationItems.extend([{"config-name": propertyName, "item": self.getWarnItem("Value is greater than the recommended maximum of {0} ".format(maxValue))}])
+ if "minimum" in recommendedDefaults[configName]["property_attributes"][propertyName] and \
+ propertyName in recommendedDefaults[configName]["properties"]:
+ userValue = convertToNumber(configurations[configName]["properties"][propertyName])
+ minValue = convertToNumber(recommendedDefaults[configName]["property_attributes"][propertyName]["minimum"])
+ if userValue < minValue:
+ validationItems.extend([{"config-name": propertyName, "item": self.getWarnItem("Value is less than the recommended minimum of {0} ".format(minValue))}])
+ items.extend(self.toConfigurationValidationProblems(validationItems, configName))
+ pass
+
+ def validateAmsSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+
+ op_mode = properties.get("timeline.metrics.service.operation.mode")
+ correct_op_mode_item = None
+ if op_mode not in ("embedded", "distributed"):
+ correct_op_mode_item = self.getErrorItem("Correct value should be set.")
+ pass
+
+ validationItems.extend([{"config-name":'timeline.metrics.service.operation.mode', "item": correct_op_mode_item }])
+ return self.toConfigurationValidationProblems(validationItems, "ams-site")
+
+ def validateAmsHbaseSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+
+ amsCollectorHosts = self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR")
+ ams_site = getSiteProperties(configurations, "ams-site")
+ core_site = getSiteProperties(configurations, "core-site")
+
+ collector_heapsize, hbase_heapsize, total_sinks_count = self.getAmsMemoryRecommendation(services, hosts)
+ recommendedDiskSpace = 10485760
+ # TODO validate configuration for multiple AMBARI_METRICS collectors
+ if len(amsCollectorHosts) > 1:
+ pass
+ else:
+ if total_sinks_count > 2000:
+ recommendedDiskSpace = 104857600 # * 1k == 100 Gb
+ elif total_sinks_count > 500:
+ recommendedDiskSpace = 52428800 # * 1k == 50 Gb
+ elif total_sinks_count > 250:
+ recommendedDiskSpace = 20971520 # * 1k == 20 Gb
+
+ validationItems = []
+
+ rootdir_item = None
+ op_mode = ams_site.get("timeline.metrics.service.operation.mode")
+ default_fs = core_site.get("fs.defaultFS") if core_site else "file:///"
+ hbase_rootdir = properties.get("hbase.rootdir")
+ hbase_tmpdir = properties.get("hbase.tmp.dir")
+ distributed = properties.get("hbase.cluster.distributed")
+ is_local_root_dir = hbase_rootdir.startswith("file://") or (default_fs.startswith("file://") and hbase_rootdir.startswith("/"))
+
+ if op_mode == "distributed" and is_local_root_dir:
+ rootdir_item = self.getWarnItem("In distributed mode hbase.rootdir should point to HDFS.")
+ elif op_mode == "embedded":
+ if distributed.lower() == "false" and hbase_rootdir.startswith('/') or hbase_rootdir.startswith("hdfs://"):
+ rootdir_item = self.getWarnItem("In embedded mode hbase.rootdir cannot point to schemaless values or HDFS, "
+ "Example - file:// for localFS")
+ pass
+
+ distributed_item = None
+ if op_mode == "distributed" and not distributed.lower() == "true":
+ distributed_item = self.getErrorItem("hbase.cluster.distributed property should be set to true for "
+ "distributed mode")
+ if op_mode == "embedded" and distributed.lower() == "true":
+ distributed_item = self.getErrorItem("hbase.cluster.distributed property should be set to false for embedded mode")
+
+ hbase_zk_client_port = properties.get("hbase.zookeeper.property.clientPort")
+ zkPort = self.getZKPort(services)
+ hbase_zk_client_port_item = None
+ if distributed.lower() == "true" and op_mode == "distributed" and \
+ hbase_zk_client_port != zkPort and hbase_zk_client_port != "{{zookeeper_clientPort}}":
+ hbase_zk_client_port_item = self.getErrorItem("In AMS distributed mode, hbase.zookeeper.property.clientPort "
+ "should be the cluster zookeeper server port : {0}".format(zkPort))
+
+ if distributed.lower() == "false" and op_mode == "embedded" and \
+ hbase_zk_client_port == zkPort and hbase_zk_client_port != "{{zookeeper_clientPort}}":
+ hbase_zk_client_port_item = self.getErrorItem("In AMS embedded mode, hbase.zookeeper.property.clientPort "
+ "should be a different port than cluster zookeeper port."
+ "(default:61181)")
+
+ validationItems.extend([{"config-name":'hbase.rootdir', "item": rootdir_item },
+ {"config-name":'hbase.cluster.distributed', "item": distributed_item },
+ {"config-name":'hbase.zookeeper.property.clientPort', "item": hbase_zk_client_port_item }])
+
+ for collectorHostName in amsCollectorHosts:
+ for host in hosts["items"]:
+ if host["Hosts"]["host_name"] == collectorHostName:
+ if op_mode == 'embedded' or is_local_root_dir:
+ validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorEnoughDiskSpace(properties, 'hbase.rootdir', host["Hosts"], recommendedDiskSpace)}])
+ validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.rootdir', host["Hosts"])}])
+ validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.tmp.dir', host["Hosts"])}])
+
+ dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE")
+ if is_local_root_dir:
+ mountPoints = []
+ for mountPoint in host["Hosts"]["disk_info"]:
+ mountPoints.append(mountPoint["mountpoint"])
+ hbase_rootdir_mountpoint = getMountPointForDir(hbase_rootdir, mountPoints)
+ hbase_tmpdir_mountpoint = getMountPointForDir(hbase_tmpdir, mountPoints)
+ preferred_mountpoints = self.getPreferredMountPoints(host['Hosts'])
+ # hbase.rootdir and hbase.tmp.dir shouldn't point to the same partition
+ # if multiple preferred_mountpoints exist
+ if hbase_rootdir_mountpoint == hbase_tmpdir_mountpoint and \
+ len(preferred_mountpoints) > 1:
+ item = self.getWarnItem("Consider not using {0} partition for storing metrics temporary data. "
+ "{0} partition is already used as hbase.rootdir to store metrics data".format(hbase_tmpdir_mountpoint))
+ validationItems.extend([{"config-name":'hbase.tmp.dir', "item": item}])
+
+ # if METRICS_COLLECTOR is co-hosted with DATANODE
+ # cross-check dfs.datanode.data.dir and hbase.rootdir
+ # they shouldn't share same disk partition IO
+ hdfs_site = getSiteProperties(configurations, "hdfs-site")
+ dfs_datadirs = hdfs_site.get("dfs.datanode.data.dir").split(",") if hdfs_site and "dfs.datanode.data.dir" in hdfs_site else []
+ if dn_hosts and collectorHostName in dn_hosts and ams_site and \
+ dfs_datadirs and len(preferred_mountpoints) > len(dfs_datadirs):
+ for dfs_datadir in dfs_datadirs:
+ dfs_datadir_mountpoint = getMountPointForDir(dfs_datadir, mountPoints)
+ if dfs_datadir_mountpoint == hbase_rootdir_mountpoint:
+ item = self.getWarnItem("Consider not using {0} partition for storing metrics data. "
+ "{0} is already used by datanode to store HDFS data".format(hbase_rootdir_mountpoint))
+ validationItems.extend([{"config-name": 'hbase.rootdir', "item": item}])
+ break
+ # If no local DN in distributed mode
+ elif collectorHostName not in dn_hosts and distributed.lower() == "true":
+ item = self.getWarnItem("It's recommended to install Datanode component on {0} "
+ "to speed up IO operations between HDFS and Metrics "
+ "Collector in distributed mode ".format(collectorHostName))
+ validationItems.extend([{"config-name": "hbase.cluster.distributed", "item": item}])
+ # Short circuit read should be enabled in distibuted mode
+ # if local DN installed
+ else:
+ validationItems.extend([{"config-name": "dfs.client.read.shortcircuit", "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, "dfs.client.read.shortcircuit")}])
+
+ return self.toConfigurationValidationProblems(validationItems, "ams-hbase-site")
+
+ def validateStormConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ # Storm AMS integration
+ if 'AMBARI_METRICS' in servicesList and "metrics.reporter.register" in properties and \
+ "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter" not in properties.get("metrics.reporter.register"):
+
+ validationItems.append({"config-name": 'metrics.reporter.register',
+ "item": self.getWarnItem(
+ "Should be set to org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter to report the metrics to Ambari Metrics service.")})
+
+ return self.toConfigurationValidationProblems(validationItems, "storm-site")
+
+ def validateStormRangerPluginConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+ ranger_plugin_properties = getSiteProperties(configurations, "ranger-storm-plugin-properties")
+ ranger_plugin_enabled = ranger_plugin_properties['ranger-storm-plugin-enabled'] if ranger_plugin_properties else 'No'
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if ranger_plugin_enabled.lower() == 'yes':
+ # ranger-storm-plugin must be enabled in ranger-env
+ ranger_env = getServicesSiteProperties(services, 'ranger-env')
+ if not ranger_env or not 'ranger-storm-plugin-enabled' in ranger_env or \
+ ranger_env['ranger-storm-plugin-enabled'].lower() != 'yes':
+ validationItems.append({"config-name": 'ranger-storm-plugin-enabled',
+ "item": self.getWarnItem(
+ "ranger-storm-plugin-properties/ranger-storm-plugin-enabled must correspond ranger-env/ranger-storm-plugin-enabled")})
+ if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()) and not 'KERBEROS' in servicesList:
+ validationItems.append({"config-name": "ranger-storm-plugin-enabled",
+ "item": self.getWarnItem(
+ "Ranger Storm plugin should not be enabled in non-kerberos environment.")})
+ return self.toConfigurationValidationProblems(validationItems, "ranger-storm-plugin-properties")
+
+ def validateKafkaRangerPluginConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+ ranger_plugin_properties = getSiteProperties(configurations, "ranger-kafka-plugin-properties")
+ ranger_plugin_enabled = ranger_plugin_properties['ranger-kafka-plugin-enabled'] if ranger_plugin_properties else 'No'
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if ranger_plugin_enabled.lower() == 'yes':
+ # ranger-kafka-plugin must be enabled in ranger-env
+ ranger_env = getServicesSiteProperties(services, 'ranger-env')
+ if not ranger_env or not 'ranger-kafka-plugin-enabled' in ranger_env or \
+ ranger_env['ranger-kafka-plugin-enabled'].lower() != 'yes':
+ validationItems.append({"config-name": 'ranger-kafka-plugin-enabled',
+ "item": self.getWarnItem(
+ "ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled must correspond ranger-env/ranger-kafka-plugin-enabled")})
+
+ if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'yes') and not 'KERBEROS' in servicesList:
+ validationItems.append({"config-name": "ranger-kafka-plugin-enabled",
+ "item": self.getWarnItem(
+ "Ranger Kafka plugin should not be enabled in non-kerberos environment.")})
+
+ return self.toConfigurationValidationProblems(validationItems, "ranger-kafka-plugin-properties")
+
+ def validateRangerAdminConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ ranger_site = properties
+ validationItems = []
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if 'RANGER' in servicesList and 'policymgr_external_url' in ranger_site:
+ policymgr_mgr_url = ranger_site['policymgr_external_url']
+ if policymgr_mgr_url.endswith('/'):
+ validationItems.append({'config-name':'policymgr_external_url',
+ 'item':self.getWarnItem('Ranger External URL should not contain trailing slash "/"')})
+ return self.toConfigurationValidationProblems(validationItems,'admin-properties')
+
+ def validateRangerConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts):
+ ranger_env_properties = properties
+ validationItems = []
+ security_enabled = False
+
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if 'KERBEROS' in servicesList:
+ security_enabled = True
+
+ if "ranger-storm-plugin-enabled" in ranger_env_properties and ranger_env_properties['ranger-storm-plugin-enabled'].lower() == 'yes' and not security_enabled:
+ validationItems.append({"config-name": "ranger-storm-plugin-enabled",
+ "item": self.getWarnItem("Ranger Storm plugin should not be enabled in non-kerberos environment.")})
+ if "ranger-kafka-plugin-enabled" in ranger_env_properties and ranger_env_properties["ranger-kafka-plugin-enabled"].lower() == 'yes' and not security_enabled:
+ validationItems.append({"config-name": "ranger-kafka-plugin-enabled",
+ "item": self.getWarnItem(
+ "Ranger Kafka plugin should not be enabled in non-kerberos environment.")})
+ return self.toConfigurationValidationProblems(validationItems, "ranger-env")
+
+ def validateRangerTagsyncConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ ranger_tagsync_properties = properties
+ validationItems = []
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+ has_atlas = False
+ if "RANGER" in servicesList:
+ has_atlas = not "ATLAS" in servicesList
+
+ if has_atlas and 'ranger.tagsync.source.atlas' in ranger_tagsync_properties and \
+ ranger_tagsync_properties['ranger.tagsync.source.atlas'].lower() == 'true':
+ validationItems.append({"config-name": "ranger.tagsync.source.atlas",
+ "item": self.getWarnItem(
+ "Need to Install ATLAS service to set ranger.tagsync.source.atlas as true.")})
+
+ return self.toConfigurationValidationProblems(validationItems, "ranger-tagsync-site")
+
+ def validateAmsHbaseEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+
+ ams_env = getSiteProperties(configurations, "ams-env")
+ amsHbaseSite = getSiteProperties(configurations, "ams-hbase-site")
+ validationItems = []
+ mb = 1024 * 1024
+ gb = 1024 * mb
+
+ regionServerItem = self.validatorLessThenDefaultValue(properties, recommendedDefaults, "hbase_regionserver_heapsize") ## FIXME if new service added
+ if regionServerItem:
+ validationItems.extend([{"config-name": "hbase_regionserver_heapsize", "item": regionServerItem}])
+
+ hbaseMasterHeapsizeItem = self.validatorLessThenDefaultValue(properties, recommendedDefaults, "hbase_master_heapsize")
+ if hbaseMasterHeapsizeItem:
+ validationItems.extend([{"config-name": "hbase_master_heapsize", "item": hbaseMasterHeapsizeItem}])
+
+ logDirItem = self.validatorEqualsPropertyItem(properties, "hbase_log_dir", ams_env, "metrics_collector_log_dir")
+ if logDirItem:
+ validationItems.extend([{"config-name": "hbase_log_dir", "item": logDirItem}])
+
+ collector_heapsize = to_number(ams_env.get("metrics_collector_heapsize"))
+ hbase_master_heapsize = to_number(properties["hbase_master_heapsize"])
+ hbase_master_xmn_size = to_number(properties["hbase_master_xmn_size"])
+ hbase_regionserver_heapsize = to_number(properties["hbase_regionserver_heapsize"])
+ hbase_regionserver_xmn_size = to_number(properties["regionserver_xmn_size"])
+
+ # Validate Xmn settings.
+ masterXmnItem = None
+ regionServerXmnItem = None
+ is_hbase_distributed = amsHbaseSite.get("hbase.cluster.distributed").lower() == 'true'
+
+ if is_hbase_distributed:
+ minMasterXmn = 0.12 * hbase_master_heapsize
+ maxMasterXmn = 0.2 * hbase_master_heapsize
+ if hbase_master_xmn_size < minMasterXmn:
+ masterXmnItem = self.getWarnItem("Value is lesser than the recommended minimum Xmn size of {0} "
+ "(12% of hbase_master_heapsize)".format(int(ceil(minMasterXmn))))
+
+ if hbase_master_xmn_size > maxMasterXmn:
+ masterXmnItem = self.getWarnItem("Value is greater than the recommended maximum Xmn size of {0} "
+ "(20% of hbase_master_heapsize)".format(int(floor(maxMasterXmn))))
+
+ minRegionServerXmn = 0.12 * hbase_regionserver_heapsize
+ maxRegionServerXmn = 0.2 * hbase_regionserver_heapsize
+ if hbase_regionserver_xmn_size < minRegionServerXmn:
+ regionServerXmnItem = self.getWarnItem("Value is lesser than the recommended minimum Xmn size of {0} "
+ "(12% of hbase_regionserver_heapsize)"
+ .format(int(ceil(minRegionServerXmn))))
+
+ if hbase_regionserver_xmn_size > maxRegionServerXmn:
+ regionServerXmnItem = self.getWarnItem("Value is greater than the recommended maximum Xmn size of {0} "
+ "(20% of hbase_regionserver_heapsize)"
+ .format(int(floor(maxRegionServerXmn))))
+ else:
+ minMasterXmn = 0.12 * (hbase_master_heapsize + hbase_regionserver_heapsize)
+ maxMasterXmn = 0.2 * (hbase_master_heapsize + hbase_regionserver_heapsize)
+ if hbase_master_xmn_size < minMasterXmn:
+ masterXmnItem = self.getWarnItem("Value is lesser than the recommended minimum Xmn size of {0} "
+ "(12% of hbase_master_heapsize + hbase_regionserver_heapsize)"
+ .format(int(ceil(minMasterXmn))))
+
+ if hbase_master_xmn_size > maxMasterXmn:
+ masterXmnItem = self.getWarnItem("Value is greater than the recommended maximum Xmn size of {0} "
+ "(20% of hbase_master_heapsize + hbase_regionserver_heapsize)"
+ .format(int(floor(maxMasterXmn))))
+ if masterXmnItem:
+ validationItems.extend([{"config-name": "hbase_master_xmn_size", "item": masterXmnItem}])
+
+ if regionServerXmnItem:
+ validationItems.extend([{"config-name": "regionserver_xmn_size", "item": regionServerXmnItem}])
+
+ if hbaseMasterHeapsizeItem is None:
+ hostMasterComponents = {}
+
+ for service in services["services"]:
+ for component in service["components"]:
+ if component["StackServiceComponents"]["hostnames"] is not None:
+ for hostName in component["StackServiceComponents"]["hostnames"]:
+ if self.isMasterComponent(component):
+ if hostName not in hostMasterComponents.keys():
+ hostMasterComponents[hostName] = []
+ hostMasterComponents[hostName].append(component["StackServiceComponents"]["component_name"])
+
+ amsCollectorHosts = self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR")
+ for collectorHostName in amsCollectorHosts:
+ for host in hosts["items"]:
+ if host["Hosts"]["host_name"] == collectorHostName:
+ # AMS Collector co-hosted with other master components in bigger clusters
+ if len(hosts['items']) > 31 and \
+ len(hostMasterComponents[collectorHostName]) > 2 and \
+ host["Hosts"]["total_mem"] < 32*mb: # < 32Gb(total_mem in k)
+ masterHostMessage = "Host {0} is used by multiple master components ({1}). " \
+ "It is recommended to use a separate host for the " \
+ "Ambari Metrics Collector component and ensure " \
+ "the host has sufficient memory available."
+
+ hbaseMasterHeapsizeItem = self.getWarnItem(masterHostMessage.format(
+ collectorHostName, str(", ".join(hostMasterComponents[collectorHostName]))))
+ if hbaseMasterHeapsizeItem:
+ validationItems.extend([{"config-name": "hbase_master_heapsize", "item": hbaseMasterHeapsizeItem}])
+
+ # Check for unused RAM on AMS Collector node
+ hostComponents = []
+ for service in services["services"]:
+ for component in service["components"]:
+ if component["StackServiceComponents"]["hostnames"] is not None:
+ if collectorHostName in component["StackServiceComponents"]["hostnames"]:
+ hostComponents.append(component["StackServiceComponents"]["component_name"])
+
+ requiredMemory = getMemorySizeRequired(hostComponents, configurations)
+ unusedMemory = host["Hosts"]["total_mem"] * 1024 - requiredMemory # in bytes
+ if unusedMemory > 4*gb: # warn user, if more than 4GB RAM is unused
+ heapPropertyToIncrease = "hbase_regionserver_heapsize" if is_hbase_distributed else "hbase_master_heapsize"
+ xmnPropertyToIncrease = "regionserver_xmn_size" if is_hbase_distributed else "hbase_master_xmn_size"
+ recommended_collector_heapsize = int((unusedMemory - 4*gb)/5) + collector_heapsize*mb
+ recommended_hbase_heapsize = int((unusedMemory - 4*gb)*4/5) + to_number(properties.get(heapPropertyToIncrease))*mb
+ recommended_hbase_heapsize = min(32*gb, recommended_hbase_heapsize) #Make sure heapsize <= 32GB
+ recommended_xmn_size = round_to_n(0.12*recommended_hbase_heapsize/mb,128)
+
+ if collector_heapsize < recommended_collector_heapsize or \
+ to_number(properties[heapPropertyToIncrease]) < recommended_hbase_heapsize:
+ collectorHeapsizeItem = self.getWarnItem("{0} MB RAM is unused on the host {1} based on components " \
+ "assigned. Consider allocating {2} MB to " \
+ "metrics_collector_heapsize in ams-env, " \
+ "{3} MB to {4} in ams-hbase-env"
+ .format(unusedMemory/mb, collectorHostName,
+ recommended_collector_heapsize/mb,
+ recommended_hbase_heapsize/mb,
+ heapPropertyToIncrease))
+ validationItems.extend([{"config-name": heapPropertyToIncrease, "item": collectorHeapsizeItem}])
+
+ if to_number(properties[xmnPropertyToIncrease]) < recommended_hbase_heapsize:
+ xmnPropertyToIncreaseItem = self.getWarnItem("Consider allocating {0} MB to use up some unused memory "
+ "on host".format(recommended_xmn_size))
+ validationItems.extend([{"config-name": xmnPropertyToIncrease, "item": xmnPropertyToIncreaseItem}])
+ pass
+
+ return self.toConfigurationValidationProblems(validationItems, "ams-hbase-env")
+
+ def validateKAFKAConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ kafka_broker = properties
+ validationItems = []
+
+ #Adding Ranger Plugin logic here
+ ranger_plugin_properties = getSiteProperties(configurations, "ranger-kafka-plugin-properties")
+ ranger_plugin_enabled = ranger_plugin_properties['ranger-kafka-plugin-enabled'] if ranger_plugin_properties else 'No'
+ prop_name = 'authorizer.class.name'
+ prop_val = "org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer"
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()):
+ if kafka_broker[prop_name] != prop_val:
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "If Ranger Kafka Plugin is enabled." \
+ "{0} needs to be set to {1}".format(prop_name,prop_val))})
+
+ return self.toConfigurationValidationProblems(validationItems, "kafka-broker")
+
+ def __find_ca(self, services):
+ for service in services['services']:
+ if 'components' in service:
+ for component in service['components']:
+ stackServiceComponent = component['StackServiceComponents']
+ if 'NIFI_CA' == stackServiceComponent['component_name'] and stackServiceComponent['hostnames']:
+ return True
+ return False
+
+ def validateConfigurationsForSite(self, configurations, recommendedDefaults, services, hosts, siteName, method):
+ if siteName == 'nifi-ambari-ssl-config':
+ return method(self.getSiteProperties(configurations, siteName), None, configurations, services, hosts)
+ else:
+ return DefaultStackAdvisor.validateConfigurationsForSite(self, configurations, recommendedDefaults, services, hosts, siteName, method)
+
+ def validateNiFiSslProperties(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+ ssl_enabled = properties['nifi.node.ssl.isenabled'] and str(properties['nifi.node.ssl.isenabled']).lower() != 'false'
+ if (self.__find_ca(services)):
+ if not properties['nifi.toolkit.tls.token']:
+ validationItems.append({"config-name": 'nifi.toolkit.tls.token', 'item': self.getErrorItem('If NiFi Certificate Authority is used, nifi.toolkit.tls.token must be set')})
+ if not ssl_enabled:
+ validationItems.append({"config-name": 'nifi.node.ssl.isenabled', 'item': self.getWarnItem('For NiFi Certificate Authority to be useful, ssl should be enabled')})
+ else:
+ if properties['nifi.toolkit.tls.token']:
+ validationItems.append({"config-name": 'nifi.toolkit.tls.token', 'item': self.getWarnItem("If NiFi Certificate Authority is not used, nifi.toolkit.tls.token doesn't do anything.")})
+ if ssl_enabled:
+ if not properties['nifi.security.keystorePasswd']:
+ validationItems.append({"config-name": 'nifi.security.keystorePasswd', 'item': self.getErrorItem('If NiFi Certificate Authority is not used and SSL is enabled, must specify nifi.security.keystorePasswd')})
+ if not properties['nifi.security.keyPasswd']:
+ validationItems.append({"config-name": 'nifi.security.keyPasswd', 'item': self.getErrorItem('If NiFi Certificate Authority is not used and SSL is enabled, must specify nifi.security.keyPasswd')})
+ if not properties['nifi.security.truststorePasswd']:
+ validationItems.append({"config-name": 'nifi.security.truststorePasswd', 'item': self.getErrorItem('If NiFi Certificate Authority is not used and SSL is enabled, must specify nifi.security.truststorePasswd')})
+ if not properties['nifi.security.keystoreType']:
+ validationItems.append({"config-name": 'nifi.security.keystoreType', 'item': self.getErrorItem('If NiFi Certificate Authority is not used and SSL is enabled, must specify nifi.security.keystoreType')})
+ if not properties['nifi.security.truststoreType']:
+ validationItems.append({"config-name": 'nifi.security.truststoreType', 'item': self.getErrorItem('If NiFi Certificate Authority is not used and SSL is enabled, must specify nifi.security.truststoreType')})
+ return self.toConfigurationValidationProblems(validationItems, "nifi-ambari-ssl-config")
+
+ def validateNiFiRangerPluginConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+ ranger_plugin_properties = getSiteProperties(configurations, "ranger-nifi-plugin-properties")
+ ranger_plugin_enabled = ranger_plugin_properties['ranger-nifi-plugin-enabled'] if ranger_plugin_properties else 'No'
+
+ if ranger_plugin_enabled.lower() == 'yes':
+ ranger_env = getServicesSiteProperties(services, 'ranger-env')
+ if not ranger_env or not 'ranger-nifi-plugin-enabled' in ranger_env or \
+ ranger_env['ranger-nifi-plugin-enabled'].lower() != 'yes':
+ validationItems.append({"config-name": 'ranger-nifi-plugin-enabled',
+ "item": self.getWarnItem(
+ "ranger-nifi-plugin-properties/ranger-nifi-plugin-enabled must correspond ranger-env/ranger-nifi-plugin-enabled")})
+
+ return self.toConfigurationValidationProblems(validationItems, "ranger-nifi-plugin-properties")
+
+ def validateServiceConfigurations(self, serviceName):
+ return self.getServiceConfigurationValidators().get(serviceName, None)
+
+ def getWarnItem(self, message):
+ return {"level": "WARN", "message": message}
+
+ def getErrorItem(self, message):
+ return {"level": "ERROR", "message": message}
+
+ def getPreferredMountPoints(self, hostInfo):
+
+ # '/etc/resolv.conf', '/etc/hostname', '/etc/hosts' are docker specific mount points
+ undesirableMountPoints = ["/", "/home", "/etc/resolv.conf", "/etc/hosts",
+ "/etc/hostname", "/tmp"]
+ undesirableFsTypes = ["devtmpfs", "tmpfs", "vboxsf", "CDFS"]
+ mountPoints = []
+ if hostInfo and "disk_info" in hostI
<TRUNCATED>
[8/8] ambari git commit: AMBARI-18385: Add HDF management pack
(jluniya)
Posted by jl...@apache.org.
AMBARI-18385: Add HDF management pack (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/37e71db7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/37e71db7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/37e71db7
Branch: refs/heads/trunk
Commit: 37e71db741cacb5acc4113131a27d2c1b7ac5791
Parents: 532b689
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Tue Sep 13 22:26:38 2016 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Tue Sep 13 22:26:38 2016 -0700
----------------------------------------------------------------------
.../hdf-ambari-mpack/.gitignore | 1 +
.../management-packs/hdf-ambari-mpack/pom.xml | 150 ++
.../src/main/assemblies/hdf-ambari-mpack.xml | 79 +
.../common-services/NIFI/1.0.0/alerts.json | 18 +
.../1.0.0/configuration/nifi-ambari-config.xml | 189 ++
.../configuration/nifi-ambari-ssl-config.xml | 226 +++
.../configuration/nifi-authorizers-env.xml | 103 +
.../1.0.0/configuration/nifi-bootstrap-env.xml | 116 ++
...nifi-bootstrap-notification-services-env.xml | 80 +
.../NIFI/1.0.0/configuration/nifi-env.xml | 116 ++
.../NIFI/1.0.0/configuration/nifi-flow-env.xml | 69 +
.../nifi-login-identity-providers-env.xml | 147 ++
.../configuration/nifi-node-logback-env.xml | 201 ++
.../1.0.0/configuration/nifi-properties.xml | 848 ++++++++
.../configuration/nifi-state-management-env.xml | 96 +
.../1.0.0/configuration/ranger-nifi-audit.xml | 91 +
.../ranger-nifi-plugin-properties.xml | 170 ++
.../configuration/ranger-nifi-policymgr-ssl.xml | 84 +
.../configuration/ranger-nifi-security.xml | 90 +
.../common-services/NIFI/1.0.0/kerberos.json | 72 +
.../common-services/NIFI/1.0.0/metainfo.xml | 149 ++
.../common-services/NIFI/1.0.0/metrics.json | 131 ++
.../1.0.0/package/scripts/alert_check_nifi.py | 49 +
.../NIFI/1.0.0/package/scripts/nifi.py | 231 +++
.../NIFI/1.0.0/package/scripts/nifi_ca.py | 114 ++
.../NIFI/1.0.0/package/scripts/nifi_ca_util.py | 110 ++
.../1.0.0/package/scripts/nifi_constants.py | 25 +
.../NIFI/1.0.0/package/scripts/params.py | 445 +++++
.../NIFI/1.0.0/package/scripts/run_ca.sh | 63 +
.../NIFI/1.0.0/package/scripts/service_check.py | 75 +
.../1.0.0/package/scripts/setup_ranger_nifi.py | 68 +
.../NIFI/1.0.0/package/scripts/status_params.py | 29 +
.../NIFI/1.0.0/package/templates/nifi.conf.j2 | 36 +
.../NIFI/1.0.0/quicklinks/quicklinks.json | 35 +
.../common-services/NIFI/1.0.0/widgets.json | 463 +++++
.../src/main/resources/hooks/after_install.py | 49 +
.../src/main/resources/mpack.json | 30 +
.../HDF/2.0/configuration/cluster-env.xml | 210 ++
.../HDF/2.0/hooks/after-INSTALL/scripts/hook.py | 37 +
.../2.0/hooks/after-INSTALL/scripts/params.py | 101 +
.../scripts/shared_initialization.py | 108 ++
.../hooks/before-ANY/files/changeToSecureUid.sh | 53 +
.../HDF/2.0/hooks/before-ANY/scripts/hook.py | 36 +
.../HDF/2.0/hooks/before-ANY/scripts/params.py | 235 +++
.../before-ANY/scripts/shared_initialization.py | 224 +++
.../2.0/hooks/before-INSTALL/scripts/hook.py | 37 +
.../2.0/hooks/before-INSTALL/scripts/params.py | 113 ++
.../scripts/repo_initialization.py | 68 +
.../scripts/shared_initialization.py | 37 +
.../2.0/hooks/before-RESTART/scripts/hook.py | 29 +
.../hooks/before-START/files/checkForFormat.sh | 65 +
.../before-START/files/fast-hdfs-resource.jar | Bin 0 -> 19285850 bytes
.../before-START/files/task-log4j.properties | 134 ++
.../hooks/before-START/files/topology_script.py | 66 +
.../HDF/2.0/hooks/before-START/scripts/hook.py | 39 +
.../2.0/hooks/before-START/scripts/params.py | 318 +++
.../before-START/scripts/rack_awareness.py | 47 +
.../scripts/shared_initialization.py | 175 ++
.../templates/commons-logging.properties.j2 | 43 +
.../templates/exclude_hosts_list.j2 | 21 +
.../templates/hadoop-metrics2.properties.j2 | 104 +
.../before-START/templates/health_check.j2 | 81 +
.../templates/include_hosts_list.j2 | 21 +
.../templates/topology_mappings.data.j2 | 24 +
.../main/resources/stacks/HDF/2.0/kerberos.json | 64 +
.../main/resources/stacks/HDF/2.0/metainfo.xml | 23 +
.../HDF/2.0/properties/stack_features.json | 281 +++
.../stacks/HDF/2.0/properties/stack_tools.json | 4 +
.../resources/stacks/HDF/2.0/repos/repoinfo.xml | 104 +
.../stacks/HDF/2.0/role_command_order.json | 55 +
.../HDF/2.0/services/AMBARI_INFRA/metainfo.xml | 26 +
.../AMBARI_INFRA/role_command_order.json | 7 +
.../2.0/services/AMBARI_METRICS/metainfo.xml | 26 +
.../KAFKA/configuration/ranger-kafka-audit.xml | 77 +
.../ranger-kafka-policymgr-ssl.xml | 37 +
.../stacks/HDF/2.0/services/KAFKA/kerberos.json | 69 +
.../stacks/HDF/2.0/services/KAFKA/metainfo.xml | 27 +
.../HDF/2.0/services/KERBEROS/metainfo.xml | 26 +
.../HDF/2.0/services/LOGSEARCH/metainfo.xml | 26 +
.../services/LOGSEARCH/role_command_order.json | 10 +
.../stacks/HDF/2.0/services/NIFI/metainfo.xml | 27 +
.../RANGER/configuration/ranger-admin-site.xml | 45 +
.../RANGER/configuration/ranger-env.xml | 66 +
.../configuration/ranger-tagsync-site.xml | 72 +
.../RANGER/configuration/ranger-ugsync-site.xml | 59 +
.../tagsync-application-properties.xml | 39 +
.../stacks/HDF/2.0/services/RANGER/metainfo.xml | 42 +
.../services/RANGER/themes/theme_version_4.json | 480 +++++
.../STORM/configuration/ranger-storm-audit.xml | 36 +
.../ranger-storm-policymgr-ssl.xml | 37 +
.../configuration/ranger-storm-security.xml | 30 +
.../services/STORM/configuration/storm-site.xml | 62 +
.../stacks/HDF/2.0/services/STORM/metainfo.xml | 28 +
.../HDF/2.0/services/ZOOKEEPER/metainfo.xml | 51 +
.../stacks/HDF/2.0/services/stack_advisor.py | 1828 ++++++++++++++++++
.../main/resources/stacks/HDF/2.0/widgets.json | 95 +
contrib/management-packs/pom.xml | 3 +
pom.xml | 1 +
98 files changed, 11337 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/.gitignore
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/.gitignore b/contrib/management-packs/hdf-ambari-mpack/.gitignore
new file mode 100644
index 0000000..1377554
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/.gitignore
@@ -0,0 +1 @@
+*.swp
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/pom.xml b/contrib/management-packs/hdf-ambari-mpack/pom.xml
new file mode 100644
index 0000000..4764fa2
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/pom.xml
@@ -0,0 +1,150 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>hdf-ambari-mpack</artifactId>
+ <packaging>pom</packaging>
+ <version>0.1.0.0-SNAPSHOT</version>
+ <name>HDF Ambari Management Pack</name>
+ <url>http://ambari.apache.org/</url>
+ <properties>
+ <minAmbariVersion>2.4.0.0</minAmbariVersion>
+ <maxAmbariVersion></maxAmbariVersion>
+ <nifiversion>1.0.0</nifiversion>
+ </properties>
+ <parent>
+ <groupId>org.apache.ambari.contrib.mpacks</groupId>
+ <artifactId>ambari-contrib-mpacks</artifactId>
+ <version>2.0.0.0-SNAPSHOT</version>
+ </parent>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>parse-version</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>parse-version</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>mpackVersion</name>
+ <value>${project.version}</value>
+ <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
+ <replacement>$1.$2.$3.$4</replacement>
+ <failIfNoMatch>false</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>unpack</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-toolkit-assembly</artifactId>
+ <classifier>bin</classifier>
+ <type>zip</type>
+ <overWrite>true</overWrite>
+ <outputDirectory>${project.build.directory}/nifi-toolkit</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>mpack.json</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptors>
+ <descriptor>src/main/assemblies/hdf-ambari-mpack.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-toolkit-assembly</artifactId>
+ <version>${nifiversion}</version>
+ <classifier>bin</classifier>
+ <type>zip</type>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/assemblies/hdf-ambari-mpack.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/assemblies/hdf-ambari-mpack.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/assemblies/hdf-ambari-mpack.xml
new file mode 100644
index 0000000..2df8075
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/assemblies/hdf-ambari-mpack.xml
@@ -0,0 +1,79 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id></id>
+ <formats>
+ <format>dir</format>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <!-- File sets. Syntax:
+ <fileSets>
+ <fileSet>
+ <useDefaultExcludes/>
+ <outputDirectory/>
+ <includes/>
+ <excludes/>
+ <fileMode/>
+ <directoryMode/>
+ <directory/>
+ <lineEnding/>
+ <filtered/>
+ </fileSet>
+ </fileSets>
+ -->
+ <fileSets>
+ <fileSet>
+ <directory>src/main/resources/hooks</directory>
+ <outputDirectory>hooks</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/common-services</directory>
+ <outputDirectory>common-services</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/stacks</directory>
+ <outputDirectory>stacks</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${project.build.directory}/nifi-toolkit</directory>
+ <outputDirectory>common-services/NIFI/1.0.0/package/files</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <!-- Single files. Syntax:
+ <files>
+ <file>
+ <source/>
+ <outputDirectory/>
+ <destName/>
+ <fileMode/>
+ <lineEnding/>
+ <filtered/>
+ </file>
+ </files>
+ -->
+ <files>
+ <file>
+ <source>target/mpack.json</source>
+ </file>
+ </files>
+ <dependencySets>
+ </dependencySets>
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/alerts.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/alerts.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/alerts.json
new file mode 100644
index 0000000..9d065a0
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/alerts.json
@@ -0,0 +1,18 @@
+{
+ "NIFI": {
+ "service": [],
+ "NIFI_MASTER": [
+ {
+ "name": "nifi_status",
+ "label": "Nifi Status",
+ "description": "This host-level alert is triggered if the Nifi service cannot be determined to be up and responding to client requests.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "SCRIPT",
+ "path": "NIFI/1.0.0/package/scripts/alert_check_nifi.py"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-config.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-config.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-config.xml
new file mode 100644
index 0000000..743fda0
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-config.xml
@@ -0,0 +1,189 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+
+
+ <!-- Note: these two properties must be in same xml file for quicklinks to correctly work -->
+ <property>
+ <name>nifi.node.port</name>
+ <value>9090</value>
+ <display-name>Nifi HTTP port (non-SSL)</display-name>
+ <description>HTTP port NiFi node runs on when SSL is not enabled</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.node.ssl.port</name>
+ <value>9091</value>
+ <display-name>Nifi HTTP port (SSL)</display-name>
+ <description>HTTP port NiFi node runs on when SSL is enabled</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- End Note -->
+
+ <property>
+ <name>nifi.node.protocol.port</name>
+ <value>9088</value>
+ <display-name>Nifi protocol port</display-name>
+ <description>HTTP port NiFi node protocol runs on</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.initial_mem</name>
+ <value>512m</value>
+ <display-name>Initial memory allocation</display-name>
+ <description>Initial memory allocation for NiFi JVM (Xms)</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.max_mem</name>
+ <value>512m</value>
+ <display-name>Max memory allocation</display-name>
+ <description>Max memory allocation for Nifi JVM (Xmx)</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+
+ <property>
+ <name>nifi.ambari_reporting_frequency</name>
+ <value>1 mins</value>
+ <display-name>Metrics reporting frequency</display-name>
+ <description>Frequency with which Nifi metrics will be reported to Ambari Metrics. Only used during initial install of service. The existence of nifi_znode is used to determine whether initial install or not. Post-install this can be reconfigured from Nifi UI under Controller Settings</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.ambari_reporting_enabled</name>
+ <value>true</value>
+ <display-name>Enable metrics reporting to Ambari?</display-name>
+ <description>Whether to setup reporting of Nifi metrics to Ambari Metrics. Only used during initial install of service. The existence of nifi_znode is used to determine whether initial install or not. Post-install this can be reconfigured from Nifi UI under Controller Settings</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.nifi_znode</name>
+ <value>/nifi</value>
+ <display-name>Zookeeper znode</display-name>
+ <description>Zookeeper znode for Nifi</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+
+ <property>
+ <name>nifi.internal.dir</name>
+ <value>/var/lib/nifi</value>
+ <display-name>Nifi internal dir</display-name>
+ <description>Internal directory for Nifi (used to store archive, templates, work dirs). This directory will be created (if not already existing)</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.config.dir</name>
+ <value>{nifi_install_dir}/conf</value>
+ <display-name>Nifi config dir</display-name>
+ <description>The directory to store config files (nifi.properties, bootstrap.conf etc). This directory will be created (if not already existing)</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.flow.config.dir</name>
+ <value>{nifi_internal_dir}/conf</value>
+ <display-name>Nifi internal config dir</display-name>
+ <description>The directory to store the state, flow.xml.gz and authorizations.xml files. This directory will be created (if not already existing)</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.state.dir</name>
+ <value>{nifi_internal_dir}/state/local</value>
+ <display-name>Nifi internal state dir</display-name>
+ <description>The directory to store the state. This directory will be created (if not already existing)</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.database.dir</name>
+ <value>/var/lib/nifi/database_repository</value>
+ <display-name>Nifi H2 database dir</display-name>
+ <description>The directory for the H2 database repository. The H2 database keeps track of user access and flow controller history. This directory will be created (if not already existing)</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.flowfile.repository.dir</name>
+ <value>/var/lib/nifi/flowfile_repository</value>
+ <display-name>Nifi flowfile repository dir</display-name>
+ <description>The directory for the flowfile repository. This directory will be created (if not already existing)</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.content.repository.dir.default</name>
+ <value>/var/lib/nifi/content_repository</value>
+ <display-name>Nifi content repository default dir</display-name>
+ <description>The default directory for the content repository. This directory will be created (if not already existing). NOTE: Multiple content repositories can be specified by using the nifi.content.repository.directory. prefix with unique suffixes and separate paths as values. For example, to provide two additional locations to act as part of the content repository, a user could also specify additional properties with keys of: nifi.content.repository.directory.content1=/repos/content1</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi.provenance.repository.dir.default</name>
+ <value>/var/lib/nifi/provenance_repository</value>
+ <display-name>Nifi provenance repository default dir</display-name>
+ <description>The default directory for the provenance repository. This directory will be created (if not already existing). NOTE: Multiple provenance repositories can be specified by using the nifi.provenance.repository.directory. prefix with unique suffixes and separate paths as values. For example, to provide two additional locations to act as part of the provenance repository, a user could also specify additional properties with keys of: nifi.provenance.repository.directory.content1=/repos/provenance1</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property require-input="true">
+ <name>nifi.sensitive.props.key</name>
+ <value/>
+ <property-type>PASSWORD</property-type>
+ <display-name>Sensitive property values encryption password</display-name>
+ <description>
+ This is the password used to encrypt any sensitive property values that are configured in processors.
+ It can be a string of any length, although the recommended minimum length is 10 characters.
+ Be aware that once this password is set and one or more sensitive processor properties have been configured, this password should not be changed.
+ </description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-ssl-config.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-ssl-config.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-ssl-config.xml
new file mode 100644
index 0000000..272535f
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-ssl-config.xml
@@ -0,0 +1,226 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+
+ <property>
+ <name>nifi.node.ssl.isenabled</name>
+ <value>false</value>
+ <display-name>Enable SSL?</display-name>
+ <description>
+ Check flag to enable SSL. A few additional properties are also required (depending on your setup).
+ Assuming NIFI's Certificate Authority (CA) was installed: If Ranger auth will be used, only 'Nifi CA Token' is required - otherwise 'Nifi CA Token', 'Initial Admin Identity', 'Node identities' are all required.
+ Assuming CA is not installed: If Ranger auth will be used, only Truststore/Keystore paths/type/passwords should be set - otherwise Truststore/Keystore paths/type/passwords as well as 'Initial Admin Identity','Node identities' are required.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.toolkit.tls.regenerate</name>
+ <value>false</value>
+ <display-name>NiFi CA Force Regenerate?</display-name>
+ <description>Will force regeneration of Tls Certificates and Configuration</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property require-input="true">
+ <name>nifi.toolkit.tls.token</name>
+ <value/>
+ <property-type>PASSWORD</property-type>
+ <display-name>NiFi CA Token</display-name>
+ <description>
+ This is a token that will be used by the NiFi Certificate Authority to verify the identity of NiFi nodes before issuing them certificates and by the NiFi nodes
+ to verify the identity of the NiFi Certificate Authority.
+
+ If relying on NiFi Certificate Authority, set this to a long, random value.
+ </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.toolkit.tls.helper.days</name>
+ <value>1095</value>
+ <on-ambari-upgrade add="true"/>
+ <display-name>NiFi CA Certificate Duration</display-name>
+ <description>The number of days that a certificate from the NiFi Certificate Authority should be valid for.</description>
+ </property>
+ <property>
+ <name>nifi.toolkit.tls.port</name>
+ <value>10443</value>
+ <on-ambari-upgrade add="true"/>
+ <display-name>NiFi Certificate Authority port</display-name>
+ <description>Port for NiFi Certificate Authority to listen on.</description>
+ </property>
+ <property>
+ <name>nifi.toolkit.dn.prefix</name>
+ <value>CN=</value>
+ <on-ambari-upgrade add="true"/>
+ <display-name>NiFi CA DN prefix</display-name>
+ <description>Portion of the distinguished name to put before the hostname in TLS certificates (Note, NiFi will sort the DN attributes according to the order they appear in https://www.ietf.org/rfc/rfc2253.txt)</description>
+ </property>
+ <property>
+ <name>nifi.toolkit.dn.suffix</name>
+ <value>, OU=NIFI</value>
+ <on-ambari-upgrade add="true"/>
+ <display-name>NiFi CA DN suffix</display-name>
+ <description>Portion of the distinguished name to put after the hostname in TLS certificates (Note, NiFi will sort the DN attributes according to the order they appear in https://www.ietf.org/rfc/rfc2253.txt)</description>
+ </property>
+ <property>
+ <name>nifi.security.keystore</name>
+ <value>{{nifi_config_dir}}/keystore.jks</value>
+ <on-ambari-upgrade add="true"/>
+ <display-name>Keystore path</display-name>
+ <description>The full path and name of the keystore. To reference paths that include hostnames you can use format such as: /etc/security/nifi-certs/{nifi_node_ssl_host}/{nifi_node_ssl_host}.jks</description>
+ </property>
+ <property>
+ <name>nifi.security.keystoreType</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <display-name>Keystore type</display-name>
+ <description>The keystore type. It is blank by default but required if nifi.node.ssl.isenabled=true. e.g. jks</description>
+ </property>
+ <property>
+ <name>nifi.security.keystorePasswd</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Keystore password</display-name>
+ <description>The keystore password. It is blank by default but required if nifi.node.ssl.isenabled=true.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.security.keyPasswd</name>
+ <value/>
+ <property-type>PASSWORD</property-type>
+ <display-name>Key password</display-name>
+ <description>The key password. It is blank by default but required if nifi.node.ssl.isenabled=true.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.security.truststore</name>
+ <value>{{nifi_config_dir}}/truststore.jks</value>
+ <on-ambari-upgrade add="true"/>
+ <display-name>Truststore path</display-name>
+ <description>The full path and name of the truststore. To reference paths that include hostnames you can use format such as: /etc/security/nifi-certs/{nifi_node_ssl_host}/truststore.jks</description>
+ </property>
+ <property>
+ <name>nifi.security.truststoreType</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <display-name>Truststore type</display-name>
+ <description>The truststore type. It is blank by default but required if nifi.node.ssl.isenabled=true. e.g. jks</description>
+ </property>
+ <property>
+ <name>nifi.security.truststorePasswd</name>
+ <value/>
+ <property-type>PASSWORD</property-type>
+ <display-name>Truststore password</display-name>
+ <description>The truststore password. It is blank by default but required if nifi.node.ssl.isenabled=true.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.security.needClientAuth</name>
+ <value>false</value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <display-name>Clients need to authenticate?</display-name>
+ <description>Specifies whether or not connecting clients must authenticate themselves. Specifically this property is used by the NiFi cluster protocol. If the Truststore properties are not set, this must be false. Otherwise, a value of true indicates that nodes in the cluster will be authenticated and must have certificates that are trusted by the Truststores.</description>
+ </property>
+
+ <property>
+ <name>nifi.initial.admin.identity</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <display-name>Initial Admin Identity</display-name>
+ <description>
+ The identity of an initial admin user that will be granted access to the UI and
+ given the ability to create additional users, groups, and policies. Must be specified when Ranger plugin for
+ Nifi will not be used for authorization.
+ Before changing value of this field, note that it will only take effect if authorizations.xml does not contain any existing policies.
+ The value of this property could be
+ a DN when using certificates or LDAP, or a Kerberos principal. This property will only be used when there
+ are no other users, groups, and policies defined. If this property is specified then a Legacy Authorized
+ Users File can not be specified.
+ </description>
+ </property>
+
+ <property>
+ <name>content</name>
+ <display-name>Node Identities</display-name>
+ <description>
+ Node Identity [unique key] - The identity of a NiFi cluster node. When clustered, a property for each node
+ should be defined, so that every node knows about every other node. If not clustered these properties can be ignored.
+ Also must be specified when Ranger Nifi plugin will not be used for authorization.
+ Before changing value of this field, note that it will only take effect if authorizations.xml does not contain any existing policies.
+ The name of each property must be unique, for example for a three node cluster:
+ "Node Identity A", "Node Identity B", "Node Identity C" or "Node Identity 1", "Node Identity 2", "Node Identity 3"
+ NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the node identities,
+ so the values should be the unmapped identities (i.e. full DN from a certificate).
+ </description>
+ <value>
+<!-- Provide the identity (typically a DN) of each node when clustered (see tool tip for detailed description of Node Identity). Must be specified when Ranger Nifi plugin will not be used for authorization. -->
+
+<!--
+<property name="Node Identity 1"></property>
+<property name="Node Identity 2"></property>
+<property name="Node Identity 3"></property>
+<property name="Node Identity 4"></property>
+-->
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-authorizers-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-authorizers-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-authorizers-env.xml
new file mode 100644
index 0000000..e6505aa
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-authorizers-env.xml
@@ -0,0 +1,103 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+ <property>
+ <name>content</name>
+ <display-name>Template for authorizers.xml</display-name>
+ <description>This is the jinja template for authorizers.xml</description>
+ <value>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file lists the authority providers to use when running securely. In order
+ to use a specific provider it must be configured here and it's identifier
+ must be specified in the nifi.properties file.
+ -->
+ <authorizers>
+
+ <!--
+ The FileAuthorizer is NiFi"s provided authorizer and has the following properties:
+ - Authorizations File - The file where the FileAuthorizer will store policies.
+ - Users File - The file where the FileAuthorizer will store users and groups.
+ - Initial Admin Identity - The identity of an initial admin user that will be granted access to the UI and
+ given the ability to create additional users, groups, and policies. The value of this property could be
+ a DN when using certificates or LDAP, or a Kerberos principal. This property will only be used when there
+ are no other users, groups, and policies defined. If this property is specified then a Legacy Authorized
+ Users File can not be specified.
+ NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the initial admin identity,
+ so the value should be the unmapped identity.
+ - Legacy Authorized Users File - The full path to an existing authorized-users.xml that will be automatically
+ converted to the new authorizations model. If this property is specified then an Initial Admin Identity can
+ not be specified, and this property will only be used when there are no other users, groups, and policies defined.
+ - Node Identity [unique key] - The identity of a NiFi cluster node. When clustered, a property for each node
+ should be defined, so that every node knows about every other node. If not clustered these properties can be ignored.
+ The name of each property must be unique, for example for a three node cluster:
+ "Node Identity A", "Node Identity B", "Node Identity C" or "Node Identity 1", "Node Identity 2", "Node Identity 3"
+ NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the node identities,
+ so the values should be the unmapped identities (i.e. full DN from a certificate).
+ -->
+ <authorizer>
+ <identifier>{{nifi_authorizer}}</identifier>
+ {% if has_ranger_admin and enable_ranger_nifi %}
+ <class>org.apache.nifi.ranger.authorization.RangerNiFiAuthorizer</class>
+ <property name="Ranger Audit Config Path">{{nifi_config_dir}}/ranger-nifi-audit.xml</property>
+ <property name="Ranger Security Config Path">{{nifi_config_dir}}/ranger-nifi-security.xml</property>
+ <property name="Ranger Service Type">nifi</property>
+ <property name="Ranger Application Id">nifi</property>
+ <property name="Allow Anonymous">true</property>
+ <property name="Ranger Admin Identity">{{ranger_admin_identity}}</property>
+ {% if security_enabled %}
+ <property name="Ranger Kerberos Enabled">true</property>
+ {% else %}
+ <property name="Ranger Kerberos Enabled">false</property>
+ {% endif %}
+ {% else %}
+ <class>org.apache.nifi.authorization.FileAuthorizer</class>
+ <property name="Authorizations File">{{nifi_flow_config_dir}}/authorizations.xml</property>
+ <property name="Users File">{{nifi_flow_config_dir}}/users.xml</property>
+ <property name="Initial Admin Identity">{{nifi_initial_admin_id}}</property>
+ <property name="Legacy Authorized Users File"></property>
+ {% endif %}
+
+ {{nifi_ssl_config_content}}
+
+ </authorizer>
+ </authorizers>
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-env.xml
new file mode 100644
index 0000000..ea4e919
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-env.xml
@@ -0,0 +1,116 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+
+ <property>
+ <name>content</name>
+ <display-name>Template for bootstrap.conf</display-name>
+ <value>
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Java command to use when running NiFi
+java=java
+
+# Username to use when running NiFi. This value will be ignored on Windows.
+run.as={{nifi_user}}
+##run.as=root
+
+# Configure where NiFi's lib and conf directories live
+lib.dir={{nifi_install_dir}}/lib
+conf.dir={{nifi_config_dir}}
+
+# How long to wait after telling NiFi to shutdown before explicitly killing the Process
+graceful.shutdown.seconds=20
+
+# Disable JSR 199 so that we can use JSP's without running a JDK
+java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
+
+# JVM memory settings
+java.arg.2=-Xms{{nifi_initial_mem}}
+java.arg.3=-Xmx{{nifi_max_mem}}
+
+# Enable Remote Debugging
+#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
+
+java.arg.4=-Djava.net.preferIPv4Stack=true
+
+# allowRestrictedHeaders is required for Cluster/Node communications to work properly
+java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
+java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
+
+# The G1GC is still considered experimental but has proven to be very advantageous in providing great
+# performance without significant "stop-the-world" delays.
+java.arg.13=-XX:+UseG1GC
+
+#Set headless mode by default
+java.arg.14=-Djava.awt.headless=true
+
+#Ambari Metrics Collector URL - passed in to flow.xml for AmbariReportingTask
+java.arg.15=-Dambari.metrics.collector.url=http://{{metrics_collector_host}}:{{metrics_collector_port}}/ws/v1/timeline/metrics
+
+#Application ID - used in flow.xml - passed into flow.xml for AmbariReportingTask
+java.arg.16=-Dambari.application.id=nifi
+
+
+###
+# Notification Services for notifying interested parties when NiFi is stopped, started, dies
+###
+
+# XML File that contains the definitions of the notification services
+notification.services.file={{nifi_config_dir}}/bootstrap-notification-services.xml
+
+# In the case that we are unable to send a notification for an event, how many times should we retry?
+notification.max.attempts=5
+
+# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
+#nifi.start.notification.services=email-notification
+
+# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
+#nifi.stop.notification.services=email-notification
+
+# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
+#nifi.dead.notification.services=email-notification
+
+ </value>
+ <description>Template for bootstrap.conf</description>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-notification-services-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-notification-services-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-notification-services-env.xml
new file mode 100644
index 0000000..3c5b535
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-notification-services-env.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+ <property>
+ <name>content</name>
+ <display-name>Template for bootstrap-notification-services.xml</display-name>
+ <description>This is the jinja template for bootstrap-notification-services.xml</description>
+ <value>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<services>
+ <!-- This file is used to define how interested parties are notified when events in NiFi's lifecycle occur. -->
+ <!-- The format of this file is:
+ <services>
+ <service>
+ <id>service-identifier</id>
+ <class>org.apache.nifi.notifications.DesiredNotificationService</class>
+ <property name="property name">property value</property>
+ <property name="another property">another property value</property>
+ </service>
+ </services>
+
+ This file can contain 0 to many different service definitions.
+ The id can then be referenced from the bootstrap.conf file in order to configure the notification service
+ to be used when particular lifecycle events occur.
+ -->
+
+<!--
+ <service>
+ <id>email-notification</id>
+ <class>org.apache.nifi.bootstrap.notification.email.EmailNotificationService</class>
+ <property name="SMTP Hostname"></property>
+ <property name="SMTP Port"></property>
+ <property name="SMTP Username"></property>
+ <property name="SMTP Password"></property>
+ <property name="SMTP TLS"></property>
+ <property name="From"></property>
+ <property name="To"></property>
+ </service>
+-->
+</services>
+
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-env.xml
new file mode 100644
index 0000000..76de1c0
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-env.xml
@@ -0,0 +1,116 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+
+ <property>
+ <name>nifi_node_log_dir</name>
+ <value>/var/log/nifi</value>
+ <display-name>NiFi Log dir</display-name>
+ <description>NiFi Log dir</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi_pid_dir</name>
+ <value>/var/run/nifi</value>
+ <display-name>NiFi PID dir</display-name>
+ <description>Dir containing process ID file</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi_user</name>
+ <value>nifi</value>
+ <property-type>USER</property-type>
+ <display-name>Nifi User</display-name>
+ <description>User nifi daemon runs as</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi_group</name>
+ <value>nifi</value>
+ <property-type>GROUP</property-type>
+ <display-name>Nifi Group</display-name>
+ <description>nifi group</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi_user_nofile_limit</name>
+ <value>50000</value>
+ <display-name>Open files limit for Nifi</display-name>
+ <description>Max open files (nofile) limit setting for Nifi user.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>nifi_user_nproc_limit</name>
+ <value>10000</value>
+ <display-name>Process limit for Nifi</display-name>
+ <description>Max number of processes (nproc) limit setting for Nifi user.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>content</name>
+ <display-name>Template for nifi-env.sh</display-name>
+ <value>
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# The java implementation to use.
+export JAVA_HOME={{jdk64_home}}
+
+export NIFI_HOME={{nifi_install_dir}}
+
+#The directory for the NiFi pid file
+export NIFI_PID_DIR="{{nifi_pid_dir}}"
+
+#The directory for NiFi log files
+export NIFI_LOG_DIR="{{nifi_node_log_dir}}"
+
+
+ </value>
+ <description>Template for nifi-env.sh</description>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-flow-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-flow-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-flow-env.xml
new file mode 100644
index 0000000..f01164a
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-flow-env.xml
@@ -0,0 +1,69 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+ <property>
+ <name>content</name>
+ <display-name>Template for flow.xml</display-name>
+ <value>
+<flowController>
+ <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
+ <maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
+ <rootGroup>
+ <id>7c84501d-d10c-407c-b9f3-1d80e38fe36a</id>
+ <name>NiFi Flow</name>
+ <position x="0.0" y="0.0"/>
+ <comment/>
+ </rootGroup>
+ <controllerServices/>
+ <reportingTasks>
+ <reportingTask>
+ <id>3b80ba0f-a6c0-48db-b721-4dbc04cef28e</id>
+ <name>AmbariReportingTask</name>
+ <comment/>
+ <class>org.apache.nifi.reporting.ambari.AmbariReportingTask</class>
+ <schedulingPeriod>{{nifi_ambari_reporting_frequency}}</schedulingPeriod>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <property>
+ <name>Metrics Collector URL</name>
+ <value>${ambari.metrics.collector.url}</value>
+ </property>
+ <property>
+ <name>Application ID</name>
+ <value>${ambari.application.id}</value>
+ </property>
+ <property>
+ <name>Hostname</name>
+ <value>${hostname(true)}</value>
+ </property>
+ </reportingTask>
+ </reportingTasks>
+</flowController>
+ </value>
+ <description>Initial template for flow.xml. Only used during initial install of Nifi service. Post-install this can be reconfigured from Nifi UI under Controller Settings</description>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-login-identity-providers-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-login-identity-providers-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-login-identity-providers-env.xml
new file mode 100644
index 0000000..636ecdb
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-login-identity-providers-env.xml
@@ -0,0 +1,147 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for login-identity-providers.xml</description>
+ <display-name>Template for login-identity-providers.xml</display-name>
+ <value>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ This file lists the login identity providers to use when running securely. In order
+ to use a specific provider it must be configured here and it's identifier
+ must be specified in the nifi.properties file.
+-->
+<loginIdentityProviders>
+ <!--
+ Identity Provider for users logging in with username/password against an LDAP server.
+
+ 'Authentication Strategy' - How the connection to the LDAP server is authenticated. Possible
+ values are ANONYMOUS, SIMPLE, or START_TLS.
+
+ 'Manager DN' - The DN of the manager that is used to bind to the LDAP server to search for users.
+ 'Manager Password' - The password of the manager that is used to bind to the LDAP server to
+ search for users.
+
+ 'TLS - Keystore' - Path to the Keystore that is used when connecting to LDAP using START_TLS.
+ 'TLS - Keystore Password' - Password for the Keystore that is used when connecting to LDAP
+ using START_TLS.
+ 'TLS - Keystore Type' - Type of the Keystore that is used when connecting to LDAP using
+ START_TLS (i.e. JKS or PKCS12).
+ 'TLS - Truststore' - Path to the Truststore that is used when connecting to LDAP using START_TLS.
+ 'TLS - Truststore Password' - Password for the Truststore that is used when connecting to
+ LDAP using START_TLS.
+ 'TLS - Truststore Type' - Type of the Truststore that is used when connecting to LDAP using
+ START_TLS (i.e. JKS or PKCS12).
+ 'TLS - Client Auth' - Client authentication policy when connecting to LDAP using START_TLS.
+ Possible values are REQUIRED, WANT, NONE.
+ 'TLS - Protocol' - Protocol to use when connecting to LDAP using START_TLS. (i.e. TLS,
+ TLSv1.1, TLSv1.2, etc).
+ 'TLS - Shutdown Gracefully' - Specifies whether the TLS should be shut down gracefully
+ before the target context is closed. Defaults to false.
+
+ 'Referral Strategy' - Strategy for handling referrals. Possible values are FOLLOW, IGNORE, THROW.
+ 'Connect Timeout' - Duration of connect timeout. (i.e. 10 secs).
+ 'Read Timeout' - Duration of read timeout. (i.e. 10 secs).
+
+ 'Url' - Url of the LDAP servier (i.e. ldap://<hostname>:<port>).
+ 'User Search Base' - Base DN for searching for users (i.e. CN=Users,DC=example,DC=com).
+ 'User Search Filter' - Filter for searching for users against the 'User Search Base'.
+ (i.e. sAMAccountName={0}). The user specified name is inserted into '{0}'.
+
+ 'Authentication Expiration' - The duration of how long the user authentication is valid
+ for. If the user never logs out, they will be required to log back in following
+ this duration.
+ -->
+ <!-- To enable the ldap-provider remove 2 lines. This is 1 of 2.
+ <provider>
+ <identifier>ldap-provider</identifier>
+ <class>org.apache.nifi.ldap.LdapProvider</class>
+ <property name="Authentication Strategy">START_TLS</property>
+
+ <property name="Manager DN"></property>
+ <property name="Manager Password"></property>
+
+ <property name="TLS - Keystore"></property>
+ <property name="TLS - Keystore Password"></property>
+ <property name="TLS - Keystore Type"></property>
+ <property name="TLS - Truststore"></property>
+ <property name="TLS - Truststore Password"></property>
+ <property name="TLS - Truststore Type"></property>
+ <property name="TLS - Client Auth"></property>
+ <property name="TLS - Protocol"></property>
+ <property name="TLS - Shutdown Gracefully"></property>
+
+ <property name="Referral Strategy">FOLLOW</property>
+ <property name="Connect Timeout">10 secs</property>
+ <property name="Read Timeout">10 secs</property>
+
+ <property name="Url"></property>
+ <property name="User Search Base"></property>
+ <property name="User Search Filter"></property>
+
+ <property name="Authentication Expiration">12 hours</property>
+ </provider>
+ To enable the ldap-provider remove 2 lines. This is 2 of 2. -->
+
+ <!--
+ Identity Provider for users logging in with username/password against a Kerberos KDC server.
+
+ 'Default Realm' - Default realm to provide when user enters incomplete user principal (i.e. NIFI.APACHE.ORG).
+ 'Authentication Expiration' - The duration of how long the user authentication is valid for. If the user never logs out, they will be required to log back in following this duration.
+ -->
+ {% if not security_enabled %}
+ <!-- To enable the kerberos-provider remove 2 lines. This is 1 of 2.
+ {% endif %}
+ <provider>
+ <identifier>kerberos-provider</identifier>
+ <class>org.apache.nifi.kerberos.KerberosProvider</class>
+ <property name="Default Realm">{{nifi_kerberos_realm}}</property>
+ <property name="Authentication Expiration">{{nifi_kerberos_authentication_expiration}}</property>
+ </provider>
+ {% if not security_enabled %}
+ To enable the kerberos-provider remove 2 lines. This is 2 of 2. -->
+ {% endif %}
+
+</loginIdentityProviders>
+
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-node-logback-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-node-logback-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-node-logback-env.xml
new file mode 100644
index 0000000..7989c05
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-node-logback-env.xml
@@ -0,0 +1,201 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+
+ <property>
+ <name>content</name>
+ <display-name>Template for logback.xml</display-name>
+ <value>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration scan="true" scanPeriod="30 seconds">
+ <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
+ <resetJUL>true</resetJUL>
+ </contextListener>
+
+ <appender name="APP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <!--
+ For daily rollover, use 'app_%d.log'.
+ For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.
+ To GZIP rolled files, replace '.log' with '.log.gz'.
+ To ZIP rolled files, replace '.log' with '.log.zip'.
+ -->
+ <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log.gz</fileNamePattern>
+ <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+ <maxFileSize>100MB</maxFileSize>
+ </timeBasedFileNamingAndTriggeringPolicy>
+ <!-- keep 30 log files worth of history -->
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
+ <immediateFlush>true</immediateFlush>
+ </encoder>
+ </appender>
+
+ <appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <!--
+ For daily rollover, use 'user_%d.log'.
+ For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
+ To GZIP rolled files, replace '.log' with '.log.gz'.
+ To ZIP rolled files, replace '.log' with '.log.zip'.
+ -->
+ <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log.gz</fileNamePattern>
+ <!-- keep 30 log files worth of history -->
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <!--
+ For daily rollover, use 'user_%d.log'.
+ For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
+ To GZIP rolled files, replace '.log' with '.log.gz'.
+ To ZIP rolled files, replace '.log' with '.log.zip'.
+ -->
+ <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log.gz</fileNamePattern>
+ <!-- keep 5 log files worth of history -->
+ <maxHistory>5</maxHistory>
+ </rollingPolicy>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+
+ <logger name="org.apache.nifi" level="INFO"/>
+ <logger name="org.apache.nifi.processors" level="WARN"/>
+ <logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
+ <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />
+
+
+ <logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
+ <logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
+ <logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />
+ <logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
+ <logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
+
+ <!-- Logger for managing logging statements for nifi clusters. -->
+ <logger name="org.apache.nifi.cluster" level="INFO"/>
+
+ <!-- Logger for logging HTTP requests received by the web server. -->
+ <logger name="org.apache.nifi.server.JettyServer" level="INFO"/>
+
+ <!-- Logger for managing logging statements for jetty -->
+ <logger name="org.eclipse.jetty" level="INFO"/>
+
+ <!-- Suppress non-error messages due to excessive logging by class or library -->
+ <logger name="com.sun.jersey.spi.container.servlet.WebComponent" level="ERROR"/>
+ <logger name="com.sun.jersey.spi.spring" level="ERROR"/>
+ <logger name="org.springframework" level="ERROR"/>
+
+ <!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->
+ <logger name="com.sun.jersey.spi.inject.Errors" level="ERROR"/>
+
+ <!--
+ Logger for capturing user events. We do not want to propagate these
+ log events to the root logger. These messages are only sent to the
+ user-log appender.
+ -->
+ <logger name="org.apache.nifi.web.security" level="INFO" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.apache.nifi.web.api.config" level="INFO" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.apache.nifi.authorization" level="INFO" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.apache.nifi.cluster.authorization" level="INFO" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.apache.nifi.web.filter.RequestLogger" level="INFO" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+
+
+ <!--
+ Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
+ -->
+ <logger name="org.apache.nifi.bootstrap" level="INFO" additivity="false">
+ <appender-ref ref="BOOTSTRAP_FILE" />
+ </logger>
+ <logger name="org.apache.nifi.bootstrap.Command" level="INFO" additivity="false">
+ <appender-ref ref="CONSOLE" />
+ <appender-ref ref="BOOTSTRAP_FILE" />
+ </logger>
+
+ <!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level -->
+ <logger name="org.apache.nifi.StdOut" level="INFO" additivity="false">
+ <appender-ref ref="BOOTSTRAP_FILE" />
+ </logger>
+
+ <!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->
+ <logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false">
+ <appender-ref ref="BOOTSTRAP_FILE" />
+ </logger>
+
+
+ <root level="INFO">
+ <appender-ref ref="APP_FILE"/>
+ </root>
+
+</configuration>
+
+ </value>
+ <description>Template for logback.xml</description>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
+
[7/8] ambari git commit: AMBARI-18385: Add HDF management pack
(jluniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-properties.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-properties.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-properties.xml
new file mode 100644
index 0000000..94055b1
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-properties.xml
@@ -0,0 +1,848 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="false">
+ <!--# Core Properties #-->
+ <property>
+ <name>nifi.version</name>
+ <value>1.0.0.{{stack_version_buildnum}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The version number of the current release.</description>
+ </property>
+
+ <property>
+ <name>nifi.flow.configuration.file</name>
+ <value>{{nifi_flow_config_dir}}/flow.xml.gz</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the flow configuration file (i.e., the file that contains what is currently displayed on the NiFi graph).</description>
+ </property>
+ <property>
+ <name>nifi.flow.configuration.archive.dir</name>
+ <value>{{nifi_internal_dir}}/archive/</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the archive directory where backup copies of the flow.xml are saved.</description>
+ </property>
+ <property>
+ <name>nifi.flowcontroller.graceful.shutdown.period</name>
+ <value>10 sec</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Indicates the shutdown period. The default value is 10 sec.</description>
+ </property>
+ <property>
+ <name>nifi.flowcontroller.autoResumeState</name>
+ <value>true</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Indicates whether -upon restart- the components on the NiFi graph should return to their last state. The default value is true.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>nifi.flowservice.writedelay.interval</name>
+ <value>500 ms</value>
+ <on-ambari-upgrade add="true"/>
+ <description>When many changes are made to the flow.xml, this property specifies how long to wait before writing out the changes, so as to batch the changes into a single write. The default value is 500 ms.</description>
+ </property>
+ <property>
+ <name>nifi.administrative.yield.duration</name>
+ <value>30 sec</value>
+ <on-ambari-upgrade add="true"/>
+ <description>If a component allows an unexpected exception to escape, it is considered a bug. As a result, the framework will pause (or administratively yield) the component for this amount of time. This is done so that the component does not use up massive amounts of system resources, since it is known to have problems in the existing state. The default value is 30 sec.</description>
+ </property>
+ <property>
+ <name>nifi.bored.yield.duration</name>
+ <value>10 millis</value>
+ <on-ambari-upgrade add="true"/>
+ <description>When a component has no work to do (i.e., is "bored"), this is the amount of time it will wait before checking to see if it has new data to work on. This way, it does not use up CPU resources by checking for new work too often. When setting this property, be aware that it could add extra latency for components that do not constantly have work to do, as once they go into this "bored" state, they will wait this amount of time before checking for more work. The default value is 10 millis.</description>
+ </property>
+
+ <property>
+ <name>nifi.authorizer.configuration.file</name>
+ <value>{{nifi_config_dir}}/authorizers.xml</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This is the location of the file that specifies how user access is authorized. </description>
+ </property>
+ <property>
+ <name>nifi.login.identity.provider.configuration.file</name>
+ <value>{{nifi_config_dir}}/login-identity-providers.xml</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This is the location of the file that specifies how username/password authentication is performed. This file is only consider if nifi.security.user.login.identity.provider configured with a provider identifier</description>
+ </property>
+ <property>
+ <name>nifi.templates.directory</name>
+ <value>{{nifi_internal_dir}}/templates</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This is the location of the directory where flow templates are saved.</description>
+ </property>
+ <property>
+ <name>nifi.ui.banner.text</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>This is banner text that may be configured to display at the top of the User Interface. It is blank by default.</description>
+ </property>
+ <property>
+ <name>nifi.ui.autorefresh.interval</name>
+ <value>30 sec</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The interval at which the User Interface auto-refreshes. The default value is 30 sec.</description>
+ </property>
+ <property>
+ <name>nifi.nar.library.directory</name>
+ <value>{{nifi_install_dir}}/lib</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the nar library. The default probably should be left as is but additional library directories can be specified by using the nifi.nar.library.directory. prefix with unique suffixes and separate paths as values e.g. nifi.nar.library.directory.lib1=/nars/lib1</description>
+ </property>
+ <property>
+ <name>nifi.nar.working.directory</name>
+ <value>{{nifi_internal_dir}}/work/nar</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the nar working directory. The default value should probably should be left as is.</description>
+ </property>
+ <property>
+ <name>nifi.documentation.working.directory</name>
+ <value>{{nifi_internal_dir}}/work/docs/components</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The documentation working directory. The default value should probably should be left as is.</description>
+ </property>
+
+ <!-- State Management -->
+ <property>
+ <name>nifi.state.management.configuration.file</name>
+ <value>{{nifi_config_dir}}/state-management.xml</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The XML file that contains configuration for the local and cluster-wide State Providers.</description>
+ </property>
+ <property>
+ <name>nifi.state.management.provider.cluster</name>
+ <value>zk-provider</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The ID of the Cluster State Provider to use. This value must match the value of the id element of one of the cluster-provider elements in the state-management.xml file. This value is ignored if not clustered but is required for nodes in a cluster.</description>
+ </property>
+ <property>
+ <name>nifi.state.management.embedded.zookeeper.start</name>
+ <value>false</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Specifies whether or not this instance of NiFi should start an embedded ZooKeeper Server. This is used in conjunction with the ZooKeeperStateProvider.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>nifi.state.management.embedded.zookeeper.properties</name>
+ <value>{{nifi_config_dir}}/zookeeper.properties</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Specifies a properties file that contains the configuration for the embedded ZooKeeper Server (if started)</description>
+ </property>
+ <property>
+ <name>nifi.state.management.provider.local</name>
+ <value>local-provider</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The property that provides the identifier of the local State Provider configured</description>
+ </property>
+
+ <!-- H2 Settings-->
+ <property>
+ <name>nifi.database.directory</name>
+ <value>{{nifi_database_dir}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the H2 database directory. The H2 database keeps track of user access and flow controller history</description>
+ </property>
+ <property>
+ <name>nifi.h2.url.append</name>
+ <value>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This property specifies additional arguments to add to the connection string for the H2 database. The default value should be used and should not be changed</description>
+ </property>
+
+ <!-- Flowfile repository-->
+ <property>
+ <name>nifi.flowfile.repository.always.sync</name>
+ <value>false</value>
+ <on-ambari-upgrade add="true"/>
+ <description>If set to true, any change to the repository will be synchronized to the disk, meaning that NiFi will ask the operating system not to cache the information. This is very expensive and can significantly reduce NiFi performance. However, if it is false, there could be the potential for data loss if either there is a sudden power loss or the operating system crashes. The default value is false.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>nifi.flowfile.repository.partitions</name>
+ <value>256</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The number of partitions. The default value is 256.</description>
+ </property>
+ <property>
+ <name>nifi.flowfile.repository.checkpoint.interval</name>
+ <value>2 mins</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The FlowFile Repository checkpoint interval. The default value is 2 mins.</description>
+ </property>
+ <property>
+ <name>nifi.flowfile.repository.directory</name>
+ <value>{{nifi_flowfile_repo_dir}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the FlowFile Repository.</description>
+ </property>
+ <property>
+ <name>nifi.flowfile.repository.implementation</name>
+ <value>org.apache.nifi.controller.repository.WriteAheadFlowFileRepository</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The FlowFile Repository implementation. The default value is org.apache.nifi.controller.repository.WriteAheadFlowFileRepository and should not be changed.</description>
+ </property>
+
+ <property>
+ <name>nifi.swap.in.period</name>
+ <value>5 sec</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The swap in period. The default value is 5 sec.</description>
+ </property>
+ <property>
+ <name>nifi.swap.in.threads</name>
+ <value>1</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The number of threads to use for swapping in. The default value is 1.</description>
+ </property>
+ <property>
+ <name>nifi.swap.out.period</name>
+ <value>5 sec</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The swap out period. The default value is 5 sec.</description>
+ </property>
+ <property>
+ <name>nifi.swap.out.threads</name>
+ <value>4</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The number of threads to use for swapping out. The default value is 4.</description>
+ </property>
+ <property>
+ <name>nifi.swap.manager.implementation</name>
+ <value>org.apache.nifi.controller.FileSystemSwapManager</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The Swap Manager implementation. The default value is org.apache.nifi.controller.FileSystemSwapManager and should not be changed.</description>
+ </property>
+ <property>
+ <name>nifi.queue.swap.threshold</name>
+ <value>20000</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The queue threshold at which NiFi starts to swap FlowFile information to disk. The default value is 20000.</description>
+ </property>
+
+ <property>
+ <name>nifi.content.repository.archive.max.usage.percentage</name>
+ <value>50%</value>
+ <on-ambari-upgrade add="true"/>
+ <description>If archiving is enabled (see nifi.content.repository.archive.enabled), then this property also must have a value to indicate the maximum percentage of disk space that may be used before archive data is removed. If this value is already met even before archiving then arhival will not be of much use. It is 50% by default.</description>
+ </property>
+ <property>
+ <name>nifi.content.claim.max.flow.files</name>
+ <value>100</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The maximum number of FlowFiles to assign to one content claim. The default value is 100.</description>
+ </property>
+ <property>
+ <name>nifi.content.repository.always.sync</name>
+ <value>false</value>
+ <on-ambari-upgrade add="true"/>
+ <description>If set to true, any change to the repository will be synchronized to the disk, meaning that NiFi will ask the operating system not to cache the information. This is very expensive and can significantly reduce NiFi performance. However, if it is false, there could be the potential for data loss if either there is a sudden power loss or the operating system crashes. The default value is false.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>nifi.content.claim.max.appendable.size</name>
+ <value>10 MB</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The maximum size for a content claim. The default value is 10 MB.</description>
+ </property>
+ <property>
+ <name>nifi.content.repository.directory.default</name>
+ <value>{{nifi_content_repo_dir_default}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the Content Repository. NOTE: Multiple content repositories can be specified by using the nifi.content.repository.directory. prefix with unique suffixes and separate paths as values e.g. nifi.content.repository.directory.content1=/repos/content1</description>
+ </property>
+ <property>
+ <name>nifi.content.repository.implementation</name>
+ <value>org.apache.nifi.controller.repository.FileSystemRepository</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The Content Repository implementation. The default value is org.apache.nifi.controller.repository.FileSystemRepository and should not be changed.</description>
+ </property>
+ <property>
+ <name>nifi.content.viewer.url</name>
+ <value>/nifi-content-viewer/</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The URL for a web-based content viewer if one is available.</description>
+ </property>
+ <property>
+ <name>nifi.content.repository.archive.enabled</name>
+ <value>true</value>
+ <on-ambari-upgrade add="true"/>
+ <description>To enable archiving, set this to true and specify a value for the nifi.content.repository.archive.max.usage.percentage property above. By default, archiving is enabled.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>nifi.content.repository.archive.max.retention.period</name>
+ <value>12 hours</value>
+ <on-ambari-upgrade add="true"/>
+ <description>
+If archiving is enabled (see nifi.content.repository.archive.enabled below), then this property specifies the maximum amount of time to keep the archived data. It is 12 hours by default.</description>
+ </property>
+
+ <property>
+ <name>nifi.provenance.repository.rollover.size</name>
+ <value>100 MB</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The amount of information to roll over at a time. The default value is 100 MB.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.max.storage.size</name>
+ <value>1 GB</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The maximum amount of data provenance information to store at a time. The default is 1 GB.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.implementation</name>
+ <value>org.apache.nifi.provenance.PersistentProvenanceRepository</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The Provenance Repository implementation. The default value is org.apache.nifi.provenance.PersistentProvenanceRepository and should not be changed.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.indexed.attributes</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>This is a comma-separated list of FlowFile Attributes that should be indexed and made searchable. It is blank by default. But some good examples to consider are filename, uuid, and mime.type as well as any custom attritubes you might use which are valuable for your use case.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.indexed.fields</name>
+ <value>EventType, FlowFileUUID, Filename, ProcessorID, Relationship</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This is a comma-separated list of FlowFile Attributes that should be indexed and made searchable. It is blank by default. But some good examples to consider are filename, uuid, and mime.type as well as any custom attritubes you might use which are valuable for your use case.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.compress.on.rollover</name>
+ <value>true</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Indicates whether to compress the provenance information when rolling it over. The default value is true.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.buffer.size</name>
+ <value>100000</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The Provenance Repository buffer size. The default value is 100000.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.index.shard.size</name>
+ <value>500 MB</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Large values for the shard size will result in more Java heap usage when searching the Provenance Repository but should provide better performance. The default value is 500 MB.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.max.attribute.length</name>
+ <value>65536</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. The default is 65536.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.directory.default</name>
+ <value>{{nifi_provenance_repo_dir_default}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the Provenance Repository.NOTE: Multiple provenance repositories can be specified by using the nifi.provenance.repository.directory. prefix with unique suffixes and separate paths as values. For example, to provide two additional locations to act as part of the provenance repository, a user could also specify additional properties with keys of: nifi.provenance.repository.directory.provenance1=/repos/provenance1</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.always.sync</name>
+ <value>false</value>
+ <on-ambari-upgrade add="true"/>
+ <description>If set to true, any change to the repository will be synchronized to the disk, meaning that NiFi will ask the operating system not to cache the information. This is very expensive and can significantly reduce NiFi performance. However, if it is false, there could be the potential for data loss if either there is a sudden power loss or the operating system crashes. The default value is false.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.journal.count</name>
+ <value>16</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The number of journal files that should be used to serialize Provenance Event data. Increasing this value will allow more tasks to simultaneously update the repository but will result in more expensive merging of the journal files later. This value should ideally be equal to the number of threads that are expected to update the repository simultaneously, but 16 tends to work well in must environments. The default value is 16.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.max.storage.time</name>
+ <value>24 hours</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The maximum amount of time to keep data provenance information. The default value is 24 hours.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.query.threads</name>
+ <value>2</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The number of threads to use for Provenance Repository queries. The default value is 2.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.rollover.time</name>
+ <value>30 secs</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The amount of time to wait before rolling over the latest data provenance information so that it is available in the User Interface. The default value is 30 secs.</description>
+ </property>
+ <property>
+ <name>nifi.provenance.repository.index.threads</name>
+ <value>1</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The number of threads to use for indexing Provenance events so that they are searchable. The default value is 1. For flows that operate on a very high number of FlowFiles, the indexing of Provenance events could become a bottleneck. If this is the case, a bulletin will appear, indicating that "The rate of the dataflow is exceeding the provenance recording rate. Slowing down flow to accommodate." If this happens, increasing the value of this property may increase the rate at which the Provenance Repository is able to process these records, resulting in better overall throughput.</description>
+ </property>
+
+
+ <property>
+ <name>nifi.remote.input.socket.port</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>The remote input socket port for Site-to-Site communication. By default, it is blank, but it must have a value in order to use Remote Process Groups.</description>
+ </property>
+ <property>
+ <name>nifi.remote.input.http.transaction.ttl</name>
+ <value>30 sec</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Specify how long a transaction can stay alive on server. If a Site-to-Site client didn\u2019t proceed to next action for this period of time, the transaction is discarded from remote NiFi instance. For example, a client creates a transaction but doesn\u2019t send or receive flow files, or send or received flow files but doesn\u2019t confirm that transaction. By default, it is set to 30 seconds.</description>
+ </property>
+ <property>
+ <name>nifi.remote.input.secure</name>
+ <value>{{nifi_ssl_enabled}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This indicates whether communication between this instance of NiFi and remote NiFi instances should be secure. By default, it is set to true. In order for secure site-to-site to work, many Security Properties must also be configured.</description>
+ </property>
+ <property>
+ <name>nifi.remote.input.http.enabled</name>
+ <value>true</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Specify if HTTP Site-to-Site should be enabled on this host. By default, it is true. HTTP non-secure Site-to-Site is enabled by default. Whether a Site-to-Site client uses HTTP or HTTPS is determined by nifi.remote.input.secure. If it is set to true, then requests are sent as HTTPS to nifi.web.https.port, if it is false, HTTP requests are sent to nifi.web.http.port.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>nifi.remote.input.host</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>The host name that will be given out to clients to connect to this NiFi instance for Site-to-Site communication. By default, it is the value from InetAddress.getLocalHost().getHostName(). On UNIX-like operating systems, this is typically the output from the hostname command.</description>
+ </property>
+
+ <property>
+ <name>nifi.components.status.snapshot.frequency</name>
+ <value>1 min</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This value indicates how often to present a snapshot of the components' status history. The default value is 1 min.</description>
+ </property>
+ <property>
+ <name>nifi.components.status.repository.buffer.size</name>
+ <value>1440</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Specifies the buffer size for the Component Status Repository. The default value is 1440.</description>
+ </property>
+
+ <property>
+ <name>nifi.components.status.repository.implementation</name>
+ <value>org.apache.nifi.controller.status.history.VolatileComponentStatusRepository</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The Component Status Repository implementation. The default value is org.apache.nifi.controller.status.history.VolatileComponentStatusRepository and should not be changed.</description>
+ </property>
+
+
+ <!--# web properties #-->
+
+ <property>
+ <name>nifi.web.war.directory</name>
+ <value>{{nifi_install_dir}}/lib</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This is the location of the web war directory.</description>
+ </property>
+ <property>
+ <name>nifi.web.http.host</name>
+ <value>{{nifi_node_nonssl_host}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The HTTP host</description>
+ </property>
+ <property>
+ <name>nifi.web.http.port</name>
+ <value>{{nifi_node_port}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The HTTP port</description>
+ </property>
+ <property>
+ <name>nifi.web.https.host</name>
+ <value>{{nifi_node_ssl_host}}</value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>The HTTPS host</description>
+ </property>
+ <property>
+ <name>nifi.web.https.port</name>
+ <value>{{nifi_node_ssl_port}}</value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>The HTTPS port</description>
+ </property>
+ <property>
+ <name>nifi.web.jetty.working.directory</name>
+ <value>{{nifi_internal_dir}}/work/jetty</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the Jetty working directory.</description>
+ </property>
+ <property>
+ <name>nifi.web.jetty.threads</name>
+ <value>200</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The number of Jetty threads. The default value is 200.</description>
+ </property>
+
+
+ <!--# security properties #-->
+ <property>
+ <name>nifi.sensitive.props.key</name>
+ <value>{{nifi_sensitive_props_key}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This is the password used to encrypt any sensitive property values that are configured in processors. By default, it is blank, but the system administrator should provide a value for it. It can be a string of any length, although the recommended minimum length is 10 characters. Be aware that once this password is set and one or more sensitive processor properties have been configured, this password should not be changed.</description>
+ </property>
+ <property>
+ <name>nifi.sensitive.props.algorithm</name>
+ <value>PBEWITHMD5AND256BITAES-CBC-OPENSSL</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The algorithm used to encrypt sensitive properties. The default value is PBEWITHMD5AND256BITAES-CBC-OPENSSL.</description>
+ </property>
+ <property>
+ <name>nifi.sensitive.props.provider</name>
+ <value>BC</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The sensitive property provider. The default value is BC.</description>
+ </property>
+ <!-- SSL related configs -->
+ <property>
+ <name>nifi.security.keystore</name>
+ <value>{{nifi_keystore}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The full path and name of the keystore. It is blank by default.</description>
+ </property>
+ <property>
+ <name>nifi.security.keystoreType</name>
+ <value>{{nifi_keystoreType}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The keystore type. It is blank by default.</description>
+ </property>
+ <property>
+ <name>nifi.security.keystorePasswd</name>
+ <value>{{nifi_keystorePasswd}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The keystore password. It is blank by default.</description>
+ </property>
+ <property>
+ <name>nifi.security.keyPasswd</name>
+ <value>{{nifi_keyPasswd}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The key password. It is blank by default.</description>
+ </property>
+ <property>
+ <name>nifi.security.truststore</name>
+ <value>{{nifi_truststore}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The full path and name of the truststore. It is blank by default.</description>
+ </property>
+ <property>
+ <name>nifi.security.truststoreType</name>
+ <value>{{nifi_truststoreType}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The truststore type. It is blank by default.</description>
+ </property>
+ <property>
+ <name>nifi.security.truststorePasswd</name>
+ <value>{{nifi_truststorePasswd}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The truststore password. It is blank by default.</description>
+ </property>
+ <property>
+ <name>nifi.security.needClientAuth</name>
+ <value>{{nifi_needClientAuth}}</value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>This indicates whether client authentication in the cluster protocol. It is blank by default.</description>
+ </property>
+
+ <property>
+ <name>nifi.security.user.authorizer</name>
+ <value>{{nifi_authorizer}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>ID of one of the authorizers from authorizers.xml that will be the active authorizer used by NiFi. Can be set to file-authorizer or ranger-authorizer (or your own custom authorizer)</description>
+ </property>
+ <property>
+ <name>nifi.security.user.login.identity.provider</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>This indicates what type of login identity provider to use. The default value is blank, can be set to the identifier from a provider in the file specified in nifi.login.identity.provider.configuration.file. Setting this property will trigger NiFi to support username/password authentication.</description>
+ </property>
+ <property>
+ <name>nifi.security.ocsp.responder.url</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>This is the URL for the Online Certificate Status Protocol (OCSP) responder if one is being used. It is blank by default.</description>
+ </property>
+ <property>
+ <name>nifi.security.ocsp.responder.certificate</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>This is the location of the OCSP responder certificate if one is being used. It is blank by default.</description>
+ </property>
+
+ <!-- Identity Mapping Properties -->
+
+
+
+ <property>
+ <name>nifi.security.identity.mapping.pattern.dn</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>Sample DN pattern property to demonstrate normalizing DNs from certificates into a common identity string. Create your own patterns using nifi.security.identity.mapping.pattern.XXX convention. Sample DN pattern: ^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$</description>
+ </property>
+
+ <property>
+ <name>nifi.security.identity.mapping.value.dn</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>Sample DN value property to demonstrate normalizing DNs from certificates into a common identity string. Set values for your own patterns using nifi.security.identity.mapping.pattern.XXX convention. Sample DN value: $1@$2</description>
+ </property>
+
+ <property>
+ <name>nifi.security.identity.mapping.pattern.kerb</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>Sample kerberos pattern property to demonstrate normalizing DNs from Kerberos principals into a common identity string. Create your own patterns using nifi.security.identity.mapping.pattern.XXX convention. Sample kerberos pattern: ^(.*?)/instance@(.*?)$</description>
+ </property>
+
+ <property>
+ <name>nifi.security.identity.mapping.value.kerb</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>Sample kerberos value property to demonstrate normalizing DNs from Kerberos principals into a common identity string. Set values for your own patterns using nifi.security.identity.mapping.pattern.XXX convention. Sample kerberos value: $1@$2</description>
+ </property>
+
+
+
+
+ <!--# cluster common properties (all nodes must have same values) #-->
+
+ <property>
+ <name>nifi.cluster.protocol.heartbeat.interval</name>
+ <value>5 sec</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The interval at which nodes should emit heartbeats to the cluster manager. The default value is 5 sec.</description>
+ </property>
+ <property>
+ <name>nifi.cluster.protocol.is.secure</name>
+ <value>{{nifi_ssl_enabled}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>This indicates whether cluster communications are secure. The default value is false.</description>
+ </property>
+
+ <!--# cluster node properties (only configure for cluster nodes) #-->
+
+ <property>
+ <name>nifi.cluster.is.node</name>
+ <value>{{nifi_is_node}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Whether the instance is a node in a cluster.</description>
+ </property>
+ <property>
+ <name>nifi.cluster.node.address</name>
+ <value>{{nifi_node_host}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The fully qualified address of the node. Will be auto-generated</description>
+ </property>
+ <property>
+ <name>nifi.cluster.node.protocol.port</name>
+ <value>{{nifi_node_protocol_port}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The node's protocol port</description>
+ </property>
+ <property>
+ <name>nifi.cluster.node.protocol.threads</name>
+ <value>10</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The number of threads used for the node protocol. The default value is 10.</description>
+ </property>
+ <property>
+ <name>nifi.cluster.node.event.history.size</name>
+ <value>25</value>
+ <on-ambari-upgrade add="true"/>
+ <description>When the state of a node in the cluster is changed, an event is generated and can be viewed in the Cluster page. This value indicates how many events to keep in memory for each node. The default value is 25.</description>
+ </property>
+ <property>
+ <name>nifi.cluster.node.connection.timeout</name>
+ <value>5 sec</value>
+ <on-ambari-upgrade add="true"/>
+ <description>When connecting to another node in the cluster, specifies how long this node should wait before considering the connection a failure. The default value is 5 secs.</description>
+ </property>
+
+ <property>
+ <name>nifi.cluster.node.read.timeout</name>
+ <value>5 sec</value>
+ <on-ambari-upgrade add="true"/>
+ <description>When communicating with another node in the cluster, specifies how long this node should wait to receive information from the remote node before considering the communication with the node a failure. The default value is 5 secs.</description>
+ </property>
+ <property>
+ <name>nifi.cluster.firewall.file</name>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the node firewall file. This is a file that may be used to list all the nodes that are allowed to connect to the cluster. It provides an additional layer of security. This value is blank by default, meaning that no firewall file is to be used.</description>
+ </property>
+
+ <property>
+ <name>nifi.cluster.flow.election.max.wait.time</name>
+ <value>5 mins</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Specifies the amount of time to wait before electing a Flow as the "correct" Flow.
+ If the number of Nodes that have voted is equal to the number specified by the nifi.cluster.flow.election.max.candidates
+ property, the cluster will not wait this long. The default is 5 minutes. Note that the time starts as soon as the first vote is cast.</description>
+ </property>
+
+ <property>
+ <name>nifi.cluster.flow.election.max.candidates</name>
+ <value>{{nifi_num_nodes}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>Specifies the number of Nodes required in the cluster to cause early election of Flows.
+ This allows the Nodes in the cluster to avoid having to wait a long time before starting processing if we reach at least this number of nodes in the cluster</description>
+ </property>
+
+ <!--# zookeeper properties, used for cluster management # -->
+ <property>
+ <name>nifi.zookeeper.connect.string</name>
+ <value>{{zookeeper_quorum}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The Connect String that is needed to connect to Apache ZooKeeper. This is a comma-separted list of hostname:port pairs. For example, localhost:2181,localhost:2182,localhost:2183. This should contain a list of all ZooKeeper instances in the ZooKeeper quorum.</description>
+ </property>
+ <property>
+ <name>nifi.zookeeper.connect.timeout</name>
+ <value>3 secs</value>
+ <on-ambari-upgrade add="true"/>
+ <description>How long to wait when connecting to ZooKeeper before considering the connection a failure. The default is 3 secs.</description>
+ </property>
+ <property>
+ <name>nifi.zookeeper.session.timeout</name>
+ <value>3 secs</value>
+ <on-ambari-upgrade add="true"/>
+ <description>How long to wait after losing a connection to ZooKeeper before the session is expired. The default is 3 secs.</description>
+ </property>
+ <property>
+ <name>nifi.zookeeper.root.node</name>
+ <value>{{nifi_znode}}</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The root ZNode that should be used in ZooKeeper. ZooKeeper provides a directory-like structure for storing data. Each directory in this structure is referred to as a ZNode. This denotes the root ZNode, or directory, that should be used for storing data. The default value is /root. This is important to set correctly, as which cluster the NiFi instance attempts to join is determined by which ZooKeeper instance it connects to and the ZooKeeper Root Node that is specified.</description>
+ </property>
+
+
+ <property>
+ <name>nifi.kerberos.krb5.file</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>The location of the krb5 file, if used. It is blank by default. At this time, only a single krb5 file is allowed to be specified per NiFi instance, so this property is configured here to support SPNEGO and service principles rather than in individual Processors. If necessary the krb5 file can support multiple realms.</description>
+ </property>
+
+ <property>
+ <name>nifi.kerberos.service.principal</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>The name of the NiFi Kerberos service principal, if used. It is blank by default. Note that this property is used to authenticate NiFi users. Example: HTTP/nifi.example.com or HTTP/nifi.example.com@EXAMPLE.COM</description>
+ </property>
+
+ <property>
+ <name>nifi.kerberos.service.keytab.location</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>The file path of the NiFi Kerberos keytab, if used. It is blank by default. Note that this property is used to authenticate NiFi users</description>
+ </property>
+
+ <property>
+ <name>nifi.kerberos.spnego.principal</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>The name of the Spnego service principal, if used. It is blank by default. Note that this property is used to authenticate NiFi users. Example: HTTP/nifi.example.com or HTTP/nifi.example.com@EXAMPLE.COM</description>
+ </property>
+
+ <property>
+ <name>nifi.kerberos.spnego.keytab.location</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <description>The file path of the NiFi Spnego Kerberos keytab, if used. It is blank by default. Note that this property is used to authenticate NiFi users</description>
+ </property>
+
+ <property>
+ <name>nifi.kerberos.spnego.authentication.expiration</name>
+ <value>12 hours</value>
+ <on-ambari-upgrade add="true"/>
+ <description>The expiration duration of a successful Kerberos user authentication, if used. It is 12 hours by default. Example: 12 hours</description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-state-management-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-state-management-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-state-management-env.xml
new file mode 100644
index 0000000..c559831
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-state-management-env.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+ <property>
+ <name>content</name>
+ <display-name>Template for state-management.xml</display-name>
+ <value>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ This file provides a mechanism for defining and configuring the State Providers
+ that should be used for storing state locally and across a NiFi cluster. In order
+ to use a specific provider, it must be configured here and its identifier
+ must be specified in the nifi.properties file.
+-->
+<stateManagement>
+ <!--
+ State Provider that stores state locally in a configurable directory. This Provider requires the following properties:
+
+ Directory - the directory to store components' state in. If the directory being used is a sub-directory of the NiFi installation, it
+ is important that the directory be copied over to the new version when upgrading NiFi.
+ -->
+ <local-provider>
+ <id>local-provider</id>
+ <class>org.apache.nifi.controller.state.providers.local.WriteAheadLocalStateProvider</class>
+ <property name="Directory">{{nifi_state_dir}}</property>
+ </local-provider>
+
+ <!--
+ State Provider that is used to store state in ZooKeeper. This Provider requires the following properties:
+
+ Root Node - the root node in ZooKeeper where state should be stored. The default is '/nifi', but it is advisable to change this to a different value if not using
+ the embedded ZooKeeper server and if multiple NiFi instances may all be using the same ZooKeeper Server.
+
+ Connect String - A comma-separated list of host:port pairs to connect to ZooKeeper. For example, myhost.mydomain:2181,host2.mydomain:5555,host3:6666
+
+ Session Timeout - Specifies how long this instance of NiFi is allowed to be disconnected from ZooKeeper before creating a new ZooKeeper Session. Default value is "30 seconds"
+
+ Access Control - Specifies which Access Controls will be applied to the ZooKeeper ZNodes that are created by this State Provider. This value must be set to one of:
+ - Open : ZNodes will be open to any ZooKeeper client.
+ - CreatorOnly : ZNodes will be accessible only by the creator. The creator will have full access to create children, read, write, delete, and administer the ZNodes.
+ This option is available only if access to ZooKeeper is secured via Kerberos or if a Username and Password are set.
+ -->
+
+
+ <cluster-provider>
+ <id>zk-provider</id>
+ <class>org.apache.nifi.controller.state.providers.zookeeper.ZooKeeperStateProvider</class>
+ <property name="Connect String">{{zookeeper_quorum}}</property>
+ <property name="Root Node">{{nifi_znode}}</property>
+ <property name="Session Timeout">30 seconds</property>
+ <property name="Access Control">Open</property>
+ </cluster-provider>
+</stateManagement>
+
+
+ </value>
+ <description>template for state-management.xml</description>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-audit.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-audit.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-audit.xml
new file mode 100644
index 0000000..cd565dd
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-audit.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>xasecure.audit.is.enabled</name>
+ <value>true</value>
+ </property>
+
+ <!-- Log4j audit provider configuration -->
+ <property>
+ <name>xasecure.audit.destination.log4j</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.log4j.logger</name>
+ <value>ranger_audit_logger</value>
+ </property>
+
+ <!-- Solr audit provider configuration -->
+ <property>
+ <name>xasecure.audit.destination.solr</name>
+ <value>false</value>
+ <display-name>Audit to SOLR</display-name>
+ <description>Is Solr audit enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.solr</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.solr.batch.filespool.dir</name>
+ <value>/tmp/audit/solr/spool</value>
+ <description>/tmp/audit/solr/spool</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.solr.urls</name>
+ <value/>
+ <description>Solr URL</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.urls</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.solr.zookeepers</name>
+ <value>NONE</value>
+ <description>Solr Zookeeper string</description>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.zookeepers</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-plugin-properties.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-plugin-properties.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-plugin-properties.xml
new file mode 100644
index 0000000..4caf07b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-plugin-properties.xml
@@ -0,0 +1,170 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <property>
+ <name>policy_user</name>
+ <value>nifi</value>
+ <display-name>Policy user for NIFI</display-name>
+ <description>This user must be system user and also present at Ranger admin portal</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>ranger-nifi-plugin-enabled</name>
+ <value>No</value>
+ <display-name>Enable Ranger for NIFI</display-name>
+ <description>Enable ranger nifi plugin ?</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>ranger-nifi-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.authentication</name>
+ <value>NONE</value>
+ <display-name>Authentication</display-name>
+ <description>Ranger Service Authentication Type</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>ranger-nifi-plugin-enabled</name>
+ </property>
+ <property>
+ <type>nifi-ambari-ssl-config</type>
+ <name>nifi.node.ssl.isenabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>SSL</value>
+ <label>ON</label>
+ </entry>
+ <entry>
+ <value>NONE</value>
+ <label>OFF</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.ssl.keystore</name>
+ <value></value>
+ <display-name>Keystore for Ranger Service Accessing NiFi</display-name>
+ <description>Used by the Ranger Service to access NiFi protected with SSL</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.ssl.keystoreType</name>
+ <value></value>
+ <display-name>Keystore Type</display-name>
+ <description>Keystore Type</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.ssl.keystorePassword</name>
+ <value></value>
+ <display-name>Keystore password</display-name>
+ <property-type>PASSWORD</property-type>
+ <description>Keystore password</description>
+ <value-attributes>
+ <type>password</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.ssl.truststore</name>
+ <value></value>
+ <display-name>Truststore for Ranger Service Accessing NiFi</display-name>
+ <description>Used by the Ranger Service to access/identify NiFi protected with SSL</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.ssl.truststoreType</name>
+ <value></value>
+ <display-name>Truststore Type</display-name>
+ <description>Truststore Type</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nifi.ssl.truststorePassword</name>
+ <value></value>
+ <display-name>Truststore password</display-name>
+ <property-type>PASSWORD</property-type>
+ <description>Truststore password</description>
+ <value-attributes>
+ <type>password</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>owner.for.certificate</name>
+ <value/>
+ <display-name>Owner for Certificate</display-name>
+ <description>Full identity of certificate owner (created when keystore was generated)</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>REPOSITORY_CONFIG_USERNAME</name>
+ <value>{{ranger_admin_username}}</value>
+ <display-name>Ranger repository config user</display-name>
+ <description>Used for repository creation on ranger admin</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>REPOSITORY_CONFIG_PASSWORD</name>
+ <value>{{ranger_admin_password}}</value>
+ <display-name>Ranger repository config password</display-name>
+ <property-type>PASSWORD</property-type>
+ <description>Used for repository creation on ranger admin</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-policymgr-ssl.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-policymgr-ssl.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-policymgr-ssl.xml
new file mode 100644
index 0000000..58c75b4
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-policymgr-ssl.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+<!-- The following properties are used for 2-way SSL client server validation -->
+<property>
+ <name>xasecure.policymgr.clientssl.keystore</name>
+ <value></value>
+ <description>Java Keystore files</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+</property>
+<property>
+ <name>xasecure.policymgr.clientssl.keystore.password</name>
+ <value>changeme</value>
+ <property-type>PASSWORD</property-type>
+ <description>password for keystore</description>
+ <value-attributes>
+ <type>password</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+</property>
+<property>
+ <name>xasecure.policymgr.clientssl.truststore</name>
+ <value></value>
+ <description>java truststore file</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+</property>
+<property>
+ <name>xasecure.policymgr.clientssl.truststore.password</name>
+ <value>changeme</value>
+ <property-type>PASSWORD</property-type>
+ <description>java truststore password</description>
+ <value-attributes>
+ <type>password</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+</property>
+<property>
+ <name>owner.for.certificate</name>
+ <value/>
+ <description>Full identity of certificate owner (created when keystore was generated)</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+</property>
+<property>
+ <name>xasecure.policymgr.clientssl.keystore.credential.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>java keystore credential file</description>
+ <on-ambari-upgrade add="true"/>
+</property>
+<property>
+ <name>xasecure.policymgr.clientssl.truststore.credential.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>java truststore credential file</description>
+ <on-ambari-upgrade add="true"/>
+</property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-security.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-security.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-security.xml
new file mode 100644
index 0000000..05d6adb
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/ranger-nifi-security.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>ranger.plugin.nifi.policy.rest.url</name>
+ <value>{{policymgr_mgr_url}}</value>
+ <description>
+ URL to Ranger Admin
+ </description>
+ <depends-on>
+ <property>
+ <type>admin-properties</type>
+ <name>policymgr_external_url</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>ranger.plugin.nifi.service.name</name>
+ <value>{{repo_name}}</value>
+ <description>
+ Name of the Ranger service containing policies for this nifi instance
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.nifi.policy.source.impl</name>
+ <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value>
+ <description>
+ Class to retrieve policies from the source
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.nifi.policy.rest.ssl.config.file</name>
+ <value>ranger-policymgr-ssl.xml</value>
+ <description>
+ Path to the file containing SSL details to contact Ranger Admin
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.nifi.policy.pollIntervalMs</name>
+ <value>30000</value>
+ <description>
+ How often to poll for changes in policies?
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.nifi.policy.cache.dir</name>
+ <value>/tmp</value>
+ <description>
+ Directory where Ranger policies are cached after successful retrieval from the source
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.nifi.policy.rest.client.connection.timeoutMs</name>
+ <value>120000</value>
+ <description>
+ RangerRestClient Connection Timeout in Milli Seconds
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.nifi.policy.rest.client.read.timeoutMs</name>
+ <value>30000</value>
+ <description>
+ RangerRestClient read Timeout in Milli Seconds
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/kerberos.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/kerberos.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/kerberos.json
new file mode 100644
index 0000000..74d84e6
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/kerberos.json
@@ -0,0 +1,72 @@
+{
+ "services": [
+ {
+ "name": "NIFI",
+ "components": [
+ {
+ "name": "NIFI_MASTER",
+ "identities": [
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "nifi-properties/nifi.kerberos.spnego.principal"
+ },
+ "keytab": {
+ "configuration": "nifi-properties/nifi.kerberos.spnego.keytab.location"
+ }
+ },
+ {
+ "name": "nifi_principal",
+ "principal": {
+ "value": "${nifi-env/nifi_user}/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "nifi-properties/nifi.kerberos.service.principal",
+ "local_username": "${nifi-env/nifi_user}"
+
+ },
+ "keytab": {
+ "file": "${keytab_dir}/nifi.service.keytab",
+ "owner": {
+ "name": "${nifi-env/nifi_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "nifi-properties/nifi.kerberos.service.keytab.location"
+ }
+ },
+ {
+ "name": "/NIFI/NIFI_MASTER/nifi_principal",
+ "principal": {
+ "configuration": "ranger-nifi-audit/xasecure.audit.jaas.Client.option.principal"
+ },
+ "keytab": {
+ "configuration": "ranger-nifi-audit/xasecure.audit.jaas.Client.option.keyTab"
+ }
+ }
+ ],
+ "configurations": [
+ {
+ "nifi-properties": {
+ "nifi.kerberos.krb5.file": "${krb5-conf/conf_dir}/krb5.conf",
+ "nifi.security.user.login.identity.provider": "kerberos-provider"
+ }
+ },
+ {
+ "ranger-nifi-audit": {
+ "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",
+ "xasecure.audit.jaas.Client.loginModuleControlFlag": "required",
+ "xasecure.audit.jaas.Client.option.useKeyTab": "true",
+ "xasecure.audit.jaas.Client.option.storeKey": "false",
+ "xasecure.audit.jaas.Client.option.serviceName": "solr",
+ "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/metainfo.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/metainfo.xml
new file mode 100644
index 0000000..ac54975
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/metainfo.xml
@@ -0,0 +1,149 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>NIFI</name>
+ <displayName>NiFi</displayName>
+ <comment>Apache NiFi is an easy to use, powerful, and reliable system to process and distribute data.</comment>
+ <version>1.0.0</version>
+ <components>
+ <component>
+ <name>NIFI_CA</name>
+ <displayName>NiFi Certificate Authority</displayName>
+ <category>SLAVE</category>
+ <cardinality>0-1</cardinality>
+ <timelineAppid>NIFI</timelineAppid>
+ <logs>
+ <log>
+ <logId>nifi_ca</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
+ <commandScript>
+ <script>scripts/nifi_ca.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>10000</timeout>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>INVALIDATE_CA_SERVER</name>
+ <commandScript>
+ <script>scripts/nifi_ca.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+ <component>
+ <name>NIFI_MASTER</name>
+ <displayName>NiFi</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <timelineAppid>NIFI</timelineAppid>
+
+ <dependencies>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+
+ <logs>
+ <log>
+ <logId>nifi_app</logId>
+ <primary>true</primary>
+ </log>
+ <log>
+ <logId>nifi_bootstrap</logId>
+ </log>
+ <log>
+ <logId>nifi_setup</logId>
+ </log>
+ <log>
+ <logId>nifi_user</logId>
+ </log>
+ </logs>
+ <commandScript>
+ <script>scripts/nifi.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>10000</timeout>
+ </commandScript>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>nifi_${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+ <packages>
+ <package>
+ <name>nifi-${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>ZOOKEEPER</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>nifi-ambari-config</config-type>
+ <config-type>nifi-ambari-ssl-config</config-type>
+ <config-type>nifi-authorizers-env</config-type>
+ <config-type>nifi-bootstrap-env</config-type>
+ <config-type>nifi-env</config-type>
+ <config-type>nifi-flow-env</config-type>
+ <config-type>nifi-node-logback-env</config-type>
+ <config-type>nifi-properties</config-type>
+ <config-type>nifi-state-management-env</config-type>
+ <config-type>ranger-nifi-audit</config-type>
+ <config-type>ranger-nifi-plugin-properties</config-type>
+ <config-type>ranger-nifi-policymgr-ssl</config-type>
+ <config-type>ranger-nifi-security</config-type>
+ </configuration-dependencies>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <widgetsFileName>widgets.json</widgetsFileName>
+ <metricsFileName>metrics.json</metricsFileName>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/metrics.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/metrics.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/metrics.json
new file mode 100644
index 0000000..5a3e82f
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/metrics.json
@@ -0,0 +1,131 @@
+{
+ "NIFI_MASTER": {
+ "Component": [{
+ "type": "ganglia",
+ "metrics": {
+ "default": {
+ "metrics/nifi/FlowFilesReceivedLast5Minutes": {
+ "metric": "FlowFilesReceivedLast5Minutes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/BytesReceivedLast5Minutes": {
+ "metric": "BytesReceivedLast5Minutes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/FlowFilesSentLast5Minutes": {
+ "metric": "FlowFilesSentLast5Minutes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/BytesSentLast5Minutes": {
+ "metric": "BytesSentLast5Minutes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/FlowFilesQueued": {
+ "metric": "FlowFilesQueued",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/BytesQueued": {
+ "metric": "BytesQueued",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/BytesReadLast5Minutes": {
+ "metric": "BytesReadLast5Minutes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/BytesWrittenLast5Minutes": {
+ "metric": "BytesWrittenLast5Minutes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/ActiveThreads": {
+ "metric": "ActiveThreads",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/TotalTaskDurationSeconds": {
+ "metric": "TotalTaskDurationSeconds",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/nifi/TotalTaskDurationNanoSeconds": {
+ "metric": "TotalTaskDurationNanoSeconds",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/uptime": {
+ "metric": "jvm.uptime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/heap_used": {
+ "metric": "jvm.heap_used",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/heap_usage": {
+ "metric": "jvm.heap_usage",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/non_heap_usage": {
+ "metric": "jvm.non_heap_usage",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/thread_states/runnable": {
+ "metric": "jvm.thread_states.runnable",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/thread_states/blocked": {
+ "metric": "jvm.thread_states.blocked",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/thread_states/timed_waiting": {
+ "metric": "jvm.thread_states.timed_waiting",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/thread_states/terminated": {
+ "metric": "jvm.thread_states.terminated",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/thread_count": {
+ "metric": "jvm.thread_count",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/daemon_thread_count": {
+ "metric": "jvm.daemon_thread_count",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/file_descriptor_usage": {
+ "metric": "jvm.file_descriptor_usage",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/gc/runs/$1": {
+ "metric": "jvm.gc.runs.(\\w+)",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/gc/time/$1": {
+ "metric": "jvm.gc.time.(\\w+)",
+ "pointInTime": true,
+ "temporal": true
+ }
+ }
+ }
+ }]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/alert_check_nifi.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/alert_check_nifi.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/alert_check_nifi.py
new file mode 100644
index 0000000..d74c676
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/alert_check_nifi.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import glob
+import sys
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.script import Script
+
+reload(sys)
+sys.setdefaultencoding('utf8')
+config = Script.get_config()
+
+nifi_pid_dir = config['configurations']['nifi-env']['nifi_pid_dir']
+
+RESULT_CODE_OK = 'OK'
+RESULT_CODE_CRITICAL = 'CRITICAL'
+RESULT_CODE_UNKNOWN = 'UNKNOWN'
+
+
+def execute(configurations={}, parameters={}, host_name=None):
+ try:
+ pid_file = nifi_pid_dir + '/nifi.pid'
+ check_process_status(pid_file)
+ except ComponentIsNotRunning as ex:
+ return (RESULT_CODE_CRITICAL, [str(ex)])
+ except:
+ return (RESULT_CODE_CRITICAL, ["Nifi is not running"])
+
+ return (RESULT_CODE_OK, ["Successful connection to Nifi"])
+